tvaLib
C:/Home/Documents/workspace/tvaLib/main.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 # tvaLib Copyright (c) 2012-2016 Paul G. St-Aubin
3 # Ecole Polytechnique de Montreal, McGill University
4 # Python 2.7; (dt) Spyder Windows 10 64-bit; ipython Ubuntu 15.04 64-bit
5 version = 'R2.3.0 u. 2017-03-22'
6 
7 ''' Dev stuff
8 import pdb; pdb.set_trace()
9 raise Exception, [9999, 'Test error']
10 import lib.vis; lib.vis.trajSample(obj, alignments=alignments).get().show()
11 import traceback;traceback.print_stack()
12 import site; site.addsitedir(r'E:\My Documents\workspace\tvaLib')
13 -e -z M:\ --test-track-params "Z:\Projects\Plateau-Arrets\Data\Trajectory Optimisation\CandidateParams.csv" -v 10
14 -i 5 -e --play-int -d "lund-scene.sqlite"
15  -z "Q:\Video" -i 0 -waue
16  -s 1 -b 1 -c 1 -f 2 -z Q:\Video --mota-opt-g -t 4 -v 10
17  -s 1 -b 1 -c 1 -f 2 -z Q:\Video -wau
18  -wa -y stop -z M:\ -b 2 -f 1
19  -euaw --plot-dmp --dir M:\ -i 5
20 ## Place these two lines of code just before main()
21 from memory_profiler import profile;fp=open('memory_profiler.log','w+')
22 @profile(stream=fp)
23 n=4 is cyclist on clark rachel GT
24 '''
25 
26 
27 
30 def main():
31 
32  try:
33 
37  import sys, os, time, subprocess, logging
38  from importlib import import_module
39  from csv import reader as csv_reader
40  from csv import writer as csv_writer
41  from csv import QUOTE_MINIMAL as csv_QUOTE_MINIMAL
42  from re import search as re_search
43  from shutil import copy as shutil_copy
44  from string import ascii_uppercase as string_ascii_uppercase
45  from string import digits as string_digits
46  from copy import deepcopy
47  from random import choice as random_choice
48  from datetime import datetime
49  from datetime import timedelta
50  from types import MethodType
51  import cPickle as pickle
52  from warnings import filterwarnings
53  filterwarnings('ignore')
54 
55 
56  os.chdir(os.path.dirname(os.path.abspath(sys.argv[0])))
57 
58  import include.config as tvaConfig
59  import include.local as tvaLocal
60 
61 
64  commands = tvaConfig.commands()
65  config = tvaConfig.Config(readonly=commands.configreadonly)
66 
67 
70  try: #Colorama, optional, required for Windows if coloured text is desired
71  from colorama import init as colorama_init
72  colorama_init(strip=False)
73  from colorama import Fore, Back, Style
74  except ImportError:
75  Fore = tvaConfig.Fore()
76  Back = tvaConfig.Back()
77  Style = tvaConfig.Style()
78  if(commands.logging or config.disable_colour):
79  config.disable_colour = True
80  Fore = tvaConfig.Fore(forceOff=True)
81  Back = tvaConfig.Back(forceOff=True)
82  Style = tvaConfig.Style(forceOff=True)
83 
84 
87  import include.runtime as tvaRuntime
88 
89 
92  if(not config.advanced_runtime): tvaRuntime.checkDependancies()
93 
94 
99  try: import numpy as np
100  except ImportError: pass
101  #Matplotlib
102  try:
103  from matplotlib import use as mpl_use
104  from matplotlib import get_backend as mpl_get_backend
105  if(commands.f_tk_interface): mpl_use('TkAgg')
106  elif(not commands.f_interface and not config.debug): mpl_use('Agg')
107  import matplotlib.pyplot as plt
108  if(commands.f_interface): commands.max_obj = config.draw_max_traj if commands.max_obj <= 0 else commands.max_obj
109  else: plt.ioff()
110  except (ImportError, RuntimeError) as e: pass
111 
112 
113 
118  if(not config.debug): oldstdout = sys.stdout;sys.stdout = tvaConfig.NullWriter()
119  try:
120  import lib.comp as tvaCompute
121  import lib.draw as tvaDraw
122  import lib.filt as tvaFilter
123  import lib.int as tvaInteraction
124  import lib.scene as tvaScene
125  import lib.stat as tvaStats
126  import lib.tools as tvaLib
127  import lib.vis as tvaVis
128  finally:
129  if(not config.debug): sys.stdout = oldstdout #Re-enable output
130 
131 
134  if(config.debug): logging.basicConfig(level=logging.DEBUG, format='[%(levelname)s] (%(threadName)-10s) %(message)s')
135 
136 
139  if(commands.dir): config.dir = commands.dir
140  if(commands.dbn): config.dbn = commands.dbn
141  if(commands.language): config.language = commands.language
142  local = tvaLocal.Local(config.language)
143  #Override module colour options
144  tvaLib.Fore = Fore;tvaLib.Back = Back;tvaLib.Style = Style
145  tvaRuntime.Fore = Fore;tvaRuntime.Back = Back;tvaRuntime.Style = Style
146 
147  if(not commands.configreadonly and (not config.path_to_trafint or (config.path_to_trafint and not os.path.exists(config.path_to_trafint)))):
148  try:
149  import Tkinter as tk
150  from tkFileDialog import askdirectory
151  root = tk.Tk()
152  root.withdraw()
153  config.setValue('path_to_trafint',askdirectory(title=local['UI_dialogue_title_sel_TI'], initialdir=''))
154  if(config.path_to_trafint): config.write()
155  except: raise Exception, [2840, 'Could not find Traffic Intelligence (at "'+config.path_to_trafint+'") and interface failed to load.']
156 
157  if(not commands.configreadonly and not os.path.exists(os.path.join(config.dir, config.dbn))):
158  try:
159  import Tkinter as tk
160  from tkFileDialog import askopenfilename
161  root = tk.Tk()
162  root.withdraw()
163  except: raise Exception, [2841, 'Could not find video database (at "'+os.path.join(config.dir, config.dbn)+'") and interface failed to load.']
164  dbFilename = askopenfilename(title=local['UI_dialogue_title_sel_db'], filetypes=[('SQLITE', '.sqlite')])
165  if(not dbFilename): raise Exception, [201, 'No suitable database chosen ('+os.path.join(config.dir, config.dbn)+' could not be found).']
166  config.dir = os.path.split(dbFilename)[0]
167  config.dbn = os.path.split(dbFilename)[1]
168  if(not commands.dir and not commands.dbn):
169  if(config.dir or config.dbn): config.write()
170 
171  try:
172  sites = tvaScene.Sites(config)
173  site_analyses = tvaScene.SiteAnalyses(config, sites=sites)
174  except:
175  if(config.debug):
176  sites = tvaScene.Sites(config)
177  site_analyses = tvaScene.SiteAnalyses(config, sites=sites)
178  else: raise Exception, [2842, 'Metadata database failed to load and is possibly corrupt.']
179  #sites.exportToCode() ## Use this manual override to backport scene.sqlite data to code to regenerate a clean scene.sqlite
180  #site_analyses.exportToCode() ## Use this manual override to backport scene.sqlite data to code to regenerate a clean scene.sqlite
181  commands.indicators = tvaLib.Parse.list1D(commands.indicators, i_type='int')
182  commands.sample_traj = tvaLib.Parse.list1D(commands.sample_traj, i_type='int')
183 
184  if(commands.logging):
185  if(commands.verbose >= 1): tvaLib.printNotice('The --log command has been selected. ALL program output will be dumped to run.log which will be moved to the relevant analysis folder if possible. No further output will be displayed onscreen.', local['gen_notice'])
186  if(commands.verbose < 2): commands.verbose = 2
187  commands.stat = True
188  if(commands.save_data_flush or commands.save_data_flush_objects): log_mode = 'w'
189  else: log_mode = 'a'
190  oldstdout = sys.stdout
191  process_file_id = ''.join(random_choice(string_ascii_uppercase + string_digits) for _ in range(20))
192  sys.stdout = open('run_'+process_file_id+'.log', log_mode)
193 
194  config.version = tvaLib.Parse.versionFromHgCommit(version)
195  if(commands.verbose): print('Running tvaLib version '+config.version)
196  if(config.debug): logging.debug('Using matplotlib backend: '+mpl_get_backend())
197  if(commands.verbose): print('Using database: '+os.path.join(config.dir,config.dbn))
198 
199 
200 
203 
204 
207  if(commands.concat):
208  if(commands.interactive): commands = tvaRuntime.interactiveSiteSelection(commands, config, sites, local)
209  siteIx = sites.interpret(commands.site)[0]
210  camIx = sites[siteIx].interpret(commands.camera)[0]
211  os.chdir(sites[siteIx][camIx].getBaseDirectory())
212 
213  if(commands.verbose): tvaLib.printTimeStamp('Concatenating video data for site: '+sites[siteIx].name+'; camera: '+sites[siteIx][camIx].name+'...')
214 
215  for dirs in os.listdir('.'):
216  if(os.path.isdir(dirs)):
217 
218  temp = 0
219  for file in os.listdir(dirs):
220  file_lower = file.lower()
221  if(file_lower.endswith('.mp4')):
222  subprocess.call(['mencoder', os.path.join(dirs, file), '-ovc', 'copy', '-nosound', '-fps', str(sites[siteIx][camIx].camera.frameRate), '-ofps', str(config.fps), '-o', os.path.join(dirs, file)+'-tmp.avi'])
223  temp = 1
224 
225  if(temp): subprocess.call('mencoder '+os.path.join(dirs, '*-tmp.avi')+' -forceidx -ovc copy -nosound -o '+dirs+'.avi', shell=True)
226 
227  if(temp):
228  for file in os.listdir(dirs):
229  if(file.endswith('-tmp.avi')):
230  os.remove(os.path.join(dirs, file))
231 
232  raise Exception, [2, 'Concatenation complete.']
233 
234 
235 
238  if(commands.framedump):
239  if(commands.interactive): commands = tvaRuntime.interactiveSiteSelection(commands, config, sites, local)
240  siteIx = sites.interpret(commands.site)[0]
241  camIxs = sites[siteIx].interpret(commands.camera)
242 
243  for camIx in camIxs:
244  fileIxs = sites[siteIx][camIx].interpret(commands.file)
245 
246  for fileIx in fileIxs:
247  if(commands.verbose): tvaLib.printTimeStamp('Dumping frame for '+sites[siteIx].name+'; camera: '+sites[siteIx][camIx].name+'; sequence: '+sites[siteIx][camIx][fileIx].name+'...')
248  tvaLib.Obj.dumpFrame(sites[siteIx][camIx][fileIx].getFullVideoFilename())
249 
250  raise Exception, [3, 'Framedump complete.']
251 
252 
253 
256  if(commands.undistort):
257  if(commands.interactive): commands = tvaRuntime.interactiveSiteSelection(commands, config, sites, local)
258  siteIx = sites.interpret(commands.site)[0]
259  camIxs = sites[siteIx].interpret(commands.camera)
260 
261  for camIx in camIxs:
262  fileIxs = sites[siteIx][camIx].interpret(commands.file)
263 
264  for fileIx in fileIxs:
265  if(not sites[siteIx][camIx].camera.camera_matrix.data): tvaLib.printNotice('The camera associated with this view does not have distortion parameters. Please verify that this camera (#'+str(sites[siteIx][camIx].camera.idx)+': '+sites[siteIx][camIx].camera.name+') requires undistortion.', local['gen_warning'])
266  else:
267  if(commands.verbose): tvaLib.printTimeStamp('Undistortion for '+sites[siteIx].name+'/'+sites[siteIx][camIx].name+'/'+sites[siteIx][camIx][fileIx].name+'...')
268  if(not os.path.exists(sites[siteIx][camIx][fileIx].getImageSpaceFrameFilename())):
269  if(commands.verbose): print(' No default video frame found... Automatically dumping new frame for '+sites[siteIx].name+'; camera: '+sites[siteIx][camIx].name+'; sequence: '+sites[siteIx][camIx][fileIx].name+'...')
270  tvaLib.Obj.dumpFrame(sites[siteIx][camIx][fileIx].getFullVideoFilename())
271  if(commands.verbose): print(' Undistorting image...')
272  tvaLib.Obj.undistortImage(sites[siteIx][camIx][fileIx].getImageSpaceFrameFilename(), sites[siteIx][camIx][fileIx].getImageSpaceFrameFilename(forceDistorted=True, allowNonExistant=True), camera_matrix=sites[siteIx][camIx].camera.camera_matrix.data, dist_coeffs=sites[siteIx][camIx].camera.dist_coeffs.data, display=commands.play, freeScalingParameter=sites[siteIx][camIx].camera.freeScalingParameter, imageScalingFactor=sites[siteIx][camIx].camera.imageScalingFactor, verbose=commands.verbose)
273  if(sites[siteIx][camIx][fileIx].getFullVideoFilename()):
274  if(commands.verbose): print(' Undistorting video...')
275  tvaLib.Obj.undistortImage(sites[siteIx][camIx][fileIx].getFullVideoFilename(), sites[siteIx][camIx][fileIx].getFullVideoFilename(forceDistorted=True, allowNonExistant=True), camera_matrix=sites[siteIx][camIx].camera.camera_matrix.data, dist_coeffs=sites[siteIx][camIx].camera.dist_coeffs.data, frames=commands.time*sites[siteIx][camIx].camera.frameRate, display=commands.play, fourcc=config.video_out_fourcc, freeScalingParameter=sites[siteIx][camIx].camera.freeScalingParameter, imageScalingFactor=sites[siteIx][camIx].camera.imageScalingFactor, verbose=commands.verbose)
276  else: tvaLib.printNotice('No video footage found for undistortion.', local['gen_warning'])
277  raise Exception, [4, 'Undistortion complete.']
278 
279 
280 
283  if(commands.extract_images):
284  import cv2
285 
286 
287  if(commands.interactive): commands = tvaRuntime.interactiveSiteSelection(commands, config, sites, local)
288  siteIxs = sites.interpret(commands.site)
289 
290 
291  if(commands.verbose): print('Automatically extracting images from available classification or ground truth files...')
292  for siteIx in siteIxs:
293  camIxs = sites[siteIx].interpret(commands.camera)
294  for camIx in camIxs:
295  imageDirectories = dict([(x,os.path.join(sites[siteIx][camIx].getClassificationTrainingBaseDirectory(local['userTypeNames'][x]))) for x in config.class_candidates])
296  fileIxs = sites[siteIx][camIx].interpret(commands.file)
297  print(' Searching "'+Fore.YELLOW+sites[siteIx].name+'/'+sites[siteIx][camIx].name+Fore.RESET+'" found. Processing... ')
298  for fileIx in fileIxs:
299 
300 
301  if(not commands.proc_gt and os.path.exists(sites[siteIx][camIx][fileIx].getFullClassifiedFilename())):
302  filename = sites[siteIx][camIx][fileIx].getFullClassifiedFilename()
303  if(commands.verbose >= 2): print(' Suitable '+Back.MAGENTA+'classification'+Back.RESET+' file for sequence "'+Fore.YELLOW+sites[siteIx][camIx][fileIx].name+Fore.RESET+'" found. Processing... ')
304  elif(os.path.exists(sites[siteIx][camIx][fileIx].getFullGroundTruthFilename())):
305  filename = sites[siteIx][camIx][fileIx].getFullGroundTruthFilename()
306  if(commands.verbose >= 2): print(' Suitable '+Back.MAGENTA+'ground truth'+Back.RESET+' file for sequence "'+Fore.YELLOW+sites[siteIx][camIx][fileIx].name+Fore.RESET+'" found. Processing... ')
307  else: continue
308 
309 
310  try: source = cv2.VideoCapture(sites[siteIx][camIx][fileIx].getFullVideoFilename())
311  except Exception:
312  if(commands.verbose): tvaLib.printWarning('Could not load video file necessary for image extraction.', local['gen_warning'])
313  continue
314  width = int(source.get(3))
315  height = int(source.get(4))
316  if(sites[siteIx][camIx].camera.camera_matrix.data):
317  R = np.identity(3)
318  newImgSize = (int(round(width*sites[siteIx][camIx].camera.imageScalingFactor)), int(round(height*sites[siteIx][camIx].camera.imageScalingFactor)))
319  new_matrix = deepcopy(sites[siteIx][camIx].camera.camera_matrix.asNpArray())
320  new_matrix[0,2] = newImgSize[0]/2.
321  new_matrix[1,2] = newImgSize[1]/2.
322  [map1, map2] = cv2.initUndistortRectifyMap(sites[siteIx][camIx].camera.camera_matrix.asNpArray(), sites[siteIx][camIx].camera.dist_coeffs.asNpArray(), R, new_matrix, newImgSize, cv2.CV_32FC1)
323 
324 
325  firstInstant = int(commands.start_time*sites[siteIx][camIx].camera.frameRate)
326  if(commands.time): lastInstant = int(commands.start_time*sites[siteIx][camIx].camera.frameRate)+int(commands.time*sites[siteIx][camIx].camera.frameRate)
327  else: lastInstant = int(sites[siteIx][camIx][fileIx].getDuration()*sites[siteIx][camIx].camera.frameRate)
328 
329 
330  seq_objects = tvaLib.Obj.loadObjects(filename, max_obj_features=config.max_obj_features)
331  seq_objects = tvaFilter.corVelIndex(seq_objects, verbose=commands.verbose)
332  seq_objects,_ = tvaFilter.filtBoundingBox(seq_objects, sites[siteIx][camIx].mask, containment_threshold=config.f_bb_containment_threshold, max_outside_dist=config.f_bb_max_outside_dist, loopback_verification_frames=config.f_bb_loopback_ver_frames, bounding_boxes_label='analysis zone', verbose=0)
333  seq_objects = tvaFilter.filtTime(seq_objects, startTime=firstInstant, endTime=lastInstant, verbose=commands.verbose)
334 
335 
336  invHomography = tvaLib.Obj.invHomography(sites[siteIx][camIx].getHomography().asNpArray())
337 
338 
339  if(commands.verbose >= 2): print(' Dumping images ('+str(len(seq_objects))+' objects)... ')
340  for obj in seq_objects:
341  if(obj.userType not in config.class_candidates): continue
342 
343  if(len(obj.getXCoordinates()) == 0): continue
344  frameNum = (obj.getLastInstant() - obj.getFirstInstant())/2 + obj.getFirstInstant()
345  source.set(1, frameNum)
346  _,img = source.read()
347  img = cv2.remap(img, map1, map2, interpolation=cv2.INTER_LINEAR)
348 
349  try:
350  if(obj.hasFeatures() and len(obj.features) >= config.class_min_n_features): imgBox = tvaLib.Obj.imageBoxTI(img, obj, frameNum=frameNum, width=newImgSize[0], height=newImgSize[1], homography=invHomography)
351  else: imgBox,_,_,_,_ = tvaLib.Obj.imageBox(img, obj, frameNum=frameNum, homography=invHomography)
352  except ZeroDivisionError: continue
353 
354  if(imgBox.shape[0]*imgBox.shape[1] < config.class_minNPixels): continue
355  if(not os.path.exists(imageDirectories[obj.userType])): os.makedirs(imageDirectories[obj.userType])
356  if(imgBox is not None): cv2.imwrite(os.path.join(imageDirectories[obj.userType],sites[siteIx][camIx][fileIx].name+'_'+str(obj.num)+'.png'),imgBox)
357  elif(commands.verbose >= 5): tvaLib.printWarning('Empty imgbox for obj #'+str(obj.num)+' ('+('has features)' if obj.hasFeatures() else 'no features)'), local['gen_warning'])
358 
359  raise Exception, [5, 'Image extraction complete.']
360 
361 
362 
365  if(commands.train_classifier):
366 
367  from cvutils import HOG as TrafIntCVUtils_HOG
368  from ml import SVM as TrafIntML_SVM
369  import cv2
370 
371 
372  if(config.class_useAllSites): imageDirectories = dict([(x,os.path.join(config.dir, config.calibration_folder, 'Data', local['userTypeNames'][x])) for x in config.class_candidates])
373  else:
374  if(commands.interactive): commands = tvaRuntime.interactiveSiteSelection(commands, config, sites, local)
375  siteIx = sites.interpret(commands.site)[0]
376  imageDirectories = dict([(x,os.path.join(sites[siteIx][camIx].getClassificationTrainingBaseDirectory(local['userTypeNames'][x]))) for x in config.class_candidates])
377  if(commands.verbose): tvaLib.printTimeStamp('Training classification model (for '+', '.join([local['userTypeNames'][x] for x in config.class_candidates])+')...')
378  if(all([os.path.exists(imageDirectories[x]) for x in imageDirectories]) == False): raise Exception, [9358, 'Could not locate training data. Consider running --extract-images on multiple sites first.']
379 
380 
381 
382  trainingSamplesPBV = dict((c,[]) for c in config.class_candidates)
383  trainingSamplesBV = dict((c,[]) for c in config.class_candidates)
384  trainingSamplesPB = dict((c,[]) for c in config.class_candidates)
385  trainingSamplesPV = dict((c,[]) for c in config.class_candidates)
386  for classNum, dirpath in imageDirectories.iteritems():
387  if(commands.verbose >= 2): print('Loading '+Fore.MAGENTA+local['userTypeNames'][classNum]+Fore.RESET+' samples...')
388  for imageDirectory, dirnames, filenames in os.walk(dirpath):
389  if(dirnames == []):
390  if(commands.verbose >= 3): print(' Adding "'+imageDirectory+'"...')
391  for filename in filenames:
392  img = cv2.imread(os.path.join(imageDirectory,filename))
393  hog = TrafIntCVUtils_HOG(img, rescaleSize=config.class_hogRescaleSize, orientations=config.class_hogNOrientations, pixelsPerCell=config.class_hogNPixelsPerCell, cellsPerBlock=config.class_hogNCellsPerBlock, blockNorm=config.class_hogBlockNorm).tolist()
394  if(classNum==1 or classNum==2 or classNum==4): trainingSamplesPBV[classNum].append(hog)
395  if(classNum==1 or classNum==4): trainingSamplesBV[classNum].append(hog)
396  if(classNum==2 or classNum==4): trainingSamplesPB[classNum].append(hog)
397  if(classNum==1 or classNum==2): trainingSamplesPV[classNum].append(hog)
398 
399  if(commands.verbose >= 4):
400  print(' Histogram of values: ')
401  hy,hx = np.histogram(tvaLib.flatten_list(trainingSamplesPBV[classNum]))
402  tvaLib.printTable([[round(x,3) for x in hx],'--',hy], padding=' | ')
403 
404 
405 
406  if(commands.verbose >= 2): print('Training Pedestrian-Cyclist-Vehicle Model...')
407  model = TrafIntML_SVM(svmType=config.class_svmType, kernelType=config.class_kernelType, degree=config.class_degree, gamma=config.class_gamma, coef0=config.class_coef0, Cvalue=config.class_cvalue, nu=config.class_nu, p=config.class_svmP)
408  result = model.train(np.array([i for s in trainingSamplesPBV.values() for i in s], dtype=np.float32), np.array([k for k,s in zip(trainingSamplesPBV.keys(),trainingSamplesPBV.values()) for i in s], dtype=np.float32), True)
409  model.save(os.path.join(config.dir, config.calibration_folder, config.class_PBV_SVMFilename))
410  if(commands.verbose >= 3): print(' '+str(result))
411 
412  if(commands.verbose >= 2): print('Training Cyclist-Vehicle Model...')
413  model = TrafIntML_SVM(svmType=config.class_svmType, kernelType=config.class_kernelType, degree=config.class_degree, gamma=config.class_gamma, coef0=config.class_coef0, Cvalue=config.class_cvalue, nu=config.class_nu, p=config.class_svmP)
414  result = model.train(np.array([i for s in trainingSamplesBV.values() for i in s], dtype=np.float32), np.array([k for k,s in zip(trainingSamplesBV.keys(),trainingSamplesBV.values()) for i in s], dtype=np.float32), True)
415  model.save(os.path.join(config.dir, config.calibration_folder, config.class_BV_SVMFilename))
416  if(commands.verbose >= 3): print(' '+str(result))
417 
418  if(commands.verbose >= 2): print('Training Pedestrian-Cyclist Model...')
419  model = TrafIntML_SVM(svmType=config.class_svmType, kernelType=config.class_kernelType, degree=config.class_degree, gamma=config.class_gamma, coef0=config.class_coef0, Cvalue=config.class_cvalue, nu=config.class_nu, p=config.class_svmP)
420  result = model.train(np.array([i for s in trainingSamplesPB.values() for i in s], dtype=np.float32), np.array([k for k,s in zip(trainingSamplesPB.keys(),trainingSamplesPB.values()) for i in s], dtype=np.float32), True)
421  model.save(os.path.join(config.dir, config.calibration_folder, config.class_PB_SVMFilename))
422  if(commands.verbose >= 3): print(' '+str(result))
423 
424  if(commands.verbose >= 2): print('Training Pedestrian-Vehicle Model...')
425  model = TrafIntML_SVM(svmType=config.class_svmType, kernelType=config.class_kernelType, degree=config.class_degree, gamma=config.class_gamma, coef0=config.class_coef0, Cvalue=config.class_cvalue, nu=config.class_nu, p=config.class_svmP)
426  result = model.train(np.array([i for s in trainingSamplesPV.values() for i in s], dtype=np.float32), np.array([k for k,s in zip(trainingSamplesPV.keys(),trainingSamplesPV.values()) for i in s], dtype=np.float32), True)
427  model.save(os.path.join(config.dir, config.calibration_folder, config.class_PV_SVMFilename))
428  if(commands.verbose >= 3): print(' '+str(result))
429 
430  raise Exception, [5, 'Classification training complete.']
431 
432 
433 
436  if(commands.train_classifier or commands.test_classifier):
437  from cvutils import HOG as TrafIntCVUtils_HOG
438  from ml import SVM as TrafIntML_SVM
439  import cv2
440 
441 
442  if(config.class_useAllSites): imageDirectories = dict([(x,os.path.join(config.dir, config.calibration_folder, 'Data', local['userTypeNames'][x])) for x in config.class_candidates])
443  else:
444  if(commands.interactive): commands = tvaRuntime.interactiveSiteSelection(commands, config, sites, local)
445  siteIx = sites.interpret(commands.site)[0]
446  imageDirectories = dict([(x,os.path.join(sites[siteIx][camIx].getClassificationTrainingBaseDirectory(local['userTypeNames'][x]))) for x in config.class_candidates])
447  if(commands.verbose): tvaLib.printTimeStamp('Testing classification model (for '+', '.join([local['userTypeNames'][x] for x in config.class_candidates])+')...')
448  if(commands.verbose): tvaLib.printNotice('Please note that this test only tests classification of images. It is a less complete than testing classification with MOTA. Consider also running a MOTA test with a ground truth file and --mot.', local['notice'])
449  if(all([os.path.exists(imageDirectories[x]) for x in imageDirectories]) == False): raise Exception, [9358, 'Could not locate testing data. Consider running --extract-images on multiple sites first.']
450 
451 
452  svmModels = dict((svmModelName,TrafIntML_SVM()) for svmModelName in [config.class_PBV_SVMFilename,config.class_BV_SVMFilename,config.class_PB_SVMFilename,config.class_PV_SVMFilename])
453  for svmModelName in svmModels: svmModels[svmModelName].load(os.path.join(config.dir, config.calibration_folder, svmModelName))
454  if(commands.verbose >= 10): print(''.rjust(4,' ')+'SVM hashes: '+', '.join([x+'=>'+str(hash(svmModels[x])) for x in svmModels]))
455 
456  tally = []
457  for svmModelName in svmModels:
458  if(commands.verbose): print('Testing model '+Fore.CYAN+svmModelName+Fore.RESET+'...')
459  tally.append({})
460  for classNum, dirpath in imageDirectories.iteritems():
461  tally[-1][classNum] = []
462  for imageDirectory, dirnames, filenames in os.walk(dirpath):
463  for filename in filenames:
464  img = cv2.imread(os.path.join(imageDirectory,filename))
465  hog = TrafIntCVUtils_HOG(img, rescaleSize=config.class_hogRescaleSize, orientations=config.class_hogNOrientations, pixelsPerCell=config.class_hogNPixelsPerCell, cellsPerBlock=config.class_hogNCellsPerBlock, blockNorm=config.class_hogBlockNorm)
466  if classNum==4.0 : import pdb; pdb.set_trace()
467  if(svmModels[svmModelName].predict(hog) == classNum): tally[-1][classNum].append(1)
468  else: tally[-1][classNum].append(0)
469  if(commands.verbose): print(' '+Fore.MAGENTA+local['userTypeNames'][classNum].ljust(9)+Fore.RESET+' result: '+str(sum(tally[-1][classNum]))+'/'+str(len(tally[-1][classNum])))
470 
471  raise Exception, [5, 'Classification testing complete.']
472 
473 
476  if(commands.homo):
477  if(commands.interactive): commands = tvaRuntime.interactiveSiteSelection(commands, config, sites, local)
478  siteIx = sites.interpret(commands.site)[0]
479  camIx = sites[siteIx].interpret(commands.camera)[0]
480  fileIxs = sites[siteIx][camIx].interpret(commands.file)
481 
482  if(commands.verbose): tvaLib.printTimeStamp('Homography for site: '+sites[siteIx].name+'; camera: '+sites[siteIx][camIx].name+'...')
483  imageSpaceFrame = ''
484  if(os.path.exists(sites[siteIx][camIx].getBaseDirectory())):
485  os.chdir(sites[siteIx][camIx].getBaseDirectory())
486  else: raise Exception, [2070, 'Directory or data for this site does not exist.']
487  if(not sites[siteIx].satres): raise Exception, [2071, 'Satellite resolution not declared in the database. Please add the satellite resolution before proceeding.']
488 
489 
490  for fileIx in fileIxs:
491  if(sites[siteIx][camIx].camera.camera_matrix.data):
492  imageSpaceFrame = sites[siteIx][camIx][fileIx].getImageSpaceFrameFilename(forceDistorted=True)
493  sites[siteIx][camIx].homographyFilename = 'homography-undistort.txt'
494  else:
495  imageSpaceFrame = sites[siteIx][camIx][fileIx].getImageSpaceFrameFilename()
496  if(imageSpaceFrame): break
497  if(not imageSpaceFrame): imageSpaceFrame = tvaLib.crawlDirsForFilePattern(pattern='*frame*.png', path=os.getcwd(), returnList=False)
498  if(not imageSpaceFrame):
499  if(commands.verbose): print(' No default video frame found... Automatically dumping new frame for '+sites[siteIx].name+'; camera: '+sites[siteIx][camIx].name+'; sequence: '+sites[siteIx][camIx][fileIx].name+'...')
500  tvaLib.Obj.dumpFrame(sites[siteIx][camIx][fileIx].getFullVideoFilename())
501  if(not sites[siteIx][camIx].camera.camera_matrix.data): imageSpaceFrame = sites[siteIx][camIx][fileIx].getImageSpaceFrameFilename()
502  else:
503  if(commands.verbose): print(' New frame needs to be undistorted...')
504  tvaLib.Obj.undistortImage(sites[siteIx][camIx][fileIx].getImageSpaceFrameFilename(), sites[siteIx][camIx][fileIx].getImageSpaceFrameFilename(forceDistorted=True, allowNonExistant=True), camera_matrix=sites[siteIx][camIx].camera.camera_matrix.data, dist_coeffs=sites[siteIx][camIx].camera.dist_coeffs.data, display=commands.play, freeScalingParameter=sites[siteIx][camIx].camera.freeScalingParameter, imageScalingFactor=sites[siteIx][camIx].camera.imageScalingFactor, verbose=commands.verbose)
505  imageSpaceFrame = sites[siteIx][camIx][fileIx].getImageSpaceFrameFilename(forceDistorted=True)
506 
507 
508  tsaiCameraSrcPath = tvaLib.crawlDirsForFilePattern(pattern='*.tacal', path=os.getcwd(), returnList=False)
509  worldPts = homography = None
510  if(tsaiCameraSrcPath):
511  worldPts, homography = tvaLib.Obj.getHomography(imageSpaceFrame, sites[siteIx].getFullSatFilename(), tsaiCameraSrcPath=tsaiCameraSrcPath, nPoints=commands.homo, unitsPerPixel=sites[siteIx].satres, worldPts=None, fig_name=sites[siteIx].name+'/'+sites[siteIx][camIx].name, verbose=commands.verbose)
512  if(commands.verbose): print('Homography processed directly from total camera calibration (using file "'+tsaiCameraSrcPath+'").')
513  if(commands.verbose and (not imageSpaceFrame or not sites[siteIx].satFilename)): tvaLib.printNotice('No video frame or satelite image available for world point visualisation.', local['gen_notice'])
514  else:
515  if(not imageSpaceFrame):
516  if(commands.verbose): tvaLib.printNotice('No video frame found.', local['gen_notice'])
517  elif(not sites[siteIx].satFilename):
518  if(commands.verbose): tvaLib.printNotice('No satelite image defined.', local['gen_notice'])
519  elif(not os.path.exists(sites[siteIx].getFullSatFilename())):
520  if(commands.verbose): tvaLib.printNotice('No satelite image found at the location: '+sites[siteIx].getFullSatFilename(), local['gen_notice'])
521  else:
522  worldPts, homography = tvaLib.Obj.getHomography(imageSpaceFrame, sites[siteIx].getFullSatFilename(), savePath=sites[siteIx][camIx].getFullHomographyFilename(force=True), nPoints=commands.homo, unitsPerPixel=sites[siteIx].satres, worldPts=False, fig_name=sites[siteIx].name+'/'+sites[siteIx][camIx].name, verbose=commands.verbose)
523  sites[siteIx][camIx].setHomography(homography)
524 
525  if(homography is not None):
526  sites[siteIx][camIx].setHomography(homography)
527  sites.update()
528  if(True in [os.path.exists(x.getFullDataFilename()) for x in sites[siteIx][camIx]]):
529  try: flush = raw_input(Back.BLUE+'Previous tracking data exists on disk and will be void with a new homography. Do you wish to fluh data for this camera view? y/n:'+Back.RESET)
530  except: flush = None
531  if(type(flush) is str and flush.lower()=='y'):
532  for seq in sites[siteIx][camIx]:
533  if(os.path.exists(seq.getFullDataFilename())): os.remove(seq.getFullDataFilename())
534  if(os.path.exists(seq.getFullSerialisedFilename())): os.remove(seq.getFullSerialisedFilename())
535  raise Exception, [5, 'Homography data entry complete.']
536 
537 
540  if(commands.test_track or commands.test_track_param_filename):
541  try: import moving as TrafIntMoving
542  except ImportError: raise Exception, [103, 'Traffic-Intelligence is not installed.']
543  if(commands.interactive): commands = tvaRuntime.interactiveSiteAnalysisSelection(commands, config, site_analyses, sites, local)
544  siteIx = sites.interpret(commands.site)[0]
545  camIx = sites[siteIx].interpret(commands.camera)[0]
546  fileIxs = sites[siteIx][camIx].interpret(commands.file)
547  saIx = site_analyses.interpret(commands.s_analysis)[0]
548  if(commands.verbose): tvaLib.printTimeStamp('Running test tracking for: '+sites[siteIx].name+'/'+sites[siteIx][camIx].name+' (over '+str(commands.time)+'s)... Use -r for visual inspection of results.')
549 
550 
551  for fileIx in fileIxs:
552  if(os.path.exists(sites[siteIx][camIx][fileIx].getFullGroundTruthFilename()) and os.path.exists(sites[siteIx][camIx][fileIx].getFullVideoFilename())):
553  fileIxs = [fileIx]
554  break
555 
556 
557  for fileIx in fileIxs[1:]+fileIxs[:1]: #The first video file is often less reliable than the others
558  if(fileIx == None): continue
559  if(not os.path.exists(sites[siteIx][camIx][fileIx].getFullVideoFilename())):
560  tvaLib.printWarning('Could not find video file on disk. Check to make sure video file exists at "'+sites[siteIx][camIx][fileIx].getFullVideoFilename()+'" or that this path is properly entered in the database.', local['gen_warning'])
561  continue
562 
563 
564  firstInstant = int(commands.start_time*sites[siteIx][camIx].camera.frameRate)
565  if(commands.time): lastInstant = int(commands.start_time*sites[siteIx][camIx].camera.frameRate)+int(commands.time*sites[siteIx][camIx].camera.frameRate)
566  else: lastInstant = int(sites[siteIx][camIx][fileIx].getDuration()*sites[siteIx][camIx].camera.frameRate)
567 
568 
569  if(commands.play and commands.threads == 1): view = '1'
570  else: view = '0'
571  if(sys.platform == 'win32'): command = os.path.join(config.path_to_trafint, 'trafficintelligence.exe')
572  else: command = 'feature-based-tracking'
573  if(not commands.legacy): undistort_cmd_stack = sites[siteIx][camIx].camera.getUndistortCmdStack()
574  else: undistort_cmd_stack = []
575  cmd_stack = [command,
576  sites[siteIx][camIx].getFullConfigurationFilename(),
577  '--video-filename', sites[siteIx][camIx][fileIx].getFullVideoFilename(),
578  '--mask-filename', sites[siteIx][camIx].getFullMaskFilename(),
579  '--homography', sites[siteIx][camIx].getFullHomographyFilename(),
580  '--frame1', str(int(commands.start_time*sites[siteIx][camIx].camera.frameRate)),
581  '--nframes', str(int(commands.time*sites[siteIx][camIx].camera.frameRate)),
582  '--display', view]+undistort_cmd_stack
583 
584 
585  results_path = os.path.splitext(commands.test_track_param_filename)[0]+'-results.txt'
586  if(os.path.exists(commands.test_track_param_filename)):
587  tracking_cases = {}
588  with open(commands.test_track_param_filename, 'r') as f:
589  reader = csv_reader(f, delimiter=',', quotechar='"')
590  headers = next(reader)
591  headers[0] = headers[0].decode('utf-8-sig')
592  for row in reader:
593  tracking_cases[row[0]] = tvaLib.flatten_list([['--'+headers[cIx],row[cIx]] for cIx in range(len(row)) if headers[cIx]!='Name'])
594  if(os.path.exists(results_path)): os.remove(results_path)
595  with open(results_path, 'w') as f: pass
596  else: tracking_cases = {'basic':['']}
597 
598 
599  for filename in tvaLib.crawlDirsForFilePattern('*_*.sqlite', config.temp_dir, returnList=True):
600  try: os.remove(filename)
601  except: pass
602 
603 
604  combined_cmd_stacks = [cmd_stack+['--database-filename', os.path.join(config.temp_dir, sites[siteIx][camIx][fileIx].name+'_'+tvaLib.Parse.clean_file_name(tracking_case)+'.sqlite')]+tracking_cases[tracking_case] for tracking_case in tracking_cases]
605  if(config.debug):
606  try: logging.debug(tvaLib.commandStacktoShellString(combined_cmd_stacks[0]+['--tf']))
607  except: pass
608  tvaLib.command_async_pool([x+['--tf'] for x in combined_cmd_stacks], threads=commands.threads, timeout=config.s_proc_max_exec_time_in_s*10.0, prependMsg=' ', verbose=commands.verbose)
609  tvaLib.command_async_pool([x+['--gf'] for x in combined_cmd_stacks], threads=commands.threads, timeout=config.s_proc_max_exec_time_in_s*10.0, prependMsg=' ', verbose=commands.verbose)
610 
611  for tracking_case in tracking_cases:
612  if(commands.verbose): print(' Testing tracking case "'+tracking_case+'"...')
613 
614  seq_objects_src = tvaLib.Obj.loadObjects(os.path.join(config.temp_dir, sites[siteIx][camIx][fileIx].name+'_'+tvaLib.Parse.clean_file_name(tracking_case)+'.sqlite'), max_obj_features=config.max_obj_features, suppress_features=True)
615  seq_objects_src = tvaFilter.compensateParralax(seq_objects_src, sites[siteIx][camIx].camOrigin, sites[siteIx][camIx].camHeight, verbose=commands.verbose)
616  seq_objects_src,_ = tvaFilter.filtBoundingBox(seq_objects_src, site_analyses[saIx].zone, containment_threshold=config.f_bb_containment_threshold, max_outside_dist=config.f_bb_max_outside_dist, loopback_verification_frames=config.f_bb_loopback_ver_frames, bounding_boxes_label='analysis zone', verbose=0)
617  seq_objects_src = tvaFilter.filtTime(seq_objects_src, startTime=firstInstant, endTime=lastInstant, verbose=commands.verbose)
618 
619  if(len(seq_objects_src) == 0):
620  tvaLib.printWarning('No objects created for this tracking case!', local['gen_warning'])
621  continue
622 
623 
624  if(os.path.exists(sites[siteIx][camIx][fileIx].getFullGroundTruthFilename())):
625  if(commands.verbose): print(' Classifiying tracking...')
626  seq_objects_src = tvaFilter.classify(seq_objects_src, sites[siteIx][camIx], fileIx=fileIx, config=config, method=config.classification_method, verbose=commands.verbose)
627 
628  if(commands.verbose): print(' Loading and processing annotations...')
629  annotations = sites[siteIx][camIx][fileIx].loadAnnotations(indent=4, verbose=commands.verbose)
630  annotations = tvaFilter.corVelIndex(annotations, verbose=commands.verbose)
631  annotations = tvaFilter.compensateParralax(annotations, sites[siteIx][camIx].camOrigin, sites[siteIx][camIx].camHeight, verbose=commands.verbose)
632  annotations,_ = tvaFilter.filtBoundingBox(annotations, site_analyses[saIx].zone, containment_threshold=config.f_bb_containment_threshold, max_outside_dist=config.f_bb_max_outside_dist, loopback_verification_frames=config.f_bb_loopback_ver_frames, bounding_boxes_label='analysis zone', verbose=0)
633  annotations = tvaFilter.filtTime(annotations, startTime=firstInstant, endTime=lastInstant, verbose=commands.verbose)
634 
635  mot_results = TrafIntMoving.computeClearMOT(annotations, seq_objects_src, config.MOT_max_distance, firstInstant, lastInstant)+(len(annotations),len(seq_objects_src))
636  annot_class = [[x.getUserType() for x in annotations].count(i)/float(len(annotations)) for i in range(len(local['userTypeNames']))]
637  tracked_class = [[x.getUserType() for x in seq_objects_src].count(i)/float(len(seq_objects_src)) for i in range(len(local['userTypeNames']))]
638  normalisedScore = tvaCompute.normaliseMOTscore(motp=mot_results[0], mota=mot_results[1], n_annot=mot_results[6], n_tracked=mot_results[7], annot_class=annot_class, tracked_class=tracked_class)
639  if(tracking_cases[tracking_case] is not ['']):
640  with open(results_path, 'a') as f: f.write(tracking_case+' '+str(normalisedScore)+' '+str(mot_results)+' '+str(annot_class)+' '+str(tracked_class))
641  print(' Score: '+Back.MAGENTA+str(normalisedScore)+Back.RESET+' '+str(mot_results))
642 
643  if(commands.draw and commands.threads == 1):
644  tvaVis.traj2D(seq_objects_src, draw_max_traj=config.draw_max_traj, alignments=sites[siteIx].alignments, mask=sites[siteIx][camIx].mask, sateliteImage=sites[siteIx].getFullSatFilename(), sateliteResolution=sites[siteIx].satres, labels=False, local=local, fig_name='Sample tracking for new homography projection', figsize=config.figsize)
645  plt.show()
646 
647 
648 
649  break
650 
651  raise Exception, [5, 'Tracking test complete.']
652 
653 
654 
657  if(commands.mota_opt_t or commands.mota_opt_g or commands.mota_opt_f):
658  try: import moving as TrafIntMoving
659  except ImportError: raise Exception, [103, 'Traffic-Intelligence is not installed.']
660  if(commands.interactive): commands = tvaRuntime.interactiveSiteAnalysisSelection(commands, config, site_analyses, sites, local)
661 
662 
663  if(not commands.s_analysis): raise Exception, [2090, 'MOT-optimization requires a specific site-analysis explicitly.']
664  siteIx, camIx, fileIx = tvaRuntime.targetSequence(commands, sites, config)
665  saIx = site_analyses.interpret(commands.s_analysis)[0]
666  process_file_ids = [''.join(random_choice(string_ascii_uppercase + string_digits) for _ in range(20)) for _ in range(config.MOT_population_size)]
667  if(not os.path.exists(sites[siteIx][camIx][fileIx].getFullGroundTruthFilename())): raise Exception, [2091, 'Could not find ground truth file for MOT-optimization: "'+sites[siteIx][camIx][fileIx].getFullGroundTruthFilename()+'".']
668  if(not sites[siteIx][camIx].getHomography()): raise Exception, [2092, 'No homography available for target camera.']
669  if(commands.verbose): tvaLib.printTimeStamp('Running MOT tracking optimization for sequence '+sites[siteIx].name+'/'+sites[siteIx][camIx].name+'/'+sites[siteIx][camIx][fileIx].name+'...')
670  firstInstant = int(commands.start_time*sites[siteIx][camIx].camera.frameRate)
671  if(commands.time): lastInstant = int(commands.start_time*sites[siteIx][camIx].camera.frameRate)+int(commands.time*sites[siteIx][camIx].camera.frameRate)
672  else: lastInstant = int(sites[siteIx][camIx][fileIx].getDuration()*sites[siteIx][camIx].camera.frameRate)
673  if(commands.threads > 1):
674  try: from pathos.multiprocessing import Pool as multiprocessing_Pool
675  except: raise Exception, [2159, 'Use of multiple threads for this operation requires the pathos module (not installed).']
676 
677 
678 
679  if(commands.verbose): print(' Loading and processing annotations...')
680  annotations = sites[siteIx][camIx][fileIx].loadAnnotations(indent=4, verbose=commands.verbose)
681  annotations = tvaFilter.corVelIndex(annotations, verbose=commands.verbose)
682  annotations = tvaFilter.compensateParralax(annotations, sites[siteIx][camIx].camOrigin, sites[siteIx][camIx].camHeight, verbose=commands.verbose)
683  annotations,_ = tvaFilter.filtBoundingBox(annotations, site_analyses[saIx].zone, containment_threshold=config.f_bb_containment_threshold, max_outside_dist=config.f_bb_max_outside_dist, loopback_verification_frames=config.f_bb_loopback_ver_frames, bounding_boxes_label='analysis zone', verbose=0)
684  annotations = tvaFilter.filtTime(annotations, startTime=firstInstant, endTime=lastInstant, verbose=commands.verbose)
685 
686 
687  if(commands.play and commands.threads == 1): view = '1'
688  else: view = '0'
689  if(sys.platform == 'win32'): command = os.path.join(config.path_to_trafint, 'trafficintelligence.exe')
690  else: command = 'feature-based-tracking'
691  main_cmd_stack = [command,
692  site_analyses[saIx].getFullConfigurationFilename(config.output_folder),
693  '--video-filename', sites[siteIx][camIx][fileIx].getFullVideoFilename(),
694  '--mask-filename', sites[siteIx][camIx].getFullMaskFilename(),
695  '--homography', sites[siteIx][camIx].getFullHomographyFilename(),
696  '--frame1', str(int(commands.start_time*sites[siteIx][camIx].camera.frameRate)),
697  '--nframes', str(int(commands.time*sites[siteIx][camIx].camera.frameRate)),
698  '--display', view]
699 
700  #Undistortion parameters
701  if(not commands.legacy): undistort_cmd_stack = sites[siteIx][camIx].camera.getUndistortCmdStack()
702  else: undistort_cmd_stack = []
703 
704 
705  parameters_t = {'t-min-feature-distanceklt': [0,6,float], #closer to 1
706  't-feature-quality': [0,0.4,float], #closer to 0.1
707  't-window-size': [3,10,int], #closer to 6, but unpredictable
708  }
709  parameters_g = {'t-min-feature-time': [1,15,int], #closer to 3
710  't-mm-connection-distance': [1.2,5.0,float], #closer to 1.85
711  't-mm-segmentation-distance': [0.5,5,float], #closer to 2.2
712  't-min-nfeatures-group': [1,5,float] #closer to 3
713  }
714  parameters_f = {'f-bb-containment_threshold': [0.7,0.95,float], #closer to 0.9
715  'f-bb-loopback_verification_frames': [5,40,int], #closer to 20
716  'f-so-hard_maxSpeed': [4.0,8.0,float], #closer to 6.0
717  'f-so-soft_maxSpeed': [1.0,3.0,float], #closer to 2.0
718  'f-so-max_angle': [35.0,90.0,float], #closer to 45.0
719  'f-te-config_min_traj_len': [10,30,int], #closer to 20
720  'f-td-minimumSeperationDistance': [3.0,10.0,float], #closer to 8.0
721  'f-td-minimumSeperationVelocity': [2.0,12.0,float], #closer to 8.0
722  'f-fs-max_dwell_time': [0.2,2.0,float], #closer to 0.66
723  'f-fs-search_radius': [2.0,12.0,float], #closer to 8.0
724  'f-fs-search_window': [10,100,float], #closer to 50
725  'f-fs-speed_adj_factor': [500.0,1500.00,float], #closer to 1000.0
726  'f-fs-speed_similarity_limit': [0.1,0.6,float], #closer to 0.3
727  'f-fs-speed_sim_limit_min': [5.0,20.0,float], #closer to 10.0
728  'f-fs-stop_speed_detection_limit': [2.0,7.0,float], #closer to 5.5
729  'f-fs-angle_similairty_deg': [40.0,120.0,float] #closer to 80.0
730  }
731 
732 
733  parameters = {}
734  if(commands.mota_opt_t): parameters.update(parameters_t)
735  if(commands.mota_opt_g): parameters.update(parameters_g)
736  if(commands.mota_opt_f): parameters.update(parameters_f)
737 
738 
739 
740  if(commands.verbose): print(' Purging any previous trajectories...')
741  try: os.remove(os.path.join(site_analyses[saIx].getFullResultsFolder(), 'MOT-calibration.txt'))
742  except: pass
743  for filename in tvaLib.crawlDirsForFilePattern('*-opt*.sqlite', config.temp_dir, returnList=True):
744  try: os.remove(filename)
745  except: pass
746  with open(os.path.join(site_analyses[saIx].getFullResultsFolder(), 'MOT-calibration.txt'),'a') as f:
747  f.write('Generation #; Score; Results; File; Params;\n')
748 
749 
750  if(commands.mota_opt_g and not commands.mota_opt_t):
751  if(commands.verbose): print(' Running pre-tracking...')
752  combined_cmd_stacks = []
753  try: os.remove(os.path.join(config.temp_dir, process_file_ids[0]+'-opt_base.sqlite'))
754  except: pass
755  if(os.path.exists(os.path.join(sites[siteIx][camIx].getBaseDirectory(), sites[siteIx][camIx][fileIx].name+'-tracking.sqlite'))):
756  try: shutil_copy(os.path.join(sites[siteIx][camIx].getBaseDirectory(), sites[siteIx][camIx][fileIx].name+'-tracking.sqlite'), os.path.join(config.temp_dir, process_file_ids[0]+'-opt_base.sqlite'))
757  except IOError: raise Exception, [2750, 'Insufficient disk space. Consider freeing up disk space on the main hard drive, or set path to external scratch disk in configuration file.']
758  else:
759  combined_cmd_stacks.append(main_cmd_stack+['--database-filename', os.path.join(config.temp_dir, process_file_ids[0]+'-opt_base.sqlite')]+undistort_cmd_stack+['--tf'])
760  if(config.debug): logging.debug(tvaLib.commandStacktoShellString(combined_cmd_stacks[0]))
761  results = tvaLib.command_async_pool(combined_cmd_stacks, threads=commands.threads, timeout=config.s_proc_max_exec_time_in_s, prependMsg=' ', verbose=commands.verbose)
762 
763 
764  if(commands.mota_opt_f):
765  if(commands.verbose): print(' Running pre-tracking and grouping...')
766  try: os.remove(os.path.join(config.temp_dir, process_file_ids[0]+'-opt.sqlite'))
767  except: pass
768  if(tvaLib.command_timeout(main_cmd_stack+['--database-filename', os.path.join(config.temp_dir, process_file_ids[0]+'-opt.sqlite')]+undistort_cmd_stack+['--tf'], config.s_proc_max_exec_time_in_s, commands.verbose)):
769  tvaLib.command_timeout(main_cmd_stack+['--database-filename', os.path.join(config.temp_dir, process_file_ids[0]+'-opt.sqlite')]+undistort_cmd_stack+['--gf'], config.s_proc_max_exec_time_in_s, commands.verbose)
770 
771  # Generate chromosomes
772  chromosomes = tvaLib.Math.geneticAlg(parameters, config.MOT_population_size)
773  bestFit = 0
774  bestSpawn = None
775 
776 
777  if(config.MOT_save_all==True and (commands.mota_opt_t or commands.mota_opt_g)):
778  mot_trial = 0
779  while True:
780  mot_trial+=1
781  if(not os.path.exists(os.path.join(sites[siteIx][camIx].getBaseDirectory(), config.MOT_save_all_dir_name, 'Trial'+str(mot_trial)))):
782  os.makedirs(os.path.join(sites[siteIx][camIx].getBaseDirectory(), config.MOT_save_all_dir_name, 'Trial'+str(mot_trial)))
783  break
784 
785 
786  generationCount=0
787  while True:
788  generationCount += 1
789  if(commands.verbose): print('---------- Generation #'+str(generationCount))
790 
791 
792  if(commands.mota_opt_t or commands.mota_opt_g):
793 
794  #Generate commands
795  config_cmd_stacks = [tvaLib.flatten_list([['--'+x[2:], str(chromosomes[cIx]['params'][x])] for x in chromosomes[cIx]['params'] if x.startswith('t-')]) for cIx in range(len(chromosomes))]
796 
797  #Run commands (tracking)
798  if(commands.mota_opt_t):
799  if(commands.verbose): print(' Running tracking (max exec: '+str(config.s_proc_max_exec_time_in_s)+'s)...')
800  combined_cmd_stacks = [main_cmd_stack+['--database-filename', os.path.join(config.temp_dir, process_file_ids[cIx]+'-opt.sqlite')]+undistort_cmd_stack+config_cmd_stacks[cIx]+['--tf'] for cIx in range(len(config_cmd_stacks))]
801  if(config.debug): logging.debug(tvaLib.commandStacktoShellString(combined_cmd_stacks[0]))
802  results = tvaLib.command_async_pool(combined_cmd_stacks, threads=commands.threads, timeout=config.s_proc_max_exec_time_in_s, prependMsg=' ', verbose=commands.verbose)
803  if(config.debug and False in results):
804  logging.debug('At least one process had to be terminated prematurely. Corresponding configurations:')
805  logging.debug([cmd if result==False else '' for cmd,result in zip(config_cmd_stacks,results)])
806  #If only grouping is being optimized, trajectory files need only be rebuilt from originals
807  elif(commands.mota_opt_g):
808  for cIx in range(config.MOT_population_size):
809  if(os.path.exists(os.path.join(config.temp_dir, process_file_ids[0]+'-opt_base.sqlite'))):
810  try: shutil_copy(os.path.join(config.temp_dir, process_file_ids[0]+'-opt_base.sqlite'), os.path.join(config.temp_dir, process_file_ids[cIx]+'-opt.sqlite'))
811  except IOError: raise Exception, [2750, 'Insufficient disk space. Consider freeing up disk space on the main hard drive, or set path to external scratch disk in configuration file.']
812  else: raise Exception, [2751, 'Could not find, base tracking file "'+os.path.join(config.temp_dir, process_file_ids[0]+'-opt_base.sqlite')+'". Did another process delete it from the scratch disks?']
813  #Run commands (grouping)
814  if(commands.verbose): print(' Running grouping (max exec: '+str(config.s_proc_max_exec_time_in_s)+'s)...')
815  combined_cmd_stacks = [main_cmd_stack+['--database-filename', os.path.join(config.temp_dir, process_file_ids[cIx]+'-opt.sqlite')]+undistort_cmd_stack+config_cmd_stacks[cIx]+['--gf'] for cIx in range(len(chromosomes))]
816  if(config.debug): logging.debug(tvaLib.commandStacktoShellString(combined_cmd_stacks[0]))
817  tvaLib.command_async_pool(combined_cmd_stacks, threads=commands.threads, timeout=config.s_proc_max_exec_time_in_s, prependMsg=' ', verbose=commands.verbose)
818 
819 
820  if(commands.verbose): print(' Loading and processing tracked trajectories...')
821  if(commands.mota_opt_f): seq_objects_src = tvaLib.Obj.loadObjects(os.path.join(config.temp_dir, process_file_ids[0]+'-opt.sqlite'), max_obj_features=config.max_obj_features, suppress_features=True)
822  else: seq_objects_src = None
823  if(config.MOT_save_all==True and (commands.mota_opt_t or commands.mota_opt_g)):
824  if(not os.path.exists(os.path.join(sites[siteIx][camIx].getBaseDirectory(), config.MOT_save_all_dir_name, 'Trial'+str(mot_trial), 'Gen'+str(generationCount)))):
825  os.makedirs(os.path.join(sites[siteIx][camIx].getBaseDirectory(), config.MOT_save_all_dir_name, 'Trial'+str(mot_trial), 'Gen'+str(generationCount)))
826 
827 
829  if(commands.verbose): print(' Running MOTA analysis...')
830  arguments = {'config':config,'commands':commands,'seq_objects_src':seq_objects_src,'zone':site_analyses[saIx].zone,'mask':sites[siteIx][camIx].mask,'frameRate':sites[siteIx][camIx].camera.frameRate,'camOrigin':sites[siteIx][camIx].camOrigin,'camHeight':sites[siteIx][camIx].camHeight,'chromosomes':chromosomes,'annotations':annotations,'firstInstant':firstInstant,'lastInstant':lastInstant,'process_file_ids':process_file_ids}
831  if(commands.threads<=1): mot_results = [tvaCompute.motaWorker(tvaLib.mergeDicts({'cIx':cIx}, arguments)) for cIx in range(len(chromosomes))]
832  else: mot_results = multiprocessing_Pool(commands.threads).map(tvaCompute.motaWorker, [tvaLib.mergeDicts({'cIx':cIx}, arguments) for cIx in range(len(chromosomes))])
833  '''except:
834  mot_results = (0.0,0.0,0,0,0,0)
835  tvaLib.printWarning('There was an unexpected problem with the MOT calculation ('+str(len(annotations))+','+str(len(seq_objects))+'). This trial will be skipped.', local['gen_warning'])
836  '''
837 
838 
839  for cIx in range(len(chromosomes)):
840  ''' Score is normalized between 0 (undesirable) and 1 (desirable), with 30% weight to MOTP, 55% weight to MOTA, and 15% weight to number of objects. Each of these is normalized between 0 and 1 as well. '''
841  normalisedScore = tvaCompute.normaliseMOTscore(motp=mot_results[0], mota=mot_results[1], n_annot=mot_results[6], n_tracked=mot_results[7])
842  print(' Chromosome #'+str(cIx+1)+'; Score: '+str(normalisedScore)+'; MOT results: '+str(mot_results[cIx]))
843  chromosomes.setScore(cIx, normalisedScore)
844 
845  if(normalisedScore>bestFit):
846  bestFit = normalisedScore
847  bestSpawn = tvaLib.flatten_list([[x[2:], str(chromosomes[cIx]['params'][x])] for x in chromosomes[cIx]['params']])
848  with open(os.path.join(site_analyses[saIx].getFullResultsFolder(), 'MOT-calibration_best.txt'),'wb') as f:
849  f.write('Generation #'+str(generationCount)+' normalized score: '+str(normalisedScore)+'\n')
850  f.write('motp, mota, mt, mme, fpt, gt'+"\n")
851  f.write(str(mot_results[cIx])+"\n")
852  f.write(str(bestSpawn))
853  if(config.MOT_save_best==True and (commands.mota_opt_t or commands.mota_opt_g)):
854  shutil_copy(os.path.join(config.temp_dir, process_file_ids[cIx]+'-opt.sqlite'), os.path.join(sites[siteIx][camIx].getBaseDirectory(), 'best-mota.sqlite'))
855 
856  with open(os.path.join(site_analyses[saIx].getFullResultsFolder(), 'MOT-calibration.txt'),'a') as f:
857  f.write(str(generationCount)+'; '+str(normalisedScore)+'; '+str(mot_results[cIx])+'; '+str(process_file_ids[cIx])+'-opt.sqlite; '+str(tvaLib.flatten_list([[x[2:], str(chromosomes[cIx]['params'][x])] for x in chromosomes[cIx]['params']]))+";\n")
858  if(config.MOT_save_all==True and (commands.mota_opt_t or commands.mota_opt_g)):
859  shutil_copy(os.path.join(config.temp_dir, process_file_ids[cIx]+'-opt.sqlite'), os.path.join(sites[siteIx][camIx].getBaseDirectory(), config.MOT_save_all_dir_name, 'Trial'+str(mot_trial), 'Gen'+str(generationCount)))
860  if(normalisedScore>config.MOT_end_criteria or (config.MOT_max_generations and generationCount>=config.MOT_max_generations)):
861  print(bestFit)
862  print(bestSpawn)
863  sys.exit()
864 
865 
866 
867  print(' =====> Best fit this generation: '+Back.MAGENTA+str(bestFit)+Back.RESET+' <=====')
868  print(' Running genetic algorithm selection...')
869  chromosomes.naturalSelection(recipe=0, scoreThreshold=0.3)
870  print(' '+chromosomes.naturalSelectionStats())
871 
872  #Purge runs
873  if(commands.verbose): print(' Purge data...')
874  for cIx in range(config.MOT_population_size):
875  try: os.remove(os.path.join(config.temp_dir, process_file_ids[cIx]+'-opt.sqlite'))
876  except: pass
877 
878  # Clean-up any remaining tracking files
879  for filename in tvaLib.crawlDirsForFilePattern('*-opt*.sqlite', config.temp_dir, returnList=True):
880  try: os.remove(filename)
881  except: pass
882 
883  raise Exception, [6, 'MOT-optimization complete.']
884 
885 
888  if(commands.trafint):
889  if(commands.interactive): commands = tvaRuntime.interactiveSiteSelection(commands, config, sites, local, showTrackingCompletion=True)
890 
891  siteIxs = sites.interpret(commands.site)
892  for siteIx in siteIxs:
893  camIxs = sites[siteIx].interpret(commands.camera)
894 
895  for camIx in camIxs:
896  os.chdir(os.path.join(sites.getBaseDirectory(), sites[siteIx].name, sites[siteIx][camIx].name))
897  fileIxs = sites[siteIx][camIx].interpret(commands.file)
898 
899  #Undistortion parameters
900  if(not commands.legacy): undistort_cmd_stack = sites[siteIx][camIx].camera.getUndistortCmdStack()
901  else: undistort_cmd_stack = []
902 
903 
904  combined_cmd_stacks = []
905  for fileIx in fileIxs:
906  if(fileIx == None): continue
907  if(commands.save_data_flush or commands.save_data_flush_objects):
908  tvaLib.printWarning('Data flushing is selected. Any previously stored trajectories and serialized data will be purged permanently for sequence: '+sites[siteIx][camIx][fileIx].name, local['gen_warning'])
909  if(os.path.exists(sites[siteIx][camIx][fileIx].getFullDataFilename())): os.remove(sites[siteIx][camIx][fileIx].getFullDataFilename())
910  if(os.path.exists(sites[siteIx][camIx][fileIx].getFullSerialisedFilename())): os.remove(sites[siteIx][camIx][fileIx].getFullSerialisedFilename())
911  if(not os.path.exists(sites[siteIx][camIx][fileIx].getFullVideoFilename())):
912  tvaLib.printWarning('Could not find video file on disk. Check to make sure video file exists at "'+sites[siteIx][camIx][fileIx].getFullVideoFilename()+'" or that this path is properly entered in the database.', local['gen_warning'])
913  continue
914 
915 
916  if(commands.play and commands.threads == 1): view = '1'
917  else: view = '0'
918  if(sys.platform == 'win32'): command = os.path.join(config.path_to_trafint, 'trafficintelligence.exe')
919  else: command = 'feature-based-tracking'
920  main_cmd_stack = [command,
921  sites[siteIx][camIx].getFullConfigurationFilename(),
922  '--video-filename', sites[siteIx][camIx][fileIx].getFullVideoFilename(),
923  '--database-filename', sites[siteIx][camIx][fileIx].getFullDataFilename(),
924  '--mask-filename', sites[siteIx][camIx].getFullMaskFilename(),
925  '--homography', sites[siteIx][camIx].getFullHomographyFilename(),
926  '--frame1', str(int(commands.start_time*sites[siteIx][camIx].camera.frameRate)),
927  '--nframes', str(int(commands.time*sites[siteIx][camIx].camera.frameRate)),
928  '--display', view]
929 
930  combined_cmd_stacks.append(main_cmd_stack+undistort_cmd_stack)
931 
932  if(commands.verbose): tvaLib.printTimeStamp('Running tracking for site: '+sites[siteIx].name+'; camera: '+sites[siteIx][camIx].name+'...')
933  if(config.debug):
934  try: logging.debug(tvaLib.commandStacktoShellString(combined_cmd_stacks[0]+['--tf']))
935  except: pass
936  tvaLib.command_async_pool([x+['--tf'] for x in combined_cmd_stacks], threads=commands.threads, timeout=0, prependMsg=' ', descriptor='sequence', verbose=commands.verbose)
937  if(commands.verbose): tvaLib.printTimeStamp('Running grouping for site: '+sites[siteIx].name+'; camera: '+sites[siteIx][camIx].name+'...')
938  tvaLib.command_async_pool([x+['--gf'] for x in combined_cmd_stacks], threads=commands.threads, timeout=0, prependMsg=' ', descriptor='sequence', verbose=commands.verbose)
939  raise Exception, [7, 'Traffic-Intelligence complete.']
940 
941 
944  if(commands.detroy_tva_data or commands.detroy_tva_data_ser_traj):
945  if(commands.interactive): commands = tvaRuntime.interactiveSiteSelection(commands, config, sites, local)
946  siteIxs = sites.interpret(commands.site)
947  if(commands.detroy_tva_data and len(siteIxs)>1): raise Exception, [2980, 'As a safety precaution, the deletion of more than one site at a time is disabled.']
948  tvaLib.printWarning('You are about to permanently delete the saved tvaLib data associated with the following sequences:', local['gen_warning'])
949 
950  for siteIx in siteIxs:
951  camIxs = sites[siteIx].interpret(commands.camera)
952 
953  for camIx in camIxs:
954  fileIxs = sites[siteIx][camIx].interpret(commands.file)
955 
956  for fileIx in fileIxs:
957  print(' '+sites[siteIx].name+'/'+sites[siteIx][camIx].name+'/'+sites[siteIx][camIx][fileIx].name)
958  tvaLib.printWarning('Are you sure you wish to continue with the deletion? This action is permanent (y/n):', local['gen_warning'])
959  result = raw_input('')
960  if(result=='y'):
961 
962  for siteIx in siteIxs:
963  camIxs = sites[siteIx].interpret(commands.camera)
964 
965  for camIx in camIxs:
966  fileIxs = sites[siteIx][camIx].interpret(commands.file)
967 
968  for fileIx in fileIxs:
969 
970  if(os.path.exists(sites[siteIx][camIx][fileIx].getFullSerialisedFilename())):
971  try:
972  os.remove(sites[siteIx][camIx][fileIx].getFullSerialisedFilename())
973  tvaLib.printNotice('Removed '+sites[siteIx][camIx][fileIx].getFullSerialisedFilename(), local['gen_notice'])
974  except: tvaLib.printWarning('Removal of '+sites[siteIx][camIx][fileIx].getFullSerialisedFilename()+' failed.', local['gen_warning'])
975 
976  if(commands.detroy_tva_data):
977  if(os.path.exists(sites[siteIx][camIx][fileIx].getFullAnnotationFilename())):
978  try:
979  os.remove(sites[siteIx][camIx][fileIx].getFullAnnotationFilename())
980  tvaLib.printNotice('Removed '+sites[siteIx][camIx][fileIx].getFullAnnotationFilename(), local['gen_notice'])
981  except: tvaLib.printWarning('Removal of '+sites[siteIx][camIx][fileIx].getFullAnnotationFilename()+' failed.', local['gen_warning'])
982  if(os.path.exists(sites[siteIx][camIx][fileIx].getFullEventsFilename())):
983  try:
984  os.remove(sites[siteIx][camIx][fileIx].getFullEventsFilename())
985  tvaLib.printNotice('Removed '+sites[siteIx][camIx][fileIx].getFullEventsFilename(), local['gen_notice'])
986  except: tvaLib.printWarning('Removal of '+sites[siteIx][camIx][fileIx].getFullEventsFilename()+' failed.', local['gen_warning'])
987  if(os.path.exists(sites[siteIx][camIx][fileIx].getFullEventsCSVFilename())):
988  try:
989  os.remove(sites[siteIx][camIx][fileIx].getFullEventsCSVFilename())
990  tvaLib.printNotice('Removed '+sites[siteIx][camIx][fileIx].getFullEventsCSVFilename(), local['gen_notice'])
991  except: tvaLib.printWarning('Removal of '+sites[siteIx][camIx][fileIx].getFullEventsCSVFilename()+' failed.', local['gen_warning'])
992  raise Exception, [8, 'tvaLib cleanup complete.']
993  if(commands.detroy_figures_by_type):
994  if(commands.detroy_figures_by_type.lower() not in ['pdf','svg', 'png', 'jpg']): raise Exception, [2986, 'it is only possible to delete figures of the following filetypes: pdf, svg, png, jpg']
995  tvaLib.printWarning('Are you sure you wish to continue with figure deletion of type "'+commands.detroy_figures_by_type.lower()+'"? This action is permanent (y/n):', local['gen_warning'])
996  result = raw_input('')
997  if(result=='y'):
998  saIxs = site_analyses.interpret(commands.s_analysis)
999  for saIx in saIxs:
1000  for rootDir, dirs, files in os.walk(site_analyses[saIx].getFullResultsFolder()):
1001  for name in files:
1002  filename, file_extension = os.path.splitext(name)
1003  if(file_extension.lower().replace('.', '')==commands.detroy_figures_by_type.lower()): os.remove(site_analyses[saIx].getFullResultsFolder())
1004 
1005  raise Exception, [8, 'tvaLib cleanup complete.']
1006 
1007 
1010  if(commands.detroy_ti_data):
1011  if(commands.interactive): commands = tvaRuntime.interactiveSiteSelection(commands, config, sites, local)
1012  siteIxs = sites.interpret(commands.site)
1013  if(len(siteIxs)>1): raise Exception, [2981, 'As a safety precaution, the deletion of more than one site at a time is disabled.']
1014  tvaLib.printWarning('You are about to permanently delete the tracking data associated with the following files:', local['gen_warning'])
1015 
1016  for siteIx in siteIxs:
1017  camIxs = sites[siteIx].interpret(commands.camera)
1018 
1019  for camIx in camIxs:
1020  fileIxs = sites[siteIx][camIx].interpret(commands.file)
1021 
1022  for fileIx in fileIxs:
1023  print(' '+sites[siteIx].name+'/'+sites[siteIx][camIx].name+'/'+sites[siteIx][camIx][fileIx].name)
1024  tvaLib.printWarning('Are you sure you wish to continue with the deletion? This action is permanent (y/n):', local['gen_warning'])
1025  result = raw_input('')
1026  if(result=='y'):
1027 
1028  for siteIx in siteIxs:
1029  camIxs = sites[siteIx].interpret(commands.camera)
1030 
1031  for camIx in camIxs:
1032  fileIxs = sites[siteIx][camIx].interpret(commands.file)
1033 
1034  for fileIx in fileIxs:
1035  try:
1036  try: os.remove(sites[siteIx][camIx][fileIx].getFullSerialisedFilename())
1037  except: pass
1038  os.remove(sites[siteIx][camIx][fileIx].getFullDataFilename())
1039  tvaLib.printNotice('Removed '+sites[siteIx][camIx][fileIx].getFullDataFilename()+'', local['gen_notice'])
1040  except: tvaLib.printWarning('Removal of '+sites[siteIx][camIx][fileIx].getFullDataFilename()+' failed.', local['gen_warning'])
1041  raise Exception, [9, 'TI tracking data cleanup complete.']
1042 
1043 
1046  if(commands.disk_space_usage):
1047  usage_data_types = {'video_extensions' : ['.mp4', '.mpg', '.avi', '.flv'],
1048  'sequences' : ['.sqlite'],
1049  'serialised_extensions' : ['.pva','.traj','.upairs'],
1050  'analysis_image_extensions' : ['.png', '.pdf', '.jpg','.svg'],
1051  'analysis_dump_extensions' : ['.csv','.xlsx'],
1052  'text_extensions' : ['.txt','.log','.cfg']}
1053  col_width0 = 35
1054  col_width1 = 20
1055  usage_data = dict((k, []) for k in usage_data_types)
1056  usage_data_other = []
1057  if(commands.verbose): tvaLib.printTimeStamp('Calculating disk space usage...')
1058  for rootDir, dirs, files in os.walk(sites.getBaseDirectory()):
1059  for name in files:
1060  filename, file_extension = os.path.splitext(name)
1061  gotIt = False
1062  for usage_data_name,usage_data_type in usage_data_types.iteritems():
1063  if(file_extension.lower() in usage_data_type):
1064  gotIt = True
1065  usage_data[usage_data_name].append(os.path.getsize(os.path.join(rootDir, name)))
1066  break
1067  if(not gotIt):
1068  usage_data_other.append(os.path.getsize(os.path.join(rootDir, name)))
1069 
1070  for usage_data_name,usage_datum in usage_data.iteritems():
1071  print((usage_data_name+': ').ljust(col_width0)+('{:12.3f}'.format(sum(usage_datum)/1048576.0)+' MB').ljust(col_width1)+'('+str(len(usage_datum))+' files)')
1072  print('====================')
1073  print('TOTAL:'.ljust(col_width0)+('{:12.3f}'.format(sum([sum(usage_datum) for _,usage_datum in usage_data.iteritems()])/1048576.0)+' MB').ljust(col_width1)+'('+str(sum([len(usage_datum) for _,usage_datum in usage_data.iteritems()]))+' files)')
1074  print(('Other files: ').ljust(col_width0)+('{:12.3f}'.format(sum(usage_data_other)/1048576.0)+' MB').ljust(col_width1)+'('+str(len(usage_data_other))+' files)')
1075 
1076  raise Exception, [10, 'Disk space usage complete.']
1077 
1078 
1079 
1082  if(commands.rebuild_file_structure or commands.rfs_from_disk):
1083  if(commands.verbose): tvaLib.printTimeStamp('Rebuilding file structure...')
1084 
1085  if(commands.rfs_from_disk):
1086 
1087  if(not os.path.exists(os.path.join(commands.rfs_from_disk,config.dbn))): raise Exception, [2356, 'Target data location "'+os.path.join(commands.rfs_from_disk,config.dbn)+'" not recognised or inaccessible. Check path and try again.']
1088  tvaLib.printWarning('This will overwrite any existing database and metadata files. Are you sure you wish to continue? This action is permanent (y/n):', local['gen_warning'])
1089  result = raw_input('')
1090  if(result!='y'): raise Exception, [11, 'File structure rebuilding aborted.']
1091 
1092  shutil_copy(os.path.join(commands.rfs_from_disk,config.dbn), os.path.join(config.dir,config.dbn))
1093  sites = tvaScene.Sites(config)
1094 
1095  for site in sites:
1096  if(not os.path.exists(os.path.join(sites.getBaseDirectory(), site.name))): os.makedirs(os.path.join(sites.getBaseDirectory(), site.name))
1097  if(commands.rfs_from_disk): shutil_copy(os.path.join(commands.rfs_from_disk, site.name, site.satFilename), site.getFullSatFilename())
1098  for cam in site:
1099  if(not os.path.exists(os.path.join(sites.getBaseDirectory(), site.name, cam.name))): os.makedirs(os.path.join(sites.getBaseDirectory(), site.name, cam.name))
1100 
1101  raise Exception, [11, 'File structure rebuilding complete.']
1102 
1103 
1104 
1107  if(commands.create_camtype or commands.create_site or commands.create_view or commands.create_seqs or commands.create_sa or commands.create_analysis):
1108  if(commands.verbose): tvaLib.printTimeStamp('Editing meta database file...')
1109  if(commands.create_camtype): tvaDraw.dataEntryCamType(tvaScene.CameraTypes(config), local=local)
1110  if(commands.create_site): tvaDraw.dataEntrySite(sites, config=config, local=local)
1111  if(commands.create_view): tvaDraw.dataEntryCameraView(sites, config, tvaScene.CameraTypes(config), local=local)
1112  if(commands.create_seqs): tvaDraw.dataEntrySequences(sites, local=local)
1113  if(commands.create_analysis): tvaDraw.dataEntryAnalysis(tvaScene.Analyses(config), local=local)
1114  if(commands.create_sa): tvaDraw.dataEntrySA(sites, site_analyses, local=local)
1115  raise Exception, [12, 'Metadata creation completion.']
1116 
1117 
1120 
1121 
1124  if(commands.draw): commands = tvaRuntime.interactiveSiteAnalysisSelection(commands, config, site_analyses, sites, local)
1125 
1126 
1129  if(len(commands.sample_traj) > 0):
1130  if(commands.interactive): commands = tvaRuntime.interactiveSiteSelection(commands, config, sites, local)
1131  if(not (commands.site and commands.camera and commands.file)): raise Exception, [300, 'Plotting mode requires explicit sequence declaration.']
1132 
1133  siteIx, camIx, fileIx = tvaRuntime.targetSequence(commands, sites, config)
1134  if(commands.verbose): tvaLib.printTimeStamp('Preparing plot using data in sequence: '+sites[siteIx].name+'/'+sites[siteIx][camIx].name+'/'+sites[siteIx][camIx][fileIx].name+'...')
1135  objects = sites[siteIx][camIx][fileIx].loadObjects(max_obj=commands.max_obj, max_obj_features=config.max_obj_features, suppress_features=True, indent=4, verbose=commands.verbose)
1136  objects = tvaFilter.compensateParralax(objects, sites[siteIx][camIx].camOrigin, sites[siteIx][camIx].camHeight, verbose=commands.verbose)
1137  if(not commands.draw_transform): objects = tvaFilter.transformTrajectories(objects, sites[siteIx][camIx][fileIx].translationX, sites[siteIx][camIx][fileIx].translationY, sites[siteIx][camIx][fileIx].rotation, verbose=commands.verbose)
1138 
1139  input_objects = []
1140  for i in range(len(objects)):
1141  if(i in commands.sample_traj):
1142  input_objects.append(objects[i])
1143  figures = [tvaVis.TrajSample(input_objects, figsize=config.figsize, traj_colour='random')]
1144  input_objects = None
1145  plt.show()
1146  plt.close()
1147  #Close up plotting operation
1148  if(not commands.draw): raise Exception, [11, 'Plotting interrupt complete.']
1149 
1150 
1151 
1154  if(commands.draw_mask or commands.draw_mask2):
1155  if(commands.interactive): commands = tvaRuntime.interactiveSiteSelection(commands, config, sites, local)
1156  if(commands.file is None): commands.file = config.default_sequence_no
1157  if(not (commands.site and commands.camera)): raise Exception, [301, 'Mask drawing requires explicit camera declaration.']
1158 
1159  siteIx, camIx, fileIx = tvaRuntime.targetSequence(commands, sites, config)
1160  if(commands.verbose): tvaLib.printTimeStamp('Preparing mask drawing using data in sequence: '+sites[siteIx].name+'/'+sites[siteIx][camIx].name+'/'+sites[siteIx][camIx][fileIx].name+'...')
1161  objects = sites[siteIx][camIx][fileIx].loadObjects(max_obj=commands.max_obj, max_obj_features=config.max_obj_features, suppress_features=True, indent=4, verbose=commands.verbose)
1162  objects = tvaFilter.compensateParralax(objects, sites[siteIx][camIx].camOrigin, sites[siteIx][camIx].camHeight, verbose=commands.verbose)
1163  objects = tvaFilter.transformTrajectories(objects, sites[siteIx][camIx][fileIx].translationX, sites[siteIx][camIx][fileIx].translationY, sites[siteIx][camIx][fileIx].rotation, verbose=commands.verbose)
1164 
1165  tvaDraw.drawMask(objects, sites, siteIx, camIx, fileIx, alternateView=commands.draw_mask, draw_max_traj=config.draw_max_traj, local=local, figsize=config.figsize)
1166  if(not commands.draw): raise Exception, [4, 'Mask drawing complete.']
1167 
1168 
1169 
1172  if(commands.draw_bounds):
1173  if(commands.interactive): commands = tvaRuntime.interactiveSiteAnalysisSelection(commands, config, site_analyses, sites, local)
1174  if(not (commands.site and commands.camera and commands.file)): raise Exception, [308, 'Bound drawing requires explicit sequence declaration.']
1175 
1176  siteIx, camIx, fileIx = tvaRuntime.targetSequence(commands, sites, config)
1177  saIx = site_analyses.interpret(commands.s_analysis)[0]
1178  if(commands.verbose): tvaLib.printTimeStamp('Preparing bound drawing using data in sequence: '+sites[siteIx].name+'/'+sites[siteIx][camIx].name+'/'+sites[siteIx][camIx][fileIx].name+'...')
1179  objects = sites[siteIx][camIx][fileIx].loadObjects(max_obj=commands.max_obj, max_obj_features=config.max_obj_features, suppress_features=True, indent=4, verbose=commands.verbose)
1180  objects = tvaFilter.compensateParralax(objects, sites[siteIx][camIx].camOrigin, sites[siteIx][camIx].camHeight, verbose=commands.verbose)
1181  objects = tvaFilter.transformTrajectories(objects, sites[siteIx][camIx][fileIx].translationX, sites[siteIx][camIx][fileIx].translationY, sites[siteIx][camIx][fileIx].rotation, verbose=commands.verbose)
1182 
1183  result = tvaDraw.drawBounds(objects, sites, site_analyses, siteIx, camIx, saIx, draw_max_traj=config.draw_max_traj, local=local, figsize=config.figsize)
1184  if(not commands.draw): raise Exception, [4, 'Bound drawing complete.']
1185 
1186 
1187 
1190  if(commands.auto_bounds):
1191  if(commands.interactive): commands = tvaRuntime.interactiveSiteAnalysisSelection(commands, config, site_analyses, sites, local)
1192 
1193  siteIx, camIx, fileIx = tvaRuntime.targetSequence(commands, sites, config)
1194  saIx = site_analyses.interpret(commands.s_analysis)[0]
1195 
1196  if(not commands.save_data_flush and site_analyses[saIx].xy_bounds): raise Exception, [311, 'Bounds already exist. Use -u to force recreation.']
1197 
1198  if(commands.verbose): tvaLib.printTimeStamp('Preparing data for automatic bound creation using data in sequence: '+sites[siteIx].name+'/'+sites[siteIx][camIx].name+'/'+sites[siteIx][camIx][fileIx].name+'...')
1199  objects = sites[siteIx][camIx][fileIx].loadObjects(max_obj=commands.max_obj, max_obj_features=config.max_obj_features, suppress_features=True, indent=4, verbose=commands.verbose)
1200  objects = tvaFilter.compensateParralax(objects, sites[siteIx][camIx].camOrigin, sites[siteIx][camIx].camHeight, verbose=commands.verbose)
1201  objects = tvaFilter.transformTrajectories(objects, sites[siteIx][camIx][fileIx].translationX, sites[siteIx][camIx][fileIx].translationY, sites[siteIx][camIx][fileIx].rotation, verbose=commands.verbose)
1202 
1203  result = tvaDraw.autoBounds(objects, site_analyses, saIx, verbose=commands.verbose)
1204  raise Exception, [4, 'Bound creation complete.']
1205 
1206 
1207 
1210  if(commands.draw_zone):
1211  if(commands.interactive): commands = tvaRuntime.interactiveSiteAnalysisSelection(commands, config, site_analyses, sites, local)
1212  if(not (commands.site and commands.camera and commands.file)): raise Exception, [302, 'Zone drawing requires explicit site-analysis declaration.']
1213 
1214  siteIx, camIx, fileIx = tvaRuntime.targetSequence(commands, sites, config)
1215  saIx = site_analyses.interpret(commands.s_analysis)[0]
1216  if(commands.verbose): tvaLib.printTimeStamp('Preparing zone drawing using data in sequence: '+sites[siteIx].name+'/'+sites[siteIx][camIx].name+'/'+sites[siteIx][camIx][fileIx].name+'...')
1217  objects = sites[siteIx][camIx][fileIx].loadObjects(max_obj=commands.max_obj, max_obj_features=config.max_obj_features, suppress_features=True, indent=4, verbose=commands.verbose)
1218  objects = tvaFilter.compensateParralax(objects, sites[siteIx][camIx].camOrigin, sites[siteIx][camIx].camHeight, verbose=commands.verbose)
1219  objects = tvaFilter.transformTrajectories(objects, sites[siteIx][camIx][fileIx].translationX, sites[siteIx][camIx][fileIx].translationY, sites[siteIx][camIx][fileIx].rotation, verbose=commands.verbose)
1220  if(not objects): raise Exception, [322, 'No objects could be loaded.']
1221 
1222  tvaDraw.drawZone(objects, sites, site_analyses, siteIx, camIx, saIx, draw_max_traj=config.draw_max_traj, local=local, figsize=config.figsize)
1223  if(not commands.draw): raise Exception, [4, 'Zone drawing complete.']
1224 
1225 
1226 
1229  if(commands.draw_align or commands.draw_align_no_trk):
1230  ignore_alternative_parts = True
1231  confine_to_masks = True
1232  if(commands.interactive): commands = tvaRuntime.interactiveSiteSelection(commands, config, sites, local, shortcutCamera=True, shortcutSequences=True)
1233  if(not (commands.site)): raise Exception, [303, 'Alignment drawing requires explicit site declaration.']
1234 
1235  siteIx = sites.interpret(commands.site)[0]
1236  camIxs = sites[siteIx].interpret()
1237  if(commands.verbose): tvaLib.printTimeStamp('Preparing alignment drawing for site: '+sites[siteIx].name+'...')
1238  objects = []
1239  if(not commands.draw_align_no_trk):
1240  for camIx in camIxs:
1241  if(ignore_alternative_parts and re_search(r'.+/P[2-9]', sites[siteIx][camIx].name)): continue
1242  if(len(sites[siteIx][camIx]) > 1): fileIxs = [config.default_sequence_no-1,]+range(len(sites[siteIx][camIx]))
1243  else: fileIxs = range(len(sites[siteIx][camIx]))
1244  for fileIx in fileIxs:
1245  print(' Loading trajectory data for sequence: '+sites[siteIx][camIx].name+'/'+sites[siteIx][camIx][fileIx].name+'...')
1246  if(os.path.exists(sites[siteIx][camIx][fileIx].getFullDataFilename())):
1247  try:
1248  seq_objects = sites[siteIx][camIx][fileIx].loadObjects(max_obj=commands.max_obj, max_obj_features=config.max_obj_features, suppress_features=True, indent=4, verbose=commands.verbose)
1249  if(confine_to_masks and sites[siteIx][camIx].mask): seq_objects, _ = tvaFilter.filtBoundingBox(seq_objects, sites[siteIx][camIx].mask, containment_threshold=config.f_bb_containment_threshold, max_outside_dist=config.f_bb_max_outside_dist, loopback_verification_frames=config.f_bb_loopback_ver_frames, bounding_boxes_label='mask', verbose=0)
1250  seq_objects = tvaFilter.compensateParralax(seq_objects, sites[siteIx][camIx].camOrigin, sites[siteIx][camIx].camHeight, verbose=commands.verbose)
1251  seq_objects = tvaFilter.transformTrajectories(seq_objects, sites[siteIx][camIx][fileIx].translationX, sites[siteIx][camIx][fileIx].translationY, sites[siteIx][camIx][fileIx].rotation, verbose=commands.verbose)
1252  if(seq_objects):
1253  objects += [seq_objects]
1254  break
1255  except: pass
1256  tvaLib.printWarning('Could not load any sequence objects for camera "'+sites[siteIx][camIx].name+'"... attempting a new sequence', local['gen_warning'])
1257  if(not objects): raise Exception, [321, 'No objects could be loaded.']
1258 
1259  tvaDraw.drawAlign(objects, sites, siteIx, draw_max_traj=config.draw_max_traj, local=local, connectorSearchDistance=config.connectorSearchDistance, intersectionDistanceFactor=config.intersectionDistanceFactor, figsize=config.figsize)
1260  if(not commands.draw): raise Exception, [4, 'Alignment drawing complete.']
1261 
1262 
1265  if(commands.draw_loops):
1266  if(commands.interactive): commands = tvaRuntime.interactiveSiteAnalysisSelection(commands, config, site_analyses, sites, local)
1267  if(not (commands.site and commands.camera and commands.file)): raise Exception, [304, 'Virtual loop drawing requires explicit site-analysis declaration.']
1268 
1269  siteIx, camIx, fileIx = tvaRuntime.targetSequence(commands, sites, config)
1270  saIx = site_analyses.interpret(commands.s_analysis)[0]
1271  if(commands.verbose): tvaLib.printTimeStamp('Preparing virtual loop drawing using data in sequence: '+sites[siteIx].name+'/'+sites[siteIx][camIx].name+'/'+sites[siteIx][camIx][fileIx].name+'...')
1272  objects = sites[siteIx][camIx][fileIx].loadObjects(max_obj=commands.max_obj, max_obj_features=config.max_obj_features, suppress_features=True, indent=4, verbose=commands.verbose)
1273  objects = tvaFilter.compensateParralax(objects, sites[siteIx][camIx].camOrigin, sites[siteIx][camIx].camHeight, verbose=commands.verbose)
1274  objects = tvaFilter.transformTrajectories(objects, sites[siteIx][camIx][fileIx].translationX, sites[siteIx][camIx][fileIx].translationY, sites[siteIx][camIx][fileIx].rotation, verbose=commands.verbose)
1275 
1276  tvaDraw.drawLoops(objects, site_analyses, saIx, draw_max_traj=config.draw_max_traj, local=local, figsize=config.figsize)
1277  if(not commands.draw): raise Exception, [4, 'Virtual loop drawing complete.']
1278 
1279 
1282  if(commands.draw_transform):
1283  if(commands.interactive): commands = tvaRuntime.interactiveSiteSelection(commands, config, sites, local)
1284  if(not (commands.site and commands.camera)): raise Exception, [305, 'Trajectory transformation requires explicit camera declaration.']
1285  if(commands.verbose and not commands.file): tvaLib.printNotice('A sequence was not explicitly declared. Coordinate transformation will be applied to ALL sequences for this camera.', local['gen_notice'])
1286 
1287  siteIx, camIx, fileIx = tvaRuntime.targetSequence(commands, sites, config)
1288  if(commands.verbose): tvaLib.printTimeStamp('Preparing trajectory transformation drawing using data in sequence: '+sites[siteIx].name+'/'+sites[siteIx][camIx].name+'/'+sites[siteIx][camIx][fileIx].name+'...')
1289  objects = sites[siteIx][camIx][fileIx].loadObjects(max_obj=commands.max_obj, max_obj_features=config.max_obj_features, suppress_features=True, indent=4, verbose=commands.verbose)
1290  objects = tvaFilter.compensateParralax(objects, sites[siteIx][camIx].camOrigin, sites[siteIx][camIx].camHeight, verbose=commands.verbose)
1291  rotation, translationX, translationY = tvaDraw.drawTransformation(objects, sites[siteIx][camIx][fileIx], draw_max_traj=config.draw_max_traj, figsize=config.figsize)
1292  if(commands.file):
1293  sites[siteIx][camIx][fileIx].setTranslation(translationX, translationY)
1294  sites[siteIx][camIx][fileIx].setRotation(rotation)
1295  else:
1296  for sequence in sites[siteIx][camIx]:
1297  sequence.setTranslation(translationX, translationY)
1298  sequence.setRotation(rotation)
1299  sites.update()
1300  if(not commands.draw): raise Exception, [4, 'Trajectory transformation drawing complete.']
1301 
1302 
1305  if(commands.snap_align):
1306  if(commands.interactive): commands = tvaRuntime.interactiveSiteSelection(commands, config, sites, local, shortcutCamera=True, shortcutSequences=True)
1307  if(not (commands.site)): raise Exception, [306, 'Alignment snapping requires explicit site declaration.']
1308 
1309  siteIx, camIx, fileIx = tvaRuntime.targetSequence(commands, sites, config)
1310  camIxs = sites[siteIx].interpret()
1311  if(not sites[siteIx].alignments): raise Exception, [310, 'Reference alignments missing. Reference alignments required to snap points.']
1312  if(commands.verbose): tvaLib.printTimeStamp('Preparing alignment snapping drawing for site: '+sites[siteIx].name+'...')
1313  objects = []
1314  for camIx in camIxs:
1315  fileIx = None
1316  for fileIx_ in range(len(sites[siteIx][camIx])):
1317  if(os.path.exists(sites[siteIx][camIx][fileIx_].getFullDataFilename())):
1318  fileIx = fileIx_
1319  break
1320  try:
1321  seq_objects = sites[siteIx][camIx][fileIx].loadObjects(max_obj=commands.max_obj, suppress_features=True, indent=4, verbose=commands.verbose)
1322  seq_objects = tvaFilter.compensateParralax(seq_objects, sites[siteIx][camIx].camOrigin, sites[siteIx][camIx].camHeight, verbose=commands.verbose)
1323  seq_objects = tvaFilter.transformTrajectories(seq_objects, sites[siteIx][camIx][fileIx].translationX, sites[siteIx][camIx][fileIx].translationY, sites[siteIx][camIx][fileIx].rotation, verbose=commands.verbose)
1324  objects += seq_objects
1325  except: tvaLib.printWarning('Could not load any sequence objects for camera '+sites[siteIx].name+'/'+sites[siteIx][camIx].name, local['gen_warning'])
1326  try: tvaDraw.drawAlignSnap(objects, site=sites[siteIx], draw_max_traj=config.draw_max_traj, local=local, figsize=config.figsize)
1327  except: pass
1328  if(not commands.draw): raise Exception, [4, 'Alignment snapping complete.']
1329 
1330 
1331 
1334  if(commands.obj_highlight):
1335  if(commands.interactive): commands = tvaRuntime.interactiveSiteSelection(commands, config, sites, local)
1336  if(not (commands.site)): raise Exception, [307, 'Object highlighting requires explicit site declaration.']
1337 
1338  siteIx, camIx, fileIx = tvaRuntime.targetSequence(commands, sites, config)
1339  if(commands.verbose): tvaLib.printTimeStamp('Preparing object highlighting drawing for sequence: '+sites[siteIx].name+'/'+sites[siteIx][camIx].name+'/'+sites[siteIx][camIx][fileIx].name+'...')
1340  objects = sites[siteIx][camIx][fileIx].loadObjects(max_obj=commands.max_obj, suppress_features=True, indent=4, verbose=commands.verbose)
1341  objects = tvaFilter.compensateParralax(objects, sites[siteIx][camIx].camOrigin, sites[siteIx][camIx].camHeight, verbose=commands.verbose)
1342  objects = tvaFilter.transformTrajectories(objects, sites[siteIx][camIx][fileIx].translationX, sites[siteIx][camIx][fileIx].translationY, sites[siteIx][camIx][fileIx].rotation, verbose=commands.verbose)
1343  objects, _ = tvaFilter.transformToCurvilinear(objects, sites[siteIx].alignments, cl_align_window_m=config.f_cl_align_window_m, verbose=commands.verbose)
1344 
1345  tvaDraw.drawObjHighlight(objects, camera=sites[siteIx][camIx], draw_max_traj=config.draw_max_traj, local=local, figsize=config.figsize)
1346  if(not commands.draw): raise Exception, [4, 'Object highlighting complete.']
1347 
1348 
1349 
1352  if(commands.mast_height_calibration or commands.draw_mast_location):
1353  if(commands.interactive): commands = tvaRuntime.interactiveSiteSelection(commands, config, sites, local, shortcutSequences=True)
1354  if(not commands.site or not commands.camera): raise Exception, [308, 'Mast drawing requires explicit site declaration.']
1355 
1356  siteIx, camIx, fileIx = tvaRuntime.targetSequence(commands, sites, config)
1357 
1358  if(not sites[siteIx][camIx].camOrigin):
1359  tvaLib.printWarning('This camera does not have a camera origin set. Pick an origin now.', local['gen_warning'])
1360  commands.draw_mast_location = True
1361 
1362  if(commands.draw_mast_location):
1363  tvaDraw.drawMastLocation(sites, siteIx, camIx, figsize=config.figsize)
1364 
1365 
1366  if(commands.mast_height_calibration):
1367 
1368  if(commands.verbose): tvaLib.printTimeStamp('Preparing mast height calibration drawing for sequence: '+sites[siteIx].name+'/'+sites[siteIx][camIx].name+'/'+sites[siteIx][camIx][fileIx].name+'...')
1369  fileIx = None
1370  for fileIx_ in range(len(sites[siteIx][camIx])):
1371  if(os.path.exists(sites[siteIx][camIx][fileIx_].getFullDataFilename())):
1372  fileIx = fileIx_
1373  break
1374  try: objects = sites[siteIx][camIx][fileIx].loadObjects(max_obj=(200 if commands.max_obj==-1 else max(commands.max_obj,200)), suppress_features=True, indent=4, verbose=commands.verbose)
1375  except: raise Exception, [391, 'Could not load any sequence objects for camera '+sites[siteIx].name+'/'+sites[siteIx][camIx].name]
1376 
1377  tvaDraw.drawMastCalibration(objects, sites, siteIx, camIx, local=local, figsize=config.figsize)
1378 
1379  if(not commands.draw): raise Exception, [4, 'Mast calibration complete.']
1380 
1381 
1382 
1385  if(commands.draw_hli):
1386  if(not commands.hli): commands.hli = 'all'
1387 
1388  tvaHLI = []
1389  if(commands.hli == 'all'):
1390  import importlib
1391  for _, dirs, files in os.walk('hli'):
1392  for file_ in files:
1393  if(os.path.splitext(file_)[0] != '__init__' and os.path.splitext(file_)[1] == '.py'):
1394  try: tvaHLI.append(importlib.import_module('hli.'+os.path.splitext(file_)[0]))
1395  except ImportError: tvaLib.printWarning('There were issues trying to import HLI module "'+file_+'".', local['gen_warning'])
1396  elif(commands.hli):
1397  import importlib
1398  try: tvaHLI.append(importlib.import_module('hli.'+commands.hli))
1399  except ImportError: tvaLib.printWarning('There were issues trying to import HLI module "'+commands.hli+'".', local['gen_warning'])
1400 
1401  if(tvaHLI):
1402  if(commands.interactive): commands = tvaRuntime.interactiveSiteAnalysisSelection(commands, config, site_analyses, sites, local)
1403  if(not (commands.site and commands.camera and commands.file)): raise Exception, [309, 'HLI drawing requires explicit site-analysis declaration.']
1404 
1405  siteIx, camIx, fileIx = tvaRuntime.targetSequence(commands, sites, config)
1406  objects=None
1407 
1408  for module in tvaHLI:
1409  if(hasattr(module, 'draw')):
1410  if(hasattr(module, 'DRAW_REQUIRES_OBJECTS') and module.DRAW_REQUIRES_OBJECTS==True):
1411  if(objects==None):
1412  if(commands.verbose): tvaLib.printTimeStamp('Preparing HLI drawing for sequence: '+sites[siteIx].name+'/'+sites[siteIx][camIx].name+'/'+sites[siteIx][camIx][fileIx].name+'...')
1413  objects = sites[siteIx][camIx][fileIx].loadObjects(max_obj=commands.max_obj, suppress_features=True, indent=4, verbose=commands.verbose)
1414  objects = tvaFilter.compensateParralax(objects, sites[siteIx][camIx].camOrigin, sites[siteIx][camIx].camHeight, verbose=commands.verbose)
1415  objects = tvaFilter.transformTrajectories(objects, sites[siteIx][camIx][fileIx].translationX, sites[siteIx][camIx][fileIx].translationY, sites[siteIx][camIx][fileIx].rotation, verbose=commands.verbose)
1416  if(module.draw(objects=objects, commands=commands, config=config, sites=sites, site_analyses=site_analyses, local=local)==False): raise Exception, [1004, 'HLI drawing module interrupted.']
1417  if(not commands.draw): raise Exception, [4, 'HLI drawing modules complete.']
1418 
1419  if(commands.draw): raise Exception, [4, 'Basic drawing complete.']
1420 
1421 
1422 
1425  if(commands.annotate):
1426  if(commands.interactive): commands = tvaRuntime.interactiveSiteSelection(commands, config, sites, local)
1427  siteIx, camIx, fileIx = tvaRuntime.targetSequence(commands, sites, config)
1428  if(siteIx is not False and camIx is not False and fileIx is not False):
1429  import lib.interface as tvaInterface
1430  tvaInterface.AnnotateVideo(sites[siteIx][camIx][fileIx].getFullVideoFilename(),
1431  sequence=sites[siteIx][camIx][fileIx],
1432  local=local,
1433  satFilename=sites[siteIx].getFullSatFilename(),
1434  satRes=sites[siteIx].satres,
1435  homography=sites[siteIx][camIx].getHomography(),
1436  alignments=sites[siteIx].alignments,
1437  fps=sites[siteIx][camIx].camera.frameRate,
1438  intrinsicCameraMatrix=sites[siteIx][camIx].camera.camera_matrix.data,
1439  distortionCoefficients=sites[siteIx][camIx].camera.dist_coeffs.data,
1440  undistortedImageScalingFactor=sites[siteIx][camIx].camera.getImageScalingFactor(),
1441  startTime=sites[siteIx][camIx][fileIx].startTime,
1442  eventLabels=config.event_labels,
1443  config=config,
1444  verbose=commands.verbose)
1445  raise Exception, [8, 'Video annotation complete.']
1446  else: raise Exception, [2081, 'Annotation for more than one sequence currently not supported. Please use -s, -c, and -f in conjunction with -p, or pick a specific sequence via interface.']
1447 
1448 
1449 
1450 
1451 
1456 
1457 
1462  if(commands.interactive): commands = tvaRuntime.interactiveSiteAnalysisSelection(commands, config, site_analyses, sites, local)
1463  if(not commands.s_analysis): raise Exception, [2020, 'Core runtime requires explicit definition of a site-analysis. If you are unsure how to select one, append the command "-e" to launch an interactive selection dialogue.']
1464  saIx = site_analyses.interpret(commands.s_analysis)[0]
1465 
1466  if(5 in commands.indicators and not site_analyses[saIx].xy_bounds): raise Exception, [2701, 'DMP indicators elected, but no bounds specified. Please create bounds first.']
1467 
1468 
1469 
1470 
1473  objects = tvaScene.ObjectSpace(spaceType='sites')
1474  objects_uncon = tvaScene.ObjectSpace(spaceType='sites')
1475  dropped_trajectories = {'ie':[], 'bb':[], 'te':[], 'td':[], 'tc':[], 'st':[], 'so':[]}
1476 
1477  siteIxs = tvaRuntime.targetSiteIxs(commands, sites, site_analyses[saIx])
1478  for siteIx in siteIxs:
1479  if(not sites[siteIx].alignments and not config.alignmentOverride):
1480  if(commands.play):
1481  if(commands.verbose): tvaLib.printWarning('No alignment data found. However, trajectory playback may continue.', local['gen_warning'])
1482  else: raise Exception, [2021, 'Alignment data is missing for site '+sites[siteIx].name+'.']
1483  objects.add(tvaScene.ObjectSpace(spaceType='cameras'), idx=siteIx, metadata=sites[siteIx])
1484  objects_uncon.add(tvaScene.ObjectSpace(spaceType='cameras'), idx=siteIx, metadata=sites[siteIx])
1485 
1486  camIxs = tvaRuntime.targetCameraIxs(commands, sites[siteIx], site_analyses[saIx])
1487  for camIx in camIxs:
1488  if(not sites[siteIx][camIx].mask): tvaLib.printWarning('Mask is missing for camera '+sites[siteIx].name+'/'+sites[siteIx][camIx].name+'.', local['gen_warning'])
1489  objects[-1].add(tvaScene.ObjectSpace(spaceType='sequences'), idx=camIx, metadata=sites[siteIx][camIx])
1490  objects_uncon[-1].add(tvaScene.ObjectSpace(spaceType='sequences'), idx=camIx, metadata=sites[siteIx][camIx])
1491 
1492  fileIxs = tvaRuntime.targetSequenceIxs(commands, sites[siteIx][camIx], site_analyses[saIx])
1493  for fileIx in fileIxs:
1494 
1495 
1498  if(not commands.export_traj and not commands.save_data_flush and not commands.save_data_flush_objects and os.path.exists(sites[siteIx][camIx][fileIx].getFullSerialisedFilename()) and not commands.proc_gt and not commands.filename_override):
1499  with open(sites[siteIx][camIx][fileIx].getFullSerialisedFilename(), 'rb') as input_data:
1500  s_version = pickle.load(input_data)
1501  if(not tvaLib.Parse.versionIsMoreRecent(s_version, config.version, config.autoflush_ver_depth, orEqual=True) and config.autoflush_version):
1502  flush = True
1503  if(commands.verbose): tvaLib.printWarning('Object data appears to be saved with an older version ('+s_version+'). The objects will be automatically flushed.', local['gen_warning'])
1504  else:
1505  flush = False
1506  if(not tvaLib.Parse.versionIsMoreRecent(s_version, config.version, config.autoflush_ver_depth, orEqual=True) and commands.verbose): tvaLib.printWarning('object data appears to be saved with an older version ('+s_version+'). The analysis will proceed, but consider flushing the data if errors occur.', local['gen_warning'])
1507  if(commands.verbose): tvaLib.printTimeStamp('Loading data from saved memory for sequence: '+Fore.YELLOW+sites[siteIx].name+'/'+sites[siteIx][camIx].name+'/'+sites[siteIx][camIx][fileIx].name+Fore.RESET+'...')
1508  try:
1509  seq_objects = pickle.load(input_data)
1510  seq_dropped_trajectories = pickle.load(input_data)
1511  if(commands.max_obj): seq_objects = tvaFilter.trimObjects(seq_objects, commands.max_obj)
1512  if(commands.verbose): print(' '+str(len(seq_objects))+' objects loaded')
1513  except EOFError:
1514  if(commands.verbose): tvaLib.printWarning('Serialized data is corrupted. Reverting to sqlite data.', local['gen_warning'])
1515  flush = True
1516  else: flush = True
1517 
1518 
1521  if(flush):
1522  if(not os.path.isfile(os.path.join(sites.getBaseDirectory(), sites[siteIx].name, sites[siteIx][camIx].name, sites[siteIx][camIx][fileIx].dataFilename))):
1523  if(commands.verbose): tvaLib.printWarning('Data for sequence: '+sites[siteIx][camIx][fileIx].name+' could not be found.', local['gen_warning'])
1524  continue
1525  if(commands.verbose): tvaLib.printTimeStamp('Loading data for sequence: '+sites[siteIx].name+'/'+sites[siteIx][camIx].name+'/'+sites[siteIx][camIx][fileIx].name+'...')
1526  seq_objects = sites[siteIx][camIx][fileIx].loadObjects(filename_override=commands.filename_override, max_obj=commands.max_obj, max_obj_features=config.max_obj_features, suppress_features=commands.suppress_features, load_gt=commands.proc_gt, indent=4, verbose=commands.verbose)
1527  #Create dropped trajectories container
1528  seq_dropped_trajectories = {}
1529  sites[siteIx][camIx][fileIx].setCountObjOld(len(seq_objects))
1530 
1531 
1534  if(not seq_objects): continue
1535  alignment_passes=0
1536  if(not commands.no_filt):
1537  if(commands.verbose): print(' Performing filtering and transformation functions...')
1538  seq_objects = tvaFilter.corVelIndex(seq_objects, verbose=commands.verbose)
1539  if(not commands.no_track_filt and not commands.proc_gt):
1540  seq_objects, seq_dropped_trajectories['ie'] = tvaFilter.verifyObjectIntegrity(seq_objects, skipTrajectoryContiguityCheck=(True if config.max_obj_features < 1000 else False), verbose=commands.verbose)
1541  seq_objects = tvaFilter.compensateParralax(seq_objects, sites[siteIx][camIx].camOrigin, sites[siteIx][camIx].camHeight, verbose=commands.verbose)
1542  seq_objects = tvaFilter.smoothPositions(seq_objects, verbose=commands.verbose)
1543  seq_objects = tvaFilter.transformTrajectories(seq_objects, sites[siteIx][camIx][fileIx].translationX, sites[siteIx][camIx][fileIx].translationY, sites[siteIx][camIx][fileIx].rotation, verbose=commands.verbose)
1544  seq_objects, seq_dropped_trajectories['bb'] = tvaFilter.filtBoundingBox(seq_objects, sites[siteIx][camIx].mask, containment_threshold=config.f_bb_containment_threshold, max_outside_dist=config.f_bb_max_outside_dist, loopback_verification_frames=config.f_bb_loopback_ver_frames, bounding_boxes_label='mask', verbose=commands.verbose)
1545  seq_objects, seq_dropped_trajectories['so'] = tvaFilter.splitbyOutlierVectors(seq_objects, hard_maxSpeed=config.f_so_hard_maxSpeed, soft_maxSpeed=config.f_so_soft_maxSpeed, maxAngle=config.f_so_max_angle, verbose=commands.verbose)
1546  seq_objects, seq_dropped_trajectories['te'] = tvaFilter.dropTrackingErrors(seq_objects, config_min_traj_len=config.f_te_config_min_traj_len, verbose=commands.verbose)
1547  seq_objects, seq_dropped_trajectories['td'] = tvaFilter.dropTrackingDuplicates(seq_objects, minimumSeperationDistance=config.f_td_min_sep_distance, minimumSeperationVelocity=config.f_td_min_sep_velocity/sites[siteIx][camIx].camera.frameRate/config.mps_kmh, verbose=commands.verbose)
1548  seq_objects, seq_dropped_trajectories['st'] = tvaFilter.objectStitchExp(seq_objects, maximum_t_seperation=13.0*sites[siteIx][camIx].camera.frameRate, verbose=commands.verbose)
1549  #seq_objects, seq_dropped_trajectories['st'] = tvaFilter.objectStitch(seq_objects, sites[siteIx][camIx].mask, sites[siteIx][camIx].camera.frameRate, max_dwell_time=config.fs_max_dwell_time, search_radius=config.fs_search_radius, search_window=config.fs_search_window, speed_adj_factor=config.fs_speed_adj_factor, speed_similarity_limit=config.fs_speed_similarity_lim, speed_sim_limit_min=config.fs_speed_sim_limit_min, stop_speed_detection_limit=config.fs_stop_speed_det_lim, angle_similairty_deg=config.fs_angle_similairty_deg, verbose=commands.verbose)
1550  if(commands.force_reclassify or len(tvaCompute.computeClassificationVariety(seq_objects))<=1):
1551  alignment_passes += 1
1552  seq_objects, seq_dropped_trajectories['tc'] = tvaFilter.transformToCurvilinear(seq_objects, sites[siteIx].alignments, restrict_by_type=False, cl_align_window_m=config.f_cl_align_window_m, local=local, passNumber=alignment_passes, verbose=commands.verbose)
1553  seq_objects = tvaFilter.classify(seq_objects, sites[siteIx][camIx], alignments=sites[siteIx].alignments, fileIx=fileIx, config=config, method=config.classification_method, commitPermanently=commands.commit_classify, verbose=commands.verbose)
1554  elif(commands.verbose): print(' Classification already exists in data. Use --force-reclassify to force a reclassification')
1555  seq_objects, seq_dropped_trajectories['tc'] = tvaFilter.transformToCurvilinear(seq_objects, sites[siteIx].alignments, cl_align_window_m=config.f_cl_align_window_m, local=local, passNumber=alignment_passes+1, verbose=commands.verbose)
1556  seq_objects = tvaFilter.buildSpeed(seq_objects, verbose=commands.verbose)
1557  seq_objects = tvaFilter.genHashID(seq_objects)
1559  sites[siteIx][camIx][fileIx].setCountsFromObjects(seq_objects)
1560  if(commands.verbose): print(' '+str(len(seq_objects))+' objects remain')
1561 
1562 
1563  if(not seq_objects): continue
1564 
1565 
1568  if(commands.stat):
1569  if(commands.verbose): tvaLib.printTimeStamp('Dumping stats for objects...')
1570  tvaStats.echoGeneralStats(seq_objects, site_analyses[saIx], camIx, fileIx, commands)
1571  tvaStats.echoFeatures(seq_objects)
1572  tvaStats.echoScreenTime(seq_objects, fps=sites[siteIx][camIx].camera.frameRate)
1573  tvaStats.echoSpeedDistributionsByType(seq_objects, fps=sites[siteIx][camIx].camera.frameRate, mps_to_kmh=config.mps_kmh, local=local)
1574  try: print(' List of stopped objects stitched: '+str([x.num for x in seq_dropped_trajectories['st']]))
1575  except: pass
1576 
1577 
1580  if(commands.save_data and not commands.proc_gt and not commands.filename_override):
1581  if(commands.verbose): tvaLib.printTimeStamp('Saving objects data to cache...')
1582  with open(os.path.join(sites.getBaseDirectory(), sites[siteIx].name, sites[siteIx][camIx].name, sites[siteIx][camIx][fileIx].name+'.traj'), 'wb') as output:
1583  pickle.dump(config.version, output, protocol=2)
1584  pickle.dump(seq_objects, output, protocol=2)
1585  pickle.dump(seq_dropped_trajectories, output, protocol=2)
1586 
1587  if(commands.export_traj and not commands.proc_gt):
1588  if(commands.verbose): tvaLib.printTimeStamp('Exporting trajectories to sqlite database...')
1589  if(os.path.exists(os.path.join(sites.getBaseDirectory(), sites[siteIx].name, sites[siteIx][camIx].name, sites[siteIx][camIx][fileIx].name+'_export.sqlite'))):
1590  if(commands.verbose): tvaLib.printWarning('A previous data export was detected for this sequence. The previous export will be overwritten with the new export.', local['gen_warning'], 4)
1591  os.remove(os.path.join(sites.getBaseDirectory(), sites[siteIx].name, sites[siteIx][camIx].name, sites[siteIx][camIx][fileIx].name+'_export.sqlite'))
1592  try: import storage as TrafIntStorage
1593  except ImportError: raise Exception, [103, 'Traffic-Intelligence is not installed.']
1594  TrafIntStorage.saveTrajectoriesToSqlite(os.path.join(sites[siteIx][camIx].getBaseDirectory(), sites[siteIx][camIx][fileIx].name+'_export.sqlite'), seq_objects, trajectoryType='object')
1595 
1596 
1599  if(commands.play and not commands.mot):
1600  siteIx, camIx, fileIx = tvaRuntime.targetSequence(commands, sites, config)
1601  if(siteIx is not False and camIx is not False and fileIx is not False):
1602  import lib.interface as tvaInterface
1603  tvaInterface.ObjectInspectorInterface(seq_objects,
1604  sites[siteIx][camIx][fileIx].getFullVideoFilename(),
1605  local=local,
1606  satFilename=sites[siteIx].getFullSatFilename(),
1607  satRes=sites[siteIx].satres,
1608  homography=sites[siteIx][camIx].getHomography(),
1609  fps=sites[siteIx][camIx].camera.frameRate,
1610  intrinsicCameraMatrix=sites[siteIx][camIx].camera.camera_matrix.data,
1611  distortionCoefficients=sites[siteIx][camIx].camera.dist_coeffs.data,
1612  undistortedImageScalingFactor=sites[siteIx][camIx].camera.getImageScalingFactor(),
1613  startTime=sites[siteIx][camIx][fileIx].startTime,
1614  config=config,
1615  verbose=commands.verbose)
1616  raise Exception, [8, 'Video playback complete.']
1617  else: raise Exception, [2081, 'Video playback for more than one segment currently not supported. Please use -s, -c, and -f in conjunction with -p.']
1618  if(commands.play_vid):
1619  siteIx, camIx, fileIx = tvaRuntime.targetSequence(commands, sites, config)
1620  if(siteIx is not False and camIx is not False and fileIx is not False):
1621  import lib.interface as tvaInterface
1622  tvaInterface.PlaybackVideoInterface(seq_objects,
1623  sites[siteIx][camIx][fileIx].getFullVideoFilename(),
1624  local=local,
1625  satFilename=sites[siteIx].getFullSatFilename(),
1626  satRes=sites[siteIx].satres,
1627  homography=sites[siteIx][camIx].getHomography(),
1628  fps=sites[siteIx][camIx].camera.frameRate,
1629  intrinsicCameraMatrix=sites[siteIx][camIx].camera.camera_matrix.data,
1630  distortionCoefficients=sites[siteIx][camIx].camera.dist_coeffs.data,
1631  undistortedImageScalingFactor=sites[siteIx][camIx].camera.getImageScalingFactor(),
1632  startTime=sites[siteIx][camIx][fileIx].startTime,
1633  config=config,
1634  verbose=commands.verbose)
1635  raise Exception, [8, 'Video playback complete.']
1636  else: raise Exception, [2081, 'Video playback for more than one segment currently not supported. Please use -s, -c, and -f in conjunction with -p.']
1637  if(commands.play_filt in seq_dropped_trajectories):
1638  siteIx, camIx, fileIx = tvaRuntime.targetSequence(commands, sites, config)
1639  if(siteIx is not False and camIx is not False and fileIx is not False):
1640  import lib.interface as tvaInterface
1641  tvaInterface.ObjectInspectorInterface(seq_dropped_trajectories[commands.play_filt],
1642  sites[siteIx][camIx][fileIx].getFullVideoFilename(),
1643  companion_objects=seq_objects,
1644  local=local,
1645  satFilename=sites[siteIx].getFullSatFilename(),
1646  satRes=sites[siteIx].satres,
1647  homography=sites[siteIx][camIx].getHomography(),
1648  fps=sites[siteIx][camIx].camera.frameRate,
1649  intrinsicCameraMatrix=sites[siteIx][camIx].camera.camera_matrix.data,
1650  distortionCoefficients=sites[siteIx][camIx].camera.dist_coeffs.data,
1651  undistortedImageScalingFactor=sites[siteIx][camIx].camera.getImageScalingFactor(),
1652  startTime=sites[siteIx][camIx][fileIx].startTime,
1653  verbose=commands.verbose)
1654  raise Exception, [8, 'Video playback complete.']
1655  else: raise Exception, [2081, 'Video playback for more than one segment currently not supported. Please use -s, -c, and -f in conjunction with -p.']
1656 
1657 
1658 
1661  if(commands.memory_clean_feature): seq_objects = tvaFilter.purgeFeatures(seq_objects)
1662 
1663 
1667  if(commands.verbose): print(' Constraining trajectories to analysis zone ('+str(len(site_analyses[saIx].zone))+' zones) and time interval...')
1668  if(commands.max_ucon_obj != 0): objects_uncon[-1][-1].add(deepcopy(seq_objects[:commands.max_ucon_obj]), idx=fileIx, metadata=sites[siteIx][camIx][fileIx])
1669  seq_objects = tvaFilter.duration(seq_objects, duration=commands.time*sites[siteIx][camIx].camera.frameRate, start=commands.start_time*sites[siteIx][camIx].camera.frameRate, verbose=commands.verbose)
1670  seq_objects, _ = tvaFilter.filtBoundingBox(seq_objects, site_analyses[saIx].zone, containment_threshold=config.f_bb_containment_threshold, max_outside_dist=config.f_bb_max_outside_dist, loopback_verification_frames=config.f_bb_loopback_ver_frames, bounding_boxes_label='analysis zone', verbose=commands.verbose)
1671  if(commands.verbose): print(' '+str(len(seq_objects))+' objects remain')
1672  if(commands.verbose): print(' Merging object space for sequence '+str(sites[siteIx][camIx][fileIx].name))
1673  objects[-1][-1].add(seq_objects, idx=fileIx, metadata=sites[siteIx][camIx][fileIx])
1674  dropped_trajectories = tvaLib.mergeDicts(dropped_trajectories, seq_dropped_trajectories)
1675 
1676 
1677 
1680  if(commands.mot):
1681  try: import moving as TrafIntMoving
1682  except ImportError: raise Exception, [103, 'Traffic-Intelligence is not installed.']
1683  if(not os.path.exists(sites[siteIx][camIx][fileIx].getFullGroundTruthFilename())): raise Exception, [2091, 'Could not find ground truth file for MOT-opimisation: "'+sites[siteIx][camIx][fileIx].getFullGroundTruthFilename()+'".']
1684  if(not sites[siteIx][camIx].getHomography()): raise Exception, [2092, 'No homography available for target camera.']
1685  if(commands.verbose): tvaLib.printTimeStamp('Performing MOT analysis...')
1686  firstInstant = int(commands.start_time*sites[siteIx][camIx].camera.frameRate)
1687  if(commands.time): lastInstant = int(commands.start_time*sites[siteIx][camIx].camera.frameRate)+int(commands.time*sites[siteIx][camIx].camera.frameRate)
1688  else: lastInstant = int(sites[siteIx][camIx][fileIx].getDuration()*sites[siteIx][camIx].camera.frameRate)
1689 
1690 
1691  annotations = sites[siteIx][camIx][fileIx].loadAnnotations(indent=4, verbose=commands.verbose)
1692  annotations = tvaFilter.corVelIndex(annotations, verbose=commands.verbose)
1693  annotations = tvaFilter.compensateParralax(annotations, sites[siteIx][camIx].camOrigin, sites[siteIx][camIx].camHeight, verbose=commands.verbose)
1694  annotations,_ = tvaFilter.filtBoundingBox(annotations, site_analyses[saIx].zone, containment_threshold=config.f_bb_containment_threshold, max_outside_dist=config.f_bb_max_outside_dist, loopback_verification_frames=config.f_bb_loopback_ver_frames, bounding_boxes_label='analysis zone', verbose=commands.verbose)
1695  annotations = tvaFilter.filtTime(annotations, startTime=firstInstant, endTime=lastInstant, verbose=commands.verbose)
1696 
1697 
1698 
1699  if(commands.stat):
1700  if(commands.verbose): tvaLib.printTimeStamp('Dumping stats for annotations...')
1701  tvaStats.echoSpeedDistributionsByType(annotations, fps=sites[siteIx][camIx].camera.frameRate, mps_to_kmh=config.mps_kmh, local=local)
1702 
1703 
1704 
1705  if(commands.play):
1706  import lib.interface as tvaInterface
1707  tvaInterface.ObjectInspectorInterface(seq_objects,
1708  sites[siteIx][camIx][fileIx].getFullVideoFilename(),
1709  companion_objects=annotations,
1710  local=local,
1711  satFilename=sites[siteIx].getFullSatFilename(),
1712  satRes=sites[siteIx].satres,
1713  homography=sites[siteIx][camIx].getHomography(),
1714  fps=sites[siteIx][camIx].camera.frameRate,
1715  intrinsicCameraMatrix=sites[siteIx][camIx].camera.camera_matrix.data,
1716  distortionCoefficients=sites[siteIx][camIx].camera.dist_coeffs.data,
1717  undistortedImageScalingFactor=sites[siteIx][camIx].camera.getImageScalingFactor(),
1718  startTime=sites[siteIx][camIx][fileIx].startTime,
1719  config=config,
1720  verbose=commands.verbose)
1721 
1722 
1723  for annotation in annotations: annotation.matches = MethodType(tvaLib.Obj.matches, annotation)
1724 
1725 
1727  mot_results = TrafIntMoving.computeClearMOT(annotations, seq_objects, config.MOT_max_distance, firstInstant, lastInstant, returnMatches=True)
1728  class1_results = tvaLib.Obj.computeClassCorrespondance(mot_results[6], annotations, seq_objects)
1729  class2_results = tvaLib.Obj.computeClassCorrespondance(mot_results[7], seq_objects, annotations)
1730  #TODO: class_i_results = tvaLib.Obj.computeClassInstantCorrespondance(mot_results[6], annotations, seq_objects)
1731  annot_class = [[x.getUserType() for x in annotations].count(i)/float(len(annotations)) for i in range(len(local['userTypeNames']))]
1732  tracked_class = [[x.getUserType() for x in seq_objects].count(i)/float(len(seq_objects)) for i in range(len(local['userTypeNames']))]
1733  normalisedScore = tvaCompute.normaliseMOTscore(motp=mot_results[0], mota=mot_results[1], n_annot=len(annotations), n_tracked=len(seq_objects), annot_class=annot_class, tracked_class=tracked_class)
1734 
1735 
1736  print(' Score: '+Back.MAGENTA+str(normalisedScore)+Back.RESET+' '+str(mot_results[0:6]+(len(annotations),len(seq_objects))))
1737  tvaLib.printTable([['User type:']+local['userTypeNames'],
1738  '==',
1739  ['GT classifications:']+ [[x.getUserType() for x in annotations].count(i) for i in range(len(local['userTypeNames']))],
1740  ['Tracked classifications:']+ [[x.getUserType() for x in seq_objects].count(i) for i in range(len(local['userTypeNames']))],
1741  '--',
1742  ['GT correspondence:']+ [str(round(x,3)*100.0)+'%' if x is not None else 'N/A' for x in class1_results],
1743  ['Tracking correspondence:']+ [str(round(x,3)*100.0)+'%' if x is not None else 'N/A' for x in class2_results],
1744  ['Average correspondence:']+ ['N/A' if x1 is None and x2 is None else (str(round(x2,3)*100.0)+'%' if x1 is None else (str(round(x1,3)*100.0)+'%' if x2 is None else str(round((x1+x2)/2.0,3)*100.0)+'%')) for x1,x2 in zip(class1_results,class2_results)]], padding=' | ')
1745 
1746 
1747 
1750  '''
1751  if(len(site_analyses[saIx].cameras) > 1):
1752  if(commands.verbose): tvaLib.printTimeStamp('Camera view trajectory stitching...')
1753  ## Determine parent camera (all trajectories will be merged to this camera and inherit its properties)
1754  max_framerate = 0
1755  parentCamIx = None
1756  for camIx in camIxs:
1757  if(sites[siteIx][camIx].camera.framerate > max_framerate):
1758  max_framerate = sites[siteIx][camIx].camera.framerate
1759  parentCamIx = camIx
1760  ## Interpolate framerate such that it is constant
1761  for camIx in camIxs:
1762  if(sites[siteIx][camIx].camera.framerate != sites[siteIx][parentCamIx].camera.framerate):
1763  fileIxs = tvaRuntime.targetSequenceIxs(commands, sites[siteIx][camIx], site_analyses[saIx])
1764  for fileIx in fileIxs:
1765  objects[siteIx][camIx][fileIx].data = tvaFilter.applyFunctionToObjects(objects[siteIx][camIx][fileIx].data, tvaLib.Obj.interpolateNewObjectFramerate, sourceFramerate=sites[siteIx][camIx].camera.frameRate, targetFramerate=sites[siteIx][parentCamIx].camera.framerate)
1766  ## Join camera sequences together
1767  parentFileIxs = tvaRuntime.targetSequenceIxs(commands, sites[siteIx][parentCamIx], site_analyses[saIx])
1768  for parentFileIx in parentFileIxs:
1769  for camIx in camIxs:
1770  if(camIx == parentCamIx): continue
1771  fileIxs = tvaRuntime.targetSequenceIxs(commands, sites[siteIx][camIx], site_analyses[saIx])
1772  for fileIx in fileIxs:
1773  if(sites[siteIx][camIx][fileIx].isParallelSequenceTo(sites[siteIx][parentCamIx][parentFileIx]):
1774  objects[siteIx][parentCamIx][parentFileIx].data += objects[siteIx][camIx][fileIx].data
1775  objects[siteIx][camIx][fileIx].data = None
1776  ## Rerun stitching algorythm on joined trajectories
1777  objects[siteIx][camIx][fileIx].data = tvaFilter.objectStitchExp(objects[siteIx][camIx][fileIx].data, maximum_t_seperation=200, verbose=commands.verbose)
1778  '''
1779 
1780  if(len(objects) < 1 or not objects.getAll()): raise Exception, [2055, 'No tracking data could be loaded for site: '+sites[siteIx].name]
1781 
1782 
1783 
1784  this_run_startTime = min([y.getStartTime() for y in tvaLib.flatten_list([x.metadatas for x in objects.getSubSpacebyId(site_analyses[saIx].site.idx)])])
1785  this_run_endTime = max([y.getEndTime() for y in tvaLib.flatten_list([x.metadatas for x in objects.getSubSpacebyId(site_analyses[saIx].site.idx)])])
1786 
1787 
1790  if(commands.save_data_gen_dmp or commands.plot_dmp):
1791  if(commands.verbose): tvaLib.printTimeStamp('Preparing prediction methods...')
1792  predictionMethods = tvaConfig.PredictionMethods(maxSpeed=site_analyses[saIx].max_speed,
1793  timeHorizon=config.timehorizon,
1794  xy_bounds=tvaLib.Obj.genXY_bounds(objects_uncon.getAll()),
1795  allowImport=not commands.save_data_gen_dmp,
1796  importLocations=[sites[siteIx][camIx].getBaseDirectory()], #site_analyses[saIx].getFullResultsFolder()
1797  customSpeedRange=[x*sites[siteIx][camIx].camera.frameRate*config.mps_kmh for x in tvaLib.Math.getPercintileBinsFromList(objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAllSpeeds(flatList=True), bins=7, includeLeftEdge=True)],
1798  results_path=site_analyses[saIx].getFullResultsFolder(),
1799  minSampleSize=config.pred_minSampleSize,
1800  probRetentionThreshold=config.pred_probability_threshold,
1801  verbose=commands.verbose,
1802  version=config.version)
1803 
1804  predictionMethods.setCameraSettings(framerate=sites[siteIx][0].camera.frameRate, alignments=sites[siteIx].alignments)
1805  for predictionMethod in predictionMethods:
1806  if(predictionMethod.getPrepare()):
1807  predictionMethod.prepare(objects_uncon.getAll())
1808  #Export prediction method data
1809  if(predictionMethod.export(path=sites[siteIx][camIx].getBaseDirectory()) and commands.verbose): print(' Exporting data for '+predictionMethod.label+'...')
1810  if(commands.save_data_gen_dmp): raise Exception, [91, 'MPA generation complete.']
1811 
1812 
1813 
1816 
1817 
1818 
1821  if(not commands.hli_only):
1822  if(commands.verbose): tvaLib.printTimeStamp('Performing data analysis...')
1823  if(commands.verbose): print(' Compiling traffic counts...')
1824  site_analyses[saIx].virtual_loops.addEdgeCounts(objects.getSubSpacebyId(site_analyses[saIx].site.idx), verbose=commands.verbose)
1825  site_analyses[saIx].virtual_loops.addCoordCounts(objects.getSubSpacebyId(site_analyses[saIx].site.idx), verbose=commands.verbose)
1826  laneChanges = tvaCompute.computeLaneChanges(objects.getSubSpacebyId(site_analyses[saIx].site.idx), alignments=site_analyses[saIx].site.alignments, mode=1, verbose=commands.verbose)
1827  ODMatrix = tvaCompute.computeLaneChanges(objects.getSubSpacebyId(site_analyses[saIx].site.idx), alignments=site_analyses[saIx].site.alignments, mode=0, verbose=commands.verbose)
1828  ODMatrix_Time = tvaCompute.computeLaneChangesOverTime(objects.getSubSpacebyId(site_analyses[saIx].site.idx), alignments=site_analyses[saIx].site.alignments, mode=0, timestep=15, verbose=commands.verbose)
1829  user_class_variety = tvaCompute.computeClassificationVariety(objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll(), verbose=commands.verbose)
1830  if(commands.verbose): print(' Compiling traffic speed data...')
1831  speed_histo = tvaCompute.computeSpeedHisto(objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll(), bins=np.arange(0,site_analyses[saIx].max_speed,1), framerate=sites[siteIx][camIx].camera.frameRate, verbose=commands.verbose)
1832  speed_by_lane,slsd = tvaCompute.computeSpeedProf(objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll(), site_analyses[saIx].site.alignments, framerate=sites[siteIx][camIx].camera.frameRate, verbose=commands.verbose)
1833  speed_histo_by_lane = tvaCompute.computeSpeedHistoByLane(objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll(), alignments=site_analyses[saIx].site.alignments, bins=np.arange(0,site_analyses[saIx].max_speed,1), framerate=sites[siteIx][camIx].camera.frameRate, verbose=commands.verbose)
1834  speed_histo_by_class = tvaCompute.computeSpeedHistoByClass(objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll(), user_class_variety=user_class_variety, bins=np.arange(0,site_analyses[saIx].max_speed,1), framerate=sites[siteIx][camIx].camera.frameRate, verbose=commands.verbose)
1835 
1836  '''
1837  if(commands.infractions):
1838  infractions = tvaCompute.computeInfractions(objects.getSubSpacebyId(site_analyses[saIx].site.idx), site_analyses[saIx].site.alignments, verbose=commands.verbose)
1839  #timeHorizon = tvaCompute.timeHorizon(objects[camOIx].getAll(), return_max=True, verbose=commands.verbose)
1840  #alignments_true = tvaLib.Obj.true_lane_xy(objects[camOIx].getAll(), site_analyses[saIx].site.alignments)
1841 
1842  '''
1843 
1844 
1848  if(len(commands.indicators) > 0):
1849  predictionMethods = tvaConfig.PredictionMethods(maxSpeed=site_analyses[saIx].max_speed,
1850  timeHorizon=config.timehorizon,
1851  xy_bounds=site_analyses[saIx].xy_bounds,
1852  allowImport=(not commands.save_data_flush_dmp and not commands.save_data_gen_dmp),
1853  importLocations=[sites[siteIx][camIx].getBaseDirectory()], #site_analyses[saIx].getFullResultsFolder()
1854  customSpeedRange=[x*sites[siteIx][camIx].camera.frameRate*config.mps_kmh for x in tvaLib.Math.getPercintileBinsFromList(objects_uncon.getSubSpacebyId(site_analyses[saIx].site.idx).getAllSpeeds(flatList=True), bins=7, includeLeftEdge=True)],
1855  results_path=site_analyses[saIx].getFullResultsFolder(),
1856  minSampleSize=config.pred_minSampleSize,
1857  probRetentionThreshold=config.pred_probability_threshold,
1858  verbose=commands.verbose,
1859  version=config.version)
1860  userPairs = [tvaScene.UPSpace(spaceType='sites') for x in predictionMethods]
1861 
1862 
1863  siteIxs = tvaRuntime.targetSiteIxs(commands, sites, site_analyses[saIx])
1864  for siteIx in siteIxs:
1865 
1866  for userPair in userPairs: userPair.add(tvaScene.UPSpace(spaceType='cameras'), idx=siteIx, metadata=sites[siteIx])
1867  camIxs = tvaRuntime.targetCameraIxs(commands, sites[siteIx], site_analyses[saIx])
1868  for camIx in camIxs:
1869  predictionMethods.setCameraSettings(framerate=sites[siteIx][camIx].camera.frameRate, alignments=sites[siteIx].alignments)
1870 
1871  for userPair in userPairs: userPair[-1].add(tvaScene.UPSpace(spaceType='sequences'), idx=camIx, metadata=sites[siteIx][camIx])
1872  fileIxs = tvaRuntime.targetSequenceIxs(commands, sites[siteIx][camIx], site_analyses[saIx])
1873  for fileIx in fileIxs:
1874 
1875  if(not objects_uncon.getSubSpacebyId(sites[siteIx].idx).getSubSpacebyId(sites[siteIx][camIx].idx).getSubSpacebyId(sites[siteIx][camIx][fileIx].idx)): continue
1876 
1877  if(commands.verbose): tvaLib.printTimeStamp('Running conflict analysis for sequence: '+sites[siteIx].name+'/'+sites[siteIx][camIx].name+'/'+sites[siteIx][camIx][fileIx].name+'...')
1878  for predictionMethod in predictionMethods:
1879  if(not predictionMethod.idx in commands.indicators): continue
1880  if(commands.verbose): print('Prediction method '+Back.MAGENTA+predictionMethod.label+Back.RESET+'...')
1881  filename = predictionMethod.label_short+'_'+sites[siteIx][camIx].name.replace('\\','').replace('/','')+'_'+sites[siteIx][camIx][fileIx].name+'.upairs'
1882 
1883 
1886  flush = True
1887  if(not commands.save_data_flush and not commands.save_data_flush_indicators):
1888 
1889  if(os.path.exists(os.path.join(site_analyses[saIx].getFullResultsFolder(), filename))):
1890  with open(os.path.join(site_analyses[saIx].getFullResultsFolder(), filename), 'rb') as input_data:
1891  s_version = pickle.load(input_data)
1892  if(not tvaLib.Parse.versionIsMoreRecent(s_version, config.version, config.autoflush_ver_depth, orEqual=True) and config.autoflush_version):
1893  if(commands.verbose): tvaLib.printWarning('Object data appears to be saved with an older version ('+s_version+'). The interactions will be automatically flushed.', local['gen_warning'])
1894  else:
1895  flush = False
1896  if(not tvaLib.Parse.versionIsMoreRecent(s_version, config.version, config.autoflush_ver_depth, orEqual=True) and commands.verbose): tvaLib.printWarning('User pairs data appears to be saved with an older version ('+s_version+'). The analysis will proceed, but consider flushing the data if errors occur.', local['gen_warning'])
1897  if(commands.verbose): print(' Loading saved user pairs data...')
1898  predictionMethod.params = pickle.load(input_data)
1899  seq_userPairs = pickle.load(input_data)
1900  repop_rate = seq_userPairs.repopulateObjectData(objects_uncon.getSubSpacebyId(sites[siteIx].idx).getSubSpacebyId(sites[siteIx][camIx].idx).getSubSpacebyId(sites[siteIx][camIx][fileIx].idx))
1901  if(repop_rate < 0.85): raise Exception, [2062, 'Failed to repopulate interactions ('+str(round(100*repop_rate, 1))+'% of user pairs remain). Saved data is probably corrupted.']
1902  elif(repop_rate < 1 and commands.verbose): tvaLib.printWarning('Failed to repopulate interactions ('+str(round(100*repop_rate, 1))+'% of user pairs remain). Analysis will proceed on the remaining user pairs.', local['gen_warning'])
1903 
1904 
1905  #Dump statistics
1906  if(commands.verbose and not flush):
1907  if(seq_userPairs.skipped_rate_avg): print(' Skipping rate: None-'+str(seq_userPairs.skipped_rate_none)+', Partial-'+str(seq_userPairs.skipped_rate_part)+', Full-'+str(seq_userPairs.skipped_rate_full)+', AvgRate-'+str(seq_userPairs.skipped_rate_avg))
1908  print(' Interaction instants with TTC indicators: '+str(seq_userPairs.getInstantWIndicatorCount())+'/'+str(seq_userPairs.getInstantCount()))
1909  print(' User pairs with TTC indicators: '+str(seq_userPairs.getIntWTTCCount())+'/'+str(len(seq_userPairs)))
1910 
1911 
1914  if(flush or commands.plot_dmp or commands.save_data_gen_dmp):
1915  #Prepare and regulate prediction method, as needed
1916  if(predictionMethod.getPrepare()):
1917  if(commands.verbose): print(' Preparing prediction method...')
1918 
1919  #MPA Protocol
1920  if(predictionMethod.protocol == 'dmp'):
1921  predictionMethod.prepare(objects_uncon.getSubSpacebyId(sites[siteIx].idx).getAll())
1922  #Export prediction method data
1923  if((commands.save_data or commands.save_data_light or commands.save_data_gen_dmp) and predictionMethod.export() and commands.verbose): print(' Exporting motion pattern memory...')
1924 
1925  if(commands.save_data_gen_dmp): raise Exception, [91, 'MPA generation complete.']
1926 
1927 
1930  if(flush):
1931  #Generate user pairs
1932  if(commands.verbose): print(' Finding user pairs...')
1933  seq_userPairs = tvaInteraction.UserPairs(objects_uncon.getSubSpacebyId(sites[siteIx].idx).getSubSpacebyId(sites[siteIx][camIx].idx).getSubSpacebyId(sites[siteIx][camIx][fileIx].idx), methodName=predictionMethod.label, maxUserPairs=config.up_maxUserPairsPerHour*sites[siteIx][camIx][fileIx].getDuration()/3600.0, max_distance=config.up_maxDistance, verbose=commands.verbose)
1934 
1935  #Compute interaction indicators
1936  if(commands.verbose): print(' Calculating TTC indicators...')
1937  seq_userPairs.calculateIndicators(predictionMethod, triage=sites[siteIx].alignments, threads=commands.threads, timeHorizon=int(config.timehorizon*sites[siteIx][camIx].camera.frameRate), verbose=commands.verbose)
1938  seq_userPairs.dropSafetyIndicatorsByValue(indicatorLowerBound=2)
1939 
1940  #Constrain to analysis zone
1941  seq_userPairs.constrainToZone(site_analyses[saIx].zone)
1942 
1943  #Dump statistics
1944  if(commands.verbose):
1945  if(seq_userPairs.skipped_rate_avg): print(' Skipping rate: None-'+str(seq_userPairs.skipped_rate_none)+', Partial-'+str(seq_userPairs.skipped_rate_part)+', Full-'+str(seq_userPairs.skipped_rate_full)+', AvgRate-'+str(seq_userPairs.skipped_rate_avg))
1946  print(' Interaction instants with TTC indicators: '+str(seq_userPairs.getInstantWIndicatorCount())+'/'+str(seq_userPairs.getInstantCount()))
1947  print(' User pairs with TTC indicators: '+str(seq_userPairs.getIntWTTCCount())+'/'+str(len(seq_userPairs)))
1948 
1949  #Build object frame lookup table
1950  if(commands.verbose): print(' Building object-frame lookup table...')
1951  objFrameTable = tvaScene.ObjFrameTable(objects_uncon.getSubSpacebyId(sites[siteIx].idx).getSubSpacebyId(sites[siteIx][camIx].idx).getSubSpacebyId(sites[siteIx][camIx][fileIx].idx))
1952  seq_userPairs.genExposure(objFrameTable, label='5 minutes', frames=5*60*sites[siteIx][camIx].camera.frameRate)
1953  seq_userPairs.genExposure(objFrameTable, label='2 minutes', frames=2*60*sites[siteIx][camIx].camera.frameRate)
1954  seq_userPairs.genExposure(objFrameTable, label='1 minute', frames=60*sites[siteIx][camIx].camera.frameRate)
1955  seq_userPairs.genExposure(objFrameTable, label='15 seconds', frames=15*sites[siteIx][camIx].camera.frameRate)
1956 
1957 
1960  if((commands.save_data or commands.save_data_light) and len(seq_userPairs) > 0 and len(predictionMethods) > 0):
1961  if(commands.verbose): print(' Saving interaction data to cache...')
1962 
1963  userPair = deepcopy(seq_userPairs)
1964  userPair.garbageCollectObjectData()
1965  predictionMethod.params.cleanUp()
1966  with open(os.path.join(site_analyses[saIx].getFullResultsFolder(), filename), 'wb') as output:
1967  pickle.dump(config.version, output, protocol=2)
1968  pickle.dump(predictionMethod.params, output, protocol=2)
1969  pickle.dump(userPair, output, protocol=2)
1970  userPair = None
1971 
1972 
1973 
1976  if(commands.play_int):
1977  import lib.interface as tvaInterface
1978  tvaInterface.InteractionsInspectionInterface(seq_userPairs,
1979  sites[siteIx][camIx][fileIx].getFullVideoFilename(),
1980  local=local,
1981  satFilename=sites[siteIx].getFullSatFilename(),
1982  satRes=sites[siteIx].satres,
1983  homography=sites[siteIx][camIx].getHomography(),
1984  fps=sites[siteIx][camIx].camera.frameRate,
1985  intrinsicCameraMatrix=sites[siteIx][camIx].camera.camera_matrix.data,
1986  distortionCoefficients=sites[siteIx][camIx].camera.dist_coeffs.data,
1987  undistortedImageScalingFactor=sites[siteIx][camIx].camera.getImageScalingFactor(),
1988  startTime=sites[siteIx][camIx][fileIx].startTime,
1989  config=config,
1990  verbose=commands.verbose)
1991 
1992 
1995  if(commands.timeseries and type(commands.timeseries) == list):
1996  for userPair in seq_userPairs:
1997  userPairNums=list(userPair.roadUserNumbers)
1998 
1999  # Target specific user number
2000  if(not commands.timeseries[0] in userPairNums): continue
2001  if(commands.timeseries[0] and not commands.timeseries[0] in userPairNums): continue
2002 
2003  smallestTTCVal = sys.maxint
2004  localTTCList = []
2005  for interaction in userPair.interactions:
2006  localTTCList.append([])
2007  for indicator in interaction:
2008  if(indicator.type == 'TTC'):
2009  if(indicator.value < smallestTTCVal): smallestTTCVal = indicator.value
2010  localTTCList[-1].append({'TTC':indicator.value, 'frame':interaction.time, 'X':indicator.x, 'Y':indicator.y, 'prob':indicator.prob})
2011 
2012  if(smallestTTCVal < 2.0*sites[siteIx][camIx].camera.frameRate):
2013  if(not commands.file): raise Exception, [2950, 'Multiple sequences not yet supported with time series analysis.']
2014  if(commands.verbose): tvaLib.printTimeStamp('Time series video playback for '+str(userPairNums[0])+' and '+str(userPairNums[1])+'...')
2015  siteIx, camIx, fileIx = tvaRuntime.targetSequence(commands, sites, config)
2016  tvaVis.plotSettings(style=commands.fig_style, size=config.plot_text_size, family=config.font_family, verbose=commands.verbose)
2017  tvaVis.traj2D([objects_uncon.getSubSpacebyId(sites[siteIx].idx).getSubSpacebyId(sites[siteIx][camIx].idx).getSubSpacebyId(sites[siteIx][camIx][fileIx].idx).getAll()[tvaLib.Obj.num2ind(objects_uncon.getSubSpacebyId(sites[siteIx].idx).getSubSpacebyId(sites[siteIx][camIx].idx).getSubSpacebyId(sites[siteIx][camIx][fileIx].idx).getAll(), userPairNums[0])], objects_uncon.getSubSpacebyId(sites[siteIx].idx).getSubSpacebyId(sites[siteIx][camIx].idx).getSubSpacebyId(sites[siteIx][camIx][fileIx].idx).getAll()[tvaLib.Obj.num2ind(objects_uncon.getSubSpacebyId(sites[siteIx].idx).getSubSpacebyId(sites[siteIx][camIx].idx).getSubSpacebyId(sites[siteIx][camIx][fileIx].idx).getAll(), userPairNums[1])]], draw_max_traj=config.draw_max_traj, xy_bounds=site_analyses[saIx].xy_bounds, mask=site_analyses[saIx].getCombinedMasks(), labels=True, velocity_vector=False, local=local, fig_name='Exp', figsize=config.figsize)
2018  plt.show()
2019  #List of avreage TTCs
2020  if(False):
2021  listAvgTTCy = []
2022  listAvgTTCx = []
2023  for instance in localTTCList:
2024  listAvgTTCy.append(sum([x['TTC']*x['prob'] for x in instance])/sum([x['prob'] for x in instance]))
2025  listAvgTTCx.append(instance[0]['frame'])
2026  print(listAvgTTCy)
2027  print(listAvgTTCx)
2028  #TTC Scatter
2029  print(tvaLib.flatten_list([[y['TTC'] for y in x] for x in localTTCList]))
2030  print(tvaLib.flatten_list([[y['frame'] for y in x] for x in localTTCList]))
2031  print(tvaLib.flatten_list([[y['prob'] for y in x] for x in localTTCList]))
2032 
2033 
2036  userPairs[predictionMethod.idx][-1][-1].add(seq_userPairs, idx=fileIx, metadata=sites[siteIx][camIx][fileIx])
2037 
2038 
2039 
2040 
2043 
2044 
2045 
2048  if((commands.fig_save or commands.plot_dmp) and not commands.hli_only):
2049  if(commands.verbose): tvaLib.printTimeStamp('Generating figures...')
2050  plotSettings = tvaVis.plotSettings(style=commands.fig_style, size=config.plot_text_size, family=config.font_family, verbose=commands.verbose)
2051  if(commands.fig_save_lan_a): fig_lan_suffix = config.language
2052  else: fig_lan_suffix = None
2053 
2054 
2057  if(commands.fig_save and not commands.hli_only):
2058  if(commands.verbose): print(' Generating basic figures...')
2059  figures = []
2060 
2061 
2062  figures.append(tvaVis.plotGeometry(alignments=site_analyses[saIx].site.alignments, sateliteImage=site_analyses[saIx].site.getFullSatFilename(), sateliteResolution=site_analyses[saIx].site.satres, xy_bounds=site_analyses[saIx].xy_bounds, zone=site_analyses[saIx].zone, virtual_loops=site_analyses[saIx].virtual_loops, labels=True, local=local, figsize=config.figsize, plotSettings=plotSettings))
2063  figures.append(tvaVis.traj2DHighlightAlignments(objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll(), xy_bounds=site_analyses[saIx].xy_bounds, alignments=site_analyses[saIx].site.alignments))
2064 
2065 
2066 
2067  figures.append(tvaVis.traj2D(objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll(), draw_max_traj=config.draw_max_traj, xy_bounds=site_analyses[saIx].xy_bounds, alignments=site_analyses[saIx].site.alignments, mask=site_analyses[saIx].getCombinedMasks(), zone=site_analyses[saIx].zone, virtual_loops=None, labels=True, objects_uncon=objects_uncon.getAll(), velocity_vector=False, local=local, figsize=config.figsize, plotSettings=plotSettings))
2068  figures.append(tvaVis.traj2D(objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll(), draw_max_traj=config.draw_max_traj, xy_bounds=site_analyses[saIx].xy_bounds, alignments=site_analyses[saIx].site.alignments, SDR=site_analyses[saIx].site.alignments.SDR, virtual_loops=None, labels=True, objects_uncon=objects_uncon.getAll(), velocity_vector=False, local=local, fig_name='Vehicle Trajectories (SDR)', figsize=config.figsize, plotSettings=plotSettings))
2069  figures.append(tvaVis.traj2D(objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll(), draw_max_traj=config.draw_max_traj, xy_bounds=site_analyses[saIx].xy_bounds, alignments=site_analyses[saIx].site.alignments, mask=site_analyses[saIx].getCombinedMasks(), zone=site_analyses[saIx].zone, virtual_loops=None, sateliteImage=site_analyses[saIx].site.getFullSatFilename(), sateliteResolution=site_analyses[saIx].site.satres, labels=True, velocity_vector=False, local=local, fig_name='Vehicle Trajectories (Satellite)', figsize=config.figsize, plotSettings=plotSettings))
2070  figures.append(tvaVis.traj2D(objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll(), draw_max_traj=config.draw_max_traj, xy_bounds=site_analyses[saIx].xy_bounds, alignments=site_analyses[saIx].site.alignments, mask=site_analyses[saIx].getCombinedMasks(), zone=site_analyses[saIx].zone, virtual_loops=site_analyses[saIx].virtual_loops, labels=False, objects_uncon=objects_uncon.getAll(), velocity_vector=False, local=local, fig_name='Loop detectors', figsize=config.figsize, plotSettings=plotSettings))
2071  figures.append(tvaVis.traj2D(objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll(), draw_max_traj=config.draw_max_traj, xy_bounds=site_analyses[saIx].xy_bounds, alignments=site_analyses[saIx].site.alignments, traj_colour='class', draw_connectors=False, alignAlpha=0.2, labels=False, velocity_vector=False, local=local, fig_name='Vehicle Trajectories (Classification)', figsize=config.figsize, plotSettings=plotSettings))
2072  figures.append(tvaVis.microTrafficIntensity(objects.getSubSpacebyId(site_analyses[saIx].site.idx), this_run_startTime, this_run_endTime, site_analyses[saIx].cameras[0].camera.frameRate, local=local, figsize=config.figsize, plotSettings=plotSettings))
2073 
2074 
2075 
2076  figures.append(tvaVis.speedMap(objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll(), ptype='heat', sm_scale=[0,site_analyses[saIx].max_speed], colour=config.cm_colour, gridsize=site_analyses[saIx].getGridHexSize(), mincnt=config.minVFcount, virtualLoops=site_analyses[saIx].virtual_loops.getEdgeCountVector(), zone=site_analyses[saIx].zone, alignments=site_analyses[saIx].site.alignments, local=local, fig_name='Speed Heat Map (Normalised)', figsize=config.figsize))
2077  figures.append(tvaVis.speedMap(objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll(), ptype='heat', colour=config.cm_colour, gridsize=site_analyses[saIx].getGridHexSize(), mincnt=config.minVFcount, virtualLoops=site_analyses[saIx].virtual_loops.getEdgeCountVector(), zone=site_analyses[saIx].zone, alignments=site_analyses[saIx].site.alignments, local=local, fig_name='Speed Heat Map', figsize=config.figsize))
2078  figures.append(tvaVis.speedMap(objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll(), ptype='vector', sm_scale=[0,site_analyses[saIx].max_speed], colour=config.cm_colour, gridsize=site_analyses[saIx].getGridHexSize(), mincnt=config.minVFcount, virtualLoops=site_analyses[saIx].virtual_loops.getEdgeCountVector(), zone=site_analyses[saIx].zone, alignments=site_analyses[saIx].site.alignments, local=local, fig_name='Speed Vector Map', figsize=config.figsize))
2079  figures.append(tvaVis.speedMap(objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll(), ptype='stream', sm_scale=[0,site_analyses[saIx].max_speed], colour=config.cm_colour, gridsize=site_analyses[saIx].getGridHexSize(), mincnt=config.minVFcount, virtualLoops=site_analyses[saIx].virtual_loops.getEdgeCountVector(), zone=site_analyses[saIx].zone, alignments=site_analyses[saIx].site.alignments, local=local, fig_name='Speed Stream Map', figsize=config.figsize))
2080  figures.append(tvaVis.speedHist(speed_histo, sm_scale=site_analyses[saIx].max_speed, local=local, figsize=config.figsize))
2081  figures.append(tvaVis.speedHist(speed_histo_by_lane.values(), labels=[site_analyses[saIx].site.alignments[align].getPrintName() for align in speed_histo_by_lane.keys()], sm_scale=site_analyses[saIx].max_speed, local=local, fig_name='Speed Distribution by Lane', figsize=config.figsize))
2082  figures.append(tvaVis.speedHist(speed_histo_by_class.values(), labels=[local['userTypeNames'][class_] for class_ in speed_histo_by_class.keys()], sm_scale=site_analyses[saIx].max_speed, local=local, fig_name='Speed Distribution by Classification', figsize=config.figsize))
2083  figures.append(tvaVis.visAlignData(speed_by_lane, slsd, local=local, plotSettings=plotSettings))
2084  figures.append(tvaVis.traj2D(objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll(), draw_max_traj=config.draw_max_traj, xy_bounds=site_analyses[saIx].xy_bounds, alignments=site_analyses[saIx].site.alignments, mask=site_analyses[saIx].getCombinedMasks(), zone=site_analyses[saIx].zone, virtual_loops=None, labels=True, objects_uncon=objects_uncon.getAll(), velocity_vector=False, local=local, constantFramerate=sites[siteIx][camIx].camera.frameRate, mps_kmh=config.mps_kmh, traj_colour='speed', sm_scale=site_analyses[saIx].max_speed, cm_colour=config.cm_colour, fig_name='Vehicle Trajectories (Speed)', figsize=config.figsize, plotSettings=plotSettings))
2085 
2086 
2087  figures.append(tvaVis.visualiseOD(ODMatrix, xy_bounds=site_analyses[saIx].xy_bounds, alignments=site_analyses[saIx].site.alignments, sateliteImage=site_analyses[saIx].site.getFullSatFilename(), sateliteResolution=site_analyses[saIx].site.satres, local=local, fig_name='OD Visualisation', figsize=config.figsize, verbose=commands.verbose).get())
2088 
2089 
2090  if(commands.infractions):
2091  pass
2092 
2095 
2096 
2097  if(commands.mot):
2098  figures.append(tvaVis.traj2D(objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll(), companion_objects=annotations, draw_max_traj=config.draw_max_traj, xy_bounds=site_analyses[saIx].xy_bounds, alignments=sites[siteIx].alignments, zone=site_analyses[saIx].zone, local=local, fig_name='MOT trajectories comparison', traj_colour_alternate='m', figsize=config.figsize, plotSettings=plotSettings))
2099 
2100 
2101  figures_false = sum([1 for figure in figures if figure == False])
2102  if(figures_false): tvaLib.printWarning('Omited figures: '+str(figures_false)+'/'+str(len(figures)), local['gen_warning'])
2103 
2104 
2105  tvaVis.Save(figures, site_analyses[saIx].getFullResultsFolder(), fig_format=commands.fig_format, fig_resize=commands.fig_resize, fig_lan_suffix=fig_lan_suffix, fig_bg_colour=config.fig_bg_colour, verbose=commands.verbose)
2106  tvaVis.close()
2107 
2108 
2109 
2112  if(commands.fig_save and config.plot_dropped_traj and not commands.hli_only):
2113  if(commands.verbose): print(' Generating filtering figures...')
2114  figures = []
2115 
2116 
2117  for alignIx in range(len(site_analyses[saIx].site.alignments)):
2118  figures.append(tvaVis.traj2D([obj for obj in objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll() if alignIx in obj.curvilinearPositions.getLanes()], draw_max_traj=config.draw_max_traj, xy_bounds=site_analyses[saIx].xy_bounds, alignments=site_analyses[saIx].site.alignments, local=local, labels=True, fig_name='Trajectories bound on '+site_analyses[saIx].site.alignments[alignIx].getPrintName(), figsize=config.figsize, plotSettings=plotSettings))
2119 
2120 
2121  figures.append(tvaVis.traj2D(dropped_trajectories['bb'], draw_max_traj=config.draw_max_traj, xy_bounds=site_analyses[saIx].xy_bounds, alignments=site_analyses[saIx].site.alignments, mask=site_analyses[saIx].getCombinedMasks(), local=local, fig_name='BoundingBox Dropped Trajectories', traj_colour='random', figsize=config.figsize, plotSettings=plotSettings))
2122  figures.append(tvaVis.traj2D(dropped_trajectories['tc'], draw_max_traj=config.draw_max_traj, xy_bounds=site_analyses[saIx].xy_bounds, alignments=site_analyses[saIx].site.alignments, mask=site_analyses[saIx].getCombinedMasks(), local=local, fig_name='Coordinate Transformation Dropped Trajectories', traj_colour='random', figsize=config.figsize, plotSettings=plotSettings))
2123  figures.append(tvaVis.traj2D(dropped_trajectories['te'], draw_max_traj=config.draw_max_traj, xy_bounds=site_analyses[saIx].xy_bounds, alignments=site_analyses[saIx].site.alignments, mask=site_analyses[saIx].getCombinedMasks(), local=local, fig_name='Tracking Error Dropped Trajectories', traj_colour='random', figsize=config.figsize, plotSettings=plotSettings))
2124  figures.append(tvaVis.traj2D(dropped_trajectories['td'], draw_max_traj=config.draw_max_traj, xy_bounds=site_analyses[saIx].xy_bounds, alignments=site_analyses[saIx].site.alignments, mask=site_analyses[saIx].getCombinedMasks(), local=local, fig_name='Duplicate Tracking Dropped Trajectories', traj_colour='random', figsize=config.figsize, plotSettings=plotSettings))
2125  figures.append(tvaVis.traj2D(dropped_trajectories['ie'], draw_max_traj=config.draw_max_traj, xy_bounds=site_analyses[saIx].xy_bounds, alignments=site_analyses[saIx].site.alignments, mask=site_analyses[saIx].getCombinedMasks(), local=local, fig_name='Integrity Error Dropped Trajectories', traj_colour='random', figsize=config.figsize, plotSettings=plotSettings))
2126  figures.append(tvaVis.traj2D(dropped_trajectories['so'], draw_max_traj=config.draw_max_traj, xy_bounds=site_analyses[saIx].xy_bounds, alignments=site_analyses[saIx].site.alignments, mask=site_analyses[saIx].getCombinedMasks(), local=local, fig_name='Outlier Split Below Retention', traj_colour='random', figsize=config.figsize, plotSettings=plotSettings))
2127 
2128 
2129  figures_false = sum([1 for figure in figures if figure == False])
2130  if(figures_false): tvaLib.printWarning('Omited figures: '+str(figures_false)+'/'+str(len(figures)), local['gen_warning'])
2131 
2132 
2133  tvaVis.Save(figures, site_analyses[saIx].getFullResultsFolder(config.output_folder, customSubPath='Filter'), fig_format=commands.fig_format, fig_resize=commands.fig_resize, fig_lan_suffix=fig_lan_suffix, fig_bg_colour=config.fig_bg_colour, verbose=commands.verbose)
2134  tvaVis.close()
2135 
2136 
2137 
2140  if(commands.fig_save and len(commands.indicators) > 0 and not commands.hli_only):
2141  if(commands.verbose): print(' Generating conflict/SSM figures...')
2142  figures = []
2143  labels = {}
2144  pointTypeArgs = [{'ptype':'CP', 'maxIndThreshold':sys.maxint},
2145  {'ptype':'CP', 'maxIndThreshold':config.TTCthreshSeconds*sites[siteIx][camIx].camera.frameRate},
2146  {'ptype':'CZ', 'maxIndThreshold':sys.maxint}]
2147  distArgs = [{'method':0, 'percentile':0.0, 'label':local['int_all_instants']},
2148  {'method':1, 'percentile':0.15, 'label':local['int_unique_15th']},
2149  {'method':1, 'percentile':0.0, 'label':local['int_unique_min']},
2150  {'method':3, 'percentile':1.0, 'label':local['int_unique_max_prob']}]
2151 
2152 
2153  listOfExistingUserTypes = list(set([None]+[i for i, x in enumerate([[x.getUserType() for x in objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll()].count(i) for i in range(len(local['userTypeNames']))]) if x != 0]))
2154 
2155 
2156  for predictionMethod in predictionMethods:
2157  userPair_list = userPairs[predictionMethod.idx].getAll()
2158  if(not userPair_list): continue
2159  labels['predictionMethod'] = str(predictionMethod.label_short)
2160 
2161 
2162  for userType1 in listOfExistingUserTypes:
2163  for userType2 in listOfExistingUserTypes[listOfExistingUserTypes.index(userType1):]:
2164 
2165 
2166  if(userType1 == None):
2167  if(userType2 == None): labels['userType'] = ''
2168  else: labels['userType'] = local['userTypeNames'][userType2]
2169  else:
2170  if(userType2 == None): labels['userType'] = local['userTypeNames'][userType1]
2171  else: labels['userType'] = local['userTypeNames'][userType1]+'-'+local['userTypeNames'][userType2]
2172 
2173 
2177  for pointTypeArg in pointTypeArgs:
2178 
2179 
2180  if(pointTypeArg['maxIndThreshold'] != sys.maxint): labels['pointProb'] = 'lt'+str(config.TTCthreshSeconds)+'s'
2181  else: labels['pointProb'] = ''
2182  labels['pointType'] = pointTypeArg['ptype']
2183 
2184  if(predictionMethod.protocol == 'dmp'):
2185  '''
2186  #Enable point scattering
2187  #CPmappingData = [{'data': userPair_list.getPointList(ptype='CP', format='columns') ....
2188  if(CPmappingDatum['data']):
2189  if(predictionMethod.getPrepare()): predictionMethod.prepare()
2190  CPmappingDatum['data'][1] = tvaVis.scatterInBins(CPmappingDatum['data'][1], predictionMethod.getPredictionClass().xrange.getRange())
2191  CPmappingDatum['data'][2] = tvaVis.scatterInBins(CPmappingDatum['data'][2], predictionMethod.getPredictionClass().yrange.getRange())
2192 
2193  figures.append(tvaVis.interactionMap(CPmappingDatum['data'], ptype = 'hexbin', weightedDensity=True, colour=config.cm_colour, xy_bounds=site_analyses[saIx].cm_bounds, z_bounds=0.05, gridsize=site_analyses[saIx].getGridHexSize(), minGridSize=config.hex_grid_min_size, alignments=sites[siteIx].alignments, mask=site_analyses[saIx].getCombinedMasks(), zone=site_analyses[saIx].zone, labels=True, local=local, zlabel=local['vis_cp_label_density'], fig_name=CPmappingDatum['figureName']+' density', verbose=commands.verbose, plotSettings=plotSettings))
2194  '''
2195  zlabel=local['vis_cp_label_prob']
2196  else: zlabel=None
2197 
2198  figures.append(tvaVis.interactionMap(userPair_list.getPointList(userType1=userType1, userType2=userType2, format='columns', **pointTypeArg), ptype='hexbin', weightedDensity=False, colour=config.cm_colour, xy_bounds=site_analyses[saIx].cm_bounds, gridsize=site_analyses[saIx].getGridHexSize(), minGridSize=config.hex_grid_min_size, alignments=sites[siteIx].alignments, mask=site_analyses[saIx].getCombinedMasks(), zone=site_analyses[saIx].zone, labels=True, local=local, zlabel=zlabel, fig_name='Map_'+'_'.join([labels[i] for i in labels if labels[i] != '']), verbose=commands.verbose, plotSettings=plotSettings))
2199 
2200 
2203  if(userPair_list.getInstantCount()):
2204  try: label_ratio=local['vis_cp_hist_label_p1']+' '+str(userPair_list.getInstantWIndicatorCount())+'\n'+local['vis_cp_hist_label_p2']+' '+str(seq_userPairs.getInstantCount())+'\n'+local['vis_cp_hist_label_p3']+' '+str(round(float(seq_userPairs.getInstantWIndicatorCount())/float(seq_userPairs.getInstantCount()),3)*100)+'%'
2205  except ZeroDivisionError: label_ratio=local['vis_cp_hist_label_p1']+' '+str(userPair_list.getInstantWIndicatorCount())+'\n'+local['vis_cp_hist_label_p2']+' '+str(seq_userPairs.getInstantCount())+'\n'+local['vis_cp_hist_label_p3']+' NAN'
2206 
2207  for dist_type in ['pdf','cdf']:
2208  figures.append(tvaVis.interactionTTCHist([[userPair_list.genIndicatorDistribution(userType1=userType1, userType2=userType2, method=distArg['method'], percentile=distArg['percentile'], dist_type=dist_type, bins=range(0,int(config.disp_timehorizon*sites[siteIx][camIx].camera.frameRate),1)), distArg['label']] for distArg in distArgs], fps=sites[siteIx][camIx].camera.frameRate, timehorizon=config.disp_timehorizon, labels=True, label_ratio=label_ratio, local=local, fig_name='TTC_'+'_'.join([dist_type,labels['predictionMethod'],labels['userType']]), dist_type=dist_type, figsize=config.figsize, verbose=commands.verbose))
2209 
2210 
2211  figures.append(tvaVis.interactionTimeseries(userPair_list, fps=sites[siteIx][camIx].camera.frameRate, local=local, fig_name='TTC_timeseries_'+labels['predictionMethod'], figsize=config.figsize, verbose=commands.verbose).get())
2212 
2213 
2214  figures_false = sum([1 for figure in figures if figure == False])
2215  if(figures_false): tvaLib.printWarning('Omited figures: '+str(figures_false)+'/'+str(len(figures)), local['gen_warning'])
2216 
2217 
2218  tvaVis.Save(figures, site_analyses[saIx].getFullResultsFolder(config.output_folder, customSubPath='SSM'), fig_format=commands.fig_format, fig_resize=commands.fig_resize, fig_lan_suffix=fig_lan_suffix, fig_bg_colour=config.fig_bg_colour, verbose=commands.verbose)
2219  tvaVis.close()
2220 
2221 
2222 
2226  if(commands.plot_dmp and not commands.hli_only):
2227  if(commands.verbose): print(' Generating Discretized Motion Pattern figures...')
2228 
2229  for predictionMethod in predictionMethods:
2230  if(predictionMethod.protocol == 'dmp'):
2231  tvaVis.DrawOther.DMPExperimental(predictionMethod.getPredictionClass(), objects_uncon.getSubSpacebyId(site_analyses[saIx].site.idx).getAll(), site_analysis=site_analyses[saIx], config=config, fig_format=commands.fig_format, plotSettings=plotSettings, verbose=commands.verbose)
2232  if(len(commands.indicators) > 0):
2233  tvaVis.DrawOther.viewInteractionAsDMP(userPairs[predictionMethod.idx].getAll(), predictionMethod.getPredictionClass(), site_analysis=site_analyses[saIx], config=config, fig_format=commands.fig_format, plotSettings=plotSettings, verbose=commands.verbose)
2234 
2235 
2236 
2239  if(not commands.hli_only):
2240  if(commands.verbose): tvaLib.printTimeStamp('Saving data to CSVs...')
2241 
2242  if(commands.verbose >= 2): print(' Saving data to results.csv...')
2243  try:
2244  with open(os.path.join(site_analyses[saIx].getFullResultsFolder(), 'results.csv'), 'wb') as f:
2245  writer = csv_writer(f)
2246 
2247 
2249  VLTableHeadings1 = []
2250  VLTableHeadings2 = []
2251  for align in site_analyses[saIx].site.alignments:
2252  VLTableHeadings1.append(align.getPrintName()+' '+local['gen_in'].encode('utf-8').strip())
2253  VLTableHeadings2.append(align.getPrintName()+' '+local['gen_out'].encode('utf-8').strip())
2254 
2255 
2256  try:
2257  writer.writerow([local['out_class'].encode('utf-8').strip()])
2258  for i in range(len(local['userTypeNames'])):
2259  writer.writerow([local['userTypeNames'][i], [x.getUserType() for x in objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll()].count(i), [x.getUserType() for x in objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll()].count(i)/float(len(objects.getSubSpacebyId(site_analyses[saIx].site.idx).getAll()))])
2260  writer.writerow(['============================'])
2261  except: pass
2262  #-------------------------------
2263  writer.writerow([local['out_lc'].encode('utf-8').strip()])
2264  writer.writerow(['']+[align.getPrintName() for align in site_analyses[saIx].site.alignments])
2265  for align,row in zip(site_analyses[saIx].site.alignments, laneChanges):
2266  writer.writerow([align.getPrintName()]+row)
2267  writer.writerow(['============================'])
2268  #-------------------------------
2269  writer.writerow([local['out_od_matrix'].encode('utf-8').strip()])
2270  writer.writerow(['']+[align.getPrintName() for align in site_analyses[saIx].site.alignments])
2271  for align,row in zip(site_analyses[saIx].site.alignments, ODMatrix):
2272  writer.writerow([align.getPrintName()]+row)
2273  writer.writerow(['============================'])
2274  #-------------------------------
2275  writer.writerow([local['out_edge_counts'].encode('utf-8').strip()])
2276  writer.writerow([local['gen_minutes'].encode('utf-8').strip(), local['gen_clock'].encode('utf-8').strip(), '|']+VLTableHeadings1+['|']+VLTableHeadings2)
2277  for i in range(len(site_analyses[saIx].virtual_loops.edge_m15)):
2278  dta15 = site_analyses[saIx].virtual_loops.getEdgeCountVector(i=i, m='m15')
2279  dta15.insert(site_analyses[saIx].virtual_loops.lanes, '|')
2280  writer.writerow([i*15, (datetime(2010,01,01) + timedelta(minutes=i*15)).strftime('%H:%M'), '|']+dta15)
2281  writer.writerow(['============================'])
2282  #-------------------------------
2283  writer.writerow([local['out_edge_speed'].encode('utf-8').strip()])
2284  writer.writerow([local['gen_minutes'].encode('utf-8').strip(), local['gen_clock'].encode('utf-8').strip(), '|']+VLTableHeadings1+['|']+VLTableHeadings2)
2285  for i in range(len(site_analyses[saIx].virtual_loops.edge_m15)):
2286  dta15 = [x*site_analyses[saIx].cameras[0].camera.frameRate*config.mps_kmh for x in site_analyses[saIx].virtual_loops.getEdgeMeanSpeedVector(i=i, m='m15')]
2287  dta15.insert(site_analyses[saIx].virtual_loops.lanes, '|')
2288  writer.writerow([i*15, (datetime(2010,01,01) + timedelta(minutes=i*15)).strftime('%H:%M'), '|']+dta15)
2289  #TODO: handle variable framerate between different cameras
2290  writer.writerow(['============================'])
2291  #-------------------------------
2292  writer.writerow([local['out_od_time'].encode('utf-8').strip()])
2293  writer.writerow([local['gen_minutes'].encode('utf-8').strip(), local['gen_clock'].encode('utf-8').strip(), '|']+tvaLib.flatten_list([[str(align.id+1)+'->'+str(align2.id+1) for align2 in site_analyses[saIx].site.alignments] for align in site_analyses[saIx].site.alignments]))
2294  for i in range(len(ODMatrix_Time)):
2295  writer.writerow([i*15, (datetime(2010,01,01) + timedelta(minutes=i*15)).strftime('%H:%M'), '|']+tvaLib.flatten_list(ODMatrix_Time[i]))
2296  writer.writerow(['============================'])
2297  #-------------------------------
2298  if(site_analyses[saIx].virtual_loops.loops > 0 and len(site_analyses[saIx].virtual_loops.coord_m15) > 0):
2299  VLTableHeadings1 = []
2300  for i in range(site_analyses[saIx].virtual_loops.loops):
2301  VLTableHeadings1.append('['+str(site_analyses[saIx].virtual_loops.coordinates[i].x)+','+str(site_analyses[saIx].virtual_loops.coordinates[i].y)+']')
2302  writer.writerow([local['out_edge_vlc'].encode('utf-8').strip()])
2303  writer.writerow([local['out_time'].encode('utf-8').strip()]+VLTableHeadings1)
2304  for i in range(len(site_analyses[saIx].virtual_loops.coord_m15)):
2305  writer.writerow([i*15]+site_analyses[saIx].virtual_loops.getCoordCountVector(i=i, m='m15'))
2306  writer.writerow(['============================'])
2307 
2308  writer.writerow([local['out_edge_vls'].encode('utf-8').strip()])
2309  writer.writerow([local['out_time'].encode('utf-8').strip()]+VLTableHeadings1)
2310  for i in range(len(site_analyses[saIx].virtual_loops.coord_m15)):
2311  writer.writerow([i*15]+[x*site_analyses[saIx].cameras[0].camera.frameRate*config.mps_kmh for x in site_analyses[saIx].virtual_loops.getCoordMeanSpeedVector(i=i, m='m15')])
2312  writer.writerow(['============================'])
2313  #-------------------------------
2314  writer.writerow(['Speed profile (each column is 1/80 of the profile; row 1 = distance along profile (m), row 2 = mean speed (km/h), row 3 = standard deviation speed)'])
2315  for align,alignment in zip(range(len(speed_by_lane)),site_analyses[saIx].site.alignments):
2316  writer.writerow([alignment.getPrintName()])
2317  writer.writerows(speed_by_lane[align][1:4])
2318  writer.writerow(['============================'])
2319  #-------------------------------
2320  writer.writerow([local['out_stop_speed_histo'].encode('utf-8').strip()])
2321  writer.writerow([local['gen_speed'].encode('utf-8').strip()]+list(speed_histo[0]))
2322  writer.writerow([local['gen_observations'].encode('utf-8').strip()]+list(speed_histo[1]))
2323  writer.writerow(['PDF']+list(speed_histo[2]))
2324  writer.writerow(['============================'])
2325  #-------------------------------
2326  writer.writerow([local['out_stop_stitch'].encode('utf-8').strip()])
2327  writer.writerow([len(dropped_trajectories['st'])])
2328  except IOError: tvaLib.printWarning('results.csv could not be opened for saving.', local['gen_warning'])
2329 
2330 
2331 
2332  try:
2333  for predictionMethod in predictionMethods:
2334  userPair_list = userPairs[predictionMethod.idx].getAll()
2335  if(not userPair_list or not hasattr(userPair_list, 'getPointList')): continue
2336  try:
2337  if(commands.verbose >= 2): print(' Saving data to TTC_'+predictionMethod.label_short+'.csv...')
2338  with open(os.path.join(site_analyses[saIx].getFullResultsFolder(), 'TTC_'+predictionMethod.label_short+'.csv'), 'wb') as f:
2339  writer = csv_writer(f, delimiter=',', quotechar='"', quoting=csv_QUOTE_MINIMAL)
2340  writer.writerow(['TTC','X','Y','Prob','Frame','Num1','Num2'])
2341  for i in userPair_list.getPointList():
2342  i[0] = i[0]/sites[siteIx][camIx].camera.frameRate
2343  writer.writerow(i)
2344  except IOError: tvaLib.printWarning('TTC_'+commands.file+'_'+predictionMethod.label_short+'.csv could not be opened for saving.', local['gen_warning'])
2345  except (NameError,ValueError,IndexError,AttributeError): pass
2346 
2347 
2350  if(commands.hli):
2351  from inspect import getargspec as inspect_getargspec
2352  if(commands.hli == 'all'):
2353  for _, dirs, files in os.walk('hli'):
2354  for file_ in files:
2355  if(os.path.splitext(file_)[0] != '__init__'):
2356  try:
2357  tvaHLI = import_module('hli.'+os.path.splitext(file_)[0])
2358  kwargs = {}
2359  hli_args = inspect_getargspec(tvaHLI.main)
2360  if('objects' in hli_args[0]): kwargs['objects'] = objects
2361  try:
2362  if('userPairs' in hli_args[0]): kwargs['userPairs'] = userPairs
2363  except UnboundLocalError: raise Exception, [2660, 'HLI module '+tvaHLI.module_name_long+' requires interactions to be loaded. Run with the argument -i and a corresponding list of prediction method IDs.']
2364  tvaHLI.main(commands=commands, config=config, objects=objects, sites=sites, site_analyses=site_analyses)
2365  except ImportError: tvaLib.printWarning('There were issues trying to import HLI module "'+file_+'".', local['gen_warning'])
2366  else:
2367  try:
2368  tvaHLI = import_module('hli.'+commands.hli)
2369  kwargs = {}
2370  hli_args = inspect_getargspec(tvaHLI.main)
2371  if('objects' in hli_args[0]): kwargs['objects'] = objects
2372  try:
2373  if('userPairs' in hli_args[0]): kwargs['userPairs'] = userPairs
2374  except UnboundLocalError: raise Exception, [2660, 'HLI module '+tvaHLI.module_name_long+' requires interactions to be loaded. Run with the argument -i and a corresponding list of prediction method IDs.']
2375  tvaHLI.main(commands=commands, config=config, sites=sites, site_analyses=site_analyses, **kwargs)
2376  except ImportError: tvaLib.printWarning('There were issues trying to import HLI module "'+commands.hli+'".', local['gen_warning'])
2377 
2378  if(config.MOT_save_all==False): sys.exit()
2379 
2380 
2384  except KeyboardInterrupt:
2385  if(commands.verbose): print(Back.GREEN+'==User exited== [0000]'+Back.RESET+'\nRuntime: {0}'.format(round(time.clock())))
2386  error=0
2387 
2388  except SystemExit, e:
2389  try:
2390  if(commands.verbose): print(Back.GREEN+'==Execution finished== [0001]'+Back.RESET+'\nRuntime: {0}'.format(round(time.clock())))
2391  except UnboundLocalError: pass
2392  error=0
2393 
2394  except Exception, e:
2395  if('commands' in dir()): verbose = commands.verbose
2396  else: verbose = 1
2397  if('config' in dir()): debug = config.debug
2398  else: debug = True
2399  from include.runtime import debug as tvaRuntime_debug
2400  error = tvaRuntime_debug(e, time, logging=logging, verbose=verbose, force=debug)
2401 
2402 
2403  finally:
2404  if('commands' in dir() and commands.logging):
2405  from include.runtime import end_logging as tvaRuntime_end_logging
2406  tvaRuntime_end_logging(oldstdout, site_analyses[saIx], os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])),'run_'+process_file_id+'.log'))
2407 
2408 
2409  if(error): os._exit(error)
2410 
2411 
2412 
2415 if __name__ == "__main__":
2416  #import cProfile, os
2417  #cProfile.run('main()', os.path.join(os.getcwd(),'main.profile'))
2418  main()
Definition: int.py:1
Definition: stat.py:1
Definition: comp.py:1
Definition: draw.py:1
def join(obj1, obj2, postSmoothing=True)
Definition: tools_obj.py:816
def main()
main()
Definition: main.py:30
def loadObjects(sequencepath, max_obj=None, max_obj_features=999, suppress_features=False, legacy=False, legacy_features_path='')
The following functions are used for manipulating object data from Traffic-Intelligence.
Definition: tools_obj.py:521
def getHomography(homoSrcFrame='', orthoSrcPath='', savePath='homography.txt', tsaiCameraSrcPath='', nPoints=4, unitsPerPixel=0.1, videoPts=None, worldPts=None, pointDrawSize=3, pointTargetDrawSize=7, pointTargetDrawThickness=2, maxDisplayResolutionX=900, fig_name='', verbose=0)
Definition: tools_obj.py:403
Definition: vis.py:1
Definition: main.py:1
Definition: filt.py:1