tvaLib
runtime.py
Go to the documentation of this file.
1 #!/usr/bin/env python
2 # tvaLib Copyright (c) 2012-2016 Paul G. St-Aubin
3 # Ecole Polytechnique de Montreal, McGill University
4 # Python 2.7; (dt) Spyder Windows 10 64-bit; ipython Ubuntu 15.04 64-bit
5 
9 import sys
10 import os
11 import cPickle as pickle
12 from importlib import import_module
13 from shutil import copy as shutil_copy
14 
15 try: from colorama import Fore, Back, Style
16 except ImportError:
17  import include.config as tvaConfig
18  Fore = tvaConfig.Fore()
19  Back = tvaConfig.Back()
20  Style = tvaConfig.Style()
21 
22 
23 
26 def interactiveSiteSelection(commands, config, sites, local, shortcutCamera=False, shortcutSequences=False, showTrackingCompletion=False):
27  import lib.tools as tvaLib
28  tvaLib.printTimeStamp(Back.BLUE+'Site listing:'+Back.RESET)
29  try: max_col_size1 = config.interactive_max_col_size1
30  except: max_col_size1 = 20
31  try: max_col_size2 = config.interactive_max_col_size2
32  except: max_col_size2 = 40
33  try: config_dir = config.dir
34  except: config_dir = ''
35 
36  if(not commands.site):
37  output = []
38  i = 1
39  for site in sites:
40  appendix = []
41  if(showTrackingCompletion):
42  tracking_completion = tvaLib.flatten_list([[1 if os.path.exists(seq.getFullDataFilename()) else 0 for seq in cam] for cam in site])
43  if(len(tracking_completion)): appendix.append(Style.BRIGHT+Fore.BLUE+'trckg:'+str(int(sum(tracking_completion)/float(len(tracking_completion))*100)).rjust(3)+'%'+Fore.RESET+Style.RESET_ALL)
44  else: appendix.append(Style.BRIGHT+Fore.BLUE+'trckg: - '+Fore.RESET+Style.RESET_ALL)
45  if(None not in [cam.getHomography() for cam in site]): appendix.append(Style.DIM+Fore.MAGENTA+'homo'+Fore.RESET+Style.RESET_ALL)
46  if(None not in [cam.camHeight for cam in site] and [] not in [cam.camOrigin.data for cam in site]): appendix.append(Style.BRIGHT+Fore.GREEN+'mhc'+Fore.RESET+Style.RESET_ALL)
47  if(site.alignments.data): appendix.append(Style.BRIGHT+Fore.RED+'align'+Fore.RESET+Style.RESET_ALL)
48  if([] not in [cam.mask.data for cam in site]): appendix.append(Style.BRIGHT+Fore.BLUE+'mask'+Fore.RESET+Style.RESET_ALL)
49 
50  camera_list = []
51  for camera in site:
52  camera_list.append(camera.name)
53  if(appendix): appendix = '['+', '.join(appendix)+']'
54  else: appendix = ''
55  if(camera_list): output.append([Back.BLUE+str(i)+Style.RESET_ALL+':', site.name, '(Cams: '+', '.join(camera_list)+')', appendix])
56  else: output.append([Back.BLUE+str(i)+Style.RESET_ALL+':', site.name, '', appendix])
57  i += 1
58  col_width0 = max(len(row[0]) for row in output) + 1
59  col_width1 = min(max(len(row[1]) for row in output), max_col_size1) + 2
60  col_width2 = min(max(len(row[2]) for row in output), max_col_size2) + 2
61  for row in output:
62  if(len(row[1]) > max_col_size1): row[1] = row[1][:(max_col_size1-3)]+'...'
63  if(len(row[2]) > max_col_size2): row[2] = row[2][:(max_col_size2-3)]+'...'
64  print(' '+row[0].ljust(col_width0)+Fore.YELLOW+row[1].ljust(col_width1)+Style.RESET_ALL+row[2].ljust(col_width2)+row[3])
65  try: commands.site = int(raw_input(Back.BLUE+'Interactive site selection mode: choose a site (defaults to all):'+Back.RESET))
66  except: commands.site = None
67 
68  if(commands.site and not commands.camera and not shortcutCamera):
69  siteIx = sites.interpret(commands.site)[0]
70  print('Alignments:')
71  for align in sites[siteIx].alignments:
72  print(' '+Fore.GREEN+align.name+Style.RESET_ALL)
73  print('Cameras:')
74  output = []
75  for camIx in range(len(sites[siteIx])):
76  appendix = [sites[siteIx][camIx].camera.name]
77  # Detect meta-data completion
78  if(sites[siteIx][camIx].getHomography()): appendix.append(Style.DIM+Fore.MAGENTA+'homo'+Fore.RESET+Style.RESET_ALL)
79  if(sites[siteIx][camIx].camHeight and sites[siteIx][camIx].camOrigin.data): appendix.append(Style.BRIGHT+Fore.GREEN+'mhc'+Fore.RESET+Style.RESET_ALL)
80  if(sites[siteIx][camIx].mask): appendix.append(Style.BRIGHT+Fore.BLUE+'mask'+Fore.RESET+Style.RESET_ALL)
81  output.append([Back.BLUE+str(camIx+1)+Style.RESET_ALL+':',Fore.YELLOW+sites[siteIx][camIx].name+Style.RESET_ALL,'['+', '.join(appendix)+']'])
82  col_width0 = max(len(row[0]) for row in output) + 1
83  col_width1 = min(max(len(row[1]) for row in output), max_col_size1) + 2
84  for row in output:
85  if(len(row[1]) > max_col_size1): row[1] = row[1][:(max_col_size1-3)]+'...'
86  print(' '+row[0].ljust(col_width0)+Fore.YELLOW+row[1].ljust(col_width1)+Style.RESET_ALL+row[2])
87  try: commands.camera = int(raw_input(Back.BLUE+'Interactive camera selection mode: choose a camera (defaults to all):'+Back.RESET))
88  except: commands.camera = None
89 
90  if(commands.site and commands.camera and not commands.file and not shortcutSequences):
91  siteIx = sites.interpret(commands.site)[0]
92  camIx = sites[siteIx].interpret(commands.camera)[0]
93  fileIxs = sites[siteIx][camIx].interpret()
94  print('Sequences:')
95  for fileIx in fileIxs:
96  appendix = []
97  if(not os.path.exists(os.path.join(config.dir, sites[siteIx].name, sites[siteIx][camIx].name, sites[siteIx][camIx][fileIx].name+'.sqlite'))): appendix.append(Back.RED+'**NO TRACKS**'+Back.RESET)
98  if(not os.path.exists(sites[siteIx][camIx][fileIx].getFullVideoFilename())): appendix.append(Back.RED+'**NO VIDEO**'+Back.RESET)
99  if(os.path.exists(sites[siteIx][camIx][fileIx].getFullGroundTruthFilename())): appendix.append(Fore.GREEN+'gt'+Fore.RESET)
100  if(os.path.exists(sites[siteIx][camIx][fileIx].getFullClassifiedFilename())): appendix.append(Fore.CYAN+'cl'+Fore.RESET)
101 
102  if(appendix): print(' '+Back.BLUE+str(fileIx+1)+Style.RESET_ALL+': '+Fore.YELLOW+sites[siteIx][camIx][fileIx].name+Style.RESET_ALL+' ['+', '.join(appendix)+']')
103  else: print(' '+Back.BLUE+str(fileIx+1)+Style.RESET_ALL+': '+Fore.YELLOW+sites[siteIx][camIx][fileIx].name+Style.RESET_ALL)
104  try: commands.file = int(raw_input(Back.BLUE+'Interactive sequence selection mode: choose a sequence (defaults to all):'+Back.RESET))
105  except: commands.file = None
106 
107  commands.interactive = False
108  return commands
109 
110 def interactiveSiteAnalysisSelection(commands, config, site_analyses, sites, local, shortcutCameraSequences=False):
111  import lib.tools as tvaLib
112  tvaLib.printTimeStamp(Back.BLUE+'Site analysis listing:'+Back.RESET)
113 
114  tvaHLI = []
115  if(commands.hli == 'all'):
116  import importlib
117  for root, dirs, files in os.walk('hli'):
118  for file in files:
119  if(os.path.splitext(file)[0] != '__init__' and os.path.splitext(file)[1] == '.py'):
120  try: tvaHLI.append(importlib.import_module('hli.'+os.path.splitext(file)[0]))
121  except ImportError: tvaLib.printWarning('There were issues trying to import HLI module "'+file+'".', local['gen_warning'])
122  elif(commands.hli):
123  import importlib
124  try: tvaHLI.append(importlib.import_module('hli.'+commands.hli))
125  except ImportError: tvaLib.printWarning('There were issues trying to import HLI module "'+commands.hli+'".', local['gen_warning'])
126 
127 
128  if(not commands.s_analysis):
129  output = []
130  #Prepare module listings in advance (more efficienct than reloading full list of all site analysis on each site analysis)
131  moduleListings = [[] for saIx in range(len(site_analyses))]
132  for module in tvaHLI:
133  try:
134  moduleListing = module.listingPlugin(site_analyses, config)
135  if(moduleListing):
136  for saIx in range(len(site_analyses)):
137  if(moduleListing[saIx]): moduleListings[saIx].append(Fore.GREEN+moduleListing[saIx]+Fore.RESET)
138  except NameError: pass
139 
140  i = 1
141  for saIx in range(len(site_analyses)):
142  appendix = []
143  if(site_analyses[saIx].site.alignments.data): appendix.append(Style.BRIGHT+Fore.RED+'align'+Fore.RESET+Style.RESET_ALL)
144  if(site_analyses[saIx].zone): appendix.append(Fore.RED+'zone'+Fore.RESET)
145  if(site_analyses[saIx].xy_bounds): appendix.append(Fore.CYAN+'bounds'+Fore.RESET)
146  mpv_string = site_analyses[saIx].findLowestMPversion()
147  if(mpv_string): appendix.append(Fore.MAGENTA+'DMP '+mpv_string+Fore.RESET)
148  appendix += moduleListings[saIx]
149  camera_list = []
150  for camera in site_analyses[saIx].cameras:
151  camera_list.append(camera.name)
152  if(appendix): appendix = '['+', '.join(appendix)+']'
153  else: appendix = ''
154  if(camera_list): output.append([Back.BLUE+str(site_analyses[saIx].idx)+Style.RESET_ALL+':', site_analyses[saIx].name, '(Cams: '+', '.join(camera_list)+')', appendix])
155  else: output.append([Back.BLUE+str(site_analyses[saIx].idx)+Style.RESET_ALL+':', site_analyses[saIx].name, '', appendix])
156  i += 1
157 
158  try:
159  col_width0 = max(len(row[0]) for row in output) + 1
160  col_width1 = min(max(len(row[1]) for row in output), config.interactive_max_col_size2) + 2
161  col_width2 = min(max(len(row[1]) for row in output), config.interactive_max_col_size1) + 2
162  for row in output:
163  if(len(row[1]) > config.interactive_max_col_size1): row[1] = row[1][:(config.interactive_max_col_size2-3)]+'...'
164  if(len(row[2]) > config.interactive_max_col_size1): row[2] = row[2][:(config.interactive_max_col_size1-3)]+'...'
165  print(' '+row[0].ljust(col_width0)+Fore.YELLOW+row[1].ljust(col_width1)+Style.RESET_ALL+row[2].ljust(col_width2)+row[3])
166  try: commands.s_analysis = raw_input(Back.BLUE+'Interactive site analysis selection mode: choose a site-analysis (defaults to all):'+Back.RESET)
167  except: commands.s_analysis = None
168  except: pass
169  tvaLib.printWarning('TimeIntervals not yet supported (the list defaults to ALL camera-sequences available).', local['gen_warning'])
170  if(commands.s_analysis and not shortcutCameraSequences):
171  s_aIx = site_analyses.interpret(commands.s_analysis)[0]
172  output = []
173  i = 1
174  sequenceMap = {}
175  if(not site_analyses[s_aIx].getCamSeqs()): raise Exception, [3101, 'No sequences have been defined for this site/camera.']
176  for cam_seq in site_analyses[s_aIx].getCamSeqs():
177  siteIx = sites.index([site for site in sites if site.idx == cam_seq.cameraView.site.idx][0])
178  camIx = sites[siteIx].index([cam for cam in sites[siteIx] if cam.idx == cam_seq.cameraView.idx][0])
179  fileIx = sites[siteIx][camIx].index([seq for seq in sites[siteIx][camIx] if seq.idx == cam_seq.idx][0])
180  sequenceMap[i] = [siteIx+1,camIx+1,fileIx+1]
181 
182  appendix = []
183  if(not os.path.exists(os.path.join(config.dir, sites[siteIx].name, sites[siteIx][camIx].name, sites[siteIx][camIx][fileIx].name+'.sqlite'))): appendix.append(Back.RED+'**NO TRACKS**'+Back.RESET)
184  if(not os.path.exists(sites[siteIx][camIx][fileIx].getFullVideoFilename())): appendix.append(Back.RED+'**NO VIDEO**'+Back.RESET)
185  if(os.path.exists(sites[siteIx][camIx][fileIx].getFullGroundTruthFilename())): appendix.append(Fore.GREEN+'gt'+Fore.RESET)
186  if(os.path.exists(sites[siteIx][camIx][fileIx].getFullClassifiedFilename())): appendix.append(Fore.CYAN+'cl'+Fore.RESET)
187 
188  if(appendix): output.append([Back.BLUE+str(i)+Style.RESET_ALL+':', cam_seq.cameraView.name+'/'+cam_seq.name, ' ('+', '.join(appendix)+')'])
189  else: output.append([Back.BLUE+str(i)+Style.RESET_ALL+':', cam_seq.cameraView.name+'/'+cam_seq.name, ' '])
190 
191  i += 1
192 
193  col_width0 = max(len(row[0]) for row in output) + 1
194  for row in output:
195  print(' '+row[0].ljust(col_width0)+Fore.YELLOW+row[1]+Style.RESET_ALL+row[2])
196  try:
197  sequence = int(raw_input(Back.BLUE+'Interactive camera-sequence selection mode: choose a camera-sequence (defaults to all):'+Back.RESET))
198  commands.site = sequenceMap[sequence][0]
199  commands.camera = sequenceMap[sequence][1]
200  commands.file = sequenceMap[sequence][2]
201  except: pass
202  else:
203  commands.site = []
204  for site_a in site_analyses:
205  if(not hasattr(site_a, 'cameras')): continue
206  for siteIx in range(len(sites)):
207  if(site_a.site.idx == sites[siteIx].idx):
208  commands.site.append(siteIx+1)
209  break
210 
211  commands.interactive = False
212  return commands
213 
214 def interactiveAnalysisSelection(commands, config, analyses, local):
215  import lib.tools as tvaLib
216  tvaLib.printTimeStamp(Back.BLUE+'Analysis listing:'+Back.RESET)
217 
218 
219  if(not commands.s_analysis):
220  output = []
221  for analysis in analyses:
222  appendix = []
223  output.append([Back.BLUE+str(analysis.idx)+Style.RESET_ALL+':', analysis.name, '('+str(len(analysis.site_analyses) if len(analysis.site_analyses) != 0 else 'all')+' SAs)', appendix])
224 
225  try:
226  col_width0 = max(len(row[0]) for row in output) + 1
227  col_width1 = min(max(len(row[1]) for row in output), config.interactive_max_col_size2) + 2
228  for row in output:
229  if(len(row[1]) > config.interactive_max_col_size1): row[1] = row[1][:(config.interactive_max_col_size2-3)]+'...'
230  if(len(row[2]) > config.interactive_max_col_size1): row[2] = row[2][:(config.interactive_max_col_size1-3)]+'...'
231  print(' '+row[0].ljust(col_width0)+Fore.YELLOW+row[1].ljust(col_width1)+Style.RESET_ALL+row[2])
232  try: commands.analysis = int(raw_input(Back.BLUE+'Interactive analysis selection mode: choose an analysis (defaults to the first one):'+Back.RESET))
233  except: commands.analysis = None
234  except: pass
235 
236  commands.interactive = False
237  return commands
238 
239 def interactiveClusterSelection(commands, config, clusters, local):
240  import lib.tools as tvaLib
241  tvaLib.printTimeStamp(Back.BLUE+'Cluster listing:'+Back.RESET)
242 
243 
244  if(not commands.s_analysis):
245  output = []
246  for cluster in clusters:
247  appendix = []
248  output.append([Back.BLUE+str(cluster.idx)+Style.RESET_ALL+':', cluster.name, '('+str(len(cluster.analyses) if len(cluster.analyses) != 0 else 'all')+' analyses)', appendix])
249 
250  try:
251  col_width0 = max(len(row[0]) for row in output) + 1
252  col_width1 = min(max(len(row[1]) for row in output), config.interactive_max_col_size2) + 2
253  for row in output:
254  if(len(row[1]) > config.interactive_max_col_size1): row[1] = row[1][:(config.interactive_max_col_size2-3)]+'...'
255  if(len(row[2]) > config.interactive_max_col_size1): row[2] = row[2][:(config.interactive_max_col_size1-3)]+'...'
256  print(' '+row[0].ljust(col_width0)+Fore.YELLOW+row[1].ljust(col_width1)+Style.RESET_ALL+row[2])
257  try: commands.s_analysis = int(raw_input(Back.BLUE+'Interactive cluster selection mode: choose a cluster (defaults to the first one):'+Back.RESET))
258  except: commands.s_analysis = None
259  except: pass
260 
261  commands.interactive = False
262  return commands
263 
264 
267 def targetSequence(commands, sites, config, force=False):
268  ''' Prepare targeted sequence. '''
269  if(commands.site):
270  siteIx = sites.interpret(commands.site)[0]
271  if(commands.camera):
272  camIx = sites[siteIx].interpret(commands.camera, interpretStrAsLiteralIndex=True)[0]
273  if(commands.file): fileIx = sites[siteIx][camIx].interpret(commands.file, interpretStrAsLiteralIndex=True)[0]
274  else: fileIx = False
275  else: camIx = fileIx = False
276  else: siteIx = camIx = fileIx = False
277 
278  if(force and (siteIx==False or camIx==False or fileIx==False)): return False, False, False
279  return siteIx, camIx, fileIx
280 
281 def targetSiteIxs(commands, sites, site_analysis):
282  ''' Prepare list of targeted sites. '''
283  if(commands.s_analysis):
284  Ixs = []
285  for siteIx in range(len(sites)):
286  if(site_analysis.site.idx == sites[siteIx].idx): Ixs.append(siteIx)
287  return Ixs
288  else: return sites.interpret(commands.site)
289 
290 def targetCameraIxs(commands, site, site_analysis):
291  ''' Prepare list of targeted cameras. '''
292  if(commands.camera): return site.interpret(commands.camera, interpretStrAsLiteralIndex=True)
293  if(commands.s_analysis):
294  Ixs = []
295  for camIx in range(len(site)):
296  for camera in site_analysis.cameras:
297  if(camera.idx == site[camIx].idx): Ixs.append(camIx)
298  return Ixs
299  else: return site.interpret(commands.camera)
300 
301 def targetSequenceIxs(commands, camera, site_analysis):
302  ''' Prepare list of targeted cameras. '''
303 
304  if(commands.file): return camera.interpret(commands.file, interpretStrAsLiteralIndex=True)
305  if(commands.s_analysis):
306  Ixs = []
307  for sequenceIx in range(len(camera)):
308  for camera_ in site_analysis.cameras:
309  if(camera_.idx == camera.idx):
310  for sequence in camera_:
311  if(sequence.idx == camera[sequenceIx].idx): Ixs.append(sequenceIx)
312  return Ixs
313  else: return camera.interpret('')
314 
315 
319  from pip import get_installed_distributions
320  required_programs = [['numpy'], ['matplotlib'], ['scipy'], ['SQLAlchemy'], ['munkres']]
321  recommended_programs = [['opencv-python','cv2'],['scikit-learn'],['scikit-image'],['pandas'],['Pillow']]
322  #Other: colorama
323  required_programs_m = []
324  recommended_programs_m = []
325  packages = [package.project_name for package in get_installed_distributions(local_only=False)]
326  for program in required_programs:
327  if(not program[0] in packages):
328  if(len(program) >=2):
329  try: import_module(program[1])
330  except ImportError: required_programs_m.append(program)
331  else: required_programs_m.append(program)
332 
333  for program in recommended_programs:
334  if(not program[0] in packages):
335  if(len(program) >=2):
336  try: import_module(program[1])
337  except ImportError: recommended_programs_m.append(program)
338  else: recommended_programs_m.append(program)
339 
340 
341 
342 
343 
344  if(recommended_programs_m): print(Back.YELLOW+Fore.RED+Style.BRIGHT+'Warning:'+Style.RESET_ALL+'The following modules are recommended, but not installed: '+','.join(recommended_programs_m))
345  if(required_programs_m): raise Exception, [101, 'The following modules are required, but not installed: '+','.join(required_programs_m)]
346 
347 
348 
351 def debug(e, time, logging=None, verbose=0, force=False):
352  if(isinstance(e, Exception) and str(e) and isinstance(e[0], list)):
353  if(verbose and len(e[0]) >= 2):
354  if(e[0][0] > 99): print(Back.RED+'==Runtime error== ['+str(e[0][0]).zfill(4)+'] '+e[0][1]+Back.RESET+'\nRuntime: {0}'.format(round(time.clock())))
355  else: print(Back.GREEN+'==Execution finished== ['+str(e[0][0]).zfill(4)+'] '+e[0][1]+Back.RESET+'\nRuntime: {0}'.format(round(time.clock())))
356  return e[0][0]
357  elif(force):
358  import sys
359  from pdb import post_mortem as pdb_post_mortem
360  from traceback import print_exc as traceback_print_exc
361  _, _, tb = sys.exc_info()
362  traceback_print_exc()
363  pdb_post_mortem(tb)
364  if(logging): logging.exception(e)
365 
366 
367 def end_logging(oldstdout, site_analysis, current_path):
368  newstdout = sys.stdout
369  sys.stdout = oldstdout
370  newstdout.close()
371  try: os.remove(os.path.join(site_analysis.getFullResultsFolder() ,'run.log'))
372  except: pass
373  shutil_copy(current_path, os.path.join(site_analysis.getFullResultsFolder() ,'run.log'))
374  os.remove(current_path)
def join(obj1, obj2, postSmoothing=True)
Definition: tools_obj.py:816
def targetSiteIxs(commands, sites, site_analysis)
Definition: runtime.py:281
def targetSequence(commands, sites, config, force=False)
Selection of site/cam/sequence/etc.
Definition: runtime.py:267
def interactiveSiteAnalysisSelection(commands, config, site_analyses, sites, local, shortcutCameraSequences=False)
Definition: runtime.py:110
def interactiveAnalysisSelection(commands, config, analyses, local)
Definition: runtime.py:214
def targetSequenceIxs(commands, camera, site_analysis)
Definition: runtime.py:301
def end_logging(oldstdout, site_analysis, current_path)
Definition: runtime.py:367
def checkDependancies()
Start-of-program operations.
Definition: runtime.py:318
def targetCameraIxs(commands, site, site_analysis)
Definition: runtime.py:290
def interactiveSiteSelection(commands, config, sites, local, shortcutCamera=False, shortcutSequences=False, showTrackingCompletion=False)
CLI-stuff.
Definition: runtime.py:26
def getHomography(homoSrcFrame='', orthoSrcPath='', savePath='homography.txt', tsaiCameraSrcPath='', nPoints=4, unitsPerPixel=0.1, videoPts=None, worldPts=None, pointDrawSize=3, pointTargetDrawSize=7, pointTargetDrawThickness=2, maxDisplayResolutionX=900, fig_name='', verbose=0)
Definition: tools_obj.py:403
def interactiveClusterSelection(commands, config, clusters, local)
Definition: runtime.py:239
def debug(e, time, logging=None, verbose=0, force=False)
End-of-program operations.
Definition: runtime.py:351