#!/usr/bin/env python """ This script is used for launching the triage tool. It takes as input a meta_id of the log, creates a roscore, loads required rosparams, and launches the triage tool. Assumes that the logs and tests have already been downloaded. """ import sys from infra.utils.load_static_tls_deps import load_static_tls_deps python_utils_dep = "zlearner/zcache/_python_utils_py2.so" if sys.version_info.major == 2 else "zlearner/zcache/_python_utils.so" load_static_tls_deps(python_utils_dep) # Moved on top to prevent: # ImportError: dlopen: cannot load any more object with static TLS from zlearner.zcache import python_utils import hashlib import json import os import shutil import socket import subprocess import tempfile import urllib import webbrowser import data.triage.util as util import infra.data_catalog.common.id_util as id_util from argus.utils.links import render_link from base.file.utils.file_utils import mkdir_p from base.git.git_build_info import GIT_BUILD_INFO from data.chum import chumpy from data.triage.pcp_triage_tests.pcp_triage_to_sdl import translate_tests from lidar.metrics.utils.variant_utils import ( delete_variant_if_exists, variants_exist, ) from vision.common.scripts.run_offline_vision import run_vision_on_log ZCACHE_VISION_BINARY = "VISION" class TriageToolRunner: def __init__(self, bag_id, **kwargs): """ The class that launches the triage tool. Simply call TriageToolRunner(bag_id, **kwargs).main() to run. All parameters except bag_id are keyword arguments :param bag_id: the BagID (see util.py) to run the triage tool on :param start_time: float time in seconds to start playback at. If None or not explicitly set, starts playback from the beginning of the BagID's time range. :param start_offset: float offset in seconds from the incident time indicating when to start playback. If none or not explicitly set (or if the BagID doesn't have an incident), starts playback from the beginning of the BagID's time range. :param edit: bool indicating whether to allow editing and creating tests (defaults to True) :param run_perception: bool indicating whether to re-run perception and visualize the results of that rather than the original bags (defaults to False) :param cached_perception: bool indicating whether to use the previously regenerated perception bags rather than re-running perception. (defaults to False) :param zcache_vision_dataset: string indicating the name of the ZCache vision dataset if to use the ZCache regenerated vision rather than re-running vision. (defaults to None) ZCache vision is used only when all of zcache_vision_dataset, zcache_vision_build and zcache_vision_version are specified. :param zcache_vision_build: string indicating the build used to generate the ZCache vision dataset if to use the ZCache regenerated vision rather than re-running vision. (defaults to None) ZCache vision is used only when all of zcache_vision_dataset, zcache_vision_build and zcache_vision_version are specified. :param zcache_vision_version: int indicating the version of the ZCache vision dataset if to use the ZCache regenerated vision rather than re-running vision. (defaults to None) ZCache vision is used only when all of zcache_vision_dataset, zcache_vision_build and zcache_vision_version are specified. :param cached_vision: bool indicating whether to use the previously regenerated vision rather than re-running vision. (defaults to False) :param run_vision: bool indicating to use an updated vision bag (either re-running vision from scratch or using a cached vision bag locally or on flashblade). If true, will also trigger a re-run of perception. (defaults to False) :param zcache_vision: bool indicating to use an updated vision bag (either re-running vision from scratch or using a cached vision bag locally or on flashblade). If true, will also trigger a re-run of perception. (defaults to False) :param alsologtostderr bool indicating whether to show info logs. :param force_ll3d: bool indicating to re-run ll3d components instead of using logged results. If true, will also trigger a re-run of perception. (defaults to False) :param validator_errors: bool indicating whether to log error messages from the PO2 validator to stdout (defaults to False) :param command_path: string indicating the path of the triage tool binary to run (defaults to data/triage/triage_tool) :param ground_truth_input_source: string containing path to a ground truth annotated spin LMDB or a chum URI containing scale data (optional). If provided, ground truth annotations will be displayed, only within the timestamp range of the ground truth input source. :param ground_truth_association_frame_lmdb_path: string containing path to an LMDB file with association data between ground truth annotations on one hand, and observations and PCP tracks on the other (optional). If provided, fusions to ground truth will be displayed. displayed, only within the timestamp range of the lmdb file. :param perception_ci_lmdb_basename: string name of an LMDB file that is part of a perception_ci suite that we want to triage. Only pass this in as an argument if you want to triage something from a perception_ci suite. :param tracking_metrics_lmdb_path: string containing the path to a tracking metrics lmdb path (optional). If provided, all the objects in Matches contained in TrackingMetricsFrames will be displayed. Will only show labeled objects, so not all spins will have them. :param control_tracking_metrics_lmdb_path: string containing the path to a control tracking metrics lmdb path (optional). If provided, the tracking metrics will be diff'ed with this control run. :param segmentation_metrics_pb_path: string containing the path to a segmentation metrics pb. If provided, all the objects evaluated in segmentation metrics will be displayed. :param control_segmentation_metrics_pb_path: string containing the path to a control segmentation metrics pb If provided, segmentation metrics will be diff'ed with this run. :param window_name: Optional string for the name of the actual GUI that is displayed at the top of the GUI. :param custom_root: Optional string for custom chum root where all output from running different offline tools will be placed. :param custom_variant_name: Optional string for an alternate name for the variant used to store regenerated perception and vision. Useful if you need to store perception outputs for two different runs of the same log at the same time. :param run_tracks_to_annotated_objects: Optional bool defaulting to False, indicating whether to run the tracks to annotated objects plugin for annotating lidar blobs with the corresponding track classification. :param show_notes: bool indicating whether to show the notes pane. :param variants: list indicating variants to use by default when --run_perception is not specified. Defaults to an empty list. :param remote_roots: list indicating remote roots to use to search for Chum files (e.g. perception-ci variants). Defaults to an empty list. :param triage_filter_name: name of the filter which to use for triaging perception-ci regressions. For example, "dynamic_only,class:pedestrian" :param triage_metric_name: name of the metric which to use for triaging perception-ci regressions. :param load_prediction: bool indicating whether to load prediction messages. Defaults to true. Should only be false if you're running into compatibility issues with new prediction messages/fields not present on your branch. :param use_ground_heights: bool indicating whether to use Lidar estimated ground heights for 3D visualization of layers including ZRN, planner-interactions and OcclusionGrid. Defaults to false. Requires CUDA. :param test_pbtxt_path: string indicating where the PCP log test pbtxt to visualize is saved. If not specified, infers it from the BagID. :param non_debug_mode: bool indicating whether to turn off debug mode when rerunning offline perception. :param dtn_debug_mode: bool indicating whether to perform stricter checkings for debugging DTN. :param log_track_attr_model_features: bool indicating whether to log features for debugging the track attributes model. :param no_yaw_computer: bool indicating whether to use the yaw computer. :param log_yaw_computer_info: bool indicating whether to log yaw computer info. :param log_extent_history: bool indicating whether to log track extent history. :param override_params_path: string the path to a pbtxt that will used to override specific tracker params. :param argus: bool indicating if we want argus triage. :param argus_port: integer denoting port at which argus server is running. """ self.bag_id = bag_id self.edit = kwargs.get('edit', True) # Set the custom start time according to the start_time or start_offset # kwargs, if applicable. self.start_time = kwargs.get('start_time', None) start_offset = kwargs.get('start_offset', None) if (self.start_time is None and start_offset is not None and self.bag_id.incident is not None): self.start_time = self.bag_id.incident + start_offset zcache_vision_dataset = kwargs.get('zcache_vision_dataset', None) zcache_vision_build = kwargs.get('zcache_vision_build', None) zcache_vision_version = kwargs.get('zcache_vision_version', None) if zcache_vision_dataset is not None and \ zcache_vision_build is not None and \ zcache_vision_version is not None: time_str = util.make_time_str(self.bag_id) # The timestamps in the usual bag IDs we see are rounded to 0.01s, # while the time ranges in ZCache jobs are in nanoseconds. To avoid # missing the job that actually matches the criteria due to # precision issues, here we slightly tweak the start time and end # time here by adding or subtracting 0.01s. if '-' in time_str: start_time_str, end_time_str = time_str.split('-') start_ts = long((float(start_time_str) + 0.01) * 1e9) end_ts = long((float(end_time_str) - 0.01) * 1e9) elif '+' in time_str: start_time_str, offset_str = time_str.split('+') start_ts = long((float(start_time_str) + 0.01) * 1e9) end_ts = start_ts + long((float(offset_str) - 0.01) * 1e9) else: start_ts = long(float(time_str) * 1e9) end_ts = start_ts chum_uris = python_utils.get_zcache_chum_uris( zcache_vision_dataset, 0, zcache_vision_build, ZCACHE_VISION_BINARY, self.bag_id.meta_id, start_ts, end_ts) assert len(chum_uris) > 0, 'No ZCache vision for {}'.format(bag_id) self.zcache_vision_uri = chum_uris[0] else: self.zcache_vision_uri = None self.cached_vision = kwargs.get('cached_vision', False) # Specifying cached_vision forces run_vision to be true. self.run_vision = kwargs.get('run_vision', False) or self.cached_vision self.force_ll3d = kwargs.get('force_ll3d', False) self.use_ground_heights = kwargs.get('use_ground_heights', False) # Set the local root for the Chum variants for regenerated PCP and # vision. self.custom_root = kwargs.get('custom_root', None) self.local_root = (util.TRIAGE_CHUM_ROOT if self.custom_root is None else self.custom_root) mkdir_p(self.local_root) self.remote_roots = kwargs.get('remote_roots', []) # Get the list of variants to use by default and make the initial (i.e. # pre-pcp rerun) Chum URI. self.variants = kwargs.get('variants', []) self.chum_uri = self._make_chum_uri() # Whether to show info logs. self.alsologtostderr = kwargs.get('alsologtostderr', False) self.cached_perception = kwargs.get('cached_perception', False) # Specifying run_vision or run_ll3d or cached_perception forces # run_perception to be true. self.run_perception = kwargs.get('run_perception', False) or \ self.run_vision or self.force_ll3d or self.cached_perception self.validator_errors = kwargs.get('validator_errors', False) # Specify which command to run. self.command_path = kwargs.get( 'command_path', 'data/triage/triage_tool') # If we want to target an lmdb in particular self.perception_ci_lmdb_basename = kwargs.get( 'perception_ci_lmdb_basename', None) # When this is true then camera triage tool app is launched. self.run_camera_triage = kwargs.get('camera_triage', False) self.ground_truth_input_source = kwargs.get( 'ground_truth_input_source', None) self.ground_truth_association_frame_lmdb_path = kwargs.get( 'ground_truth_association_frame_lmdb_path', None) self.tracking_metrics_lmdb_path = kwargs.get( 'tracking_metrics_lmdb_path', None) self.control_tracking_metrics_lmdb_path = kwargs.get( 'control_tracking_metrics_lmdb_path', None) self.occlusion_ground_truth_lmdb_path = kwargs.get( 'occlusion_ground_truth_lmdb_path', None) self.segmentation_metrics_pb_path = kwargs.get( 'segmentation_metrics_pb_path', None) self.control_segmentation_metrics_pb_path = kwargs.get( 'control_segmentation_metrics_pb_path', None) self.window_name = kwargs.get('window_name', None) if self.window_name is None: self.window_name = self._get_autogenerated_window_name() self.custom_variant_name = kwargs.get('custom_variant_name', None) self.run_tracks_to_annotated_objects = kwargs.get( 'run_tracks_to_annotated_objects', False) self.show_notes = kwargs.get('show_notes', False) self.triage_metric_name = kwargs.get('triage_metric_name', None) self.triage_filter_name = kwargs.get('triage_filter_name', None) self.load_prediction = kwargs.get('load_prediction', True) self.test_pbtxt_path = kwargs.get('test_pbtxt_path', None) if self.test_pbtxt_path is None: self.test_pbtxt_path = util.get_test_pbtxt_path(self.bag_id) self.output_directory = os.path.join( self.local_root, '{}-{}'.format( self.bag_id.meta_id, util.make_time_str(self.bag_id))) mkdir_p(self.output_directory) self.override_params_path = kwargs.get('override_params_path', "") self.non_debug_mode = kwargs.get('non_debug_mode', False) self.dtn_debug_mode = kwargs.get('dtn_debug_mode', False) self.log_track_attr_model_features = kwargs.get( 'log_track_attr_model_features', False) self.no_yaw_computer = kwargs.get('no_yaw_computer', False) self.log_yaw_computer_info = kwargs.get('log_yaw_computer_info', False) self.log_extent_history = kwargs.get('log_extent_history', False) self.argus = kwargs.get('argus', False) self.argus_port = kwargs.get('argus_port', 8080) def main(self): """ The high level plan: 1. Parse command line arguments. 2. Download required data from s3 into a local directory (default /tmp). 3. Create a roscore with the necessary parameters loaded. 4. Launch the triage tool. """ if self.run_camera_triage: self.run_camera_triage_server() if self.need_remapped_hero_state(): self.chum_uri = self.get_remapped_hero_state_chum_uri() if self.run_perception: if self.zcache_vision_uri is not None: # When specifying to use ZCache vision, do not run vision or # use cached vision. self.chum_uri = self.zcache_vision_uri elif self.run_vision: self.chum_uri = self.get_vision_chum_uri() self.chum_uri = self.get_perception_chum_uri() if self.argus: self.launch_argus() else: self.launch_triage_tool() return 0 def _make_chum_uri(self): """ Make a Chum URI for the current state of the TriageToolRunner, incorporating all the currently used variants. """ all_roots = [self.local_root] + self.remote_roots chum_uri = '{}?i={}'.format( util.bag_id_to_chum_uri(self.bag_id), ','.join(all_roots)) if self.variants: chum_uri += '&v={}'.format(','.join(self.variants)) return chum_uri def _make_variant_name(self, variant_type): """ Make a variant name for regenerated perception/vision. Outputted variant will be of the form "--[- is either "", "-", or "all" and is either "pcp" or "vision". :param variant_type: string demarcator for this variant (usually either "pcp" or "vision"). :return: string variant. """ variant = 'triage/{meta_id}-{time_str}-{variant_type}'.format( meta_id=self.bag_id.meta_id, time_str=util.make_time_str(self.bag_id), variant_type=variant_type) if self.custom_variant_name is not None: variant = '{}-{}'.format(variant, self.custom_variant_name) return variant def _get_autogenerated_window_name(self): """ Makes an autogenerated window name by querying data catalog for the timestamp and run id and the current branch name. :return: string containing autogenerated window name. """ # Get issue string issue_string = util.bag_id_to_input_string(self.bag_id) # Get branch name or else fall back to git sha. if GIT_BUILD_INFO.git_branch != 'HEAD': branch_string = GIT_BUILD_INFO.git_branch else: branch_string = GIT_BUILD_INFO.git_sha return "{} [{}]".format(issue_string, branch_string) def get_vision_chum_uri(self): """ Run the offline vision pipeline on this incident and return a Chum URI containing the Chum variant offline vision was saved to. If a cached version of the regenerated visiona already exists and self.cached_vision is true, will not bother running vision. :return: string Chum URI containing the regenerated vision. """ variant = self._make_variant_name('vision') self.variants.insert(0, variant) new_chum_uri = self._make_chum_uri() if self.cached_vision and variants_exist(self.local_root, [variant]): print 'Using cached offline vision' else: # If we're forcing vision, delete the variant if it already exists to # prevent us from using the old variant or overwriting it. delete_variant_if_exists(self.local_root, variant) # Compute a time interval from an incident timestamp if a time # interval doesn't already exist. start_ts, end_ts = util.get_start_and_end_times(self.bag_id) _, vehicle = id_util.human_readable_id_to_dt_vehicle( self.bag_id.meta_id) # Generate vision chum. PLAY_VISION_IN_COZMOS = False run_vision_on_log( run_id=self.bag_id.meta_id, output_chum_root=self.local_root, output_chum_variant=variant, play_in_real_time=False, play_in_cozmos=PLAY_VISION_IN_COZMOS, start_time=start_ts, end_time=end_ts) return new_chum_uri def run_camera_triage_server(self): url = "" if self.bag_id.incident is not None: url = "http://camera-triage.zooxlabs.com:9000/?meta_id={0}@{1}"\ .format(self.bag_id.meta_id, self.bag_id.incident) elif (self.bag_id.start_time is not None and self.bag_id.end_time is not None): url = "http://camera-triage.zooxlabs.com:9000/?meta_id={0}@{1}-{2}"\ .format(self.bag_id.meta_id, self.bag_id.start_time, self.bag_id.end_time) else: url = "http://camera-triage.zooxlabs.com:9000/?meta_id={0}"\ .format(self.bag_id.meta_id) webbrowser.open_new_tab(url) print "Running camera triage server" def need_remapped_hero_state(self): """ Checks if the current Chum URI contains the recorded hero state topic. If so, this URI is the result of simulation, we will need to remap hero state topics so that perception runs on the recorded hero states, instead of the regenerated ones. :return: bool whether remapping hero state topics is needed or not. """ recorded_hero_state_topic = "/recorded/planner/hero_state" store, range = chumpy.parseChumUri(self.chum_uri) return recorded_hero_state_topic in range.topics def get_remapped_hero_state_chum_uri(self): """ Run remap_hero_state_topics to remap the hero state topics generated by simulation. This makes sure the recorded hero states get remapped into the original topic so that perception can use it as an input. :return: string Chum URI containing the remapped hero state topics. """ # Append to the variant type with a hash of the Chum URI so that # different simulation runs on the same log will have different variant # names. This allows the triage tool to be run on multiple runs at the # same time, e.g. for comparison. variant_type = 'remapped_hero_state_{}'.format( hashlib.sha256(self.chum_uri).hexdigest()) variant = self._make_variant_name(variant_type) self.variants.insert(0, variant) new_chum_uri = self._make_chum_uri() # If we remapped the topics for this run before, no need to do it # again. if variants_exist(self.local_root, [variant]): print 'Using remapped hero states from cache' return new_chum_uri cmd_args = ['data/triage/utils/remap_hero_state_topics', '--chum_uri_in', self.chum_uri, '--chum_root_out', self.local_root, '--chum_variant_out', variant] print 'Remapping hero state topics' print cmd_args subprocess.check_call(cmd_args) return new_chum_uri def get_perception_chum_uri(self): """ Run offline_perception_main on the logs for this incident, store the resulting obstacles in a Chum variant, and return a Chum URI that includes that regenerated perception. :return: string Chum URI containing the regenerated perception. """ variant = self._make_variant_name('pcp') self.variants.insert(0, variant) new_chum_uri = self._make_chum_uri() # If cached_perception is on and the output variant already exists, # don't bother re-running offline_perception_main. if self.cached_perception and variants_exist(self.local_root, [variant]): print 'Using cached perception' return new_chum_uri # If we aren't reusing this variant, delete it if it exists. delete_variant_if_exists(self.local_root, variant) cmd_args = ['vehicle/perception/pipeline/offline_perception_main', # Allows a user to ignore logged metaspins by setting # environment variable FLAGS_ignore_logged_metaspins=true. '--tryfromenv=ignore_logged_metaspins', '--chum_uri', self.chum_uri, '--variant_out', variant, '--chum_root_out', self.local_root, '--override_params_path', self.override_params_path] if not self.non_debug_mode: cmd_args += ['--debug_mode'] if self.run_tracks_to_annotated_objects: self.annotated_objects_lmdb_path = os.path.join( self.output_directory, '{}.annotated_objects.lmdb'.format( util.make_time_str(self.bag_id))) cmd_args += ['--tracks_to_annotated_objects_lmdb_out', self.annotated_objects_lmdb_path] if self.force_ll3d: cmd_args += ['--force_ll3d'] if self.dtn_debug_mode: cmd_args += ['--dtn_debug_mode'] if self.log_track_attr_model_features: cmd_args += ['--log_track_attr_model_features'] if self.no_yaw_computer: cmd_args += ['--no_yaw_computer'] if self.log_yaw_computer_info: cmd_args += ['--log_yaw_computer_info'] if self.log_extent_history: cmd_args += ['--log_extent_history'] if self.use_ground_heights: cmd_args += ['--publish_ground'] print 'Running offline_perception_main' print cmd_args subprocess.check_call(cmd_args) return new_chum_uri def launch_argus(self): # Hacky method to get local IP address from # https://stackoverflow.com/questions/166506/finding-local-ip-addresses-using-pythons-stdlib s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.connect(("8.8.8.8", 80)) hostname = s.getsockname()[0] s.close() # Set up layout for argus. link = render_link( self.chum_uri, comparison_uri=None, layout="pcp_tracker", time=self.start_time, hostname=hostname, port=self.argus_port, secure=False) print 'Open this URL in your browser: {}'.format(link) print ('NOTE: You will need a local argus instance running. ' 'If you don\'t already have one running, you can start ' 'a new one by ./argus/run.sh from the root of the repo.') def launch_triage_tool(self): cmd = [ self.command_path, '--chum_uri', self.chum_uri, '--test_pbtxt_path', self.test_pbtxt_path, # Allows a user to ignore logged metaspins by setting environment # variable FLAGS_ignore_logged_metaspins=true. '--tryfromenv=ignore_logged_metaspins', ] if self.alsologtostderr: cmd.append('--alsologtostderr') if self.edit: cmd.append('--edit_tests') if self.validator_errors: cmd += [ '--vmodule=triage_tool_controller=1', '--alsologtostderr', ] if self.ground_truth_input_source: cmd += ['--ground_truth_input_source', self.ground_truth_input_source] if self.ground_truth_association_frame_lmdb_path: cmd += ['--ground_truth_association_frame_lmdb_path', self.ground_truth_association_frame_lmdb_path] if self.tracking_metrics_lmdb_path: cmd += ['--tracking_metrics_lmdb_path', self.tracking_metrics_lmdb_path] if self.control_tracking_metrics_lmdb_path: cmd += ['--control_tracking_metrics_lmdb_path', self.control_tracking_metrics_lmdb_path] if self.segmentation_metrics_pb_path: cmd += ['--segmentation_metrics_pb_path', self.segmentation_metrics_pb_path] if self.control_segmentation_metrics_pb_path: cmd += ['--control_segmentation_metrics_pb_path', self.control_segmentation_metrics_pb_path] if self.triage_metric_name: cmd += ['--triage_metric_name', self.triage_metric_name] if self.triage_filter_name: cmd += ['--triage_filter_name', self.triage_filter_name] if self.occlusion_ground_truth_lmdb_path: cmd += ['--occlusion_ground_truth_lmdb_path', self.occlusion_ground_truth_lmdb_path] if self.use_ground_heights: cmd += ['--use_ground_heights'] if not self.load_prediction: cmd += ['--noload_prediction'] if self.window_name: cmd += ['--window_name', self.window_name] if self.show_notes: cmd += ['--show_notes'] if self.start_time: cmd += ['--start_time', str(self.start_time)] print 'Running {}'.format(' '.join(cmd)) subprocess.check_call(cmd) if self.edit: print 'Translating tests' translate_tests(self.bag_id)