This plot shows an aggregate of actions considered at a given Decision tick. An action is \"considered\" if it passes preconditions and has a rollout. Not all considered actions are terminal (ie. successful) because they may fail constraints. Actions are run in parallel so latency should not scale linearly with the number of actions. A step increase in latency is expected if more than seven actions are considered at a Decision tick.
""" ) reference_details = Div( text="""This plot shows an aggregate of references considered at a given Decision tick. A reference is \"considered\" if geometric planning and action reference smoothing is executed. Not all considered references are feasible and reference generation can fail. Not all feasible references are used by an action; contrarily, a given reference may be used by multiple actions. References are generated in parallel but do contend for GPU resources.
""" ) # build actions considered box plots df_actions_candidate = experiment.aggregate_zci_tick( candidate, "num_actions_considered" ) df_actions_control = experiment.aggregate_zci_tick( control, "num_actions_considered" ) p_actions = zci_tick_box_plot( df_actions_control, "num_actions_considered", control ) d_actions = zci_tick_box_plot( df_actions_candidate, "num_actions_considered", candidate ) # build references considered box plots df_refs_candidate = experiment.aggregate_zci_tick( candidate, "num_references_considered" ) df_refs_control = experiment.aggregate_zci_tick( control, "num_references_considered" ) p_refs = zci_tick_box_plot( df_refs_control, "num_references_considered", control ) d_refs = zci_tick_box_plot( df_refs_candidate, "num_references_considered", candidate ) # align plot bounds max_y = max( [ p_actions.y_range.end, d_actions.y_range.end, p_refs.y_range.end, d_refs.y_range.end, ] ) p_actions.y_range = bokeh_plots.Range1d(0, max_y) d_actions.y_range = bokeh_plots.Range1d(0, max_y) p_refs.y_range = bokeh_plots.Range1d(0, max_y) d_refs.y_range = bokeh_plots.Range1d(0, max_y) self.action_search_plots = gridplot( [ [action_details], [p_actions, d_actions], [reference_details], [p_refs, d_refs], ], sizing_mode="stretch_width", ) def get_coverage_sub_tab(self): # Render stats stats = pd.concat( [self.control_stats.sum(), self.candidate_stats.sum()], axis=1 ) stats.columns = ["Control", "Candidate"] # Break into actions and subgoals ticks = stats.loc["ticks"] stats_actions = stats.filter(regex="^ACTION_.*$", axis=0).copy() stats_subgoals = stats.filter(regex="^((?!^ACTION_).)*$", axis=0).copy() stats_subgoals = stats_subgoals.drop(["ticks"]).copy() stats_actions.loc[:, "Diff"] = ( stats_actions["Candidate"] - stats_actions["Control"] ) stats_subgoals.loc[:, "Diff"] = ( stats_subgoals["Candidate"] - stats_subgoals["Control"] ) stats_actions.index.name = "Actions" stats_subgoals.index.name = "Subgoals" action_subgoal_help = Div( text="""Compares the number of ticks where actions or subgoals are active. If there are many 0s in the candidate and control, it likely means the test suite does not exercise many of the planner features.
""".format( PRECISION_THRESH ) ) action_summary = ( stats_actions.sort_values(by="Control", ascending=True) .style.applymap(color_ok, subset=["Diff"]) .applymap(color_zero, subset=["Candidate", "Control"]) .format( { "Control": "{{:0,}}/{:0,} ticks".format(ticks["Control"]), "Candidate": "{{:0,}}/{:0,} ticks".format( ticks["Candidate"] ), } ) .set_table_styles(self.table_styles) .set_properties(**{"border-width": 1}) .render() ) subgoal_summary = ( stats_subgoals.sort_values(by="Control", ascending=True) .style.applymap(color_ok, subset=["Diff"]) .applymap(color_zero, subset=["Candidate", "Control"]) .format( { "Control": "{{:0,}}/{:0,} ticks".format(ticks["Control"]), "Candidate": "{{:0,}}/{:0,} ticks".format( ticks["Candidate"] ), } ) .set_table_styles(self.table_styles) .set_properties(**{"border-width": 1}) .render() ) action_stats_table = Div(text=action_summary) subgoal_stats_table = Div(text=subgoal_summary) coverage_layout = layout( [[action_subgoal_help], [action_stats_table, subgoal_stats_table]], sizing_mode="stretch_width", align=("center", "start"), ) return Panel(child=coverage_layout, title="Test Coverage") def get_auxilary_sub_tabs(self): return [ Panel( child=self.action_search_plots, title="Action Search Statistics" ) ] def get_key_metric_definitions(self): return dict( vx_max_diff="""Max absolute deviation in decision longitudinal velocity in its first state at all ticks.
""", accel_max_diff="""Max absolute deviation in decision longitudinal acceleration in its first control at all ticks.
""", ars_max_diff="""Max absolute deviation in action reference longitudinal acceleration in its first control at all ticks.
""", accel_rmse="""Root mean-squared error in decision longitudinal acceleration in its first control at all ticks.
""", actions_max_diff="""Max absolute deviation in the number of actions considered at every tick.
""", references_max_diff="""Max absolute deviation in the number of action references considered at every tick.
""", ars_accel_rmse="""Root mean-squared error in action refrence longitudinal acceleration in its first control at all ticks.
""", corridor_left_max_rmse="""Computes the root mean-squared error between the left-hand corridor of the two SHAs at every tick and reports the maximum.
""", corridor_right_max_rmse="""Computes the root-mean-squared error between the right-hand corridor of the two SHAs at every tick and reports the maximum.
""", ey_max_diff="""Max absolute deviation in the decision lateral deviation from the route reference in its first state at all ticks.
""", ey_rmse="""Root mean-squared error in the decision lateral deviation from the route reference in its first state at all ticks.
""", ars_avg_num_states_diff="""Average difference in the number of action reference smoother states between the two SHAs.
""", registry_num_diff="""Max absolute deviation in the number of registry entries considered at every tick.
""", registry_ped_diff="""Max absolute deviation in the number of registry pedestrain entries considered at every tick.
""", ) def _get_metric_formats(self): pos_fmt = "{:.2f}m" num_fmt = "{:.3f}" return { "ctrl_dist_to_ar": pos_fmt, "cand_dist_to_ar": pos_fmt, "accel_max_diff": ACCEL_FMT, "ars_max_diff": ACCEL_FMT, "lane_ref_max_diff": ACCEL_FMT, "accel_rmse": ACCEL_FMT, "ars_accel_rmse": ACCEL_FMT, "lane_ref_accel_rmse": ACCEL_FMT, "vx_max_diff": VEL_FMT, "corridor_left_max_rmse": pos_fmt, "corridor_right_max_rmse": pos_fmt, "ars_avg_s_sampling_diff": pos_fmt, "ars_avg_num_states_diff": num_fmt, "ey_max_diff": pos_fmt, "ey_rmse": pos_fmt, "ctrl_latprox_left": num_fmt, "ctrl_latprox_right": num_fmt, "cand_latprox_left": num_fmt, "cand_latprox_right": num_fmt, "ctrl_num_actions": INT_FMT, "cand_num_actions": INT_FMT, "actions_max_diff": INT_FMT, "ctrl_num_references": INT_FMT, "cand_num_references": INT_FMT, "references_max_diff": INT_FMT, "registry_num_diff": INT_FMT, "registry_ped_diff": INT_FMT, } def _get_display_cols(self): return [ # Meta data columns "scenario", "links", # Planner metrics columns "vx_max_diff", "accel_max_diff", "ars_max_diff", "ey_max_diff", "accel_rmse", "ars_accel_rmse", "ey_rmse", "ars_avg_num_states_diff", "corridor_left_max_rmse", "corridor_right_max_rmse", "actions_max_diff", "references_max_diff", "registry_num_diff", "registry_ped_diff", ] class SteeringComparisonReportTab(ReportTab): """The tab for reporting VH6 2ws vs 4ws comparisons.""" def __init__( self, dataframe, branch, sha, styles, argus_host, pipe_id, tracking ): df_traj = self._build_summary_table(dataframe).reset_index(drop=True) self.traj_gradient = df_traj.columns self.targets = dataframe["sdl_target"] self.sha = sha self.branch = branch self.pipe_id = pipe_id self.tracking = tracking links = dataframe[["ctrl_uri", "cand_uri"]].apply( format_argus_link( f"{branch}/2ws", f"{branch}/4ws", "planner_comparison", argus_host, ), axis=1, ) df_traj[("", "links")] = links.apply( lambda url: f"""""" ) df_traj[("", "scenario")] = self.targets # use SDL target as scenario df_traj[("", "ctrl_uri")] = dataframe["ctrl_uri"] df_traj[("", "cand_uri")] = dataframe["cand_uri"] df_traj.sort_index(axis=1, inplace=True) super().__init__(df_traj, styles, argus_host) @property def type(self): return "VH6 2ws vs 4ws" def get_key_metric_definitions(self): """Returns a dict from Planner Data Comparator metric to definition.""" return { ("decision", "Non-expected heading error diff(deg)"): ( "Unaccounted heading error (error not due to change in " "wheelbase) when projecting VH6 4ws decision pose onto VH6 2ws " "decision pose" ), ("decision", "Projected Lateral error(m)"): ( "Lateral error when projecting VH6 4ws decision pose onto VH6 " "2ws decision pose" ), ("decision", "Projected Velocity vx error(m/s)"): ( "Velocity error when projecting VH6 4ws decision pose onto VH6 " "2ws decision pose" ), } def get_comparison_sub_tab(self): df_traj = self.df[self._get_display_cols()] # Create the trajectory projection errors traj_html = ( df_traj.style.apply( highlight_violations(STEERING_MODE_SCENARIO_THRESHOLDS), subset=self.traj_gradient, targets=self.targets, ) .hide_index() .set_precision(3) .set_properties(**{"font-family": "monospace"}) .format(self._get_metric_formats()) .render() ) traj_html = "\n".join( set_data_attributes( df_traj, traj_html, subset=self._get_key_metrics() ) ) traj_table = DataTables(text=traj_html, page_length=25) comparison_help = Div( text=f"""Compares {self.branch} to itself using VH6 2ws and
4ws. See
mined_metric/builder/metrics_impl/pisces:steering_mode_log_tests
for the set of tests run.
Definitions of the columns can be found on Confluence and are reproduced here for convenience.
""" ) comparison_defs = Div( text=f"""Computes the projected lateral, heading and velocity
vx errors by projecting poses from the VH6 4ws
trajectory onto the VH6 2ws trajectory from
{self.branch}.
It is the non-zero heading error difference expected to be present between the baseline VH6 2ws and comparison 4ws run. In this situation, a non-zero heading error difference is expected to be present by design for not being able to steer one of the axles on the 2ws run. This non-zero error is calculated based on the lever arm distance from the center of the non-steered axle to the wheelbase center.
The non-expected heading error difference equates to the difference between the projected heading errors and the expected heading error difference.
The tool also computes errors in Decision trajectory's
s values for longitudinal comparison between the VH6
2ws run and the VH6 4ws run on {self.branch}.
If you are seeing unexpected delta in the 2ws/4ws comparison, you can investigate further by running the Planner Motion Comparator tool with output metrics JSON logging enabled.
git checkout {self.sha}
./sim/launch.sh local planner SCENARIO --save_chum \\
--simulator_args="--params-kv sim/override_autonomous_mode=PURE_PLAYBACK_MODE \\
--params-kv sim/vehicle_type=vh6 \\
--params-kv sim/sim_model=vh6a \\
--params-kv planner/config_variant_id=vh6_2ws"
./sim/launch.sh local planner SCENARIO --save_chum \\
--simulator_args="--params-kv sim/override_autonomous_mode=PURE_PLAYBACK_MODE \\
--params-kv sim/vehicle_type=vh6 \\
--params-kv sim/sim_model=vh6a \\
--params-kv planner/config_variant_id=vh6_4ws"
bazel run //vis/data_analysis/data_comparator:planner_motion_comparator -- '<2ws chum uri>' '<4ws chum uri>' --decision_only
If the issue exists on both the candidate and the control SHAs,
merge the latest upstream changes to the control branch and repeat
the above step to run the comparator tool on the control SHA to
confirm this issue has not been fixed by an upstream change.
If the issue still persists, post on #planner-metrics-help for
visibility.
"""
)
steering_mode_comparison = Panel(
child=layout(
[[comparison_help], [comparison_defs], [traj_table]],
sizing_mode="stretch_width",
align=("center", "start"),
),
title="Comparison",
)
return steering_mode_comparison
def get_key_metric_df(self):
key_df = super().get_key_metric_df()
key_df = key_df.apply(
get_violations(STEERING_MODE_SCENARIO_THRESHOLDS),
targets=self.targets,
)
return key_df
def record_steering_mode_comparison_data(self, comparison_data, stats):
"""
Computes and records data for steering mode comparison that can be
analyzed over time at
https://pmav.zooxlabs.com/tracking?metric=VH6_2WS_4WS_DIFF&branch=The maximum difference of L2 distance of two most likely trajectories between the control and candidate experiments among all matched entities across the log range. The entity id and timestamp are the match keys.
""", max_trajectory_count_diff="""The maximum difference of trajectory counts between the control and candidate experiments among all matched entities across the log range. The entity id and timestamp are the match keys.
""", max_trajectory_weight_diff="""The maximum difference of trajectory weights between the control and candidate experiments among all matched entities across the log range. The entity id and timestamp are the match keys.
""", ) def _get_metric_formats(self): num_fmt = "{:.5f}" pos_fmt = "{:.3f}m" cnt_fmt = "{:d}" return { "max_most_likely_at_5s_l2_diff": pos_fmt, "max_trajectory_count_diff": cnt_fmt, "max_trajectory_weight_diff": num_fmt, } def _get_display_cols(self): cols = ["scenario", "argus", "comparison_valid"] # Meta data columns cols.extend(self._get_key_metrics()) return cols class PerceptionReportTab(ReportTab): def __init__(self, dataframe, styles, argus_host, control, candidate): super().__init__(dataframe, styles, argus_host) self.df = add_url_column_to_dataframe( self.df, argus_host, "prediction_comparison", self._get_key_metrics(), ) self.add_pcp_track_highlight( self.df, argus_host, "prediction_comparison" ) self.df["argus"] = self.df["argus"].apply( lambda x: """""".format( x ) ) def add_pcp_track_highlight( self, df, argus_host, argus_layout="prediction_comparison" ): def get_argus_layout(diff_tracks_A=None, diff_tracks_B=None): # diff_tracks_A is the different track ids and timestamps for sha A # diff_tracks_B is the different track ids and timestamps for sha B layout_json = read_layout_to_dict(argus_layout) if diff_tracks_A and diff_tracks_B: track_id_A = list(diff_tracks_A.keys()) timestamps_A = list(diff_tracks_A.values()) track_id_B = list(diff_tracks_B.keys()) timestamps_B = list(diff_tracks_B.values()) layout_json = add_highlight_entities( layout_json, track_id_A, timestamps_A, 0 ) layout_json = add_highlight_entities( layout_json, track_id_B, timestamps_B, 1 ) return layout_json return layout_json def optionally_render_link(diff_tracks_A_key, diff_tracks_B_key): def render(x): if pd.notnull(x[diff_tracks_A_key]) and pd.notnull( x[diff_tracks_B_key] ): return render_link( x["ctrl_uri"], x["cand_uri"], get_argus_layout( x[diff_tracks_A_key], x[diff_tracks_B_key] ), time=None, hostname=argus_host, secure=(argus_host == "argus.zooxlabs.com"), ) else: return None return render renderer = optionally_render_link("diff_tracks_A", "diff_tracks_B") df["argus"] = df.apply(renderer, axis=1) return df def get_key_metric_definitions(self): return dict( fraction_diff="""Fraction of tracks which are new in either of the branches.
""" ) def _get_metric_formats(self): num_fmt = "{:.5f}" pos_fmt = "{:.3f}m" cnt_fmt = "{:d}" return {"fraction_diff": num_fmt} def _get_display_cols(self): cols = ["scenario", "argus"] # Meta data columns cols.extend(self._get_key_metrics()) return cols @property def type(self): return "Perception" class TrackerReportTab(ReportTab): """The tab for reporting Tracker-related comparisons.""" def __init__(self, dataframe, styles, argus_host): super().__init__(dataframe, styles, argus_host) @property def type(self): return "Tracker" def get_key_metric_definitions(self): """Returns a dict from Tracker metric to definition.""" definitions = {} for field in LowLevelControl.DESCRIPTOR.fields: if is_comparable_low_level_control_field(field): definitions[ field.name + "_max_diff" ] = """Max absolute deviation in the Tracker {} considered at every tick.
""".format( field.name ) return definitions def _get_metric_formats(self): """Returns a dict from Tracker metric to display format.""" ang_fmt = "{:.3f}rad" ang_vel_fmt = "{:.3f}rad/s" return { "steering_angle_front_max_diff": ang_fmt, "steering_angle_rear_max_diff": ang_fmt, "steering_angle_rate_front_max_diff": ang_vel_fmt, "steering_angle_rate_rear_max_diff": ang_vel_fmt, "velocity_max_diff": VEL_FMT, "acceleration_max_diff": ACCEL_FMT, "tracker_state_max_diff": INT_FMT, "velocity_control_state_max_diff": INT_FMT, "drive_direction_max_diff": INT_FMT, "drive_gear_max_diff": INT_FMT, "gravity_acceleration_max_diff": ACCEL_FMT, "current_odd_max_diff": INT_FMT, } def _get_display_cols(self): """Returns the list of Tracker metrics to display.""" columns = ["scenario", "links"] for field in LowLevelControl.DESCRIPTOR.fields: if is_comparable_low_level_control_field(field): columns.append(field.name + "_max_diff") return columns class SummaryTab: def __init__(self, report_tabs, styles): self.report_tabs = report_tabs self.styles = styles self.summary_df = None def get_changed_metrics(self): """ Get the counts of metrics that are changed if no changes 0's would be returned """ if self.summary_df is None: raise RuntimeError( "Can't get summary metrics without building the summary tab first" ) # The columns of self.summary_df should be the same as the ones defined # in the `self.build()` function return dict(self.summary_df.groupby(["Metric Type"])["Count"].sum()) def build(self, title="Summary"): summary = {} scenario_and_uris_df = pd.DataFrame() diff_cnt = 0 for report_tab in self.report_tabs: key_metric_df = report_tab.get_key_metric_df() if ( isinstance(report_tab, SteeringComparisonReportTab) and report_tab.tracking ): scenario_and_uris_df = report_tab.get_scenario_and_uris_df() violated_df = key_metric_df[ (key_metric_df > PRECISION_THRESH).any(1) ] scenario_and_uris_df = scenario_and_uris_df[ scenario_and_uris_df.index.isin(violated_df.index) ] scenario_and_uris_df = pd.concat( [scenario_and_uris_df, violated_df], axis=1 ) total_cnt_gt_thresh = (key_metric_df > PRECISION_THRESH).sum() total_cnt = key_metric_df.count() for ( def_key, def_description, ) in report_tab.get_key_metric_definitions().items(): diff_cnt = total_cnt_gt_thresh[def_key] total = total_cnt[def_key] diff_summary = "{:0}/{:0} differences".format(diff_cnt, total) # If the key already exists, _diff_ the diff_cnt (diff of diffs) # for the key to display differences in the underlying # comparison. # # This is predominantly used for 2ws vs 4ws comparisons. # # This behavior will be undefined if we try to update a record # multiple times. if def_key in summary: existing_record = summary[def_key] # # ********************** WARNING ************************** # # This is super brittle. It relies on the fact that the # candidate tab is _first_ in the list of report tabs. # Thus, when there are overlaps, the existing record is # on the candidate, while the new value of `diff_cnt` is # from the control. So the code is trying to diff the number # of metrics exceeding threshold on the candidate to those # on the control. diff_of_diff_cnt = existing_record[3] - diff_cnt diff_summary = "{:0}/{:0} differences".format( diff_of_diff_cnt, total ) diff_cnt = diff_of_diff_cnt summary.update( { def_key: [ report_tab.type, def_description, diff_summary, diff_cnt, ] } ) if ( isinstance(report_tab, SteeringComparisonReportTab) and report_tab.tracking ): stats = {"diff": int(diff_cnt > 0)} report_tab.record_steering_mode_comparison_data( scenario_and_uris_df, stats ) self.summary_df = pd.DataFrame.from_dict( summary, orient="index", columns=["Metric Type", "Definition", "Summary", "Count"], ) summary_df = self.summary_df[ ["Metric Type", "Definition", "Summary"] ].sort_values(by="Metric Type") summary_html = ( summary_df.style.applymap( color_str_ok, negative_color="#569edd", subset=["Summary"] ) .set_table_styles(self.styles) .set_properties(**{"border-width": 1, "max-width": "600px"}) .render() ) summary_table = Div(text=summary_html) summary_help = Div( text="""There is no behavior change between branches when all metrics are green (or blue, improvements to 2ws vs 4ws).
For the Planner, Prediction, Perception or
Tracker metric types, failures occur when the key metrics
exceed a threshold of {:.1e}.
For the VH6 2ws vs 4ws metric type, failures occur when the key metrics exceed the scenario-specific thresholds on the candidate SHA alone. The scenario-specific thresholds are specified in VH6 2ws vs 4ws Thresholds.
If there are any deviations, please use information in the corresponding tabs to investigate.
""".format( PRECISION_THRESH ) ) summary_layout = layout( [[summary_table], [summary_help]], sizing_mode="stretch_width", align=("center", "start"), ) return Panel(child=summary_layout, title=title)