""" Create TRG tickets based on events during the run """ import argparse import json import logging import os import time from collections import OrderedDict from base.jira import JiraClient from base.lock import lock from base.logging import zoox_logger from base.metrics import increment from bravado.exception import HTTPConflict, HTTPNotFound from data.automated_triage.triagers.vanilla\ .vanilla_triager import VanillaTriager from data.automated_triage.triagers.pose_invalid\ .pose_invalid_triager import PoseInvalidTriager from data.automated_triage.triagers.traffic_light_states\ .traffic_light_states_triager import TrafficLightStatesTriager from data.automated_triage.triagers.traffic_light_states\ .tl_utils import MissingZrnError from data.automated_triage.triagers.pcp_triager.pcp_triager import PcpTriager from data.automated_triage.event_classifier import lib from data.ingest.cluster.utils import ( MISSING_ZRN_ISSUE_NAME, DataCatalogQueryError, publish_ingest_job_metrics, publish_ingest_metric, register_ingest_issue, ) from data.ingest.cluster.ingest_events import lib as events_lib from infra.data_catalog.client import data_rest_api from zclient.client import make_client_from_url logger = logging.getLogger(__name__) CREATE_META_LOCK_PREFIX = 'data.automated_triage.event_classifier.create_meta' METRICS_PREFIX = 'data.automated_triage.event_classifier' VEHICLE_TICKETS_URL = 'https://vehicle-tickets.zooxlabs.com' VEHICLE_TICKETS_VERSION = 0 # The following vehicles will not have event classifier run for them EXEMPT_VEHICLES = [ 'labbot_01', 'vh6a_13', 'vh6a_07', 'zimbeast_20', 'zimbeast_16', ] def exempt_from_classifier(run): # Returns true if vehicle is in list of exempt vehicles # Returns false otherwise vehicle = run.get('vehicle') if vehicle in EXEMPT_VEHICLES: logger.info('Vehicle {} found in list of exempt vehicles'.format(vehicle)) return True return False def add_event_ticket(ticket_client, event_id, ticket_id=None, ticket_key=None): # Create an EventTicket Entry with the Event ID and (possibly) Jira Information event_ticket = ticket_client.get_model('EventTicket')( event_id=event_id, ticket=ticket_id, key=ticket_key) try: response = ticket_client.event_tickets.addEventTicket( eventTicket=event_ticket).result() except HTTPConflict: logger.warn('EventTicket entry already exists for this event.') except Exception as e: raise lib.VehicleTicketsError('Could not add Event Ticket Entry: {}'.format(str(e))) def patch_event_ticket(ticket_client, event_id, ticket_id, ticket_key): # Update EventTicket with Jira information patch = ticket_client.get_model("EventTicketPatch")(ticket=ticket_id, key=ticket_key) try: patch_response = ticket_client.event_tickets.modifyEventTicket( eventId=event_id, eventTicketPatch=patch).result() except Exception as e: logger.warn("Event {} {}\n".format(event_id, str(patch))) raise lib.VehicleTicketsError('Could not patch event ticket: {}'.format(str(e))) def create_or_update_coredump_meta_ticket(jira_client, event, issue, release_version): issue_key = issue['key'] line = lib.extract_failed_line(event) if not line: logger.warning("Can not extract failed line from backtrace," \ " meta ticket will not be created") return # Prevent parallel creation of the same Meta ticket with lock(CREATE_META_LOCK_PREFIX + "[" + release_version + "]" + line, blocking=True): meta_issues = lib.find_meta_issues(jira_client, release_version, line) if not meta_issues: description = "This is a meta ticket for core dump issues of {}\n"\ "corresponding to the failure in: {} \n".format(release_version, line) title = lib.get_meta_summary(release_version, line) labels = ['meta', 'meta_core_dump'] meta_issue = jira_client.create_issue(title, description, release_version=release_version, labels=labels).raw meta_issue_key = meta_issue['key'] jira_client.link_child_issue(meta_issue, issue) logger.info('Created a meta ticket:' + meta_issue_key) else: if len(meta_issues) > 1: logger.warning("Found more than 1 meta ticket for release" \ "version: '{}', line: '{}'".format(release_version, line)) meta_issue = meta_issues[0] meta_issue_key = meta_issue['key'] jira_client.link_child_issue(meta_issue, issue) logger.info("Linked meta jira ticket: '{}' " \ "with issue: '{}'".format(meta_issue_key, issue['key'])) def classify_events(meta_id, event_type, event_id, dry_run, force): event_client = make_client_from_url(events_lib.VEHICLE_EVENTS_URL, version=events_lib.VEHICLE_EVENTS_VERSION) ticket_client = make_client_from_url(VEHICLE_TICKETS_URL, version=VEHICLE_TICKETS_VERSION) jira_client = JiraClient() response = data_rest_api.get_run(meta_id) if not response['success']: raise DataCatalogQueryError("Could not retrieve data from run: {}".format(meta_id)) run = response['run'] if exempt_from_classifier(run): logger.info('Run {} is exempt from running event classifier'.format(meta_id)) return run_type = run.get('run_type') run_events = lib.get_run_events(event_client, lib.get_event_metadata(event_client, meta_id, event_id), meta_id, event_id) if not event_type: events = sum([run_events[key] for key in run_events.keys()], []) else: logger.info('Running on event: {}'.format(event_type)) events = run_events.get(event_type, []) # Get timestamps of all notes and disengagements in run. note_ts, disengagement_ts = lib.get_event_timestamps( lib.RosParamsProcessor(meta_id), run_events) # Instantiate all the triagers triagers = { VanillaTriager.name: VanillaTriager(note_ts, disengagement_ts), PoseInvalidTriager.name: PoseInvalidTriager(run['start_time'], run['end_time']), 'BrakeTap': PcpTriager( meta_id, 'vehicle/perception/pcp_auto_triage/' 'brake_tap_triager_params_default.pbtxt'), } # Triager is cpp python binding, difficult to handle errors # so checking chum topic first if lib.no_corrupt_chum_in_traffic_light_states_zrn(meta_id, dry_run): try: triagers[TrafficLightStatesTriager.name] =\ TrafficLightStatesTriager(meta_id) except MissingZrnError: logger.warn('Did not find ZRN for {}: skipping' ' TrafficLightStatesTriager'.format(meta_id)) register_ingest_issue(meta_id, MISSING_ZRN_ISSUE_NAME, dry_run) release_version = lib.get_release_version(meta_id) nogo_dict = {nogo.event.id: nogo for nogo in run_events['NOGO']} events.sort(key=lambda x: x.event.timestamp) for event in events: reports = lib.run_classifiers(ticket_client, event, triagers) reports = lib.process_reports(event, reports, triagers) if dry_run: for triager_name, report in reports.items(): logger.info('(dry run). Event {} by triager {} was classified: {} \n'. format(event.event.id, triager_name, report.result)) logger.info(str(report) + '\n') continue # Upload all the reports to the database desired_reports = lib.upload_reports(ticket_client, reports.values(), force=force) # Check if EventTicket entry already exists for this event try: event_ticket_obj = ticket_client.event_tickets.getEventTicket( eventId=event.event.id).result() event_ticket_exists = True except HTTPNotFound as e: event_ticket_exists = False # Don't file ticket if none of the triager reports want one if not any(report.result for report in desired_reports): logger.info("Event {} did not trigger any triagers, skipping.".format(event.event.id)) # Add EventTicket entry with no ticket information if not event_ticket_exists: add_event_ticket(ticket_client, event.event.id) continue title = lib.generate_standard_ticket_title(event) labels = lib.generate_ticket_labels(event, run, desired_reports, triagers) disengagement_reason = None nogo_for_disengagement = None if event.event.type == 'DISENGAGEMENT': disengagement_reason = event.reason nogo_for_disengagement = nogo_dict.get(event.nogo_id) # Retrieve the Ticket from JIRA issue_list = lib.get_event_issue(jira_client, event.event.id) if issue_list: issue = issue_list[0] issue_id, issue_key = issue['id'], issue['key'] labels = issue['fields'].get('labels', labels) description = issue['fields'].get('description', '') logger.info('Issue {} already exists!' 'Updating release version...'.format(issue_key)) jira_client.update_issue(issue_key, title, description, labels=labels, release_version=release_version) increment('jira_issue.updated', prefix=METRICS_PREFIX) else: logger.info("Need to create a new issue!") issue = jira_client.create_issue( title, 'classifier:py, need to obtain the issue key' ' before the body is generated', event_id=event.event.id, argus_link=lib.get_argus_link(event), bucket=lib.extract_triage_bucket(event), priority=lib.get_event_priority(event), release_version=release_version, run_type=run_type, disengagement_reason=disengagement_reason, bucket_reclassified_count=0).raw issue_id, issue_key = issue['id'], issue['key'] description = lib.generate_description(event, run, desired_reports, triagers, nogo_for_disengagement, issue_key) jira_client.update_issue(issue_key, title, description, labels=list(OrderedDict.fromkeys(labels))) increment('jira_issue.created', prefix=METRICS_PREFIX) lib.notify_slack(meta_id, issue_key, title) if event.event.type == 'COREDUMP': if release_version: create_or_update_coredump_meta_ticket(jira_client, event, issue, release_version) else: logger.warning("No release version, can't create/update Meta-core ticket") if event_ticket_exists: # Patch EventTicket entry if it already exists patch_event_ticket(ticket_client, event.event.id, issue_id, issue_key) else: # Add new EventTicket entry add_event_ticket(ticket_client, event.event.id, issue_id, issue_key) def parse_args(): parser = argparse.ArgumentParser( "Analyze all events from a run and add classifications to vehicle_ticket microservice") parser.add_argument("--ingest_meta_id", type=str, required=True) parser.add_argument('--dry-run', action='store_true', help='boolean flag to print instead of storing classifications') parser.add_argument('--force', action='store_true', help='boolean flag to force update on event reports') parser.add_argument("--event_type", type=str, default='', help='string flag to filter single event type to be classified') parser.add_argument("--event_id", type=str, default='', help='string flag to specify single event id to be classified') return parser.parse_args() if __name__ == "__main__": zoox_logger.configureLogging('event_classifier') args = parse_args() start_time = time.time() publish_ingest_metric( args.ingest_meta_id, 'event_classifier', 'started', dry_run=args.dry_run, time=start_time ) classify_events( args.ingest_meta_id, args.event_type.upper(), args.event_id, args.dry_run, args.force, ) publish_ingest_job_metrics( args.ingest_meta_id, 'event_classifier', 'completed', start_time, dry_run=args.dry_run, registration_time=True )