#!/usr/bin/env python """ Script to launch all caffe2 MobileLab tests MobieLab uses `flow-cli` to launch jobs To install flow-cli on your devserver, run ``` sudo feature install fblearner_flow ``` Example usage: - Run all tests: ~/fbsource/xplat/caffe2/scripts/run_mobilelab.py --diff_id 56041720 --base_commit 583c0d7396c8d0469a225bf2bb19d8ecd25ba3af - Run all android tests: ~/fbsource/xplat/caffe2/scripts/run_mobilelab.py --diff_id 56041720 --base_commit --android - Run all android tests: ~/fbsource/xplat/caffe2/scripts/run_mobilelab.py --diff_id 56041720 --base_commit --iOS - Run all tests and let LabBot post lab run results on your diff: ~/fbsource/xplat/caffe2/scripts/run_mobilelab.py --diff_id 56041720 --base_commit 583c0d7396c8d0469a225bf2bb19d8ecd25ba3af --comment_on_revision D8756131 """ from __future__ import absolute_import from __future__ import division from __future__ import print_function from __future__ import unicode_literals import argparse import json import os import subprocess import sys # comment out the tests you don't need to run android_tests = [ "android_microbenchmark.aml.microbenchmark_caffe2.correctness.nexus6", "android_microbenchmark.aml.microbenchmark_caffe2.body_tracking_cold_delay.nexus6", "android_microbenchmark.aml.microbenchmark_caffe2.body_tracking_delay.nexus6", "android_microbenchmark.aml.microbenchmark_caffe2.body_tracking_memory.nexus6", "android_microbenchmark.aml.microbenchmark_caffe2.body_tracking_operator_delay.nexus6", "android_microbenchmark.aml.microbenchmark_caffe2.hand_tracking_cold_delay.nexus6", "android_microbenchmark.aml.microbenchmark_caffe2.hand_tracking_delay.nexus6", "android_microbenchmark.aml.microbenchmark_caffe2.hand_tracking_memory.nexus6", "android_microbenchmark.aml.microbenchmark_caffe2.hand_tracking_operator_delay.nexus6", "android_microbenchmark.aml.microbenchmark_caffe2.person_segmentation_cold_delay.nexus6", "android_microbenchmark.aml.microbenchmark_caffe2.person_segmentation_delay.nexus6", "android_microbenchmark.aml.microbenchmark_caffe2.person_segmentation_memory.nexus6", "android_microbenchmark.aml.microbenchmark_caffe2.person_segmentation_operator_delay.nexus6", "android_microbenchmark.aml.microbenchmark_caffe2.xray_mobile_42_cold_delay.nexus6", "android_microbenchmark.aml.microbenchmark_caffe2.xray_mobile_42_delay.nexus6", "android_microbenchmark.aml.microbenchmark_caffe2.xray_mobile_42_memory.nexus6", "android_microbenchmark.aml.microbenchmark_caffe2.xray_mobile_42_operator_delay.nexus6", "android_microbenchmark.aml.microbenchmark_caffe2.xray_mobile_84_v1_cold_delay.nexus6", "android_microbenchmark.aml.microbenchmark_caffe2.xray_mobile_84_v1_delay.nexus6", "android_microbenchmark.aml.microbenchmark_caffe2.xray_mobile_84_v1_memory.nexus6", "android_microbenchmark.aml.microbenchmark_caffe2.xray_mobile_84_v1_operator_delay.nexus6", ] iOS_tests = [ "xcui.caffe2.correctness.ios10.iPhone7", "xcui.caffe2.body_tracking_cold_delay.ios10.iPhone7", "xcui.caffe2.body_tracking_delay.ios10.iPhone7", "xcui.caffe2.body_tracking_memory.ios10.iPhone7", "xcui.caffe2.hand_tracking_cold_delay.ios10.iPhone7", "xcui.caffe2.hand_tracking_delay.ios10.iPhone7", "xcui.caffe2.hand_tracking_memory.ios10.iPhone7", "xcui.caffe2.person_segmentation_cold_delay.ios10.iPhone7", "xcui.caffe2.person_segmentation_delay.ios10.iPhone7", "xcui.caffe2.person_segmentation_memory.ios10.iPhone7", "xcui.caffe2.xray_mobile_42_cold_delay.ios10.iPhone7", "xcui.caffe2.xray_mobile_42_delay.ios10.iPhone7", "xcui.caffe2.xray_mobile_42_memory.ios10.iPhone7", "xcui.caffe2.xray_mobile_84_v1_cold_delay.ios10.iPhone7", "xcui.caffe2.xray_mobile_84_v1_delay.ios10.iPhone7", "xcui.caffe2.xray_mobile_84_v1_memory.ios10.iPhone7", ] android_job_inputs = json.loads( """{ "test_config_name": "android_microbenchmark.aml.microbenchmark_caffe2.correctness.nexus6", "workflow_inputs": { "gating_overrides": null, "android_microbenchmark": { "experiment_store_config": { "metadata": { "entry_point": "frontend" } }, "one_world_priority": -10, "experiment_request": { "treatment": { "build_request": { "build_spec": { "diff_id": "replaceme" }, "priority": 0 } }, "control": { "build_request": { "build_spec": { "commit_hash": "replaceme" }, "priority": 0 } } }, "comment_on_revision": "replaceme" } } }""" ) iOS_job_inputs = json.loads( """{ "test_config_name": "xcui.caffe2.correctness.ios10.iPhone7", "workflow_inputs": { "xcui": { "gating_overrides": null, "experiment_store_config": { "metadata": { "entry_point": "frontend" } }, "one_world_priority": -10, "experiment_request": { "treatment": { "build_request": { "build_spec": { "diff_id": "replaceme" }, "priority": 0 } }, "control": { "build_request": { "build_spec": { "commit_hash": "replaceme" }, "priority": 0 } } }, "comment_on_revision": "replaceme" } } }""" ) parser = argparse.ArgumentParser(description="Launch all caffe2 mobilelab jobs") parser.add_argument( "--diff_id", required=True, help="the diff version (not the DNNNNNNNN revision) for your change, e.g. 55434539", ) parser.add_argument( "--base_commit", required=True, help="base commit hash, e.g. 583c0d7396c8d0469a225bf2bb19d8ecd25ba3af", ) parser.add_argument( "--comment_on_revision", default=None, help="the phabricator revision (the DNNNNNNNN revision), eg. D8756131 or D8756131", ) parser.add_argument("--android", action="store_true", help="Run all android tests") parser.add_argument("--iOS", action="store_true", help="Run all iOS tests") def main(args): options = parser.parse_args(args) diff_id = options.diff_id base_commit = options.base_commit comment_on_revision = options.comment_on_revision if comment_on_revision and comment_on_revision.startswith("D"): comment_on_revision = comment_on_revision[1:] # Launch all jobs if the user didn't specify android or iOS run_all_jobs = True if not (options.android) and not (options.iOS) else False # Must launch jobs from fbcode os.system("cd ~/fbsource/fbcode") if run_all_jobs or options.android: for t in android_tests: android_job_inputs["workflow_inputs"]["android_microbenchmark"][ "experiment_request" ]["treatment"]["build_request"]["build_spec"]["diff_id"] = diff_id android_job_inputs["workflow_inputs"]["android_microbenchmark"][ "experiment_request" ]["control"]["build_request"]["build_spec"]["commit_hash"] = base_commit android_job_inputs["workflow_inputs"]["android_microbenchmark"][ "comment_on_revision" ] = comment_on_revision android_job_inputs["test_config_name"] = t subprocess.check_call( [ "flow-cli", "launch", "lab.run_test.entry", "--parameters-json", json.dumps(android_job_inputs), "--entitlement", "oneworld_prn", "--name", "Run with test config " + t, ] ) if run_all_jobs or options.iOS: for t in iOS_tests: iOS_job_inputs["workflow_inputs"]["xcui"]["experiment_request"][ "treatment" ]["build_request"]["build_spec"]["diff_id"] = diff_id iOS_job_inputs["workflow_inputs"]["xcui"]["experiment_request"]["control"][ "build_request" ]["build_spec"]["commit_hash"] = base_commit iOS_job_inputs["workflow_inputs"]["xcui"][ "comment_on_revision" ] = comment_on_revision iOS_job_inputs["test_config_name"] = t subprocess.check_call( [ "flow-cli", "launch", "lab.run_test.entry", "--parameters-json", json.dumps(iOS_job_inputs), "--entitlement", "oneworld_prn", "--name", "Run with test config " + t, ] ) if __name__ == "__main__": sys.exit(main(sys.argv[1:]))