build: Refactor build_with_bazel.py to discover targets

Currently, the build_with_bazel.py script has an internal list
of targets. As we add/subtract targets, this becomes more difficult
to maintain. It also makes adding extra targets defined by
msm_kernel_extensions.bzl a manual process.

Instead, query the Bazel DAG to dynamically determine the
build targets based on input target name.

Change-Id: Id39889260b8f8af5c0c157ff091ee6e0d253c90f
Signed-off-by: John Moon <quic_johmoo@quicinc.com>

build_with_bazel.py

@@ -5,100 +5,219 @@
 import argparse
 import logging
 import os
+import re
 import sys
-from subprocess import Popen
+import subprocess
 
-BAZEL_BIN = "./tools/bazel"
 CPU = "aarch64"
 HOST_CROSSTOOL = "@bazel_tools//tools/cpp:toolchain"
+HOST_TARGETS = ["dtc"]
+DEFAULT_SKIP_LIST = ["abi", "test_mapping"]
 
 
-def get_cross_cli_opts(toolchain):
+class BazelBuilder:
-    return [
+    """Helper class for building with Bazel"""
-        "--cpu={}".format(CPU),
-        "--crosstool_top={}".format(toolchain),
-        "--host_crosstool_top={}".format(HOST_CROSSTOOL),
-    ]
 
+    def __init__(self, target_list, skip_list, user_opts):
+        self.workspace = os.path.realpath(
+            os.path.join(os.path.dirname(os.path.realpath(__file__)), "..")
+        )
+        self.bazel_bin = os.path.join(self.workspace, "tools", "bazel")
+        if not os.path.exists(self.bazel_bin):
+            logging.error("failed to find Bazel binary at %s", self.bazel_bin)
+            sys.exit(1)
+        self.kernel_dir = os.path.basename(
+            (os.path.dirname(os.path.realpath(__file__)))
+        )
+        self.target_list = target_list
+        self.skip_list = skip_list
+        self.user_opts = user_opts
+        self.process_list = []
 
-def bazel(bazel_subcommand, targets, extra_options=[], us_cross_toolchain=None):
+    def __del__(self):
-    cmdline = [BAZEL_BIN, bazel_subcommand]
+        for proc in self.process_list:
-    cmdline.extend(extra_options)
+            proc.kill()
-    if us_cross_toolchain:
+            proc.wait()
-        cmdline.extend(get_cross_cli_opts(us_cross_toolchain))
-    cmdline.extend(targets)
-    cmdline_str = " ".join(cmdline)
-    try:
-        logging.debug('Running "%s"', cmdline_str)
-        build_proc = Popen(cmdline_str, shell=True)
-        build_proc.wait()
-        if build_proc.returncode != 0:
-            sys.exit(build_proc.returncode)
-    except Exception as e:
-        logging.error(e),
-        sys.exit(1)
 
+    @staticmethod
+    def get_cross_cli_opts(toolchain):
+        """Form cross toolchain Bazel options"""
+        return [
+            "--cpu={}".format(CPU),
+            "--crosstool_top={}".format(toolchain),
+            "--host_crosstool_top={}".format(HOST_CROSSTOOL),
+        ]
 
-def build_targets(targets, user_opts=[], us_cross_toolchain=None):
+    def get_build_targets(self):
-    if not targets:
+        """Query for build targets"""
-        logging.warning("no targets to build")
+        logging.info("Querying build targets...")
-    bazel("build", targets, user_opts, us_cross_toolchain)
 
+        cross_target_list = []
+        host_target_list = []
+        for t, v in self.target_list:
+            if v == "ALL":
+                skip_list_re = [
+                    re.compile(r"//{}:{}_.*_{}_dist".format(self.kernel_dir, t, s))
+                    for s in self.skip_list
+                ]
+                host_target_list_re = [
+                    re.compile(r"//{}:{}_.*_{}_dist".format(self.kernel_dir, t, h))
+                    for h in HOST_TARGETS
+                ]
+                query = 'filter("{}_.*_dist$", attr(generator_function, define_msm_platforms, {}/...))'.format(
+                    t, self.kernel_dir
+                )
+            else:
+                skip_list_re = [
+                    re.compile(r"//{}:{}_{}_{}_dist".format(self.kernel_dir, t, v, s))
+                    for s in self.skip_list
+                ]
+                host_target_list_re = [
+                    re.compile(r"//{}:{}_{}_{}_dist".format(self.kernel_dir, t, v, h))
+                    for h in HOST_TARGETS
+                ]
+                query = 'filter("{}_{}.*_dist$", attr(generator_function, define_msm_platforms, {}/...))'.format(
+                    t, v, self.kernel_dir
+                )
 
-def run_targets(targets, user_opts=[], us_cross_toolchain=None):
+            cmdline = [
-    # bazel cannot _run_ multiple targets at once, so run them in serial
+                self.bazel_bin,
-    for target in targets:
+                "query",
-        bazel("run", [target], user_opts, us_cross_toolchain)
+                "--ui_event_filters=-info",
+                "--noshow_progress",
+                query,
+            ]
 
+            logging.debug('Running "%s"', " ".join(cmdline))
 
-def build(
+            try:
-    target_list,
+                query_cmd = subprocess.Popen(
-    user_opts=[],
+                    cmdline, cwd=self.workspace, stdout=subprocess.PIPE
-    us_cross_toolchain=None,
+                )
-    extra_targets=[],
+                self.process_list.append(query_cmd)
-    skip_kernel=False,
+                target_list = query_cmd.stdout.read().splitlines()
-    skip_abl=False,
+            except Exception as e:
-    skip_host_tools=False,
+                logging.error(e)
-    **ignored
+                sys.exit(1)
-):
+
-    msm_targets = ["//msm-kernel:{}_{}".format(t, v) for t, v in target_list]
+            self.process_list.remove(query_cmd)
-    extra_msm_targets = []
+
-    for extra in extra_targets:
+            if not target_list:
-        extra_msm_targets.extend(
+                logging.error(
-            ["//msm-kernel:{}_{}_{}".format(t, v, extra) for t, v in target_list]
+                    "failed to find any Bazel targets for target/variant combo %s_%s",
+                    t,
+                    v,
+                )
+                sys.exit(1)
+
+            for target in target_list:
+                if any((skip_re.match(target) for skip_re in skip_list_re)):
+                    continue
+                if any((host_re.match(target) for host_re in host_target_list_re)):
+                    host_target_list.append(target)
+                else:
+                    cross_target_list.append(target)
+
+        return (cross_target_list, host_target_list)
+
+    def get_cross_toolchain(self):
+        """Query for a custom toolchain if one is defined"""
+        logging.info("Querying toolchains...")
+        query = 'filter("crosstool_suite", {}/...)'.format(self.kernel_dir)
+        cmdline = [
+            self.bazel_bin,
+            "query",
+            "--ui_event_filters=-info",
+            "--noshow_progress",
+            query,
+        ]
+
+        logging.debug('Running "%s"', " ".join(cmdline))
+
+        try:
+            query_cmd = subprocess.Popen(
+                cmdline, cwd=self.workspace, stdout=subprocess.PIPE
+            )
+            self.process_list.append(query_cmd)
+            toolchain = query_cmd.stdout.read().strip()
+        except Exception as e:
+            logging.error(e)
+            sys.exit(1)
+
+        self.process_list.remove(query_cmd)
+
+        if not toolchain:
+            logging.debug("no userspace cross toolchain found")
+            return None
+
+        logging.debug("using userspace cross toolchain %s", toolchain)
+        return toolchain
+
+    def bazel(
+        self, bazel_subcommand, targets, extra_options=None, us_cross_toolchain=None
+    ):
+        """Execute a bazel command"""
+        cmdline = [self.bazel_bin, bazel_subcommand]
+        if extra_options:
+            cmdline.extend(extra_options)
+        if us_cross_toolchain:
+            cmdline.extend(self.get_cross_cli_opts(us_cross_toolchain))
+        cmdline.extend(targets)
+        cmdline_str = " ".join(cmdline)
+        try:
+            logging.info('Running "%s"', cmdline_str)
+            build_proc = subprocess.Popen(cmdline_str, cwd=self.workspace, shell=True)
+            self.process_list.append(build_proc)
+            build_proc.wait()
+            if build_proc.returncode != 0:
+                sys.exit(build_proc.returncode)
+        except Exception as e:
+            logging.error(e)
+            sys.exit(1)
+
+        self.process_list.remove(build_proc)
+
+    def build_targets(self, targets, user_opts=None, us_cross_toolchain=None):
+        """Run "bazel build" on all targets in parallel"""
+        if not targets:
+            logging.warning("no targets to build")
+        self.bazel("build", targets, user_opts, us_cross_toolchain)
+
+    def run_targets(self, targets, user_opts=None, us_cross_toolchain=None):
+        """Run "bazel run" on all targets in serial (since bazel run cannot have multiple targets)"""
+        for target in targets:
+            self.bazel("run", [target], user_opts, us_cross_toolchain)
+
+    def build(self):
+        """Determine which targets to build, then build them"""
+        cross_targets_to_build, host_targets_to_build = self.get_build_targets()
+
+        logging.debug(
+            "Building the following %s targets:\n%s",
+            CPU,
+            "\n".join(cross_targets_to_build),
+        )
+        logging.debug(
+            "Building the following host targets:\n%s", "\n".join(host_targets_to_build)
         )
 
-    targets_to_build = []
+        us_cross_toolchain = self.get_cross_toolchain()
-    host_targets_to_build = []
-    for t in msm_targets:
-        logging.info(
-            "skipping kernel build"
-        ) if skip_kernel else targets_to_build.append(t)
-        logging.info("skipping ABL build") if skip_abl else targets_to_build.append(
-            t + "_abl"
-        )
-        logging.info(
-            "skipping host tools build"
-        ) if skip_host_tools else host_targets_to_build.append(t + "_dtc")
 
-    targets_to_build.append(t + "_test_mapping")
+        if not cross_targets_to_build and not host_targets_to_build:
+            logging.error("no targets to build")
+            sys.exit(1)
 
-    targets_to_build += extra_msm_targets
+        logging.info("Building %s targets...", CPU)
+        self.build_targets(cross_targets_to_build, self.user_opts, us_cross_toolchain)
+        self.run_targets(cross_targets_to_build, self.user_opts, us_cross_toolchain)
 
-    if not targets_to_build:
+        logging.info("Building host targets...")
-        logging.error("no targets to build")
+        self.build_targets(host_targets_to_build, self.user_opts)
-        sys.exit(1)
+        self.run_targets(host_targets_to_build, self.user_opts)
-
-    logging.info("Building {} targets...".format(CPU))
-    build_targets(targets_to_build, user_opts, us_cross_toolchain)
-    run_targets([t + "_dist" for t in targets_to_build], user_opts, us_cross_toolchain)
-
-    logging.info("Building host targets...")
-    build_targets(host_targets_to_build, user_opts)
-    run_targets([t + "_dist" for t in host_targets_to_build], user_opts)
 
 
 def main():
+    """Main script entrypoint"""
     parser = argparse.ArgumentParser(description="Build kernel platform with Bazel")
 
     parser.add_argument(
@@ -108,47 +227,22 @@ def main():
         action="append",
         nargs=2,
         required=True,
-        help="Target and variant to build (e.g. -t kalama gki). May be passed multiple times.",
+        help='Target and variant to build (e.g. -t kalama gki). May be passed multiple times. A special VARIANT may be passed, "ALL", which will build all variants for a particular target',
     )
     parser.add_argument(
-        "-u",
+        "-s",
-        "--userspace_cross_toolchain",
+        "--skip",
-        default=None,
+        metavar="BUILD_RULE",
-        metavar="TOOLCHAIN",
-        help="Set a Bazel toolchain for cross-compiled userspace programs",
-    )
-    parser.add_argument(
-        "-e",
-        "--extra_build_target",
-        metavar="BUILD_TARGET",
         action="append",
         default=[],
-        help="""
+        help="Skip specific build rules (e.g. --skip abl will skip the //msm-kernel:<target>_<variant>_abl build)",
-           Additional Bazel target to build and run dist. For example, "-t kalama gki --extra_target foo"
-           would run "bazel build //msm-kernel:kalama_gki_foo && bazel run //msms-kernel:kalama_gki_foo_dist".
-           May be passed multiple times.
-        """,
-    )
-    parser.add_argument(
-        "--skip_kernel",
-        action="store_true",
-        help="Skip building the kernel (note: may still need to build kernel if building ABL or tests)",
-    )
-    parser.add_argument(
-        "--skip_abl",
-        action="store_true",
-        help="Skip building the Android Boot Loader (ABL)",
-    )
-    parser.add_argument(
-        "--skip_host_tools",
-        action="store_true",
-        help="Skip building host tools (DTC, etc.)",
     )
     parser.add_argument(
         "--log",
-        help="Log level (debug, info, warning, error)",
+        metavar="LEVEL",
-        default="debug",
+        default="info",
         choices=["debug", "info", "warning", "error"],
+        help="Log level (debug, info, warning, error)",
     )
 
     args, user_opts = parser.parse_known_args(sys.argv[1:])
@@ -158,15 +252,15 @@ def main():
         format="[{}] %(levelname)s: %(message)s".format(os.path.basename(sys.argv[0])),
     )
 
-    extra_targets = list(args.extra_build_target)
+    args.skip.extend(DEFAULT_SKIP_LIST)
 
-    build(
+    builder = BazelBuilder(args.target, args.skip, user_opts)
-        args.target,
+    try:
-        user_opts,
+        builder.build()
-        args.userspace_cross_toolchain,
+    except KeyboardInterrupt:
-        extra_targets,
+        logging.info("Received keyboard interrupt... exiting")
-        **vars(args)
+        del builder
-    )
+        sys.exit(1)
 
     logging.info("Build completed successfully!")