Embed minibp into soong_ui.

This requires linking Blueprint into soong_ui. It lets us avoid the
complicated dance of Ninja files and shell scripts: now the information
as to how soong_build is built is passed directly to Blueprint using a
struct that contains all the information the command line arguments used
to contain.

The ability to run Blueprint from the command line is kept (for now).

Some variables in bootstrap/command.go needed public accessor functions
because soong_build reads them. This will be disentangled by moving the
flag parsing to soong_build.

The presence of the flag definitions in Blueprint means that soong_ui
now also accepts them. This is not a problem in practice because they
are ignored and because soong_ui itself is hidden behind a few layers of
shell scripts.

Test: Presubmits + the new bootstrap_test.sh .
Change-Id: I6dca478f356f56a8aee1e457d71439272351390b

5 files changed, 96 insertions, 49 deletions

diff --git a/ui/build/Android.bp b/ui/build/Android.bp
index 32b6edade..d17b4645c 100644
--- a/ui/build/Android.bp
+++ b/ui/build/Android.bp
@@ -32,6 +32,8 @@ bootstrap_go_package {
     name: "soong-ui-build",
     pkgPath: "android/soong/ui/build",
     deps: [
+        "blueprint",
+        "blueprint-bootstrap",
         "soong-ui-build-paths",
         "soong-ui-logger",
         "soong-ui-metrics",
diff --git a/ui/build/build.go b/ui/build/build.go
index 215a6c8ce..3692f4fb4 100644
--- a/ui/build/build.go
+++ b/ui/build/build.go
@@ -218,6 +218,11 @@ func Build(ctx Context, config Config, what int) {
 		what = what &^ BuildKati
 	}
 
+	if config.SkipNinja() {
+		ctx.Verboseln("Skipping Ninja as requested")
+		what = what &^ BuildNinja
+	}
+
 	if config.StartGoma() {
 		// Ensure start Goma compiler_proxy
 		startGoma(ctx, config)
@@ -290,7 +295,7 @@ func Build(ctx Context, config Config, what int) {
 		}
 
 		// Run ninja
-		runNinja(ctx, config)
+		runNinjaForBuild(ctx, config)
 	}
 
 	// Currently, using Bazel requires Kati and Soong to run first, so check whether to run Bazel last.
diff --git a/ui/build/config.go b/ui/build/config.go
index 1152cd790..4816d1f15 100644
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -48,6 +48,7 @@ type configImpl struct {
 	dist           bool
 	skipConfig     bool
 	skipKati       bool
+	skipNinja      bool
 	skipSoongTests bool
 
 	// From the product config
@@ -552,6 +553,8 @@ func (c *configImpl) parseArgs(ctx Context, args []string) {
 		if arg == "--make-mode" {
 		} else if arg == "showcommands" {
 			c.verbose = true
+		} else if arg == "--skip-ninja" {
+			c.skipNinja = true
 		} else if arg == "--skip-make" {
 			c.skipConfig = true
 			c.skipKati = true
@@ -772,6 +775,10 @@ func (c *configImpl) SkipKati() bool {
 	return c.skipKati
 }
 
+func (c *configImpl) SkipNinja() bool {
+	return c.skipNinja
+}
+
 func (c *configImpl) SkipConfig() bool {
 	return c.skipConfig
 }
diff --git a/ui/build/ninja.go b/ui/build/ninja.go
index 779976653..893fd6db8 100644
--- a/ui/build/ninja.go
+++ b/ui/build/ninja.go
@@ -30,7 +30,7 @@ import (
 // Constructs and runs the Ninja command line with a restricted set of
 // environment variables. It's important to restrict the environment Ninja runs
 // for hermeticity reasons, and to avoid spurious rebuilds.
-func runNinja(ctx Context, config Config) {
+func runNinjaForBuild(ctx Context, config Config) {
 	ctx.BeginTrace(metrics.PrimaryNinja, "ninja")
 	defer ctx.EndTrace()
 
diff --git a/ui/build/soong.go b/ui/build/soong.go
index 884e95741..a8a65b1ea 100644
--- a/ui/build/soong.go
+++ b/ui/build/soong.go
@@ -23,6 +23,8 @@ import (
 	"android/soong/shared"
 
 	soong_metrics_proto "android/soong/ui/metrics/metrics_proto"
+	"github.com/google/blueprint"
+	"github.com/google/blueprint/bootstrap"
 
 	"github.com/golang/protobuf/proto"
 	"github.com/google/blueprint/microfactory"
@@ -42,17 +44,75 @@ func writeEnvironmentFile(ctx Context, envFile string, envDeps map[string]string
 
 // This uses Android.bp files and various tools to generate <builddir>/build.ninja.
 //
-// However, the execution of <builddir>/build.ninja happens later in build/soong/ui/build/build.go#Build()
+// However, the execution of <builddir>/build.ninja happens later in
+// build/soong/ui/build/build.go#Build()
 //
-// We want to rely on as few prebuilts as possible, so there is some bootstrapping here.
+// We want to rely on as few prebuilts as possible, so we need to bootstrap
+// Soong. The process is as follows:
 //
-// "Microfactory" is a tool for compiling Go code. We use it to build two other tools:
-// - minibp, used to generate build.ninja files. This is really build/blueprint/bootstrap/command.go#Main()
-// - bpglob, used during incremental builds to identify files in a glob that have changed
-//
-// In reality, several build.ninja files are generated and/or used during the bootstrapping and build process.
-// See build/blueprint/bootstrap/doc.go for more information.
+// 1. We use "Microfactory", a simple tool to compile Go code, to build
+//    first itself, then soong_ui from soong_ui.bash. This binary contains
+//    parts of soong_build that are needed to build itself.
+// 2. This simplified version of soong_build then reads the Blueprint files
+//    that describe itself and emits .bootstrap/build.ninja that describes
+//    how to build its full version and use that to produce the final Ninja
+//    file Soong emits.
+// 3. soong_ui executes .bootstrap/build.ninja
 //
+// (After this, Kati is executed to parse the Makefiles, but that's not part of
+// bootstrapping Soong)
+
+// A tiny struct used to tell Blueprint that it's in bootstrap mode. It would
+// probably be nicer to use a flag in bootstrap.Args instead.
+type BlueprintConfig struct {
+	srcDir        string
+	buildDir      string
+	ninjaBuildDir string
+}
+
+func (c BlueprintConfig) GeneratingPrimaryBuilder() bool {
+	return true
+}
+
+func (c BlueprintConfig) SrcDir() string {
+	return "."
+}
+
+func (c BlueprintConfig) BuildDir() string {
+	return c.buildDir
+}
+
+func (c BlueprintConfig) NinjaBuildDir() string {
+	return c.ninjaBuildDir
+}
+
+func bootstrapBlueprint(ctx Context, config Config) {
+	ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
+	defer ctx.EndTrace()
+
+	var args bootstrap.Args
+
+	args.RunGoTests = !config.skipSoongTests
+	args.UseValidations = true // Use validations to depend on tests
+	args.BuildDir = config.SoongOutDir()
+	args.NinjaBuildDir = config.OutDir()
+	args.TopFile = "Android.bp"
+	args.ModuleListFile = filepath.Join(config.FileListDir(), "Android.bp.list")
+	args.OutFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja")
+	args.DepFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/build.ninja.d")
+	args.GlobFile = shared.JoinPath(config.SoongOutDir(), ".bootstrap/soong-build-globs.ninja")
+
+	blueprintCtx := blueprint.NewContext()
+	blueprintCtx.SetIgnoreUnknownModuleTypes(true)
+	blueprintConfig := BlueprintConfig{
+		srcDir:        os.Getenv("TOP"),
+		buildDir:      config.SoongOutDir(),
+		ninjaBuildDir: config.OutDir(),
+	}
+
+	bootstrap.RunBlueprint(args, blueprintCtx, blueprintConfig)
+}
+
 func runSoong(ctx Context, config Config) {
 	ctx.BeginTrace(metrics.RunSoong, "soong")
 	defer ctx.EndTrace()
@@ -63,33 +123,15 @@ func runSoong(ctx Context, config Config) {
 	// unused variables were changed?
 	envFile := filepath.Join(config.SoongOutDir(), "soong.environment.available")
 
-	// Use an anonymous inline function for tracing purposes (this pattern is used several times below).
-	func() {
-		ctx.BeginTrace(metrics.RunSoong, "blueprint bootstrap")
-		defer ctx.EndTrace()
-
-		// Use validations to depend on tests.
-		args := []string{"-n"}
-
-		if !config.skipSoongTests {
-			// Run tests.
-			args = append(args, "-t")
+	for _, n := range []string{".bootstrap", ".minibootstrap"} {
+		dir := filepath.Join(config.SoongOutDir(), n)
+		if err := os.MkdirAll(dir, 0755); err != nil {
+			ctx.Fatalf("Cannot mkdir " + dir)
 		}
+	}
 
-		cmd := Command(ctx, config, "blueprint bootstrap", "build/blueprint/bootstrap.bash", args...)
-
-		cmd.Environment.Set("BLUEPRINTDIR", "./build/blueprint")
-		cmd.Environment.Set("BOOTSTRAP", "./build/blueprint/bootstrap.bash")
-		cmd.Environment.Set("BUILDDIR", config.SoongOutDir())
-		cmd.Environment.Set("GOROOT", "./"+filepath.Join("prebuilts/go", config.HostPrebuiltTag()))
-		cmd.Environment.Set("BLUEPRINT_LIST_FILE", filepath.Join(config.FileListDir(), "Android.bp.list"))
-		cmd.Environment.Set("NINJA_BUILDDIR", config.OutDir())
-		cmd.Environment.Set("SRCDIR", ".")
-		cmd.Environment.Set("TOPNAME", "Android.bp")
-		cmd.Sandbox = soongSandbox
-
-		cmd.RunAndPrintOrFatal()
-	}()
+	// This is done unconditionally, but does not take a measurable amount of time
+	bootstrapBlueprint(ctx, config)
 
 	soongBuildEnv := config.Environment().Copy()
 	soongBuildEnv.Set("TOP", os.Getenv("TOP"))
@@ -105,6 +147,11 @@ func runSoong(ctx Context, config Config) {
 	soongBuildEnv.Set("BAZEL_WORKSPACE", absPath(ctx, "."))
 	soongBuildEnv.Set("BAZEL_METRICS_DIR", config.BazelMetricsDir())
 
+	// For Soong bootstrapping tests
+	if os.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" {
+		soongBuildEnv.Set("ALLOW_MISSING_DEPENDENCIES", "true")
+	}
+
 	err := writeEnvironmentFile(ctx, envFile, soongBuildEnv.AsMap())
 	if err != nil {
 		ctx.Fatalf("failed to write environment file %s: %s", envFile, err)
@@ -130,16 +177,6 @@ func runSoong(ctx Context, config Config) {
 	cfg.TrimPath = absPath(ctx, ".")
 
 	func() {
-		ctx.BeginTrace(metrics.RunSoong, "minibp")
-		defer ctx.EndTrace()
-
-		minibp := filepath.Join(config.SoongOutDir(), ".minibootstrap/minibp")
-		if _, err := microfactory.Build(&cfg, minibp, "github.com/google/blueprint/bootstrap/minibp"); err != nil {
-			ctx.Fatalln("Failed to build minibp:", err)
-		}
-	}()
-
-	func() {
 		ctx.BeginTrace(metrics.RunSoong, "bpglob")
 		defer ctx.EndTrace()
 
@@ -187,10 +224,6 @@ func runSoong(ctx Context, config Config) {
 		cmd.Sandbox = soongSandbox
 		cmd.RunAndStreamOrFatal()
 	}
-
-	// This build generates .bootstrap/build.ninja, which is used in the next step.
-	ninja("minibootstrap", ".minibootstrap/build.ninja")
-
 	// This build generates <builddir>/build.ninja, which is used later by build/soong/ui/build/build.go#Build().
 	ninja("bootstrap", ".bootstrap/build.ninja")