summaryrefslogtreecommitdiff
path: root/python/python.go
diff options
context:
space:
mode:
Diffstat (limited to 'python/python.go')
-rw-r--r--python/python.go278
1 files changed, 186 insertions, 92 deletions
diff --git a/python/python.go b/python/python.go
index df5999d8a..d462af124 100644
--- a/python/python.go
+++ b/python/python.go
@@ -24,6 +24,7 @@ import (
"strings"
"github.com/google/blueprint"
+ "github.com/google/blueprint/proptools"
"android/soong/android"
)
@@ -35,58 +36,70 @@ func init() {
}
// the version properties that apply to python libraries and binaries.
-type PythonVersionProperties struct {
+type VersionProperties struct {
// true, if the module is required to be built with this version.
- Enabled *bool
-
- // if specified, common src files are converted to specific version with converter tool.
- // Converter bool
+ Enabled *bool `android:"arch_variant"`
// non-empty list of .py files under this strict Python version.
// srcs may reference the outputs of other modules that produce source files like genrule
// or filegroup using the syntax ":module".
- Srcs []string
+ Srcs []string `android:"arch_variant"`
+
+ // list of source files that should not be used to build the Python module.
+ // This is most useful in the arch/multilib variants to remove non-common files
+ Exclude_srcs []string `android:"arch_variant"`
// list of the Python libraries under this Python version.
- Libs []string
+ Libs []string `android:"arch_variant"`
+
+ // true, if the binary is required to be built with embedded launcher.
+ // TODO(nanzhang): Remove this flag when embedded Python3 is supported later.
+ Embedded_launcher *bool `android:"arch_variant"`
}
// properties that apply to python libraries and binaries.
-type PythonBaseModuleProperties struct {
+type BaseProperties struct {
// the package path prefix within the output artifact at which to place the source/data
// files of the current module.
// eg. Pkg_path = "a/b/c"; Other packages can reference this module by using
// (from a.b.c import ...) statement.
// if left unspecified, all the source/data files of current module are copied to
// "runfiles/" tree directory directly.
- Pkg_path string
+ Pkg_path string `android:"arch_variant"`
+
+ // true, if the Python module is used internally, eg, Python std libs.
+ Is_internal *bool `android:"arch_variant"`
// list of source (.py) files compatible both with Python2 and Python3 used to compile the
// Python module.
// srcs may reference the outputs of other modules that produce source files like genrule
// or filegroup using the syntax ":module".
// Srcs has to be non-empty.
- Srcs []string
+ Srcs []string `android:"arch_variant"`
+
+ // list of source files that should not be used to build the C/C++ module.
+ // This is most useful in the arch/multilib variants to remove non-common files
+ Exclude_srcs []string `android:"arch_variant"`
// list of files or filegroup modules that provide data that should be installed alongside
// the test. the file extension can be arbitrary except for (.py).
- Data []string
+ Data []string `android:"arch_variant"`
// list of the Python libraries compatible both with Python2 and Python3.
- Libs []string
+ Libs []string `android:"arch_variant"`
Version struct {
// all the "srcs" or Python dependencies that are to be used only for Python2.
- Py2 PythonVersionProperties
+ Py2 VersionProperties `android:"arch_variant"`
// all the "srcs" or Python dependencies that are to be used only for Python3.
- Py3 PythonVersionProperties
- }
+ Py3 VersionProperties `android:"arch_variant"`
+ } `android:"arch_variant"`
// the actual version each module uses after variations created.
// this property name is hidden from users' perspectives, and soong will populate it during
// runtime.
- ActualVersion string `blueprint:"mutated"`
+ Actual_version string `blueprint:"mutated"`
}
type pathMapping struct {
@@ -94,11 +107,21 @@ type pathMapping struct {
src android.Path
}
-type pythonBaseModule struct {
+type Module struct {
android.ModuleBase
- subModule PythonSubModule
- properties PythonBaseModuleProperties
+ properties BaseProperties
+
+ // initialize before calling Init
+ hod android.HostOrDeviceSupported
+ multilib android.Multilib
+
+ // the bootstrapper is used to bootstrap .par executable.
+ // bootstrapper might be nil (Python library module).
+ bootstrapper bootstrapper
+
+ // the installer might be nil.
+ installer installer
// the Python files of current module after expanding source dependencies.
// pathMapping: <dest: runfile_path, src: source_path>
@@ -108,83 +131,97 @@ type pythonBaseModule struct {
// pathMapping: <dest: runfile_path, src: source_path>
dataPathMappings []pathMapping
+ // soong_zip arguments of all its dependencies.
+ depsParSpecs []parSpec
+
+ // Python runfiles paths of all its dependencies.
+ depsPyRunfiles []string
+
+ // (.intermediate) module output path as installation source.
+ installSource android.OptionalPath
+
// the soong_zip arguments for zipping current module source/data files.
parSpec parSpec
- // the installer might be nil.
- installer installer
-
subAndroidMkOnce map[subAndroidMkProvider]bool
}
-type PythonSubModule interface {
- GeneratePythonBuildActions(ctx android.ModuleContext) android.OptionalPath
-}
-
-type PythonDependency interface {
- GetSrcsPathMappings() []pathMapping
- GetDataPathMappings() []pathMapping
- GetParSpec() parSpec
+func newModule(hod android.HostOrDeviceSupported, multilib android.Multilib) *Module {
+ return &Module{
+ hod: hod,
+ multilib: multilib,
+ }
}
-type pythonDecorator struct {
- baseInstaller *pythonInstaller
+type bootstrapper interface {
+ bootstrapperProps() []interface{}
+ bootstrap(ctx android.ModuleContext, Actual_version string, embedded_launcher bool,
+ srcsPathMappings []pathMapping, parSpec parSpec,
+ depsPyRunfiles []string, depsParSpecs []parSpec) android.OptionalPath
}
type installer interface {
install(ctx android.ModuleContext, path android.Path)
}
-func (p *pythonBaseModule) GetSrcsPathMappings() []pathMapping {
+type PythonDependency interface {
+ GetSrcsPathMappings() []pathMapping
+ GetDataPathMappings() []pathMapping
+ GetParSpec() parSpec
+}
+
+func (p *Module) GetSrcsPathMappings() []pathMapping {
return p.srcsPathMappings
}
-func (p *pythonBaseModule) GetDataPathMappings() []pathMapping {
+func (p *Module) GetDataPathMappings() []pathMapping {
return p.dataPathMappings
}
-func (p *pythonBaseModule) GetParSpec() parSpec {
+func (p *Module) GetParSpec() parSpec {
return p.parSpec
}
-var _ PythonDependency = (*pythonBaseModule)(nil)
+var _ PythonDependency = (*Module)(nil)
-var _ android.AndroidMkDataProvider = (*pythonBaseModule)(nil)
+var _ android.AndroidMkDataProvider = (*Module)(nil)
-func InitPythonBaseModule(baseModule *pythonBaseModule, subModule PythonSubModule,
- hod android.HostOrDeviceSupported) android.Module {
+func (p *Module) Init() android.Module {
- baseModule.subModule = subModule
-
- baseModule.AddProperties(&baseModule.properties)
+ p.AddProperties(&p.properties)
+ if p.bootstrapper != nil {
+ p.AddProperties(p.bootstrapper.bootstrapperProps()...)
+ }
- android.InitAndroidArchModule(baseModule, hod, android.MultilibCommon)
+ android.InitAndroidArchModule(p, p.hod, p.multilib)
- return baseModule
+ return p
}
-// the tag used to mark dependencies within "py_libs" attribute.
-type pythonDependencyTag struct {
+type dependencyTag struct {
blueprint.BaseDependencyTag
+ name string
}
-var pyDependencyTag pythonDependencyTag
-
var (
+ pythonLibTag = dependencyTag{name: "pythonLib"}
+ launcherTag = dependencyTag{name: "launcher"}
pyIdentifierRegexp = regexp.MustCompile(`^([a-z]|[A-Z]|_)([a-z]|[A-Z]|[0-9]|_)*$`)
pyExt = ".py"
pyVersion2 = "PY2"
pyVersion3 = "PY3"
initFileName = "__init__.py"
mainFileName = "__main__.py"
+ entryPointFile = "entry_point.txt"
parFileExt = ".zip"
runFiles = "runfiles"
+ internal = "internal"
)
// create version variants for modules.
func versionSplitMutator() func(android.BottomUpMutatorContext) {
return func(mctx android.BottomUpMutatorContext) {
- if base, ok := mctx.Module().(*pythonBaseModule); ok {
+ if base, ok := mctx.Module().(*Module); ok {
versionNames := []string{}
if base.properties.Version.Py2.Enabled != nil &&
*(base.properties.Version.Py2.Enabled) == true {
@@ -197,36 +234,61 @@ func versionSplitMutator() func(android.BottomUpMutatorContext) {
modules := mctx.CreateVariations(versionNames...)
for i, v := range versionNames {
// set the actual version for Python module.
- modules[i].(*pythonBaseModule).properties.ActualVersion = v
+ modules[i].(*Module).properties.Actual_version = v
}
}
}
}
-func (p *pythonBaseModule) DepsMutator(ctx android.BottomUpMutatorContext) {
+func (p *Module) isEmbeddedLauncherEnabled(actual_version string) bool {
+ switch actual_version {
+ case pyVersion2:
+ return proptools.Bool(p.properties.Version.Py2.Embedded_launcher)
+ case pyVersion3:
+ return proptools.Bool(p.properties.Version.Py3.Embedded_launcher)
+ }
+
+ return false
+}
+
+func (p *Module) DepsMutator(ctx android.BottomUpMutatorContext) {
// deps from "data".
android.ExtractSourcesDeps(ctx, p.properties.Data)
// deps from "srcs".
android.ExtractSourcesDeps(ctx, p.properties.Srcs)
- switch p.properties.ActualVersion {
+ switch p.properties.Actual_version {
case pyVersion2:
// deps from "version.py2.srcs" property.
android.ExtractSourcesDeps(ctx, p.properties.Version.Py2.Srcs)
- ctx.AddVariationDependencies(nil, pyDependencyTag,
+ ctx.AddVariationDependencies(nil, pythonLibTag,
uniqueLibs(ctx, p.properties.Libs, "version.py2.libs",
p.properties.Version.Py2.Libs)...)
+
+ if p.bootstrapper != nil && p.isEmbeddedLauncherEnabled(pyVersion2) {
+ ctx.AddVariationDependencies(nil, pythonLibTag, "py2-stdlib")
+ ctx.AddFarVariationDependencies([]blueprint.Variation{
+ {"arch", ctx.Target().String()},
+ }, launcherTag, "py2-launcher")
+ }
+
case pyVersion3:
// deps from "version.py3.srcs" property.
android.ExtractSourcesDeps(ctx, p.properties.Version.Py3.Srcs)
- ctx.AddVariationDependencies(nil, pyDependencyTag,
+ ctx.AddVariationDependencies(nil, pythonLibTag,
uniqueLibs(ctx, p.properties.Libs, "version.py3.libs",
p.properties.Version.Py3.Libs)...)
+
+ if p.bootstrapper != nil && p.isEmbeddedLauncherEnabled(pyVersion3) {
+ //TODO(nanzhang): Add embedded launcher for Python3.
+ ctx.PropertyErrorf("version.py3.embedded_launcher",
+ "is not supported yet for Python3.")
+ }
default:
- panic(fmt.Errorf("unknown Python actualVersion: %q for module: %q.",
- p.properties.ActualVersion, ctx.ModuleName()))
+ panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
+ p.properties.Actual_version, ctx.ModuleName()))
}
}
@@ -258,27 +320,43 @@ func uniqueLibs(ctx android.BottomUpMutatorContext,
return ret
}
-func (p *pythonBaseModule) GenerateAndroidBuildActions(ctx android.ModuleContext) {
- installSource := p.subModule.GeneratePythonBuildActions(ctx)
+func (p *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) {
+ p.GeneratePythonBuildActions(ctx)
- if p.installer != nil && installSource.Valid() {
- p.installer.install(ctx, installSource.Path())
+ if p.bootstrapper != nil {
+ // TODO(nanzhang): Since embedded launcher is not supported for Python3 for now,
+ // so we initialize "embedded_launcher" to false.
+ embedded_launcher := false
+ if p.properties.Actual_version == pyVersion2 {
+ embedded_launcher = p.isEmbeddedLauncherEnabled(pyVersion2)
+ }
+ p.installSource = p.bootstrapper.bootstrap(ctx, p.properties.Actual_version,
+ embedded_launcher, p.srcsPathMappings, p.parSpec, p.depsPyRunfiles,
+ p.depsParSpecs)
+ }
+
+ if p.installer != nil && p.installSource.Valid() {
+ p.installer.install(ctx, p.installSource.Path())
}
+
}
-func (p *pythonBaseModule) GeneratePythonBuildActions(ctx android.ModuleContext) android.OptionalPath {
+func (p *Module) GeneratePythonBuildActions(ctx android.ModuleContext) {
// expand python files from "srcs" property.
srcs := p.properties.Srcs
- switch p.properties.ActualVersion {
+ exclude_srcs := p.properties.Exclude_srcs
+ switch p.properties.Actual_version {
case pyVersion2:
srcs = append(srcs, p.properties.Version.Py2.Srcs...)
+ exclude_srcs = append(exclude_srcs, p.properties.Version.Py2.Exclude_srcs...)
case pyVersion3:
srcs = append(srcs, p.properties.Version.Py3.Srcs...)
+ exclude_srcs = append(exclude_srcs, p.properties.Version.Py3.Exclude_srcs...)
default:
- panic(fmt.Errorf("unknown Python actualVersion: %q for module: %q.",
- p.properties.ActualVersion, ctx.ModuleName()))
+ panic(fmt.Errorf("unknown Python Actual_version: %q for module: %q.",
+ p.properties.Actual_version, ctx.ModuleName()))
}
- expandedSrcs := ctx.ExpandSources(srcs, nil)
+ expandedSrcs := ctx.ExpandSources(srcs, exclude_srcs)
if len(expandedSrcs) == 0 {
ctx.ModuleErrorf("doesn't have any source files!")
}
@@ -292,15 +370,26 @@ func (p *pythonBaseModule) GeneratePythonBuildActions(ctx android.ModuleContext)
pkg_path = filepath.Clean(p.properties.Pkg_path)
if pkg_path == ".." || strings.HasPrefix(pkg_path, "../") ||
strings.HasPrefix(pkg_path, "/") {
- ctx.PropertyErrorf("pkg_path", "%q is not a valid format.",
+ ctx.PropertyErrorf("pkg_path",
+ "%q must be a relative path contained in par file.",
p.properties.Pkg_path)
- return android.OptionalPath{}
+ return
+ }
+ if p.properties.Is_internal != nil && *p.properties.Is_internal {
+ // pkg_path starts from "internal/" implicitly.
+ pkg_path = filepath.Join(internal, pkg_path)
+ } else {
+ // pkg_path starts from "runfiles/" implicitly.
+ pkg_path = filepath.Join(runFiles, pkg_path)
}
- // pkg_path starts from "runfiles/" implicitly.
- pkg_path = filepath.Join(runFiles, pkg_path)
} else {
- // pkg_path starts from "runfiles/" implicitly.
- pkg_path = runFiles
+ if p.properties.Is_internal != nil && *p.properties.Is_internal {
+ // pkg_path starts from "runfiles/" implicitly.
+ pkg_path = internal
+ } else {
+ // pkg_path starts from "runfiles/" implicitly.
+ pkg_path = runFiles
+ }
}
p.genModulePathMappings(ctx, pkg_path, expandedSrcs, expandedData)
@@ -308,13 +397,11 @@ func (p *pythonBaseModule) GeneratePythonBuildActions(ctx android.ModuleContext)
p.parSpec = p.dumpFileList(ctx, pkg_path)
p.uniqWholeRunfilesTree(ctx)
-
- return android.OptionalPath{}
}
// generate current module unique pathMappings: <dest: runfiles_path, src: source_path>
// for python/data files.
-func (p *pythonBaseModule) genModulePathMappings(ctx android.ModuleContext, pkg_path string,
+func (p *Module) genModulePathMappings(ctx android.ModuleContext, pkg_path string,
expandedSrcs, expandedData android.Paths) {
// fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
// check duplicates.
@@ -355,7 +442,7 @@ func (p *pythonBaseModule) genModulePathMappings(ctx android.ModuleContext, pkg_
}
// register build actions to dump filelist to disk.
-func (p *pythonBaseModule) dumpFileList(ctx android.ModuleContext, pkg_path string) parSpec {
+func (p *Module) dumpFileList(ctx android.ModuleContext, pkg_path string) parSpec {
relativeRootMap := make(map[string]android.Paths)
// the soong_zip params in order to pack current module's Python/data files.
ret := parSpec{rootPrefix: pkg_path}
@@ -365,7 +452,8 @@ func (p *pythonBaseModule) dumpFileList(ctx android.ModuleContext, pkg_path stri
// "srcs" or "data" properties may have filegroup so it might happen that
// the relative root for each source path is different.
for _, path := range pathMappings {
- relativeRoot := strings.TrimSuffix(path.src.String(), path.src.Rel())
+ var relativeRoot string
+ relativeRoot = strings.TrimSuffix(path.src.String(), path.src.Rel())
if v, found := relativeRootMap[relativeRoot]; found {
relativeRootMap[relativeRoot] = append(v, path.src)
} else {
@@ -392,8 +480,19 @@ func (p *pythonBaseModule) dumpFileList(ctx android.ModuleContext, pkg_path stri
return ret
}
-// check Python/data files duplicates from current module and its whole dependencies.
-func (p *pythonBaseModule) uniqWholeRunfilesTree(ctx android.ModuleContext) {
+func isPythonLibModule(module blueprint.Module) bool {
+ if m, ok := module.(*Module); ok {
+ // Python library has no bootstrapper or installer.
+ if m.bootstrapper != nil || m.installer != nil {
+ return false
+ }
+ return true
+ }
+ return false
+}
+
+// check Python source/data files duplicates from current module and its whole dependencies.
+func (p *Module) uniqWholeRunfilesTree(ctx android.ModuleContext) {
// fetch <runfiles_path, source_path> pairs from "src" and "data" properties to
// check duplicates.
destToPySrcs := make(map[string]string)
@@ -408,16 +507,15 @@ func (p *pythonBaseModule) uniqWholeRunfilesTree(ctx android.ModuleContext) {
// visit all its dependencies in depth first.
ctx.VisitDepsDepthFirst(func(module blueprint.Module) {
- // module can only depend on Python library.
- if base, ok := module.(*pythonBaseModule); ok {
- if _, ok := base.subModule.(*PythonLibrary); !ok {
- panic(fmt.Errorf(
- "the dependency %q of module %q is not Python library!",
- ctx.ModuleName(), ctx.OtherModuleName(module)))
- }
- } else {
+ if ctx.OtherModuleDependencyTag(module) != pythonLibTag {
return
}
+ // Python module cannot depend on modules, except for Python library.
+ if !isPythonLibModule(module) {
+ panic(fmt.Errorf(
+ "the dependency %q of module %q is not Python library!",
+ ctx.ModuleName(), ctx.OtherModuleName(module)))
+ }
if dep, ok := module.(PythonDependency); ok {
srcs := dep.GetSrcsPathMappings()
for _, path := range srcs {
@@ -428,9 +526,7 @@ func (p *pythonBaseModule) uniqWholeRunfilesTree(ctx android.ModuleContext) {
}
// binary needs the Python runfiles paths from all its
// dependencies to fill __init__.py in each runfiles dir.
- if sub, ok := p.subModule.(*pythonBinaryBase); ok {
- sub.depsPyRunfiles = append(sub.depsPyRunfiles, path.dest)
- }
+ p.depsPyRunfiles = append(p.depsPyRunfiles, path.dest)
}
data := dep.GetDataPathMappings()
for _, path := range data {
@@ -440,9 +536,7 @@ func (p *pythonBaseModule) uniqWholeRunfilesTree(ctx android.ModuleContext) {
}
// binary needs the soong_zip arguments from all its
// dependencies to generate executable par file.
- if sub, ok := p.subModule.(*pythonBinaryBase); ok {
- sub.depsParSpecs = append(sub.depsParSpecs, dep.GetParSpec())
- }
+ p.depsParSpecs = append(p.depsParSpecs, dep.GetParSpec())
}
})
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-03 20:22:19 +0000