1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
|
// Copyright 2020 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package build
// This file contains the functionality to upload data from one location to
// another.
import (
"io/ioutil"
"os"
"path/filepath"
"time"
"android/soong/ui/metrics"
"github.com/golang/protobuf/proto"
upload_proto "android/soong/ui/metrics/upload_proto"
)
const (
// Used to generate a raw protobuf file that contains information
// of the list of metrics files from host to destination storage.
uploadPbFilename = ".uploader.pb"
)
var (
// For testing purpose.
tmpDir = ioutil.TempDir
)
// pruneMetricsFiles iterates the list of paths, checking if a path exist.
// If a path is a file, it is added to the return list. If the path is a
// directory, a recursive call is made to add the children files of the
// path.
func pruneMetricsFiles(paths []string) []string {
var metricsFiles []string
for _, p := range paths {
fi, err := os.Stat(p)
// Some paths passed may not exist. For example, build errors protobuf
// file may not exist since the build was successful.
if err != nil {
continue
}
if fi.IsDir() {
if l, err := ioutil.ReadDir(p); err == nil {
files := make([]string, 0, len(l))
for _, fi := range l {
files = append(files, filepath.Join(p, fi.Name()))
}
metricsFiles = append(metricsFiles, pruneMetricsFiles(files)...)
}
} else {
metricsFiles = append(metricsFiles, p)
}
}
return metricsFiles
}
// UploadMetrics uploads a set of metrics files to a server for analysis. An
// uploader full path is specified in ANDROID_ENABLE_METRICS_UPLOAD environment
// variable in order to upload the set of metrics files. The metrics files are
// first copied to a temporary directory and the uploader is then executed in
// the background to allow the user/system to continue working. Soong communicates
// to the uploader through the upload_proto raw protobuf file.
func UploadMetrics(ctx Context, config Config, simpleOutput bool, buildStarted time.Time, paths ...string) {
ctx.BeginTrace(metrics.RunSetupTool, "upload_metrics")
defer ctx.EndTrace()
uploader := config.MetricsUploaderApp()
if uploader == "" {
// If the uploader path was not specified, no metrics shall be uploaded.
return
}
// Several of the files might be directories.
metricsFiles := pruneMetricsFiles(paths)
if len(metricsFiles) == 0 {
return
}
// The temporary directory cannot be deleted as the metrics uploader is started
// in the background and requires to exist until the operation is done. The
// uploader can delete the directory as it is specified in the upload proto.
tmpDir, err := tmpDir("", "upload_metrics")
if err != nil {
ctx.Fatalf("failed to create a temporary directory to store the list of metrics files: %v\n", err)
}
for i, src := range metricsFiles {
dst := filepath.Join(tmpDir, filepath.Base(src))
if _, err := copyFile(src, dst); err != nil {
ctx.Fatalf("failed to copy %q to %q: %v\n", src, dst, err)
}
metricsFiles[i] = dst
}
// For platform builds, the branch and target name is hardcoded to specific
// values for later extraction of the metrics in the data metrics pipeline.
data, err := proto.Marshal(&upload_proto.Upload{
CreationTimestampMs: proto.Uint64(uint64(buildStarted.UnixNano() / int64(time.Millisecond))),
CompletionTimestampMs: proto.Uint64(uint64(time.Now().UnixNano() / int64(time.Millisecond))),
BranchName: proto.String("developer-metrics"),
TargetName: proto.String("platform-build-systems-metrics"),
MetricsFiles: metricsFiles,
DirectoriesToDelete: []string{tmpDir},
})
if err != nil {
ctx.Fatalf("failed to marshal metrics upload proto buffer message: %v\n", err)
}
pbFile := filepath.Join(tmpDir, uploadPbFilename)
if err := ioutil.WriteFile(pbFile, data, 0644); err != nil {
ctx.Fatalf("failed to write the marshaled metrics upload protobuf to %q: %v\n", pbFile, err)
}
// Start the uploader in the background as it takes several milliseconds to start the uploader
// and prepare the metrics for upload. This affects small shell commands like "lunch".
cmd := Command(ctx, config, "upload metrics", uploader, "--upload-metrics", pbFile)
if simpleOutput {
cmd.RunOrFatal()
} else {
cmd.RunAndStreamOrFatal()
}
}
|
