Merge "Preparing to merge metalava build tasks" into rvc-dev

13 files changed, 783 insertions, 96 deletions

diff --git a/src/main/java/com/android/tools/metalava/ApiLint.kt b/src/main/java/com/android/tools/metalava/ApiLint.kt
index 821c9e2..61b84cb 100644
--- a/src/main/java/com/android/tools/metalava/ApiLint.kt
+++ b/src/main/java/com/android/tools/metalava/ApiLint.kt
@@ -165,7 +165,7 @@ import java.util.function.Predicate
  * The [ApiLint] analyzer checks the API against a known set of preferred API practices
  * by the Android API council.
  */
-class ApiLint(private val codebase: Codebase, private val oldCodebase: Codebase?) : ApiVisitor(
+class ApiLint(private val codebase: Codebase, private val oldCodebase: Codebase?, private val reporter: Reporter) : ApiVisitor(
     // Sort by source order such that warnings follow source line number order
     methodComparator = MethodItem.sourceOrderComparator,
     fieldComparator = FieldItem.comparator,
@@ -239,7 +239,7 @@ class ApiLint(private val codebase: Codebase, private val oldCodebase: Codebase?
 
     // The previous Kotlin interop tests are also part of API lint now (though they can be
     // run independently as well; therefore, only run them here if not running separately)
-    private val kotlinInterop = if (!options.checkKotlinInterop) KotlinInteropChecks() else null
+    private val kotlinInterop = if (!options.checkKotlinInterop) KotlinInteropChecks(reporter) else null
 
     override fun visitClass(cls: ClassItem) {
         val methods = cls.filteredMethods(filterReference).asSequence()
@@ -3634,8 +3634,8 @@ class ApiLint(private val codebase: Codebase, private val oldCodebase: Codebase?
             }
         }
 
-        fun check(codebase: Codebase, oldCodebase: Codebase?) {
-            ApiLint(codebase, oldCodebase).check()
+        fun check(codebase: Codebase, oldCodebase: Codebase?, reporter: Reporter) {
+            ApiLint(codebase, oldCodebase, reporter).check()
         }
     }
 }
diff --git a/src/main/java/com/android/tools/metalava/Baseline.kt b/src/main/java/com/android/tools/metalava/Baseline.kt
index 8444d08..2ad1638 100644
--- a/src/main/java/com/android/tools/metalava/Baseline.kt
+++ b/src/main/java/com/android/tools/metalava/Baseline.kt
@@ -40,6 +40,8 @@ import kotlin.text.Charsets.UTF_8
 const val DEFAULT_BASELINE_NAME = "baseline.txt"
 
 class Baseline(
+    /** Description of this baseline. e.g. "api-lint. */
+    val description: String,
     val file: File?,
     var updateFile: File?,
     var merge: Boolean = false,
@@ -176,8 +178,9 @@ class Baseline(
         return path.replace('\\', '/')
     }
 
-    fun close() {
-        write()
+    /** Close the baseline file. If "update file" is set, update this file, and returns TRUE. If not, returns false. */
+    fun close(): Boolean {
+        return write()
     }
 
     private fun read() {
@@ -217,8 +220,8 @@ class Baseline(
         }
     }
 
-    private fun write() {
-        val updateFile = this.updateFile ?: return
+    private fun write(): Boolean {
+        val updateFile = this.updateFile ?: return false
         if (!map.isEmpty() || !options.deleteEmptyBaselines) {
             val sb = StringBuilder()
             sb.append(format.header())
@@ -245,6 +248,7 @@ class Baseline(
         } else {
             updateFile.delete()
         }
+        return true
     }
 
     fun dumpStats(writer: PrintWriter) {
@@ -255,7 +259,7 @@ class Baseline(
             counts[issue] = count
         }
 
-        writer.println("Baseline issue type counts:")
+        writer.println("Baseline issue type counts for $description baseline:")
         writer.println("" +
             "    Count Issue Id                       Severity\n" +
             "    ---------------------------------------------\n")
@@ -273,4 +277,42 @@ class Baseline(
             "    ${String.format("%5d", total)}")
         writer.println()
     }
+
+    /**
+     * Builder for [Baseline]. [build] will return a non-null [Baseline] if either [file] or
+     * [updateFile] is set.
+     */
+    class Builder {
+        var description: String = ""
+
+        var file: File? = null
+            set(value) {
+                if (field != null) {
+                    throw DriverException("Only one baseline is allowed; found both $field and $value")
+                }
+                field = value
+            }
+        var merge: Boolean = false
+
+        var updateFile: File? = null
+            set(value) {
+                if (field != null) {
+                    throw DriverException("Only one update-baseline is allowed; found both $field and $value")
+                }
+                field = value
+            }
+
+        var headerComment: String = ""
+
+        fun build(): Baseline? {
+            // If neither file nor updateFile is set, don't return an instance.
+            if (file == null && updateFile == null) {
+                return null
+            }
+            if (description.isEmpty()) {
+                throw DriverException("Baseline description must be set")
+            }
+            return Baseline(description, file, updateFile, merge, headerComment)
+        }
+    }
 }
diff --git a/src/main/java/com/android/tools/metalava/CompatibilityCheck.kt b/src/main/java/com/android/tools/metalava/CompatibilityCheck.kt
index 6292dce..d10c684 100644
--- a/src/main/java/com/android/tools/metalava/CompatibilityCheck.kt
+++ b/src/main/java/com/android/tools/metalava/CompatibilityCheck.kt
@@ -48,7 +48,8 @@ class CompatibilityCheck(
     val filterReference: Predicate<Item>,
     private val oldCodebase: Codebase,
     private val apiType: ApiType,
-    private val base: Codebase? = null
+    private val base: Codebase? = null,
+    private val reporter: Reporter
 ) : ComparisonVisitor() {
 
     /**
@@ -868,7 +869,7 @@ class CompatibilityCheck(
             base: Codebase? = null
         ) {
             val filter = apiType.getEmitFilter()
-            val checker = CompatibilityCheck(filter, previous, apiType, base)
+            val checker = CompatibilityCheck(filter, previous, apiType, base, getReporterForReleaseType(releaseType))
             val issueConfiguration = releaseType.getIssueConfiguration()
             val previousConfiguration = configuration
             try {
@@ -885,5 +886,10 @@ class CompatibilityCheck(
                 throw DriverException(exitCode = -1, stderr = message)
             }
         }
+
+        private fun getReporterForReleaseType(releaseType: ReleaseType): Reporter = when (releaseType) {
+            ReleaseType.DEV -> options.reporterCompatibilityCurrent
+            ReleaseType.RELEASED -> options.reporterCompatibilityReleased
+        }
     }
 }
diff --git a/src/main/java/com/android/tools/metalava/Driver.kt b/src/main/java/com/android/tools/metalava/Driver.kt
index 6053324..5159cba 100644
--- a/src/main/java/com/android/tools/metalava/Driver.kt
+++ b/src/main/java/com/android/tools/metalava/Driver.kt
@@ -114,7 +114,7 @@ fun run(
 
         processFlags()
 
-        if (reporter.hasErrors() && !options.passBaselineUpdates) {
+        if (options.allReporters.any { it.hasErrors() } && !options.passBaselineUpdates) {
             exitCode = -1
         }
         exitValue = true
@@ -133,17 +133,25 @@ fun run(
         Disposer.dispose(LintCoreApplicationEnvironment.get().parentDisposable)
     }
 
-    if (options.updateBaseline) {
+    // Update and close all baseline files.
+    options.allBaselines.forEach { baseline ->
         if (options.verbose) {
-            options.baseline?.dumpStats(options.stdout)
+            baseline.dumpStats(options.stdout)
         }
-        if (!options.quiet) {
-            stdout.println("$PROGRAM_NAME wrote updated baseline to ${options.baseline?.updateFile}")
+        if (baseline.close()) {
+            if (!options.quiet) {
+                stdout.println("$PROGRAM_NAME wrote updated baseline to ${baseline.updateFile}")
+            }
         }
     }
-    options.baseline?.close()
+
     options.reportEvenIfSuppressedWriter?.close()
 
+    // Show failure messages, if any.
+    options.allReporters.forEach {
+        it.writeErrorMessage(stderr)
+    }
+
     stdout.flush()
     stderr.flush()
 
@@ -794,8 +802,10 @@ private fun loadFromSources(): Codebase {
     options.nullabilityAnnotationsValidator?.report()
     analyzer.handleStripping()
 
+    val apiLintReporter = options.reporterApiLint
+
     if (options.checkKotlinInterop) {
-        KotlinInteropChecks().check(codebase)
+        KotlinInteropChecks(apiLintReporter).check(codebase)
     }
 
     // General API checks for Android APIs
@@ -815,8 +825,8 @@ private fun loadFromSources(): Codebase {
                     kotlinStyleNulls = options.inputKotlinStyleNulls
                 )
             }
-        ApiLint.check(codebase, previous)
-        progress("$PROGRAM_NAME ran api-lint in ${localTimer.elapsed(SECONDS)} seconds")
+        ApiLint.check(codebase, previous, apiLintReporter)
+        progress("$PROGRAM_NAME ran api-lint in ${localTimer.elapsed(SECONDS)} seconds with ${apiLintReporter.getBaselineDescription()}")
     }
 
     // Compute default constructors (and add missing package private constructors
diff --git a/src/main/java/com/android/tools/metalava/KotlinInteropChecks.kt b/src/main/java/com/android/tools/metalava/KotlinInteropChecks.kt
index 8c37f34..50a49ad 100644
--- a/src/main/java/com/android/tools/metalava/KotlinInteropChecks.kt
+++ b/src/main/java/com/android/tools/metalava/KotlinInteropChecks.kt
@@ -36,7 +36,7 @@ import org.jetbrains.uast.kotlin.KotlinUField
 //   https://android.github.io/kotlin-guides/interop.html
 //
 // Also potentially makes other API suggestions.
-class KotlinInteropChecks {
+class KotlinInteropChecks(val reporter: Reporter) {
     fun check(codebase: Codebase) {
 
         codebase.accept(object : ApiVisitor(
diff --git a/src/main/java/com/android/tools/metalava/Options.kt b/src/main/java/com/android/tools/metalava/Options.kt
index cbc96be..ed4877f 100644
--- a/src/main/java/com/android/tools/metalava/Options.kt
+++ b/src/main/java/com/android/tools/metalava/Options.kt
@@ -149,8 +149,12 @@ const val ARG_DEX_API_MAPPING = "--dex-api-mapping"
 const val ARG_GENERATE_DOCUMENTATION = "--generate-documentation"
 const val ARG_REPLACE_DOCUMENTATION = "--replace-documentation"
 const val ARG_BASELINE = "--baseline"
+const val ARG_BASELINE_API_LINT = "--baseline:api-lint"
+const val ARG_BASELINE_CHECK_COMPATIBILITY_RELEASED = "--baseline:compatibility:released"
 const val ARG_REPORT_EVEN_IF_SUPPRESSED = "--report-even-if-suppressed"
 const val ARG_UPDATE_BASELINE = "--update-baseline"
+const val ARG_UPDATE_BASELINE_API_LINT = "--update-baseline:api-lint"
+const val ARG_UPDATE_BASELINE_CHECK_COMPATIBILITY_RELEASED = "--update-baseline:compatibility:released"
 const val ARG_MERGE_BASELINE = "--merge-baseline"
 const val ARG_STUB_PACKAGES = "--stub-packages"
 const val ARG_STUB_IMPORT_PACKAGES = "--stub-import-packages"
@@ -159,6 +163,9 @@ const val ARG_SUBTRACT_API = "--subtract-api"
 const val ARG_TYPEDEFS_IN_SIGNATURES = "--typedefs-in-signatures"
 const val ARG_FORCE_CONVERT_TO_WARNING_NULLABILITY_ANNOTATIONS = "--force-convert-to-warning-nullability-annotations"
 const val ARG_IGNORE_CLASSES_ON_CLASSPATH = "--ignore-classes-on-classpath"
+const val ARG_ERROR_MESSAGE_API_LINT = "--error-message:api-lint"
+const val ARG_ERROR_MESSAGE_CHECK_COMPATIBILITY_RELEASED = "--error-message:compatibility:released"
+const val ARG_ERROR_MESSAGE_CHECK_COMPATIBILITY_CURRENT = "--error-message:compatibility:current"
 
 class Options(
     private val args: Array<String>,
@@ -559,8 +566,48 @@ class Options(
     /** A baseline to check against */
     var baseline: Baseline? = null
 
-    /** Whether all baseline files need to be updated */
-    var updateBaseline = false
+    /** A baseline to check against, specifically used for "API lint" (i.e. [ARG_API_LINT]) */
+    var baselineApiLint: Baseline? = null
+
+    /**
+     * A baseline to check against, specifically used for "check-compatibility:*:released"
+     * (i.e. [ARG_CHECK_COMPATIBILITY_API_RELEASEED] and [ARG_CHECK_COMPATIBILITY_REMOVED_RELEASEED])
+     */
+    var baselineCompatibilityReleased: Baseline? = null
+
+    var allBaselines: List<Baseline>
+
+    /** If set, metalava will show this error message when "API lint" (i.e. [ARG_API_LINT]) fails. */
+    var errorMessageApiLint: String? = null
+
+    /**
+     * If set, metalava will show this error message when "check-compatibility:*:released" fails.
+     * (i.e. [ARG_CHECK_COMPATIBILITY_API_RELEASEED] and [ARG_CHECK_COMPATIBILITY_REMOVED_RELEASEED])
+     */
+    var errorMessageCompatibilityReleased: String? = null
+
+    /**
+     * If set, metalava will show this error message when "check-compatibility:*:current" fails.
+     * (i.e. [ARG_CHECK_COMPATIBILITY_API_CURRENT] and [ARG_CHECK_COMPATIBILITY_REMOVED_CURRENT])
+     */
+    var errorMessageCompatibilityCurrent: String? = null
+
+    /** [Reporter] for "api-lint" */
+    var reporterApiLint: Reporter
+
+    /**
+     * [Reporter] for "check-compatibility:*:released".
+     * (i.e. [ARG_CHECK_COMPATIBILITY_API_RELEASEED] and [ARG_CHECK_COMPATIBILITY_REMOVED_RELEASEED])
+     */
+    var reporterCompatibilityReleased: Reporter
+
+    /**
+     * [Reporter] for "check-compatibility:*:current".
+     * (i.e. [ARG_CHECK_COMPATIBILITY_API_CURRENT] and [ARG_CHECK_COMPATIBILITY_REMOVED_CURRENT])
+     */
+    var reporterCompatibilityCurrent: Reporter
+
+    var allReporters: List<Reporter>
 
     /** If updating baselines, don't fail the build */
     var passBaselineUpdates = false
@@ -647,12 +694,21 @@ class Options(
 
         var androidJarPatterns: MutableList<String>? = null
         var currentJar: File? = null
-        var updateBaselineFile: File? = null
-        var baselineFile: File? = null
-        var mergeBaseline = false
         var delayedCheckApiFiles = false
         var skipGenerateAnnotations = false
 
+        var baselineBuilder = Baseline.Builder().apply { description = "base" }
+        var baselineApiLintBuilder = Baseline.Builder().apply { description = "api-lint" }
+        var baselineCompatibilityReleasedBuilder = Baseline.Builder().apply { description = "compatibility:released" }
+
+        fun getBaselineBuilderForArg(flag: String): Baseline.Builder = when (flag) {
+                ARG_BASELINE, ARG_UPDATE_BASELINE, ARG_MERGE_BASELINE -> baselineBuilder
+                ARG_BASELINE_API_LINT, ARG_UPDATE_BASELINE_API_LINT -> baselineApiLintBuilder
+                ARG_BASELINE_CHECK_COMPATIBILITY_RELEASED, ARG_UPDATE_BASELINE_CHECK_COMPATIBILITY_RELEASED
+                    -> baselineCompatibilityReleasedBuilder
+                else -> error("Internal error: Invalid flag: $flag")
+            }
+
         var index = 0
         while (index < args.size) {
             val arg = args[index]
@@ -852,10 +908,10 @@ class Options(
                     allowClassesFromClasspath = false
                 }
 
-                ARG_BASELINE -> {
-                    val relative = getValue(args, ++index)
-                    assert(baselineFile == null) { "Only one baseline is allowed; found both $baselineFile and $relative" }
-                    baselineFile = stringToExistingFile(relative)
+                ARG_BASELINE, ARG_BASELINE_API_LINT, ARG_BASELINE_CHECK_COMPATIBILITY_RELEASED -> {
+                    val nextArg = getValue(args, ++index)
+                    val builder = getBaselineBuilderForArg(arg)
+                    builder.file = stringToExistingFile(nextArg)
                 }
 
                 ARG_REPORT_EVEN_IF_SUPPRESSED -> {
@@ -867,18 +923,22 @@ class Options(
                     reportEvenIfSuppressedWriter = reportEvenIfSuppressed?.printWriter()
                 }
 
-                ARG_UPDATE_BASELINE, ARG_MERGE_BASELINE -> {
-                    updateBaseline = true
-                    mergeBaseline = arg == ARG_MERGE_BASELINE
+                ARG_MERGE_BASELINE, ARG_UPDATE_BASELINE, ARG_UPDATE_BASELINE_API_LINT, ARG_UPDATE_BASELINE_CHECK_COMPATIBILITY_RELEASED -> {
+                    val builder = getBaselineBuilderForArg(arg)
+                    builder.merge = (arg == ARG_MERGE_BASELINE)
                     if (index < args.size - 1) {
                         val nextArg = args[index + 1]
                         if (!nextArg.startsWith("-")) {
-                            val file = stringToNewOrExistingFile(nextArg)
                             index++
-                            updateBaselineFile = file
+                            builder.updateFile = stringToNewOrExistingFile(nextArg)
                         }
                     }
                 }
+
+                ARG_ERROR_MESSAGE_API_LINT -> errorMessageApiLint = getValue(args, ++index)
+                ARG_ERROR_MESSAGE_CHECK_COMPATIBILITY_RELEASED -> errorMessageCompatibilityReleased = getValue(args, ++index)
+                ARG_ERROR_MESSAGE_CHECK_COMPATIBILITY_CURRENT -> errorMessageCompatibilityCurrent = getValue(args, ++index)
+
                 ARG_PASS_BASELINE_UPDATES -> passBaselineUpdates = true
                 ARG_DELETE_EMPTY_BASELINES -> deleteEmptyBaselines = true
 
@@ -1557,28 +1617,51 @@ class Options(
             removedDexApiFile = null
         }
 
-        if (baselineFile == null) {
+        // Fix up [Baseline] files and [Reporter]s.
+
+        val baselineHeaderComment = if (isBuildingAndroid())
+            "// See tools/metalava/API-LINT.md for how to update this file.\n\n"
+        else
+            ""
+        baselineBuilder.headerComment = baselineHeaderComment
+        baselineApiLintBuilder.headerComment = baselineHeaderComment
+        baselineCompatibilityReleasedBuilder.headerComment = baselineHeaderComment
+
+        if (baselineBuilder.file == null) {
+            // If default baseline is a file, use it.
             val defaultBaselineFile = getDefaultBaselineFile()
             if (defaultBaselineFile != null && defaultBaselineFile.isFile) {
-                if (updateBaseline && updateBaselineFile == null) {
-                    updateBaselineFile = defaultBaselineFile
-                }
-                baseline = Baseline(defaultBaselineFile, updateBaselineFile, mergeBaseline)
-            } else if (updateBaselineFile != null) {
-                baseline = Baseline(null, updateBaselineFile, mergeBaseline)
+                baselineBuilder.file = defaultBaselineFile
             }
-        } else {
-            // Add helpful doc in AOSP baseline files?
-            val headerComment = if (isBuildingAndroid())
-                "// See tools/metalava/API-LINT.md for how to update this file.\n\n"
-            else
-                ""
-            if (updateBaseline && updateBaselineFile == null) {
-                updateBaselineFile = baselineFile
-            }
-            baseline = Baseline(baselineFile, updateBaselineFile, mergeBaseline, headerComment)
         }
 
+        baseline = baselineBuilder.build()
+        baselineApiLint = baselineApiLintBuilder.build()
+        baselineCompatibilityReleased = baselineCompatibilityReleasedBuilder.build()
+
+        reporterApiLint = Reporter(
+            baselineApiLint ?: baseline,
+            errorMessageApiLint
+        )
+        reporterCompatibilityReleased = Reporter(
+            baselineCompatibilityReleased ?: baseline,
+            errorMessageCompatibilityReleased
+        )
+        reporterCompatibilityCurrent = Reporter(
+            // Note, the compat-check:current shouldn't take a baseline file, so we don't have
+            // a task specific baseline file, but we still respect the global baseline file.
+            baseline,
+            errorMessageCompatibilityCurrent
+        )
+
+        // Build "all baselines" and "all reporters"
+
+        // Baselines are nullable, so selectively add to the list.
+        allBaselines = listOfNotNull(baseline, baselineApiLint, baselineCompatibilityReleased)
+
+        // Reporters are non-null.
+        allReporters = listOf<Reporter>(reporterApiLint, reporterCompatibilityReleased, reporterCompatibilityCurrent)
+
         checkFlagConsistency()
     }
 
@@ -2139,6 +2222,13 @@ class Options(
                 "If some warnings have been fixed, this will delete them from the baseline files. If a file " +
                 "is provided, the updated baseline is written to the given file; otherwise the original source " +
                 "baseline file is updated.",
+            "$ARG_BASELINE_API_LINT <file> $ARG_UPDATE_BASELINE_API_LINT [file]", "Same as $ARG_BASELINE and " +
+                "$ARG_UPDATE_BASELINE respectively, but used specifically for API lint issues performed by " +
+                "$ARG_API_LINT.",
+            "$ARG_BASELINE_CHECK_COMPATIBILITY_RELEASED <file> $ARG_UPDATE_BASELINE_CHECK_COMPATIBILITY_RELEASED [file]",
+                "Same as $ARG_BASELINE and " +
+                "$ARG_UPDATE_BASELINE respectively, but used specifically for API compatibility issues performed by " +
+                "$ARG_CHECK_COMPATIBILITY_API_RELEASED and $ARG_CHECK_COMPATIBILITY_REMOVED_RELEASED.",
             "$ARG_MERGE_BASELINE [file]", "Like $ARG_UPDATE_BASELINE, but instead of always replacing entries " +
                 "in the baseline, it will merge the existing baseline with the new baseline. This is useful " +
                 "if $PROGRAM_NAME runs multiple times on the same source tree with different flags at different " +
@@ -2148,6 +2238,11 @@ class Options(
                 "all the baselines in the source tree can be updated in one go.",
             ARG_DELETE_EMPTY_BASELINES, "Whether to delete baseline files if they are updated and there is nothing " +
                 "to include.",
+            "$ARG_ERROR_MESSAGE_API_LINT <message>", "If set, $PROGRAM_NAME shows it when errors are detected in $ARG_API_LINT.",
+            "$ARG_ERROR_MESSAGE_CHECK_COMPATIBILITY_RELEASED <message>", "If set, $PROGRAM_NAME shows it " +
+                " when errors are detected in $ARG_CHECK_COMPATIBILITY_API_RELEASED and $ARG_CHECK_COMPATIBILITY_REMOVED_RELEASED.",
+            "$ARG_ERROR_MESSAGE_CHECK_COMPATIBILITY_CURRENT <message>", "If set, $PROGRAM_NAME shows it " +
+                " when errors are detected in $ARG_CHECK_COMPATIBILITY_API_CURRENT and $ARG_CHECK_COMPATIBILITY_REMOVED_CURRENT.",
 
             "", "\nJDiff:",
             "$ARG_XML_API <file>", "Like $ARG_API, but emits the API in the JDiff XML format instead",
diff --git a/src/main/java/com/android/tools/metalava/Reporter.kt b/src/main/java/com/android/tools/metalava/Reporter.kt
index 0af715a..f8ea605 100644
--- a/src/main/java/com/android/tools/metalava/Reporter.kt
+++ b/src/main/java/com/android/tools/metalava/Reporter.kt
@@ -29,6 +29,7 @@ import com.android.tools.metalava.model.Item
 import com.android.tools.metalava.model.configuration
 import com.android.tools.metalava.model.psi.PsiItem
 import com.android.tools.metalava.model.text.TextItem
+import com.google.common.annotations.VisibleForTesting
 import com.intellij.openapi.util.TextRange
 import com.intellij.openapi.vfs.VfsUtilCore
 import com.intellij.psi.PsiCompiledElement
@@ -36,8 +37,13 @@ import com.intellij.psi.PsiElement
 import com.intellij.psi.PsiModifierListOwner
 import com.intellij.psi.impl.light.LightElement
 import java.io.File
+import java.io.PrintWriter
 
-var reporter = Reporter()
+/**
+ * "Global" [Reporter] used by most operations.
+ * Certain operations, such as api-lint and compatibility check, may use a custom [Reporter]
+ */
+val reporter = Reporter(null, null)
 
 enum class Severity(private val displayName: String) {
     INHERIT("inherit"),
@@ -71,7 +77,17 @@ enum class Severity(private val displayName: String) {
     override fun toString(): String = displayName
 }
 
-open class Reporter(private val rootFolder: File? = File("").absoluteFile) {
+class Reporter(
+    /** [Baseline] file associated with this [Reporter]. If null, the global baseline is used. */
+    // See the comment on [getBaseline] for why it's nullable.
+    private val customBaseline: Baseline?,
+
+    /**
+     * An error message associated with this [Reporter], which should be shown to the user
+     * when metalava finishes with errors.
+     */
+    private val errorMessage: String?
+) {
     var errorCount = 0
         private set
     var warningCount = 0
@@ -80,6 +96,10 @@ open class Reporter(private val rootFolder: File? = File("").absoluteFile) {
 
     private var hasErrors = false
 
+    // Note we can't set [options.baseline] as the default for [customBaseline], because
+    // options.baseline will be initialized after the global [Reporter] is instantiated.
+    fun getBaseline(): Baseline? = customBaseline ?: options.baseline
+
     fun report(id: Issues.Issue, element: PsiElement?, message: String): Boolean {
         val severity = configuration.getSeverity(id)
 
@@ -87,7 +107,7 @@ open class Reporter(private val rootFolder: File? = File("").absoluteFile) {
             return false
         }
 
-        val baseline = options.baseline
+        val baseline = getBaseline()
         if (element != null && baseline != null && baseline.mark(element, message, id)) {
             return false
         }
@@ -102,7 +122,7 @@ open class Reporter(private val rootFolder: File? = File("").absoluteFile) {
             return false
         }
 
-        val baseline = options.baseline
+        val baseline = getBaseline()
         if (file != null && baseline != null && baseline.mark(file, message, id)) {
             return false
         }
@@ -126,6 +146,7 @@ open class Reporter(private val rootFolder: File? = File("").absoluteFile) {
             else -> which(severity, null as String?, message, id)
         }
 
+        // Optionally write to the --report-even-if-suppressed file.
         dispatch(this::reportEvenIfSuppressed)
 
         if (isSuppressed(id, item, message)) {
@@ -144,7 +165,7 @@ open class Reporter(private val rootFolder: File? = File("").absoluteFile) {
             }
         }
 
-        val baseline = options.baseline
+        val baseline = getBaseline()
         if (item != null && baseline != null && baseline.mark(item, message, id)) {
             return false
         } else if (psi != null && baseline != null && baseline.mark(psi, message, id)) {
@@ -269,7 +290,7 @@ open class Reporter(private val rootFolder: File? = File("").absoluteFile) {
     private fun doReport(severity: Severity, location: String?, message: String, id: Issues.Issue?) =
         report(severity, location, message, id)
 
-    open fun report(
+    fun report(
         severity: Severity,
         location: String?,
         message: String,
@@ -296,7 +317,7 @@ open class Reporter(private val rootFolder: File? = File("").absoluteFile) {
             warningCount++
         }
 
-        print(format(effectiveSeverity, location, message, id, color, options.omitLocations))
+        reportPrinter(format(effectiveSeverity, location, message, id, color, options.omitLocations))
         return true
     }
 
@@ -396,11 +417,34 @@ open class Reporter(private val rootFolder: File? = File("").absoluteFile) {
         return true
     }
 
-    open fun print(message: String) {
-        options.stdout.println()
-        options.stdout.print(message.trim())
-        options.stdout.flush()
+    fun hasErrors(): Boolean = hasErrors
+
+    /** Write the error message set to this [Reporter], if any errors have been detected. */
+    fun writeErrorMessage(writer: PrintWriter) {
+        if (hasErrors()) {
+            errorMessage ?. let { writer.write(it) }
+        }
+    }
+
+    fun getBaselineDescription(): String {
+        val file = getBaseline()?.file
+        return if (file != null) {
+            "baseline ${file.path}"
+        } else {
+            "no baseline"
+        }
     }
 
-    fun hasErrors(): Boolean = hasErrors
+    companion object {
+        /** root folder, which needs to be changed for unit tests. */
+        @VisibleForTesting
+        internal var rootFolder: File? = File("").absoluteFile
+
+        /** Injection point for unit tests. */
+        internal var reportPrinter: (String) -> Unit = { message ->
+            options.stdout.println()
+            options.stdout.print(message.trim())
+            options.stdout.flush()
+        }
+    }
 }
diff --git a/src/test/java/com/android/tools/metalava/ApiLintBaselineTest.kt b/src/test/java/com/android/tools/metalava/ApiLintBaselineTest.kt
new file mode 100644
index 0000000..12b9a5a
--- /dev/null
+++ b/src/test/java/com/android/tools/metalava/ApiLintBaselineTest.kt
@@ -0,0 +1,181 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.tools.metalava
+
+import org.junit.Test
+
+/**
+ * Test for [ApiLint] specifically with baseline arguments.
+ */
+class ApiLintBaselineTest : DriverTest() {
+    @Test
+    fun `Test with global baseline`() {
+        check(
+            apiLint = "", // enabled
+            compatibilityMode = false,
+            baseline = """
+                // Baseline format: 1.0
+                Enum: android.pkg.MyEnum:
+                    Enums are discouraged in Android APIs
+            """,
+            sourceFiles = arrayOf(
+                java(
+                    """
+                    package android.pkg;
+
+                    public enum MyEnum {
+                       FOO, BAR
+                    }
+                    """
+                )
+            )
+        )
+    }
+
+    @Test
+    fun `Test with api-lint specific baseline`() {
+        check(
+            apiLint = "", // enabled
+            compatibilityMode = false,
+            baselineApiLint = """
+                // Baseline format: 1.0
+                Enum: android.pkg.MyEnum:
+                    Enums are discouraged in Android APIs
+            """,
+            sourceFiles = arrayOf(
+                java(
+                    """
+                    package android.pkg;
+
+                    public enum MyEnum {
+                       FOO, BAR
+                    }
+                    """
+                )
+            )
+        )
+    }
+
+    @Test
+    fun `Test with api-lint specific baseline with update`() {
+        check(
+            apiLint = "", // enabled
+            compatibilityMode = false,
+            baselineApiLint = """
+                """,
+            updateBaselineApiLint = """
+                // Baseline format: 1.0
+                Enum: android.pkg.MyEnum:
+                    Enums are discouraged in Android APIs
+                """,
+            sourceFiles = arrayOf(
+                java(
+                    """
+                    package android.pkg;
+
+                    public enum MyEnum {
+                       FOO, BAR
+                    }
+                    """
+                )
+            )
+        )
+    }
+
+    @Test
+    fun `Test with non-api-lint specific baseline`() {
+        check(
+            apiLint = "", // enabled
+            compatibilityMode = false,
+            baselineCheckCompatibilityReleased = """
+                // Baseline format: 1.0
+                Enum: android.pkg.MyEnum:
+                    Enums are discouraged in Android APIs
+            """,
+            warnings = """
+                src/android/pkg/MyEnum.java:3: error: Enums are discouraged in Android APIs [Enum] [Rule F5 in go/android-api-guidelines]
+                """,
+            sourceFiles = arrayOf(
+                java(
+                    """
+                    package android.pkg;
+
+                    public enum MyEnum {
+                       FOO, BAR
+                    }
+                    """
+                )
+            )
+        )
+    }
+
+    @Test
+    fun `Test api-lint error message`() {
+        check(
+            apiLint = "", // enabled
+            compatibilityMode = false,
+            baselineApiLint = "",
+            errorMessageApiLint = "*** api-lint failed ***",
+            warnings = """
+                src/android/pkg/MyClassImpl.java:3: error: Don't expose your implementation details: `MyClassImpl` ends with `Impl` [EndsWithImpl]
+                """,
+            sourceFiles = arrayOf(
+                java(
+                    """
+                    package android.pkg;
+
+                    public class MyClassImpl {
+                    }
+                    """
+                )
+            ),
+            expectedFail = "",
+            expectedOutput = """
+                1 new API lint issues were found.
+                See tools/metalava/API-LINT.md for how to handle these.
+                *** api-lint failed ***
+                """
+        )
+    }
+
+    @Test
+    fun `Test no api-lint error message`() {
+        check(
+            apiLint = "", // enabled
+            compatibilityMode = false,
+            baselineApiLint = "",
+            warnings = """
+                src/android/pkg/MyClassImpl.java:3: error: Don't expose your implementation details: `MyClassImpl` ends with `Impl` [EndsWithImpl]
+                """,
+            sourceFiles = arrayOf(
+                java(
+                    """
+                    package android.pkg;
+
+                    public class MyClassImpl {
+                    }
+                    """
+                )
+            ),
+            expectedFail = "",
+            expectedOutput = """
+                1 new API lint issues were found.
+                See tools/metalava/API-LINT.md for how to handle these.
+                """
+        )
+    }
+}
\ No newline at end of file
diff --git a/src/test/java/com/android/tools/metalava/ApiLintTest.kt b/src/test/java/com/android/tools/metalava/ApiLintTest.kt
index 5c70ae4..7ae2d28 100644
--- a/src/test/java/com/android/tools/metalava/ApiLintTest.kt
+++ b/src/test/java/com/android/tools/metalava/ApiLintTest.kt
@@ -1281,9 +1281,11 @@ class ApiLintTest : DriverTest() {
     @Test
     fun `Check exception related issues`() {
         check(
-            extraArguments = arrayOf(ARG_API_LINT,
+            extraArguments = arrayOf(
+                ARG_API_LINT,
                 // Conflicting advice:
-                ARG_HIDE, "BannedThrow"),
+                ARG_HIDE, "BannedThrow"
+            ),
             compatibilityMode = false,
             warnings = """
                 src/android/pkg/MyClass.java:6: error: Methods must not throw generic exceptions (`java.lang.Exception`) [GenericException] [Rule S1 in go/android-api-guidelines]
@@ -2682,7 +2684,8 @@ class ApiLintTest : DriverTest() {
                         }
                     """
                 ),
-                kotlin("""
+                kotlin(
+                    """
                     package android.pkg
 
                     object Bar
diff --git a/src/test/java/com/android/tools/metalava/BaselineTest.kt b/src/test/java/com/android/tools/metalava/BaselineTest.kt
index a993349..112f665 100644
--- a/src/test/java/com/android/tools/metalava/BaselineTest.kt
+++ b/src/test/java/com/android/tools/metalava/BaselineTest.kt
@@ -57,7 +57,6 @@ class BaselineTest : DriverTest() {
                 ReferencesHidden: test.pkg.Foo#method(test.pkg.Hidden1, test.pkg.Hidden2) parameter #1:
                     Class test.pkg.Hidden2 is hidden but was referenced (as parameter type) from public parameter hidden2 in test.pkg.Foo.method(test.pkg.Hidden1 hidden1, test.pkg.Hidden2 hidden2)
             """,
-            updateBaseline = false,
             // Commented out above:
             warnings = """
                 src/test/pkg/Foo.java:9: error: Class test.pkg.Hidden2 is hidden but was referenced (as return type) from public method test.pkg.Foo.getHidden2() [ReferencesHidden]
@@ -167,11 +166,12 @@ class BaselineTest : DriverTest() {
                 ARG_API_LINT
             ),
             baseline = """
+            """,
+            updateBaseline = """
                 // Baseline format: 1.0
                 PairedRegistration: android.pkg.RegistrationMethods#registerUnpaired2Callback(Runnable):
                     Found registerUnpaired2Callback but not unregisterUnpaired2Callback in android.pkg.RegistrationMethods
             """,
-            updateBaseline = true,
             warnings = "",
             sourceFiles = arrayOf(
                 java(
@@ -258,7 +258,6 @@ class BaselineTest : DriverTest() {
                 ReferencesHidden: test.pkg.Foo#hidden2:
                     Class test.pkg.Hidden2 is hidden but was referenced (as field type) from public field test.pkg.Foo.hidden2
             """,
-            updateBaseline = false,
             mergeBaseline = """
                 // Baseline format: 1.0
                 BothPackageInfoAndHtml: test/visible/package-info.java:
diff --git a/src/test/java/com/android/tools/metalava/CompatibilityCheckBaselineTest.kt b/src/test/java/com/android/tools/metalava/CompatibilityCheckBaselineTest.kt
new file mode 100644
index 0000000..9a48326
--- /dev/null
+++ b/src/test/java/com/android/tools/metalava/CompatibilityCheckBaselineTest.kt
@@ -0,0 +1,213 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.tools.metalava
+
+import org.junit.Test
+
+class CompatibilityCheckBaselineTest : DriverTest() {
+    @Test
+    fun `Test released-API check, with error message`() {
+        // Global baseline works on both released- and current- check.
+        check(
+            warnings = """
+                TESTROOT/released-api.txt:1: error: Removed package test.pkg [RemovedPackage]
+                """,
+            compatibilityMode = false,
+            errorMessageCheckCompatibilityReleased = "*** release-api check failed ***",
+            errorMessageCheckCompatibilityCurrent = "*** current-api check failed ***",
+            checkCompatibilityApiReleased = """
+                package test.pkg {
+                  public class MyTest1 {
+                  }
+                }
+                """,
+            signatureSource = """
+                """,
+            expectedFail = """
+                Aborting: Found compatibility problems checking the public API against the API in TESTROOT/project/released-api.txt
+                *** release-api check failed ***
+                """
+        )
+    }
+
+    @Test
+    fun `Test current-API check, with error message`() {
+        // Global baseline works on both released- and current- check.
+        check(
+            warnings = """
+                TESTROOT/current-api.txt:1: error: Removed package test.pkg [RemovedPackage]
+                """,
+            compatibilityMode = false,
+            errorMessageCheckCompatibilityReleased = "*** release-api check failed ***",
+            errorMessageCheckCompatibilityCurrent = "*** current-api check failed ***",
+            checkCompatibilityApi = """
+                package test.pkg {
+                  public class MyTest1 {
+                  }
+                }
+                """,
+            signatureSource = """
+                """,
+            expectedFail = """
+                Aborting: Found compatibility problems checking the public API against the API in TESTROOT/project/current-api.txt
+                *** current-api check failed ***
+                """
+        )
+    }
+
+    @Test
+    fun `Test released-API check, with global baseline`() {
+        // Global baseline works on both released- and current- check.
+        check(
+            warnings = """
+                """,
+            compatibilityMode = false,
+            baseline = """
+                // Baseline format: 1.0
+                ChangedScope: test.pkg.MyTest1:
+                    Class test.pkg.MyTest1 changed visibility from public to private
+                """,
+            checkCompatibilityApiReleased = """
+                package test.pkg {
+                  public class MyTest1 {
+                  }
+                }
+                """,
+            signatureSource = """
+                package test.pkg {
+                  private class MyTest1 { // visibility changed
+                  }
+                }
+                """
+        )
+    }
+
+    @Test
+    fun `Test current-API check, with global baseline`() {
+        // Global baseline works on both released- and current- check.
+        check(
+            warnings = """
+                """,
+            compatibilityMode = false,
+            baseline = """
+                // Baseline format: 1.0
+                ChangedScope: test.pkg.MyTest1:
+                    Class test.pkg.MyTest1 changed visibility from public to private
+                """,
+            checkCompatibilityApi = """
+                package test.pkg {
+                  public class MyTest1 {
+                  }
+                }
+                """,
+            signatureSource = """
+                package test.pkg {
+                  private class MyTest1 { // visibility changed
+                  }
+                }
+                """
+        )
+    }
+
+    @Test
+    fun `Test released-API check, with compatibility-released baseline`() {
+        // Use released-API check baseline, which should work in released-API check.
+        check(
+            warnings = """
+                """,
+            compatibilityMode = false,
+            baselineCheckCompatibilityReleased = """
+                // Baseline format: 1.0
+                ChangedScope: test.pkg.MyTest1:
+                    Class test.pkg.MyTest1 changed visibility from public to private
+                """,
+            checkCompatibilityApiReleased = """
+                package test.pkg {
+                  public class MyTest1 {
+                  }
+                }
+                """,
+            signatureSource = """
+                package test.pkg {
+                  private class MyTest1 { // visibility changed
+                  }
+                }
+                """
+        )
+    }
+
+    @Test
+    fun `Test released-API check, with compatibility-released baseline, and update baseline`() {
+        // Use released-API check baseline, which should work in released-API check.
+        check(
+            warnings = """
+                """,
+            compatibilityMode = false,
+            baselineCheckCompatibilityReleased = """
+                """,
+            updateBaselineCheckCompatibilityReleased = """
+                // Baseline format: 1.0
+                ChangedScope: test.pkg.MyTest1:
+                    Class test.pkg.MyTest1 changed visibility from public to private
+                """,
+            checkCompatibilityApiReleased = """
+                package test.pkg {
+                  public class MyTest1 {
+                  }
+                }
+                """,
+            signatureSource = """
+                package test.pkg {
+                  private class MyTest1 { // visibility changed
+                  }
+                }
+                """
+        )
+    }
+
+    @Test
+    fun `Test current-API check, but with compatibility-released baseline`() {
+        // Use released-API check baseline, which shouldn't be used in current-API check.
+        check(
+            compatibilityMode = false,
+            warnings = """
+                TESTROOT/load-api.txt:2: error: Class test.pkg.MyTest1 changed visibility from public to private [ChangedScope]
+                """,
+            // This is a "current" compat check, so this baseline should be ignored.
+            baselineCheckCompatibilityReleased = """
+                // Baseline format: 1.0
+                ChangedScope: test.pkg.MyTest1:
+                    Class test.pkg.MyTest1 changed visibility from public to private
+                """,
+            checkCompatibilityApi = """
+                package test.pkg {
+                  public class MyTest1 {
+                  }
+                }
+                """,
+            signatureSource = """
+                package test.pkg {
+                  private class MyTest1 { // visibility changed
+                  }
+                }
+                """,
+            expectedFail = """
+                Aborting: Found compatibility problems checking the public API against the API in TESTROOT/project/current-api.txt
+                """
+        )
+    }
+}
diff --git a/src/test/java/com/android/tools/metalava/DriverTest.kt b/src/test/java/com/android/tools/metalava/DriverTest.kt
index 6a022b9..f0bdb09 100644
--- a/src/test/java/com/android/tools/metalava/DriverTest.kt
+++ b/src/test/java/com/android/tools/metalava/DriverTest.kt
@@ -130,8 +130,7 @@ abstract class DriverTest {
                         // the signature was passed at the same time
                         // ignore
                     } else {
-                        assertEquals(expectedFail, actualFail)
-                        fail(actualFail)
+                        assertEquals(expectedFail.trimIndent(), actualFail)
                     }
                 }
             }
@@ -214,6 +213,14 @@ abstract class DriverTest {
         return System.getenv("JAVA_HOME")
     }
 
+    private fun <T> buildOptionalArgs(option: T?, converter: (T) -> Array<String>): Array<String> {
+        if (option != null) {
+            return converter(option)
+        } else {
+            return emptyArray<String>()
+        }
+    }
+
     /** File conversion tasks */
     data class ConvertData(
         val fromApi: String,
@@ -378,12 +385,30 @@ abstract class DriverTest {
          * directory
          */
         projectSetup: ((File) -> Unit)? = null,
-        /** Baseline file to use, if any */
+        /** Content of the baseline file to use, if any */
         baseline: String? = null,
-        /** Whether to create the baseline if it does not exist. Requires [baseline] to be set. */
-        updateBaseline: Boolean = false,
+        /** If non-null, we expect the baseline file to be updated to this. [baseline] must also be set. */
+        updateBaseline: String? = null,
         /** Merge instead of replacing the baseline */
         mergeBaseline: String? = null,
+
+        /** [ARG_BASELINE_API_LINT] */
+        baselineApiLint: String? = null,
+        /** [ARG_UPDATE_BASELINE_API_LINT] */
+        updateBaselineApiLint: String? = null,
+
+        /** [ARG_BASELINE_CHECK_COMPATIBILITY_RELEASED] */
+        baselineCheckCompatibilityReleased: String? = null,
+        /** [ARG_UPDATE_BASELINE_CHECK_COMPATIBILITY_RELEASED] */
+        updateBaselineCheckCompatibilityReleased: String? = null,
+
+        /** [ARG_ERROR_MESSAGE_API_LINT] */
+        errorMessageApiLint: String? = null,
+        /** [ARG_ERROR_MESSAGE_CHECK_COMPATIBILITY_RELEASED] */
+        errorMessageCheckCompatibilityReleased: String? = null,
+        /** [ARG_ERROR_MESSAGE_CHECK_COMPATIBILITY_CURRENT] */
+        errorMessageCheckCompatibilityCurrent: String? = null,
+
         /**
          * If non null, enable API lint. If non-blank, a codebase where only new APIs not in the codebase
          * are linted.
@@ -495,10 +520,9 @@ abstract class DriverTest {
         }
 
         val reportedWarnings = StringBuilder()
-        reporter = object : Reporter(project) {
-            override fun print(message: String) {
-                reportedWarnings.append(cleanupString(message, project).trim()).append('\n')
-            }
+        Reporter.rootFolder = project
+        Reporter.reportPrinter = { message ->
+            reportedWarnings.append(cleanupString(message, project).trim()).append('\n')
         }
 
         val mergeAnnotationsArgs = if (mergeXmlAnnotations != null) {
@@ -921,22 +945,46 @@ abstract class DriverTest {
             emptyArray()
         }
 
-        var baselineFile: File? = null
-        val baselineArgs = if (baseline != null) {
-            baselineFile = temporaryFolder.newFile("baseline.txt")
-            baselineFile?.writeText(baseline.trimIndent())
-            if (!(updateBaseline || mergeBaseline != null)) {
-                arrayOf(ARG_BASELINE, baselineFile.path)
+        fun buildBaselineArgs(
+            argBaseline: String,
+            argUpdateBaseline: String,
+            argMergeBaseline: String,
+            filename: String,
+            baselineContent: String?,
+            updateContent: String?,
+            merge: Boolean
+        ): Pair<Array<String>, File?> {
+            if (baselineContent != null) {
+                val baselineFile = temporaryFolder.newFile(filename)
+                baselineFile?.writeText(baselineContent.trimIndent())
+                if (!(updateContent != null || merge)) {
+                    return Pair(arrayOf(argBaseline, baselineFile.path), baselineFile)
+                } else {
+                    return Pair(arrayOf(argBaseline,
+                        baselineFile.path,
+                        if (mergeBaseline != null) argMergeBaseline else argUpdateBaseline,
+                        baselineFile.path), baselineFile)
+                }
             } else {
-                arrayOf(ARG_BASELINE,
-                    baselineFile.path,
-                    if (mergeBaseline != null) ARG_MERGE_BASELINE else ARG_UPDATE_BASELINE,
-                    baselineFile.path)
+                return Pair(emptyArray(), null)
             }
-        } else {
-            emptyArray()
         }
 
+        val (baselineArgs, baselineFile) = buildBaselineArgs(
+            ARG_BASELINE, ARG_UPDATE_BASELINE, ARG_MERGE_BASELINE, "baseline.txt",
+            baseline, updateBaseline, mergeBaseline != null
+        )
+        val (baselineApiLintArgs, baselineApiLintFile) = buildBaselineArgs(
+            ARG_BASELINE_API_LINT, ARG_UPDATE_BASELINE_API_LINT, "",
+            "baseline-api-lint.txt",
+            baselineApiLint, updateBaselineApiLint, false
+        )
+        val (baselineCheckCompatibilityReleasedArgs, baselineCheckCompatibilityReleasedFile) = buildBaselineArgs(
+            ARG_BASELINE_CHECK_COMPATIBILITY_RELEASED, ARG_UPDATE_BASELINE_CHECK_COMPATIBILITY_RELEASED, "",
+            "baseline-check-released.txt",
+            baselineCheckCompatibilityReleased, updateBaselineCheckCompatibilityReleased, false
+        )
+
         val importedPackageArgs = mutableListOf<String>()
         importedPackages.forEach {
             importedPackageArgs.add("--stub-import-packages")
@@ -1030,6 +1078,16 @@ abstract class DriverTest {
             emptyArray()
         }
 
+        val errorMessageApiLintArgs = buildOptionalArgs(errorMessageApiLint) {
+            arrayOf(ARG_ERROR_MESSAGE_API_LINT, it)
+        }
+        val errorMessageCheckCompatibilityReleasedArgs = buildOptionalArgs(errorMessageCheckCompatibilityReleased) {
+            arrayOf(ARG_ERROR_MESSAGE_CHECK_COMPATIBILITY_RELEASED, it)
+        }
+        val errorMessageCheckCompatibilityCurrentArgs = buildOptionalArgs(errorMessageCheckCompatibilityCurrent) {
+            arrayOf(ARG_ERROR_MESSAGE_CHECK_COMPATIBILITY_CURRENT, it)
+        }
+
         // Run optional additional setup steps on the project directory
         projectSetup?.invoke(project)
 
@@ -1093,6 +1151,8 @@ abstract class DriverTest {
             *convertArgs,
             *applyApiLevelsXmlArgs,
             *baselineArgs,
+            *baselineApiLintArgs,
+            *baselineCheckCompatibilityReleasedArgs,
             *showAnnotationArguments,
             *hideAnnotationArguments,
             *hideMetaAnnotationArguments,
@@ -1109,6 +1169,9 @@ abstract class DriverTest {
             *signatureFormatArgs,
             *sourceList,
             *extraArguments,
+            *errorMessageApiLintArgs,
+            *errorMessageCheckCompatibilityReleasedArgs,
+            *errorMessageCheckCompatibilityCurrentArgs,
             expectedFail = expectedFail
         )
 
@@ -1135,15 +1198,28 @@ abstract class DriverTest {
             parseDocument(apiXmlFile.readText(UTF_8), false)
         }
 
-        if (baseline != null && baselineFile != null) {
+        fun checkBaseline(arg: String, baselineContent: String?, updateBaselineContent: String?, mergeBaselineContent: String?, file: File?) {
+            if (file == null) {
+                return
+            }
             assertTrue(
-                "${baselineFile.path} does not exist even though $ARG_BASELINE was used",
-                baselineFile.exists()
+                "${file.path} does not exist even though $arg was used",
+                file.exists()
             )
-            val actualText = readFile(baselineFile, stripBlankLines, trim)
-            val sourceFile = mergeBaseline ?: baseline
+            val actualText = readFile(file, stripBlankLines, trim)
+
+            // Compare against:
+            // If "merged baseline" is set, use it.
+            // If "update baseline" is set, use it.
+            // Otherwise, the original baseline.
+            val sourceFile = mergeBaselineContent ?: updateBaselineContent ?: baselineContent ?: ""
             assertEquals(stripComments(sourceFile, stripLineComments = false).trimIndent(), actualText)
         }
+        checkBaseline(ARG_BASELINE, baseline, updateBaseline, mergeBaseline, baselineFile)
+        checkBaseline(ARG_BASELINE_API_LINT, baselineApiLint, updateBaselineApiLint, null, baselineApiLintFile)
+        checkBaseline(ARG_BASELINE_CHECK_COMPATIBILITY_RELEASED, baselineCheckCompatibilityReleased,
+            updateBaselineCheckCompatibilityReleased, null, baselineCheckCompatibilityReleasedFile
+        )
 
         if (convertFiles.isNotEmpty()) {
             for (i in 0 until convertToJDiff.size) {
diff --git a/src/test/java/com/android/tools/metalava/OptionsTest.kt b/src/test/java/com/android/tools/metalava/OptionsTest.kt
index ed7c4e6..32636e3 100644
--- a/src/test/java/com/android/tools/metalava/OptionsTest.kt
+++ b/src/test/java/com/android/tools/metalava/OptionsTest.kt
@@ -280,6 +280,14 @@ Diffs and Checks:
                                              some warnings have been fixed, this will delete them from the baseline
                                              files. If a file is provided, the updated baseline is written to the given
                                              file; otherwise the original source baseline file is updated.
+--baseline:api-lint <file> --update-baseline:api-lint [file]
+                                             Same as --baseline and --update-baseline respectively, but used
+                                             specifically for API lint issues performed by --api-lint.
+--baseline:compatibility:released <file> --update-baseline:compatibility:released [file]
+                                             Same as --baseline and --update-baseline respectively, but used
+                                             specifically for API compatibility issues performed by
+                                             --check-compatibility:api:released and
+                                             --check-compatibility:removed:released.
 --merge-baseline [file]                      
                                              Like --update-baseline, but instead of always replacing entries in the
                                              baseline, it will merge the existing baseline with the new baseline. This
@@ -293,6 +301,16 @@ Diffs and Checks:
 --delete-empty-baselines                     
                                              Whether to delete baseline files if they are updated and there is nothing
                                              to include.
+--error-message:api-lint <message>           
+                                             If set, metalava shows it when errors are detected in --api-lint.
+--error-message:compatibility:released <message>
+                                             If set, metalava shows it  when errors are detected in
+                                             --check-compatibility:api:released and
+                                             --check-compatibility:removed:released.
+--error-message:compatibility:current <message>
+                                             If set, metalava shows it  when errors are detected in
+                                             --check-compatibility:api:current and
+                                             --check-compatibility:removed:current.
 
 
 JDiff: