Add support and VTS test for RSA OAEP MGF1.

Test: atest VtsAidlKeyMintV1_0TargetTest
Bug: 160968519
Change-Id: I7093b26217b69ea36b4be8837b42cb9446887685

1 files changed, 101 insertions, 0 deletions

diff --git a/security/keymint/aidl/vts/functional/KeyMintTest.cpp b/security/keymint/aidl/vts/functional/KeyMintTest.cpp
index 30601538dd..eeb74915dc 100644
--- a/security/keymint/aidl/vts/functional/KeyMintTest.cpp
+++ b/security/keymint/aidl/vts/functional/KeyMintTest.cpp
@@ -2056,6 +2056,107 @@ TEST_P(EncryptionOperationsTest, RsaOaepTooLarge) {
 }
 
 /*
+ * EncryptionOperationsTest.RsaOaepWithMGFDigestSuccess
+ *
+ * Verifies that RSA-OAEP encryption operations work, with all SHA 256 digests and all type of MGF1
+ * digests.
+ */
+TEST_P(EncryptionOperationsTest, RsaOaepWithMGFDigestSuccess) {
+    auto digests = ValidDigests(false /* withNone */, true /* withMD5 */);
+
+    size_t key_size = 2048;  // Need largish key for SHA-512 test.
+    ASSERT_EQ(ErrorCode::OK, GenerateKey(AuthorizationSetBuilder()
+                                                 .OaepMGFDigest(digests)
+                                                 .Authorization(TAG_NO_AUTH_REQUIRED)
+                                                 .RsaEncryptionKey(key_size, 65537)
+                                                 .Padding(PaddingMode::RSA_OAEP)
+                                                 .Digest(Digest::SHA_2_256)));
+
+    string message = "Hello";
+
+    for (auto digest : digests) {
+        auto params = AuthorizationSetBuilder()
+                              .Authorization(TAG_RSA_OAEP_MGF_DIGEST, digest)
+                              .Digest(Digest::SHA_2_256)
+                              .Padding(PaddingMode::RSA_OAEP);
+        string ciphertext1 = EncryptMessage(message, params);
+        if (HasNonfatalFailure()) std::cout << "-->" << digest << std::endl;
+        EXPECT_EQ(key_size / 8, ciphertext1.size());
+
+        string ciphertext2 = EncryptMessage(message, params);
+        EXPECT_EQ(key_size / 8, ciphertext2.size());
+
+        // OAEP randomizes padding so every result should be different (with astronomically high
+        // probability).
+        EXPECT_NE(ciphertext1, ciphertext2);
+
+        string plaintext1 = DecryptMessage(ciphertext1, params);
+        EXPECT_EQ(message, plaintext1) << "RSA-OAEP failed with digest " << digest;
+        string plaintext2 = DecryptMessage(ciphertext2, params);
+        EXPECT_EQ(message, plaintext2) << "RSA-OAEP failed with digest " << digest;
+
+        // Decrypting corrupted ciphertext should fail.
+        size_t offset_to_corrupt = random() % ciphertext1.size();
+        char corrupt_byte;
+        do {
+            corrupt_byte = static_cast<char>(random() % 256);
+        } while (corrupt_byte == ciphertext1[offset_to_corrupt]);
+        ciphertext1[offset_to_corrupt] = corrupt_byte;
+
+        EXPECT_EQ(ErrorCode::OK, Begin(KeyPurpose::DECRYPT, params));
+        string result;
+        EXPECT_EQ(ErrorCode::UNKNOWN_ERROR, Finish(ciphertext1, &result));
+        EXPECT_EQ(0U, result.size());
+    }
+}
+
+/*
+ * EncryptionOperationsTest.RsaOaepWithMGFIncompatibleDigest
+ *
+ * Verifies that RSA-OAEP encryption operations fail in the correct way when asked to operate
+ * with incompatible MGF digest.
+ */
+TEST_P(EncryptionOperationsTest, RsaOaepWithMGFIncompatibleDigest) {
+    ASSERT_EQ(ErrorCode::OK,
+              GenerateKey(AuthorizationSetBuilder()
+                                  .Authorization(TAG_RSA_OAEP_MGF_DIGEST, Digest::SHA_2_256)
+                                  .Authorization(TAG_NO_AUTH_REQUIRED)
+                                  .RsaEncryptionKey(2048, 65537)
+                                  .Padding(PaddingMode::RSA_OAEP)
+                                  .Digest(Digest::SHA_2_256)));
+    string message = "Hello World!";
+
+    auto params = AuthorizationSetBuilder()
+                          .Padding(PaddingMode::RSA_OAEP)
+                          .Digest(Digest::SHA_2_256)
+                          .Authorization(TAG_RSA_OAEP_MGF_DIGEST, Digest::SHA_2_224);
+    EXPECT_EQ(ErrorCode::INCOMPATIBLE_MGF_DIGEST, Begin(KeyPurpose::ENCRYPT, params));
+}
+
+/*
+ * EncryptionOperationsTest.RsaOaepWithMGFUnsupportedDigest
+ *
+ * Verifies that RSA-OAEP encryption operations fail in the correct way when asked to operate
+ * with unsupported MGF digest.
+ */
+TEST_P(EncryptionOperationsTest, RsaOaepWithMGFUnsupportedDigest) {
+    ASSERT_EQ(ErrorCode::OK,
+              GenerateKey(AuthorizationSetBuilder()
+                                  .Authorization(TAG_RSA_OAEP_MGF_DIGEST, Digest::SHA_2_256)
+                                  .Authorization(TAG_NO_AUTH_REQUIRED)
+                                  .RsaEncryptionKey(2048, 65537)
+                                  .Padding(PaddingMode::RSA_OAEP)
+                                  .Digest(Digest::SHA_2_256)));
+    string message = "Hello World!";
+
+    auto params = AuthorizationSetBuilder()
+                          .Padding(PaddingMode::RSA_OAEP)
+                          .Digest(Digest::SHA_2_256)
+                          .Authorization(TAG_RSA_OAEP_MGF_DIGEST, Digest::NONE);
+    EXPECT_EQ(ErrorCode::UNSUPPORTED_MGF_DIGEST, Begin(KeyPurpose::ENCRYPT, params));
+}
+
+/*
  * EncryptionOperationsTest.RsaPkcs1Success
  *
  * Verifies that RSA PKCS encryption/decrypts works.