core: add CFG_RSA_PUB_EXPONENT_3

When generating RSA key pairs, OP-TEE currently enforces a minimum public
exponent size of 65537 per NIST SP800-56B recommendations. However, AOSP
KeyMint VTS (EncryptionOperationsTest.RsaNoPaddingSuccess [1]) requires
implementations to support public exponent 3 for backwards compatibility.
Add CFG_RSA_PUB_EXPONENT_3 to allow public exponents >= 3.

Link: https://android.googlesource.com/platform/hardware/interfaces/+/refs/heads/main/security/keymint/aidl/vts/functional/KeyMintTest.cpp#5258 [1]
Signed-off-by: Sami Tolvanen <samitolvanen@google.com>
Reviewed-by: Jerome Forissier <jerome.forissier@linaro.org>
Reviewed-by: Jens Wiklander <jens.wiklander@linaro.org>
Reviewed-by: Etienne Carriere <etienne.carriere@foss.st.com>

core/tee/tee_svc_cryp.c

@@ -2087,30 +2087,36 @@ static TEE_Result check_pub_rsa_key(struct bignum *e)
 
 	/*
 	 * NIST SP800-56B requires public RSA key to be an odd integer in
-	 * the range 65537 <= e < 2^256.
+	 * the range 65537 <= e < 2^256. AOSP requires implementations to
+	 * support public exponents >= 3, which can be allowed by enabling
+	 * CFG_RSA_PUB_EXPONENT_3.
 	 */
 
-	if (n > sizeof(bin_key) || n < 3)
+	if (n > sizeof(bin_key) || n < 1)
 		return TEE_ERROR_BAD_PARAMETERS;
 
 	crypto_bignum_bn2bin(e, bin_key);
 
 	if (!(bin_key[n - 1] & 1)) /* key must be odd */
 		return TEE_ERROR_BAD_PARAMETERS;
 
-	if (n == 3) {
+	if (n <= 3) {
+		uint32_t min_key = 65537;
 		uint32_t key = 0;
+		size_t m = 0;
 
-		for (n = 0; n < 3; n++) {
+		if (IS_ENABLED(CFG_RSA_PUB_EXPONENT_3))
+			min_key = 3;
+
+		for (m = 0; m < n; m++) {
 			key <<= 8;
-			key |= bin_key[n];
+			key |= bin_key[m];
 		}
 
-		if (key < 65537)
+		if (key < min_key)
 			return TEE_ERROR_BAD_PARAMETERS;
 	}
 
-	/* key is larger than 65537 */
 	return TEE_SUCCESS;
 }
 

mk/config.mk

@@ -1127,6 +1127,15 @@ CFG_TA_OPTEE_CORE_API_COMPAT_1_1 ?= n
 # Note that this violates GP requirements of HMAC size range.
 CFG_HMAC_64_1024_RANGE ?= n
 
+# CFG_RSA_PUB_EXPONENT_3, when enabled, allows RSA public exponents in the
+# range 3 <= e < 2^256. This is needed to pass AOSP KeyMint VTS tests:
+#    Link to tests: https://android.googlesource.com/platform/hardware/interfaces/+/refs/heads/main/security/keymint/aidl/vts/functional/KeyMintTest.cpp
+#    Module: VtsAidlKeyMintTargetTest
+#    Testcases: - PerInstance/EncryptionOperationsTest.RsaNoPaddingSuccess
+# When CFG_RSA_PUB_EXPONENT_3 is disabled, RSA public exponents must conform
+# to NIST SP800-56B recommendation and be in the range 65537 <= e < 2^256.
+CFG_RSA_PUB_EXPONENT_3 ?= n
+
 # Enable a hardware pbkdf2 function
 # By default use standard pbkdf2 implementation
 CFG_CRYPTO_HW_PBKDF2 ?= n