never trust openssl

Turns out you can't trust it to safely compute the max salt length
allowed for PSS, so now we get to do it ourselves. We also check for
whether the key size is large enough for the selected hash function
(PSS only for now, PKCS1 coming in another PR)

3 files changed, 184 insertions, 18 deletions

diff --git a/cryptography/hazmat/backends/openssl/backend.py b/cryptography/hazmat/backends/openssl/backend.py
index 1495e75c..d7c0f882 100644
--- a/cryptography/hazmat/backends/openssl/backend.py
+++ b/cryptography/hazmat/backends/openssl/backend.py
@@ -15,6 +15,7 @@ from __future__ import absolute_import, division, print_function
 
 import collections
 import itertools
+import math
 
 import six
 
@@ -695,11 +696,24 @@ class _HMACContext(object):
         return self._backend._ffi.buffer(buf)[:outlen[0]]
 
 
+def _get_rsa_pss_salt_length(mgf, key_size, digest_size):
+    if mgf._salt_length is MGF1.MAX_LENGTH:
+        # bit length - 1 per RFC 3447
+        emlen = int(math.ceil((key_size - 1) / 8.0))
+        salt_length = emlen - digest_size - 2
+        assert salt_length >= 0
+        return salt_length
+    else:
+        return mgf._salt_length
+
+
 @utils.register_interface(interfaces.AsymmetricSignatureContext)
 class _RSASignatureContext(object):
     def __init__(self, backend, private_key, padding, algorithm):
         self._backend = backend
         self._private_key = private_key
+        key_size_bytes = int(math.ceil(private_key.key_size / 8.0))
+
         if not isinstance(padding, interfaces.AsymmetricPadding):
             raise TypeError(
                 "Expected provider of interfaces.AsymmetricPadding")
@@ -716,6 +730,11 @@ class _RSASignatureContext(object):
                     "Only MGF1 is supported by this backend"
                 )
 
+            # Size of key in bytes - 2 is the maximum
+            # PSS signature length (salt length is checked later)
+            if key_size_bytes - algorithm.digest_size - 2 < 0:
+                raise ValueError("Digest too large for key size.")
+
             if not self._backend.mgf1_hash_supported(padding._mgf._algorithm):
                 raise UnsupportedHash(
                     "When OpenSSL is older than 1.0.1 then only SHA1 is "
@@ -773,8 +792,15 @@ class _RSASignatureContext(object):
         assert res > 0
         if isinstance(self._padding, PSS):
             res = self._backend._lib.EVP_PKEY_CTX_set_rsa_pss_saltlen(
-                pkey_ctx, self._get_salt_length())
+                pkey_ctx,
+                _get_rsa_pss_salt_length(
+                    self._padding._mgf,
+                    self._private_key.key_size,
+                    self._hash_ctx.algorithm.digest_size
+                )
+            )
             assert res > 0
+
             if self._backend._lib.Cryptography_HAS_MGF1_MD:
                 # MGF1 MD is configurable in OpenSSL 1.0.1+
                 mgf1_md = self._backend._lib.EVP_get_digestbyname(
@@ -834,7 +860,11 @@ class _RSASignatureContext(object):
             padded,
             data_to_sign,
             evp_md,
-            self._get_salt_length()
+            _get_rsa_pss_salt_length(
+                self._padding._mgf,
+                self._private_key.key_size,
+                len(data_to_sign)
+            )
         )
         if res != 1:
             errors = self._backend._consume_errors()
@@ -854,12 +884,6 @@ class _RSASignatureContext(object):
         assert sig_len != -1
         return self._backend._ffi.buffer(sig_buf)[:sig_len]
 
-    def _get_salt_length(self):
-        if self._padding._mgf._salt_length is MGF1.MAX_LENGTH:
-            return -2
-        else:
-            return self._padding._mgf._salt_length
-
 
 @utils.register_interface(interfaces.AsymmetricVerificationContext)
 class _RSAVerificationContext(object):
@@ -867,6 +891,8 @@ class _RSAVerificationContext(object):
         self._backend = backend
         self._public_key = public_key
         self._signature = signature
+        key_size_bytes = int(math.ceil(public_key.key_size / 8.0))
+
         if not isinstance(padding, interfaces.AsymmetricPadding):
             raise TypeError(
                 "Expected provider of interfaces.AsymmetricPadding")
@@ -883,6 +909,11 @@ class _RSAVerificationContext(object):
                     "Only MGF1 is supported by this backend"
                 )
 
+            # Size of key in bytes - 2 is the maximum
+            # PSS signature length (salt length is checked later)
+            if key_size_bytes - algorithm.digest_size - 2 < 0:
+                raise ValueError("Digest too large for key size.")
+
             if not self._backend.mgf1_hash_supported(padding._mgf._algorithm):
                 raise UnsupportedHash(
                     "When OpenSSL is older than 1.0.1 then only SHA1 is "
@@ -936,7 +967,13 @@ class _RSAVerificationContext(object):
         assert res > 0
         if isinstance(self._padding, PSS):
             res = self._backend._lib.EVP_PKEY_CTX_set_rsa_pss_saltlen(
-                pkey_ctx, self._get_salt_length())
+                pkey_ctx,
+                _get_rsa_pss_salt_length(
+                    self._padding._mgf,
+                    self._public_key.key_size,
+                    self._hash_ctx.algorithm.digest_size
+                )
+            )
             assert res > 0
             if self._backend._lib.Cryptography_HAS_MGF1_MD:
                 # MGF1 MD is configurable in OpenSSL 1.0.1+
@@ -1011,18 +1048,16 @@ class _RSAVerificationContext(object):
             data_to_verify,
             evp_md,
             buf,
-            self._get_salt_length()
+            _get_rsa_pss_salt_length(
+                self._padding._mgf,
+                self._public_key.key_size,
+                len(data_to_verify)
+            )
         )
         if res != 1:
             errors = self._backend._consume_errors()
             assert errors
             raise InvalidSignature
 
-    def _get_salt_length(self):
-        if self._padding._mgf._salt_length is MGF1.MAX_LENGTH:
-            return -2
-        else:
-            return self._padding._mgf._salt_length
-
 
 backend = Backend()
diff --git a/tests/hazmat/primitives/test_rsa.py b/tests/hazmat/primitives/test_rsa.py
index a1ed8959..34f49f94 100644
--- a/tests/hazmat/primitives/test_rsa.py
+++ b/tests/hazmat/primitives/test_rsa.py
@@ -28,7 +28,7 @@ from cryptography.exceptions import (
 from cryptography.hazmat.primitives import hashes, interfaces
 from cryptography.hazmat.primitives.asymmetric import rsa, padding
 
-from .utils import generate_rsa_pss_test
+from .utils import generate_rsa_pss_test, rsa_pss_signing_test
 from ...utils import (
     load_pkcs1_vectors, load_vectors_from_file, load_rsa_nist_vectors
 )
@@ -483,6 +483,79 @@ class TestRSASignature(object):
         verifier.update(binascii.unhexlify(example["message"]))
         verifier.verify()
 
+    @pytest.mark.supported(
+        only_if=lambda backend: backend.mgf1_hash_supported(hashes.SHA224()),
+        skip_message="Does not support SHA224 with MGF1."
+    )
+    def test_pss_signing_sha224(self, backend):
+        rsa_pss_signing_test(backend, hashes.SHA224())
+
+    @pytest.mark.supported(
+        only_if=lambda backend: backend.mgf1_hash_supported(hashes.SHA256()),
+        skip_message="Does not support SHA256 with MGF1."
+    )
+    def test_pss_signing_sha256(self, backend):
+        rsa_pss_signing_test(backend, hashes.SHA256())
+
+    @pytest.mark.supported(
+        only_if=lambda backend: backend.mgf1_hash_supported(hashes.SHA384()),
+        skip_message="Does not support SHA384 with MGF1."
+    )
+    def test_pss_signing_sha384(self, backend):
+        rsa_pss_signing_test(backend, hashes.SHA384())
+
+    @pytest.mark.supported(
+        only_if=lambda backend: backend.mgf1_hash_supported(hashes.SHA512()),
+        skip_message="Does not support SHA512 with MGF1."
+    )
+    def test_pss_signing_sha512(self, backend):
+        rsa_pss_signing_test(backend, hashes.SHA512())
+
+    @pytest.mark.supported(
+        only_if=lambda backend: backend.hash_supported(hashes.SHA512()),
+        skip_message="Does not support SHA512."
+    )
+    def test_pss_minimum_key_size_for_digest(self, backend):
+        private_key = rsa.RSAPrivateKey.generate(
+            public_exponent=65537,
+            key_size=522,
+            backend=backend
+        )
+        signer = private_key.signer(
+            padding.PSS(
+                mgf=padding.MGF1(
+                    algorithm=hashes.SHA1(),
+                    salt_length=padding.MGF1.MAX_LENGTH
+                )
+            ),
+            hashes.SHA512(),
+            backend
+        )
+        signer.update(b"no failure")
+        signer.finalize()
+
+    @pytest.mark.supported(
+        only_if=lambda backend: backend.hash_supported(hashes.SHA512()),
+        skip_message="Does not support SHA512."
+    )
+    def test_pss_signing_digest_too_large_for_key_size(self, backend):
+        private_key = rsa.RSAPrivateKey.generate(
+            public_exponent=65537,
+            key_size=512,
+            backend=backend
+        )
+        with pytest.raises(ValueError):
+            private_key.signer(
+                padding.PSS(
+                    mgf=padding.MGF1(
+                        algorithm=hashes.SHA1(),
+                        salt_length=padding.MGF1.MAX_LENGTH
+                    )
+                ),
+                hashes.SHA512(),
+                backend
+            )
+
     def test_pss_signing_salt_length_too_long(self, backend):
         private_key = rsa.RSAPrivateKey.generate(
             public_exponent=65537,
@@ -643,7 +716,7 @@ class TestRSAVerification(object):
             padding.PSS(
                 mgf=padding.MGF1(
                     algorithm=hashes.SHA1(),
-                    salt_length=padding.MGF1.MAX_LENGTH
+                    salt_length=20
                 )
             ),
             hashes.SHA1(),
@@ -804,6 +877,34 @@ class TestRSAVerification(object):
             public_key.verifier(b"sig", padding.PSS(mgf=DummyMGF()),
                                 hashes.SHA1(), backend)
 
+    @pytest.mark.supported(
+        only_if=lambda backend: backend.hash_supported(hashes.SHA512()),
+        skip_message="Does not support SHA512."
+    )
+    def test_pss_verify_digest_too_large_for_key_size(self, backend):
+        private_key = rsa.RSAPrivateKey.generate(
+            public_exponent=65537,
+            key_size=512,
+            backend=backend
+        )
+        signature = binascii.unhexlify(
+            b"8b9a3ae9fb3b64158f3476dd8d8a1f1425444e98940e0926378baa9944d219d8"
+            b"534c050ef6b19b1bdc6eb4da422e89161106a6f5b5cc16135b11eb6439b646bd"
+        )
+        public_key = private_key.public_key()
+        with pytest.raises(ValueError):
+            public_key.verifier(
+                signature,
+                padding.PSS(
+                    mgf=padding.MGF1(
+                        algorithm=hashes.SHA1(),
+                        salt_length=padding.MGF1.MAX_LENGTH
+                    )
+                ),
+                hashes.SHA512(),
+                backend
+            )
+
     def test_pss_verify_salt_length_too_long(self, backend):
         signature = binascii.unhexlify(
             b"8b9a3ae9fb3b64158f3476dd8d8a1f1425444e98940e0926378baa9944d219d8"
diff --git a/tests/hazmat/primitives/utils.py b/tests/hazmat/primitives/utils.py
index 31491023..5d3b4d15 100644
--- a/tests/hazmat/primitives/utils.py
+++ b/tests/hazmat/primitives/utils.py
@@ -406,3 +406,33 @@ def rsa_pss_test(backend, params, hash_alg):
     )
     verifier.update(binascii.unhexlify(params["msg"]))
     verifier.verify()
+
+
+def rsa_pss_signing_test(backend, hash_alg):
+    private_key = rsa.RSAPrivateKey.generate(
+        public_exponent=65537,
+        key_size=768,
+        backend=backend
+    )
+    public_key = private_key.public_key()
+    pss = padding.PSS(
+        mgf=padding.MGF1(
+            algorithm=hash_alg,
+            salt_length=padding.MGF1.MAX_LENGTH
+        )
+    )
+    signer = private_key.signer(
+        pss,
+        hash_alg,
+        backend
+    )
+    signer.update(b"testing signature")
+    signature = signer.finalize()
+    verifier = public_key.verifier(
+        signature,
+        pss,
+        hash_alg,
+        backend
+    )
+    verifier.update(b"testing signature")
+    verifier.verify()