support RSA verify with prehashing (#3265)

* support RSA verify with prehashing

* review feedback

* more dedupe

* refactor and move to a separate module

2 files changed, 37 insertions, 18 deletions

diff --git a/src/cryptography/hazmat/backends/openssl/rsa.py b/src/cryptography/hazmat/backends/openssl/rsa.py
index 85d06525..8996d884 100644
--- a/src/cryptography/hazmat/backends/openssl/rsa.py
+++ b/src/cryptography/hazmat/backends/openssl/rsa.py
@@ -10,10 +10,12 @@ from cryptography import utils
 from cryptography.exceptions import (
     InvalidSignature, UnsupportedAlgorithm, _Reasons
 )
+from cryptography.hazmat.backends.openssl.utils import (
+    _calculate_digest_and_algorithm
+)
 from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives.asymmetric import (
-    AsymmetricSignatureContext, AsymmetricVerificationContext, rsa,
-    utils as asym_utils
+    AsymmetricSignatureContext, AsymmetricVerificationContext, rsa
 )
 from cryptography.hazmat.primitives.asymmetric.padding import (
     AsymmetricPadding, MGF1, OAEP, PKCS1v15, PSS, calculate_max_pss_salt_length
@@ -453,19 +455,9 @@ class _RSAPrivateKey(object):
         padding_enum = _rsa_sig_determine_padding(
             self._backend, self, padding, algorithm
         )
-        if not isinstance(algorithm, asym_utils.Prehashed):
-            hash_ctx = hashes.Hash(algorithm, self._backend)
-            hash_ctx.update(data)
-            data = hash_ctx.finalize()
-        else:
-            algorithm = algorithm._algorithm
-
-        if len(data) != algorithm.digest_size:
-            raise ValueError(
-                "The provided data must be the same length as the hash "
-                "algorithm's digest size."
-            )
-
+        data, algorithm = _calculate_digest_and_algorithm(
+            self._backend, data, algorithm
+        )
         return _rsa_sig_sign(
             self._backend, padding, padding_enum,
             algorithm, self, data
@@ -523,6 +515,13 @@ class _RSAPublicKey(object):
         )
 
     def verify(self, signature, data, padding, algorithm):
-        verifier = self.verifier(signature, padding, algorithm)
-        verifier.update(data)
-        verifier.verify()
+        padding_enum = _rsa_sig_determine_padding(
+            self._backend, self, padding, algorithm
+        )
+        data, algorithm = _calculate_digest_and_algorithm(
+            self._backend, data, algorithm
+        )
+        return _rsa_sig_verify(
+            self._backend, padding, padding_enum, algorithm, self,
+            signature, data
+        )
diff --git a/src/cryptography/hazmat/backends/openssl/utils.py b/src/cryptography/hazmat/backends/openssl/utils.py
index 001121f9..c88e3189 100644
--- a/src/cryptography/hazmat/backends/openssl/utils.py
+++ b/src/cryptography/hazmat/backends/openssl/utils.py
@@ -6,6 +6,9 @@ from __future__ import absolute_import, division, print_function
 
 import six
 
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.asymmetric.utils import Prehashed
+
 
 def _truncate_digest(digest, order_bits):
     digest_len = len(digest)
@@ -24,3 +27,20 @@ def _truncate_digest(digest, order_bits):
         digest = digest[:-1] + six.int2byte(six.indexbytes(digest, -1) & mask)
 
     return digest
+
+
+def _calculate_digest_and_algorithm(backend, data, algorithm):
+    if not isinstance(algorithm, Prehashed):
+        hash_ctx = hashes.Hash(algorithm, backend)
+        hash_ctx.update(data)
+        data = hash_ctx.finalize()
+    else:
+        algorithm = algorithm._algorithm
+
+    if len(data) != algorithm.digest_size:
+        raise ValueError(
+            "The provided data must be the same length as the hash "
+            "algorithm's digest size."
+        )
+
+    return (data, algorithm)