Fix PDF decryption of ancient 40-bit RC4 with R=2

1 files changed, 16 insertions, 5 deletions

diff --git a/DeDRM_plugin/ineptpdf.py b/DeDRM_plugin/ineptpdf.py
index ab48130..e227ddc 100755
--- a/DeDRM_plugin/ineptpdf.py
+++ b/DeDRM_plugin/ineptpdf.py
@@ -1369,11 +1369,12 @@ class PDFDocument(object):
             keylen = len(key)
             iv = bytes([0x00]*keylen)
 
+        aes = AES.new(key, AES.MODE_CBC, iv)
+
         if not encrypt:
-            plaintext = AES.new(key,AES.MODE_CBC,iv, True).decrypt(data)
+            plaintext = aes.decrypt(data)
             return plaintext
         else:
-            aes = AES.new(key, AES.MODE_CBC, iv, False)
             new_data = bytes(data * repetitions)
             crypt = aes.encrypt(new_data)
             return crypt
@@ -1478,14 +1479,23 @@ class PDFDocument(object):
             EncMetadata = b'True'
         if (EncMetadata == ('False' or 'false') or V < 4) and R >= 4:
             hash.update(codecs.decode(b'ffffffff','hex'))
+
+        # Finish hash:
+        hash = hash.digest()
+        
         if R >= 3:
             # 8
             for _ in range(50):
-                hash = hashlib.md5(hash.digest()[:length//8])
-        key = hash.digest()[:length//8]
+                hash = hashlib.md5(hash[:length//8]).digest()
+        if R == 2:
+            # R=2 only uses first five bytes.
+            key = hash[:5]
+        else:
+            key = hash[:length//8]
+
         if R == 2:
             # Algorithm 3.4
-            u1 = ARC4.new(key).decrypt(password)
+            u1 = ARC4.new(key).decrypt(self.PASSWORD_PADDING)
         elif R >= 3:
             # Algorithm 3.5
             hash = hashlib.md5(self.PASSWORD_PADDING) # 2
@@ -1498,6 +1508,7 @@ class PDFDocument(object):
                     k = b''.join(bytes([c ^ i]) for c in key )
                 x = ARC4.new(k).decrypt(x)
             u1 = x+x # 32bytes total
+        
         if R == 2:
             is_authenticated = (u1 == U)
         else: