Back to index

tetex-bin  3.0
Decrypt.cc
Go to the documentation of this file.
00001 //========================================================================
00002 //
00003 // Decrypt.cc
00004 //
00005 // Copyright 1996-2003 Glyph & Cog, LLC
00006 //
00007 //========================================================================
00008 
00009 #include <aconf.h>
00010 
00011 #ifdef USE_GCC_PRAGMAS
00012 #pragma implementation
00013 #endif
00014 
00015 #include "gmem.h"
00016 #include "Decrypt.h"
00017 
00018 static void rc4InitKey(Guchar *key, int keyLen, Guchar *state);
00019 static Guchar rc4DecryptByte(Guchar *state, Guchar *x, Guchar *y, Guchar c);
00020 static void md5(Guchar *msg, int msgLen, Guchar *digest);
00021 
00022 static Guchar passwordPad[32] = {
00023   0x28, 0xbf, 0x4e, 0x5e, 0x4e, 0x75, 0x8a, 0x41,
00024   0x64, 0x00, 0x4e, 0x56, 0xff, 0xfa, 0x01, 0x08, 
00025   0x2e, 0x2e, 0x00, 0xb6, 0xd0, 0x68, 0x3e, 0x80, 
00026   0x2f, 0x0c, 0xa9, 0xfe, 0x64, 0x53, 0x69, 0x7a
00027 };
00028 
00029 //------------------------------------------------------------------------
00030 // Decrypt
00031 //------------------------------------------------------------------------
00032 
00033 Decrypt::Decrypt(Guchar *fileKey, int keyLength, int objNum, int objGen) {
00034   int i;
00035 
00036   // construct object key
00037   for (i = 0; i < keyLength; ++i) {
00038     objKey[i] = fileKey[i];
00039   }
00040   objKey[keyLength] = objNum & 0xff;
00041   objKey[keyLength + 1] = (objNum >> 8) & 0xff;
00042   objKey[keyLength + 2] = (objNum >> 16) & 0xff;
00043   objKey[keyLength + 3] = objGen & 0xff;
00044   objKey[keyLength + 4] = (objGen >> 8) & 0xff;
00045   md5(objKey, keyLength + 5, objKey);
00046 
00047   // set up for decryption
00048   x = y = 0;
00049   if ((objKeyLength = keyLength + 5) > 16) {
00050     objKeyLength = 16;
00051   }
00052   rc4InitKey(objKey, objKeyLength, state);
00053 }
00054 
00055 void Decrypt::reset() {
00056   x = y = 0;
00057   rc4InitKey(objKey, objKeyLength, state);
00058 }
00059 
00060 Guchar Decrypt::decryptByte(Guchar c) {
00061   return rc4DecryptByte(state, &x, &y, c);
00062 }
00063 
00064 GBool Decrypt::makeFileKey(int encVersion, int encRevision, int keyLength,
00065                         GString *ownerKey, GString *userKey,
00066                         int permissions, GString *fileID,
00067                         GString *ownerPassword, GString *userPassword,
00068                         Guchar *fileKey, GBool *ownerPasswordOk) {
00069   Guchar test[32], test2[32];
00070   GString *userPassword2;
00071   Guchar fState[256];
00072   Guchar tmpKey[16];
00073   Guchar fx, fy;
00074   int len, i, j;
00075 
00076   // try using the supplied owner password to generate the user password
00077   *ownerPasswordOk = gFalse;
00078   if (ownerPassword) {
00079     len = ownerPassword->getLength();
00080     if (len < 32) {
00081       memcpy(test, ownerPassword->getCString(), len);
00082       memcpy(test + len, passwordPad, 32 - len);
00083     } else {
00084       memcpy(test, ownerPassword->getCString(), 32);
00085     }
00086     md5(test, 32, test);
00087     if (encRevision == 3) {
00088       for (i = 0; i < 50; ++i) {
00089        md5(test, 16, test);
00090       }
00091     }
00092     if (encRevision == 2) {
00093       rc4InitKey(test, keyLength, fState);
00094       fx = fy = 0;
00095       for (i = 0; i < 32; ++i) {
00096        test2[i] = rc4DecryptByte(fState, &fx, &fy, ownerKey->getChar(i));
00097       }
00098     } else {
00099       memcpy(test2, ownerKey->getCString(), 32);
00100       for (i = 19; i >= 0; --i) {
00101        for (j = 0; j < keyLength; ++j) {
00102          tmpKey[j] = test[j] ^ i;
00103        }
00104        rc4InitKey(tmpKey, keyLength, fState);
00105        fx = fy = 0;
00106        for (j = 0; j < 32; ++j) {
00107          test2[j] = rc4DecryptByte(fState, &fx, &fy, test2[j]);
00108        }
00109       }
00110     }
00111     userPassword2 = new GString((char *)test2, 32);
00112     if (makeFileKey2(encVersion, encRevision, keyLength, ownerKey, userKey,
00113                    permissions, fileID, userPassword2, fileKey)) {
00114       *ownerPasswordOk = gTrue;
00115       delete userPassword2;
00116       return gTrue;
00117     }
00118     delete userPassword2;
00119   }
00120 
00121   // try using the supplied user password
00122   return makeFileKey2(encVersion, encRevision, keyLength, ownerKey, userKey,
00123                     permissions, fileID, userPassword, fileKey);
00124 }
00125 
00126 GBool Decrypt::makeFileKey2(int encVersion, int encRevision, int keyLength,
00127                          GString *ownerKey, GString *userKey,
00128                          int permissions, GString *fileID,
00129                          GString *userPassword, Guchar *fileKey) {
00130   Guchar *buf;
00131   Guchar test[32];
00132   Guchar fState[256];
00133   Guchar tmpKey[16];
00134   Guchar fx, fy;
00135   int len, i, j;
00136   GBool ok;
00137 
00138   // generate file key
00139   buf = (Guchar *)gmalloc(68 + fileID->getLength());
00140   if (userPassword) {
00141     len = userPassword->getLength();
00142     if (len < 32) {
00143       memcpy(buf, userPassword->getCString(), len);
00144       memcpy(buf + len, passwordPad, 32 - len);
00145     } else {
00146       memcpy(buf, userPassword->getCString(), 32);
00147     }
00148   } else {
00149     memcpy(buf, passwordPad, 32);
00150   }
00151   memcpy(buf + 32, ownerKey->getCString(), 32);
00152   buf[64] = permissions & 0xff;
00153   buf[65] = (permissions >> 8) & 0xff;
00154   buf[66] = (permissions >> 16) & 0xff;
00155   buf[67] = (permissions >> 24) & 0xff;
00156   memcpy(buf + 68, fileID->getCString(), fileID->getLength());
00157   md5(buf, 68 + fileID->getLength(), fileKey);
00158   if (encRevision == 3) {
00159     for (i = 0; i < 50; ++i) {
00160       md5(fileKey, keyLength, fileKey);
00161     }
00162   }
00163 
00164   // test user password
00165   if (encRevision == 2) {
00166     rc4InitKey(fileKey, keyLength, fState);
00167     fx = fy = 0;
00168     for (i = 0; i < 32; ++i) {
00169       test[i] = rc4DecryptByte(fState, &fx, &fy, userKey->getChar(i));
00170     }
00171     ok = memcmp(test, passwordPad, 32) == 0;
00172   } else if (encRevision == 3) {
00173     memcpy(test, userKey->getCString(), 32);
00174     for (i = 19; i >= 0; --i) {
00175       for (j = 0; j < keyLength; ++j) {
00176        tmpKey[j] = fileKey[j] ^ i;
00177       }
00178       rc4InitKey(tmpKey, keyLength, fState);
00179       fx = fy = 0;
00180       for (j = 0; j < 32; ++j) {
00181        test[j] = rc4DecryptByte(fState, &fx, &fy, test[j]);
00182       }
00183     }
00184     memcpy(buf, passwordPad, 32);
00185     memcpy(buf + 32, fileID->getCString(), fileID->getLength());
00186     md5(buf, 32 + fileID->getLength(), buf);
00187     ok = memcmp(test, buf, 16) == 0;
00188   } else {
00189     ok = gFalse;
00190   }
00191 
00192   gfree(buf);
00193   return ok;
00194 }
00195 
00196 //------------------------------------------------------------------------
00197 // RC4-compatible decryption
00198 //------------------------------------------------------------------------
00199 
00200 static void rc4InitKey(Guchar *key, int keyLen, Guchar *state) {
00201   Guchar index1, index2;
00202   Guchar t;
00203   int i;
00204 
00205   for (i = 0; i < 256; ++i)
00206     state[i] = i;
00207   index1 = index2 = 0;
00208   for (i = 0; i < 256; ++i) {
00209     index2 = (key[index1] + state[i] + index2) % 256;
00210     t = state[i];
00211     state[i] = state[index2];
00212     state[index2] = t;
00213     index1 = (index1 + 1) % keyLen;
00214   }
00215 }
00216 
00217 static Guchar rc4DecryptByte(Guchar *state, Guchar *x, Guchar *y, Guchar c) {
00218   Guchar x1, y1, tx, ty;
00219 
00220   x1 = *x = (*x + 1) % 256;
00221   y1 = *y = (state[*x] + *y) % 256;
00222   tx = state[x1];
00223   ty = state[y1];
00224   state[x1] = ty;
00225   state[y1] = tx;
00226   return c ^ state[(tx + ty) % 256];
00227 }
00228 
00229 //------------------------------------------------------------------------
00230 // MD5 message digest
00231 //------------------------------------------------------------------------
00232 
00233 // this works around a bug in older Sun compilers
00234 static inline Gulong rotateLeft(Gulong x, int r) {
00235   x &= 0xffffffff;
00236   return ((x << r) | (x >> (32 - r))) & 0xffffffff;
00237 }
00238 
00239 static inline Gulong md5Round1(Gulong a, Gulong b, Gulong c, Gulong d,
00240                             Gulong Xk,  Gulong s, Gulong Ti) {
00241   return b + rotateLeft((a + ((b & c) | (~b & d)) + Xk + Ti), s);
00242 }
00243 
00244 static inline Gulong md5Round2(Gulong a, Gulong b, Gulong c, Gulong d,
00245                             Gulong Xk,  Gulong s, Gulong Ti) {
00246   return b + rotateLeft((a + ((b & d) | (c & ~d)) + Xk + Ti), s);
00247 }
00248 
00249 static inline Gulong md5Round3(Gulong a, Gulong b, Gulong c, Gulong d,
00250                             Gulong Xk,  Gulong s, Gulong Ti) {
00251   return b + rotateLeft((a + (b ^ c ^ d) + Xk + Ti), s);
00252 }
00253 
00254 static inline Gulong md5Round4(Gulong a, Gulong b, Gulong c, Gulong d,
00255                             Gulong Xk,  Gulong s, Gulong Ti) {
00256   return b + rotateLeft((a + (c ^ (b | ~d)) + Xk + Ti), s);
00257 }
00258 
00259 static void md5(Guchar *msg, int msgLen, Guchar *digest) {
00260   Gulong x[16];
00261   Gulong a, b, c, d, aa, bb, cc, dd;
00262   int n64;
00263   int i, j, k;
00264 
00265   // compute number of 64-byte blocks
00266   // (length + pad byte (0x80) + 8 bytes for length)
00267   n64 = (msgLen + 1 + 8 + 63) / 64;
00268 
00269   // initialize a, b, c, d
00270   a = 0x67452301;
00271   b = 0xefcdab89;
00272   c = 0x98badcfe;
00273   d = 0x10325476;
00274 
00275   // loop through blocks
00276   k = 0;
00277   for (i = 0; i < n64; ++i) {
00278 
00279     // grab a 64-byte block
00280     for (j = 0; j < 16 && k < msgLen - 3; ++j, k += 4)
00281       x[j] = (((((msg[k+3] << 8) + msg[k+2]) << 8) + msg[k+1]) << 8) + msg[k];
00282     if (i == n64 - 1) {
00283       if (k == msgLen - 3)
00284        x[j] = 0x80000000 + (((msg[k+2] << 8) + msg[k+1]) << 8) + msg[k];
00285       else if (k == msgLen - 2)
00286        x[j] = 0x800000 + (msg[k+1] << 8) + msg[k];
00287       else if (k == msgLen - 1)
00288        x[j] = 0x8000 + msg[k];
00289       else
00290        x[j] = 0x80;
00291       ++j;
00292       while (j < 16)
00293        x[j++] = 0;
00294       x[14] = msgLen << 3;
00295     }
00296 
00297     // save a, b, c, d
00298     aa = a;
00299     bb = b;
00300     cc = c;
00301     dd = d;
00302 
00303     // round 1
00304     a = md5Round1(a, b, c, d, x[0],   7, 0xd76aa478);
00305     d = md5Round1(d, a, b, c, x[1],  12, 0xe8c7b756);
00306     c = md5Round1(c, d, a, b, x[2],  17, 0x242070db);
00307     b = md5Round1(b, c, d, a, x[3],  22, 0xc1bdceee);
00308     a = md5Round1(a, b, c, d, x[4],   7, 0xf57c0faf);
00309     d = md5Round1(d, a, b, c, x[5],  12, 0x4787c62a);
00310     c = md5Round1(c, d, a, b, x[6],  17, 0xa8304613);
00311     b = md5Round1(b, c, d, a, x[7],  22, 0xfd469501);
00312     a = md5Round1(a, b, c, d, x[8],   7, 0x698098d8);
00313     d = md5Round1(d, a, b, c, x[9],  12, 0x8b44f7af);
00314     c = md5Round1(c, d, a, b, x[10], 17, 0xffff5bb1);
00315     b = md5Round1(b, c, d, a, x[11], 22, 0x895cd7be);
00316     a = md5Round1(a, b, c, d, x[12],  7, 0x6b901122);
00317     d = md5Round1(d, a, b, c, x[13], 12, 0xfd987193);
00318     c = md5Round1(c, d, a, b, x[14], 17, 0xa679438e);
00319     b = md5Round1(b, c, d, a, x[15], 22, 0x49b40821);
00320 
00321     // round 2
00322     a = md5Round2(a, b, c, d, x[1],   5, 0xf61e2562);
00323     d = md5Round2(d, a, b, c, x[6],   9, 0xc040b340);
00324     c = md5Round2(c, d, a, b, x[11], 14, 0x265e5a51);
00325     b = md5Round2(b, c, d, a, x[0],  20, 0xe9b6c7aa);
00326     a = md5Round2(a, b, c, d, x[5],   5, 0xd62f105d);
00327     d = md5Round2(d, a, b, c, x[10],  9, 0x02441453);
00328     c = md5Round2(c, d, a, b, x[15], 14, 0xd8a1e681);
00329     b = md5Round2(b, c, d, a, x[4],  20, 0xe7d3fbc8);
00330     a = md5Round2(a, b, c, d, x[9],   5, 0x21e1cde6);
00331     d = md5Round2(d, a, b, c, x[14],  9, 0xc33707d6);
00332     c = md5Round2(c, d, a, b, x[3],  14, 0xf4d50d87);
00333     b = md5Round2(b, c, d, a, x[8],  20, 0x455a14ed);
00334     a = md5Round2(a, b, c, d, x[13],  5, 0xa9e3e905);
00335     d = md5Round2(d, a, b, c, x[2],   9, 0xfcefa3f8);
00336     c = md5Round2(c, d, a, b, x[7],  14, 0x676f02d9);
00337     b = md5Round2(b, c, d, a, x[12], 20, 0x8d2a4c8a);
00338 
00339     // round 3
00340     a = md5Round3(a, b, c, d, x[5],   4, 0xfffa3942);
00341     d = md5Round3(d, a, b, c, x[8],  11, 0x8771f681);
00342     c = md5Round3(c, d, a, b, x[11], 16, 0x6d9d6122);
00343     b = md5Round3(b, c, d, a, x[14], 23, 0xfde5380c);
00344     a = md5Round3(a, b, c, d, x[1],   4, 0xa4beea44);
00345     d = md5Round3(d, a, b, c, x[4],  11, 0x4bdecfa9);
00346     c = md5Round3(c, d, a, b, x[7],  16, 0xf6bb4b60);
00347     b = md5Round3(b, c, d, a, x[10], 23, 0xbebfbc70);
00348     a = md5Round3(a, b, c, d, x[13],  4, 0x289b7ec6);
00349     d = md5Round3(d, a, b, c, x[0],  11, 0xeaa127fa);
00350     c = md5Round3(c, d, a, b, x[3],  16, 0xd4ef3085);
00351     b = md5Round3(b, c, d, a, x[6],  23, 0x04881d05);
00352     a = md5Round3(a, b, c, d, x[9],   4, 0xd9d4d039);
00353     d = md5Round3(d, a, b, c, x[12], 11, 0xe6db99e5);
00354     c = md5Round3(c, d, a, b, x[15], 16, 0x1fa27cf8);
00355     b = md5Round3(b, c, d, a, x[2],  23, 0xc4ac5665);
00356 
00357     // round 4
00358     a = md5Round4(a, b, c, d, x[0],   6, 0xf4292244);
00359     d = md5Round4(d, a, b, c, x[7],  10, 0x432aff97);
00360     c = md5Round4(c, d, a, b, x[14], 15, 0xab9423a7);
00361     b = md5Round4(b, c, d, a, x[5],  21, 0xfc93a039);
00362     a = md5Round4(a, b, c, d, x[12],  6, 0x655b59c3);
00363     d = md5Round4(d, a, b, c, x[3],  10, 0x8f0ccc92);
00364     c = md5Round4(c, d, a, b, x[10], 15, 0xffeff47d);
00365     b = md5Round4(b, c, d, a, x[1],  21, 0x85845dd1);
00366     a = md5Round4(a, b, c, d, x[8],   6, 0x6fa87e4f);
00367     d = md5Round4(d, a, b, c, x[15], 10, 0xfe2ce6e0);
00368     c = md5Round4(c, d, a, b, x[6],  15, 0xa3014314);
00369     b = md5Round4(b, c, d, a, x[13], 21, 0x4e0811a1);
00370     a = md5Round4(a, b, c, d, x[4],   6, 0xf7537e82);
00371     d = md5Round4(d, a, b, c, x[11], 10, 0xbd3af235);
00372     c = md5Round4(c, d, a, b, x[2],  15, 0x2ad7d2bb);
00373     b = md5Round4(b, c, d, a, x[9],  21, 0xeb86d391);
00374 
00375     // increment a, b, c, d
00376     a += aa;
00377     b += bb;
00378     c += cc;
00379     d += dd;
00380   }
00381 
00382   // break digest into bytes
00383   digest[0] = (Guchar)(a & 0xff);
00384   digest[1] = (Guchar)((a >>= 8) & 0xff);
00385   digest[2] = (Guchar)((a >>= 8) & 0xff);
00386   digest[3] = (Guchar)((a >>= 8) & 0xff);
00387   digest[4] = (Guchar)(b & 0xff);
00388   digest[5] = (Guchar)((b >>= 8) & 0xff);
00389   digest[6] = (Guchar)((b >>= 8) & 0xff);
00390   digest[7] = (Guchar)((b >>= 8) & 0xff);
00391   digest[8] = (Guchar)(c & 0xff);
00392   digest[9] = (Guchar)((c >>= 8) & 0xff);
00393   digest[10] = (Guchar)((c >>= 8) & 0xff);
00394   digest[11] = (Guchar)((c >>= 8) & 0xff);
00395   digest[12] = (Guchar)(d & 0xff);
00396   digest[13] = (Guchar)((d >>= 8) & 0xff);
00397   digest[14] = (Guchar)((d >>= 8) & 0xff);
00398   digest[15] = (Guchar)((d >>= 8) & 0xff);
00399 }