Back to index

lightning-sunbird  0.9+nobinonly
tokenize.c
Go to the documentation of this file.
00001 /*
00002 ** 2001 September 15
00003 **
00004 ** The author disclaims copyright to this source code.  In place of
00005 ** a legal notice, here is a blessing:
00006 **
00007 **    May you do good and not evil.
00008 **    May you find forgiveness for yourself and forgive others.
00009 **    May you share freely, never taking more than you give.
00010 **
00011 *************************************************************************
00012 ** An tokenizer for SQL
00013 **
00014 ** This file contains C code that splits an SQL input string up into
00015 ** individual tokens and sends those tokens one-by-one over to the
00016 ** parser for analysis.
00017 **
00018 ** $Id: tokenize.c,v 1.118 2006/04/04 01:54:55 drh Exp $
00019 */
00020 #include "sqliteInt.h"
00021 #include "os.h"
00022 #include <ctype.h>
00023 #include <stdlib.h>
00024 
00025 /*
00026 ** The charMap() macro maps alphabetic characters into their
00027 ** lower-case ASCII equivalent.  On ASCII machines, this is just
00028 ** an upper-to-lower case map.  On EBCDIC machines we also need
00029 ** to adjust the encoding.  Only alphabetic characters and underscores
00030 ** need to be translated.
00031 */
00032 #ifdef SQLITE_ASCII
00033 # define charMap(X) sqlite3UpperToLower[(unsigned char)X]
00034 #endif
00035 #ifdef SQLITE_EBCDIC
00036 # define charMap(X) ebcdicToAscii[(unsigned char)X]
00037 const unsigned char ebcdicToAscii[] = {
00038 /* 0   1   2   3   4   5   6   7   8   9   A   B   C   D   E   F */
00039    0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  /* 0x */
00040    0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  /* 1x */
00041    0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  /* 2x */
00042    0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  /* 3x */
00043    0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  /* 4x */
00044    0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  /* 5x */
00045    0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0, 95,  0,  0,  /* 6x */
00046    0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  /* 7x */
00047    0, 97, 98, 99,100,101,102,103,104,105,  0,  0,  0,  0,  0,  0,  /* 8x */
00048    0,106,107,108,109,110,111,112,113,114,  0,  0,  0,  0,  0,  0,  /* 9x */
00049    0,  0,115,116,117,118,119,120,121,122,  0,  0,  0,  0,  0,  0,  /* Ax */
00050    0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  /* Bx */
00051    0, 97, 98, 99,100,101,102,103,104,105,  0,  0,  0,  0,  0,  0,  /* Cx */
00052    0,106,107,108,109,110,111,112,113,114,  0,  0,  0,  0,  0,  0,  /* Dx */
00053    0,  0,115,116,117,118,119,120,121,122,  0,  0,  0,  0,  0,  0,  /* Ex */
00054    0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  /* Fx */
00055 };
00056 #endif
00057 
00058 /*
00059 ** The sqlite3KeywordCode function looks up an identifier to determine if
00060 ** it is a keyword.  If it is a keyword, the token code of that keyword is 
00061 ** returned.  If the input is not a keyword, TK_ID is returned.
00062 **
00063 ** The implementation of this routine was generated by a program,
00064 ** mkkeywordhash.h, located in the tool subdirectory of the distribution.
00065 ** The output of the mkkeywordhash.c program is written into a file
00066 ** named keywordhash.h and then included into this source file by
00067 ** the #include below.
00068 */
00069 #include "keywordhash.h"
00070 
00071 
00072 /*
00073 ** If X is a character that can be used in an identifier then
00074 ** IdChar(X) will be true.  Otherwise it is false.
00075 **
00076 ** For ASCII, any character with the high-order bit set is
00077 ** allowed in an identifier.  For 7-bit characters, 
00078 ** sqlite3IsIdChar[X] must be 1.
00079 **
00080 ** For EBCDIC, the rules are more complex but have the same
00081 ** end result.
00082 **
00083 ** Ticket #1066.  the SQL standard does not allow '$' in the
00084 ** middle of identfiers.  But many SQL implementations do. 
00085 ** SQLite will allow '$' in identifiers for compatibility.
00086 ** But the feature is undocumented.
00087 */
00088 #ifdef SQLITE_ASCII
00089 const char sqlite3IsIdChar[] = {
00090 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
00091     0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* 2x */
00092     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 3x */
00093     0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 4x */
00094     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,  /* 5x */
00095     0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 6x */
00096     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,  /* 7x */
00097 };
00098 #define IdChar(C)  (((c=C)&0x80)!=0 || (c>0x1f && sqlite3IsIdChar[c-0x20]))
00099 #endif
00100 #ifdef SQLITE_EBCDIC
00101 const char sqlite3IsIdChar[] = {
00102 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
00103     0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 4x */
00104     0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0,  /* 5x */
00105     0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0,  /* 6x */
00106     0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0,  /* 7x */
00107     0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0,  /* 8x */
00108     0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0,  /* 9x */
00109     1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0,  /* Ax */
00110     0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  /* Bx */
00111     0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1,  /* Cx */
00112     0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1,  /* Dx */
00113     0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1,  /* Ex */
00114     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0,  /* Fx */
00115 };
00116 #define IdChar(C)  (((c=C)>=0x42 && sqlite3IsIdChar[c-0x40]))
00117 #endif
00118 
00119 
00120 /*
00121 ** Return the length of the token that begins at z[0]. 
00122 ** Store the token type in *tokenType before returning.
00123 */
00124 static int getToken(const unsigned char *z, int *tokenType){
00125   int i, c;
00126   switch( *z ){
00127     case ' ': case '\t': case '\n': case '\f': case '\r': {
00128       for(i=1; isspace(z[i]); i++){}
00129       *tokenType = TK_SPACE;
00130       return i;
00131     }
00132     case '-': {
00133       if( z[1]=='-' ){
00134         for(i=2; (c=z[i])!=0 && c!='\n'; i++){}
00135         *tokenType = TK_COMMENT;
00136         return i;
00137       }
00138       *tokenType = TK_MINUS;
00139       return 1;
00140     }
00141     case '(': {
00142       *tokenType = TK_LP;
00143       return 1;
00144     }
00145     case ')': {
00146       *tokenType = TK_RP;
00147       return 1;
00148     }
00149     case ';': {
00150       *tokenType = TK_SEMI;
00151       return 1;
00152     }
00153     case '+': {
00154       *tokenType = TK_PLUS;
00155       return 1;
00156     }
00157     case '*': {
00158       *tokenType = TK_STAR;
00159       return 1;
00160     }
00161     case '/': {
00162       if( z[1]!='*' || z[2]==0 ){
00163         *tokenType = TK_SLASH;
00164         return 1;
00165       }
00166       for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){}
00167       if( c ) i++;
00168       *tokenType = TK_COMMENT;
00169       return i;
00170     }
00171     case '%': {
00172       *tokenType = TK_REM;
00173       return 1;
00174     }
00175     case '=': {
00176       *tokenType = TK_EQ;
00177       return 1 + (z[1]=='=');
00178     }
00179     case '<': {
00180       if( (c=z[1])=='=' ){
00181         *tokenType = TK_LE;
00182         return 2;
00183       }else if( c=='>' ){
00184         *tokenType = TK_NE;
00185         return 2;
00186       }else if( c=='<' ){
00187         *tokenType = TK_LSHIFT;
00188         return 2;
00189       }else{
00190         *tokenType = TK_LT;
00191         return 1;
00192       }
00193     }
00194     case '>': {
00195       if( (c=z[1])=='=' ){
00196         *tokenType = TK_GE;
00197         return 2;
00198       }else if( c=='>' ){
00199         *tokenType = TK_RSHIFT;
00200         return 2;
00201       }else{
00202         *tokenType = TK_GT;
00203         return 1;
00204       }
00205     }
00206     case '!': {
00207       if( z[1]!='=' ){
00208         *tokenType = TK_ILLEGAL;
00209         return 2;
00210       }else{
00211         *tokenType = TK_NE;
00212         return 2;
00213       }
00214     }
00215     case '|': {
00216       if( z[1]!='|' ){
00217         *tokenType = TK_BITOR;
00218         return 1;
00219       }else{
00220         *tokenType = TK_CONCAT;
00221         return 2;
00222       }
00223     }
00224     case ',': {
00225       *tokenType = TK_COMMA;
00226       return 1;
00227     }
00228     case '&': {
00229       *tokenType = TK_BITAND;
00230       return 1;
00231     }
00232     case '~': {
00233       *tokenType = TK_BITNOT;
00234       return 1;
00235     }
00236     case '`':
00237     case '\'':
00238     case '"': {
00239       int delim = z[0];
00240       for(i=1; (c=z[i])!=0; i++){
00241         if( c==delim ){
00242           if( z[i+1]==delim ){
00243             i++;
00244           }else{
00245             break;
00246           }
00247         }
00248       }
00249       if( c ){
00250         *tokenType = TK_STRING;
00251         return i+1;
00252       }else{
00253         *tokenType = TK_ILLEGAL;
00254         return i;
00255       }
00256     }
00257     case '.': {
00258 #ifndef SQLITE_OMIT_FLOATING_POINT
00259       if( !isdigit(z[1]) )
00260 #endif
00261       {
00262         *tokenType = TK_DOT;
00263         return 1;
00264       }
00265       /* If the next character is a digit, this is a floating point
00266       ** number that begins with ".".  Fall thru into the next case */
00267     }
00268     case '0': case '1': case '2': case '3': case '4':
00269     case '5': case '6': case '7': case '8': case '9': {
00270       *tokenType = TK_INTEGER;
00271       for(i=0; isdigit(z[i]); i++){}
00272 #ifndef SQLITE_OMIT_FLOATING_POINT
00273       if( z[i]=='.' ){
00274         i++;
00275         while( isdigit(z[i]) ){ i++; }
00276         *tokenType = TK_FLOAT;
00277       }
00278       if( (z[i]=='e' || z[i]=='E') &&
00279            ( isdigit(z[i+1]) 
00280             || ((z[i+1]=='+' || z[i+1]=='-') && isdigit(z[i+2]))
00281            )
00282       ){
00283         i += 2;
00284         while( isdigit(z[i]) ){ i++; }
00285         *tokenType = TK_FLOAT;
00286       }
00287 #endif
00288       return i;
00289     }
00290     case '[': {
00291       for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
00292       *tokenType = TK_ID;
00293       return i;
00294     }
00295     case '?': {
00296       *tokenType = TK_VARIABLE;
00297       for(i=1; isdigit(z[i]); i++){}
00298       return i;
00299     }
00300     case '#': {
00301       for(i=1; isdigit(z[i]); i++){}
00302       if( i>1 ){
00303         /* Parameters of the form #NNN (where NNN is a number) are used
00304         ** internally by sqlite3NestedParse.  */
00305         *tokenType = TK_REGISTER;
00306         return i;
00307       }
00308       /* Fall through into the next case if the '#' is not followed by
00309       ** a digit. Try to match #AAAA where AAAA is a parameter name. */
00310     }
00311 #ifndef SQLITE_OMIT_TCL_VARIABLE
00312     case '$':
00313 #endif
00314     case '@':  /* For compatibility with MS SQL Server */
00315     case ':': {
00316       int n = 0;
00317       *tokenType = TK_VARIABLE;
00318       for(i=1; (c=z[i])!=0; i++){
00319         if( IdChar(c) ){
00320           n++;
00321 #ifndef SQLITE_OMIT_TCL_VARIABLE
00322         }else if( c=='(' && n>0 ){
00323           do{
00324             i++;
00325           }while( (c=z[i])!=0 && !isspace(c) && c!=')' );
00326           if( c==')' ){
00327             i++;
00328           }else{
00329             *tokenType = TK_ILLEGAL;
00330           }
00331           break;
00332         }else if( c==':' && z[i+1]==':' ){
00333           i++;
00334 #endif
00335         }else{
00336           break;
00337         }
00338       }
00339       if( n==0 ) *tokenType = TK_ILLEGAL;
00340       return i;
00341     }
00342 #ifndef SQLITE_OMIT_BLOB_LITERAL
00343     case 'x': case 'X': {
00344       if( (c=z[1])=='\'' || c=='"' ){
00345         int delim = c;
00346         *tokenType = TK_BLOB;
00347         for(i=2; (c=z[i])!=0; i++){
00348           if( c==delim ){
00349             if( i%2 ) *tokenType = TK_ILLEGAL;
00350             break;
00351           }
00352           if( !isxdigit(c) ){
00353             *tokenType = TK_ILLEGAL;
00354             return i;
00355           }
00356         }
00357         if( c ) i++;
00358         return i;
00359       }
00360       /* Otherwise fall through to the next case */
00361     }
00362 #endif
00363     default: {
00364       if( !IdChar(*z) ){
00365         break;
00366       }
00367       for(i=1; IdChar(z[i]); i++){}
00368       *tokenType = keywordCode((char*)z, i);
00369       return i;
00370     }
00371   }
00372   *tokenType = TK_ILLEGAL;
00373   return 1;
00374 }
00375 int sqlite3GetToken(const unsigned char *z, int *tokenType){
00376   return getToken(z, tokenType);
00377 }
00378 
00379 /*
00380 ** Run the parser on the given SQL string.  The parser structure is
00381 ** passed in.  An SQLITE_ status code is returned.  If an error occurs
00382 ** and pzErrMsg!=NULL then an error message might be written into 
00383 ** memory obtained from malloc() and *pzErrMsg made to point to that
00384 ** error message.  Or maybe not.
00385 */
00386 int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
00387   int nErr = 0;
00388   int i;
00389   void *pEngine;
00390   int tokenType;
00391   int lastTokenParsed = -1;
00392   sqlite3 *db = pParse->db;
00393   extern void *sqlite3ParserAlloc(void*(*)(int));
00394   extern void sqlite3ParserFree(void*, void(*)(void*));
00395   extern int sqlite3Parser(void*, int, Token, Parse*);
00396 
00397   db->flags &= ~SQLITE_Interrupt;
00398   pParse->rc = SQLITE_OK;
00399   i = 0;
00400   pEngine = sqlite3ParserAlloc((void*(*)(int))sqlite3MallocX);
00401   if( pEngine==0 ){
00402     return SQLITE_NOMEM;
00403   }
00404   assert( pParse->sLastToken.dyn==0 );
00405   assert( pParse->pNewTable==0 );
00406   assert( pParse->pNewTrigger==0 );
00407   assert( pParse->nVar==0 );
00408   assert( pParse->nVarExpr==0 );
00409   assert( pParse->nVarExprAlloc==0 );
00410   assert( pParse->apVarExpr==0 );
00411   pParse->zTail = pParse->zSql = zSql;
00412   while( !sqlite3MallocFailed() && zSql[i]!=0 ){
00413     assert( i>=0 );
00414     pParse->sLastToken.z = (u8*)&zSql[i];
00415     assert( pParse->sLastToken.dyn==0 );
00416     pParse->sLastToken.n = getToken((unsigned char*)&zSql[i],&tokenType);
00417     i += pParse->sLastToken.n;
00418     switch( tokenType ){
00419       case TK_SPACE:
00420       case TK_COMMENT: {
00421         if( (db->flags & SQLITE_Interrupt)!=0 ){
00422           pParse->rc = SQLITE_INTERRUPT;
00423           sqlite3SetString(pzErrMsg, "interrupt", (char*)0);
00424           goto abort_parse;
00425         }
00426         break;
00427       }
00428       case TK_ILLEGAL: {
00429         if( pzErrMsg ){
00430           sqliteFree(*pzErrMsg);
00431           *pzErrMsg = sqlite3MPrintf("unrecognized token: \"%T\"",
00432                           &pParse->sLastToken);
00433         }
00434         nErr++;
00435         goto abort_parse;
00436       }
00437       case TK_SEMI: {
00438         pParse->zTail = &zSql[i];
00439         /* Fall thru into the default case */
00440       }
00441       default: {
00442         sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse);
00443         lastTokenParsed = tokenType;
00444         if( pParse->rc!=SQLITE_OK ){
00445           goto abort_parse;
00446         }
00447         break;
00448       }
00449     }
00450   }
00451 abort_parse:
00452   if( zSql[i]==0 && nErr==0 && pParse->rc==SQLITE_OK ){
00453     if( lastTokenParsed!=TK_SEMI ){
00454       sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse);
00455       pParse->zTail = &zSql[i];
00456     }
00457     sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse);
00458   }
00459   sqlite3ParserFree(pEngine, sqlite3FreeX);
00460   if( sqlite3MallocFailed() ){
00461     pParse->rc = SQLITE_NOMEM;
00462   }
00463   if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){
00464     sqlite3SetString(&pParse->zErrMsg, sqlite3ErrStr(pParse->rc), (char*)0);
00465   }
00466   if( pParse->zErrMsg ){
00467     if( pzErrMsg && *pzErrMsg==0 ){
00468       *pzErrMsg = pParse->zErrMsg;
00469     }else{
00470       sqliteFree(pParse->zErrMsg);
00471     }
00472     pParse->zErrMsg = 0;
00473     if( !nErr ) nErr++;
00474   }
00475   if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){
00476     sqlite3VdbeDelete(pParse->pVdbe);
00477     pParse->pVdbe = 0;
00478   }
00479 #ifndef SQLITE_OMIT_SHARED_CACHE
00480   if( pParse->nested==0 ){
00481     sqliteFree(pParse->aTableLock);
00482     pParse->aTableLock = 0;
00483     pParse->nTableLock = 0;
00484   }
00485 #endif
00486   sqlite3DeleteTable(pParse->db, pParse->pNewTable);
00487   sqlite3DeleteTrigger(pParse->pNewTrigger);
00488   sqliteFree(pParse->apVarExpr);
00489   if( nErr>0 && (pParse->rc==SQLITE_OK || pParse->rc==SQLITE_DONE) ){
00490     pParse->rc = SQLITE_ERROR;
00491   }
00492   return nErr;
00493 }