Back to index

lightning-sunbird  0.9+nobinonly
Defines | Functions | Variables
utf.c File Reference
#include "sqliteInt.h"
#include <assert.h>
#include "vdbeInt.h"

Go to the source code of this file.

Defines

#define READ_UTF8(zIn, c)
#define SKIP_UTF8(zIn)
#define WRITE_UTF8(zOut, c)
#define WRITE_UTF16LE(zOut, c)
#define WRITE_UTF16BE(zOut, c)
#define READ_UTF16LE(zIn, c)
#define READ_UTF16BE(zIn, c)
#define SKIP_UTF16BE(zIn)
#define SKIP_UTF16LE(zIn)
#define RSKIP_UTF16LE(zIn)
#define RSKIP_UTF16BE(zIn)

Functions

int sqlite3ReadUtf8 (const unsigned char *z)
int sqlite3VdbeMemTranslate (Mem *pMem, u8 desiredEnc)
int sqlite3VdbeMemHandleBom (Mem *pMem)
int sqlite3utf8CharLen (const char *z, int nByte)
char * sqlite3utf16to8 (const void *z, int nByte)
int sqlite3utf16ByteLen (const void *zIn, int nChar)
void sqlite3utf16Substr (sqlite3_context *context, int argc, sqlite3_value **argv)

Variables

static const u8 xtra_utf8_bytes [256]
static const int xtra_utf8_bits [4]

Define Documentation

#define READ_UTF16BE (   zIn,
  c 
)
Value:
{                                         \
  c = ((*zIn++)<<8);                                                  \
  c += (*zIn++);                                                      \
  if( c>=0xD800 && c<=0xE000 ){                                       \
    int c2 = ((*zIn++)<<8);                                           \
    c2 += (*zIn++);                                                   \
    c = (c2&0x03FF) + ((c&0x003F)<<10) + (((c&0x03C0)+0x0040)<<10);   \
  }                                                                   \
}

Definition at line 187 of file utf.c.

#define READ_UTF16LE (   zIn,
  c 
)
Value:
{                                         \
  c = (*zIn++);                                                       \
  c += ((*zIn++)<<8);                                                 \
  if( c>=0xD800 && c<=0xE000 ){                                       \
    int c2 = (*zIn++);                                                \
    c2 += ((*zIn++)<<8);                                              \
    c = (c2&0x03FF) + ((c&0x003F)<<10) + (((c&0x03C0)+0x0040)<<10);   \
  }                                                                   \
}

Definition at line 177 of file utf.c.

#define READ_UTF8 (   zIn,
  c 
)
Value:
{ \
  int xtra;                                            \
  c = *(zIn)++;                                        \
  xtra = xtra_utf8_bytes[c];                           \
  switch( xtra ){                                      \
    case 255: c = (int)0xFFFD; break;                  \
    case 3: c = (c<<6) + *(zIn)++;                     \
    case 2: c = (c<<6) + *(zIn)++;                     \
    case 1: c = (c<<6) + *(zIn)++;                     \
    c -= xtra_utf8_bits[xtra];                         \
  }                                                    \
}

Definition at line 111 of file utf.c.

#define RSKIP_UTF16BE (   zIn)
Value:
{                                            \
  zIn--;                                                              \
  if( *zIn>=0xD8 && (*zIn<0xE0 || (*zIn==0xE0 && *(zIn+1)==0x00)) ){  \
    zIn -= 3;                                                         \
  }else{                                                              \
    zIn -= 1;                                                         \
  }                                                                   \
}

Definition at line 220 of file utf.c.

#define RSKIP_UTF16LE (   zIn)
Value:
{                                            \
  if( *zIn>=0xD8 && (*zIn<0xE0 || (*zIn==0xE0 && *(zIn-1)==0x00)) ){  \
    zIn -= 4;                                                         \
  }else{                                                              \
    zIn -= 2;                                                         \
  }                                                                   \
}

Definition at line 213 of file utf.c.

#define SKIP_UTF16BE (   zIn)
Value:
{                                            \
  if( *zIn>=0xD8 && (*zIn<0xE0 || (*zIn==0xE0 && *(zIn+1)==0x00)) ){  \
    zIn += 4;                                                         \
  }else{                                                              \
    zIn += 2;                                                         \
  }                                                                   \
}

Definition at line 197 of file utf.c.

#define SKIP_UTF16LE (   zIn)
Value:
{                                            \
  zIn++;                                                              \
  if( *zIn>=0xD8 && (*zIn<0xE0 || (*zIn==0xE0 && *(zIn-1)==0x00)) ){  \
    zIn += 3;                                                         \
  }else{                                                              \
    zIn += 1;                                                         \
  }                                                                   \
}

Definition at line 204 of file utf.c.

#define SKIP_UTF8 (   zIn)
Value:
{                               \
  zIn += (xtra_utf8_bytes[*(u8 *)zIn] + 1);            \
}

Definition at line 129 of file utf.c.

#define WRITE_UTF16BE (   zOut,
  c 
)
Value:
{                                \
  if( c<=0xFFFF ){                                              \
    *zOut++ = ((c>>8)&0x00FF);                                  \
    *zOut++ = (c&0x00FF);                                       \
  }else{                                                        \
    *zOut++ = (0x00D8 + (((c-0x10000)>>18)&0x03));              \
    *zOut++ = (((c>>10)&0x003F) + (((c-0x10000)>>10)&0x00C0));  \
    *zOut++ = (0x00DC + ((c>>8)&0x03));                         \
    *zOut++ = (c&0x00FF);                                       \
  }                                                             \
}

Definition at line 165 of file utf.c.

#define WRITE_UTF16LE (   zOut,
  c 
)
Value:
{                                \
  if( c<=0xFFFF ){                                              \
    *zOut++ = (c&0x00FF);                                       \
    *zOut++ = ((c>>8)&0x00FF);                                  \
  }else{                                                        \
    *zOut++ = (((c>>10)&0x003F) + (((c-0x10000)>>10)&0x00C0));  \
    *zOut++ = (0x00D8 + (((c-0x10000)>>18)&0x03));              \
    *zOut++ = (c&0x00FF);                                       \
    *zOut++ = (0x00DC + ((c>>8)&0x03));                         \
  }                                                             \
}

Definition at line 153 of file utf.c.

#define WRITE_UTF8 (   zOut,
  c 
)
Value:
{                          \
  if( c<0x00080 ){                                     \
    *zOut++ = (c&0xFF);                                \
  }                                                    \
  else if( c<0x00800 ){                                \
    *zOut++ = 0xC0 + ((c>>6)&0x1F);                    \
    *zOut++ = 0x80 + (c & 0x3F);                       \
  }                                                    \
  else if( c<0x10000 ){                                \
    *zOut++ = 0xE0 + ((c>>12)&0x0F);                   \
    *zOut++ = 0x80 + ((c>>6) & 0x3F);                  \
    *zOut++ = 0x80 + (c & 0x3F);                       \
  }else{                                               \
    *zOut++ = 0xF0 + ((c>>18) & 0x07);                 \
    *zOut++ = 0x80 + ((c>>12) & 0x3F);                 \
    *zOut++ = 0x80 + ((c>>6) & 0x3F);                  \
    *zOut++ = 0x80 + (c & 0x3F);                       \
  }                                                    \
}

Definition at line 133 of file utf.c.


Function Documentation

int sqlite3ReadUtf8 ( const unsigned char *  z)

Definition at line 123 of file utf.c.

                                           {
  int c;
  READ_UTF8(z, c);
  return c;
}

Here is the caller graph for this function:

int sqlite3utf16ByteLen ( const void zIn,
int  nChar 
)

Definition at line 477 of file utf.c.

                                                   {
  int c = 1;
  char const *z = zIn;
  int n = 0;
  if( SQLITE_UTF16NATIVE==SQLITE_UTF16BE ){
    /* Using an "if (SQLITE_UTF16NATIVE==SQLITE_UTF16BE)" construct here
    ** and in other parts of this file means that at one branch will
    ** not be covered by coverage testing on any single host. But coverage
    ** will be complete if the tests are run on both a little-endian and 
    ** big-endian host. Because both the UTF16NATIVE and SQLITE_UTF16BE
    ** macros are constant at compile time the compiler can determine
    ** which branch will be followed. It is therefore assumed that no runtime
    ** penalty is paid for this "if" statement.
    */
    while( c && ((nChar<0) || n<nChar) ){
      READ_UTF16BE(z, c);
      n++;
    }
  }else{
    while( c && ((nChar<0) || n<nChar) ){
      READ_UTF16LE(z, c);
      n++;
    }
  }
  return (z-(char const *)zIn)-((c==0)?2:0);
}

Here is the caller graph for this function:

void sqlite3utf16Substr ( sqlite3_context context,
int  argc,
sqlite3_value **  argv 
)

Definition at line 507 of file utf.c.

 {
  int y, z;
  unsigned char const *zStr;
  unsigned char const *zStrEnd;
  unsigned char const *zStart;
  unsigned char const *zEnd;
  int i;

  zStr = (unsigned char const *)sqlite3_value_text16(argv[0]);
  zStrEnd = &zStr[sqlite3_value_bytes16(argv[0])];
  y = sqlite3_value_int(argv[1]);
  z = sqlite3_value_int(argv[2]);

  if( y>0 ){
    y = y-1;
    zStart = zStr;
    if( SQLITE_UTF16BE==SQLITE_UTF16NATIVE ){
      for(i=0; i<y && zStart<zStrEnd; i++) SKIP_UTF16BE(zStart);
    }else{
      for(i=0; i<y && zStart<zStrEnd; i++) SKIP_UTF16LE(zStart);
    }
  }else{
    zStart = zStrEnd;
    if( SQLITE_UTF16BE==SQLITE_UTF16NATIVE ){
      for(i=y; i<0 && zStart>zStr; i++) RSKIP_UTF16BE(zStart);
    }else{
      for(i=y; i<0 && zStart>zStr; i++) RSKIP_UTF16LE(zStart);
    }
    for(; i<0; i++) z -= 1;
  }

  zEnd = zStart;
  if( SQLITE_UTF16BE==SQLITE_UTF16NATIVE ){
    for(i=0; i<z && zEnd<zStrEnd; i++) SKIP_UTF16BE(zEnd);
  }else{
    for(i=0; i<z && zEnd<zStrEnd; i++) SKIP_UTF16LE(zEnd);
  }

  sqlite3_result_text16(context, zStart, zEnd-zStart, SQLITE_TRANSIENT);
}

Here is the call graph for this function:

Here is the caller graph for this function:

char* sqlite3utf16to8 ( const void z,
int  nByte 
)

Definition at line 460 of file utf.c.

                                               {
  Mem m;
  memset(&m, 0, sizeof(m));
  sqlite3VdbeMemSetStr(&m, z, nByte, SQLITE_UTF16NATIVE, SQLITE_STATIC);
  sqlite3VdbeChangeEncoding(&m, SQLITE_UTF8);
  assert( m.flags & MEM_Term );
  assert( m.flags & MEM_Str );
  return (m.flags & MEM_Dyn)!=0 ? m.z : sqliteStrDup(m.z);
}

Here is the call graph for this function:

Here is the caller graph for this function:

int sqlite3utf8CharLen ( const char *  z,
int  nByte 
)

Definition at line 436 of file utf.c.

                                                {
  int r = 0;
  const char *zTerm;
  if( nByte>=0 ){
    zTerm = &z[nByte];
  }else{
    zTerm = (const char *)(-1);
  }
  assert( z<=zTerm );
  while( *z!=0 && z<zTerm ){
    SKIP_UTF8(z);
    r++;
  }
  return r;
}

Here is the caller graph for this function:

Definition at line 390 of file utf.c.

                                      {
  int rc = SQLITE_OK;
  u8 bom = 0;

  if( pMem->n<0 || pMem->n>1 ){
    u8 b1 = *(u8 *)pMem->z;
    u8 b2 = *(((u8 *)pMem->z) + 1);
    if( b1==0xFE && b2==0xFF ){
      bom = SQLITE_UTF16BE;
    }
    if( b1==0xFF && b2==0xFE ){
      bom = SQLITE_UTF16LE;
    }
  }
  
  if( bom ){
    /* This function is called as soon as a string is stored in a Mem*,
    ** from within sqlite3VdbeMemSetStr(). At that point it is not possible
    ** for the string to be stored in Mem.zShort, or for it to be stored
    ** in dynamic memory with no destructor.
    */
    assert( !(pMem->flags&MEM_Short) );
    assert( !(pMem->flags&MEM_Dyn) || pMem->xDel );
    if( pMem->flags & MEM_Dyn ){
      void (*xDel)(void*) = pMem->xDel;
      char *z = pMem->z;
      pMem->z = 0;
      pMem->xDel = 0;
      rc = sqlite3VdbeMemSetStr(pMem, &z[2], pMem->n-2, bom, SQLITE_TRANSIENT);
      xDel(z);
    }else{
      rc = sqlite3VdbeMemSetStr(pMem, &pMem->z[2], pMem->n-2, bom, 
          SQLITE_TRANSIENT);
    }
  }
  return rc;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int sqlite3VdbeMemTranslate ( Mem pMem,
u8  desiredEnc 
)

Definition at line 241 of file utf.c.

                                                     {
  unsigned char zShort[NBFS]; /* Temporary short output buffer */
  int len;                    /* Maximum length of output string in bytes */
  unsigned char *zOut;                  /* Output buffer */
  unsigned char *zIn;                   /* Input iterator */
  unsigned char *zTerm;                 /* End of input */
  unsigned char *z;                     /* Output iterator */
  int c;

  assert( pMem->flags&MEM_Str );
  assert( pMem->enc!=desiredEnc );
  assert( pMem->enc!=0 );
  assert( pMem->n>=0 );

#if defined(TRANSLATE_TRACE) && defined(SQLITE_DEBUG)
  {
    char zBuf[100];
    sqlite3VdbeMemPrettyPrint(pMem, zBuf);
    fprintf(stderr, "INPUT:  %s\n", zBuf);
  }
#endif

  /* If the translation is between UTF-16 little and big endian, then 
  ** all that is required is to swap the byte order. This case is handled
  ** differently from the others.
  */
  if( pMem->enc!=SQLITE_UTF8 && desiredEnc!=SQLITE_UTF8 ){
    u8 temp;
    int rc;
    rc = sqlite3VdbeMemMakeWriteable(pMem);
    if( rc!=SQLITE_OK ){
      assert( rc==SQLITE_NOMEM );
      return SQLITE_NOMEM;
    }
    zIn = (u8*)pMem->z;
    zTerm = &zIn[pMem->n];
    while( zIn<zTerm ){
      temp = *zIn;
      *zIn = *(zIn+1);
      zIn++;
      *zIn++ = temp;
    }
    pMem->enc = desiredEnc;
    goto translate_out;
  }

  /* Set len to the maximum number of bytes required in the output buffer. */
  if( desiredEnc==SQLITE_UTF8 ){
    /* When converting from UTF-16, the maximum growth results from
    ** translating a 2-byte character to a 4-byte UTF-8 character.
    ** A single byte is required for the output string
    ** nul-terminator.
    */
    len = pMem->n * 2 + 1;
  }else{
    /* When converting from UTF-8 to UTF-16 the maximum growth is caused
    ** when a 1-byte UTF-8 character is translated into a 2-byte UTF-16
    ** character. Two bytes are required in the output buffer for the
    ** nul-terminator.
    */
    len = pMem->n * 2 + 2;
  }

  /* Set zIn to point at the start of the input buffer and zTerm to point 1
  ** byte past the end.
  **
  ** Variable zOut is set to point at the output buffer. This may be space
  ** obtained from malloc(), or Mem.zShort, if it large enough and not in
  ** use, or the zShort array on the stack (see above).
  */
  zIn = (u8*)pMem->z;
  zTerm = &zIn[pMem->n];
  if( len>NBFS ){
    zOut = sqliteMallocRaw(len);
    if( !zOut ) return SQLITE_NOMEM;
  }else{
    zOut = zShort;
  }
  z = zOut;

  if( pMem->enc==SQLITE_UTF8 ){
    if( desiredEnc==SQLITE_UTF16LE ){
      /* UTF-8 -> UTF-16 Little-endian */
      while( zIn<zTerm ){
        READ_UTF8(zIn, c); 
        WRITE_UTF16LE(z, c);
      }
    }else{
      assert( desiredEnc==SQLITE_UTF16BE );
      /* UTF-8 -> UTF-16 Big-endian */
      while( zIn<zTerm ){
        READ_UTF8(zIn, c); 
        WRITE_UTF16BE(z, c);
      }
    }
    pMem->n = z - zOut;
    *z++ = 0;
  }else{
    assert( desiredEnc==SQLITE_UTF8 );
    if( pMem->enc==SQLITE_UTF16LE ){
      /* UTF-16 Little-endian -> UTF-8 */
      while( zIn<zTerm ){
        READ_UTF16LE(zIn, c); 
        WRITE_UTF8(z, c);
      }
    }else{
      /* UTF-16 Little-endian -> UTF-8 */
      while( zIn<zTerm ){
        READ_UTF16BE(zIn, c); 
        WRITE_UTF8(z, c);
      }
    }
    pMem->n = z - zOut;
  }
  *z = 0;
  assert( (pMem->n+(desiredEnc==SQLITE_UTF8?1:2))<=len );

  sqlite3VdbeMemRelease(pMem);
  pMem->flags &= ~(MEM_Static|MEM_Dyn|MEM_Ephem|MEM_Short);
  pMem->enc = desiredEnc;
  if( zOut==zShort ){
    memcpy(pMem->zShort, zOut, len);
    zOut = (u8*)pMem->zShort;
    pMem->flags |= (MEM_Term|MEM_Short);
  }else{
    pMem->flags |= (MEM_Term|MEM_Dyn);
  }
  pMem->z = (char*)zOut;

translate_out:
#if defined(TRANSLATE_TRACE) && defined(SQLITE_DEBUG)
  {
    char zBuf[100];
    sqlite3VdbeMemPrettyPrint(pMem, zBuf);
    fprintf(stderr, "OUTPUT: %s\n", zBuf);
  }
#endif
  return SQLITE_OK;
}

Here is the call graph for this function:

Here is the caller graph for this function:


Variable Documentation

const int xtra_utf8_bits[4] [static]
Initial value:
  {
0,
12416,          
925824,         
63447168        
}

Definition at line 104 of file utf.c.

const u8 xtra_utf8_bytes[256] [static]
Initial value:
 {

0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0,     0, 0, 0, 0, 0, 0, 0, 0,


255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,


1, 1, 1, 1, 1, 1, 1, 1,     1, 1, 1, 1, 1, 1, 1, 1,
1, 1, 1, 1, 1, 1, 1, 1,     1, 1, 1, 1, 1, 1, 1, 1,


2, 2, 2, 2, 2, 2, 2, 2,     2, 2, 2, 2, 2, 2, 2, 2,


3, 3, 3, 3, 3, 3, 3, 3,     255, 255, 255, 255, 255, 255, 255, 255,
}

Definition at line 70 of file utf.c.