Back to index

lightning-sunbird  0.9+nobinonly
Classes | Defines | Functions | Variables
mozStorageAsyncIO.cpp File Reference
#include "mozStorageService.h"
#include "nsAutoLock.h"
#include "nsEventQueueUtils.h"
#include "nsIConsoleService.h"
#include "nsIPrompt.h"
#include "nsIRunnable.h"
#include "nsIStringBundle.h"
#include "nsIThread.h"
#include "nsMemory.h"
#include "nsNetCID.h"
#include "nsProxyRelease.h"
#include "nsString.h"
#include "plstr.h"
#include "prlock.h"
#include "prcvar.h"
#include "prtypes.h"
#include "sqlite3.h"
#include "sqlite3file.h"

Go to the source code of this file.

Classes

struct  AsyncOsFile
struct  AsyncMessage
struct  AsyncMessageBarrierData
class  AsyncWriteThread
class  nsAsyncWriteErrorDisplayer

Defines

#define SQLITE_ASYNC_TWO_FILEHANDLES   1
 This code is partially based on test_async.c distributed with sqlite.
#define ASYNC_WRITE   1
#define ASYNC_SYNC   2
#define ASYNC_TRUNCATE   3
#define ASYNC_CLOSE   4
#define ASYNC_OPENDIRECTORY   5
#define ASYNC_SETFULLSYNC   6
#define ASYNC_DELETE   7
#define ASYNC_OPENEXCLUSIVE   8
#define ASYNC_SYNCDIRECTORY   9
#define ASYNC_BARRIER   10

Functions

static int AsyncOpenReadWrite (const char *aName, OsFile **aFile, int *aReadOnly)
static int AsyncOpenExclusive (const char *aName, OsFile **aFile, int aDelFlag)
static int AsyncOpenReadOnly (const char *aName, OsFile **aFile)
static int AsyncDelete (const char *aName)
static int AsyncSyncDirectory (const char *aName)
static int AsyncFileExists (const char *aName)
static int AsyncClose (OsFile **aFile)
static int AsyncWrite (OsFile *aFile, const void *aBuf, int aCount)
static int AsyncTruncate (OsFile *aFile, sqlite_int64 aNumBytes)
static int AsyncOpenDirectory (OsFile *aFile, const char *aName)
static int AsyncSync (OsFile *aFile, int aFullsync)
static void AsyncSetFullSync (OsFile *aFile, int aValue)
static int AsyncRead (OsFile *aFile, void *aBuffer, int aCount)
static int AsyncSeek (OsFile *aFile, sqlite_int64 aOffset)
static int AsyncFileSize (OsFile *aFile, sqlite_int64 *aSize)
static int AsyncFileHandle (OsFile *aFile)
static int AsyncLock (OsFile *aFile, int aLockType)
static int AsyncUnlock (OsFile *aFile, int aLockType)
static int AsyncCheckReservedLock (OsFile *aFile)
static int AsyncLockState (OsFile *aFile)
static int AsyncBarrier (PRLock *aLock, PRCondVar *aCondVar)
static int AsyncOpenFile (const char *aName, AsyncOsFile **aFile, OsFile *aBaseRead, PRBool aOpenForWriting)
static void ProcessAsyncMessages ()
static int ProcessOneMessage (AsyncMessage *aMessage)
static void AppendAsyncMessage (AsyncMessage *aMessage)
static int AppendNewAsyncMessage (AsyncOsFile *aFile, PRUint32 aOp, sqlite_int64 aOffset, PRInt32 aDataSize, const char *aData)
static void DisplayAsyncWriteError ()
 HandleWriteErrorDisplayPLEvent (PLEvent *aEvent)
 DestroyWriteErrorDisplayerPLEvent (PLEvent *aEvent)

Variables

static AsyncMessageAsyncQueueFirst = nsnull
static AsyncMessageAsyncQueueLast = nsnull
static PRBool AsyncWriterHaltWhenIdle = PR_FALSE
static int AsyncWriteError = SQLITE_OK
static nsIThreadAsyncWriteThreadInstance = nsnull
static PRLockAsyncQueueLock = nsnull
static PRCondVarAsyncQueueCondition = nsnull
static int(* sqliteOrigOpenReadWrite )(const char *, OsFile **, int *) = nsnull
static int(* sqliteOrigOpenExclusive )(const char *, OsFile **, int) = nsnull
static int(* sqliteOrigOpenReadOnly )(const char *, OsFile **) = nsnull
static int(* sqliteOrigDelete )(const char *) = nsnull
static int(* sqliteOrigFileExists )(const char *) = nsnull
static int(* sqliteOrigSyncDirectory )(const char *) = nsnull
static int(* sqliteOrigClose )(OsFile **) = nsnull
static int(* sqliteOrigRead )(OsFile *, void *, int amt) = nsnull
static int(* sqliteOrigWrite )(OsFile *, const void *, int amt) = nsnull
static int(* sqliteOrigFileSize )(OsFile *, sqlite_int64 *pSize) = nsnull
static int(* sqliteOrigSeek )(OsFile *, sqlite_int64 offset) = nsnull
static int(* sqliteOrigSync )(OsFile *, int) = nsnull
static int(* sqliteOrigTruncate )(OsFile *, sqlite_int64 size) = nsnull
static int(* sqliteOrigOpenDirectory )(OsFile *, const char *)
static void(* sqliteOrigSetFullSync )(OsFile *, int setting)

Class Documentation

struct AsyncMessageBarrierData

Definition at line 366 of file mozStorageAsyncIO.cpp.

Collaboration diagram for AsyncMessageBarrierData:
Class Members
PRCondVar * mCondVar
PRLock * mLock

Define Documentation

Definition at line 382 of file mozStorageAsyncIO.cpp.

Definition at line 376 of file mozStorageAsyncIO.cpp.

Definition at line 379 of file mozStorageAsyncIO.cpp.

Definition at line 377 of file mozStorageAsyncIO.cpp.

Definition at line 380 of file mozStorageAsyncIO.cpp.

Definition at line 378 of file mozStorageAsyncIO.cpp.

#define ASYNC_SYNC   2

Definition at line 374 of file mozStorageAsyncIO.cpp.

Definition at line 381 of file mozStorageAsyncIO.cpp.

Definition at line 375 of file mozStorageAsyncIO.cpp.

Definition at line 373 of file mozStorageAsyncIO.cpp.

This code is partially based on test_async.c distributed with sqlite.

Error handling:

When there is a write error, we don't know what to do, since the program has continued and has assumed that the operation has succeeded. Instead, we set the flag AsyncWriteError. When set, every file operation will fail, so the errors will get passed to the program. Since the error occured asynchronously, there is no easy way to figure out what was happening, undo it, and recover.

COMMENTS FROM SQLITE DEMO FILE

This file contains an example implementation of an asynchronous IO backend for SQLite.

WHAT IS ASYNCHRONOUS I/O?

With asynchronous I/O, write requests are handled by a separate thread running in the background. This means that the thread that initiates a database write does not have to wait for (sometimes slow) disk I/O to occur. The write seems to happen very quickly, though in reality it is happening at its usual slow pace in the background.

Asynchronous I/O appears to give better responsiveness, but at a price. You lose the Durable property. With the default I/O backend of SQLite, once a write completes, you know that the information you wrote is safely on disk. With the asynchronous I/O, this is no the case. If your program crashes or if you take a power lose after the database write but before the asynchronous write thread has completed, then the database change might never make it to disk and the next user of the database might not see your change.

You lose Durability with asynchronous I/O, but you still retain the other parts of ACID: Atomic, Consistent, and Isolated. Many appliations get along fine without the Durablity.

HOW IT WORKS

Asynchronous I/O works by overloading the OS-layer disk I/O routines with modified versions that store the data to be written in queue of pending write operations. Look at the asyncEnable() subroutine to see how overloading works. Six os-layer routines are overloaded:

sqlite3OsOpenReadWrite;
sqlite3OsOpenReadOnly;
sqlite3OsOpenExclusive;
sqlite3OsDelete;
sqlite3OsFileExists;
sqlite3OsSyncDirectory;

The original implementations of these routines are saved and are used by the writer thread to do the real I/O. The substitute implementations typically put the I/O operation on a queue to be handled later by the writer thread, though read operations must be handled right away, obviously.

Asynchronous I/O is disabled by setting the os-layer interface routines back to their original values.

LIMITATIONS

This demonstration code is deliberately kept simple in order to keep the main ideas clear and easy to understand. Real applications that want to do asynchronous I/O might want to add additional capabilities. For example, in this demonstration if writes are happening at a steady stream that exceeds the I/O capability of the background writer thread, the queue of pending write operations will grow without bound until we run out of memory. Users of this technique may want to keep track of the quantity of pending writes and stop accepting new write requests when the buffer gets to be too big.

THREAD SAFETY NOTES

Basic rules:

* Both read and write access to the global write-op queue must be 
  protected by the async.queueMutex.

* The file handles from the underlying system are assumed not to 
  be thread safe.

* See the last two paragraphs under "The Writer Thread" for
  an assumption to do with file-handle synchronization by the Os.

File system operations (invoked by SQLite thread):

xOpenXXX (three versions)
xDelete
xFileExists
xSyncDirectory

File handle operations (invoked by SQLite thread):

    asyncWrite, asyncClose, asyncTruncate, asyncSync,
    asyncSetFullSync, asyncOpenDirectory.

The operations above add an entry to the global write-op list. They
prepare the entry, acquire the async.queueMutex momentarily while
list pointers are  manipulated to insert the new entry, then release
the mutex and signal the writer thread to wake up in case it happens
to be asleep.

    asyncRead, asyncFileSize.

Read operations. Both of these read from both the underlying file
first then adjust their result based on pending writes in the
write-op queue.   So async.queueMutex is held for the duration
of these operations to prevent other threads from changing the
queue in mid operation.

    asyncLock, asyncUnlock, asyncLockState, asyncCheckReservedLock

These locking primitives become no-ops. Files are always opened for
exclusive access when using this IO backend.

    asyncFileHandle.

The sqlite3OsFileHandle() function is currently only used when
debugging the pager module. Unless sqlite3OsClose() is called on the
file (shouldn't be possible for other reasons), the underlying
implementations are safe to call without grabbing any mutex. So we just
go ahead and call it no matter what any other threads are doing.

    asyncSeek.

Calling this method just manipulates the AsyncFile.iOffset variable.
Since this variable is never accessed by writer thread, this
function does not require the mutex.  Actual calls to OsSeek() take
place just before OsWrite() or OsRead(), which are always protected by
the mutex.

The writer thread:

The async.writerMutex is used to make sure only there is only
a single writer thread running at a time.

Inside the writer thread is a loop that works like this:

    WHILE (write-op list is not empty)
        Do IO operation at head of write-op list
        Remove entry from head of write-op list
    END WHILE

The async.queueMutex is always held during the <write-op list is
not empty> test, and when the entry is removed from the head
of the write-op list. Sometimes it is held for the interim
period (while the IO is performed), and sometimes it is
relinquished. It is relinquished if (a) the IO op is an
ASYNC_CLOSE or (b) when the file handle was opened, two of
the underlying systems handles were opened on the same
file-system entry.

If condition (b) above is true, then one file-handle
(AsyncFile.pBaseRead) is used exclusively by sqlite threads to read the
file, the other (AsyncFile.pBaseWrite) by sqlite3_async_flush()
threads to perform write() operations. This means that read
operations are not blocked by asynchronous writes (although
asynchronous writes may still be blocked by reads).

This assumes that the OS keeps two handles open on the same file
properly in sync. That is, any read operation that starts after a
write operation on the same file system entry has completed returns
data consistent with the write. We also assume that if one thread
reads a file while another is writing it all bytes other than the
ones actually being written contain valid data.

If the above assumptions are not true, set the preprocessor symbol
SQLITE_ASYNC_TWO_FILEHANDLES to 0.

VERSIONING

This file was originally based on version 1.5 of test_async.c in sqlite,
see http://www.sqlite.org/cvstrac/rlog?f=sqlite/src/test_async.c
Versions 1.6 and 1.7 were based on errors found and fixed here.

We've incorportated the patches for versions 1.12, 1.13, 1.15, 1.17
(which backs out some of the changes in 1.13)

FIXME: It would be nice to have "fake" in-process file locking as in
versions 1.11.

Our error handling is a little more coarse than the ones implemented
in versions 1.8 and 1.14. Those ones count the number of files and
reset the error value when all files are closed. This allows one to
potentially recover by closing all the connections and reopening them.
We don't handle this from the calling code, so there's no point in
implementing that here. Instead we just fail all operations on error.

Definition at line 256 of file mozStorageAsyncIO.cpp.


Function Documentation

void AppendAsyncMessage ( AsyncMessage aMessage) [static]

Definition at line 750 of file mozStorageAsyncIO.cpp.

{
  // We must hold the queue mutex in order to modify the queue pointers
  PR_Lock(AsyncQueueLock);

  // Add the record to the end of the write-op queue
  NS_ASSERTION(! aMessage->mNext, "New messages should not have next pointers");
  if (AsyncQueueLast) {
    NS_ASSERTION(AsyncQueueFirst, "If we have a last item, we need to have a first one");
    AsyncQueueLast->mNext = aMessage;
  } else {
    AsyncQueueFirst = aMessage;
  }
  AsyncQueueLast = aMessage;

  // The writer thread might have been idle because there was nothing on the
  // write-op queue for it to do. So wake it up.
  if (AsyncWriteThreadInstance) {
    PR_NotifyCondVar(AsyncQueueCondition);
    PR_Unlock(AsyncQueueLock);
  } else {
    // single threaded mode: call the writer to process this message
    NS_ASSERTION(AsyncWriterHaltWhenIdle, "In single-threaded mode, the writer thread should always halt when idle");
    PR_Unlock(AsyncQueueLock);
    ProcessAsyncMessages();
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

int AppendNewAsyncMessage ( AsyncOsFile aFile,
PRUint32  aOp,
sqlite_int64  aOffset,
PRInt32  aDataSize,
const char *  aData 
) [static]

Definition at line 788 of file mozStorageAsyncIO.cpp.

{
  // allocate one buffer, we will put the buffer immediately after our struct
  AsyncMessage* p = NS_STATIC_CAST(AsyncMessage*,
      nsMemory::Alloc(sizeof(AsyncMessage) + (aData ? aDataSize : 0)));
  if (! p)
    return SQLITE_NOMEM;

  p->mOp = aOp;
  p->mOffset = aOffset;
  p->mBytes = aDataSize;
  p->mFile = aFile;
  p->mNext = nsnull;
  if (aData) {
    // this gets the address of the data immediately following our structure
    p->mBuf = (char*)&p[1];
    memcpy(p->mBuf, aData, aDataSize);
  } else {
    p->mBuf = nsnull;
  }
  AppendAsyncMessage(p);
  return SQLITE_OK;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int AsyncBarrier ( PRLock aLock,
PRCondVar aCondVar 
) [static]

Definition at line 1347 of file mozStorageAsyncIO.cpp.

{
  AsyncMessageBarrierData bd;

  bd.mLock = aLock;
  bd.mCondVar = aCondVar;

  return AppendNewAsyncMessage(nsnull, ASYNC_BARRIER, 0,
                               sizeof(AsyncMessageBarrierData), (const char*) &bd);
}

Here is the call graph for this function:

Here is the caller graph for this function:

int AsyncCheckReservedLock ( OsFile aFile) [static]

Definition at line 1318 of file mozStorageAsyncIO.cpp.

{
  if (AsyncWriteError != SQLITE_OK)
    return AsyncWriteError;
  return SQLITE_OK;
}

Here is the caller graph for this function:

int AsyncClose ( OsFile **  aFile) [static]

Definition at line 973 of file mozStorageAsyncIO.cpp.

{
  if (AsyncWriteError != SQLITE_OK)
    return AsyncWriteError;
  AsyncOsFile* asyncfile = NS_STATIC_CAST(AsyncOsFile*, *aFile);
  if (! asyncfile->mOpen) {
    NS_NOTREACHED("Attempting to write to a file with a close pending!");
    return SQLITE_INTERNAL;
  }
  asyncfile->mOpen = PR_FALSE;
  return AppendNewAsyncMessage(asyncfile, ASYNC_CLOSE, 0, 0, 0);
}

Here is the call graph for this function:

Here is the caller graph for this function:

int AsyncDelete ( const char *  aName) [static]

Definition at line 909 of file mozStorageAsyncIO.cpp.

Here is the call graph for this function:

Here is the caller graph for this function:

int AsyncFileExists ( const char *  aName) [static]

Definition at line 944 of file mozStorageAsyncIO.cpp.

{
  if (AsyncWriteError != SQLITE_OK)
    return AsyncWriteError;
  nsAutoLock lock(AsyncQueueLock);

  // See if the real file system contains the specified file.
  int ret = sqliteOrigFileExists(aName);

  for (AsyncMessage* p = AsyncQueueFirst; p != nsnull; p = p->mNext) {
    if (p->mOp == ASYNC_DELETE && 0 == strcmp(p->mBuf, aName)) {
      ret = 0;
    } else if (p->mOp == ASYNC_OPENEXCLUSIVE && 0 == strcmp(p->mBuf, aName)) {
      ret = 1;
    }
  }
  return ret;
}

Here is the caller graph for this function:

int AsyncFileHandle ( OsFile aFile) [static]

Definition at line 1274 of file mozStorageAsyncIO.cpp.

{
  if (AsyncWriteError != SQLITE_OK)
    return AsyncWriteError;
  NS_NOTREACHED("Don't call FileHandle in async mode");
  return SQLITE_OK;

  // If you actually wanted the file handle you would do this:
  //AsyncOsFile* asyncfile = NS_STATIC_CAST(AsyncOsFile*, aFile);
  //return sqlite3OsFileHandle(asyncfile->mBaseRead);
}

Here is the caller graph for this function:

int AsyncFileSize ( OsFile aFile,
sqlite_int64 aSize 
) [static]

Definition at line 1223 of file mozStorageAsyncIO.cpp.

{
  nsAutoLock lock(AsyncQueueLock);

  if (AsyncWriteError != SQLITE_OK)
    return AsyncWriteError;

  AsyncOsFile* asyncfile = NS_STATIC_CAST(AsyncOsFile*, aFile);
  if (! asyncfile->mOpen) {
    NS_NOTREACHED("Attempting to write to a file with a close pending!");
    return SQLITE_INTERNAL;
  }
  int rc = SQLITE_OK;
  sqlite_int64 size = 0;

  // Read the filesystem size from the base file. If pBaseRead is NULL, this
  // means the file hasn't been opened yet. In this case all relevant data must
  // be in the write-op queue anyway, so we can omit reading from the
  // file-system.
  OsFile* pBase = asyncfile->mBaseRead;
  if (pBase) {
    NS_ASSERTION(sqliteOrigFileSize, "Original file size pointer uninitialized!");
    rc = sqliteOrigFileSize(pBase, &size);
  }

  if (rc == SQLITE_OK) {
    for (AsyncMessage* p = AsyncQueueFirst; p != nsnull; p = p->mNext) {
      if (p->mFile == asyncfile) {
        switch (p->mOp) {
          case ASYNC_WRITE:
            size = PR_MAX(p->mOffset + p->mBytes, size);
            break;
          case ASYNC_TRUNCATE:
            size = PR_MIN(size, p->mOffset);
            break;
        }
      }
    }
    *aSize = size;
  }
  return rc;
}

Here is the caller graph for this function:

int AsyncLock ( OsFile aFile,
int  aLockType 
) [static]

Definition at line 1293 of file mozStorageAsyncIO.cpp.

{
  if (AsyncWriteError != SQLITE_OK)
    return AsyncWriteError;
  return SQLITE_OK;
}

Here is the caller graph for this function:

int AsyncLockState ( OsFile aFile) [static]

Definition at line 1332 of file mozStorageAsyncIO.cpp.

{
  if (AsyncWriteError != SQLITE_OK)
    return AsyncWriteError;
  NS_NOTREACHED("Don't call LockState in async mode");
  return SQLITE_OK;
}

Here is the caller graph for this function:

int AsyncOpenDirectory ( OsFile aFile,
const char *  aName 
) [static]

Definition at line 1037 of file mozStorageAsyncIO.cpp.

{
  if (AsyncWriteError != SQLITE_OK)
    return AsyncWriteError;
  AsyncOsFile* asyncfile = NS_STATIC_CAST(AsyncOsFile*, aFile);
  if (! asyncfile->mOpen) {
    NS_NOTREACHED("Attempting to write to a file with a close pending!");
    return SQLITE_INTERNAL;
  }
  return AppendNewAsyncMessage(asyncfile, ASYNC_OPENDIRECTORY, 0,
                          strlen(aName) + 1, aName);
}

Here is the call graph for this function:

Here is the caller graph for this function:

int AsyncOpenExclusive ( const char *  aName,
OsFile **  aFile,
int  aDelFlag 
) [static]

Definition at line 837 of file mozStorageAsyncIO.cpp.

{
  if (AsyncWriteError != SQLITE_OK)
    return AsyncWriteError;
  // Create a new async file with no base reader that is not writable. Nothing
  // will be able to be done with this until the message is processed.
  AsyncOsFile* osfile;
  int rc = AsyncOpenFile(aName, &osfile, nsnull, PR_FALSE);
  if (rc != SQLITE_OK)
    return rc;

  rc = AppendNewAsyncMessage(osfile, ASYNC_OPENEXCLUSIVE, aDelFlag,
                             PL_strlen(aName) + 1, aName);
  if (rc != SQLITE_OK) {
    nsMemory::Free(osfile);
    osfile = nsnull;
  }
  *aFile = osfile;
  return rc;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int AsyncOpenFile ( const char *  aName,
AsyncOsFile **  aFile,
OsFile aBaseRead,
PRBool  aOpenForWriting 
) [static]

Definition at line 673 of file mozStorageAsyncIO.cpp.

{
  int rc;
  OsFile *baseWrite = nsnull;

  if (! sqliteOrigClose) {
    sqliteOrigClose = aBaseRead->pMethod->xClose;
    sqliteOrigRead = aBaseRead->pMethod->xRead;
    sqliteOrigWrite = aBaseRead->pMethod->xWrite;
    sqliteOrigFileSize = aBaseRead->pMethod->xFileSize;
    sqliteOrigSeek = aBaseRead->pMethod->xSeek;
    sqliteOrigSync = aBaseRead->pMethod->xSync;
    sqliteOrigTruncate = aBaseRead->pMethod->xTruncate;
    sqliteOrigOpenDirectory = aBaseRead->pMethod->xOpenDirectory;
    sqliteOrigSetFullSync = aBaseRead->pMethod->xSetFullSync;
  }

  static IoMethod iomethod = {
    AsyncClose,
    AsyncOpenDirectory,
    AsyncRead,
    AsyncWrite,
    AsyncSeek,
    AsyncTruncate,
    AsyncSync,
    AsyncSetFullSync,
    AsyncFileHandle,
    AsyncFileSize,
    AsyncLock,
    AsyncUnlock,
    AsyncLockState,
    AsyncCheckReservedLock
  };

  if (aOpenForWriting && SQLITE_ASYNC_TWO_FILEHANDLES) {
    int dummy;
    rc = sqliteOrigOpenReadWrite(aName, &baseWrite, &dummy);
    if (rc != SQLITE_OK)
      goto error_out;
  }

  *aFile = NS_STATIC_CAST(AsyncOsFile*, nsMemory::Alloc(sizeof(AsyncOsFile)));
  if (! *aFile) {
    rc = SQLITE_NOMEM;
    goto error_out;
  }
  memset(*aFile, 0, sizeof(AsyncOsFile));

  (*aFile)->mFilename = new nsCString(aName);
  (*aFile)->pMethod = &iomethod;
  (*aFile)->mOpen = PR_TRUE;
  (*aFile)->mBaseRead = aBaseRead;
  (*aFile)->mBaseWrite = baseWrite;

  return SQLITE_OK;

error_out:
  NS_ASSERTION(!*aFile, "File not cleared on error");
  sqliteOrigClose(&aBaseRead);
  sqliteOrigClose(&baseWrite);
  return rc;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int AsyncOpenReadOnly ( const char *  aName,
OsFile **  aFile 
) [static]

Definition at line 863 of file mozStorageAsyncIO.cpp.

{
  if (AsyncWriteError != SQLITE_OK)
    return AsyncWriteError;
  OsFile* base = nsnull;
  int rc = sqliteOrigOpenReadOnly(aName, &base);
  if (rc == SQLITE_OK) {
    AsyncOsFile* asyncfile;
    rc = AsyncOpenFile(aName, &asyncfile, base, PR_FALSE);
    if (rc == SQLITE_OK)
      *aFile = asyncfile;
    else
      *aFile = nsnull;
  }
  return rc;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int AsyncOpenReadWrite ( const char *  aName,
OsFile **  aFile,
int aReadOnly 
) [static]

Definition at line 884 of file mozStorageAsyncIO.cpp.

{
  if (AsyncWriteError != SQLITE_OK)
    return AsyncWriteError;
  OsFile* base = nsnull;
  int rc = sqliteOrigOpenReadWrite(aName, &base, aReadOnly);
  if (rc == SQLITE_OK) {
    AsyncOsFile* asyncfile;
    rc = AsyncOpenFile(aName, &asyncfile, base, !(*aReadOnly));
    if (rc == SQLITE_OK)
      *aFile = asyncfile;
    else
      *aFile = nsnull;
  }
  return rc;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int AsyncRead ( OsFile aFile,
void aBuffer,
int  aCount 
) [static]

Definition at line 1099 of file mozStorageAsyncIO.cpp.

{
  if (AsyncWriteError != SQLITE_OK)
    return AsyncWriteError;
  int rc = SQLITE_OK;

  // Grab the write queue mutex for the duration of the call. We don't want
  // the writer thread going and writing stuff to the file or processing
  // any messages while we do this. Open exclusive may also change mBaseRead.
  nsAutoLock lock(AsyncQueueLock);

  AsyncOsFile* asyncfile = NS_STATIC_CAST(AsyncOsFile*, aFile);
  if (! asyncfile->mOpen) {
    NS_NOTREACHED("Attempting to write to a file with a close pending!");
    return SQLITE_INTERNAL;
  }

  OsFile* pBase = asyncfile->mBaseRead;
  if (pBase) {
    // Only do any actual file reading if there is a reader structure. For
    // pending OpenExclusives, there will not be any so we don't want to do
    // file reading. Reading while an OpenExclusive is pending will read
    // entirely from the write queue. Since OpenExclusive can not work for
    // prevously existing files, we know anything in the file is in our write
    // queue.
    sqlite_int64 filesize;
    NS_ASSERTION(sqliteOrigFileSize, "Original file size pointer uninitialized!");
    rc = sqliteOrigFileSize(pBase, &filesize);
    if (rc != SQLITE_OK)
      goto asyncread_out;

    // This may seek beyond EOF if there is appended data waiting in the write
    // buffer. The OS should be OK with this. We will only try reading if there
    // is stuff for us to read there.
    NS_ASSERTION(sqliteOrigSeek, "Original seek pointer uninitialized!");
    rc = sqliteOrigSeek(pBase, asyncfile->mOffset);
    if (rc != SQLITE_OK)
      goto asyncread_out;

    // Here, we try to read as much data as we want up to EOF.
    int numread = PR_MIN(filesize - asyncfile->mOffset, aCount);
    if (numread > 0) {
      NS_ASSERTION(pBase, "Original read pointer uninitialized!");
      rc = sqliteOrigRead(pBase, aBuffer, numread);
    }
  }

  if (rc == SQLITE_OK) {
    sqlite_int64 blockOffset = asyncfile->mOffset; // Current seek offset

    // Now we need to bring our data up-do-date with any pending writes.
    for (AsyncMessage* p = AsyncQueueFirst; p != nsnull; p = p->mNext) {
      if (p->mFile == asyncfile && p->mOp == ASYNC_WRITE) {

        // What we're reading:
        //
        //      [==================================================]
        //      ^- aFile.mOffset = blockOffset
        //      <--------------------aCount------------------------>
        //
        // Possibly pending writes:
        //
        // [==============]
        // ^- p.mOffset
        //      <---------> copycount
        //
        //               [================]
        //               ^- p.mOffset
        //               <----------------> copycount
        //
        //                                              [=============]
        //                                              ^- p.mOffset
        //                                              <----------> copycount
        PRInt32 beginIn = PR_MAX(0, blockOffset - p->mOffset);
        PRInt32 beginOut = PR_MAX(0, p->mOffset - blockOffset);
        PRInt32 copycount = PR_MIN(p->mBytes - beginIn, aCount - beginOut);

        if (copycount > 0) {
          memcpy(&NS_STATIC_CAST(char*, aBuffer)[beginOut],
                 &p->mBuf[beginIn], copycount);
        }
      }
    }

    // successful read, update virtual current seek offset
    asyncfile->mOffset += aCount;
  }

asyncread_out:
  return rc;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int AsyncSeek ( OsFile aFile,
sqlite_int64  aOffset 
) [static]

Definition at line 1199 of file mozStorageAsyncIO.cpp.

{
  if (AsyncWriteError != SQLITE_OK)
    return AsyncWriteError;
  AsyncOsFile* asyncfile = NS_STATIC_CAST(AsyncOsFile*, aFile);
  if (! asyncfile->mOpen) {
    NS_NOTREACHED("Attempting to write to a file with a close pending!");
    return SQLITE_INTERNAL;
  }
  asyncfile->mOffset = aOffset;
  return SQLITE_OK;
}

Here is the caller graph for this function:

void AsyncSetFullSync ( OsFile aFile,
int  aValue 
) [static]

Definition at line 1076 of file mozStorageAsyncIO.cpp.

{
  if (AsyncWriteError != SQLITE_OK)
    return;
  AsyncOsFile* asyncfile = NS_STATIC_CAST(AsyncOsFile*, aFile);
  if (! asyncfile->mOpen) {
    NS_NOTREACHED("Attempting to write to a file with a close pending!");
    return;
  }
  AppendNewAsyncMessage(asyncfile, ASYNC_SETFULLSYNC, 0, aValue, 0);
}

Here is the call graph for this function:

Here is the caller graph for this function:

int AsyncSync ( OsFile aFile,
int  aFullsync 
) [static]

Definition at line 1057 of file mozStorageAsyncIO.cpp.

{
  if (AsyncWriteError != SQLITE_OK)
    return AsyncWriteError;
  AsyncOsFile* asyncfile = NS_STATIC_CAST(AsyncOsFile*, aFile);
  if (! asyncfile->mOpen) {
    NS_NOTREACHED("Attempting to write to a file with a close pending!");
    return SQLITE_INTERNAL;
  }
  return AppendNewAsyncMessage(asyncfile, ASYNC_SYNC, 0, aFullsync, 0);
}

Here is the call graph for this function:

Here is the caller graph for this function:

int AsyncSyncDirectory ( const char *  aName) [static]

Definition at line 923 of file mozStorageAsyncIO.cpp.

Here is the call graph for this function:

Here is the caller graph for this function:

int AsyncTruncate ( OsFile aFile,
sqlite_int64  aNumBytes 
) [static]

Definition at line 1017 of file mozStorageAsyncIO.cpp.

{
  if (AsyncWriteError != SQLITE_OK)
    return AsyncWriteError;
  AsyncOsFile* asyncfile = NS_STATIC_CAST(AsyncOsFile*, aFile);
  if (! asyncfile->mOpen) {
    NS_NOTREACHED("Attempting to write to a file with a close pending!");
    return SQLITE_INTERNAL;
  }
  return AppendNewAsyncMessage(asyncfile, ASYNC_TRUNCATE, aNumBytes, 0, 0);
}

Here is the call graph for this function:

Here is the caller graph for this function:

int AsyncUnlock ( OsFile aFile,
int  aLockType 
) [static]

Definition at line 1304 of file mozStorageAsyncIO.cpp.

{
  if (AsyncWriteError != SQLITE_OK)
    return AsyncWriteError;
  return SQLITE_OK;
}

Here is the caller graph for this function:

int AsyncWrite ( OsFile aFile,
const void aBuf,
int  aCount 
) [static]

Definition at line 995 of file mozStorageAsyncIO.cpp.

{
  if (AsyncWriteError != SQLITE_OK)
    return AsyncWriteError;
  AsyncOsFile* asyncfile = NS_STATIC_CAST(AsyncOsFile*, aFile);
  if (! asyncfile->mOpen) {
    NS_NOTREACHED("Attempting to write to a file with a close pending!");
    return SQLITE_INTERNAL;
  }
  int rc = AppendNewAsyncMessage(asyncfile, ASYNC_WRITE, asyncfile->mOffset,
                                 aCount, NS_STATIC_CAST(const char*, aBuf));
  asyncfile->mOffset += aCount;
  return rc;
}

Here is the call graph for this function:

Here is the caller graph for this function:

Definition at line 1639 of file mozStorageAsyncIO.cpp.

{
  nsAsyncWriteErrorDisplayer* displayer = NS_REINTERPRET_CAST(
      nsAsyncWriteErrorDisplayer*, aEvent);
  delete displayer;
}

Here is the caller graph for this function:

Definition at line 1689 of file mozStorageAsyncIO.cpp.

{
  nsAsyncWriteErrorDisplayer* displayer = new nsAsyncWriteErrorDisplayer();
  if (! displayer) {
    NS_NOTREACHED("Can't create the event to display a write error");
    return;
  }
  if (NS_FAILED(displayer->Post())) {
    NS_NOTREACHED("Can't call main thread");
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

Definition at line 1601 of file mozStorageAsyncIO.cpp.

{
  nsresult rv;
  nsCOMPtr<nsIPrompt> prompt = do_CreateInstance(
      NS_DEFAULTPROMPT_CONTRACTID, &rv);
  if (NS_FAILED(rv)) {
    NS_NOTREACHED("Can't get prompt service");
    return nsnull;
  }

  nsCOMPtr<nsIStringBundleService> bundleService = do_GetService(
      "@mozilla.org/intl/stringbundle;1", &rv);
  if (NS_FAILED(rv)) {
    NS_NOTREACHED("Can't get string bundle");
    return nsnull;
  }

  nsCOMPtr<nsIStringBundle> bundle;
  rv = bundleService->CreateBundle(
      "chrome://global/locale/storage.properties", getter_AddRefs(bundle));
  if (NS_FAILED(rv)) {
    NS_NOTREACHED("Can't get storage properties");
    return nsnull;
  }

  nsXPIDLString message;
  rv = bundle->GetStringFromName(NS_LITERAL_STRING("storageWriteError").get(),
                                 getter_Copies(message));
  if (NS_FAILED(rv)) {
    NS_NOTREACHED("Can't get error string");
    return nsnull;
  }

  prompt->Alert(nsnull, message.get());
  return nsnull;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void ProcessAsyncMessages ( ) [static]

Definition at line 1507 of file mozStorageAsyncIO.cpp.

{
  AsyncMessage *message = 0;
  int rc = SQLITE_OK;

  while (PR_TRUE) {
    {
      // wait for a message to come in
      nsAutoLock lock(AsyncQueueLock);
      while ((message = AsyncQueueFirst) == 0) {
        if (AsyncWriterHaltWhenIdle) {
          // We've been asked to stop, so exit the thread
          return;
        } else {
          // This will unlock AsyncQueueLock and wait for the condition to
          // be true. This condition is set when somebody adds an item to our
          // queue.
          NS_ASSERTION(AsyncQueueLock, "We need to be in multi threaded mode if we're going to wait");
          PR_WaitCondVar(AsyncQueueCondition, PR_INTERVAL_NO_TIMEOUT);
        }
      }

      // this function may release the lock in the middle, but should always
      // put it back when it's done
      rc = ProcessOneMessage(message);

      // check for error
      if (rc != SQLITE_OK) {
        AsyncWriteError = rc;
        NS_NOTREACHED("FILE ERROR");

        // log error to console
        nsresult rv;
        nsCOMPtr<nsIConsoleService> consoleSvc =
            do_GetService("@mozilla.org/consoleservice;1", &rv);
        if (NS_FAILED(rv)) {
          NS_WARNING("Couldn't get the console service for logging file error");
        } else {
          nsAutoString logMessage;
          logMessage.AssignLiteral("mozStorage: error code ");
          logMessage.AppendInt(rc);
          logMessage.AppendLiteral(" for database ");
          if (message->mFile && message->mFile->mFilename)
            logMessage.Append(NS_ConvertUTF8toUTF16(*message->mFile->mFilename));
          rv = consoleSvc->LogStringMessage(logMessage.get());
          NS_WARN_IF_FALSE(NS_SUCCEEDED(rv), "Couldn't log message on async error");
        }

        // tell user to restart
        DisplayAsyncWriteError();
        return;
      }

      // remove the message from the end of the message queue and release it
      if (message == AsyncQueueLast)
        AsyncQueueLast = nsnull;
      AsyncQueueFirst = message->mNext;
      nsMemory::Free(message);

      // free any out-of-memory flags in the library
      sqlite3ApiExit(nsnull, 0);
    }
    // Drop the queue mutex before continuing to the next write operation
    // in order to give other threads a chance to work with the write queue
    // (that should have been done by the autolock in exiting the scope that
    // just closed). We want writers to the queue to generally have priority.
    #ifdef IO_DELAY_INTERVAL_MS
      // this simulates slow disk
      PR_Sleep(PR_MillisecondsToInterval(IO_DELAY_INTERVAL_MS));
    #else
      // yield so the UI thread is more responsive
      PR_Sleep(PR_INTERVAL_NO_WAIT);
    #endif
  }
}

Here is the call graph for this function:

Here is the caller graph for this function:

int ProcessOneMessage ( AsyncMessage aMessage) [static]

Definition at line 1378 of file mozStorageAsyncIO.cpp.

{
  PRBool regainMutex = PR_FALSE;
  OsFile* pBase = nsnull;

  if (aMessage->mFile) {
    pBase = aMessage->mFile->mBaseWrite;
    if (aMessage->mOp == ASYNC_CLOSE || 
        aMessage->mOp == ASYNC_OPENEXCLUSIVE ||
        (pBase && (aMessage->mOp == ASYNC_SYNC ||
                   aMessage->mOp == ASYNC_WRITE))) {
      regainMutex = PR_TRUE;
      PR_Unlock(AsyncQueueLock);
    }
    if (! pBase)
      pBase = aMessage->mFile->mBaseRead;
  }

  int rc = SQLITE_OK;
  switch (aMessage->mOp) {
    case ASYNC_WRITE:
      NS_ASSERTION(pBase, "Must have base writer for writing");
      rc = sqliteOrigSeek(pBase, aMessage->mOffset);
      if (rc == SQLITE_OK)
        rc = sqliteOrigWrite(pBase, (const void *)(aMessage->mBuf), aMessage->mBytes);
      break;

    case ASYNC_SYNC:
      NS_ASSERTION(pBase, "Must have base writer for writing");
      rc = sqliteOrigSync(pBase, aMessage->mBytes);
      break;

    case ASYNC_TRUNCATE:
      NS_ASSERTION(pBase, "Must have base writer for writing");
      NS_ASSERTION(sqliteOrigTruncate, "No truncate pointer");
      rc = sqliteOrigTruncate(pBase, aMessage->mOffset);
      break;

    case ASYNC_CLOSE:
      // note that the sqlite close function accepts NULL pointers here and
      // will return success if given one (I think the order we close these
      // two handles matters here)
      sqliteOrigClose(&aMessage->mFile->mBaseWrite);
      sqliteOrigClose(&aMessage->mFile->mBaseRead);
      if (aMessage->mFile->mFilename)
        delete aMessage->mFile->mFilename;
      nsMemory::Free(aMessage->mFile);
      aMessage->mFile = nsnull;
      break;

    case ASYNC_OPENDIRECTORY:
      NS_ASSERTION(pBase, "Must have base writer for writing");
      NS_ASSERTION(sqliteOrigOpenDirectory, "No open directory pointer");
      sqliteOrigOpenDirectory(pBase, aMessage->mBuf);
      break;

    case ASYNC_SETFULLSYNC:
      NS_ASSERTION(pBase, "Must have base writer for writing");
      sqliteOrigSetFullSync(pBase, aMessage->mBytes);
      break;

    case ASYNC_DELETE:
      NS_ASSERTION(sqliteOrigDelete, "No delete pointer");
      rc = sqliteOrigDelete(aMessage->mBuf);
      break;

    case ASYNC_SYNCDIRECTORY:
      NS_ASSERTION(sqliteOrigSyncDirectory, "No sync directory pointer");
      rc = sqliteOrigSyncDirectory(aMessage->mBuf);
      break;

    case ASYNC_OPENEXCLUSIVE: {
      AsyncOsFile *pFile = aMessage->mFile;
      int delFlag = ((aMessage->mOffset) ? 1 : 0);
      OsFile* pBase = nsnull;
      NS_ASSERTION(! pFile->mBaseRead && ! pFile->mBaseWrite,
                   "OpenExclusive expects no file pointers");
      rc = sqliteOrigOpenExclusive(aMessage->mBuf, &pBase, delFlag);

      // exclusive opens actually go and write to the OsFile structure to set
      // the file object. We therefore need to be locked so the main thread
      // doesn't try to use it to do synchronous reading.
      PR_Lock(AsyncQueueLock);
      regainMutex = PR_FALSE;
      if (rc == SQLITE_OK)
        pFile->mBaseRead = pBase;
      break;
    }

    case ASYNC_BARRIER: {
      AsyncMessageBarrierData *bd = (AsyncMessageBarrierData*) aMessage->mBuf;
      PR_Lock(bd->mLock);
      PR_NotifyCondVar(bd->mCondVar);
      PR_Unlock(bd->mLock);
      break;
    }

    default:
      NS_NOTREACHED("Illegal value for AsyncMessage.mOp");
  }

  if (regainMutex) {
    PR_Lock(AsyncQueueLock);
  }
  return rc;
}

Here is the call graph for this function:

Here is the caller graph for this function:


Variable Documentation

Definition at line 436 of file mozStorageAsyncIO.cpp.

Definition at line 413 of file mozStorageAsyncIO.cpp.

Definition at line 414 of file mozStorageAsyncIO.cpp.

Definition at line 435 of file mozStorageAsyncIO.cpp.

Definition at line 428 of file mozStorageAsyncIO.cpp.

Definition at line 420 of file mozStorageAsyncIO.cpp.

Definition at line 434 of file mozStorageAsyncIO.cpp.

int(* sqliteOrigClose)(OsFile **) = nsnull [static]

Definition at line 448 of file mozStorageAsyncIO.cpp.

int(* sqliteOrigDelete)(const char *) = nsnull [static]

Definition at line 442 of file mozStorageAsyncIO.cpp.

int(* sqliteOrigFileExists)(const char *) = nsnull [static]

Definition at line 443 of file mozStorageAsyncIO.cpp.

int(* sqliteOrigFileSize)(OsFile *, sqlite_int64 *pSize) = nsnull [static]

Definition at line 451 of file mozStorageAsyncIO.cpp.

int(* sqliteOrigOpenDirectory)(OsFile *, const char *) [static]

Definition at line 455 of file mozStorageAsyncIO.cpp.

int(* sqliteOrigOpenExclusive)(const char *, OsFile **, int) = nsnull [static]

Definition at line 440 of file mozStorageAsyncIO.cpp.

int(* sqliteOrigOpenReadOnly)(const char *, OsFile **) = nsnull [static]

Definition at line 441 of file mozStorageAsyncIO.cpp.

int(* sqliteOrigOpenReadWrite)(const char *, OsFile **, int *) = nsnull [static]

Definition at line 439 of file mozStorageAsyncIO.cpp.

int(* sqliteOrigRead)(OsFile *, void *, int amt) = nsnull [static]

Definition at line 449 of file mozStorageAsyncIO.cpp.

Definition at line 452 of file mozStorageAsyncIO.cpp.

void(* sqliteOrigSetFullSync)(OsFile *, int setting) [static]

Definition at line 456 of file mozStorageAsyncIO.cpp.

int(* sqliteOrigSync)(OsFile *, int) = nsnull [static]

Definition at line 453 of file mozStorageAsyncIO.cpp.

int(* sqliteOrigSyncDirectory)(const char *) = nsnull [static]

Definition at line 444 of file mozStorageAsyncIO.cpp.

Definition at line 454 of file mozStorageAsyncIO.cpp.

int(* sqliteOrigWrite)(OsFile *, const void *, int amt) = nsnull [static]

Definition at line 450 of file mozStorageAsyncIO.cpp.