From 7c24610e640edd2d547c11fa87a185ffeef7e94f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 12 Apr 2010 19:00:29 +0000 Subject: [PATCH 01/89] Import experimental write-ahead-logging code. FossilOrigin-Name: 409d61baeb0a19d1700c973f16c8acef7b8506cd --- main.mk | 8 +- manifest | 33 +- manifest.uuid | 2 +- src/log.c | 1243 +++++++++++++++++++++++++++++++++++++++++++ src/os_unix.c | 6 + src/pager.c | 585 ++++++++++++-------- src/pager.h | 1 + src/pragma.c | 48 +- src/vdbe.c | 15 + test/wal.test | 551 +++++++++++++++++++ test/walcrash.test | 248 +++++++++ test/walslow.test | 69 +++ test/walthread.test | 192 +++++++ tool/mksqlite3c.tcl | 2 + 14 files changed, 2772 insertions(+), 231 deletions(-) create mode 100644 src/log.c create mode 100644 test/wal.test create mode 100644 test/walcrash.test create mode 100644 test/walslow.test create mode 100644 test/walthread.test diff --git a/main.mk b/main.mk index f0b7636e54..89f1c6ada9 100644 --- a/main.mk +++ b/main.mk @@ -56,7 +56,7 @@ LIBOBJ+= alter.o analyze.o attach.o auth.o \ fts3.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o fts3_write.o \ func.o global.o hash.o \ - icu.o insert.o journal.o legacy.o loadext.o \ + icu.o insert.o journal.o legacy.o loadext.o log.o \ main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \ memjournal.o \ mutex.o mutex_noop.o mutex_os2.o mutex_unix.o mutex_w32.o \ @@ -101,6 +101,8 @@ SRC = \ $(TOP)/src/journal.c \ $(TOP)/src/legacy.c \ $(TOP)/src/loadext.c \ + $(TOP)/src/log.c \ + $(TOP)/src/log.h \ $(TOP)/src/main.c \ $(TOP)/src/malloc.c \ $(TOP)/src/mem0.c \ @@ -255,8 +257,8 @@ TESTSRC = \ TESTSRC2 = \ $(TOP)/src/attach.c $(TOP)/src/backup.c $(TOP)/src/btree.c \ $(TOP)/src/build.c $(TOP)/src/date.c \ - $(TOP)/src/expr.c $(TOP)/src/func.c $(TOP)/src/insert.c $(TOP)/src/mem5.c \ - $(TOP)/src/os.c \ + $(TOP)/src/expr.c $(TOP)/src/func.c $(TOP)/src/insert.c $(TOP)/src/log.c \ + $(TOP)/src/mem5.c $(TOP)/src/os.c \ $(TOP)/src/os_os2.c $(TOP)/src/os_unix.c $(TOP)/src/os_win.c \ $(TOP)/src/pager.c $(TOP)/src/pragma.c $(TOP)/src/prepare.c \ $(TOP)/src/printf.c $(TOP)/src/random.c $(TOP)/src/pcache.c \ diff --git a/manifest b/manifest index 55eab897f8..4983ed4acd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swhere\sa\sprocess\sin\sexclusive\smode\scould\sdelete\sa\shot-journal\sfile\swithout\srolling\sit\sback\sfrom\swithin\ssqlite3_close()\sor\sDETACH.\sThis\sproblem\swas\sintroduced\sby\sthe\sprevious\scommit,\sit\sis\snot\spresent\sin\sany\sreleases. -D 2010-04-12T17:08:45 +C Import\sexperimental\swrite-ahead-logging\scode. +D 2010-04-12T19:00:30 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -89,7 +89,7 @@ F ext/rtree/tkt3363.test 2bf324f7908084a5f463de3109db9c6e607feb1b F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk d286b99eb87db41cfc5394e346604ef49509867d +F main.mk f12991ace528dd01d018420988ff053350ae81f8 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -131,6 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 +F src/log.c 6e8f296f6c566a297cd074c4165f1695fd1df5b7 F src/main.c c0e7192bad5b90544508b241eb2487ac661de890 F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -150,15 +151,15 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f -F src/os_unix.c 148d2f625db3727250c0b880481ae7630b6d0eb0 +F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053 -F src/pager.c da5ed17bb729c27a16c45fe38e9531c240a1c6a4 -F src/pager.h ef8a2cf10084f60ab45ee2dfded8bf8b0c655ddf +F src/pager.c d61318cd04a42076c6524cb22c14fe036adbddcb +F src/pager.h 80c57ba672fcd24215a68abf1c98d474a5080b9b F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 F src/pcache1.c 6dc1871ce8ead9187161c370a58cd06c84221f76 -F src/pragma.c e166ea41544f8e57a08db86dbe87212b7d378fe8 +F src/pragma.c 55f34050510940e7b60fa2b01141c16f87b649c5 F src/prepare.c fd1398cb1da54385ba5bd68d93928f10d10a1d9c F src/printf.c 5f5b65a83e63f2096a541a340722a509fa0240a7 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -212,7 +213,7 @@ F src/update.c c0dc6b75ad28b76b619042d934f337b02acee208 F src/utf.c 1baeeac91707a4df97ccc6141ec0f808278af685 F src/util.c 32aebf04c10e51ad3977a928b7416bed671b620b F src/vacuum.c b1d542c8919d4d11119f78069e1906a1ad07e0ee -F src/vdbe.c 2abd931ea2aec3eacc6426677f40cc5a1071d34e +F src/vdbe.c 23e5462d1cd0a6d525948cfec3382890abc31d3f F src/vdbe.h 471f6a3dcec4817ca33596fe7f6654d56c0e75f3 F src/vdbeInt.h 19ebc8c2a2e938340051ee65af3f377fb99102d1 F src/vdbeapi.c 74c25680046a116b24b95393914d3669c23305dc @@ -754,6 +755,10 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d +F test/wal.test 653a870ac4b312d421e006ec45a293ecf2135b49 +F test/walcrash.test 76a6848c5088596cf37f4042e7e4c108a8c323d4 +F test/walslow.test 17bfe8052a5d1c64e171eeab45c445fa90f4d4a6 +F test/walthread.test 82cc77f8853217a036fc82e4313ecebd9aa368d7 F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 F test/where2.test 45eacc126aabb37959a387aa83e59ce1f1f03820 F test/where3.test 97d3936e6a443b968f1a61cdcc0f673252000e94 @@ -777,7 +782,7 @@ F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/mkkeywordhash.c d2e6b4a5965e23afb80fbe74bb54648cd371f309 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 -F tool/mksqlite3c.tcl 4c6924c7e877defa8f9a12ef1e6867de614acf3f +F tool/mksqlite3c.tcl 25ec827588893857eba2d24a645ace1bb7cdab73 F tool/mksqlite3h.tcl eb100dce83f24b501b325b340f8b5eb8e5106b3b F tool/mksqlite3internalh.tcl 7b43894e21bcb1bb39e11547ce7e38a063357e87 F tool/omittest.tcl 27d6f6e3b1e95aeb26a1c140e6eb57771c6d794a @@ -797,7 +802,11 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 562d20e662da474ea326165730ecfdfcf9b414ee -R 8333162e7f153eb6e6f78b7cbd811ab4 +P 51a613950824698687c0db83b7884db33d45f7f5 +R d1eaede94883ac0829acce36d14527da +T *bgcolor * #ffd0c0 +T *branch * wal +T *sym-wal * +T -sym-trunk * U dan -Z 65fbb5bed20edd048b9a7fedeedd0df1 +Z 9f87e000c4066ee64fca3bc82365cd7e diff --git a/manifest.uuid b/manifest.uuid index 37bbe815aa..bb821cd344 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -51a613950824698687c0db83b7884db33d45f7f5 \ No newline at end of file +409d61baeb0a19d1700c973f16c8acef7b8506cd \ No newline at end of file diff --git a/src/log.c b/src/log.c new file mode 100644 index 0000000000..7d37dfa1de --- /dev/null +++ b/src/log.c @@ -0,0 +1,1243 @@ + +/* +** This file contains the implementation of a log file used in +** "journal_mode=wal" mode. +*/ + +#include "log.h" + +#include +#include +#include + +typedef struct LogSummaryHdr LogSummaryHdr; +typedef struct LogSummary LogSummary; +typedef struct LogCheckpoint LogCheckpoint; + + +/* +** The following structure may be used to store the same data that +** is stored in the log-summary header. +** +** Member variables iCheck1 and iCheck2 contain the checksum for the +** last frame written to the log, or 2 and 3 respectively if the log +** is currently empty. +*/ +struct LogSummaryHdr { + u32 iChange; /* Counter incremented each transaction */ + u32 pgsz; /* Database page size in bytes */ + u32 iLastPg; /* Address of last valid frame in log */ + u32 nPage; /* Size of database in pages */ + u32 iCheck1; /* Checkpoint value 1 */ + u32 iCheck2; /* Checkpoint value 2 */ +}; + +/* Size of serialized LogSummaryHdr object. */ +#define LOGSUMMARY_HDR_NFIELD (sizeof(LogSummaryHdr) / sizeof(u32)) + +#define LOGSUMMARY_FRAME_OFFSET \ + (LOGSUMMARY_HDR_NFIELD + LOG_CKSM_BYTES/sizeof(u32)) + +/* Size of frame header */ +#define LOG_FRAME_HDRSIZE 20 + +/* +** There is one instance of this structure for each log-summary object +** that this process has a connection to. They are stored in a linked +** list starting at pLogSummary (global variable). +** +** TODO: LogSummary.fd is a unix file descriptor. Unix APIs are used +** directly in this implementation because the VFS does not support +** the required blocking file-locks. +*/ +struct LogSummary { + sqlite3_mutex *mutex; /* Mutex used to protect this object */ + int nRef; /* Number of pointers to this structure */ + int fd; /* File descriptor open on log-summary */ + char *zPath; /* Path to associated WAL file */ + LogSummary *pNext; /* Next in global list */ + int nData; /* Size of aData allocation/mapping */ + u32 *aData; /* File body */ +}; + +/* +** List of all LogSummary objects created by this process. Protected by +** static mutex LOG_SUMMARY_MUTEX. TODO: Should have a dedicated mutex +** here instead of borrowing the LRU mutex. +*/ +#define LOG_SUMMARY_MUTEX SQLITE_MUTEX_STATIC_LRU +static LogSummary *pLogSummary = 0; + +struct Log { + LogSummary *pSummary; /* Log file summary data */ + sqlite3_vfs *pVfs; /* The VFS used to create pFd */ + sqlite3_file *pFd; /* File handle for log file */ + int sync_flags; /* Flags to use with OsSync() */ + int isLocked; /* True if a snapshot is held open */ + int isWriteLocked; /* True if this is the writer connection */ + LogSummaryHdr hdr; /* Log summary header for current snapshot */ +}; + +/* +** This structure is used to implement an iterator that iterates through +** all frames in the log in database page order. Where two or more frames +** correspond to the same database page, the iterator visits only the +** frame most recently written to the log. +** +** The internals of this structure are only accessed by: +** +** logCheckpointInit() - Create a new iterator, +** logCheckpointNext() - Step an iterator, +** logCheckpointFree() - Free an iterator. +** +** This functionality is used by the checkpoint code (see logCheckpoint()). +*/ +struct LogCheckpoint { + int nSegment; /* Size of LogCheckpoint.aSummary[] array */ + int nFinal; /* Elements in segment nSegment-1 */ + struct LogSegment { + int iNext; /* Next aIndex index */ + u8 *aIndex; /* Pointer to index array */ + u32 *aDbPage; /* Pointer to db page array */ + } aSegment[1]; +}; + +/* +** Generate an 8 byte checksum based on the data in array aByte[] and the +** initial values of aCksum[0] and aCksum[1]. The checksum is written into +** aCksum[] before returning. +*/ +#define LOG_CKSM_BYTES 8 +static void logChecksumBytes(u8 *aByte, int nByte, u32 *aCksum){ + u32 *z32 = (u32 *)aByte; + int n32 = nByte / sizeof(u32); + int i; + + assert( LOG_CKSM_BYTES==2*sizeof(u32) ); + assert( (nByte&0x00000003)==0 ); + + u32 cksum0 = aCksum[0]; + u32 cksum1 = aCksum[1]; + + for(i=0; i> 8) + (cksum0 ^ z32[i]); + cksum1 = (cksum1 >> 8) + (cksum1 ^ z32[i]); + } + + aCksum[0] = cksum0; + aCksum[1] = cksum1; +} + +/* +** Argument zPath must be a nul-terminated string containing a path-name. +** This function modifies the string in-place by removing any "./" or "../" +** elements in the path. For example, the following input: +** +** "/home/user/plans/good/../evil/./world_domination.txt" +** +** is overwritten with the 'normalized' version: +** +** "/home/user/plans/evil/world_domination.txt" +*/ +static void logNormalizePath(char *zPath){ + int i, j; + char *z = zPath; + int n = strlen(z); + + while( n>1 && z[n-1]=='/' ){ n--; } + for(i=j=0; i0 && z[j-1]!='/' ){ j--; } + if( j>0 ){ j--; } + i += 2; + continue; + } + } + z[j++] = z[i]; + } + z[j] = 0; +} + +/* +** Lock the summary file pSummary->fd. +*/ +static int logSummaryLock(LogSummary *pSummary){ + int rc; + struct flock f; + memset(&f, 0, sizeof(f)); + f.l_type = F_WRLCK; + f.l_whence = SEEK_SET; + f.l_start = 0; + f.l_len = 1; + rc = fcntl(pSummary->fd, F_SETLKW, &f); + if( rc!=0 ){ + return SQLITE_IOERR; + } + return SQLITE_OK; +} + +/* +** Unlock the summary file pSummary->fd. +*/ +static int logSummaryUnlock(LogSummary *pSummary){ + int rc; + struct flock f; + memset(&f, 0, sizeof(f)); + f.l_type = F_UNLCK; + f.l_whence = SEEK_SET; + f.l_start = 0; + f.l_len = 1; + rc = fcntl(pSummary->fd, F_SETLK, &f); + if( rc!=0 ){ + return SQLITE_IOERR; + } + return SQLITE_OK; +} + +/* +** Memory map the first nByte bytes of the summary file opened with +** pSummary->fd at pSummary->aData. If the summary file is smaller than +** nByte bytes in size when this function is called, ftruncate() is +** used to expand it before it is mapped. +** +** It is assumed that an exclusive lock is held on the summary file +** by the caller (to protect the ftruncate()). +*/ +static int logSummaryMap(LogSummary *pSummary, int nByte){ + struct stat sStat; + int rc; + int fd = pSummary->fd; + void *pMap; + + assert( pSummary->aData==0 ); + + /* If the file is less than nByte bytes in size, cause it to grow. */ + rc = fstat(fd, &sStat); + if( rc!=0 ) return SQLITE_IOERR; + if( sStat.st_sizeaData = (u32 *)pMap; + pSummary->nData = nByte; + + return SQLITE_OK; +} + +/* +** Unmap the log-summary mapping and close the file-descriptor. If +** the isTruncate argument is non-zero, truncate the log-summary file +** region to zero bytes. +** +** Regardless of the value of isTruncate, close the file-descriptor +** opened on the log-summary file. +*/ +static int logSummaryUnmap(LogSummary *pSummary, int isTruncate){ + int rc = SQLITE_OK; + if( pSummary->aData ){ + assert( pSummary->fd>0 ); + munmap(pSummary->aData, pSummary->nData); + pSummary->aData = 0; + if( isTruncate ){ + rc = (ftruncate(pSummary->fd, 0) ? SQLITE_IOERR : SQLITE_OK); + } + } + if( pSummary->fd>0 ){ + close(pSummary->fd); + pSummary->fd = -1; + } + return rc; +} + + +static void logSummaryWriteHdr(LogSummary *pSummary, LogSummaryHdr *pHdr){ + u32 *aData = pSummary->aData; + memcpy(aData, pHdr, sizeof(LogSummaryHdr)); + aData[LOGSUMMARY_HDR_NFIELD] = 1; + aData[LOGSUMMARY_HDR_NFIELD+1] = 1; + logChecksumBytes( + (u8 *)aData, sizeof(LogSummaryHdr), &aData[LOGSUMMARY_HDR_NFIELD] + ); +} + +/* +** This function encodes a single frame header and writes it to a buffer +** supplied by the caller. A log frame-header is made up of a series of +** 4-byte big-endian integers, as follows: +** +** 0: Database page size in bytes. +** 4: Page number. +** 8: New database size (for commit frames, otherwise zero). +** 12: Frame checksum 1. +** 16: Frame checksum 2. +*/ +static void logEncodeFrame( + u32 *aCksum, /* IN/OUT: Checksum values */ + u32 iPage, /* Database page number for frame */ + u32 nTruncate, /* New db size (or 0 for non-commit frames) */ + int nData, /* Database page size (size of aData[]) */ + u8 *aData, /* Pointer to page data (for checksum) */ + u8 *aFrame /* OUT: Write encoded frame here */ +){ + assert( LOG_FRAME_HDRSIZE==20 ); + + sqlite3Put4byte(&aFrame[0], nData); + sqlite3Put4byte(&aFrame[4], iPage); + sqlite3Put4byte(&aFrame[8], nTruncate); + + logChecksumBytes(aFrame, 12, aCksum); + logChecksumBytes(aData, nData, aCksum); + + sqlite3Put4byte(&aFrame[12], aCksum[0]); + sqlite3Put4byte(&aFrame[16], aCksum[1]); +} + +/* +** Return 1 and populate *piPage, *pnTruncate and aCksum if the +** frame checksum looks Ok. Otherwise return 0. +*/ +static int logDecodeFrame( + u32 *aCksum, /* IN/OUT: Checksum values */ + u32 *piPage, /* OUT: Database page number for frame */ + u32 *pnTruncate, /* OUT: New db size (or 0 if not commit) */ + int nData, /* Database page size (size of aData[]) */ + u8 *aData, /* Pointer to page data (for checksum) */ + u8 *aFrame /* Frame data */ +){ + logChecksumBytes(aFrame, 12, aCksum); + logChecksumBytes(aData, nData, aCksum); + + if( aCksum[0]!=sqlite3Get4byte(&aFrame[12]) + || aCksum[1]!=sqlite3Get4byte(&aFrame[16]) + ){ + /* Checksum failed. */ + return 0; + } + + *piPage = sqlite3Get4byte(&aFrame[4]); + *pnTruncate = sqlite3Get4byte(&aFrame[8]); + return 1; +} + +static void logMergesort8( + Pgno *aContent, /* Pages in log */ + u8 *aBuffer, /* Buffer of at least *pnList items to use */ + u8 *aList, /* IN/OUT: List to sort */ + int *pnList /* IN/OUT: Number of elements in aList[] */ +){ + int nList = *pnList; + if( nList>1 ){ + int nLeft = nList / 2; /* Elements in left list */ + int nRight = nList - nLeft; /* Elements in right list */ + u8 *aLeft = aList; /* Left list */ + u8 *aRight = &aList[nLeft]; /* Right list */ + int iLeft = 0; /* Current index in aLeft */ + int iRight = 0; /* Current index in aright */ + int iOut = 0; /* Current index in output buffer */ + + /* TODO: Change to non-recursive version. */ + logMergesort8(aContent, aBuffer, aLeft, &nLeft); + logMergesort8(aContent, aBuffer, aRight, &nRight); + + while( iRight=nRight || aContent[aLeft[iLeft]]=nLeft || aContent[aLeft[iLeft]]>dbpage ); + assert( iRight>=nRight || aContent[aRight[iRight]]>dbpage ); + } + memcpy(aList, aBuffer, sizeof(aList[0])*iOut); + *pnList = iOut; + } + +#ifdef SQLITE_DEBUG + { + int i; + for(i=1; i<*pnList; i++){ + assert( aContent[aList[i]] > aContent[aList[i-1]] ); + } + } +#endif +} + + +/* +** Return the index in the LogSummary.aData array that corresponds to +** frame iFrame. The log-summary file consists of a header, followed by +** alternating "map" and "index" blocks. +*/ +static int logSummaryEntry(u32 iFrame){ + return ((((iFrame-1)>>8)<<6) + iFrame-1 + 2 + LOGSUMMARY_HDR_NFIELD); +} + + +/* +** Set an entry in the log-summary map to map log frame iFrame to db +** page iPage. Values are always appended to the log-summary (i.e. the +** value of iFrame is always exactly one more than the value passed to +** the previous call), but that restriction is not enforced or asserted +** here. +*/ +static void logSummaryAppend(LogSummary *pSummary, u32 iFrame, u32 iPage){ + u32 iSlot = logSummaryEntry(iFrame); + + /* Set the log-summary entry itself */ + pSummary->aData[iSlot] = iPage; + + /* If the frame number is a multiple of 256 (frames are numbered starting + ** at 1), build an index of the most recently added 256 frames. + */ + if( (iFrame&0x000000FF)==0 ){ + int i; /* Iterator used while initializing aIndex */ + u32 *aFrame; /* Pointer to array of 256 frames */ + int nIndex; /* Number of entries in index */ + u8 *aIndex; /* 256 bytes to build index in */ + u8 *aTmp; /* Scratch space to use while sorting */ + + aFrame = &pSummary->aData[iSlot-255]; + aIndex = (u8 *)&pSummary->aData[iSlot+1]; + aTmp = &aIndex[256]; + + nIndex = 256; + for(i=0; i<256; i++) aIndex[i] = (u8)i; + logMergesort8(aFrame, aTmp, aIndex, &nIndex); + memset(&aIndex[nIndex], aIndex[nIndex-1], 256-nIndex); + } +} + + +/* +** Recover the log-summary by reading the log file. The caller must hold +** an exclusive lock on the log-summary file. +*/ +static int logSummaryRecover(LogSummary *pSummary, sqlite3_file *pFd){ + int rc; /* Return Code */ + i64 nSize; /* Size of log file */ + LogSummaryHdr hdr; /* Recovered log-summary header */ + + memset(&hdr, 0, sizeof(hdr)); + + rc = sqlite3OsFileSize(pFd, &nSize); + if( rc!=SQLITE_OK ){ + return rc; + } + + if( nSize>LOG_FRAME_HDRSIZE ){ + u8 aBuf[LOG_FRAME_HDRSIZE]; /* Buffer to load first frame header into */ + u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */ + int nFrame; /* Number of bytes at aFrame */ + u8 *aData; /* Pointer to data part of aFrame buffer */ + int iFrame; /* Index of last frame read */ + i64 iOffset; /* Next offset to read from log file */ + int nPgsz; /* Page size according to the log */ + u32 aCksum[2] = {2, 3}; /* Running checksum */ + + /* Read in the first frame header in the file (to determine the + ** database page size). + */ + rc = sqlite3OsRead(pFd, aBuf, LOG_FRAME_HDRSIZE, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* If the database page size is not a power of two, or is greater than + ** SQLITE_MAX_PAGE_SIZE, conclude that the log file contains no valid data. + */ + nPgsz = sqlite3Get4byte(&aBuf[0]); + if( nPgsz&(nPgsz-1) || nPgsz>SQLITE_MAX_PAGE_SIZE ){ + goto finished; + } + + /* Malloc a buffer to read frames into. */ + nFrame = nPgsz + LOG_FRAME_HDRSIZE; + aFrame = (u8 *)sqlite3_malloc(nFrame); + if( !aFrame ){ + return SQLITE_NOMEM; + } + aData = &aFrame[LOG_FRAME_HDRSIZE]; + + /* Read all frames from the log file. */ + iFrame = 0; + iOffset = 0; + for(iOffset=0; (iOffset+nFrame)fd<0 ); + assert( pSummary->aData==0 ); + assert( pSummary->nRef>0 ); + assert( pSummary->zPath ); + + /* Open a file descriptor on the summary file. */ + zFile = sqlite3_mprintf("%s-summary", pSummary->zPath); + if( !zFile ){ + return SQLITE_NOMEM; + } + pSummary->fd = open(zFile, O_RDWR|O_CREAT, S_IWUSR|S_IRUSR); + sqlite3_free(zFile); + if( pSummary->fd<0 ){ + return SQLITE_IOERR; + } + + /* Grab an exclusive lock the summary file. Then mmap() it. TODO: This + ** code needs to be enhanced to support a growable mapping. For now, just + ** make the mapping very large to start with. + */ + rc = logSummaryLock(pSummary); + if( rc!=SQLITE_OK ) return rc; + rc = logSummaryMap(pSummary, 512*1024); + if( rc!=SQLITE_OK ) goto out; + + /* Grab a SHARED lock on the log file. Then try to upgrade to an EXCLUSIVE + ** lock. If successful, then this is the first (and only) connection to + ** the database. In this case assume the contents of the log-summary + ** cannot be trusted. Zero the log-summary header to make sure. + ** + ** The SHARED lock on the log file is not released until the connection + ** to the database is closed. + */ + rc = sqlite3OsLock(pFd, SQLITE_LOCK_SHARED); + if( rc!=SQLITE_OK ) goto out; + rc = sqlite3OsLock(pFd, SQLITE_LOCK_EXCLUSIVE); + if( rc==SQLITE_OK ){ + /* This is the first and only connection. */ + memset(pSummary->aData, 0, (LOGSUMMARY_HDR_NFIELD+2)*sizeof(u32) ); + rc = sqlite3OsUnlock(pFd, SQLITE_LOCK_SHARED); + }else if( rc==SQLITE_BUSY ){ + rc = SQLITE_OK; + } + + out: + logSummaryUnlock(pSummary); + return rc; +} + +/* +** Open a connection to the log file associated with database zDb. The +** database file does not actually have to exist. zDb is used only to +** figure out the name of the log file to open. If the log file does not +** exist it is created by this call. +*/ +int sqlite3LogOpen( + sqlite3_vfs *pVfs, /* vfs module to open log file with */ + const char *zDb, /* Name of database file */ + Log **ppLog /* OUT: Allocated Log handle */ +){ + int rc; /* Return Code */ + Log *pRet; /* Object to allocate and return */ + LogSummary *pSummary = 0; /* Summary object */ + sqlite3_mutex *mutex = 0; /* LOG_SUMMARY_MUTEX mutex */ + int flags; /* Flags passed to OsOpen() */ + char *zWal = 0; /* Path to WAL file */ + int nWal; /* Length of zWal in bytes */ + + /* Zero output variables */ + assert( zDb ); + *ppLog = 0; + + /* Allocate an instance of struct Log to return. */ + pRet = (Log *)sqlite3MallocZero(sizeof(Log) + pVfs->szOsFile); + if( !pRet ) goto out; + pRet->pVfs = pVfs; + pRet->pFd = (sqlite3_file *)&pRet[1]; + pRet->sync_flags = SQLITE_SYNC_NORMAL; + + /* Normalize the path name. */ + zWal = sqlite3_mprintf("%s-wal", zDb); + if( !zWal ) goto out; + logNormalizePath(zWal); + flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_MAIN_DB); + nWal = sqlite3Strlen30(zWal); + + /* Enter the mutex that protects the linked-list of LogSummary structures */ + if( sqlite3GlobalConfig.bCoreMutex ){ + mutex = sqlite3_mutex_alloc(LOG_SUMMARY_MUTEX); + } + sqlite3_mutex_enter(mutex); + + /* Search for an existing log summary object in the linked list. If one + ** cannot be found, allocate and initialize a new object. + */ + for(pSummary=pLogSummary; pSummary; pSummary=pSummary->pNext){ + int nPath = sqlite3Strlen30(pSummary->zPath); + if( nWal==nPath && 0==memcmp(pSummary->zPath, zWal, nPath) ) break; + } + if( !pSummary ){ + int nByte = sizeof(LogSummary) + nWal + 1; + pSummary = (LogSummary *)sqlite3MallocZero(nByte); + if( !pSummary ){ + rc = SQLITE_NOMEM; + goto out; + } + if( sqlite3GlobalConfig.bCoreMutex ){ + pSummary->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_RECURSIVE); + } + pSummary->zPath = (char *)&pSummary[1]; + pSummary->fd = -1; + memcpy(pSummary->zPath, zWal, nWal); + pSummary->pNext = pLogSummary; + pLogSummary = pSummary; + } + pSummary->nRef++; + pRet->pSummary = pSummary; + + /* Exit the mutex protecting the linked-list of LogSummary objects. */ + sqlite3_mutex_leave(mutex); + mutex = 0; + + /* Open file handle on the log file. */ + rc = sqlite3OsOpen(pVfs, pSummary->zPath, pRet->pFd, flags, &flags); + if( rc!=SQLITE_OK ) goto out; + + /* Object pSummary is shared between all connections to the database made + ** by this process. So at this point it may or may not be connected to + ** the log-summary. If it is not, connect it. Otherwise, just take the + ** SHARED lock on the log file. + */ + sqlite3_mutex_enter(pSummary->mutex); + mutex = pSummary->mutex; + if( pSummary->fd<0 ){ + rc = logSummaryInit(pSummary, pRet->pFd); + }else{ + rc = sqlite3OsLock(pRet->pFd, SQLITE_LOCK_SHARED); + } + + out: + sqlite3_mutex_leave(mutex); + sqlite3_free(zWal); + if( rc!=SQLITE_OK ){ + assert(0); + if( pRet ){ + sqlite3OsClose(pRet->pFd); + sqlite3_free(pRet); + } + assert( !pSummary || pSummary->nRef==0 ); + sqlite3_free(pSummary); + } + *ppLog = pRet; + return rc; +} + +static int logCheckpointNext( + LogCheckpoint *p, /* Iterator */ + u32 *piPage, /* OUT: Next db page to write */ + u32 *piFrame /* OUT: Log frame to read from */ +){ + u32 iMin = *piPage; + u32 iRet = 0xFFFFFFFF; + int i; + int nBlock = p->nFinal; + + for(i=p->nSegment-1; i>=0; i--){ + struct LogSegment *pSegment = &p->aSegment[i]; + while( pSegment->iNextaDbPage[pSegment->aIndex[pSegment->iNext]]; + if( iPg>iMin ){ + if( iPgaIndex[pSegment->iNext]; + } + break; + } + pSegment->iNext++; + } + + nBlock = 256; + } + + *piPage = iRet; + return (iRet==0xFFFFFFFF); +} + +static LogCheckpoint *logCheckpointInit(Log *pLog){ + u32 *aData = pLog->pSummary->aData; + LogCheckpoint *p; /* Return value */ + int nSegment; /* Number of segments to merge */ + u32 iLast; /* Last frame in log */ + int nByte; /* Number of bytes to allocate */ + int i; /* Iterator variable */ + int nFinal; /* Number of unindexed entries */ + struct LogSegment *pFinal; /* Final (unindexed) segment */ + u8 *aTmp; /* Temp space used by merge-sort */ + + iLast = pLog->hdr.iLastPg; + nSegment = (iLast >> 8) + 1; + nFinal = (iLast & 0x000000FF); + + nByte = sizeof(LogCheckpoint) + (nSegment-1)*sizeof(struct LogSegment) + 512; + p = (LogCheckpoint *)sqlite3_malloc(nByte); + if( p ){ + memset(p, 0, nByte); + p->nSegment = nSegment; + p->nFinal = nFinal; + } + + for(i=0; iaSegment[i].aDbPage = &aData[logSummaryEntry(i*256+1)]; + p->aSegment[i].aIndex = (u8 *)&aData[logSummaryEntry(i*256+1)+256]; + } + pFinal = &p->aSegment[nSegment-1]; + + pFinal->aDbPage = &aData[logSummaryEntry((nSegment-1)*256+1)]; + pFinal->aIndex = (u8 *)&pFinal[1]; + aTmp = &pFinal->aIndex[256]; + for(i=0; iaIndex[i] = i; + } + logMergesort8(pFinal->aDbPage, aTmp, pFinal->aIndex, &nFinal); + p->nFinal = nFinal; + + return p; +} + +/* +** Free a log iterator allocated by logCheckpointInit(). +*/ +static void logCheckpointFree(LogCheckpoint *p){ + sqlite3_free(p); +} + +/* +** Checkpoint the contents of the log file. +*/ +static int logCheckpoint( + Log *pLog, /* Log connection */ + sqlite3_file *pFd, /* File descriptor open on db file */ + u8 *zBuf /* Temporary buffer to use */ +){ + int rc; /* Return code */ + int pgsz = pLog->hdr.pgsz; /* Database page-size */ + LogCheckpoint *pIter = 0; /* Log iterator context */ + u32 iDbpage = 0; /* Next database page to write */ + u32 iFrame; /* Log frame containing data for iDbpage */ + + /* Allocate the iterator */ + pIter = logCheckpointInit(pLog); + if( !pIter ) return SQLITE_NOMEM; + + /* Sync the log file to disk */ + rc = sqlite3OsSync(pLog->pFd, pLog->sync_flags); + if( rc!=SQLITE_OK ) goto out; + + /* Iterate through the contents of the log, copying data to the db file. */ + while( 0==logCheckpointNext(pIter, &iDbpage, &iFrame) ){ + rc = sqlite3OsRead(pLog->pFd, zBuf, pgsz, + (iFrame-1) * (pgsz+LOG_FRAME_HDRSIZE) + LOG_FRAME_HDRSIZE + ); + if( rc!=SQLITE_OK ) goto out; + rc = sqlite3OsWrite(pFd, zBuf, pgsz, (iDbpage-1)*pgsz); + if( rc!=SQLITE_OK ) goto out; + } + + /* Truncate the database file */ + rc = sqlite3OsTruncate(pFd, ((i64)pLog->hdr.nPage*(i64)pgsz)); + if( rc!=SQLITE_OK ) goto out; + + /* Sync the database file. If successful, update the log-summary. */ + rc = sqlite3OsSync(pFd, pLog->sync_flags); + if( rc!=SQLITE_OK ) goto out; + pLog->hdr.iLastPg = 0; + pLog->hdr.iCheck1 = 2; + pLog->hdr.iCheck2 = 3; + logSummaryWriteHdr(pLog->pSummary, &pLog->hdr); + + /* TODO: If a crash occurs and the current log is copied into the + ** database there is no problem. However, if a crash occurs while + ** writing the next transaction into the start of the log, such that: + ** + ** * The first transaction currently in the log is left intact, but + ** * The second (or subsequent) transaction is damaged, + ** + ** then the database could become corrupt. + ** + ** The easiest thing to do would be to write and sync a dummy header + ** into the log at this point. Unfortunately, that turns out to be + ** an unwelcome performance hit. Alternatives are... + */ +#if 0 + memset(zBuf, 0, LOG_FRAME_HDRSIZE); + rc = sqlite3OsWrite(pLog->pFd, zBuf, LOG_FRAME_HDRSIZE, 0); + if( rc!=SQLITE_OK ) goto out; + rc = sqlite3OsSync(pLog->pFd, pLog->sync_flags); +#endif + + out: + logCheckpointFree(pIter); + return rc; +} + +/* +** Close a connection to a log file. +*/ +int sqlite3LogClose( + Log *pLog, /* Log to close */ + sqlite3_file *pFd, /* Database file */ + u8 *zBuf /* Buffer of at least page-size bytes */ +){ + int rc = SQLITE_OK; + if( pLog ){ + LogSummary *pSummary = pLog->pSummary; + sqlite3_mutex *mutex = 0; + + if( sqlite3GlobalConfig.bCoreMutex ){ + mutex = sqlite3_mutex_alloc(LOG_SUMMARY_MUTEX); + } + sqlite3_mutex_enter(mutex); + + /* Decrement the reference count on the log summary. If this is the last + ** reference to the log summary object in this process, the object will + ** be freed. If this is also the last connection to the database, then + ** checkpoint the database and truncate the log and log-summary files + ** to zero bytes in size. + **/ + pSummary->nRef--; + if( pSummary->nRef==0 ){ + LogSummary **pp; + + rc = logSummaryLock(pSummary); + if( rc==SQLITE_OK ){ + int isTruncate = 0; + int rc2 = sqlite3OsLock(pLog->pFd, SQLITE_LOCK_EXCLUSIVE); + if( rc2==SQLITE_OK ){ + /* This is the last connection to the database (including other + ** processes). Do three things: + ** + ** 1. Checkpoint the db. + ** 2. Truncate the log file to zero bytes. + ** 3. Truncate the log-summary file to zero bytes. + */ + rc2 = logCheckpoint(pLog, pFd, zBuf); + if( rc2==SQLITE_OK ){ + rc2 = sqlite3OsTruncate(pLog->pFd, 0); + } + isTruncate = 1; + }else if( rc2==SQLITE_BUSY ){ + rc2 = SQLITE_OK; + } + logSummaryUnmap(pSummary, isTruncate); + sqlite3OsUnlock(pLog->pFd, SQLITE_LOCK_NONE); + rc = logSummaryUnlock(pSummary); + if( rc2!=SQLITE_OK ) rc = rc2; + } + + /* Remove the LogSummary object from the global list. Then free the + ** mutex and the object itself. + */ + for(pp=&pLogSummary; *pp!=pSummary; pp=&(*pp)->pNext); + *pp = (*pp)->pNext; + sqlite3_mutex_free(pSummary->mutex); + sqlite3_free(pSummary); + } + + sqlite3_mutex_leave(mutex); + + /* Close the connection to the log file and free the Log handle. */ + sqlite3OsClose(pLog->pFd); + sqlite3_free(pLog); + } + return rc; +} + +/* +** Set the flags to pass to the sqlite3OsSync() function when syncing +** the log file. +*/ +#if 0 +void sqlite3LogSetSyncflags(Log *pLog, int sync_flags){ + assert( sync_flags==SQLITE_SYNC_NORMAL || sync_flags==SQLITE_SYNC_FULL ); + pLog->sync_flags = sync_flags; +} +#endif + +/* +** Enter and leave the log-summary mutex. In this context, entering the +** log-summary mutex means: +** +** 1. Obtaining mutex pLog->pSummary->mutex, and +** 2. Taking an exclusive lock on the log-summary file. +** +** i.e. this mutex locks out other processes as well as other threads +** hosted in this address space. +*/ +static int logEnterMutex(Log *pLog){ + LogSummary *pSummary = pLog->pSummary; + int rc; + + sqlite3_mutex_enter(pSummary->mutex); + rc = logSummaryLock(pSummary); + if( rc!=SQLITE_OK ){ + sqlite3_mutex_leave(pSummary->mutex); + } + return rc; +} +static void logLeaveMutex(Log *pLog){ + LogSummary *pSummary = pLog->pSummary; + logSummaryUnlock(pSummary); + sqlite3_mutex_leave(pSummary->mutex); +} + +/* +** The caller must hold a SHARED lock on the database file. +** +** If this call obtains a new read-lock and the database contents have been +** modified since the most recent call to LogCloseSnapshot() on this Log +** connection, then *pChanged is set to 1 before returning. Otherwise, it +** is left unmodified. This is used by the pager layer to determine whether +** or not any cached pages may be safely reused. +*/ +int sqlite3LogOpenSnapshot(Log *pLog, int *pChanged){ + int rc = SQLITE_OK; + if( pLog->isLocked==0 ){ + if( SQLITE_OK==(rc = logEnterMutex(pLog)) ){ + u32 aCksum[2] = {1, 1}; + u32 aHdr[LOGSUMMARY_HDR_NFIELD+2]; + memcpy(aHdr, pLog->pSummary->aData, sizeof(aHdr)); + + /* Verify the checksum on the log-summary header. If it fails, + ** recover the log-summary from the log file. + */ + logChecksumBytes((u8*)aHdr, sizeof(u32)*LOGSUMMARY_HDR_NFIELD, aCksum); + if( aCksum[0]!=aHdr[LOGSUMMARY_HDR_NFIELD] + || aCksum[1]!=aHdr[LOGSUMMARY_HDR_NFIELD+1] + ){ + rc = logSummaryRecover(pLog->pSummary, pLog->pFd); + memcpy(aHdr, pLog->pSummary->aData, sizeof(aHdr)); + *pChanged = 1; + } + if( rc==SQLITE_OK ){ + pLog->isLocked = 1; + if( memcmp(&pLog->hdr, aHdr, sizeof(LogSummaryHdr)) ){ + *pChanged = 1; + memcpy(&pLog->hdr, aHdr, LOGSUMMARY_HDR_NFIELD*sizeof(u32)); + } + } + logLeaveMutex(pLog); + } + } + return rc; +} + +/* +** Unlock the current snapshot. +*/ +void sqlite3LogCloseSnapshot(Log *pLog){ + pLog->isLocked = 0; +} + + + +/* +** Read a page from the log, if it is present. +*/ +int sqlite3LogRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut){ + u32 iRead = 0; + u32 *aData = pLog->pSummary->aData; + int iFrame = (pLog->hdr.iLastPg & 0xFFFFFF00); + + /* Do a linear search of the unindexed block of page-numbers (if any) + ** at the end of the log-summary. An alternative to this would be to + ** build an index in private memory each time a read transaction is + ** opened on a new snapshot. + */ + if( pLog->hdr.iLastPg ){ + u32 *pi = &aData[logSummaryEntry(pLog->hdr.iLastPg)]; + u32 *piStop = pi - (pLog->hdr.iLastPg & 0xFF); + while( *pi!=pgno && pi!=piStop ) pi--; + if( pi!=piStop ){ + iRead = (pi-piStop) + iFrame; + } + } + assert( iRead==0 || aData[logSummaryEntry(iRead)]==pgno ); + + while( iRead==0 && iFrame>0 ){ + int iLow = 0; + int iHigh = 255; + u32 *aFrame; + u8 *aIndex; + + iFrame -= 256; + aFrame = &aData[logSummaryEntry(iFrame+1)]; + aIndex = (u8 *)&aFrame[256]; + + while( iLow<=iHigh ){ + int iTest = (iLow+iHigh)>>1; + u32 iPg = aFrame[aIndex[iTest]]; + + if( iPg==pgno ){ + iRead = iFrame + 1 + aIndex[iTest]; + break; + } + else if( iPghdr.pgsz+LOG_FRAME_HDRSIZE); + iOffset += LOG_FRAME_HDRSIZE; + *pInLog = 1; + return sqlite3OsRead(pLog->pFd, pOut, pLog->hdr.pgsz, iOffset); + } + + *pInLog = 0; + return SQLITE_OK; +} + + +/* +** Set *pPgno to the size of the database file (or zero, if unknown). +*/ +void sqlite3LogMaxpgno(Log *pLog, Pgno *pPgno){ + assert( pLog->isLocked ); + *pPgno = pLog->hdr.nPage; +} + +/* +** The caller must hold at least a RESERVED lock on the database file +** when invoking this function. +** +** This function returns SQLITE_OK if the caller may write to the database. +** Otherwise, if the caller is operating on a snapshot that has already +** been overwritten by another writer, SQLITE_OBE is returned. +*/ +int sqlite3LogWriteLock(Log *pLog, int op){ + assert( pLog->isLocked ); + if( op ){ + if( memcmp(&pLog->hdr, pLog->pSummary->aData, sizeof(pLog->hdr)) ){ + return SQLITE_BUSY; + } + pLog->isWriteLocked = 1; + }else if( pLog->isWriteLocked ){ + memcpy(&pLog->hdr, pLog->pSummary->aData, sizeof(pLog->hdr)); + pLog->isWriteLocked = 0; + } + return SQLITE_OK; +} + +/* +** Write a set of frames to the log. The caller must hold at least a +** RESERVED lock on the database file. +*/ +int sqlite3LogFrames( + Log *pLog, /* Log handle to write to */ + int nPgsz, /* Database page-size in bytes */ + PgHdr *pList, /* List of dirty pages to write */ + Pgno nTruncate, /* Database size after this commit */ + int isCommit, /* True if this is a commit */ + int isSync /* True to sync the log file */ +){ + /* Each frame has a 20 byte header, as follows: + ** + ** + Pseudo-random salt (4 bytes) + ** + Page number (4 bytes) + ** + New database size, or 0 if not a commit frame (4 bytes) + ** + Checksum (CHECKSUM_BYTES bytes); + ** + ** The checksum is computed based on the following: + ** + ** + The previous checksum, or {2, 3} for the first frame in the log. + ** + The non-checksum fields of the frame header, and + ** + The frame contents (page data). + ** + ** This format must also be understood by the code in logSummaryRecover(). + ** The size of the frame header is used by LogRead() and LogCheckpoint(). + */ + int rc; /* Used to catch return codes */ + u32 iFrame; /* Next frame address */ + u8 aFrame[LOG_FRAME_HDRSIZE]; + PgHdr *p; /* Iterator to run through pList with. */ + u32 aCksum[2]; + + PgHdr *pLast; /* Last frame in list */ + int nLast = 0; /* Number of extra copies of last page */ + + assert( LOG_FRAME_HDRSIZE==(4 * 3 + LOG_CKSM_BYTES) ); + assert( pList ); + + aCksum[0] = pLog->hdr.iCheck1; + aCksum[1] = pLog->hdr.iCheck2; + + /* Write the log file. */ + iFrame = pLog->hdr.iLastPg; + for(p=pList; p; p=p->pDirty){ + u32 nDbsize; /* Db-size field for frame header */ + i64 iOffset; /* Write offset in log file */ + + iFrame++; + iOffset = (iFrame-1) * (nPgsz+sizeof(aFrame)); + + /* Populate and write the frame header */ + nDbsize = (isCommit && p->pDirty==0) ? nTruncate : 0; + logEncodeFrame(aCksum, p->pgno, nDbsize, nPgsz, p->pData, aFrame); + rc = sqlite3OsWrite(pLog->pFd, aFrame, sizeof(aFrame), iOffset); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Write the page data */ + rc = sqlite3OsWrite(pLog->pFd, p->pData, nPgsz, iOffset + sizeof(aFrame)); + if( rc!=SQLITE_OK ){ + return rc; + } + pLast = p; + } + + /* Sync the log file if the 'isSync' flag was specified. */ + if( isSync ){ +#if 0 + i64 iSegment = sqlite3OsSectorSize(pLog->pFd); + i64 iOffset = iFrame * (nPgsz+sizeof(aFrame)); + + if( iSegmentpgno,nTruncate,nPgsz,pLast->pData,aFrame); + rc = sqlite3OsWrite(pLog->pFd, aFrame, sizeof(aFrame), iOffset); + if( rc!=SQLITE_OK ){ + return rc; + } + + iOffset += LOG_FRAME_HDRSIZE; + rc = sqlite3OsWrite(pLog->pFd, pLast->pData, nPgsz, iOffset); + if( rc!=SQLITE_OK ){ + return rc; + } + nLast++; + iOffset += nPgsz; + } +#endif + + rc = sqlite3OsSync(pLog->pFd, pLog->sync_flags); + if( rc!=SQLITE_OK ){ + return rc; + } + } + + /* Append data to the log summary. It is not necessary to lock the + ** log-summary to do this as the RESERVED lock held on the db file + ** guarantees that there are no other writers, and no data that may + ** be in use by existing readers is being overwritten. + */ + iFrame = pLog->hdr.iLastPg; + for(p=pList; p; p=p->pDirty){ + iFrame++; + logSummaryAppend(pLog->pSummary, iFrame, p->pgno); + } + while( nLast>0 ){ + iFrame++; + nLast--; + logSummaryAppend(pLog->pSummary, iFrame, pLast->pgno); + } + + /* Update the private copy of the header. */ + pLog->hdr.pgsz = nPgsz; + pLog->hdr.iLastPg = iFrame; + if( isCommit ){ + pLog->hdr.iChange++; + pLog->hdr.nPage = nTruncate; + } + pLog->hdr.iCheck1 = aCksum[0]; + pLog->hdr.iCheck2 = aCksum[1]; + + /* If this is a commit, update the log-summary header too. */ + if( isCommit && SQLITE_OK==(rc = logEnterMutex(pLog)) ){ + logSummaryWriteHdr(pLog->pSummary, &pLog->hdr); + logLeaveMutex(pLog); + } + + return SQLITE_OK; +} + +/* +** Checkpoint the database. When this function is called the caller +** must hold an exclusive lock on the database file. +*/ +int sqlite3LogCheckpoint( + Log *pLog, /* Log connection */ + sqlite3_file *pFd, /* File descriptor open on db file */ + u8 *zBuf /* Temporary buffer to use */ +){ + + /* Assert() that the caller is holding an EXCLUSIVE lock on the + ** database file. + */ +#ifdef SQLITE_DEBUG + int lock; + sqlite3OsFileControl(pFd, SQLITE_FCNTL_LOCKSTATE, &lock); + assert( lock>=4 ); +#endif + + return logCheckpoint(pLog, pFd, zBuf); +} + diff --git a/src/os_unix.c b/src/os_unix.c index 769e75df39..80ce9e0b00 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -1536,9 +1536,11 @@ static int _posixUnlock(sqlite3_file *id, int locktype, int handleNFSUnlock){ ** the file has changed and hence might not know to flush their ** cache. The use of a stale cache can lead to database corruption. */ +#if 0 assert( pFile->inNormalWrite==0 || pFile->dbUpdate==0 || pFile->transCntrChng==1 ); +#endif pFile->inNormalWrite = 0; #endif @@ -2956,10 +2958,12 @@ static int unixRead( /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ +#if 0 assert( pFile->pUnused==0 || offset>=PENDING_BYTE+512 || offset+amt<=PENDING_BYTE ); +#endif got = seekAndRead(pFile, offset, pBuf, amt); if( got==amt ){ @@ -3031,10 +3035,12 @@ static int unixWrite( /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ +#if 0 assert( pFile->pUnused==0 || offset>=PENDING_BYTE+512 || offset+amt<=PENDING_BYTE ); +#endif #ifndef NDEBUG /* If we are doing a normal write to a database file (as opposed to diff --git a/src/pager.c b/src/pager.c index d5c236e24c..383c0fe6bd 100644 --- a/src/pager.c +++ b/src/pager.c @@ -20,6 +20,7 @@ */ #ifndef SQLITE_OMIT_DISKIO #include "sqliteInt.h" +#include "log.h" /* ******************** NOTES ON THE DESIGN OF THE PAGER ************************ @@ -397,6 +398,7 @@ struct Pager { char *pTmpSpace; /* Pager.pageSize bytes of space for tmp use */ PCache *pPCache; /* Pointer to page cache object */ sqlite3_backup *pBackup; /* Pointer to list of ongoing backup processes */ + Log *pLog; /* Log used by "journal_mode=wal" */ }; /* @@ -1185,6 +1187,27 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ return rc; } +/* +** Open a connection to the write-ahead log file for pager pPager. +*/ +static int pagerOpenLog(Pager *pPager){ + if( !pPager->pLog ){ + int rc; /* Return code from LogOpen() */ + + rc = sqlite3LogOpen(pPager->pVfs, pPager->zFilename, &pPager->pLog); + if( rc!=SQLITE_OK ) return rc; + } + return SQLITE_OK; +} + +/* +** Return true if this pager uses a write-ahead log instead of the usual +** rollback journal. Otherwise false. +*/ +static int pagerUseLog(Pager *pPager){ + return (pPager->pLog!=0); +} + /* ** Unlock the database file. This function is a no-op if the pager ** is in exclusive mode. @@ -1197,7 +1220,7 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ */ static void pager_unlock(Pager *pPager){ if( !pPager->exclusiveMode ){ - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ /* Always close the journal file when dropping the database lock. ** Otherwise, another connection with journal_mode=delete might @@ -1216,6 +1239,9 @@ static void pager_unlock(Pager *pPager){ */ pPager->dbSizeValid = 0; + if( pagerUseLog(pPager) ){ + sqlite3LogCloseSnapshot(pPager->pLog); + } rc = osUnlock(pPager->fd, NO_LOCK); if( rc ){ pPager->errCode = rc; @@ -1365,6 +1391,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ assert( isOpen(pPager->jfd) || pPager->pInJournal==0 ); if( isOpen(pPager->jfd) ){ + assert( !pagerUseLog(pPager) ); /* Finalize the journal file. */ if( sqlite3IsMemJournal(pPager->jfd) ){ @@ -1408,7 +1435,10 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ pPager->nRec = 0; sqlite3PcacheCleanAll(pPager->pPCache); - if( !pPager->exclusiveMode ){ + if( pagerUseLog(pPager) ){ + rc2 = sqlite3LogWriteLock(pPager->pLog, 0); + pPager->state = PAGER_SHARED; + }else if( !pPager->exclusiveMode ){ rc2 = osUnlock(pPager->fd, SHARED_LOCK); pPager->state = PAGER_SHARED; pPager->changeCountDone = 0; @@ -2120,6 +2150,9 @@ end_playback: if( rc==SQLITE_OK && pPager->noSync==0 && pPager->state>=PAGER_EXCLUSIVE ){ rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); } + if( rc==SQLITE_OK && pPager->noSync==0 && pPager->state>=PAGER_EXCLUSIVE ){ + rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); + } if( rc==SQLITE_OK ){ rc = pager_end_transaction(pPager, zMaster[0]!='\0'); testcase( rc!=SQLITE_OK ); @@ -2140,6 +2173,97 @@ end_playback: return rc; } + +/* +** Read the content for page pPg out of the database file and into +** pPg->pData. A shared lock or greater must be held on the database +** file before this function is called. +** +** If page 1 is read, then the value of Pager.dbFileVers[] is set to +** the value read from the database file. +** +** If an IO error occurs, then the IO error is returned to the caller. +** Otherwise, SQLITE_OK is returned. +*/ +static int readDbPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */ + Pgno pgno = pPg->pgno; /* Page number to read */ + int rc; /* Return code */ + i64 iOffset; /* Byte offset of file to read from */ + int isInLog = 0; /* True if page is in log file */ + + assert( pPager->state>=PAGER_SHARED && !MEMDB ); + assert( isOpen(pPager->fd) ); + + if( NEVER(!isOpen(pPager->fd)) ){ + assert( pPager->tempFile ); + memset(pPg->pData, 0, pPager->pageSize); + return SQLITE_OK; + } + + if( pagerUseLog(pPager) ){ + /* Try to pull the page from the write-ahead log. */ + rc = sqlite3LogRead(pPager->pLog, pgno, &isInLog, pPg->pData); + } + if( rc==SQLITE_OK && !isInLog ){ + iOffset = (pgno-1)*(i64)pPager->pageSize; + rc = sqlite3OsRead(pPager->fd, pPg->pData, pPager->pageSize, iOffset); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; + } + } + + if( pgno==1 ){ + if( rc ){ + /* If the read is unsuccessful, set the dbFileVers[] to something + ** that will never be a valid file version. dbFileVers[] is a copy + ** of bytes 24..39 of the database. Bytes 28..31 should always be + ** zero. Bytes 32..35 and 35..39 should be page numbers which are + ** never 0xffffffff. So filling pPager->dbFileVers[] with all 0xff + ** bytes should suffice. + ** + ** For an encrypted database, the situation is more complex: bytes + ** 24..39 of the database are white noise. But the probability of + ** white noising equaling 16 bytes of 0xff is vanishingly small so + ** we should still be ok. + */ + memset(pPager->dbFileVers, 0xff, sizeof(pPager->dbFileVers)); + }else{ + u8 *dbFileVers = &((u8*)pPg->pData)[24]; + memcpy(&pPager->dbFileVers, dbFileVers, sizeof(pPager->dbFileVers)); + } + } + CODEC1(pPager, pPg->pData, pgno, 3, rc = SQLITE_NOMEM); + + PAGER_INCR(sqlite3_pager_readdb_count); + PAGER_INCR(pPager->nRead); + IOTRACE(("PGIN %p %d\n", pPager, pgno)); + PAGERTRACE(("FETCH %d page %d hash(%08x)\n", + PAGERID(pPager), pgno, pager_pagehash(pPg))); + + return rc; +} + +static int pagerRollbackLog(Pager *pPager){ + int rc = SQLITE_OK; + PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); + pPager->dbSize = pPager->dbOrigSize; + while( pList && rc==SQLITE_OK ){ + PgHdr *pNext = pList->pDirty; + if( sqlite3PcachePageRefcount(pList)==0 ){ + sqlite3PagerLookup(pPager, pList->pgno); + sqlite3PcacheDrop(pList); + }else{ + rc = readDbPage(pList); + if( rc==SQLITE_OK ){ + pPager->xReiniter(pList); + } + } + pList = pNext; + } + return rc; +} + /* ** Playback savepoint pSavepoint. Or, if pSavepoint==NULL, then playback ** the entire master journal file. The case pSavepoint==NULL occurs when @@ -2197,12 +2321,17 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ */ pPager->dbSize = pSavepoint ? pSavepoint->nOrig : pPager->dbOrigSize; + if( !pSavepoint && pagerUseLog(pPager) ){ + return pagerRollbackLog(pPager); + } + /* Use pPager->journalOff as the effective size of the main rollback ** journal. The actual file might be larger than this in ** PAGER_JOURNALMODE_TRUNCATE or PAGER_JOURNALMODE_PERSIST. But anything ** past pPager->journalOff is off-limits to us. */ szJ = pPager->journalOff; + assert( pagerUseLog(pPager)==0 || szJ==0 ); /* Begin by rolling back records from the main journal starting at ** PagerSavepoint.iOffset and continuing to the next journal header. @@ -2211,7 +2340,7 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ ** will be skipped automatically. Pages are added to pDone as they ** are played back. */ - if( pSavepoint ){ + if( pSavepoint && !pagerUseLog(pPager) ){ iHdrOff = pSavepoint->iHdrOffset ? pSavepoint->iHdrOffset : szJ; pPager->journalOff = pSavepoint->iOffset; while( rc==SQLITE_OK && pPager->journalOffdbSizeValid ){ @@ -2567,15 +2696,23 @@ int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ int rc; /* Error returned by OsFileSize() */ i64 n = 0; /* File size in bytes returned by OsFileSize() */ - assert( isOpen(pPager->fd) || pPager->tempFile ); - if( isOpen(pPager->fd) && (0 != (rc = sqlite3OsFileSize(pPager->fd, &n))) ){ - pager_error(pPager, rc); - return rc; + if( pagerUseLog(pPager) ){ + sqlite3LogMaxpgno(pPager->pLog, &nPage); } - if( n>0 && npageSize ){ - nPage = 1; - }else{ - nPage = (Pgno)(n / pPager->pageSize); + + if( nPage==0 ){ + assert( isOpen(pPager->fd) || pPager->tempFile ); + if( isOpen(pPager->fd) ){ + if( SQLITE_OK!=(rc = sqlite3OsFileSize(pPager->fd, &n)) ){ + pager_error(pPager, rc); + return rc; + } + } + if( n>0 && npageSize ){ + nPage = 1; + }else{ + nPage = (Pgno)(n / pPager->pageSize); + } } if( pPager->state!=PAGER_UNLOCK ){ pPager->dbSize = nPage; @@ -2698,6 +2835,7 @@ void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ assertTruncateConstraint(pPager); } + /* ** This function is called before attempting a hot-journal rollback. It ** syncs the journal file to disk, then sets pPager->journalHdr to the @@ -2738,10 +2876,14 @@ static int pagerSyncHotJournal(Pager *pPager){ ** to the caller. */ int sqlite3PagerClose(Pager *pPager){ + u8 *pTmp = (u8 *)pPager->pTmpSpace; + disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); pPager->errCode = 0; pPager->exclusiveMode = 0; + sqlite3LogClose(pPager->pLog, pPager->fd, pTmp); + pPager->pLog = 0; pager_reset(pPager); if( MEMDB ){ pager_unlock(pPager); @@ -2762,7 +2904,7 @@ int sqlite3PagerClose(Pager *pPager){ PAGERTRACE(("CLOSE %d\n", PAGERID(pPager))); IOTRACE(("CLOSE %p\n", pPager)) sqlite3OsClose(pPager->fd); - sqlite3PageFree(pPager->pTmpSpace); + sqlite3PageFree(pTmp); sqlite3PcacheClose(pPager->pPCache); #ifdef SQLITE_HAS_CODEC @@ -3066,7 +3208,10 @@ static int subjournalPage(PgHdr *pPg){ CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); PAGERTRACE(("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno)); - assert( pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize ); + assert( pagerUseLog(pPager) + || pageInJournal(pPg) + || pPg->pgno>pPager->dbOrigSize + ); rc = write32bits(pPager->sjfd, offset, pPg->pgno); if( rc==SQLITE_OK ){ rc = sqlite3OsWrite(pPager->sjfd, pData2, pPager->pageSize, offset+4); @@ -3107,6 +3252,8 @@ static int pagerStress(void *p, PgHdr *pPg){ assert( pPg->pPager==pPager ); assert( pPg->flags&PGHDR_DIRTY ); + if( pagerUseLog(pPager) ) return SQLITE_OK; + /* The doNotSync flag is set by the sqlite3PagerWrite() function while it ** is journalling a set of two or more database pages that are stored ** on the same disk sector. Syncing the journal is not allowed while @@ -3582,67 +3729,6 @@ static int hasHotJournal(Pager *pPager, int *pExists){ return rc; } -/* -** Read the content for page pPg out of the database file and into -** pPg->pData. A shared lock or greater must be held on the database -** file before this function is called. -** -** If page 1 is read, then the value of Pager.dbFileVers[] is set to -** the value read from the database file. -** -** If an IO error occurs, then the IO error is returned to the caller. -** Otherwise, SQLITE_OK is returned. -*/ -static int readDbPage(PgHdr *pPg){ - Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */ - Pgno pgno = pPg->pgno; /* Page number to read */ - int rc; /* Return code */ - i64 iOffset; /* Byte offset of file to read from */ - - assert( pPager->state>=PAGER_SHARED && !MEMDB ); - assert( isOpen(pPager->fd) ); - - if( NEVER(!isOpen(pPager->fd)) ){ - assert( pPager->tempFile ); - memset(pPg->pData, 0, pPager->pageSize); - return SQLITE_OK; - } - iOffset = (pgno-1)*(i64)pPager->pageSize; - rc = sqlite3OsRead(pPager->fd, pPg->pData, pPager->pageSize, iOffset); - if( rc==SQLITE_IOERR_SHORT_READ ){ - rc = SQLITE_OK; - } - if( pgno==1 ){ - if( rc ){ - /* If the read is unsuccessful, set the dbFileVers[] to something - ** that will never be a valid file version. dbFileVers[] is a copy - ** of bytes 24..39 of the database. Bytes 28..31 should always be - ** zero. Bytes 32..35 and 35..39 should be page numbers which are - ** never 0xffffffff. So filling pPager->dbFileVers[] with all 0xff - ** bytes should suffice. - ** - ** For an encrypted database, the situation is more complex: bytes - ** 24..39 of the database are white noise. But the probability of - ** white noising equaling 16 bytes of 0xff is vanishingly small so - ** we should still be ok. - */ - memset(pPager->dbFileVers, 0xff, sizeof(pPager->dbFileVers)); - }else{ - u8 *dbFileVers = &((u8*)pPg->pData)[24]; - memcpy(&pPager->dbFileVers, dbFileVers, sizeof(pPager->dbFileVers)); - } - } - CODEC1(pPager, pPg->pData, pgno, 3, rc = SQLITE_NOMEM); - - PAGER_INCR(sqlite3_pager_readdb_count); - PAGER_INCR(pPager->nRead); - IOTRACE(("PGIN %p %d\n", pPager, pgno)); - PAGERTRACE(("FETCH %d page %d hash(%08x)\n", - PAGERID(pPager), pgno, pager_pagehash(pPg))); - - return rc; -} - /* ** This function is called to obtain a shared lock on the database file. ** It is illegal to call sqlite3PagerAcquire() until after this function @@ -3696,22 +3782,34 @@ int sqlite3PagerSharedLock(Pager *pPager){ pager_reset(pPager); } - if( pPager->state==PAGER_UNLOCK || isErrorReset ){ + if( pPager->noReadlock ){ + assert( pPager->readOnly ); + pPager->state = PAGER_SHARED; + }else{ + rc = pager_wait_on_lock(pPager, SHARED_LOCK); + if( rc!=SQLITE_OK ){ + assert( pPager->state==PAGER_UNLOCK ); + return pager_error(pPager, rc); + } + } + assert( pPager->state>=SHARED_LOCK ); + + if( pagerUseLog(pPager) ){ + int changed = 0; + rc = sqlite3LogOpenSnapshot(pPager->pLog, &changed); + if( rc==SQLITE_OK ){ + if( changed ){ + pager_reset(pPager); + assert( pPager->errCode || pPager->dbSizeValid==0 ); + } + pPager->state = PAGER_SHARED; + rc = sqlite3PagerPagecount(pPager, &changed); + } + }else if( pPager->state==PAGER_UNLOCK || isErrorReset ){ sqlite3_vfs * const pVfs = pPager->pVfs; int isHotJournal = 0; assert( !MEMDB ); assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); - if( pPager->noReadlock ){ - assert( pPager->readOnly ); - pPager->state = PAGER_SHARED; - }else{ - rc = pager_wait_on_lock(pPager, SHARED_LOCK); - if( rc!=SQLITE_OK ){ - assert( pPager->state==PAGER_UNLOCK ); - return pager_error(pPager, rc); - } - } - assert( pPager->state>=SHARED_LOCK ); /* If a journal file exists, and there is no RESERVED lock on the ** database file, then it either needs to be played back or deleted. @@ -3785,7 +3883,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ pPager->journalOff = 0; pPager->setMaster = 0; pPager->journalHdr = 0; - + /* Make sure the journal file has been synced to disk. */ /* Playback and delete the journal. Drop the database write @@ -3992,8 +4090,8 @@ int sqlite3PagerAcquire( if( MEMDB || nMax<(int)pgno || noContent || !isOpen(pPager->fd) ){ if( pgno>pPager->mxPgno ){ - rc = SQLITE_FULL; - goto pager_acquire_err; + rc = SQLITE_FULL; + goto pager_acquire_err; } if( noContent ){ /* Failure to set the bits in the InJournal bit-vectors is benign. @@ -4088,7 +4186,7 @@ void sqlite3PagerUnref(DbPage *pPg){ */ static int openSubJournal(Pager *pPager){ int rc = SQLITE_OK; - if( isOpen(pPager->jfd) && !isOpen(pPager->sjfd) ){ + if( (pagerUseLog(pPager) || isOpen(pPager->jfd)) && !isOpen(pPager->sjfd) ){ if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->subjInMemory ){ sqlite3MemJournalOpen(pPager->sjfd); }else{ @@ -4237,6 +4335,18 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ } } + if( rc==SQLITE_OK && pagerUseLog(pPager) ){ + /* Grab the write lock on the log file. If successful, upgrade to + ** PAGER_EXCLUSIVE state. Otherwise, return an error code to the caller. + ** The busy-handler is not invoked if another connection already + ** holds the write-lock. If possible, the upper layer will call it. + */ + rc = sqlite3LogWriteLock(pPager->pLog, 1); + if( rc==SQLITE_OK ){ + pPager->dbOrigSize = pPager->dbSize; + } + } + /* No need to open the journal file at this time. It will be ** opened before it is written to. If we defer opening the journal, ** we might save the work of creating a file if the transaction @@ -4249,6 +4359,7 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ ** kept open and either was truncated to 0 bytes or its header was ** overwritten with zeros. */ + assert( pagerUseLog(pPager)==0 ); assert( pPager->nRec==0 ); assert( pPager->dbOrigSize==0 ); assert( pPager->pInJournal==0 ); @@ -4303,6 +4414,7 @@ static int pager_write(PgHdr *pPg){ */ sqlite3PcacheMakeDirty(pPg); if( pageInJournal(pPg) && !subjRequiresPage(pPg) ){ + assert( !pagerUseLog(pPager) ); pPager->dbModified = 1; }else{ @@ -4318,7 +4430,10 @@ static int pager_write(PgHdr *pPg){ if( rc!=SQLITE_OK ){ return rc; } - if( !isOpen(pPager->jfd) && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ + if( !isOpen(pPager->jfd) + && pPager->journalMode!=PAGER_JOURNALMODE_OFF + && pPager->journalMode!=PAGER_JOURNALMODE_WAL + ){ assert( pPager->useJournal ); rc = pager_open_journal(pPager); if( rc!=SQLITE_OK ) return rc; @@ -4330,6 +4445,7 @@ static int pager_write(PgHdr *pPg){ ** the transaction journal if it is not there already. */ if( !pageInJournal(pPg) && isOpen(pPager->jfd) ){ + assert( !pagerUseLog(pPager) ); if( pPg->pgno<=pPager->dbOrigSize ){ u32 cksum; char *pData2; @@ -4710,129 +4826,138 @@ int sqlite3PagerCommitPhaseOne( */ sqlite3BackupRestart(pPager->pBackup); }else if( pPager->state!=PAGER_SYNCED && pPager->dbModified ){ - - /* The following block updates the change-counter. Exactly how it - ** does this depends on whether or not the atomic-update optimization - ** was enabled at compile time, and if this transaction meets the - ** runtime criteria to use the operation: - ** - ** * The file-system supports the atomic-write property for - ** blocks of size page-size, and - ** * This commit is not part of a multi-file transaction, and - ** * Exactly one page has been modified and store in the journal file. - ** - ** If the optimization was not enabled at compile time, then the - ** pager_incr_changecounter() function is called to update the change - ** counter in 'indirect-mode'. If the optimization is compiled in but - ** is not applicable to this transaction, call sqlite3JournalCreate() - ** to make sure the journal file has actually been created, then call - ** pager_incr_changecounter() to update the change-counter in indirect - ** mode. - ** - ** Otherwise, if the optimization is both enabled and applicable, - ** then call pager_incr_changecounter() to update the change-counter - ** in 'direct' mode. In this case the journal file will never be - ** created for this transaction. - */ -#ifdef SQLITE_ENABLE_ATOMIC_WRITE - PgHdr *pPg; - assert( isOpen(pPager->jfd) || pPager->journalMode==PAGER_JOURNALMODE_OFF ); - if( !zMaster && isOpen(pPager->jfd) - && pPager->journalOff==jrnlBufferSize(pPager) - && pPager->dbSize>=pPager->dbFileSize - && (0==(pPg = sqlite3PcacheDirtyList(pPager->pPCache)) || 0==pPg->pDirty) - ){ - /* Update the db file change counter via the direct-write method. The - ** following call will modify the in-memory representation of page 1 - ** to include the updated change counter and then write page 1 - ** directly to the database file. Because of the atomic-write - ** property of the host file-system, this is safe. - */ - rc = pager_incr_changecounter(pPager, 1); - }else{ - rc = sqlite3JournalCreate(pPager->jfd); - if( rc==SQLITE_OK ){ - rc = pager_incr_changecounter(pPager, 0); + if( pagerUseLog(pPager) ){ + PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); + if( pList ){ + rc = sqlite3LogFrames(pPager->pLog, pPager->pageSize, pList, + pPager->dbSize, 1, 1 + ); } - } -#else - rc = pager_incr_changecounter(pPager, 0); -#endif - if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - - /* If this transaction has made the database smaller, then all pages - ** being discarded by the truncation must be written to the journal - ** file. This can only happen in auto-vacuum mode. - ** - ** Before reading the pages with page numbers larger than the - ** current value of Pager.dbSize, set dbSize back to the value - ** that it took at the start of the transaction. Otherwise, the - ** calls to sqlite3PagerGet() return zeroed pages instead of - ** reading data from the database file. - ** - ** When journal_mode==OFF the dbOrigSize is always zero, so this - ** block never runs if journal_mode=OFF. - */ -#ifndef SQLITE_OMIT_AUTOVACUUM - if( pPager->dbSizedbOrigSize - && ALWAYS(pPager->journalMode!=PAGER_JOURNALMODE_OFF) - ){ - Pgno i; /* Iterator variable */ - const Pgno iSkip = PAGER_MJ_PGNO(pPager); /* Pending lock page */ - const Pgno dbSize = pPager->dbSize; /* Database image size */ - pPager->dbSize = pPager->dbOrigSize; - for( i=dbSize+1; i<=pPager->dbOrigSize; i++ ){ - if( !sqlite3BitvecTest(pPager->pInJournal, i) && i!=iSkip ){ - PgHdr *pPage; /* Page to journal */ - rc = sqlite3PagerGet(pPager, i, &pPage); - if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - rc = sqlite3PagerWrite(pPage); - sqlite3PagerUnref(pPage); - if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + sqlite3PcacheCleanAll(pPager->pPCache); + }else{ + /* The following block updates the change-counter. Exactly how it + ** does this depends on whether or not the atomic-update optimization + ** was enabled at compile time, and if this transaction meets the + ** runtime criteria to use the operation: + ** + ** * The file-system supports the atomic-write property for + ** blocks of size page-size, and + ** * This commit is not part of a multi-file transaction, and + ** * Exactly one page has been modified and store in the journal file. + ** + ** If the optimization was not enabled at compile time, then the + ** pager_incr_changecounter() function is called to update the change + ** counter in 'indirect-mode'. If the optimization is compiled in but + ** is not applicable to this transaction, call sqlite3JournalCreate() + ** to make sure the journal file has actually been created, then call + ** pager_incr_changecounter() to update the change-counter in indirect + ** mode. + ** + ** Otherwise, if the optimization is both enabled and applicable, + ** then call pager_incr_changecounter() to update the change-counter + ** in 'direct' mode. In this case the journal file will never be + ** created for this transaction. + */ + #ifdef SQLITE_ENABLE_ATOMIC_WRITE + PgHdr *pPg; + assert( isOpen(pPager->jfd) || pPager->journalMode==PAGER_JOURNALMODE_OFF ); + if( !zMaster && isOpen(pPager->jfd) + && pPager->journalOff==jrnlBufferSize(pPager) + && pPager->dbSize>=pPager->dbFileSize + && (0==(pPg = sqlite3PcacheDirtyList(pPager->pPCache)) || 0==pPg->pDirty) + ){ + /* Update the db file change counter via the direct-write method. The + ** following call will modify the in-memory representation of page 1 + ** to include the updated change counter and then write page 1 + ** directly to the database file. Because of the atomic-write + ** property of the host file-system, this is safe. + */ + rc = pager_incr_changecounter(pPager, 1); + }else{ + rc = sqlite3JournalCreate(pPager->jfd); + if( rc==SQLITE_OK ){ + rc = pager_incr_changecounter(pPager, 0); } - } - pPager->dbSize = dbSize; - } -#endif - - /* Write the master journal name into the journal file. If a master - ** journal file name has already been written to the journal file, - ** or if zMaster is NULL (no master journal), then this call is a no-op. - */ - rc = writeMasterJournal(pPager, zMaster); - if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - - /* Sync the journal file. If the atomic-update optimization is being - ** used, this call will not create the journal file or perform any - ** real IO. - */ - rc = syncJournal(pPager); - if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - - /* Write all dirty pages to the database file. */ - rc = pager_write_pagelist(sqlite3PcacheDirtyList(pPager->pPCache)); - if( rc!=SQLITE_OK ){ - assert( rc!=SQLITE_IOERR_BLOCKED ); - goto commit_phase_one_exit; - } - sqlite3PcacheCleanAll(pPager->pPCache); - - /* If the file on disk is not the same size as the database image, - ** then use pager_truncate to grow or shrink the file here. - */ - if( pPager->dbSize!=pPager->dbFileSize ){ - Pgno nNew = pPager->dbSize - (pPager->dbSize==PAGER_MJ_PGNO(pPager)); - assert( pPager->state>=PAGER_EXCLUSIVE ); - rc = pager_truncate(pPager, nNew); + } + #else + rc = pager_incr_changecounter(pPager, 0); + #endif if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + + /* If this transaction has made the database smaller, then all pages + ** being discarded by the truncation must be written to the journal + ** file. This can only happen in auto-vacuum mode. + ** + ** Before reading the pages with page numbers larger than the + ** current value of Pager.dbSize, set dbSize back to the value + ** that it took at the start of the transaction. Otherwise, the + ** calls to sqlite3PagerGet() return zeroed pages instead of + ** reading data from the database file. + ** + ** When journal_mode==OFF the dbOrigSize is always zero, so this + ** block never runs if journal_mode=OFF. + */ + #ifndef SQLITE_OMIT_AUTOVACUUM + if( pPager->dbSizedbOrigSize + && ALWAYS(pPager->journalMode!=PAGER_JOURNALMODE_OFF) + ){ + Pgno i; /* Iterator variable */ + const Pgno iSkip = PAGER_MJ_PGNO(pPager); /* Pending lock page */ + const Pgno dbSize = pPager->dbSize; /* Database image size */ + pPager->dbSize = pPager->dbOrigSize; + for( i=dbSize+1; i<=pPager->dbOrigSize; i++ ){ + if( !sqlite3BitvecTest(pPager->pInJournal, i) && i!=iSkip ){ + PgHdr *pPage; /* Page to journal */ + rc = sqlite3PagerGet(pPager, i, &pPage); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + rc = sqlite3PagerWrite(pPage); + sqlite3PagerUnref(pPage); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + } + } + pPager->dbSize = dbSize; + } + #endif + + /* Write the master journal name into the journal file. If a master + ** journal file name has already been written to the journal file, + ** or if zMaster is NULL (no master journal), then this call is a no-op. + */ + rc = writeMasterJournal(pPager, zMaster); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + + /* Sync the journal file. If the atomic-update optimization is being + ** used, this call will not create the journal file or perform any + ** real IO. + */ + rc = syncJournal(pPager); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + + /* Write all dirty pages to the database file. */ + rc = pager_write_pagelist(sqlite3PcacheDirtyList(pPager->pPCache)); + if( rc!=SQLITE_OK ){ + assert( rc!=SQLITE_IOERR_BLOCKED ); + goto commit_phase_one_exit; + } + sqlite3PcacheCleanAll(pPager->pPCache); + + /* If the file on disk is not the same size as the database image, + ** then use pager_truncate to grow or shrink the file here. + */ + if( pPager->dbSize!=pPager->dbFileSize ){ + Pgno nNew = pPager->dbSize - (pPager->dbSize==PAGER_MJ_PGNO(pPager)); + assert( pPager->state>=PAGER_EXCLUSIVE ); + rc = pager_truncate(pPager, nNew); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + } + + /* Finally, sync the database file. */ + if( !pPager->noSync && !noSync ){ + rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); + } + IOTRACE(("DBSYNC %p\n", pPager)) } - /* Finally, sync the database file. */ - if( !pPager->noSync && !noSync ){ - rc = sqlite3OsSync(pPager->fd, pPager->sync_flags); - } - IOTRACE(("DBSYNC %p\n", pPager)) - pPager->state = PAGER_SYNCED; } @@ -4940,7 +5065,12 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ int sqlite3PagerRollback(Pager *pPager){ int rc = SQLITE_OK; /* Return code */ PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager))); - if( !pPager->dbModified || !isOpen(pPager->jfd) ){ + if( pagerUseLog(pPager) ){ + int rc2; + rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1); + rc2 = pager_end_transaction(pPager, pPager->setMaster); + if( rc==SQLITE_OK ) rc = rc2; + }else if( !pPager->dbModified || !isOpen(pPager->jfd) ){ rc = pager_end_transaction(pPager, pPager->setMaster); }else if( pPager->errCode && pPager->errCode!=SQLITE_FULL ){ if( pPager->state>=PAGER_EXCLUSIVE ){ @@ -5158,7 +5288,7 @@ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ ** not yet been opened. In this case there have been no changes to ** the database file, so the playback operation can be skipped. */ - else if( isOpen(pPager->jfd) ){ + else if( pagerUseLog(pPager) || isOpen(pPager->jfd) ){ PagerSavepoint *pSavepoint = (nNew==0)?0:&pPager->aSavepoint[nNew-1]; rc = pagerPlaybackSavepoint(pPager, pSavepoint); assert(rc!=SQLITE_DONE); @@ -5435,6 +5565,7 @@ int sqlite3PagerLockingMode(Pager *pPager, int eMode){ ** PAGER_JOURNALMODE_PERSIST ** PAGER_JOURNALMODE_OFF ** PAGER_JOURNALMODE_MEMORY +** PAGER_JOURNALMODE_WAL ** ** If the parameter is not _QUERY, then the journal_mode is set to the ** value specified if the change is allowed. The change is disallowed @@ -5453,11 +5584,12 @@ int sqlite3PagerJournalMode(Pager *pPager, int eMode){ || eMode==PAGER_JOURNALMODE_TRUNCATE || eMode==PAGER_JOURNALMODE_PERSIST || eMode==PAGER_JOURNALMODE_OFF + || eMode==PAGER_JOURNALMODE_WAL || eMode==PAGER_JOURNALMODE_MEMORY ); assert( PAGER_JOURNALMODE_QUERY<0 ); if( eMode>=0 - && (!MEMDB || eMode==PAGER_JOURNALMODE_MEMORY - || eMode==PAGER_JOURNALMODE_OFF) + && (pPager->tempFile==0 || eMode!=PAGER_JOURNALMODE_WAL) + && (!MEMDB || eMode==PAGER_JOURNALMODE_MEMORY||eMode==PAGER_JOURNALMODE_OFF) && !pPager->dbModified && (!isOpen(pPager->jfd) || 0==pPager->journalOff) ){ @@ -5473,6 +5605,16 @@ int sqlite3PagerJournalMode(Pager *pPager, int eMode){ && !pPager->exclusiveMode ){ sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); } + + if( eMode==PAGER_JOURNALMODE_WAL ){ + int rc = pagerOpenLog(pPager); + if( rc!=SQLITE_OK ){ + /* TODO: The error code should not just get dropped here. Change + ** this to set a flag to force the log to be opened the first time + ** it is actually required. */ + return (int)pPager->journalMode; + } + } pPager->journalMode = (u8)eMode; } return (int)pPager->journalMode; @@ -5501,4 +5643,19 @@ sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){ return &pPager->pBackup; } +/* +** This function is called when the user invokes "PRAGMA checkpoint". +*/ +int sqlite3PagerCheckpoint(Pager *pPager, int nMin, int nMax, int doSync){ + int rc = SQLITE_OK; + if( pPager->pLog ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + if( rc==SQLITE_OK ){ + u8 *zBuf = (u8 *)pPager->pTmpSpace; + rc = sqlite3LogCheckpoint(pPager->pLog, pPager->fd, zBuf); + } + } + return rc; +} + #endif /* SQLITE_OMIT_DISKIO */ diff --git a/src/pager.h b/src/pager.h index 7d778c82cd..f329e43d81 100644 --- a/src/pager.h +++ b/src/pager.h @@ -76,6 +76,7 @@ typedef struct PgHdr DbPage; #define PAGER_JOURNALMODE_OFF 2 /* Journal omitted. */ #define PAGER_JOURNALMODE_TRUNCATE 3 /* Commit by truncating journal */ #define PAGER_JOURNALMODE_MEMORY 4 /* In-memory journal file */ +#define PAGER_JOURNALMODE_WAL 5 /* Use write-ahead logging */ /* ** The remainder of this file contains the declarations of the functions diff --git a/src/pragma.c b/src/pragma.c index f03078f246..8e24387d33 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -445,6 +445,31 @@ void sqlite3Pragma( returnSingleInt(pParse, "secure_delete", b); }else + /* + ** PRAGMA [database.]secure_delete + ** PRAGMA [database.]secure_delete=ON/OFF + ** + ** The first form reports the current setting for the + ** secure_delete flag. The second form changes the secure_delete + ** flag setting and reports thenew value. + */ + if( sqlite3StrICmp(zLeft,"secure_delete")==0 ){ + Btree *pBt = pDb->pBt; + int b = -1; + assert( pBt!=0 ); + if( zRight ){ + b = getBoolean(zRight); + } + if( pId2->n==0 && b>=0 ){ + int ii; + for(ii=0; iinDb; ii++){ + sqlite3BtreeSecureDelete(db->aDb[ii].pBt, b); + } + } + b = sqlite3BtreeSecureDelete(pBt, b); + returnSingleInt(pParse, "secure_delete", b); + }else + /* ** PRAGMA [database.]page_count ** @@ -515,7 +540,7 @@ void sqlite3Pragma( if( sqlite3StrICmp(zLeft,"journal_mode")==0 ){ int eMode; static char * const azModeName[] = { - "delete", "persist", "off", "truncate", "memory" + "delete", "persist", "off", "truncate", "memory", "wal" }; if( zRight==0 ){ @@ -561,6 +586,7 @@ void sqlite3Pragma( || eMode==PAGER_JOURNALMODE_TRUNCATE || eMode==PAGER_JOURNALMODE_PERSIST || eMode==PAGER_JOURNALMODE_OFF + || eMode==PAGER_JOURNALMODE_WAL || eMode==PAGER_JOURNALMODE_MEMORY ); sqlite3VdbeSetNumCols(v, 1); sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "journal_mode", SQLITE_STATIC); @@ -1383,6 +1409,26 @@ void sqlite3Pragma( }else #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + if( sqlite3StrICmp(zLeft, "checkpoint")==0 ){ + int nMin = 0; + int nMax = 0; + int nosync = 0; + + if( zRight ){ + char *z = zRight; + sqlite3GetInt32(z, &nMin); + while( sqlite3Isdigit(*z) ) z++; + while( *z && !sqlite3Isdigit(*z) ) z++; + sqlite3GetInt32(z, &nMax); + while( sqlite3Isdigit(*z) ) z++; + while( *z && !sqlite3Isdigit(*z) ) z++; + sqlite3GetInt32(z, &nosync); + } + sqlite3VdbeUsesBtree(v, iDb); + sqlite3VdbeAddOp2(v, OP_Transaction, iDb, 1); + sqlite3VdbeAddOp3(v, OP_Checkpoint, iDb, nMin, nMax); + }else + #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* ** Report the current state of file logs for all databases diff --git a/src/vdbe.c b/src/vdbe.c index c1b0eea313..8e27942331 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5186,6 +5186,21 @@ case OP_AggFinal: { break; } +/* Opcode: Checkpoint P1 P2 P3 * P5 +*/ +case OP_Checkpoint: { + Btree *pBt; /* Btree to checkpoint */ + int nMin = pOp->p2; /* Minimum number of pages to copy */ + int nMax = pOp->p3; /* Maximum number of pages to copy */ + int doNotSync = pOp->p5; /* True to sync database */ + + assert( pOp->p1>=0 && pOp->p1nDb ); + assert( (p->btreeMask & (1<p1))!=0 ); + pBt = db->aDb[pOp->p1].pBt; + + rc = sqlite3PagerCheckpoint(sqlite3BtreePager(pBt), nMin, nMax, !doNotSync); + break; +}; #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) /* Opcode: Vacuum * * * * * diff --git a/test/wal.test b/test/wal.test new file mode 100644 index 0000000000..57fccce288 --- /dev/null +++ b/test/wal.test @@ -0,0 +1,551 @@ + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +proc range {args} { + set ret [list] + foreach {start end} $args { + for {set i $start} {$i <= $end} {incr i} { + lappend ret $i + } + } + set ret +} + +proc reopen_db {} { + db close + file delete -force test.db test.db-wal + sqlite3_wal db test.db + #register_logtest +} +proc register_logtest {{db db}} { + register_logsummary_module $db + execsql { CREATE VIRTUAL TABLE temp.logsummary USING logsummary } $db + execsql { CREATE VIRTUAL TABLE temp.logcontent USING logcontent } $db + execsql { CREATE VIRTUAL TABLE temp.loglock USING loglock } $db +} + +proc sqlite3_wal {args} { + eval sqlite3 $args + [lindex $args 0] eval { PRAGMA journal_mode = wal } +} + +# +# These are 'warm-body' tests used while developing the WAL code. They +# serve to prove that a few really simple cases work: +# +# wal-1.*: Read and write the database. +# wal-2.*: Test MVCC with one reader, one writer. +# wal-3.*: Test transaction rollback. +# wal-4.*: Test savepoint/statement rollback. +# wal-5.*: Test the temp database. +# wal-6.*: Test creating databases with different page sizes. +# + +do_test wal-0.1 { + execsql { PRAGMA journal_mode = wal } +} {wal} + +do_test wal-1.0 { + execsql { + BEGIN; + CREATE TABLE t1(a, b); + } + list [file exists test.db-journal] [file exists test.db-wal] +} {0 1} +do_test wal-1.1 { + execsql COMMIT + list [file exists test.db-journal] [file exists test.db-wal] +} {0 1} +do_test wal-1.2 { + # There are now two pages in the log. + file size test.db-wal +} [expr (20+1024)*2] + +do_test wal-1.3 { + execsql { SELECT * FROM sqlite_master } +} {table t1 t1 2 {CREATE TABLE t1(a, b)}} + +do_test wal-1.4 { + execsql { INSERT INTO t1 VALUES(1, 2) } + execsql { INSERT INTO t1 VALUES(3, 4) } + execsql { INSERT INTO t1 VALUES(5, 6) } + execsql { INSERT INTO t1 VALUES(7, 8) } + execsql { INSERT INTO t1 VALUES(9, 10) } +} {} + +do_test wal-1.5 { + execsql { SELECT * FROM t1 } +} {1 2 3 4 5 6 7 8 9 10} + +do_test wal-2.1 { + sqlite3_wal db2 ./test.db + execsql { BEGIN; SELECT * FROM t1 } db2 +} {1 2 3 4 5 6 7 8 9 10} + +do_test wal-2.2 { + execsql { INSERT INTO t1 VALUES(11, 12) } + execsql { SELECT * FROM t1 } +} {1 2 3 4 5 6 7 8 9 10 11 12} + +do_test wal-2.3 { + execsql { SELECT * FROM t1 } db2 +} {1 2 3 4 5 6 7 8 9 10} + +do_test wal-2.4 { + execsql { INSERT INTO t1 VALUES(13, 14) } + execsql { SELECT * FROM t1 } +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14} + +do_test wal-2.5 { + execsql { SELECT * FROM t1 } db2 +} {1 2 3 4 5 6 7 8 9 10} + +do_test wal-2.6 { + execsql { COMMIT; SELECT * FROM t1 } db2 +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14} + +do_test wal-3.1 { + execsql { BEGIN; DELETE FROM t1 } + execsql { SELECT * FROM t1 } +} {} +do_test wal-3.2 { + execsql { SELECT * FROM t1 } db2 +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14} +do_test wal-3.3 { + execsql { ROLLBACK } + execsql { SELECT * FROM t1 } +} {1 2 3 4 5 6 7 8 9 10 11 12 13 14} +db2 close + +do_test wal-4.1 { + execsql { + DELETE FROM t1; + BEGIN; + INSERT INTO t1 VALUES('a', 'b'); + SAVEPOINT sp; + INSERT INTO t1 VALUES('c', 'd'); + SELECT * FROM t1; + } +} {a b c d} +do_test wal-4.2 { + execsql { + ROLLBACK TO sp; + SELECT * FROM t1; + } +} {a b} +do_test wal-4.3 { + execsql { + COMMIT; + SELECT * FROM t1; + } +} {a b} + +do_test wal-5.1 { + execsql { + CREATE TEMP TABLE t2(a, b); + INSERT INTO t2 VALUES(1, 2); + } +} {} +do_test wal-5.2 { + execsql { + BEGIN; + INSERT INTO t2 VALUES(3, 4); + SELECT * FROM t2; + } +} {1 2 3 4} +do_test wal-5.3 { + execsql { + ROLLBACK; + SELECT * FROM t2; + } +} {1 2} +do_test wal-5.4 { + execsql { + CREATE TEMP TABLE t3(x UNIQUE); + BEGIN; + INSERT INTO t2 VALUES(3, 4); + INSERT INTO t3 VALUES('abc'); + } + catchsql { INSERT INTO t3 VALUES('abc') } +} {1 {column x is not unique}} +do_test wal-5.5 { + execsql { + COMMIT; + SELECT * FROM t2; + } +} {1 2 3 4} +db close + + +foreach sector {512 4096} { + sqlite3_simulate_device -sectorsize $sector + foreach pgsz {512 1024 2048 4096} { + file delete -force test.db test.db-wal + do_test wal-6.$sector.$pgsz.1 { + sqlite3_wal db test.db -vfs devsym + execsql " + PRAGMA page_size = $pgsz ; + " + execsql " + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + " + db close + file size test.db + } [expr $pgsz*2] + + do_test wal-6.$sector.$pgsz.2 { + file size test.db-wal + } {0} + } +} + +do_test wal-7.1 { + file delete -force test.db test.db-wal + sqlite3_wal db test.db + execsql { + PRAGMA page_size = 1024; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + } + + list [file size test.db] [file size test.db-wal] +} [list 0 [expr (1024+20)*3]] +do_test wal-7.2 { + execsql { PRAGMA checkpoint } + list [file size test.db] [file size test.db-wal] +} [list 2048 [expr (1024+20)*3]] + +# db close +# sqlite3_wal db test.db +# register_logsummary_module db +# # Warm-body tests of the virtual tables used for testing. +# # +# do_test wal-8.1 { +# execsql { CREATE VIRTUAL TABLE temp.logsummary USING logsummary } +# execsql { CREATE VIRTUAL TABLE temp.logcontent USING logcontent } +# execsql { CREATE VIRTUAL TABLE temp.loglock USING loglock } +# execsql { SELECT * FROM logsummary } +# } [list [file join [pwd] test.db] 1 0 0 0 0 0 0] +# +# do_test wal-8.2 { +# sqlite3_wal db2 test.db +# execsql { SELECT * FROM logsummary } +# } [list [file join [pwd] test.db] 2 0 0 0 0 0 0] +# do_test wal-8.3 { +# db2 close +# execsql { SELECT * FROM logsummary } +# } [list [file join [pwd] test.db] 1 0 0 0 0 0 0] +# do_test wal-8.4 { +# execsql { INSERT INTO t1 VALUES(3, 4) } +# execsql { SELECT * FROM logsummary } +# } [list [file join [pwd] test.db] 1 1024 2 2 3 0 0] +# do_test wal-8.5 { +# execsql { PRAGMA checkpoint } +# execsql { SELECT * FROM logsummary } +# } [list [file join [pwd] test.db] 1 1024 0 0 0 0 0] +# +# do_test wal-8.6 { +# execsql { INSERT INTO t1 VALUES(5, 6) } +# execsql { PRAGMA checkpoint('1 1 1') } +# execsql { SELECT * FROM logsummary } +# } [list [file join [pwd] test.db] 1 1024 2 0 3 0 0] +# do_test wal-8.7 { +# execsql { SELECT logpage, dbpage FROM logcontent } +# } {} +# do_test wal-8.8 { +# execsql { INSERT INTO t1 VALUES(7, 8) } +# execsql { SELECT logpage, dbpage FROM logcontent } +# } {4 T:4 5 2} +# do_test wal-8.9 { +# execsql { SELECT * FROM logsummary } +# } [list [file join [pwd] test.db] 1 1024 2 4 5 0 0] +# do_test wal-8.10 { +# execsql { SELECT * FROM loglock } +# } [list [file join [pwd] test.db] 0 0 0] +# do_test wal-8.11 { +# execsql { BEGIN; SELECT * FROM t1; } +# execsql { SELECT * FROM loglock } +# } [list [file join [pwd] test.db] 0 0 4] +# +# # Try making the log wrap around. +# # +# reopen_db +# +# do_test wal-9.1 { +# execsql { +# BEGIN; +# CREATE TABLE t1(a PRIMARY KEY, b); +# } +# for {set i 0} {$i < 100} {incr i} { +# execsql { INSERT INTO t1 VALUES($i, randomblob(100)) } +# } +# execsql COMMIT +# execsql { SELECT * FROM logsummary } +# } [list [file join [pwd] test.db] 1 1024 2 2 17 0 0] +# do_test wal-9.2 { +# execsql { SELECT logpage, dbpage FROM logcontent } +# } {2 T:2 3 1 4 2 5 3 6 4 7 5 8 6 9 7 10 8 11 9 12 10 13 11 14 12 15 13 16 14 17 15} +# do_test wal-9.3 { +# execsql { PRAGMA checkpoint('12, 12') } +# execsql { SELECT * FROM logsummary } +# } [list [file join [pwd] test.db] 1 1024 2 15 17 0 0] +# do_test wal-9.4 { +# execsql { SELECT logpage, dbpage FROM logcontent } +# } {15 13 16 14 17 15} +# do_test wal-9.5 { +# execsql { SELECT count(*) FROM t1 } +# } {100} +# do_test wal-9.6 { +# execsql { INSERT INTO t1 VALUES(100, randomblob(100)) } +# execsql { SELECT * FROM logsummary } +# } [list [file join [pwd] test.db] 1 1024 2 15 20 0 0] +# +# do_test wal-9.7 { +# execsql { SELECT count(*) FROM t1 } +# } {101} +# do_test wal-9.8 { +# db close +# sqlite3_wal db test.db +# register_logtest +# execsql { SELECT count(*) FROM t1 } +# } {101} +# do_test wal-9.9 { +# execsql { SELECT * FROM logsummary } +# } [list [file join [pwd] test.db] 1 0 0 0 0 0 0] +# +# reopen_db +# do_test wal-10.1 { +# execsql { +# PRAGMA page_size = 1024; +# CREATE TABLE t1(x PRIMARY KEY); +# INSERT INTO t1 VALUES(randomblob(900)); +# INSERT INTO t1 VALUES(randomblob(900)); +# INSERT INTO t1 SELECT randomblob(900) FROM t1; -- 4 +# INSERT INTO t1 SELECT randomblob(900) FROM t1; -- 8 +# INSERT INTO t1 SELECT randomblob(900) FROM t1; -- 16 +# } +# list [file size test.db] [file size test.db-wal] +# } {0 55296} +# do_test wal-10.2 { +# execsql { PRAGMA checkpoint('20 30') } +# } {} +# do_test wal-10.3 { +# execsql { SELECT * FROM logsummary } +# } [list [file join [pwd] test.db] 1 1024 34 38 54 0 0] +# do_test wal-10.4 { +# execsql { SELECT dbpage FROM logcontent } +# } {21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37} +# do_test wal-10.5 { +# execsql { INSERT INTO t1 VALUES(randomblob(900)) } +# execsql { SELECT * FROM logsummary } +# } [list [file join [pwd] test.db] 1 1024 34 38 6 34 54] +# do_test wal-10.6 { +# execsql { SELECT count(*) FROM t1 WHERE x NOT NULL } +# } {17} +# do_test wal-10.8 { +# execsql { SELECT logpage FROM logcontent } +# } [range 38 54 1 6] +# do_test wal-10.9 { +# execsql { INSERT INTO t1 SELECT randomblob(900) FROM t1 } +# execsql { SELECT * FROM logsummary } +# } [list [file join [pwd] test.db] 1 1024 34 38 68 34 54] +# +# do_test wal-10.10 { +# execsql { SELECT logpage FROM logcontent } +# } [range 38 54 1 33 55 68] +# +# do_test wal-10.11 { +# execsql { SELECT count(*) FROM t1 WHERE x NOT NULL } +# } {34} +# +# do_test wal-10.12 { +# execsql { PRAGMA checkpoint('35 35') } +# } {} +# do_test wal-10.13 { +# execsql { SELECT logpage FROM logcontent } +# } [range 22 68] +# do_test wal-10.13a { +# execsql { SELECT dbpage FROM logcontent } +# } [list \ +# 50 51 52 53 54 55 56 57 58 59 60 61 \ +# 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 \ +# 62 63 64 65 66 67 68 69 70 71 72 73 74 75 \ +# ] +# +# do_test wal-10.14 { +# execsql { SELECT count(*) FROM t1 WHERE x NOT NULL } +# } {34} +# do_test wal-10.15 { +# execsql { PRAGMA integrity_check } +# } {ok} +# do_test wal-10.16 { +# execsql { PRAGMA checkpoint('20 20') } +# execsql { SELECT * FROM logsummary } +# } [list [file join [pwd] test.db] 1 1024 7 63 68 0 0] +# do_test wal-10.17 { +# execsql { SELECT logpage FROM logcontent } +# } [range 63 68] +# do_test wal-10.17a { +# execsql { SELECT dbpage FROM logcontent } +# } {70 71 72 73 74 75} +# +# do_test wal-10.18 { +# execsql { INSERT INTO t1 SELECT randomblob(900) FROM t1 } +# execsql { SELECT logpage FROM logcontent } +# } [range 63 147] +# integrity_check wal-10.19 +# +# do_test wal-10.20 { +# execsql { PRAGMA checkpoint('52 52') } +# execsql { SELECT logpage FROM logcontent } +# } [range 116 147] +# do_test wal-10.20a { +# execsql { SELECT * FROM logsummary } +# } [list [file join [pwd] test.db] 1 1024 69 116 147 0 0] +# integrity_check wal-10.20.integrity +# +# do_test wal-10.21 { +# execsql { INSERT INTO t1 VALUES( randomblob(900) ) } +# execsql { SELECT logpage FROM logcontent } +# } [range 116 152] +# do_test wal-10.22 { +# execsql { PRAGMA integrity_check } +# } {ok} +# +# file delete -force testX.db testX.db-wal +# file copy test.db testX.db +# file copy test.db-wal testX.db-wal +# do_test wal-10.23 { +# sqlite3_wal db2 testX.db +# register_logtest db2 +# execsql { SELECT logpage FROM logcontent WHERE db LIKE '%testX%' } db2 +# } [range 34 54 1 33 55 152] +# +# do_test wal-10.24 { +# execsql { PRAGMA integrity_check } db2 +# } {ok} +# db2 close +# +# do_test wal-11.1 { +# reopen_db +# sqlite3_wal db2 test.db +# +# execsql { +# BEGIN; +# CREATE TABLE t1(x); +# CREATE TABLE t2(x PRIMARY KEY); +# INSERT INTO t1 VALUES(randomblob(900)); +# INSERT INTO t1 VALUES(randomblob(900)); +# INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4 */ +# INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8 */ +# INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16 */ +# INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 32 */ +# INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 64 */ +# +# INSERT INTO t2 VALUES('x'); +# INSERT INTO t2 VALUES('y'); +# INSERT INTO t2 VALUES('z'); +# COMMIT; +# SELECT * FROM logsummary; +# } +# } [list [file join [pwd] test.db] 2 1024 2 2 70 0 0] +# +# do_test wal-11.2 { +# execsql { +# BEGIN; SELECT x FROM t2; +# } db2 +# } {x y z} +# do_test wal-11.2 { +# execsql { +# INSERT INTO t1 VALUES(randomblob(900)); +# PRAGMA checkpoint('10 100'); +# INSERT INTO t1 VALUES(randomblob(900)); +# INSERT INTO t2 VALUES('0'); +# SELECT * FROM logsummary; +# } +# } [list [file join [pwd] test.db] 2 1024 71 71 7 71 73] +# do_test wal-12.3 { +# execsql { PRAGMA integrity_check } db2 +# } {ok} +# db2 close + + +# Execute some transactions in auto-vacuum mode to test database file +# truncation. +# +do_test wal-12.1 { + reopen_db + execsql { + PRAGMA auto_vacuum = 1; + PRAGMA auto_vacuum; + } +} {1} +do_test wal-12.2 { + execsql { + PRAGMA page_size = 1024; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(randomblob(900)); + INSERT INTO t1 VALUES(randomblob(900)); + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 32 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 64 */ + PRAGMA checkpoint; + } + file size test.db +} [expr 67*1024] +do_test wal-12.3 { + execsql { + DELETE FROM t1 WHERE rowid<54; + PRAGMA checkpoint('1 100000'); + } + file size test.db +} [expr 14*1024] + +# Run some "warm-body" tests to ensure that log-summary files with more +# than 256 entries (log summaries that contain index blocks) work Ok. +# +do_test wal-13.1 { + reopen_db + execsql { + PRAGMA page_size = 1024; + CREATE TABLE t1(x PRIMARY KEY); + INSERT INTO t1 VALUES(randomblob(900)); + INSERT INTO t1 VALUES(randomblob(900)); + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 32 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 64 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 128 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 256 */ + } + file size test.db +} 0 +do_test wal-13.2 { + sqlite3_wal db2 test.db + execsql {PRAGMA integrity_check } db2 +} {ok} + +do_test wal-13.3 { + file delete -force test2.db test2.db-wal + file copy test.db test2.db + file copy test.db-wal test2.db-wal + sqlite3_wal db3 test2.db + execsql {PRAGMA integrity_check } db3 +} {ok} +db3 close + +do_test wal-13.4 { +breakpoint + execsql { PRAGMA checkpoint } + db2 close + sqlite3_wal db2 test.db + execsql {PRAGMA integrity_check } db2 +} {ok} + +finish_test + diff --git a/test/walcrash.test b/test/walcrash.test new file mode 100644 index 0000000000..b3e6cd2b97 --- /dev/null +++ b/test/walcrash.test @@ -0,0 +1,248 @@ +# 2010 February 8 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +# +# These are 'warm-body' tests of database recovery used while developing +# the WAL code. They serve to prove that a few really simple cases work: +# +# walcrash-1.*: Recover a database. +# walcrash-2.*: Recover a database where the failed transaction spanned more +# than one page. +# walcrash-3.*: Recover multiple databases where the failed transaction +# was a multi-file transaction. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +db close + +set seed 0 +set REPEATS 100 + +proc sqlite3_wal {args} { + eval sqlite3 $args + [lindex $args 0] eval { PRAGMA journal_mode = wal } +} + +# walcrash-1.* +# +for {set i 1} {$i < $REPEATS} {incr i} { + file delete -force test.db test.db-wal + do_test walcrash-1.$i.1 { + crashsql -delay 4 -file test.db-wal -seed [incr seed] { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 1); + INSERT INTO t1 VALUES(2, 3); + INSERT INTO t1 VALUES(3, 6); + } + } {1 {child process exited abnormally}} + do_test walcrash-1.$i.2 { + sqlite3_wal db test.db + execsql { SELECT sum(a)==max(b) FROM t1 } + } {1} + integrity_check walcrash-1.$i.3 + db close + + do_test walcrash-1.$i.4 { + crashsql -delay 2 -file test.db-wal -seed [incr seed] { + PRAGMA journal_mode = WAL; + PRAGMA journal_mode = WAL; + INSERT INTO t1 VALUES(4, (SELECT sum(a) FROM t1) + 4); + INSERT INTO t1 VALUES(5, (SELECT sum(a) FROM t1) + 5); + } + } {1 {child process exited abnormally}} + do_test walcrash-1.$i.5 { + sqlite3_wal db test.db + execsql { SELECT sum(a)==max(b) FROM t1 } + } {1} + integrity_check walcrash-1.$i.6 + db close +} + +# walcrash-2.* +# +for {set i 1} {$i < $REPEATS} {incr i} { + file delete -force test.db test.db-wal + do_test walcrash-2.$i.1 { + crashsql -delay 4 -file test.db-wal -seed [incr seed] { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 9); + } + } {1 {child process exited abnormally}} + do_test walcrash-2.$i.2 { + sqlite3_wal db test.db + execsql { SELECT sum(a)==max(b) FROM t1 } + } {1} + integrity_check walcrash-2.$i.3 + db close + + do_test walcrash-2.$i.4 { + crashsql -delay 2 -file test.db-wal -seed [incr seed] { + PRAGMA journal_mode = WAL; + INSERT INTO t1 VALUES(6, (SELECT sum(a) FROM t1) + 6); + INSERT INTO t1 VALUES(7, (SELECT sum(a) FROM t1) + 7); + } + } {1 {child process exited abnormally}} + do_test walcrash-2.$i.5 { + sqlite3_wal db test.db + execsql { SELECT sum(a)==max(b) FROM t1 } + } {1} + integrity_check walcrash-2.$i.6 + db close +} + +# walcrash-3.* +# +# for {set i 1} {$i < $REPEATS} {incr i} { +# file delete -force test.db test.db-wal +# file delete -force test2.db test2.db-wal +# +# do_test walcrash-3.$i.1 { +# crashsql -delay 2 -file test2.db-wal -seed [incr seed] { +# PRAGMA journal_mode = WAL; +# ATTACH 'test2.db' AS aux; +# CREATE TABLE t1(a PRIMARY KEY, b); +# CREATE TABLE aux.t2(a PRIMARY KEY, b); +# BEGIN; +# INSERT INTO t1 VALUES(1, 2); +# INSERT INTO t2 VALUES(1, 2); +# COMMIT; +# } +# } {1 {child process exited abnormally}} +# +# do_test walcrash-3.$i.2 { +# sqlite3_wal db test.db +# execsql { +# ATTACH 'test2.db' AS aux; +# SELECT * FROM t1 EXCEPT SELECT * FROM t2; +# } +# } {} +# do_test walcrash-3.$i.3 { execsql { PRAGMA main.integrity_check } } {ok} +# do_test walcrash-3.$i.4 { execsql { PRAGMA aux.integrity_check } } {ok} +# +# db close +# } + +# walcrash-4.* +# +for {set i 1} {$i < $REPEATS} {incr i} { + file delete -force test.db test.db-wal + file delete -force test2.db test2.db-wal + + do_test walcrash-4.$i.1 { + crashsql -delay 3 -file test.db-wal -seed [incr seed] -blocksize 4096 { + PRAGMA journal_mode = WAL; + PRAGMA page_size = 1024; + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + } + } {1 {child process exited abnormally}} + + do_test walcrash-4.$i.2 { + sqlite3_wal db test.db + execsql { + SELECT * FROM t1 WHERE a = 1; + } + } {1 2} + do_test walcrash-4.$i.3 { execsql { PRAGMA main.integrity_check } } {ok} + + db close +} + +# walcrash-5.* +# +for {set i 1} {$i < $REPEATS} {incr i} { + file delete -force test.db test.db-wal + file delete -force test2.db test2.db-wal + + do_test walcrash-5.$i.1 { + crashsql -delay 11 -file test.db-wal -seed [incr seed] -blocksize 4096 { + PRAGMA journal_mode = WAL; + PRAGMA page_size = 1024; + BEGIN; + CREATE TABLE t1(x PRIMARY KEY); + INSERT INTO t1 VALUES(randomblob(900)); + INSERT INTO t1 VALUES(randomblob(900)); + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4 */ + COMMIT; + INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 8 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 12 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 16 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 20 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 24 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 28 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 32 */ + + PRAGMA checkpoint('70 70'); + INSERT INTO t1 VALUES(randomblob(900)); + INSERT INTO t1 VALUES(randomblob(900)); + INSERT INTO t1 VALUES(randomblob(900)); + } + } {1 {child process exited abnormally}} + + do_test walcrash-5.$i.2 { + sqlite3_wal db test.db + execsql { SELECT count(*)==33 OR count(*)==34 FROM t1 WHERE x != 1 } + } {1} + do_test walcrash-5.$i.3 { execsql { PRAGMA main.integrity_check } } {ok} + + db close +} + +# walcrash-6.* +# +for {set i 1} {$i < $REPEATS} {incr i} { + file delete -force test.db test.db-wal + file delete -force test2.db test2.db-wal + + do_test walcrash-6.$i.1 { + crashsql -delay 12 -file test.db-wal -seed [incr seed] -blocksize 512 { + PRAGMA journal_mode = WAL; + PRAGMA page_size = 1024; + BEGIN; + CREATE TABLE t1(x PRIMARY KEY); + INSERT INTO t1 VALUES(randomblob(900)); + INSERT INTO t1 VALUES(randomblob(900)); + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4 */ + COMMIT; + INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 8 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 12 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 16 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 20 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 24 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 28 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 32 */ + + PRAGMA checkpoint('70 70'); + INSERT INTO t1 VALUES(randomblob(900)); + INSERT INTO t1 VALUES(randomblob(900)); + INSERT INTO t1 VALUES(randomblob(900)); + } + } {1 {child process exited abnormally}} + + do_test walcrash-6.$i.2 { + sqlite3_wal db test.db + execsql { SELECT count(*)==34 OR count(*)==35 FROM t1 WHERE x != 1 } + } {1} + do_test walcrash-6.$i.3 { execsql { PRAGMA main.integrity_check } } {ok} + + db close +} + +finish_test + diff --git a/test/walslow.test b/test/walslow.test new file mode 100644 index 0000000000..cec4e26d98 --- /dev/null +++ b/test/walslow.test @@ -0,0 +1,69 @@ +# 2010 March 17 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +proc reopen_db {} { + catch { db close } + file delete -force test.db test.db-wal + sqlite3 db test.db + execsql { PRAGMA journal_mode = wal } +} + +db close +save_prng_state +for {set seed 1} {$seed<10} {incr seed} { + expr srand($seed) + restore_prng_state + reopen_db + do_test walslow-1.seed=$seed.0 { + execsql { CREATE TABLE t1(a, b) } + execsql { CREATE INDEX i1 ON t1(a) } + execsql { CREATE INDEX i2 ON t1(b) } + } {} + + for {set iTest 1} {$iTest < 100} {incr iTest} { + + do_test walslow-1.seed=$seed.$iTest.1 { + set w [expr int(rand()*2000)] + set x [expr int(rand()*2000)] + set y [expr int(rand()*9)+1] + set z [expr int(rand()*2)] + execsql { INSERT INTO t1 VALUES(randomblob($w), randomblob($x)) } + execsql { PRAGMA integrity_check } + } {ok} + + do_test walslow-1.seed=$seed.$iTest.2 { + execsql "PRAGMA checkpoint('$y $y $z')" + execsql { PRAGMA integrity_check } + } {ok} + + do_test walslow-1.seed=$seed.$iTest.3 { + file delete -force testX.db testX.db-wal + file copy test.db testX.db + file copy test.db-wal testX.db-wal + + sqlite3 db2 testX.db + execsql { PRAGMA journal_mode = WAL } db2 + execsql { PRAGMA integrity_check } db2 + } {ok} + + do_test walslow-1.seed=$seed.$iTest.4 { + execsql { SELECT count(*) FROM t1 WHERE a!=b } db2 + } [execsql { SELECT count(*) FROM t1 WHERE a!=b }] + db2 close + } +} + + +finish_test diff --git a/test/walthread.test b/test/walthread.test new file mode 100644 index 0000000000..46474e2cb5 --- /dev/null +++ b/test/walthread.test @@ -0,0 +1,192 @@ +# 2007 September 7 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] + +source $testdir/tester.tcl +if {[run_thread_tests]==0} { finish_test ; return } + +do_test walthread-1.1 { + execsql { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(x PRIMARY KEY); + INSERT INTO t1 VALUES(randomblob(100)); + INSERT INTO t1 VALUES(randomblob(100)); + INSERT INTO t1 SELECT md5sum(x) FROM t1; + } +} {wal} +do_test walthread-1.2 { + execsql { + SELECT (SELECT count(*) FROM t1), ( + SELECT md5sum(x) FROM t1 WHERE oid != (SELECT max(oid) FROM t1) + ) == ( + SELECT x FROM t1 WHERE oid = (SELECT max(oid) FROM t1) + ) + } +} {3 1} +do_test walthread-1.3 { + execsql { PRAGMA integrity_check } +} {ok} +do_test walthread-1.4 { + execsql { PRAGMA lock_status } +} {main unlocked temp unknown} + +#-------------------------------------------------------------------------- +# Start N threads. Each thread performs both read and write transactions. +# Each read transaction consists of: +# +# 1) Reading the md5sum of all but the last table row, +# 2) Running integrity check. +# 3) Reading the value stored in the last table row, +# 4) Check that the values read in steps 1 and 3 are the same, and that +# the md5sum of all but the last table row has not changed. +# +# Each write transaction consists of: +# +# 1) Modifying the contents of t1 (inserting, updating, deleting rows). +# 2) Appending a new row to the table containing the md5sum() of all +# rows in the table. +# +# Each of the N threads runs N read transactions followed by a single write +# transaction in a loop as fast as possible. +# +# There is also a single checkpointer thread. It runs the following loop: +# +# 1) Execute "CHECKPOINT main 32 -1 1" +# 2) Sleep for 500 ms. +# + +set thread_program { + proc rest {ms} { + set ::rest 0 + after $ms {set ::rest 1} + vwait ::rest + } + + proc dosql {DB sql} { + set res "" + set stmt [sqlite3_prepare_v2 $DB $sql -1 dummy_tail] + set rc [sqlite3_step $stmt] + if {$rc eq "SQLITE_ROW"} { + set res [sqlite3_column_text $stmt 0] + } + set rc [sqlite3_finalize $stmt] + + if {$rc ne "SQLITE_OK"} { + error $rc + } + return $res + } + + proc read_transaction {DB} { + dosql $DB BEGIN + + set md5_1 [dosql $DB { + SELECT md5sum(x) FROM t1 WHERE rowid != (SELECT max(rowid) FROM t1) + }] + set check [dosql $DB { PRAGMA integrity_check }] + set md5_2 [dosql $DB { + SELECT x FROM t1 WHERE rowid = (SELECT max(rowid) FROM t1) + }] + set md5_3 [dosql $DB { + SELECT md5sum(x) FROM t1 WHERE rowid != (SELECT max(rowid) FROM t1) + }] + + dosql $DB COMMIT + + if {$check ne "ok" + || $md5_1 ne $md5_2 + || $md5_2 ne $md5_3 + } { + error "Failed read transaction $check $md5_1 $md5_2 $md5_3" + } + } + + proc write_transaction {DB} { + dosql $DB BEGIN + dosql $DB "INSERT INTO t1 VALUES(randomblob(100))" + dosql $DB "INSERT INTO t1 VALUES(randomblob(100))" + dosql $DB "INSERT INTO t1 SELECT md5sum(x) FROM t1" + dosql $DB COMMIT + } + + proc checkpointer {DB} { + while { !$::finished } { + dosql $DB "PRAGMA checkpoint(32)" + rest 1000 + } + } + + proc worker {DB N} { + set j 0 + while { !$::finished } { + for {set i 0} {$i < $N} {incr i} { read_transaction $DB } + write_transaction $DB + rest 1 + } + } + + set ::finished 0 + after [expr $seconds*1000] {set ::finished 1} + + set ::DB [sqlthread open test.db] + dosql $::DB { PRAGMA journal_mode = WAL } + + set rc [catch { + if {$role eq "worker"} { worker $DB $N } + if {$role eq "checkpointer"} { checkpointer $DB } + } msg] + + sqlite3_close $::DB + + if {$rc==0} { set msg OK } + set msg +} + +set NTHREAD 6 +set SECONDS 30 + +#set prg "set N $NTHREAD ; set seconds $SECONDS" +set prg "set N 1 ; set seconds $SECONDS" + +array unset finished +for {set i 0} {$i < $NTHREAD} {incr i} { + thread_spawn finished($i) {set role worker} $prg $thread_program +} +thread_spawn finished(C) {set role checkpointer} $prg $thread_program +#set finished(C) 1 + +puts "... test runs for approximately $SECONDS seconds ..." +for {set i 0} {$i < $::NTHREAD} {incr i} { + if {![info exists finished($i)]} { + vwait finished($i) + } + do_test walthread-2.$i { + set ::finished($i) + } OK +} +do_test walthread-2.C { + if {![info exists finished(C)]} { vwait finished(C) } + set ::finished(C) +} OK + +set logsize 0 + +set rows [execsql { SELECT count(*) FROM t1 }] +catch { set logsize [expr [file size test.db-wal] / 1024] } +set dbsize [expr [file size test.db] / 1024] + +puts "rows=$rows db=${dbsize}K log=${logsize}K" + +finish_test + + diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 38cee50ec2..12e4a5ce00 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -93,6 +93,7 @@ foreach hdr { hash.h hwtime.h keywordhash.h + log.h mutex.h opcodes.h os_common.h @@ -243,6 +244,7 @@ foreach file { pcache.c pcache1.c rowset.c + log.c pager.c btmutex.c From 6700d02cceca8de56e28505274fddb5fa28ba778 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 12 Apr 2010 19:05:58 +0000 Subject: [PATCH 02/89] Add "log.h", which should have been part of the previous commit. FossilOrigin-Name: 8de9f010044fc992b5b45a7bb0da1284b404ecdb --- manifest | 15 +++++-------- manifest.uuid | 2 +- src/log.h | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 10 deletions(-) create mode 100644 src/log.h diff --git a/manifest b/manifest index 4983ed4acd..e6ea0e5413 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Import\sexperimental\swrite-ahead-logging\scode. -D 2010-04-12T19:00:30 +C Add\s"log.h",\swhich\sshould\shave\sbeen\spart\sof\sthe\sprevious\scommit. +D 2010-04-12T19:05:58 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -132,6 +132,7 @@ F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 F src/log.c 6e8f296f6c566a297cd074c4165f1695fd1df5b7 +F src/log.h e691f7d935d6a8ad63b9de2e6014627056f01e1a F src/main.c c0e7192bad5b90544508b241eb2487ac661de890 F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -802,11 +803,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 51a613950824698687c0db83b7884db33d45f7f5 -R d1eaede94883ac0829acce36d14527da -T *bgcolor * #ffd0c0 -T *branch * wal -T *sym-wal * -T -sym-trunk * +P 409d61baeb0a19d1700c973f16c8acef7b8506cd +R 05e422fded750e42fd289d80ab078a37 U dan -Z 9f87e000c4066ee64fca3bc82365cd7e +Z 97a0dd7c04bfbe6a9f80405a5df46925 diff --git a/manifest.uuid b/manifest.uuid index bb821cd344..375208965b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -409d61baeb0a19d1700c973f16c8acef7b8506cd \ No newline at end of file +8de9f010044fc992b5b45a7bb0da1284b404ecdb \ No newline at end of file diff --git a/src/log.h b/src/log.h new file mode 100644 index 0000000000..63fa8ae433 --- /dev/null +++ b/src/log.h @@ -0,0 +1,61 @@ +/* +** 2010 February 1 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the interface to the write-ahead logging +** system. Refer to the comments below and the header comment attached to +** the implementation of each function in log.c for further details. +*/ + +#ifndef _LOG_H_ +#define _LOG_H_ + +#include "sqliteInt.h" + +/* Flags that may be set in the 'flags' argument to sqlite3LogWrite(): */ +#define LOG_MASK_COMMIT 0x08 +#define LOG_MASK_MASTERJOURNAL 0x10 +#define LOG_MASK_TRUNCATE 0x20 + + +#define LOG_TRUNCATE_BIT 0x80000000 + +/* Connection to a log file. There is one object of this type for each pager. */ +typedef struct Log Log; + +/* Open and close a connection to a log file. */ +int sqlite3LogOpen(sqlite3_vfs*, const char *zDb, Log **ppLog); +int sqlite3LogClose(Log *pLog, sqlite3_file *pFd, u8 *zBuf); + +/* Configure the log connection. */ +void sqlite3LogSetSyncflags(Log *, int sync_flags); + +/* Used by readers to open (lock) and close (unlock) a database snapshot. */ +int sqlite3LogOpenSnapshot(Log *pLog, int *); +void sqlite3LogCloseSnapshot(Log *pLog); + +/* Read a page from the log, if it is present. */ +int sqlite3LogRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut); +void sqlite3LogMaxpgno(Log *pLog, Pgno *pPgno); + +/* Obtain or release the WRITER lock. */ +int sqlite3LogWriteLock(Log *pLog, int op); + +/* Write a segment to the log. */ +int sqlite3LogFrames(Log *pLog, int, PgHdr *, Pgno, int, int); + +/* Copy pages from the log to the database file */ +int sqlite3LogCheckpoint( + Log *pLog, /* Log connection */ + sqlite3_file *pFd, /* File descriptor open on db file */ + u8 *zBuf /* Temporary buffer to use */ +); + +#endif /* _LOG_H_ */ From 63ca1dba5227c1a7efefc68a721c02cea36973cd Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 12 Apr 2010 19:31:59 +0000 Subject: [PATCH 03/89] Remove duplicate secure_delete pragma implemention. FossilOrigin-Name: c15fda145b621fc86402914aa5b015e53590555a --- manifest | 24 +++++++++++++++++------- manifest.uuid | 2 +- src/pragma.c | 25 ------------------------- 3 files changed, 18 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index e6ea0e5413..2f9e709d53 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,8 @@ -C Add\s"log.h",\swhich\sshould\shave\sbeen\spart\sof\sthe\sprevious\scommit. -D 2010-04-12T19:05:58 +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 + +C Remove\sduplicate\ssecure_delete\spragma\simplemention. +D 2010-04-12T19:32:00 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -160,7 +163,7 @@ F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 F src/pcache1.c 6dc1871ce8ead9187161c370a58cd06c84221f76 -F src/pragma.c 55f34050510940e7b60fa2b01141c16f87b649c5 +F src/pragma.c 5a68ee9ad23ec7486f36e6b874945348f501fcb7 F src/prepare.c fd1398cb1da54385ba5bd68d93928f10d10a1d9c F src/printf.c 5f5b65a83e63f2096a541a340722a509fa0240a7 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -803,7 +806,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 409d61baeb0a19d1700c973f16c8acef7b8506cd -R 05e422fded750e42fd289d80ab078a37 -U dan -Z 97a0dd7c04bfbe6a9f80405a5df46925 +P 8de9f010044fc992b5b45a7bb0da1284b404ecdb +R 8d5d43ae19836a365923e6a5505ada34 +U drh +Z ed2aa69bda36500ab38a65459c32fa94 +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.6 (GNU/Linux) + +iD8DBQFLw3UzoxKgR168RlERAvKMAJ411flPN1Hpy71eZLlqKus1l1HRZwCdFOYO +H++hcxwG2yJTYj8rinuj2EU= +=pmZ3 +-----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index 375208965b..b5ec0432b8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8de9f010044fc992b5b45a7bb0da1284b404ecdb \ No newline at end of file +c15fda145b621fc86402914aa5b015e53590555a \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 8e24387d33..2ecc852957 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -445,31 +445,6 @@ void sqlite3Pragma( returnSingleInt(pParse, "secure_delete", b); }else - /* - ** PRAGMA [database.]secure_delete - ** PRAGMA [database.]secure_delete=ON/OFF - ** - ** The first form reports the current setting for the - ** secure_delete flag. The second form changes the secure_delete - ** flag setting and reports thenew value. - */ - if( sqlite3StrICmp(zLeft,"secure_delete")==0 ){ - Btree *pBt = pDb->pBt; - int b = -1; - assert( pBt!=0 ); - if( zRight ){ - b = getBoolean(zRight); - } - if( pId2->n==0 && b>=0 ){ - int ii; - for(ii=0; iinDb; ii++){ - sqlite3BtreeSecureDelete(db->aDb[ii].pBt, b); - } - } - b = sqlite3BtreeSecureDelete(pBt, b); - returnSingleInt(pParse, "secure_delete", b); - }else - /* ** PRAGMA [database.]page_count ** From 9281bf2a9f4dcdf9892fd447031334c7d6751c03 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 13 Apr 2010 11:18:40 +0000 Subject: [PATCH 04/89] Fix a pager bug introduced by the wal code. FossilOrigin-Name: 4a0b29dc5e573a5d696ede17db5d7acf8d23aa98 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 24 ++++++++++++++++-------- test/quick.test | 3 +++ 4 files changed, 27 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index d477b9f2ae..99790f6114 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Sync\swal\sbranch\swith\s[012cf101bf]. -D 2010-04-13T06:20:45 +C Fix\sa\spager\sbug\sintroduced\sby\sthe\swal\scode. +D 2010-04-13T11:18:40 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -154,7 +154,7 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053 -F src/pager.c d61318cd04a42076c6524cb22c14fe036adbddcb +F src/pager.c 020c29713e77755f316cd95c233e75582a63766e F src/pager.h 80c57ba672fcd24215a68abf1c98d474a5080b9b F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf @@ -537,7 +537,7 @@ F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea F test/progress.test 5b075c3c790c7b2a61419bc199db87aaf48b8301 F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc -F test/quick.test d6591e74f3ac19da7fd076845f06dca48fd43cff +F test/quick.test b72d77e97ab2e6bd890bdc0ed4b7f57ec2f7ef93 F test/quote.test 215897dbe8de1a6f701265836d6601cc6ed103e6 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test 1084050991e9ba22c1c10edd8d84673b501cc25a @@ -803,7 +803,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P c15fda145b621fc86402914aa5b015e53590555a 012cf101bf8be9e39c138786ea5a5039b8131e55 -R a82603ab9844babf112f19212a1fe8c5 +P 9d690f24f634696f76fa38372461cfa69263e91e +R 8f595e612b9933c5a5c59646f7c7d5da U dan -Z 1e9c7b3cdd0156e9796faa9c1b265349 +Z 45325555f32460c0270b99616d9c91ef diff --git a/manifest.uuid b/manifest.uuid index 1d692935b5..165ffdb1f4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9d690f24f634696f76fa38372461cfa69263e91e \ No newline at end of file +4a0b29dc5e573a5d696ede17db5d7acf8d23aa98 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 383c0fe6bd..a1b49ef113 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3782,20 +3782,17 @@ int sqlite3PagerSharedLock(Pager *pPager){ pager_reset(pPager); } - if( pPager->noReadlock ){ - assert( pPager->readOnly ); - pPager->state = PAGER_SHARED; - }else{ + if( pagerUseLog(pPager) ){ + int changed = 0; + + /* TODO: Change the following block to grab a WAL read-lock. Or, + ** combine obtaining the read-lock with LogOpenSnapshot()? */ rc = pager_wait_on_lock(pPager, SHARED_LOCK); if( rc!=SQLITE_OK ){ assert( pPager->state==PAGER_UNLOCK ); return pager_error(pPager, rc); } - } - assert( pPager->state>=SHARED_LOCK ); - if( pagerUseLog(pPager) ){ - int changed = 0; rc = sqlite3LogOpenSnapshot(pPager->pLog, &changed); if( rc==SQLITE_OK ){ if( changed ){ @@ -3810,6 +3807,17 @@ int sqlite3PagerSharedLock(Pager *pPager){ int isHotJournal = 0; assert( !MEMDB ); assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); + if( pPager->noReadlock ){ + assert( pPager->readOnly ); + pPager->state = PAGER_SHARED; + }else{ + rc = pager_wait_on_lock(pPager, SHARED_LOCK); + if( rc!=SQLITE_OK ){ + assert( pPager->state==PAGER_UNLOCK ); + return pager_error(pPager, rc); + } + } + assert( pPager->state>=SHARED_LOCK ); /* If a journal file exists, and there is no RESERVED lock on the ** database file, then it either needs to be played back or deleted. diff --git a/test/quick.test b/test/quick.test index 431b829ef7..4ab211ade6 100644 --- a/test/quick.test +++ b/test/quick.test @@ -101,6 +101,9 @@ set EXCLUDE { vtab_err.test veryquick.test mallocAll.test + + walslow.test + walcrash.test } if {[sqlite3 -has-codec]} { From d3f8f94652e1849bf2ae25bd7009b1da0885ed71 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 13 Apr 2010 11:35:01 +0000 Subject: [PATCH 05/89] Fix test script issues preventing walthread.test from running. FossilOrigin-Name: d90b1dde0702ceb0eae2a3da36733a7e5b1a6b3a --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- test/tester.tcl | 1 + test/thread_common.tcl | 2 +- test/wal.test | 14 ++++++++++++++ test/walcrash.test | 5 ++++- test/walslow.test | 4 ++++ test/walthread.test | 6 +++++- 8 files changed, 41 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 99790f6114..39a867ddd4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\spager\sbug\sintroduced\sby\sthe\swal\scode. -D 2010-04-13T11:18:40 +C Fix\stest\sscript\sissues\spreventing\swalthread.test\sfrom\srunning. +D 2010-04-13T11:35:01 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -602,7 +602,7 @@ F test/tclsqlite.test bf4227eb236a4c097aa7974a2bf7d3225acf34be F test/tempdb.test 1bf52da28a9c24e29717362a87722dff08feb72b F test/temptable.test f42121a0d29a62f00f93274464164177ab1cc24a F test/temptrigger.test b0273db072ce5f37cf19140ceb1f0d524bbe9f05 -F test/tester.tcl e1f581c7a2648a0aaa51135c4d2e7be68f4b9292 +F test/tester.tcl 49d76f12940160d623da104f995530fc6ee8f46f F test/thread001.test a3e6a7254d1cb057836cb3145b60c10bf5b7e60f F test/thread002.test afd20095e6e845b405df4f2c920cb93301ca69db F test/thread003.test b824d4f52b870ae39fc5bae4d8070eca73085dca @@ -610,7 +610,7 @@ F test/thread004.test f51dfc3936184aaf73ee85f315224baad272a87f F test/thread005.test bf5c374ca65dd89fd56c8fe511ccfb46875bda5e F test/thread1.test 862dd006d189e8b0946935db17399dcac2f8ef91 F test/thread2.test 6e0997f7beabb6a7e471bd18740ed04805c785f4 -F test/thread_common.tcl b65e6b1d1d90dc885e10ad080896c6c56eef0819 +F test/thread_common.tcl 0b07423d29ddb73d4bacbac69268c8d37b6cc5d2 F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 F test/tkt-2ea2425d34.test 1cf13e6f75d149b3209a0cb32927a82d3d79fb28 @@ -756,10 +756,10 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test 653a870ac4b312d421e006ec45a293ecf2135b49 -F test/walcrash.test 76a6848c5088596cf37f4042e7e4c108a8c323d4 -F test/walslow.test 17bfe8052a5d1c64e171eeab45c445fa90f4d4a6 -F test/walthread.test 82cc77f8853217a036fc82e4313ecebd9aa368d7 +F test/wal.test 2c279aad6d34a9f8b8226d65c6d464375fc10343 +F test/walcrash.test df3b4f84d7bb2252914b34ab0995ce4d249779b5 +F test/walslow.test ba50a02439f7ecfc9c92f6ba7b2013a2a707ac80 +F test/walthread.test b03790383ff7ba0119e04f230fcef3b635aab32f F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 F test/where2.test 45eacc126aabb37959a387aa83e59ce1f1f03820 F test/where3.test 97d3936e6a443b968f1a61cdcc0f673252000e94 @@ -803,7 +803,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 9d690f24f634696f76fa38372461cfa69263e91e -R 8f595e612b9933c5a5c59646f7c7d5da +P 4a0b29dc5e573a5d696ede17db5d7acf8d23aa98 +R 2e081cfd5144f338793cfa49bfc89545 U dan -Z 45325555f32460c0270b99616d9c91ef +Z 734473cb8e02a3d72068235b175d9c7c diff --git a/manifest.uuid b/manifest.uuid index 165ffdb1f4..18cbe3e73a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4a0b29dc5e573a5d696ede17db5d7acf8d23aa98 \ No newline at end of file +d90b1dde0702ceb0eae2a3da36733a7e5b1a6b3a \ No newline at end of file diff --git a/test/tester.tcl b/test/tester.tcl index 8fe877ec01..44798a7098 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -143,6 +143,7 @@ proc reset_db {} { catch {db close} file delete -force test.db file delete -force test.db-journal + file delete -force test.db-wal sqlite3 db ./test.db set ::DB [sqlite3_connection_pointer db] if {[info exists ::SETUP_SQL]} { diff --git a/test/thread_common.tcl b/test/thread_common.tcl index bbd9389ea0..673afdd802 100644 --- a/test/thread_common.tcl +++ b/test/thread_common.tcl @@ -80,7 +80,7 @@ set thread_procs { } proc thread_spawn {varname args} { - sqlthread spawn $varname [join $args ;] + sqlthread spawn $varname [join $args {;}] } # Return true if this build can run the multi-threaded tests. diff --git a/test/wal.test b/test/wal.test index 57fccce288..ed9c821ef2 100644 --- a/test/wal.test +++ b/test/wal.test @@ -1,3 +1,17 @@ +# 2010 April 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL" mode. +# set testdir [file dirname $argv0] source $testdir/tester.tcl diff --git a/test/walcrash.test b/test/walcrash.test index b3e6cd2b97..8d6f720d1b 100644 --- a/test/walcrash.test +++ b/test/walcrash.test @@ -8,7 +8,10 @@ # May you share freely, never taking more than you give. # #*********************************************************************** -# This file implements regression tests for SQLite library. +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library when +# recovering a database following a simulated system failure in +# "PRAGMA journal_mode=WAL" mode. # # diff --git a/test/walslow.test b/test/walslow.test index cec4e26d98..d4133f899c 100644 --- a/test/walslow.test +++ b/test/walslow.test @@ -8,6 +8,10 @@ # May you share freely, never taking more than you give. # #*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL" mode. The tests in this file use +# brute force methods, so may take a while to run. # set testdir [file dirname $argv0] diff --git a/test/walthread.test b/test/walthread.test index 46474e2cb5..934d63325c 100644 --- a/test/walthread.test +++ b/test/walthread.test @@ -1,4 +1,4 @@ -# 2007 September 7 +# 2010 April 13 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: @@ -8,6 +8,9 @@ # May you share freely, never taking more than you give. # #*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL" mode. # set testdir [file dirname $argv0] @@ -141,6 +144,7 @@ set thread_program { set ::DB [sqlthread open test.db] dosql $::DB { PRAGMA journal_mode = WAL } + set rc [catch { if {$role eq "worker"} { worker $DB $N } if {$role eq "checkpointer"} { checkpointer $DB } From 80a1526a9a8bf2f6758340a63c6b2fe32d06c4dc Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 13 Apr 2010 11:45:31 +0000 Subject: [PATCH 06/89] Fix other problems in the WAL test scripts. FossilOrigin-Name: dcb31181db246dd3f81ce63b144dd0cf95f06ee3 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/quick.test | 1 + test/wal.test | 5 ++++- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 39a867ddd4..2ced2fc40a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stest\sscript\sissues\spreventing\swalthread.test\sfrom\srunning. -D 2010-04-13T11:35:01 +C Fix\sother\sproblems\sin\sthe\sWAL\stest\sscripts. +D 2010-04-13T11:45:31 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -537,7 +537,7 @@ F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea F test/progress.test 5b075c3c790c7b2a61419bc199db87aaf48b8301 F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc -F test/quick.test b72d77e97ab2e6bd890bdc0ed4b7f57ec2f7ef93 +F test/quick.test 6f202befe1cfae0b63df96b3120a8022ab11f574 F test/quote.test 215897dbe8de1a6f701265836d6601cc6ed103e6 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test 1084050991e9ba22c1c10edd8d84673b501cc25a @@ -756,7 +756,7 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test 2c279aad6d34a9f8b8226d65c6d464375fc10343 +F test/wal.test 5dc92880060c255c74376b9f5c0beb85a7399b93 F test/walcrash.test df3b4f84d7bb2252914b34ab0995ce4d249779b5 F test/walslow.test ba50a02439f7ecfc9c92f6ba7b2013a2a707ac80 F test/walthread.test b03790383ff7ba0119e04f230fcef3b635aab32f @@ -803,7 +803,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 4a0b29dc5e573a5d696ede17db5d7acf8d23aa98 -R 2e081cfd5144f338793cfa49bfc89545 +P d90b1dde0702ceb0eae2a3da36733a7e5b1a6b3a +R a841ebbbfc361d1f0f1f7f50f82ea9fc U dan -Z 734473cb8e02a3d72068235b175d9c7c +Z 96ff40b46c8873b7b202d8b466c8d9da diff --git a/manifest.uuid b/manifest.uuid index 18cbe3e73a..89554b3f89 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d90b1dde0702ceb0eae2a3da36733a7e5b1a6b3a \ No newline at end of file +dcb31181db246dd3f81ce63b144dd0cf95f06ee3 \ No newline at end of file diff --git a/test/quick.test b/test/quick.test index 4ab211ade6..044951f707 100644 --- a/test/quick.test +++ b/test/quick.test @@ -104,6 +104,7 @@ set EXCLUDE { walslow.test walcrash.test + walthread.test } if {[sqlite3 -has-codec]} { diff --git a/test/wal.test b/test/wal.test index ed9c821ef2..81ed9ebb78 100644 --- a/test/wal.test +++ b/test/wal.test @@ -510,7 +510,7 @@ do_test wal-12.2 { PRAGMA checkpoint; } file size test.db -} [expr 67*1024] +} [expr 68*1024] do_test wal-12.3 { execsql { DELETE FROM t1 WHERE rowid<54; @@ -561,5 +561,8 @@ breakpoint execsql {PRAGMA integrity_check } db2 } {ok} +foreach handle {db db2 db3} { catch { $handle close } } +unset handle + finish_test From f05c86dd73023b32d32545b4e83d70352400680a Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 13 Apr 2010 11:56:03 +0000 Subject: [PATCH 07/89] Remove old code to interpret an argument (no longer used) to "PRAGMA checkpoint". FossilOrigin-Name: 27dc5977c19e717afd65d3805557e38dec7bedcb --- manifest | 26 +++++++++++++------------- manifest.uuid | 2 +- src/pager.c | 2 +- src/pager.h | 1 + src/pragma.c | 16 +--------------- src/vdbe.c | 8 ++------ test/wal.test | 2 +- test/walcrash.test | 4 ++-- test/walslow.test | 4 +--- test/walthread.test | 2 +- 10 files changed, 24 insertions(+), 43 deletions(-) diff --git a/manifest b/manifest index 2ced2fc40a..170cf0e546 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sother\sproblems\sin\sthe\sWAL\stest\sscripts. -D 2010-04-13T11:45:31 +C Remove\sold\scode\sto\sinterpret\san\sargument\s(no\slonger\sused)\sto\s"PRAGMA\scheckpoint". +D 2010-04-13T11:56:04 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -154,13 +154,13 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053 -F src/pager.c 020c29713e77755f316cd95c233e75582a63766e -F src/pager.h 80c57ba672fcd24215a68abf1c98d474a5080b9b +F src/pager.c 88cbb683c96080bf2ecff2b355da10d567a0353f +F src/pager.h ce5d076f3860a5f2d7460c582cd68383343b33cf F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 F src/pcache1.c 6dc1871ce8ead9187161c370a58cd06c84221f76 -F src/pragma.c 5a68ee9ad23ec7486f36e6b874945348f501fcb7 +F src/pragma.c 9f1d53d40b47743f3d6151df15915970367c9de5 F src/prepare.c fd1398cb1da54385ba5bd68d93928f10d10a1d9c F src/printf.c 5f5b65a83e63f2096a541a340722a509fa0240a7 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -214,7 +214,7 @@ F src/update.c c0dc6b75ad28b76b619042d934f337b02acee208 F src/utf.c 1baeeac91707a4df97ccc6141ec0f808278af685 F src/util.c 32aebf04c10e51ad3977a928b7416bed671b620b F src/vacuum.c b1d542c8919d4d11119f78069e1906a1ad07e0ee -F src/vdbe.c 23e5462d1cd0a6d525948cfec3382890abc31d3f +F src/vdbe.c 2e2aaa765de667dd15e0462cf853efd1b2f97998 F src/vdbe.h 471f6a3dcec4817ca33596fe7f6654d56c0e75f3 F src/vdbeInt.h 19ebc8c2a2e938340051ee65af3f377fb99102d1 F src/vdbeapi.c 74c25680046a116b24b95393914d3669c23305dc @@ -756,10 +756,10 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test 5dc92880060c255c74376b9f5c0beb85a7399b93 -F test/walcrash.test df3b4f84d7bb2252914b34ab0995ce4d249779b5 -F test/walslow.test ba50a02439f7ecfc9c92f6ba7b2013a2a707ac80 -F test/walthread.test b03790383ff7ba0119e04f230fcef3b635aab32f +F test/wal.test d5916b3a40d74dd5bf6ff21b42e6fa7f255efd13 +F test/walcrash.test 45cfbab30bb7cbe0b2e9d5cabe90dbcad10cb89b +F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 +F test/walthread.test 1592116f74261d77ea697d9503b0cd1ab1c15e74 F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 F test/where2.test 45eacc126aabb37959a387aa83e59ce1f1f03820 F test/where3.test 97d3936e6a443b968f1a61cdcc0f673252000e94 @@ -803,7 +803,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P d90b1dde0702ceb0eae2a3da36733a7e5b1a6b3a -R a841ebbbfc361d1f0f1f7f50f82ea9fc +P dcb31181db246dd3f81ce63b144dd0cf95f06ee3 +R 6f02a41d259c74c25289be520ca51251 U dan -Z 96ff40b46c8873b7b202d8b466c8d9da +Z 68876c4a3171b03c883dda51dc5e0740 diff --git a/manifest.uuid b/manifest.uuid index 89554b3f89..31c9695615 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dcb31181db246dd3f81ce63b144dd0cf95f06ee3 \ No newline at end of file +27dc5977c19e717afd65d3805557e38dec7bedcb \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index a1b49ef113..ff696c64e7 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5654,7 +5654,7 @@ sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){ /* ** This function is called when the user invokes "PRAGMA checkpoint". */ -int sqlite3PagerCheckpoint(Pager *pPager, int nMin, int nMax, int doSync){ +int sqlite3PagerCheckpoint(Pager *pPager){ int rc = SQLITE_OK; if( pPager->pLog ){ rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); diff --git a/src/pager.h b/src/pager.h index f329e43d81..1e14d2ea60 100644 --- a/src/pager.h +++ b/src/pager.h @@ -133,6 +133,7 @@ int sqlite3PagerRollback(Pager*); int sqlite3PagerOpenSavepoint(Pager *pPager, int n); int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); int sqlite3PagerSharedLock(Pager *pPager); +int sqlite3PagerCheckpoint(Pager *pPager); /* Functions used to query pager state and configuration. */ u8 sqlite3PagerIsreadonly(Pager*); diff --git a/src/pragma.c b/src/pragma.c index 2ecc852957..95310e336c 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1385,23 +1385,9 @@ void sqlite3Pragma( #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ if( sqlite3StrICmp(zLeft, "checkpoint")==0 ){ - int nMin = 0; - int nMax = 0; - int nosync = 0; - - if( zRight ){ - char *z = zRight; - sqlite3GetInt32(z, &nMin); - while( sqlite3Isdigit(*z) ) z++; - while( *z && !sqlite3Isdigit(*z) ) z++; - sqlite3GetInt32(z, &nMax); - while( sqlite3Isdigit(*z) ) z++; - while( *z && !sqlite3Isdigit(*z) ) z++; - sqlite3GetInt32(z, &nosync); - } sqlite3VdbeUsesBtree(v, iDb); sqlite3VdbeAddOp2(v, OP_Transaction, iDb, 1); - sqlite3VdbeAddOp3(v, OP_Checkpoint, iDb, nMin, nMax); + sqlite3VdbeAddOp3(v, OP_Checkpoint, iDb, 0, 0); }else #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) diff --git a/src/vdbe.c b/src/vdbe.c index 8e27942331..42562cee09 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5186,19 +5186,15 @@ case OP_AggFinal: { break; } -/* Opcode: Checkpoint P1 P2 P3 * P5 +/* Opcode: Checkpoint P1 * * * * */ case OP_Checkpoint: { Btree *pBt; /* Btree to checkpoint */ - int nMin = pOp->p2; /* Minimum number of pages to copy */ - int nMax = pOp->p3; /* Maximum number of pages to copy */ - int doNotSync = pOp->p5; /* True to sync database */ assert( pOp->p1>=0 && pOp->p1nDb ); assert( (p->btreeMask & (1<p1))!=0 ); pBt = db->aDb[pOp->p1].pBt; - - rc = sqlite3PagerCheckpoint(sqlite3BtreePager(pBt), nMin, nMax, !doNotSync); + rc = sqlite3PagerCheckpoint(sqlite3BtreePager(pBt)); break; }; diff --git a/test/wal.test b/test/wal.test index 81ed9ebb78..b12782b663 100644 --- a/test/wal.test +++ b/test/wal.test @@ -514,7 +514,7 @@ do_test wal-12.2 { do_test wal-12.3 { execsql { DELETE FROM t1 WHERE rowid<54; - PRAGMA checkpoint('1 100000'); + PRAGMA checkpoint; } file size test.db } [expr 14*1024] diff --git a/test/walcrash.test b/test/walcrash.test index 8d6f720d1b..dd4c572941 100644 --- a/test/walcrash.test +++ b/test/walcrash.test @@ -191,7 +191,7 @@ for {set i 1} {$i < $REPEATS} {incr i} { INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 28 */ INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 32 */ - PRAGMA checkpoint('70 70'); + PRAGMA checkpoint; INSERT INTO t1 VALUES(randomblob(900)); INSERT INTO t1 VALUES(randomblob(900)); INSERT INTO t1 VALUES(randomblob(900)); @@ -231,7 +231,7 @@ for {set i 1} {$i < $REPEATS} {incr i} { INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 28 */ INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 32 */ - PRAGMA checkpoint('70 70'); + PRAGMA checkpoint; INSERT INTO t1 VALUES(randomblob(900)); INSERT INTO t1 VALUES(randomblob(900)); INSERT INTO t1 VALUES(randomblob(900)); diff --git a/test/walslow.test b/test/walslow.test index d4133f899c..73f93a48df 100644 --- a/test/walslow.test +++ b/test/walslow.test @@ -41,14 +41,12 @@ for {set seed 1} {$seed<10} {incr seed} { do_test walslow-1.seed=$seed.$iTest.1 { set w [expr int(rand()*2000)] set x [expr int(rand()*2000)] - set y [expr int(rand()*9)+1] - set z [expr int(rand()*2)] execsql { INSERT INTO t1 VALUES(randomblob($w), randomblob($x)) } execsql { PRAGMA integrity_check } } {ok} do_test walslow-1.seed=$seed.$iTest.2 { - execsql "PRAGMA checkpoint('$y $y $z')" + execsql "PRAGMA checkpoint;" execsql { PRAGMA integrity_check } } {ok} diff --git a/test/walthread.test b/test/walthread.test index 934d63325c..7727ab3759 100644 --- a/test/walthread.test +++ b/test/walthread.test @@ -124,7 +124,7 @@ set thread_program { proc checkpointer {DB} { while { !$::finished } { - dosql $DB "PRAGMA checkpoint(32)" + dosql $DB "PRAGMA checkpoint" rest 1000 } } From 622194c0d205e4391d92afa68bff8c0687b80f33 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 13 Apr 2010 15:30:53 +0000 Subject: [PATCH 08/89] Fix an uninitialized variable in readDbPage of pager.c. FossilOrigin-Name: f4e1150fed2c520c7c52612cb1019429d78dc32a --- manifest | 24 +++++++++++++++++------- manifest.uuid | 2 +- src/pager.c | 2 +- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 170cf0e546..8eff149f3c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,8 @@ -C Remove\sold\scode\sto\sinterpret\san\sargument\s(no\slonger\sused)\sto\s"PRAGMA\scheckpoint". -D 2010-04-13T11:56:04 +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 + +C Fix\san\suninitialized\svariable\sin\sreadDbPage\sof\spager.c. +D 2010-04-13T15:30:53 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -154,7 +157,7 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053 -F src/pager.c 88cbb683c96080bf2ecff2b355da10d567a0353f +F src/pager.c c4937e7175f0aa66b9122d05cc163c039f854855 F src/pager.h ce5d076f3860a5f2d7460c582cd68383343b33cf F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf @@ -803,7 +806,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P dcb31181db246dd3f81ce63b144dd0cf95f06ee3 -R 6f02a41d259c74c25289be520ca51251 -U dan -Z 68876c4a3171b03c883dda51dc5e0740 +P 27dc5977c19e717afd65d3805557e38dec7bedcb +R ca7988a67487ccd526ae5248710a4503 +U drh +Z c2421237a2b7de8b8aa2d4d5608541bc +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.6 (GNU/Linux) + +iD8DBQFLxI4woxKgR168RlERAp8sAJ9x+oJYnqOMgUNsC198WiuQpy6qyQCfcaVa +jjZp98/Y+EUb/i8plNNLbiM= +=2Ctn +-----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index 31c9695615..95954aef22 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27dc5977c19e717afd65d3805557e38dec7bedcb \ No newline at end of file +f4e1150fed2c520c7c52612cb1019429d78dc32a \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index ff696c64e7..4e8213cf9b 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2188,7 +2188,7 @@ end_playback: static int readDbPage(PgHdr *pPg){ Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */ Pgno pgno = pPg->pgno; /* Page number to read */ - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ i64 iOffset; /* Byte offset of file to read from */ int isInLog = 0; /* True if page is in log file */ From 64d039e512c8a2cf63dab37616b4fb362790d3b3 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 13 Apr 2010 19:27:31 +0000 Subject: [PATCH 09/89] Add experimental locking scheme. FossilOrigin-Name: 3f958e87c33d667d299b03ffdef58db5dc6363f4 --- manifest | 28 ++---- manifest.uuid | 2 +- src/log.c | 232 ++++++++++++++++++++++++++++++++++++++++++++++---- src/log.h | 4 +- src/pager.c | 49 +++++------ 5 files changed, 248 insertions(+), 67 deletions(-) diff --git a/manifest b/manifest index 8eff149f3c..faab67f851 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,5 @@ ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA1 - -C Fix\san\suninitialized\svariable\sin\sreadDbPage\sof\spager.c. -D 2010-04-13T15:30:53 +C Add\sexperimental\slocking\sscheme. +D 2010-04-13T19:27:31 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -134,8 +131,8 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c 6e8f296f6c566a297cd074c4165f1695fd1df5b7 -F src/log.h e691f7d935d6a8ad63b9de2e6014627056f01e1a +F src/log.c d89988bb26a3cd414858c97642a612b4ce6e540f +F src/log.h a2654af46ce7b5732f4d5a731abfdd180f0a06d9 F src/main.c c0e7192bad5b90544508b241eb2487ac661de890 F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -157,7 +154,7 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053 -F src/pager.c c4937e7175f0aa66b9122d05cc163c039f854855 +F src/pager.c 9e9ee38c923fd225d73127751b7959bd826d0686 F src/pager.h ce5d076f3860a5f2d7460c582cd68383343b33cf F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf @@ -806,14 +803,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 27dc5977c19e717afd65d3805557e38dec7bedcb -R ca7988a67487ccd526ae5248710a4503 -U drh -Z c2421237a2b7de8b8aa2d4d5608541bc ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.6 (GNU/Linux) - -iD8DBQFLxI4woxKgR168RlERAp8sAJ9x+oJYnqOMgUNsC198WiuQpy6qyQCfcaVa -jjZp98/Y+EUb/i8plNNLbiM= -=2Ctn ------END PGP SIGNATURE----- +P f4e1150fed2c520c7c52612cb1019429d78dc32a +R da172d13acb33c194abd32bf4ed20cc9 +U dan +Z cd10eeca0187a805032040fdd27191f0 diff --git a/manifest.uuid b/manifest.uuid index 95954aef22..43b6c91f02 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f4e1150fed2c520c7c52612cb1019429d78dc32a \ No newline at end of file +3f958e87c33d667d299b03ffdef58db5dc6363f4 \ No newline at end of file diff --git a/src/log.c b/src/log.c index 7d37dfa1de..2b2764f361 100644 --- a/src/log.c +++ b/src/log.c @@ -13,6 +13,7 @@ typedef struct LogSummaryHdr LogSummaryHdr; typedef struct LogSummary LogSummary; typedef struct LogCheckpoint LogCheckpoint; +typedef struct LogLock LogLock; /* @@ -55,29 +56,51 @@ struct LogSummary { int nRef; /* Number of pointers to this structure */ int fd; /* File descriptor open on log-summary */ char *zPath; /* Path to associated WAL file */ + LogLock *pLock; /* Linked list of locks on this object */ LogSummary *pNext; /* Next in global list */ int nData; /* Size of aData allocation/mapping */ u32 *aData; /* File body */ }; + /* -** List of all LogSummary objects created by this process. Protected by -** static mutex LOG_SUMMARY_MUTEX. TODO: Should have a dedicated mutex -** here instead of borrowing the LRU mutex. +** The four lockable regions associated with each log-summary. A connection +** may take either a SHARED or EXCLUSIVE lock on each. */ -#define LOG_SUMMARY_MUTEX SQLITE_MUTEX_STATIC_LRU -static LogSummary *pLogSummary = 0; +#define LOG_REGION_A 0x01 +#define LOG_REGION_B 0x02 +#define LOG_REGION_C 0x04 +#define LOG_REGION_D 0x08 + +/* +** A single instance of this structure is allocated as part of each +** connection to a database log. All structures associated with the +** same log file are linked together into a list using LogLock.pNext +** starting at LogSummary.pLock. +** +** The mLock field of the structure describes the locks (if any) +** currently held by the connection. If a SHARED lock is held on +** any of the four locking regions, then the associated LOG_REGION_X +** bit (see above) is set. If an EXCLUSIVE lock is held on the region, +** then the (LOG_REGION_X << 8) bit is set. +*/ +struct LogLock { + LogLock *pNext; /* Next lock on the same log */ + u32 mLock; /* Mask of locks */ +}; struct Log { LogSummary *pSummary; /* Log file summary data */ sqlite3_vfs *pVfs; /* The VFS used to create pFd */ sqlite3_file *pFd; /* File handle for log file */ int sync_flags; /* Flags to use with OsSync() */ - int isLocked; /* True if a snapshot is held open */ + int isLocked; /* Non-zero if a snapshot is held open */ int isWriteLocked; /* True if this is the writer connection */ LogSummaryHdr hdr; /* Log summary header for current snapshot */ + LogLock lock; /* Lock held by this connection (if any) */ }; + /* ** This structure is used to implement an iterator that iterates through ** all frames in the log in database page order. Where two or more frames @@ -102,6 +125,15 @@ struct LogCheckpoint { } aSegment[1]; }; + +/* +** List of all LogSummary objects created by this process. Protected by +** static mutex LOG_SUMMARY_MUTEX. TODO: Should have a dedicated mutex +** here instead of borrowing the LRU mutex. +*/ +#define LOG_SUMMARY_MUTEX SQLITE_MUTEX_STATIC_LRU +static LogSummary *pLogSummary = 0; + /* ** Generate an 8 byte checksum based on the data in array aByte[] and the ** initial values of aCksum[0] and aCksum[1]. The checksum is written into @@ -664,6 +696,9 @@ int sqlite3LogOpen( rc = sqlite3OsLock(pRet->pFd, SQLITE_LOCK_SHARED); } + pRet->lock.pNext = pSummary->pLock; + pSummary->pLock = &pRet->lock; + out: sqlite3_mutex_leave(mutex); sqlite3_free(zWal); @@ -838,9 +873,15 @@ int sqlite3LogClose( ){ int rc = SQLITE_OK; if( pLog ){ + LogLock **ppL; LogSummary *pSummary = pLog->pSummary; sqlite3_mutex *mutex = 0; + sqlite3_mutex_enter(pSummary->mutex); + for(ppL=&pSummary->pLock; *ppL!=&pLog->lock; ppL=&(*ppL)->pNext); + *ppL = pLog->lock.pNext; + sqlite3_mutex_leave(pSummary->mutex); + if( sqlite3GlobalConfig.bCoreMutex ){ mutex = sqlite3_mutex_alloc(LOG_SUMMARY_MUTEX); } @@ -939,7 +980,108 @@ static void logLeaveMutex(Log *pLog){ } /* -** The caller must hold a SHARED lock on the database file. +** Values for the second parameter to logLockRegion(). +*/ +#define LOG_UNLOCK 0 +#define LOG_RDLOCK 1 +#define LOG_WRLOCK 2 + +static int logLockRegion(Log *pLog, u32 mRegion, int op){ + LogSummary *pSummary = pLog->pSummary; + LogLock *p; /* Used to iterate through in-process locks */ + u32 mNew; /* New locks on file */ + u32 mOld; /* Old locks on file */ + u32 mNewLock; /* New locks held by pLog */ + + assert( + /* Writer lock operations */ + (op==LOG_WRLOCK && mRegion==(LOG_REGION_C|LOG_REGION_D)) + || (op==LOG_UNLOCK && mRegion==(LOG_REGION_C|LOG_REGION_D)) + + /* Reader lock operations */ + || (op==LOG_RDLOCK && mRegion==(LOG_REGION_A|LOG_REGION_B)) + || (op==LOG_RDLOCK && mRegion==(LOG_REGION_D)) + || (op==LOG_UNLOCK && mRegion==(LOG_REGION_A)) + || (op==LOG_UNLOCK && mRegion==(LOG_REGION_B)) + || (op==LOG_UNLOCK && mRegion==(LOG_REGION_D)) + + /* Checkpointer lock operations */ + || (op==LOG_WRLOCK && mRegion==(LOG_REGION_B|LOG_REGION_C)) + || (op==LOG_WRLOCK && mRegion==(LOG_REGION_A)) + || (op==LOG_UNLOCK && mRegion==(LOG_REGION_A|LOG_REGION_B|LOG_REGION_C)) + || (op==LOG_UNLOCK && mRegion==(LOG_REGION_B|LOG_REGION_C)) + ); + + sqlite3_mutex_enter(pSummary->mutex); + + /* If obtaining (not releasing) a lock, check if there exist any + ** conflicting locks in process. Return SQLITE_BUSY in this case. + */ + if( op ){ + u32 mConflict = (mRegion<<8) | ((op==LOG_WRLOCK) ? mRegion : 0); + for(p=pSummary->pLock; p; p=p->pNext){ + if( p!=&pLog->lock && (p->mLock & mConflict) ){ + sqlite3_mutex_leave(pSummary->mutex); + return SQLITE_BUSY; + } + } + } + + /* Determine the new lock mask for this log connection */ + switch( op ){ + case LOG_UNLOCK: + mNewLock = (pLog->lock.mLock & ~(mRegion|(mRegion<<8))); + break; + case LOG_RDLOCK: + mNewLock = ((pLog->lock.mLock & ~(mRegion<<8)) | mRegion); + break; + default: + assert( op==LOG_WRLOCK ); + mNewLock = (pLog->lock.mLock | (mRegion<<8) | mRegion); + break; + } + + /* Determine the current and desired sets of locks at the file level. */ + mNew = 0; + for(p=pSummary->pLock; p; p=p->pNext){ + assert( (p->mLock & (p->mLock<<8))==(p->mLock & 0x00000F00) ); + if( p!=&pLog->lock ) mNew |= p->mLock; + } + mOld = mNew | pLog->lock.mLock; + mNew = mNew | mNewLock; + + if( mNew!=mOld ){ + int rc; + u32 mChange = (mNew^mOld) | ((mNew^mOld)>>8); + struct flock f; + memset(&f, 0, sizeof(f)); + f.l_type = (op==LOG_WRLOCK?F_WRLCK:(op==LOG_RDLOCK?F_RDLCK:F_UNLCK)); + f.l_whence = SEEK_SET; + + if( mChange & LOG_REGION_A ) f.l_start = 12; + else if( mChange & LOG_REGION_B ) f.l_start = 13; + else if( mChange & LOG_REGION_C ) f.l_start = 14; + else if( mChange & LOG_REGION_D ) f.l_start = 15; + + if( mChange & LOG_REGION_D ) f.l_len = 16 - f.l_start; + else if( mChange & LOG_REGION_C ) f.l_len = 15 - f.l_start; + else if( mChange & LOG_REGION_B ) f.l_len = 14 - f.l_start; + else if( mChange & LOG_REGION_A ) f.l_len = 13 - f.l_start; + + rc = fcntl(pSummary->fd, F_SETLK, &f); + if( rc!=0 ){ + sqlite3_mutex_leave(pSummary->mutex); + return SQLITE_BUSY; + } + } + + pLog->lock.mLock = mNewLock; + sqlite3_mutex_leave(pSummary->mutex); + return SQLITE_OK; +} + +/* +** Lock a snapshot. ** ** If this call obtains a new read-lock and the database contents have been ** modified since the most recent call to LogCloseSnapshot() on this Log @@ -950,6 +1092,36 @@ static void logLeaveMutex(Log *pLog){ int sqlite3LogOpenSnapshot(Log *pLog, int *pChanged){ int rc = SQLITE_OK; if( pLog->isLocked==0 ){ + int nAttempt; + + /* Obtain a snapshot-lock on the log-summary file. The procedure + ** for obtaining the snapshot log is: + ** + ** 1. Attempt a SHARED lock on regions A and B. + ** 2a. If step 1 is successful, drop the lock on region B. + ** 2b. If step 1 is unsuccessful, attempt a SHARED lock on region D. + ** 3. Repeat the above until the lock attempt in step 1 or 2b is + ** successful. + ** + ** If neither of the locks can be obtained after 5 tries, presumably + ** something is wrong (i.e. a process not following the locking protocol). + ** Return an error code in this case. + */ + rc = SQLITE_BUSY; + for(nAttempt=0; nAttempt<5 && rc==SQLITE_BUSY; nAttempt++){ + rc = logLockRegion(pLog, LOG_REGION_A|LOG_REGION_B, LOG_RDLOCK); + if( rc==SQLITE_BUSY ){ + rc = logLockRegion(pLog, LOG_REGION_D, LOG_RDLOCK); + if( rc==SQLITE_OK ) pLog->isLocked = LOG_REGION_D; + }else{ + logLockRegion(pLog, LOG_REGION_B, LOG_UNLOCK); + pLog->isLocked = LOG_REGION_A; + } + } + if( rc!=SQLITE_OK ){ + return rc; + } + if( SQLITE_OK==(rc = logEnterMutex(pLog)) ){ u32 aCksum[2] = {1, 1}; u32 aHdr[LOGSUMMARY_HDR_NFIELD+2]; @@ -967,7 +1139,6 @@ int sqlite3LogOpenSnapshot(Log *pLog, int *pChanged){ *pChanged = 1; } if( rc==SQLITE_OK ){ - pLog->isLocked = 1; if( memcmp(&pLog->hdr, aHdr, sizeof(LogSummaryHdr)) ){ *pChanged = 1; memcpy(&pLog->hdr, aHdr, LOGSUMMARY_HDR_NFIELD*sizeof(u32)); @@ -975,6 +1146,11 @@ int sqlite3LogOpenSnapshot(Log *pLog, int *pChanged){ } logLeaveMutex(pLog); } + + if( rc!=SQLITE_OK ){ + /* An error occured while attempting log recovery. */ + sqlite3LogCloseSnapshot(pLog); + } } return rc; } @@ -983,6 +1159,10 @@ int sqlite3LogOpenSnapshot(Log *pLog, int *pChanged){ ** Unlock the current snapshot. */ void sqlite3LogCloseSnapshot(Log *pLog){ + if( pLog->isLocked ){ + assert( pLog->isLocked==LOG_REGION_A || pLog->isLocked==LOG_REGION_D ); + logLockRegion(pLog, pLog->isLocked, LOG_UNLOCK); + } pLog->isLocked = 0; } @@ -1072,11 +1252,20 @@ void sqlite3LogMaxpgno(Log *pLog, Pgno *pPgno){ int sqlite3LogWriteLock(Log *pLog, int op){ assert( pLog->isLocked ); if( op ){ + + /* Obtain the writer lock */ + int rc = logLockRegion(pLog, LOG_REGION_C|LOG_REGION_D, LOG_WRLOCK); + if( rc!=SQLITE_OK ){ + return rc; + } + if( memcmp(&pLog->hdr, pLog->pSummary->aData, sizeof(pLog->hdr)) ){ return SQLITE_BUSY; } pLog->isWriteLocked = 1; + }else if( pLog->isWriteLocked ){ + logLockRegion(pLog, LOG_REGION_C|LOG_REGION_D, LOG_UNLOCK); memcpy(&pLog->hdr, pLog->pSummary->aData, sizeof(pLog->hdr)); pLog->isWriteLocked = 0; } @@ -1226,18 +1415,25 @@ int sqlite3LogFrames( int sqlite3LogCheckpoint( Log *pLog, /* Log connection */ sqlite3_file *pFd, /* File descriptor open on db file */ - u8 *zBuf /* Temporary buffer to use */ + u8 *zBuf, /* Temporary buffer to use */ + int (*xBusyHandler)(void *), /* Pointer to busy-handler function */ + void *pBusyHandlerArg /* Argument to pass to xBusyHandler */ ){ + int rc; - /* Assert() that the caller is holding an EXCLUSIVE lock on the - ** database file. - */ -#ifdef SQLITE_DEBUG - int lock; - sqlite3OsFileControl(pFd, SQLITE_FCNTL_LOCKSTATE, &lock); - assert( lock>=4 ); -#endif + do { + rc = logLockRegion(pLog, LOG_REGION_B|LOG_REGION_C, LOG_WRLOCK); + }while( rc==SQLITE_BUSY && xBusyHandler(pBusyHandlerArg) ); + if( rc!=SQLITE_OK ) return rc; + + do { + rc = logLockRegion(pLog, LOG_REGION_A, LOG_WRLOCK); + }while( rc==SQLITE_BUSY && xBusyHandler(pBusyHandlerArg) ); + if( rc!=SQLITE_OK ) return rc; - return logCheckpoint(pLog, pFd, zBuf); + rc = logCheckpoint(pLog, pFd, zBuf); + + logLockRegion(pLog, LOG_REGION_A|LOG_REGION_B|LOG_REGION_C, LOG_UNLOCK); + return rc; } diff --git a/src/log.h b/src/log.h index 63fa8ae433..816f9354eb 100644 --- a/src/log.h +++ b/src/log.h @@ -55,7 +55,9 @@ int sqlite3LogFrames(Log *pLog, int, PgHdr *, Pgno, int, int); int sqlite3LogCheckpoint( Log *pLog, /* Log connection */ sqlite3_file *pFd, /* File descriptor open on db file */ - u8 *zBuf /* Temporary buffer to use */ + u8 *zBuf, /* Temporary buffer to use */ + int (*xBusyHandler)(void *), /* Pointer to busy-handler function */ + void *pBusyHandlerArg /* Argument to pass to xBusyHandler */ ); #endif /* _LOG_H_ */ diff --git a/src/pager.c b/src/pager.c index 4e8213cf9b..cd310ab887 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3120,6 +3120,7 @@ static int pager_write_pagelist(PgHdr *pList){ ** EXCLUSIVE, it means the database file has been changed and any rollback ** will require a journal playback. */ + assert( !pagerUseLog(pList->pPager) ); assert( pPager->state>=PAGER_RESERVED ); rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); @@ -3785,21 +3786,13 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( pagerUseLog(pPager) ){ int changed = 0; - /* TODO: Change the following block to grab a WAL read-lock. Or, - ** combine obtaining the read-lock with LogOpenSnapshot()? */ - rc = pager_wait_on_lock(pPager, SHARED_LOCK); - if( rc!=SQLITE_OK ){ - assert( pPager->state==PAGER_UNLOCK ); - return pager_error(pPager, rc); - } - rc = sqlite3LogOpenSnapshot(pPager->pLog, &changed); if( rc==SQLITE_OK ){ if( changed ){ pager_reset(pPager); assert( pPager->errCode || pPager->dbSizeValid==0 ); } - pPager->state = PAGER_SHARED; + pPager->state = PAGER_SHARED; /* TODO: Is this right? */ rc = sqlite3PagerPagecount(pPager, &changed); } }else if( pPager->state==PAGER_UNLOCK || isErrorReset ){ @@ -4330,20 +4323,7 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ assert( pPager->pInJournal==0 ); assert( !MEMDB && !pPager->tempFile ); - /* Obtain a RESERVED lock on the database file. If the exFlag parameter - ** is true, then immediately upgrade this to an EXCLUSIVE lock. The - ** busy-handler callback can be used when upgrading to the EXCLUSIVE - ** lock, but not when obtaining the RESERVED lock. - */ - rc = sqlite3OsLock(pPager->fd, RESERVED_LOCK); - if( rc==SQLITE_OK ){ - pPager->state = PAGER_RESERVED; - if( exFlag ){ - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); - } - } - - if( rc==SQLITE_OK && pagerUseLog(pPager) ){ + if( pagerUseLog(pPager) ){ /* Grab the write lock on the log file. If successful, upgrade to ** PAGER_EXCLUSIVE state. Otherwise, return an error code to the caller. ** The busy-handler is not invoked if another connection already @@ -4352,6 +4332,20 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ rc = sqlite3LogWriteLock(pPager->pLog, 1); if( rc==SQLITE_OK ){ pPager->dbOrigSize = pPager->dbSize; + pPager->state = PAGER_RESERVED; + } + }else{ + /* Obtain a RESERVED lock on the database file. If the exFlag parameter + ** is true, then immediately upgrade this to an EXCLUSIVE lock. The + ** busy-handler callback can be used when upgrading to the EXCLUSIVE + ** lock, but not when obtaining the RESERVED lock. + */ + rc = sqlite3OsLock(pPager->fd, RESERVED_LOCK); + if( rc==SQLITE_OK ){ + pPager->state = PAGER_RESERVED; + if( exFlag ){ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + } } } @@ -5657,11 +5651,10 @@ sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){ int sqlite3PagerCheckpoint(Pager *pPager){ int rc = SQLITE_OK; if( pPager->pLog ){ - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); - if( rc==SQLITE_OK ){ - u8 *zBuf = (u8 *)pPager->pTmpSpace; - rc = sqlite3LogCheckpoint(pPager->pLog, pPager->fd, zBuf); - } + u8 *zBuf = (u8 *)pPager->pTmpSpace; + rc = sqlite3LogCheckpoint(pPager->pLog, pPager->fd, + zBuf, pPager->xBusyHandler, pPager->pBusyHandlerArg + ); } return rc; } From b9bf16bfde7bb7efcacc92e21929a1ca76c1c492 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 14 Apr 2010 11:23:30 +0000 Subject: [PATCH 10/89] Fixes for locking issues in WAL mode. FossilOrigin-Name: a9617eff39177250e2f118f25fdd4b3acb8b0478 --- manifest | 18 +-- manifest.uuid | 2 +- src/log.c | 128 +++++++++++---- src/pragma.c | 1 - test/wal.test | 370 +++++++++++++------------------------------- test/walthread.test | 4 +- 6 files changed, 212 insertions(+), 311 deletions(-) diff --git a/manifest b/manifest index faab67f851..1f3fc5b551 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sexperimental\slocking\sscheme. -D 2010-04-13T19:27:31 +C Fixes\sfor\slocking\sissues\sin\sWAL\smode. +D 2010-04-14T11:23:30 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c d89988bb26a3cd414858c97642a612b4ce6e540f +F src/log.c 78575e5bed43578580a0ee86fd3c2e924abe4a88 F src/log.h a2654af46ce7b5732f4d5a731abfdd180f0a06d9 F src/main.c c0e7192bad5b90544508b241eb2487ac661de890 F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a @@ -160,7 +160,7 @@ F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 F src/pcache1.c 6dc1871ce8ead9187161c370a58cd06c84221f76 -F src/pragma.c 9f1d53d40b47743f3d6151df15915970367c9de5 +F src/pragma.c f12cb58a8aa0d80cfed282ef87a285ed71beb793 F src/prepare.c fd1398cb1da54385ba5bd68d93928f10d10a1d9c F src/printf.c 5f5b65a83e63f2096a541a340722a509fa0240a7 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -756,10 +756,10 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test d5916b3a40d74dd5bf6ff21b42e6fa7f255efd13 +F test/wal.test a4be3c7a36e3db3d9d276dc7664c2ed5eece1e8a F test/walcrash.test 45cfbab30bb7cbe0b2e9d5cabe90dbcad10cb89b F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 -F test/walthread.test 1592116f74261d77ea697d9503b0cd1ab1c15e74 +F test/walthread.test 58cd64b06f186251f09f64e4918fb74a7e52c963 F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 F test/where2.test 45eacc126aabb37959a387aa83e59ce1f1f03820 F test/where3.test 97d3936e6a443b968f1a61cdcc0f673252000e94 @@ -803,7 +803,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P f4e1150fed2c520c7c52612cb1019429d78dc32a -R da172d13acb33c194abd32bf4ed20cc9 +P 3f958e87c33d667d299b03ffdef58db5dc6363f4 +R 91ba1bf1aecd8bbf6dbeda941d0dbe58 U dan -Z cd10eeca0187a805032040fdd27191f0 +Z d3b26d31320c12902d5354af18b08451 diff --git a/manifest.uuid b/manifest.uuid index 43b6c91f02..8cd6641af0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3f958e87c33d667d299b03ffdef58db5dc6363f4 \ No newline at end of file +a9617eff39177250e2f118f25fdd4b3acb8b0478 \ No newline at end of file diff --git a/src/log.c b/src/log.c index 2b2764f361..c7fb8738d2 100644 --- a/src/log.c +++ b/src/log.c @@ -617,7 +617,7 @@ int sqlite3LogOpen( const char *zDb, /* Name of database file */ Log **ppLog /* OUT: Allocated Log handle */ ){ - int rc; /* Return Code */ + int rc = SQLITE_OK; /* Return Code */ Log *pRet; /* Object to allocate and return */ LogSummary *pSummary = 0; /* Summary object */ sqlite3_mutex *mutex = 0; /* LOG_SUMMARY_MUTEX mutex */ @@ -806,7 +806,7 @@ static int logCheckpoint( int pgsz = pLog->hdr.pgsz; /* Database page-size */ LogCheckpoint *pIter = 0; /* Log iterator context */ u32 iDbpage = 0; /* Next database page to write */ - u32 iFrame; /* Log frame containing data for iDbpage */ + u32 iFrame = 0; /* Log frame containing data for iDbpage */ /* Allocate the iterator */ pIter = logCheckpointInit(pLog); @@ -1080,6 +1080,77 @@ static int logLockRegion(Log *pLog, u32 mRegion, int op){ return SQLITE_OK; } +/* +** Try to read the log-summary header. Attempt to verify the header +** checksum. If the checksum can be verified, copy the log-summary +** header into structure pLog->hdr. If the contents of pLog->hdr are +** modified by this and pChanged is not NULL, set *pChanged to 1. +** Otherwise leave *pChanged unmodified. +** +** If the checksum cannot be verified return SQLITE_ERROR. +*/ +int logSummaryTryHdr(Log *pLog, int *pChanged){ + u32 aCksum[2] = {1, 1}; + u32 aHdr[LOGSUMMARY_HDR_NFIELD+2]; + + /* First try to read the header without a lock. Verify the checksum + ** before returning. This will almost always work. + */ + memcpy(aHdr, pLog->pSummary->aData, sizeof(aHdr)); + logChecksumBytes((u8*)aHdr, sizeof(u32)*LOGSUMMARY_HDR_NFIELD, aCksum); + if( aCksum[0]!=aHdr[LOGSUMMARY_HDR_NFIELD] + || aCksum[1]!=aHdr[LOGSUMMARY_HDR_NFIELD+1] + ){ + return SQLITE_ERROR; + } + + if( memcmp(&pLog->hdr, aHdr, sizeof(LogSummaryHdr)) ){ + if( pChanged ){ + *pChanged = 1; + } + memcpy(&pLog->hdr, aHdr, sizeof(LogSummaryHdr)); + } + return SQLITE_OK; +} + +/* +** Read the log-summary header from the log-summary file into structure +** pLog->hdr. If attempting to verify the header checksum fails, try +** to recover the log before returning. +** +** If the log-summary header is successfully read, return SQLITE_OK. +** Otherwise an SQLite error code. +*/ +int logSummaryReadHdr(Log *pLog, int *pChanged){ + int rc; + + /* First try to read the header without a lock. Verify the checksum + ** before returning. This will almost always work. + */ + if( SQLITE_OK==logSummaryTryHdr(pLog, pChanged) ){ + return SQLITE_OK; + } + + /* If the first attempt to read the header failed, lock the log-summary + ** file and try again. If the header checksum verification fails this + ** time as well, run log recovery. + */ + if( SQLITE_OK==(rc = logEnterMutex(pLog)) ){ + if( SQLITE_OK!=logSummaryTryHdr(pLog, pChanged) ){ + if( pChanged ){ + *pChanged = 1; + } + rc = logSummaryRecover(pLog->pSummary, pLog->pFd); + if( rc==SQLITE_OK ){ + rc = logSummaryTryHdr(pLog, 0); + } + } + logLeaveMutex(pLog); + } + + return rc; +} + /* ** Lock a snapshot. ** @@ -1122,31 +1193,7 @@ int sqlite3LogOpenSnapshot(Log *pLog, int *pChanged){ return rc; } - if( SQLITE_OK==(rc = logEnterMutex(pLog)) ){ - u32 aCksum[2] = {1, 1}; - u32 aHdr[LOGSUMMARY_HDR_NFIELD+2]; - memcpy(aHdr, pLog->pSummary->aData, sizeof(aHdr)); - - /* Verify the checksum on the log-summary header. If it fails, - ** recover the log-summary from the log file. - */ - logChecksumBytes((u8*)aHdr, sizeof(u32)*LOGSUMMARY_HDR_NFIELD, aCksum); - if( aCksum[0]!=aHdr[LOGSUMMARY_HDR_NFIELD] - || aCksum[1]!=aHdr[LOGSUMMARY_HDR_NFIELD+1] - ){ - rc = logSummaryRecover(pLog->pSummary, pLog->pFd); - memcpy(aHdr, pLog->pSummary->aData, sizeof(aHdr)); - *pChanged = 1; - } - if( rc==SQLITE_OK ){ - if( memcmp(&pLog->hdr, aHdr, sizeof(LogSummaryHdr)) ){ - *pChanged = 1; - memcpy(&pLog->hdr, aHdr, LOGSUMMARY_HDR_NFIELD*sizeof(u32)); - } - } - logLeaveMutex(pLog); - } - + rc = logSummaryReadHdr(pLog, pChanged); if( rc!=SQLITE_OK ){ /* An error occured while attempting log recovery. */ sqlite3LogCloseSnapshot(pLog); @@ -1409,8 +1456,13 @@ int sqlite3LogFrames( } /* -** Checkpoint the database. When this function is called the caller -** must hold an exclusive lock on the database file. +** Checkpoint the database: +** +** 1. Wait for an EXCLUSIVE lock on regions B and C. +** 2. Wait for an EXCLUSIVE lock on region A. +** 3. Copy the contents of the log into the database file. +** 4. Zero the log-summary header (so new readers will ignore the log). +** 5. Drop the locks obtained in steps 1 and 2. */ int sqlite3LogCheckpoint( Log *pLog, /* Log connection */ @@ -1419,20 +1471,30 @@ int sqlite3LogCheckpoint( int (*xBusyHandler)(void *), /* Pointer to busy-handler function */ void *pBusyHandlerArg /* Argument to pass to xBusyHandler */ ){ - int rc; + int rc; /* Return code */ + /* Wait for a write-lock on regions B and C. */ do { rc = logLockRegion(pLog, LOG_REGION_B|LOG_REGION_C, LOG_WRLOCK); }while( rc==SQLITE_BUSY && xBusyHandler(pBusyHandlerArg) ); if( rc!=SQLITE_OK ) return rc; + /* Wait for a write-lock on region A. */ do { rc = logLockRegion(pLog, LOG_REGION_A, LOG_WRLOCK); }while( rc==SQLITE_BUSY && xBusyHandler(pBusyHandlerArg) ); - if( rc!=SQLITE_OK ) return rc; - - rc = logCheckpoint(pLog, pFd, zBuf); + if( rc!=SQLITE_OK ){ + logLockRegion(pLog, LOG_REGION_B|LOG_REGION_C, LOG_UNLOCK); + return rc; + } + /* Copy data from the log to the database file. */ + rc = logSummaryReadHdr(pLog, 0); + if( rc==SQLITE_OK ){ + rc = logCheckpoint(pLog, pFd, zBuf); + } + + /* Release the locks. */ logLockRegion(pLog, LOG_REGION_A|LOG_REGION_B|LOG_REGION_C, LOG_UNLOCK); return rc; } diff --git a/src/pragma.c b/src/pragma.c index 95310e336c..137ff510d9 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1386,7 +1386,6 @@ void sqlite3Pragma( if( sqlite3StrICmp(zLeft, "checkpoint")==0 ){ sqlite3VdbeUsesBtree(v, iDb); - sqlite3VdbeAddOp2(v, OP_Transaction, iDb, 1); sqlite3VdbeAddOp3(v, OP_Checkpoint, iDb, 0, 0); }else diff --git a/test/wal.test b/test/wal.test index b12782b663..cafda909a4 100644 --- a/test/wal.test +++ b/test/wal.test @@ -27,7 +27,7 @@ proc range {args} { } proc reopen_db {} { - db close + catch { db close } file delete -force test.db test.db-wal sqlite3_wal db test.db #register_logtest @@ -223,7 +223,6 @@ do_test wal-7.1 { CREATE TABLE t1(a, b); INSERT INTO t1 VALUES(1, 2); } - list [file size test.db] [file size test.db-wal] } [list 0 [expr (1024+20)*3]] do_test wal-7.2 { @@ -231,272 +230,17 @@ do_test wal-7.2 { list [file size test.db] [file size test.db-wal] } [list 2048 [expr (1024+20)*3]] -# db close -# sqlite3_wal db test.db -# register_logsummary_module db -# # Warm-body tests of the virtual tables used for testing. -# # -# do_test wal-8.1 { -# execsql { CREATE VIRTUAL TABLE temp.logsummary USING logsummary } -# execsql { CREATE VIRTUAL TABLE temp.logcontent USING logcontent } -# execsql { CREATE VIRTUAL TABLE temp.loglock USING loglock } -# execsql { SELECT * FROM logsummary } -# } [list [file join [pwd] test.db] 1 0 0 0 0 0 0] -# -# do_test wal-8.2 { -# sqlite3_wal db2 test.db -# execsql { SELECT * FROM logsummary } -# } [list [file join [pwd] test.db] 2 0 0 0 0 0 0] -# do_test wal-8.3 { -# db2 close -# execsql { SELECT * FROM logsummary } -# } [list [file join [pwd] test.db] 1 0 0 0 0 0 0] -# do_test wal-8.4 { -# execsql { INSERT INTO t1 VALUES(3, 4) } -# execsql { SELECT * FROM logsummary } -# } [list [file join [pwd] test.db] 1 1024 2 2 3 0 0] -# do_test wal-8.5 { -# execsql { PRAGMA checkpoint } -# execsql { SELECT * FROM logsummary } -# } [list [file join [pwd] test.db] 1 1024 0 0 0 0 0] -# -# do_test wal-8.6 { -# execsql { INSERT INTO t1 VALUES(5, 6) } -# execsql { PRAGMA checkpoint('1 1 1') } -# execsql { SELECT * FROM logsummary } -# } [list [file join [pwd] test.db] 1 1024 2 0 3 0 0] -# do_test wal-8.7 { -# execsql { SELECT logpage, dbpage FROM logcontent } -# } {} -# do_test wal-8.8 { -# execsql { INSERT INTO t1 VALUES(7, 8) } -# execsql { SELECT logpage, dbpage FROM logcontent } -# } {4 T:4 5 2} -# do_test wal-8.9 { -# execsql { SELECT * FROM logsummary } -# } [list [file join [pwd] test.db] 1 1024 2 4 5 0 0] -# do_test wal-8.10 { -# execsql { SELECT * FROM loglock } -# } [list [file join [pwd] test.db] 0 0 0] -# do_test wal-8.11 { -# execsql { BEGIN; SELECT * FROM t1; } -# execsql { SELECT * FROM loglock } -# } [list [file join [pwd] test.db] 0 0 4] -# -# # Try making the log wrap around. -# # -# reopen_db -# -# do_test wal-9.1 { -# execsql { -# BEGIN; -# CREATE TABLE t1(a PRIMARY KEY, b); -# } -# for {set i 0} {$i < 100} {incr i} { -# execsql { INSERT INTO t1 VALUES($i, randomblob(100)) } -# } -# execsql COMMIT -# execsql { SELECT * FROM logsummary } -# } [list [file join [pwd] test.db] 1 1024 2 2 17 0 0] -# do_test wal-9.2 { -# execsql { SELECT logpage, dbpage FROM logcontent } -# } {2 T:2 3 1 4 2 5 3 6 4 7 5 8 6 9 7 10 8 11 9 12 10 13 11 14 12 15 13 16 14 17 15} -# do_test wal-9.3 { -# execsql { PRAGMA checkpoint('12, 12') } -# execsql { SELECT * FROM logsummary } -# } [list [file join [pwd] test.db] 1 1024 2 15 17 0 0] -# do_test wal-9.4 { -# execsql { SELECT logpage, dbpage FROM logcontent } -# } {15 13 16 14 17 15} -# do_test wal-9.5 { -# execsql { SELECT count(*) FROM t1 } -# } {100} -# do_test wal-9.6 { -# execsql { INSERT INTO t1 VALUES(100, randomblob(100)) } -# execsql { SELECT * FROM logsummary } -# } [list [file join [pwd] test.db] 1 1024 2 15 20 0 0] -# -# do_test wal-9.7 { -# execsql { SELECT count(*) FROM t1 } -# } {101} -# do_test wal-9.8 { -# db close -# sqlite3_wal db test.db -# register_logtest -# execsql { SELECT count(*) FROM t1 } -# } {101} -# do_test wal-9.9 { -# execsql { SELECT * FROM logsummary } -# } [list [file join [pwd] test.db] 1 0 0 0 0 0 0] -# -# reopen_db -# do_test wal-10.1 { -# execsql { -# PRAGMA page_size = 1024; -# CREATE TABLE t1(x PRIMARY KEY); -# INSERT INTO t1 VALUES(randomblob(900)); -# INSERT INTO t1 VALUES(randomblob(900)); -# INSERT INTO t1 SELECT randomblob(900) FROM t1; -- 4 -# INSERT INTO t1 SELECT randomblob(900) FROM t1; -- 8 -# INSERT INTO t1 SELECT randomblob(900) FROM t1; -- 16 -# } -# list [file size test.db] [file size test.db-wal] -# } {0 55296} -# do_test wal-10.2 { -# execsql { PRAGMA checkpoint('20 30') } -# } {} -# do_test wal-10.3 { -# execsql { SELECT * FROM logsummary } -# } [list [file join [pwd] test.db] 1 1024 34 38 54 0 0] -# do_test wal-10.4 { -# execsql { SELECT dbpage FROM logcontent } -# } {21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37} -# do_test wal-10.5 { -# execsql { INSERT INTO t1 VALUES(randomblob(900)) } -# execsql { SELECT * FROM logsummary } -# } [list [file join [pwd] test.db] 1 1024 34 38 6 34 54] -# do_test wal-10.6 { -# execsql { SELECT count(*) FROM t1 WHERE x NOT NULL } -# } {17} -# do_test wal-10.8 { -# execsql { SELECT logpage FROM logcontent } -# } [range 38 54 1 6] -# do_test wal-10.9 { -# execsql { INSERT INTO t1 SELECT randomblob(900) FROM t1 } -# execsql { SELECT * FROM logsummary } -# } [list [file join [pwd] test.db] 1 1024 34 38 68 34 54] -# -# do_test wal-10.10 { -# execsql { SELECT logpage FROM logcontent } -# } [range 38 54 1 33 55 68] -# -# do_test wal-10.11 { -# execsql { SELECT count(*) FROM t1 WHERE x NOT NULL } -# } {34} -# -# do_test wal-10.12 { -# execsql { PRAGMA checkpoint('35 35') } -# } {} -# do_test wal-10.13 { -# execsql { SELECT logpage FROM logcontent } -# } [range 22 68] -# do_test wal-10.13a { -# execsql { SELECT dbpage FROM logcontent } -# } [list \ -# 50 51 52 53 54 55 56 57 58 59 60 61 \ -# 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 \ -# 62 63 64 65 66 67 68 69 70 71 72 73 74 75 \ -# ] -# -# do_test wal-10.14 { -# execsql { SELECT count(*) FROM t1 WHERE x NOT NULL } -# } {34} -# do_test wal-10.15 { -# execsql { PRAGMA integrity_check } -# } {ok} -# do_test wal-10.16 { -# execsql { PRAGMA checkpoint('20 20') } -# execsql { SELECT * FROM logsummary } -# } [list [file join [pwd] test.db] 1 1024 7 63 68 0 0] -# do_test wal-10.17 { -# execsql { SELECT logpage FROM logcontent } -# } [range 63 68] -# do_test wal-10.17a { -# execsql { SELECT dbpage FROM logcontent } -# } {70 71 72 73 74 75} -# -# do_test wal-10.18 { -# execsql { INSERT INTO t1 SELECT randomblob(900) FROM t1 } -# execsql { SELECT logpage FROM logcontent } -# } [range 63 147] -# integrity_check wal-10.19 -# -# do_test wal-10.20 { -# execsql { PRAGMA checkpoint('52 52') } -# execsql { SELECT logpage FROM logcontent } -# } [range 116 147] -# do_test wal-10.20a { -# execsql { SELECT * FROM logsummary } -# } [list [file join [pwd] test.db] 1 1024 69 116 147 0 0] -# integrity_check wal-10.20.integrity -# -# do_test wal-10.21 { -# execsql { INSERT INTO t1 VALUES( randomblob(900) ) } -# execsql { SELECT logpage FROM logcontent } -# } [range 116 152] -# do_test wal-10.22 { -# execsql { PRAGMA integrity_check } -# } {ok} -# -# file delete -force testX.db testX.db-wal -# file copy test.db testX.db -# file copy test.db-wal testX.db-wal -# do_test wal-10.23 { -# sqlite3_wal db2 testX.db -# register_logtest db2 -# execsql { SELECT logpage FROM logcontent WHERE db LIKE '%testX%' } db2 -# } [range 34 54 1 33 55 152] -# -# do_test wal-10.24 { -# execsql { PRAGMA integrity_check } db2 -# } {ok} -# db2 close -# -# do_test wal-11.1 { -# reopen_db -# sqlite3_wal db2 test.db -# -# execsql { -# BEGIN; -# CREATE TABLE t1(x); -# CREATE TABLE t2(x PRIMARY KEY); -# INSERT INTO t1 VALUES(randomblob(900)); -# INSERT INTO t1 VALUES(randomblob(900)); -# INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4 */ -# INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8 */ -# INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16 */ -# INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 32 */ -# INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 64 */ -# -# INSERT INTO t2 VALUES('x'); -# INSERT INTO t2 VALUES('y'); -# INSERT INTO t2 VALUES('z'); -# COMMIT; -# SELECT * FROM logsummary; -# } -# } [list [file join [pwd] test.db] 2 1024 2 2 70 0 0] -# -# do_test wal-11.2 { -# execsql { -# BEGIN; SELECT x FROM t2; -# } db2 -# } {x y z} -# do_test wal-11.2 { -# execsql { -# INSERT INTO t1 VALUES(randomblob(900)); -# PRAGMA checkpoint('10 100'); -# INSERT INTO t1 VALUES(randomblob(900)); -# INSERT INTO t2 VALUES('0'); -# SELECT * FROM logsummary; -# } -# } [list [file join [pwd] test.db] 2 1024 71 71 7 71 73] -# do_test wal-12.3 { -# execsql { PRAGMA integrity_check } db2 -# } {ok} -# db2 close - - # Execute some transactions in auto-vacuum mode to test database file # truncation. # -do_test wal-12.1 { +do_test wal-8.1 { reopen_db execsql { PRAGMA auto_vacuum = 1; PRAGMA auto_vacuum; } } {1} -do_test wal-12.2 { +do_test wal-8.2 { execsql { PRAGMA page_size = 1024; CREATE TABLE t1(x); @@ -511,7 +255,7 @@ do_test wal-12.2 { } file size test.db } [expr 68*1024] -do_test wal-12.3 { +do_test wal-8.3 { execsql { DELETE FROM t1 WHERE rowid<54; PRAGMA checkpoint; @@ -522,7 +266,7 @@ do_test wal-12.3 { # Run some "warm-body" tests to ensure that log-summary files with more # than 256 entries (log summaries that contain index blocks) work Ok. # -do_test wal-13.1 { +do_test wal-9.1 { reopen_db execsql { PRAGMA page_size = 1024; @@ -539,12 +283,12 @@ do_test wal-13.1 { } file size test.db } 0 -do_test wal-13.2 { +do_test wal-9.2 { sqlite3_wal db2 test.db execsql {PRAGMA integrity_check } db2 } {ok} -do_test wal-13.3 { +do_test wal-9.3 { file delete -force test2.db test2.db-wal file copy test.db test2.db file copy test.db-wal test2.db-wal @@ -553,8 +297,7 @@ do_test wal-13.3 { } {ok} db3 close -do_test wal-13.4 { -breakpoint +do_test wal-9.4 { execsql { PRAGMA checkpoint } db2 close sqlite3_wal db2 test.db @@ -564,5 +307,102 @@ breakpoint foreach handle {db db2 db3} { catch { $handle close } } unset handle +#------------------------------------------------------------------------- +# The following block of tests - wal-10.* - test that the WAL locking +# scheme works for clients in a single process. +# +reopen_db +sqlite3_wal db2 test.db +sqlite3_wal db3 test.db + +do_test wal-10.1 { + execsql { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + BEGIN; + INSERT INTO t1 VALUES(3, 4); + } + execsql "SELECT * FROM t1" db2 +} {1 2} +do_test wal-10.2 { + execsql { COMMIT } + execsql "SELECT * FROM t1" db2 +} {1 2 3 4} +do_test wal-10.3 { + execsql { + BEGIN; + SELECT * FROM t1; + } db2 +} {1 2 3 4} +do_test wal-10.4 { + catchsql { PRAGMA checkpoint } +} {1 {database is locked}} +do_test wal-10.5 { + execsql { INSERT INTO t1 VALUES(5, 6) } + execsql { SELECT * FROM t1 } db2 +} {1 2 3 4} + +# Connection [db2] is holding a lock on a snapshot, preventing [db] from +# checkpointing the database. Add a busy-handler to [db]. If [db2] completes +# its transaction from within the busy-handler, [db] is able to complete +# the checkpoint operation. +# +proc busyhandler x { + if {$x==4} { + execsql { COMMIT } db2 + } + if {$x<5} {return 0} + return 1 +} +db busy busyhandler +do_test wal-10.6 { + execsql { PRAGMA checkpoint } +} {} + +# Similar to the test above. Except this time, a new read transaction is +# started (db3) while the checkpointer is waiting for an old one to finish. +# The checkpointer can finish, but any subsequent write operations must +# wait until after db3 has closed the read transaction. +# +db busy {} +do_test wal-10.7 { + execsql { + BEGIN; + SELECT * FROM t1; + } db2 +} {1 2 3 4 5 6} +do_test wal-10.8 { + execsql { INSERT INTO t1 VALUES(7, 8) } + catchsql { PRAGMA checkpoint } +} {1 {database is locked}} +proc busyhandler x { + if {$x==3} { execsql { BEGIN; SELECT * FROM t1 } db3 } + if {$x==4} { execsql { COMMIT } db2 } + if {$x<5} { return 0 } + return 1 +} +db busy busyhandler +do_test wal-10.9 { + execsql { PRAGMA checkpoint } +} {} +do_test wal-10.10 { + execsql { SELECT * FROM t1 } db3 +} {1 2 3 4 5 6 7 8} +do_test wal-10.11 { + catchsql { INSERT INTO t1 VALUES(9, 10) } +} {1 {database is locked}} +do_test wal-10.12 { + execsql { SELECT * FROM t1 } +} {1 2 3 4 5 6 7 8} +do_test wal-10.13 { + execsql { COMMIT } db3 +} {} +do_test wal-10.14 { + execsql { INSERT INTO t1 VALUES(9, 10) } + execsql { SELECT * FROM t1 } +} {1 2 3 4 5 6 7 8 9 10} + +foreach handle {db db2 db3} { catch { $handle close } } +unset handle finish_test diff --git a/test/walthread.test b/test/walthread.test index 7727ab3759..fbd6c653cc 100644 --- a/test/walthread.test +++ b/test/walthread.test @@ -62,7 +62,7 @@ do_test walthread-1.4 { # Each of the N threads runs N read transactions followed by a single write # transaction in a loop as fast as possible. # -# There is also a single checkpointer thread. It runs the following loop: +# Ther is also a single checkpointer thread. It runs the following loop: # # 1) Execute "CHECKPOINT main 32 -1 1" # 2) Sleep for 500 ms. @@ -85,7 +85,7 @@ set thread_program { set rc [sqlite3_finalize $stmt] if {$rc ne "SQLITE_OK"} { - error $rc + error "$rc: [sqlite3_errmsg $DB]" } return $res } From 02bb59664928c74552cbbcb3390ccbb61bbaf961 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 14 Apr 2010 15:49:40 +0000 Subject: [PATCH 11/89] Improve the logLockRegion() function in log.c. FossilOrigin-Name: 5e9dd3bd8e829376408925fb4cfcd5bb1eb1105f --- manifest | 12 ++--- manifest.uuid | 2 +- src/log.c | 134 +++++++++++++++++++++++++++++++++++--------------- 3 files changed, 100 insertions(+), 48 deletions(-) diff --git a/manifest b/manifest index 1f3fc5b551..611f989a4f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sfor\slocking\sissues\sin\sWAL\smode. -D 2010-04-14T11:23:30 +C Improve\sthe\slogLockRegion()\sfunction\sin\slog.c. +D 2010-04-14T15:49:40 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c 78575e5bed43578580a0ee86fd3c2e924abe4a88 +F src/log.c e01488f4e515d17ecb21d6cebc5a37bec5b0e56c F src/log.h a2654af46ce7b5732f4d5a731abfdd180f0a06d9 F src/main.c c0e7192bad5b90544508b241eb2487ac661de890 F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a @@ -803,7 +803,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 3f958e87c33d667d299b03ffdef58db5dc6363f4 -R 91ba1bf1aecd8bbf6dbeda941d0dbe58 +P a9617eff39177250e2f118f25fdd4b3acb8b0478 +R 7008c4f8854b7fbb33a10340b4b449d8 U dan -Z d3b26d31320c12902d5354af18b08451 +Z d0e2fbdd9e82a46dc4aa039b5fc8e3ce diff --git a/manifest.uuid b/manifest.uuid index 8cd6641af0..8fdc50e3bd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a9617eff39177250e2f118f25fdd4b3acb8b0478 \ No newline at end of file +5e9dd3bd8e829376408925fb4cfcd5bb1eb1105f \ No newline at end of file diff --git a/src/log.c b/src/log.c index c7fb8738d2..9b25216df4 100644 --- a/src/log.c +++ b/src/log.c @@ -989,84 +989,129 @@ static void logLeaveMutex(Log *pLog){ static int logLockRegion(Log *pLog, u32 mRegion, int op){ LogSummary *pSummary = pLog->pSummary; LogLock *p; /* Used to iterate through in-process locks */ - u32 mNew; /* New locks on file */ - u32 mOld; /* Old locks on file */ - u32 mNewLock; /* New locks held by pLog */ + u32 mOther; /* Locks held by other connections */ + u32 mNew; /* New mask for pLog */ assert( /* Writer lock operations */ (op==LOG_WRLOCK && mRegion==(LOG_REGION_C|LOG_REGION_D)) || (op==LOG_UNLOCK && mRegion==(LOG_REGION_C|LOG_REGION_D)) - /* Reader lock operations */ + /* Normal reader lock operations */ || (op==LOG_RDLOCK && mRegion==(LOG_REGION_A|LOG_REGION_B)) - || (op==LOG_RDLOCK && mRegion==(LOG_REGION_D)) || (op==LOG_UNLOCK && mRegion==(LOG_REGION_A)) || (op==LOG_UNLOCK && mRegion==(LOG_REGION_B)) + + /* Region D reader lock operations */ + || (op==LOG_RDLOCK && mRegion==(LOG_REGION_D)) || (op==LOG_UNLOCK && mRegion==(LOG_REGION_D)) /* Checkpointer lock operations */ || (op==LOG_WRLOCK && mRegion==(LOG_REGION_B|LOG_REGION_C)) || (op==LOG_WRLOCK && mRegion==(LOG_REGION_A)) - || (op==LOG_UNLOCK && mRegion==(LOG_REGION_A|LOG_REGION_B|LOG_REGION_C)) || (op==LOG_UNLOCK && mRegion==(LOG_REGION_B|LOG_REGION_C)) + || (op==LOG_UNLOCK && mRegion==(LOG_REGION_A|LOG_REGION_B|LOG_REGION_C)) ); + /* Assert that a connection never tries to go from an EXCLUSIVE to a + ** SHARED lock on a region. Moving from SHARED to EXCLUSIVE sometimes + ** happens though (when a region D reader upgrades to a writer). + */ + assert( op!=LOG_RDLOCK || 0==(pLog->lock.mLock & (mRegion<<8)) ); + sqlite3_mutex_enter(pSummary->mutex); - /* If obtaining (not releasing) a lock, check if there exist any - ** conflicting locks in process. Return SQLITE_BUSY in this case. + /* Calculate a mask of logs held by all connections in this process apart + ** from this one. The least significant byte of the mask contains a mask + ** of the SHARED logs held. The next least significant byte of the mask + ** indicates the EXCLUSIVE locks held. For example, to test if some other + ** connection is holding a SHARED lock on region A, or an EXCLUSIVE lock + ** on region C, do: + ** + ** hasSharedOnA = (mOther & (LOG_REGION_A<<0)); + ** hasExclusiveOnC = (mOther & (LOG_REGION_C<<8)); + ** + ** In all masks, if the bit in the EXCLUSIVE byte mask is set, so is the + ** corresponding bit in the SHARED mask. */ - if( op ){ - u32 mConflict = (mRegion<<8) | ((op==LOG_WRLOCK) ? mRegion : 0); - for(p=pSummary->pLock; p; p=p->pNext){ - if( p!=&pLog->lock && (p->mLock & mConflict) ){ - sqlite3_mutex_leave(pSummary->mutex); - return SQLITE_BUSY; - } + mOther = 0; + for(p=pSummary->pLock; p; p=p->pNext){ + assert( (p->mLock & (p->mLock<<8))==(p->mLock&0x0000FF00) ); + if( p!=&pLog->lock ){ + mOther |= p->mLock; } } - /* Determine the new lock mask for this log connection */ + /* If this call is to lock a region (not to unlock one), test if locks held + ** by any other connection in this process prevent the new locks from + ** begin granted. If so, exit the summary mutex and return SQLITE_BUSY. + */ + if( op && (mOther & (mRegion << (op==LOG_RDLOCK ? 8 : 0))) ){ + sqlite3_mutex_leave(pSummary->mutex); + return SQLITE_BUSY; + } + + /* Figure out the new log mask for this connection. */ switch( op ){ case LOG_UNLOCK: - mNewLock = (pLog->lock.mLock & ~(mRegion|(mRegion<<8))); + mNew = (pLog->lock.mLock & ~(mRegion|(mRegion<<8))); break; case LOG_RDLOCK: - mNewLock = ((pLog->lock.mLock & ~(mRegion<<8)) | mRegion); + mNew = (pLog->lock.mLock | mRegion); break; default: assert( op==LOG_WRLOCK ); - mNewLock = (pLog->lock.mLock | (mRegion<<8) | mRegion); + mNew = (pLog->lock.mLock | (mRegion<<8) | mRegion); break; } - /* Determine the current and desired sets of locks at the file level. */ - mNew = 0; - for(p=pSummary->pLock; p; p=p->pNext){ - assert( (p->mLock & (p->mLock<<8))==(p->mLock & 0x00000F00) ); - if( p!=&pLog->lock ) mNew |= p->mLock; + /* Now modify the locks held on the log-summary file descriptor. This + ** file descriptor is shared by all log connections in this process. + ** Therefore: + ** + ** + If one or more log connections in this process hold a SHARED lock + ** on a region, the file-descriptor should hold a SHARED lock on + ** the file region. + ** + ** + If a log connection in this process holds an EXCLUSIVE lock on a + ** region, the file-descriptor should also hold an EXCLUSIVE lock on + ** the region in question. + ** + ** If this is an LOG_UNLOCK operation, only regions for which no other + ** connection holds a lock should actually be unlocked. And if this + ** is a LOG_RDLOCK operation and other connections already hold all + ** the required SHARED locks, then no system call is required. + */ + if( op==LOG_UNLOCK ){ + mRegion = (mRegion & ~mOther); } - mOld = mNew | pLog->lock.mLock; - mNew = mNew | mNewLock; + if( (op==LOG_WRLOCK) + || (op==LOG_UNLOCK && mRegion) + || (op==LOG_RDLOCK && (mOther&mRegion)!=mRegion) + ){ + struct LockMap { + int iStart; /* Byte offset to start locking operation */ + int iLen; /* Length field for locking operation */ + } aMap[] = { + /* 0000 */ {0, 0}, /* 0001 */ {4, 1}, + /* 0010 */ {3, 1}, /* 0011 */ {3, 2}, + /* 0100 */ {2, 1}, /* 0101 */ {0, 0}, + /* 0110 */ {2, 2}, /* 0111 */ {2, 3}, + /* 1000 */ {1, 1}, /* 1001 */ {0, 0}, + /* 1010 */ {0, 0}, /* 1011 */ {0, 0}, + /* 1100 */ {1, 2}, /* 1101 */ {0, 0}, + /* 1110 */ {1, 3}, /* 1111 */ {0, 0} + }; + int rc; /* Return code of fcntl() */ + struct flock f; /* Locking operation */ + + assert( mRegion>8); - struct flock f; memset(&f, 0, sizeof(f)); f.l_type = (op==LOG_WRLOCK?F_WRLCK:(op==LOG_RDLOCK?F_RDLCK:F_UNLCK)); f.l_whence = SEEK_SET; - - if( mChange & LOG_REGION_A ) f.l_start = 12; - else if( mChange & LOG_REGION_B ) f.l_start = 13; - else if( mChange & LOG_REGION_C ) f.l_start = 14; - else if( mChange & LOG_REGION_D ) f.l_start = 15; - - if( mChange & LOG_REGION_D ) f.l_len = 16 - f.l_start; - else if( mChange & LOG_REGION_C ) f.l_len = 15 - f.l_start; - else if( mChange & LOG_REGION_B ) f.l_len = 14 - f.l_start; - else if( mChange & LOG_REGION_A ) f.l_len = 13 - f.l_start; + f.l_start = 32 + aMap[mRegion].iStart; + f.l_len = aMap[mRegion].iLen; rc = fcntl(pSummary->fd, F_SETLK, &f); if( rc!=0 ){ @@ -1075,7 +1120,7 @@ static int logLockRegion(Log *pLog, u32 mRegion, int op){ } } - pLog->lock.mLock = mNewLock; + pLog->lock.mLock = mNew; sqlite3_mutex_leave(pSummary->mutex); return SQLITE_OK; } @@ -1306,6 +1351,13 @@ int sqlite3LogWriteLock(Log *pLog, int op){ return rc; } + /* TODO: What if this is a region D reader? And after writing this + ** transaction it continues to hold a read-lock on the db? Maybe we + ** need to switch it to a region A reader here so that unlocking C|D + ** does not leave the connection with no lock at all. + */ + assert( pLog->isLocked!=LOG_REGION_D ); + if( memcmp(&pLog->hdr, pLog->pSummary->aData, sizeof(pLog->hdr)) ){ return SQLITE_BUSY; } From e264d983d1c8baffca80214d5a1cda88e9335abb Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 14 Apr 2010 18:06:50 +0000 Subject: [PATCH 12/89] Add tests to check inter-process WAL locking. FossilOrigin-Name: 9435f3135849e0d38fde1669201db508561a6308 --- manifest | 17 +-- manifest.uuid | 2 +- src/log.c | 2 +- test/lock2.test | 63 +--------- test/lock_common.tcl | 77 ++++++++++++ test/wal.test | 280 +++++++++++++++++++++++++++---------------- 6 files changed, 264 insertions(+), 177 deletions(-) create mode 100644 test/lock_common.tcl diff --git a/manifest b/manifest index 611f989a4f..92200a9b4f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\slogLockRegion()\sfunction\sin\slog.c. -D 2010-04-14T15:49:40 +C Add\stests\sto\scheck\sinter-process\sWAL\slocking. +D 2010-04-14T18:06:51 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c e01488f4e515d17ecb21d6cebc5a37bec5b0e56c +F src/log.c a3558ae5a48f808134db9b77a893141ba79402e7 F src/log.h a2654af46ce7b5732f4d5a731abfdd180f0a06d9 F src/main.c c0e7192bad5b90544508b241eb2487ac661de890 F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a @@ -473,12 +473,13 @@ F test/limit.test 2db7b3b34fb925b8e847d583d2eb67531d0ce67e F test/loadext.test 0393ce12d9616aa87597dd0ec88181de181f6db0 F test/loadext2.test 0bcaeb4d81cd5b6e883fdfea3c1bdbe1f173cbca F test/lock.test 842e80b6be816c79525a20b098cca066989feed7 -F test/lock2.test 7bb642551df59b3de135291d62ee82409420181e +F test/lock2.test ec208a5f394d92affaf599fde3f374361657d0ff F test/lock3.test f271375930711ae044080f4fe6d6eda930870d00 F test/lock4.test f4f36271aa5ae1da449646bf43c7341f6b2b4c4e F test/lock5.test 6b1f78f09ad1522843dad571b76b321e6f439bf7 F test/lock6.test 862aa71e97b288d6b3f92ba3313f51bd0b003776 F test/lock7.test 64006c84c1c616657e237c7ad6532b765611cf64 +F test/lock_common.tcl 58aa21f38c28223cc1107b5b2c9d7d61aa428e79 F test/lookaside.test 1dd350dc6dff015c47c07fcc5a727a72fc5bae02 F test/main.test 2be2352ac77ac5b238c6337a5469aeeef57677e6 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 @@ -756,7 +757,7 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test a4be3c7a36e3db3d9d276dc7664c2ed5eece1e8a +F test/wal.test 812dde0a689f69ce9b2d897ce4f08d752bd06749 F test/walcrash.test 45cfbab30bb7cbe0b2e9d5cabe90dbcad10cb89b F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 F test/walthread.test 58cd64b06f186251f09f64e4918fb74a7e52c963 @@ -803,7 +804,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P a9617eff39177250e2f118f25fdd4b3acb8b0478 -R 7008c4f8854b7fbb33a10340b4b449d8 +P 5e9dd3bd8e829376408925fb4cfcd5bb1eb1105f +R b62050501d5438f40040e5ea09ab63d7 U dan -Z d0e2fbdd9e82a46dc4aa039b5fc8e3ce +Z 3f45f30ce9e36f0f15e4d7966accef62 diff --git a/manifest.uuid b/manifest.uuid index 8fdc50e3bd..e7339388fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5e9dd3bd8e829376408925fb4cfcd5bb1eb1105f \ No newline at end of file +9435f3135849e0d38fde1669201db508561a6308 \ No newline at end of file diff --git a/src/log.c b/src/log.c index 9b25216df4..f1481bf92a 100644 --- a/src/log.c +++ b/src/log.c @@ -1100,7 +1100,7 @@ static int logLockRegion(Log *pLog, u32 mRegion, int op){ /* 1000 */ {1, 1}, /* 1001 */ {0, 0}, /* 1010 */ {0, 0}, /* 1011 */ {0, 0}, /* 1100 */ {1, 2}, /* 1101 */ {0, 0}, - /* 1110 */ {1, 3}, /* 1111 */ {0, 0} + /* 1110 */ {0, 0}, /* 1111 */ {0, 0} }; int rc; /* Return code of fcntl() */ struct flock f; /* Locking operation */ diff --git a/test/lock2.test b/test/lock2.test index a2b75ca311..63319535d1 100644 --- a/test/lock2.test +++ b/test/lock2.test @@ -16,69 +16,8 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +source $testdir/lock_common.tcl -# Launch another testfixture process to be controlled by this one. A -# channel name is returned that may be passed as the first argument to proc -# 'testfixture' to execute a command. The child testfixture process is shut -# down by closing the channel. -proc launch_testfixture {} { - set prg [info nameofexec] - if {$prg eq ""} { - set prg [file join . testfixture] - } - set chan [open "|$prg tf_main.tcl" r+] - fconfigure $chan -buffering line - return $chan -} - -# Execute a command in a child testfixture process, connected by two-way -# channel $chan. Return the result of the command, or an error message. -proc testfixture {chan cmd} { - puts $chan $cmd - puts $chan OVER - set r "" - while { 1 } { - set line [gets $chan] - if { $line == "OVER" } { - return $r - } - if {[eof $chan]} { - return "ERROR: Child process hung up" - } - append r $line - } -} - -# Write the main loop for the child testfixture processes into file -# tf_main.tcl. The parent (this script) interacts with the child processes -# via a two way pipe. The parent writes a script to the stdin of the child -# process, followed by the word "OVER" on a line of its own. The child -# process evaluates the script and writes the results to stdout, followed -# by an "OVER" of its own. -set f [open tf_main.tcl w] -puts $f { - set l [open log w] - set script "" - while {![eof stdin]} { - flush stdout - set line [gets stdin] - puts $l "READ $line" - if { $line == "OVER" } { - catch {eval $script} result - puts $result - puts $l "WRITE $result" - puts OVER - puts $l "WRITE OVER" - flush stdout - set script "" - } else { - append script $line - append script " ; " - } - } - close $l -} -close $f # Simple locking test case: # diff --git a/test/lock_common.tcl b/test/lock_common.tcl new file mode 100644 index 0000000000..31c04e8535 --- /dev/null +++ b/test/lock_common.tcl @@ -0,0 +1,77 @@ +# 2010 April 14 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file contains code used by several different test scripts. The +# code in this file allows testfixture to control another process (or +# processes) to test locking. +# + +# Launch another testfixture process to be controlled by this one. A +# channel name is returned that may be passed as the first argument to proc +# 'testfixture' to execute a command. The child testfixture process is shut +# down by closing the channel. +proc launch_testfixture {} { + set prg [info nameofexec] + if {$prg eq ""} { + set prg [file join . testfixture] + } + set chan [open "|$prg tf_main.tcl" r+] + fconfigure $chan -buffering line + return $chan +} + +# Execute a command in a child testfixture process, connected by two-way +# channel $chan. Return the result of the command, or an error message. +proc testfixture {chan cmd} { + puts $chan $cmd + puts $chan OVER + set r "" + while { 1 } { + set line [gets $chan] + if { $line == "OVER" } { + return $r + } + if {[eof $chan]} { + return "ERROR: Child process hung up" + } + append r $line + } +} + +# Write the main loop for the child testfixture processes into file +# tf_main.tcl. The parent (this script) interacts with the child processes +# via a two way pipe. The parent writes a script to the stdin of the child +# process, followed by the word "OVER" on a line of its own. The child +# process evaluates the script and writes the results to stdout, followed +# by an "OVER" of its own. +set f [open tf_main.tcl w] +puts $f { + set l [open log w] + set script "" + while {![eof stdin]} { + flush stdout + set line [gets stdin] + puts $l "READ $line" + if { $line == "OVER" } { + catch {eval $script} result + puts $result + puts $l "WRITE $result" + puts OVER + puts $l "WRITE OVER" + flush stdout + set script "" + } else { + append script $line + append script " ; " + } + } + close $l +} +close $f diff --git a/test/wal.test b/test/wal.test index cafda909a4..99a19fe400 100644 --- a/test/wal.test +++ b/test/wal.test @@ -15,28 +15,12 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl - -proc range {args} { - set ret [list] - foreach {start end} $args { - for {set i $start} {$i <= $end} {incr i} { - lappend ret $i - } - } - set ret -} +source $testdir/lock_common.tcl proc reopen_db {} { catch { db close } file delete -force test.db test.db-wal sqlite3_wal db test.db - #register_logtest -} -proc register_logtest {{db db}} { - register_logsummary_module $db - execsql { CREATE VIRTUAL TABLE temp.logsummary USING logsummary } $db - execsql { CREATE VIRTUAL TABLE temp.logcontent USING logcontent } $db - execsql { CREATE VIRTUAL TABLE temp.loglock USING loglock } $db } proc sqlite3_wal {args} { @@ -309,100 +293,186 @@ unset handle #------------------------------------------------------------------------- # The following block of tests - wal-10.* - test that the WAL locking -# scheme works for clients in a single process. +# scheme works in simple cases. This block of tests is run twice. Once +# using multiple connections in the address space of the current process, +# and once with all connections except one running in external processes. # -reopen_db -sqlite3_wal db2 test.db -sqlite3_wal db3 test.db +foreach code [list { + set ::code2_chan [launch_testfixture] + set ::code3_chan [launch_testfixture] + proc code2 {tcl} { testfixture $::code2_chan $tcl } + proc code3 {tcl} { testfixture $::code3_chan $tcl } + set tn 1 +} { + proc code2 {tcl} { uplevel #0 $tcl } + proc code3 {tcl} { uplevel #0 $tcl } + set tn 2 +}] { -do_test wal-10.1 { - execsql { - CREATE TABLE t1(a, b); - INSERT INTO t1 VALUES(1, 2); - BEGIN; - INSERT INTO t1 VALUES(3, 4); - } - execsql "SELECT * FROM t1" db2 -} {1 2} -do_test wal-10.2 { - execsql { COMMIT } - execsql "SELECT * FROM t1" db2 -} {1 2 3 4} -do_test wal-10.3 { - execsql { - BEGIN; + eval $code + reopen_db + + # Open connections [db2] and [db3]. Depending on which iteration this + # is, the connections may be created in this interpreter, or in + # interpreters running in other OS processes. As such, the [db2] and [db3] + # commands should only be accessed within [code2] and [code3] blocks, + # respectively. + # + code2 { sqlite3 db2 test.db ; db2 eval { PRAGMA journal_mode = WAL } } + code3 { sqlite3 db3 test.db ; db3 eval { PRAGMA journal_mode = WAL } } + + # Shorthand commands. Execute SQL using database connection [db2] or + # [db3]. Return the results. + # + proc sql2 {sql} { code2 [list db2 eval $sql] } + proc sql3 {sql} { code3 [list db3 eval $sql] } + + # Initialize the database schema and contents. + # + do_test wal-10.$tn.1 { + execsql { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); SELECT * FROM t1; - } db2 -} {1 2 3 4} -do_test wal-10.4 { - catchsql { PRAGMA checkpoint } -} {1 {database is locked}} -do_test wal-10.5 { - execsql { INSERT INTO t1 VALUES(5, 6) } - execsql { SELECT * FROM t1 } db2 -} {1 2 3 4} + } + } {1 2} -# Connection [db2] is holding a lock on a snapshot, preventing [db] from -# checkpointing the database. Add a busy-handler to [db]. If [db2] completes -# its transaction from within the busy-handler, [db] is able to complete -# the checkpoint operation. -# -proc busyhandler x { - if {$x==4} { - execsql { COMMIT } db2 + # Open a transaction and write to the database using [db]. Check that [db2] + # is still able to read the snapshot before the transaction was opened. + # + do_test wal-10.$tn.2 { + execsql { BEGIN; INSERT INTO t1 VALUES(3, 4); } + sql2 {SELECT * FROM t1} + } {1 2} + + # Have [db] commit the transaction. Check that [db2] is now seeing the + # new, updated snapshot. + # + do_test wal-10.$tn.3 { + execsql { COMMIT } + sql2 {SELECT * FROM t1} + } {1 2 3 4} + + # Have [db2] open a read transaction. Then write to the db via [db]. Check + # that [db2] is still seeing the original snapshot. Then read with [db3]. + # [db3] should see the newly committed data. + # + do_test wal-10.$tn.4 { + sql2 { BEGIN ; SELECT * FROM t1} + } {1 2 3 4} + do_test wal-10.$tn.5 { + execsql { INSERT INTO t1 VALUES(5, 6); } + sql2 {SELECT * FROM t1} + } {1 2 3 4} + do_test wal-10.$tn.6 { + sql3 {SELECT * FROM t1} + } {1 2 3 4 5 6} + do_test wal-10.$tn.7 { + sql2 COMMIT + } {} + + # Have [db2] open a write transaction. Then attempt to write to the + # database via [db]. This should fail (writer lock cannot be obtained). + # + # Then open a read-transaction with [db]. Commit the [db2] transaction + # to disk. Verify that [db] still cannot write to the database (because + # it is reading an old snapshot). + # + # Close the current [db] transaction. Open a new one. [db] can now write + # to the database (as it is not locked and [db] is reading the latest + # snapshot). + # + do_test wal-10.$tn.7 { + sql2 { BEGIN; INSERT INTO t1 VALUES(7, 8) ; } + catchsql { INSERT INTO t1 VALUES(9, 10) } + } {1 {database is locked}} + do_test wal-10.$tn.8 { + execsql { BEGIN ; SELECT * FROM t1 } + } {1 2 3 4 5 6} + do_test wal-10.$tn.9 { + sql2 COMMIT + catchsql { INSERT INTO t1 VALUES(9, 10) } + } {1 {database is locked}} + do_test wal-10.$tn.10 { + execsql { COMMIT; BEGIN; INSERT INTO t1 VALUES(9, 10); COMMIT; } + execsql { SELECT * FROM t1 } + } {1 2 3 4 5 6 7 8 9 10} + + # Open a read transaction with [db2]. Check that this prevents [db] from + # checkpointing the database. But not from writing to it. + # + do_test wal-10.$tn.11 { + sql2 { BEGIN; SELECT * FROM t1 } + } {1 2 3 4 5 6 7 8 9 10} + do_test wal-10.$tn.12 { + catchsql { PRAGMA checkpoint } + } {1 {database is locked}} + do_test wal-10.$tn.13 { + execsql { INSERT INTO t1 VALUES(11, 12) } + sql2 {SELECT * FROM t1} + } {1 2 3 4 5 6 7 8 9 10} + + # Connection [db2] is holding a lock on a snapshot, preventing [db] from + # checkpointing the database. Add a busy-handler to [db]. If [db2] completes + # its transaction from within the busy-handler, [db] is able to complete + # the checkpoint operation. + # + proc busyhandler x { + if {$x==4} { sql2 COMMIT } + if {$x<5} { return 0 } + return 1 } - if {$x<5} {return 0} - return 1 -} -db busy busyhandler -do_test wal-10.6 { - execsql { PRAGMA checkpoint } -} {} + db busy busyhandler + do_test wal-10.$tn.14 { + execsql { PRAGMA checkpoint } + } {} -# Similar to the test above. Except this time, a new read transaction is -# started (db3) while the checkpointer is waiting for an old one to finish. -# The checkpointer can finish, but any subsequent write operations must -# wait until after db3 has closed the read transaction. -# -db busy {} -do_test wal-10.7 { - execsql { - BEGIN; - SELECT * FROM t1; - } db2 -} {1 2 3 4 5 6} -do_test wal-10.8 { - execsql { INSERT INTO t1 VALUES(7, 8) } - catchsql { PRAGMA checkpoint } -} {1 {database is locked}} -proc busyhandler x { - if {$x==3} { execsql { BEGIN; SELECT * FROM t1 } db3 } - if {$x==4} { execsql { COMMIT } db2 } - if {$x<5} { return 0 } - return 1 -} -db busy busyhandler -do_test wal-10.9 { - execsql { PRAGMA checkpoint } -} {} -do_test wal-10.10 { - execsql { SELECT * FROM t1 } db3 -} {1 2 3 4 5 6 7 8} -do_test wal-10.11 { - catchsql { INSERT INTO t1 VALUES(9, 10) } -} {1 {database is locked}} -do_test wal-10.12 { - execsql { SELECT * FROM t1 } -} {1 2 3 4 5 6 7 8} -do_test wal-10.13 { - execsql { COMMIT } db3 -} {} -do_test wal-10.14 { - execsql { INSERT INTO t1 VALUES(9, 10) } - execsql { SELECT * FROM t1 } -} {1 2 3 4 5 6 7 8 9 10} + # Similar to the test above. Except this time, a new read transaction is + # started (db3) while the checkpointer is waiting for an old one (db2) to + # finish. The checkpointer can finish, but any subsequent write operations + # must wait until after db3 has closed the read transaction, as db3 is a + # "region D" writer. + # + db busy {} + do_test wal-10.$tn.15 { + sql2 { BEGIN; SELECT * FROM t1; } + } {1 2 3 4 5 6 7 8 9 10 11 12} + do_test wal-10.$tn.16 { + catchsql { PRAGMA checkpoint } + } {1 {database is locked}} + proc busyhandler x { + if {$x==3} { sql3 { BEGIN; SELECT * FROM t1 } } + if {$x==4} { sql2 COMMIT } + if {$x<5} { return 0 } + return 1 + } + db busy busyhandler + do_test wal-10.$tn.9 { + execsql { PRAGMA checkpoint } + } {} + do_test wal-10.$tn.10 { + sql3 { SELECT * FROM t1 } + } {1 2 3 4 5 6 7 8 9 10 11 12} + do_test wal-10.$tn.11 { + catchsql { INSERT INTO t1 VALUES(13, 14) } + } {1 {database is locked}} + do_test wal-10.$tn.12 { + execsql { SELECT * FROM t1 } + } {1 2 3 4 5 6 7 8 9 10 11 12} + do_test wal-10.$tn.13 { + sql3 COMMIT + } {} + do_test wal-10.$tn.14 { + execsql { INSERT INTO t1 VALUES(13, 14) } + execsql { SELECT * FROM t1 } + } {1 2 3 4 5 6 7 8 9 10 11 12 13 14} + + catch { db close } + catch { code2 { db2 close } } + catch { code3 { db3 close } } + catch { close $::code2_chan } + catch { close $::code3_chan } +} -foreach handle {db db2 db3} { catch { $handle close } } -unset handle finish_test From 49320f8568e71fb2a8796d14233256e69b67095b Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 14 Apr 2010 18:50:08 +0000 Subject: [PATCH 13/89] Add tests and fix bugs in WAL locking mechanism. FossilOrigin-Name: c18077f2465fc34830f11c9832e76be5746eaeea --- manifest | 14 +++++------ manifest.uuid | 2 +- src/log.c | 28 ++++++++++++++------- test/wal.test | 69 ++++++++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 90 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 92200a9b4f..af8652faeb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sto\scheck\sinter-process\sWAL\slocking. -D 2010-04-14T18:06:51 +C Add\stests\sand\sfix\sbugs\sin\sWAL\slocking\smechanism. +D 2010-04-14T18:50:08 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c a3558ae5a48f808134db9b77a893141ba79402e7 +F src/log.c 165addfd51fa9581936ec04914b17a3b274c49bd F src/log.h a2654af46ce7b5732f4d5a731abfdd180f0a06d9 F src/main.c c0e7192bad5b90544508b241eb2487ac661de890 F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a @@ -757,7 +757,7 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test 812dde0a689f69ce9b2d897ce4f08d752bd06749 +F test/wal.test 8f480128b140e54253684bc395f7af0254dc4e03 F test/walcrash.test 45cfbab30bb7cbe0b2e9d5cabe90dbcad10cb89b F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 F test/walthread.test 58cd64b06f186251f09f64e4918fb74a7e52c963 @@ -804,7 +804,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 5e9dd3bd8e829376408925fb4cfcd5bb1eb1105f -R b62050501d5438f40040e5ea09ab63d7 +P 9435f3135849e0d38fde1669201db508561a6308 +R 271de1f1c02e1bce44bf2011bef4ef6b U dan -Z 3f45f30ce9e36f0f15e4d7966accef62 +Z 6918d2483c611afa56b459efd78f482c diff --git a/manifest.uuid b/manifest.uuid index e7339388fa..6a496e5b8e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9435f3135849e0d38fde1669201db508561a6308 \ No newline at end of file +c18077f2465fc34830f11c9832e76be5746eaeea \ No newline at end of file diff --git a/src/log.c b/src/log.c index f1481bf92a..30d9d8ded6 100644 --- a/src/log.c +++ b/src/log.c @@ -1004,6 +1004,7 @@ static int logLockRegion(Log *pLog, u32 mRegion, int op){ /* Region D reader lock operations */ || (op==LOG_RDLOCK && mRegion==(LOG_REGION_D)) + || (op==LOG_RDLOCK && mRegion==(LOG_REGION_A)) || (op==LOG_UNLOCK && mRegion==(LOG_REGION_D)) /* Checkpointer lock operations */ @@ -1334,12 +1335,9 @@ void sqlite3LogMaxpgno(Log *pLog, Pgno *pPgno){ } /* -** The caller must hold at least a RESERVED lock on the database file -** when invoking this function. -** ** This function returns SQLITE_OK if the caller may write to the database. ** Otherwise, if the caller is operating on a snapshot that has already -** been overwritten by another writer, SQLITE_OBE is returned. +** been overwritten by another writer, SQLITE_BUSY is returned. */ int sqlite3LogWriteLock(Log *pLog, int op){ assert( pLog->isLocked ); @@ -1351,14 +1349,26 @@ int sqlite3LogWriteLock(Log *pLog, int op){ return rc; } - /* TODO: What if this is a region D reader? And after writing this - ** transaction it continues to hold a read-lock on the db? Maybe we - ** need to switch it to a region A reader here so that unlocking C|D - ** does not leave the connection with no lock at all. + /* If this is connection is a region D, then the SHARED lock on region + ** D has just been upgraded to EXCLUSIVE. But no lock at all is held on + ** region A. This means that if the write-transaction is committed + ** and this connection downgrades to a reader, it will be left with no + ** lock at all. And its snapshot could get clobbered by a checkpoint + ** operation. + ** + ** To stop this from happening, grab a SHARED lock on region A now. + ** This should always be successful, as the only time a client holds + ** an EXCLUSIVE lock on region A, it must also be holding an EXCLUSIVE + ** lock on region C (a checkpointer does this). This is not possible, + ** as this connection currently has the EXCLUSIVE lock on region C. */ - assert( pLog->isLocked!=LOG_REGION_D ); + if( pLog->isLocked==LOG_REGION_D ){ + logLockRegion(pLog, LOG_REGION_A, LOG_RDLOCK); + pLog->isLocked = LOG_REGION_A; + } if( memcmp(&pLog->hdr, pLog->pSummary->aData, sizeof(pLog->hdr)) ){ + logLockRegion(pLog, LOG_REGION_C|LOG_REGION_D, LOG_UNLOCK); return SQLITE_BUSY; } pLog->isWriteLocked = 1; diff --git a/test/wal.test b/test/wal.test index 99a19fe400..9ac00be677 100644 --- a/test/wal.test +++ b/test/wal.test @@ -447,26 +447,83 @@ foreach code [list { return 1 } db busy busyhandler - do_test wal-10.$tn.9 { + do_test wal-10.$tn.17 { execsql { PRAGMA checkpoint } } {} - do_test wal-10.$tn.10 { + do_test wal-10.$tn.18 { sql3 { SELECT * FROM t1 } } {1 2 3 4 5 6 7 8 9 10 11 12} - do_test wal-10.$tn.11 { + do_test wal-10.$tn.19 { catchsql { INSERT INTO t1 VALUES(13, 14) } } {1 {database is locked}} - do_test wal-10.$tn.12 { + do_test wal-10.$tn.20 { execsql { SELECT * FROM t1 } } {1 2 3 4 5 6 7 8 9 10 11 12} - do_test wal-10.$tn.13 { + do_test wal-10.$tn.21 { sql3 COMMIT } {} - do_test wal-10.$tn.14 { + do_test wal-10.$tn.22 { execsql { INSERT INTO t1 VALUES(13, 14) } execsql { SELECT * FROM t1 } } {1 2 3 4 5 6 7 8 9 10 11 12 13 14} + # Set [db3] up as a "region D" reader again. Then upgrade it to a writer + # and back down to a reader. Then, check that a checkpoint is not possible + # (as [db3] still has a snapshot locked). + # + do_test wal-10.$tn.23 { + execsql { PRAGMA checkpoint } + } {} + do_test wal-10.$tn.24 { + sql2 { BEGIN; SELECT * FROM t1; } + } {1 2 3 4 5 6 7 8 9 10 11 12 13 14} + do_test wal-10.$tn.25 { + execsql { PRAGMA checkpoint } + } {} + do_test wal-10.$tn.26 { + catchsql { INSERT INTO t1 VALUES(15, 16) } + } {1 {database is locked}} + do_test wal-10.$tn.27 { + sql3 { INSERT INTO t1 VALUES(15, 16) } + } {} + do_test wal-10.$tn.28 { + code3 { + set ::STMT [sqlite3_prepare db3 "SELECT * FROM t1" -1 TAIL] + sqlite3_step $::STMT + } + sql3 COMMIT + execsql { SELECT * FROM t1 } + } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16} + db busy {} + do_test wal-10.$tn.29 { + execsql { INSERT INTO t1 VALUES(17, 18) } + catchsql { PRAGMA checkpoint } + } {1 {database is locked}} + do_test wal-10.$tn.30 { + code3 { sqlite3_finalize $::STMT } + execsql { PRAGMA checkpoint } + } {} + + # At one point, if a reader failed to upgrade to a writer because it + # was reading an old snapshot, the write-locks were not being released. + # Test that this bug has been fixed. + # + do_test wal-10.$tn.31 { + execsql { BEGIN ; SELECT * FROM t1 } + sql2 { INSERT INTO t1 VALUES(19, 20) } + catchsql { INSERT INTO t1 VALUES(21, 22) } + } {1 {database is locked}} + do_test wal-10.$tn.32 { + # This statement would fail when the bug was present. + sql2 { INSERT INTO t1 VALUES(21, 22) } + } {} + do_test wal-10.$tn.33 { + execsql { SELECT * FROM t1 ; COMMIT } + } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18} + do_test wal-10.$tn.34 { + execsql { SELECT * FROM t1 } + } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22} + catch { db close } catch { code2 { db2 close } } catch { code3 { db3 close } } From 39c79f587cfe6cb423099fa89e8b169a71b5842a Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 15 Apr 2010 10:58:51 +0000 Subject: [PATCH 14/89] Change the way checksums are calculated. FossilOrigin-Name: 84955c2e9ce526c5a3ed479aa09f093a7e37c7d0 --- manifest | 24 +++++++----------------- manifest.uuid | 2 +- src/log.c | 44 ++++++++++++++++++++++++-------------------- 3 files changed, 32 insertions(+), 38 deletions(-) diff --git a/manifest b/manifest index d7f5bd5c6c..4dcbaf750e 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,5 @@ ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA1 - -C Bring\sover\sthe\srecent\squery\splanner\senhancements\sfrom\sthe\strunk. -D 2010-04-15T02:37:11 +C Change\sthe\sway\schecksums\sare\scalculated. +D 2010-04-15T10:58:52 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -134,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c 165addfd51fa9581936ec04914b17a3b274c49bd +F src/log.c bf7ce56249b2a08d26b4029430c32a01efc4a64e F src/log.h a2654af46ce7b5732f4d5a731abfdd180f0a06d9 F src/main.c c0e7192bad5b90544508b241eb2487ac661de890 F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a @@ -807,14 +804,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P c18077f2465fc34830f11c9832e76be5746eaeea defaf0d99a807027f8883bf821b6482025f9f54e -R 5f5abc6d19fbeba7fb34263a5662faa7 -U drh -Z 9dfe9b346dd22ca03d0d17a2ddbb8a3c ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.6 (GNU/Linux) - -iD8DBQFLxnvaoxKgR168RlERAkpHAJ9umCCf7Sfy2xa1T3mJ1bqzrPY7tgCgjEA2 -oN5S7m+hWn0GMbvXKOIs1cY= -=ADT1 ------END PGP SIGNATURE----- +P 82969f27e5ea843cb379666d8a02e4a3fddc03b2 +R b356f412a9c351edeb3f07a632ecce9f +U dan +Z 01a21dba5524ff578c531f4245d5f626 diff --git a/manifest.uuid b/manifest.uuid index 232556f3c5..c8288d828e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -82969f27e5ea843cb379666d8a02e4a3fddc03b2 \ No newline at end of file +84955c2e9ce526c5a3ed479aa09f093a7e37c7d0 \ No newline at end of file diff --git a/src/log.c b/src/log.c index 30d9d8ded6..112b907f4c 100644 --- a/src/log.c +++ b/src/log.c @@ -141,23 +141,21 @@ static LogSummary *pLogSummary = 0; */ #define LOG_CKSM_BYTES 8 static void logChecksumBytes(u8 *aByte, int nByte, u32 *aCksum){ - u32 *z32 = (u32 *)aByte; - int n32 = nByte / sizeof(u32); - int i; + u64 sum1 = aCksum[0]; + u64 sum2 = aCksum[1]; + u32 *a32 = (u32 *)aByte; + u32 *aEnd = (u32 *)&aByte[nByte]; assert( LOG_CKSM_BYTES==2*sizeof(u32) ); assert( (nByte&0x00000003)==0 ); - u32 cksum0 = aCksum[0]; - u32 cksum1 = aCksum[1]; + do { + sum1 += (*a32++); + sum2 += sum1; + } while( a32> 8) + (cksum0 ^ z32[i]); - cksum1 = (cksum1 >> 8) + (cksum1 ^ z32[i]); - } - - aCksum[0] = cksum0; - aCksum[1] = cksum1; + aCksum[0] = sum1 + (sum1>>24); + aCksum[1] = sum2 + (sum2>>24); } /* @@ -1259,8 +1257,6 @@ void sqlite3LogCloseSnapshot(Log *pLog){ pLog->isLocked = 0; } - - /* ** Read a page from the log, if it is present. */ @@ -1269,6 +1265,8 @@ int sqlite3LogRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut){ u32 *aData = pLog->pSummary->aData; int iFrame = (pLog->hdr.iLastPg & 0xFFFFFF00); + assert( pLog->isLocked ); + /* Do a linear search of the unindexed block of page-numbers (if any) ** at the end of the log-summary. An alternative to this would be to ** build an index in private memory each time a read transaction is @@ -1349,11 +1347,11 @@ int sqlite3LogWriteLock(Log *pLog, int op){ return rc; } - /* If this is connection is a region D, then the SHARED lock on region - ** D has just been upgraded to EXCLUSIVE. But no lock at all is held on - ** region A. This means that if the write-transaction is committed + /* If this is connection is a region D reader, then the SHARED lock on + ** region D has just been upgraded to EXCLUSIVE. But no lock at all is + ** held on region A. This means that if the write-transaction is committed ** and this connection downgrades to a reader, it will be left with no - ** lock at all. And its snapshot could get clobbered by a checkpoint + ** lock at all. And so its snapshot could get clobbered by a checkpoint ** operation. ** ** To stop this from happening, grab a SHARED lock on region A now. @@ -1367,6 +1365,10 @@ int sqlite3LogWriteLock(Log *pLog, int op){ pLog->isLocked = LOG_REGION_A; } + /* If this connection is not reading the most recent database snapshot, + ** it is not possible to write to the database. In this case release + ** the write locks and return SQLITE_BUSY. + */ if( memcmp(&pLog->hdr, pLog->pSummary->aData, sizeof(pLog->hdr)) ){ logLockRegion(pLog, LOG_REGION_C|LOG_REGION_D, LOG_UNLOCK); return SQLITE_BUSY; @@ -1535,13 +1537,15 @@ int sqlite3LogCheckpoint( ){ int rc; /* Return code */ - /* Wait for a write-lock on regions B and C. */ + assert( !pLog->isLocked ); + + /* Wait for an EXCLUSIVE lock on regions B and C. */ do { rc = logLockRegion(pLog, LOG_REGION_B|LOG_REGION_C, LOG_WRLOCK); }while( rc==SQLITE_BUSY && xBusyHandler(pBusyHandlerArg) ); if( rc!=SQLITE_OK ) return rc; - /* Wait for a write-lock on region A. */ + /* Wait for an EXCLUSIVE lock on region A. */ do { rc = logLockRegion(pLog, LOG_REGION_A, LOG_WRLOCK); }while( rc==SQLITE_BUSY && xBusyHandler(pBusyHandlerArg) ); From 521a72b1fac223743da8489a08dbcb7402c96feb Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 15 Apr 2010 12:36:06 +0000 Subject: [PATCH 15/89] Fix a problem in the result set size estimation logic of the query planner - a problem introduced by the two previous changes. FossilOrigin-Name: 33b1f584ef712625c4df8e2aefe895fa89f6a795 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/where.c | 21 ++++++++++++++++----- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index d7f5bd5c6c..1f60425958 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 -C Bring\sover\sthe\srecent\squery\splanner\senhancements\sfrom\sthe\strunk. -D 2010-04-15T02:37:11 +C Fix\sa\sproblem\sin\sthe\sresult\sset\ssize\sestimation\slogic\sof\sthe\squery\splanner\s-\s\na\sproblem\sintroduced\sby\sthe\stwo\sprevious\schanges. +D 2010-04-15T12:36:07 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -227,7 +227,7 @@ F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f -F src/where.c 74ff6f0d96b3190f8abe2ceaa09858441749e8ff +F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 F test/all.test 14165b3e32715b700b5f0cbf8f6e3833dda0be45 @@ -807,14 +807,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P c18077f2465fc34830f11c9832e76be5746eaeea defaf0d99a807027f8883bf821b6482025f9f54e -R 5f5abc6d19fbeba7fb34263a5662faa7 +P 82969f27e5ea843cb379666d8a02e4a3fddc03b2 +R 351077133ab404e2e491a33b905acb9d U drh -Z 9dfe9b346dd22ca03d0d17a2ddbb8a3c +Z 2a5a2f50be97ee24579935014cd533e4 -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.6 (GNU/Linux) -iD8DBQFLxnvaoxKgR168RlERAkpHAJ9umCCf7Sfy2xa1T3mJ1bqzrPY7tgCgjEA2 -oN5S7m+hWn0GMbvXKOIs1cY= -=ADT1 +iD8DBQFLxwg6oxKgR168RlERAn1aAJ96LitidK5axyWqFd8S2OYRbsKviACdE9xg +/OWvfn581zFJtn2LS8VJjjA= +=qS5Y -----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index 232556f3c5..a4611cbd35 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -82969f27e5ea843cb379666d8a02e4a3fddc03b2 \ No newline at end of file +33b1f584ef712625c4df8e2aefe895fa89f6a795 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 77ecdfef46..e13ca51d03 100644 --- a/src/where.c +++ b/src/where.c @@ -2737,12 +2737,23 @@ static void bestBtreeIndex( ** matters if the current index is the least costly, so do not bother ** with this step if we already know this index will not be chosen. ** Also, never reduce the output row count below 2 using this step. + ** + ** Do not reduce the output row count if pSrc is the only table that + ** is notReady; if notReady is a power of two. This will be the case + ** when the main sqlite3WhereBegin() loop is scanning for a table with + ** and "optimal" index, and on such a scan the output row count + ** reduction is not valid because it does not update the "pCost->used" + ** bitmap. The notReady bitmap will also be a power of two when we + ** are scanning for the last table in a 64-way join. We are willing + ** to bypass this optimization in that corner case. */ - if( nRow>2 && cost<=pCost->rCost ){ - int k; - int nSkipEq = nEq; - int nSkipRange = nBound; - Bitmask thisTab = getMask(pWC->pMaskSet, iCur); + if( nRow>2 && cost<=pCost->rCost && (notReady & (notReady-1))!=0 ){ + int k; /* Loop counter */ + int nSkipEq = nEq; /* Number of == constraints to skip */ + int nSkipRange = nBound; /* Number of < constraints to skip */ + Bitmask thisTab; /* Bitmap for pSrc */ + + thisTab = getMask(pWC->pMaskSet, iCur); for(pTerm=pWC->a, k=pWC->nTerm; nRow>2 && k; k--, pTerm++){ if( pTerm->wtFlags & TERM_VIRTUAL ) continue; if( (pTerm->prereqAll & notReady)!=thisTab ) continue; From 4cc6fb6165087eed723409ed70c440c528888ffc Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 15 Apr 2010 16:45:34 +0000 Subject: [PATCH 16/89] Allow writers to write dirty pages to the log mid-transaction in order to free memory. FossilOrigin-Name: ecd828f96909895535d7dc744e5a8530e234e04d --- manifest | 14 ++--- manifest.uuid | 2 +- src/pager.c | 140 ++++++++++++++++++++++++++------------------------ test/wal.test | 90 ++++++++++++++++++++++++++++++++ 4 files changed, 170 insertions(+), 76 deletions(-) diff --git a/manifest b/manifest index 2132014ce6..f04ac19026 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\stwo\sleaves\son\sthe\sWAL\sbranch. -D 2010-04-15T13:33:18 +C Allow\swriters\sto\swrite\sdirty\spages\sto\sthe\slog\smid-transaction\sin\sorder\sto\sfree\smemory. +D 2010-04-15T16:45:35 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -154,7 +154,7 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053 -F src/pager.c 9e9ee38c923fd225d73127751b7959bd826d0686 +F src/pager.c 80054194c18631342f6b8253e0bb090020c0cd55 F src/pager.h ce5d076f3860a5f2d7460c582cd68383343b33cf F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf @@ -757,7 +757,7 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test 8f480128b140e54253684bc395f7af0254dc4e03 +F test/wal.test b0b6c02bbb0b2a647b4edb69325d4c59ac73259b F test/walcrash.test 45cfbab30bb7cbe0b2e9d5cabe90dbcad10cb89b F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 F test/walthread.test 58cd64b06f186251f09f64e4918fb74a7e52c963 @@ -804,7 +804,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 84955c2e9ce526c5a3ed479aa09f093a7e37c7d0 33b1f584ef712625c4df8e2aefe895fa89f6a795 -R bcaf3c70b81f92c4b906909c026342d3 +P c9ed66cc3994b3b0d67a6c950b552a1a869f2ed2 +R 0dae17f227f2dd17e65e49bd0f5ce0a8 U dan -Z 3782bee1cb14af26018fee50d035e5d0 +Z b4af52effa56ad5510bfc4469b8a847d diff --git a/manifest.uuid b/manifest.uuid index a18185b651..b8f7810b58 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c9ed66cc3994b3b0d67a6c950b552a1a869f2ed2 \ No newline at end of file +ecd828f96909895535d7dc744e5a8530e234e04d \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index cd310ab887..cd83fdb72c 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3253,76 +3253,80 @@ static int pagerStress(void *p, PgHdr *pPg){ assert( pPg->pPager==pPager ); assert( pPg->flags&PGHDR_DIRTY ); - if( pagerUseLog(pPager) ) return SQLITE_OK; - - /* The doNotSync flag is set by the sqlite3PagerWrite() function while it - ** is journalling a set of two or more database pages that are stored - ** on the same disk sector. Syncing the journal is not allowed while - ** this is happening as it is important that all members of such a - ** set of pages are synced to disk together. So, if the page this function - ** is trying to make clean will require a journal sync and the doNotSync - ** flag is set, return without doing anything. The pcache layer will - ** just have to go ahead and allocate a new page buffer instead of - ** reusing pPg. - ** - ** Similarly, if the pager has already entered the error state, do not - ** try to write the contents of pPg to disk. - */ - if( NEVER(pPager->errCode) - || (pPager->doNotSync && pPg->flags&PGHDR_NEED_SYNC) - ){ - return SQLITE_OK; - } - - /* Sync the journal file if required. */ - if( pPg->flags&PGHDR_NEED_SYNC ){ - rc = syncJournal(pPager); - if( rc==SQLITE_OK && pPager->fullSync && - !(pPager->journalMode==PAGER_JOURNALMODE_MEMORY) && - !(sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) + if( pagerUseLog(pPager) ){ + /* Write a single frame for this page to the log. */ + assert( pPg->pDirty==0 ); + rc = sqlite3LogFrames(pPager->pLog, pPager->pageSize, pPg, 0, 0, 0); + }else{ + /* The doNotSync flag is set by the sqlite3PagerWrite() function while it + ** is journalling a set of two or more database pages that are stored + ** on the same disk sector. Syncing the journal is not allowed while + ** this is happening as it is important that all members of such a + ** set of pages are synced to disk together. So, if the page this function + ** is trying to make clean will require a journal sync and the doNotSync + ** flag is set, return without doing anything. The pcache layer will + ** just have to go ahead and allocate a new page buffer instead of + ** reusing pPg. + ** + ** Similarly, if the pager has already entered the error state, do not + ** try to write the contents of pPg to disk. + */ + if( NEVER(pPager->errCode) + || (pPager->doNotSync && pPg->flags&PGHDR_NEED_SYNC) ){ - pPager->nRec = 0; - rc = writeJournalHdr(pPager); + return SQLITE_OK; + } + + /* Sync the journal file if required. */ + if( pPg->flags&PGHDR_NEED_SYNC ){ + rc = syncJournal(pPager); + if( rc==SQLITE_OK && pPager->fullSync && + !(pPager->journalMode==PAGER_JOURNALMODE_MEMORY) && + !(sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) + ){ + pPager->nRec = 0; + rc = writeJournalHdr(pPager); + } + } + + /* If the page number of this page is larger than the current size of + ** the database image, it may need to be written to the sub-journal. + ** This is because the call to pager_write_pagelist() below will not + ** actually write data to the file in this case. + ** + ** Consider the following sequence of events: + ** + ** BEGIN; + ** + ** + ** SAVEPOINT sp; + ** + ** pagerStress(page X) + ** ROLLBACK TO sp; + ** + ** If (X>Y), then when pagerStress is called page X will not be written + ** out to the database file, but will be dropped from the cache. Then, + ** following the "ROLLBACK TO sp" statement, reading page X will read + ** data from the database file. This will be the copy of page X as it + ** was when the transaction started, not as it was when "SAVEPOINT sp" + ** was executed. + ** + ** The solution is to write the current data for page X into the + ** sub-journal file now (if it is not already there), so that it will + ** be restored to its current value when the "ROLLBACK TO sp" is + ** executed. + */ + if( NEVER( + rc==SQLITE_OK && pPg->pgno>pPager->dbSize && subjRequiresPage(pPg) + ) ){ + rc = subjournalPage(pPg); + } + + /* Write the contents of the page out to the database file. */ + if( rc==SQLITE_OK ){ + pPg->pDirty = 0; + rc = pager_write_pagelist(pPg); } - } - - /* If the page number of this page is larger than the current size of - ** the database image, it may need to be written to the sub-journal. - ** This is because the call to pager_write_pagelist() below will not - ** actually write data to the file in this case. - ** - ** Consider the following sequence of events: - ** - ** BEGIN; - ** - ** - ** SAVEPOINT sp; - ** - ** pagerStress(page X) - ** ROLLBACK TO sp; - ** - ** If (X>Y), then when pagerStress is called page X will not be written - ** out to the database file, but will be dropped from the cache. Then, - ** following the "ROLLBACK TO sp" statement, reading page X will read - ** data from the database file. This will be the copy of page X as it - ** was when the transaction started, not as it was when "SAVEPOINT sp" - ** was executed. - ** - ** The solution is to write the current data for page X into the - ** sub-journal file now (if it is not already there), so that it will - ** be restored to its current value when the "ROLLBACK TO sp" is - ** executed. - */ - if( NEVER( - rc==SQLITE_OK && pPg->pgno>pPager->dbSize && subjRequiresPage(pPg) - ) ){ - rc = subjournalPage(pPg); - } - - /* Write the contents of the page out to the database file. */ - if( rc==SQLITE_OK ){ - pPg->pDirty = 0; - rc = pager_write_pagelist(pPg); } /* Mark the page as clean. */ diff --git a/test/wal.test b/test/wal.test index 9ac00be677..799e49dc81 100644 --- a/test/wal.test +++ b/test/wal.test @@ -531,5 +531,95 @@ foreach code [list { catch { close $::code3_chan } } +#------------------------------------------------------------------------- +# This block of tests, wal-11.*, test that nothing goes terribly wrong +# if frames must be written to the log file before a transaction is +# committed (in order to free up memory). +# +do_test wal-11.1 { + reopen_db + execsql { + PRAGMA cache_size = 10; + PRAGMA page_size = 1024; + CREATE TABLE t1(x PRIMARY KEY); + } + list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] +} {0 3} +do_test wal-11.2 { + execsql { PRAGMA checkpoint } + list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] +} {3 3} +do_test wal-11.3 { + execsql { INSERT INTO t1 VALUES( randomblob(900) ) } + list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] +} {3 4} + +do_test wal-11.4 { + execsql { + BEGIN; + INSERT INTO t1 SELECT randomblob(900) FROM t1; -- 2 + INSERT INTO t1 SELECT randomblob(900) FROM t1; -- 4 + INSERT INTO t1 SELECT randomblob(900) FROM t1; -- 8 + INSERT INTO t1 SELECT randomblob(900) FROM t1; -- 16 + } + list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] +} {3 33} +do_test wal-11.5 { + execsql { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } +} {16 ok} +do_test wal-11.6 { + execsql COMMIT + list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] +} {3 42} +do_test wal-11.7 { + execsql { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } +} {16 ok} +do_test wal-11.8 { + execsql { PRAGMA checkpoint } + list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] +} {37 42} +do_test wal-11.9 { + db close + sqlite3_wal db test.db + list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] +} {37 0} + +do_test wal-11.10 { + execsql { + PRAGMA cache_size = 10; + BEGIN; + INSERT INTO t1 SELECT randomblob(900) FROM t1; -- 32 + SELECT count(*) FROM t1; + } + list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] +} {37 38} +do_test wal-11.11 { + execsql { + SELECT count(*) FROM t1; + ROLLBACK; + SELECT count(*) FROM t1; + } +} {32 16} +do_test wal-11.12 { + list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] +} {37 38} +do_test wal-11.13 { + execsql { + INSERT INTO t1 VALUES( randomblob(900) ); + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } +} {17 ok} +do_test wal-11.14 { + list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] +} {37 38} + + finish_test From 4a4b01dc1cd4265ca06e371176a7241fa528cfab Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 16 Apr 2010 11:30:18 +0000 Subject: [PATCH 17/89] Fix bug in log recovery (last frame in log was being ignored). Also remove an incorrect assert statement. FossilOrigin-Name: 67d2a89ec2d593a077eb19a6ea2b06cb1c2e9ba8 --- manifest | 16 +++++++------- manifest.uuid | 2 +- src/log.c | 40 ++++++++++++++++++----------------- src/pager.c | 3 +-- test/wal.test | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 89 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index f04ac19026..547b763339 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\swriters\sto\swrite\sdirty\spages\sto\sthe\slog\smid-transaction\sin\sorder\sto\sfree\smemory. -D 2010-04-15T16:45:35 +C Fix\sbug\sin\slog\srecovery\s(last\sframe\sin\slog\swas\sbeing\signored).\sAlso\sremove\san\sincorrect\sassert\sstatement. +D 2010-04-16T11:30:18 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c 3647edbc85ac745db7f18bc1102344aa2ee5259a +F src/log.c 40e0c5780e7a33556f421142562edbf77f8e6639 F src/log.h a2654af46ce7b5732f4d5a731abfdd180f0a06d9 F src/main.c c0e7192bad5b90544508b241eb2487ac661de890 F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a @@ -154,7 +154,7 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053 -F src/pager.c 80054194c18631342f6b8253e0bb090020c0cd55 +F src/pager.c 35c7e3b5bbad76f04e7143d2d4676a269a8ba9fc F src/pager.h ce5d076f3860a5f2d7460c582cd68383343b33cf F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf @@ -757,7 +757,7 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test b0b6c02bbb0b2a647b4edb69325d4c59ac73259b +F test/wal.test ff3c101453044e1f9930d0421a164e09619a678c F test/walcrash.test 45cfbab30bb7cbe0b2e9d5cabe90dbcad10cb89b F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 F test/walthread.test 58cd64b06f186251f09f64e4918fb74a7e52c963 @@ -804,7 +804,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P c9ed66cc3994b3b0d67a6c950b552a1a869f2ed2 -R 0dae17f227f2dd17e65e49bd0f5ce0a8 +P ecd828f96909895535d7dc744e5a8530e234e04d +R 6491b6f312e37bd3163c20e935da6a05 U dan -Z b4af52effa56ad5510bfc4469b8a847d +Z 46ab8b741bc2f3b7a6a7fee62de40d58 diff --git a/manifest.uuid b/manifest.uuid index b8f7810b58..c682fa9a6a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ecd828f96909895535d7dc744e5a8530e234e04d \ No newline at end of file +67d2a89ec2d593a077eb19a6ea2b06cb1c2e9ba8 \ No newline at end of file diff --git a/src/log.c b/src/log.c index 032b210a28..67bf9c0841 100644 --- a/src/log.c +++ b/src/log.c @@ -12,7 +12,7 @@ typedef struct LogSummaryHdr LogSummaryHdr; typedef struct LogSummary LogSummary; -typedef struct LogCheckpoint LogCheckpoint; +typedef struct LogIterator LogIterator; typedef struct LogLock LogLock; @@ -109,14 +109,14 @@ struct Log { ** ** The internals of this structure are only accessed by: ** -** logCheckpointInit() - Create a new iterator, -** logCheckpointNext() - Step an iterator, -** logCheckpointFree() - Free an iterator. +** logIteratorInit() - Create a new iterator, +** logIteratorNext() - Step an iterator, +** logIteratorFree() - Free an iterator. ** ** This functionality is used by the checkpoint code (see logCheckpoint()). */ -struct LogCheckpoint { - int nSegment; /* Size of LogCheckpoint.aSummary[] array */ +struct LogIterator { + int nSegment; /* Size of LogIterator.aSegment[] array */ int nFinal; /* Elements in segment nSegment-1 */ struct LogSegment { int iNext; /* Next aIndex index */ @@ -346,6 +346,8 @@ static int logDecodeFrame( u8 *aData, /* Pointer to page data (for checksum) */ u8 *aFrame /* Frame data */ ){ + assert( LOG_FRAME_HDRSIZE==20 ); + logChecksumBytes(aFrame, 12, aCksum); logChecksumBytes(aData, nData, aCksum); @@ -513,7 +515,7 @@ static int logSummaryRecover(LogSummary *pSummary, sqlite3_file *pFd){ /* Read all frames from the log file. */ iFrame = 0; iOffset = 0; - for(iOffset=0; (iOffset+nFrame)pSummary->aData; - LogCheckpoint *p; /* Return value */ + LogIterator *p; /* Return value */ int nSegment; /* Number of segments to merge */ u32 iLast; /* Last frame in log */ int nByte; /* Number of bytes to allocate */ @@ -759,8 +761,8 @@ static LogCheckpoint *logCheckpointInit(Log *pLog){ nSegment = (iLast >> 8) + 1; nFinal = (iLast & 0x000000FF); - nByte = sizeof(LogCheckpoint) + (nSegment-1)*sizeof(struct LogSegment) + 512; - p = (LogCheckpoint *)sqlite3_malloc(nByte); + nByte = sizeof(LogIterator) + (nSegment-1)*sizeof(struct LogSegment) + 512; + p = (LogIterator *)sqlite3_malloc(nByte); if( p ){ memset(p, 0, nByte); p->nSegment = nSegment; @@ -786,9 +788,9 @@ static LogCheckpoint *logCheckpointInit(Log *pLog){ } /* -** Free a log iterator allocated by logCheckpointInit(). +** Free a log iterator allocated by logIteratorInit(). */ -static void logCheckpointFree(LogCheckpoint *p){ +static void logIteratorFree(LogIterator *p){ sqlite3_free(p); } @@ -802,7 +804,7 @@ static int logCheckpoint( ){ int rc; /* Return code */ int pgsz = pLog->hdr.pgsz; /* Database page-size */ - LogCheckpoint *pIter = 0; /* Log iterator context */ + LogIterator *pIter = 0; /* Log iterator context */ u32 iDbpage = 0; /* Next database page to write */ u32 iFrame = 0; /* Log frame containing data for iDbpage */ @@ -811,7 +813,7 @@ static int logCheckpoint( } /* Allocate the iterator */ - pIter = logCheckpointInit(pLog); + pIter = logIteratorInit(pLog); if( !pIter ) return SQLITE_NOMEM; /* Sync the log file to disk */ @@ -819,7 +821,7 @@ static int logCheckpoint( if( rc!=SQLITE_OK ) goto out; /* Iterate through the contents of the log, copying data to the db file. */ - while( 0==logCheckpointNext(pIter, &iDbpage, &iFrame) ){ + while( 0==logIteratorNext(pIter, &iDbpage, &iFrame) ){ rc = sqlite3OsRead(pLog->pFd, zBuf, pgsz, (iFrame-1) * (pgsz+LOG_FRAME_HDRSIZE) + LOG_FRAME_HDRSIZE ); @@ -861,7 +863,7 @@ static int logCheckpoint( #endif out: - logCheckpointFree(pIter); + logIteratorFree(pIter); return rc; } diff --git a/src/pager.c b/src/pager.c index cd83fdb72c..3b16b86dd8 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3253,9 +3253,9 @@ static int pagerStress(void *p, PgHdr *pPg){ assert( pPg->pPager==pPager ); assert( pPg->flags&PGHDR_DIRTY ); + pPg->pDirty = 0; if( pagerUseLog(pPager) ){ /* Write a single frame for this page to the log. */ - assert( pPg->pDirty==0 ); rc = sqlite3LogFrames(pPager->pLog, pPager->pageSize, pPg, 0, 0, 0); }else{ /* The doNotSync flag is set by the sqlite3PagerWrite() function while it @@ -3324,7 +3324,6 @@ static int pagerStress(void *p, PgHdr *pPg){ /* Write the contents of the page out to the database file. */ if( rc==SQLITE_OK ){ - pPg->pDirty = 0; rc = pager_write_pagelist(pPg); } } diff --git a/test/wal.test b/test/wal.test index 799e49dc81..79b3b52d50 100644 --- a/test/wal.test +++ b/test/wal.test @@ -621,5 +621,63 @@ do_test wal-11.14 { } {37 38} +#------------------------------------------------------------------------- +# This block of tests, wal-12.*, tests a problem... +# +reopen_db +do_test wal-12.1 { + execsql { + PRAGMA page_size = 1024; + CREATE TABLE t1(x, y); + CREATE TABLE t2(x, y); + INSERT INTO t1 VALUES('A', 1); + } + list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] +} {0 5} +do_test wal-12.2 { + db close + sqlite3_wal db test.db + execsql { + UPDATE t1 SET y = 0 WHERE x = 'A'; + } + list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] +} {3 1} +do_test wal-12.3 { + execsql { INSERT INTO t2 VALUES('B', 1) } + list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] +} {3 2} + +do_test wal-12.4 { + file copy -force test.db test2.db + file copy -force test.db-wal test2.db-wal + sqlite3_wal db2 test2.db +breakpoint + execsql { SELECT * FROM t2 } db2 +} {B 1} +db2 close + +file copy -force test.db-wal A +do_test wal-12.5 { + execsql { + PRAGMA checkpoint; + UPDATE t2 SET y = 2 WHERE x = 'B'; + PRAGMA checkpoint; + UPDATE t1 SET y = 1 WHERE x = 'A'; + PRAGMA checkpoint; + UPDATE t1 SET y = 0 WHERE x = 'A'; + SELECT * FROM t2; + } +} {B 2} +file copy -force test.db-wal B + +do_test wal-12.4 { + file copy -force test.db test2.db + file copy -force test.db-wal test2.db-wal + sqlite3_wal db2 test2.db + execsql { SELECT * FROM t2 } db2 +} {B 2} +db2 close + + finish_test From 97a313554d2452fcdfe60e443b1eac7f1a4e9f87 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 16 Apr 2010 13:59:31 +0000 Subject: [PATCH 18/89] Change the log file format to include a small (12 byte) header at the start of the file. FossilOrigin-Name: 9865d14d6041874bc1239ce7a061d5c75f2d33c9 --- manifest | 14 +++--- manifest.uuid | 2 +- src/log.c | 128 ++++++++++++++++++++++++++++++++------------------ test/wal.test | 52 +++++++++++--------- 4 files changed, 120 insertions(+), 76 deletions(-) diff --git a/manifest b/manifest index 547b763339..ba84322467 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sbug\sin\slog\srecovery\s(last\sframe\sin\slog\swas\sbeing\signored).\sAlso\sremove\san\sincorrect\sassert\sstatement. -D 2010-04-16T11:30:18 +C Change\sthe\slog\sfile\sformat\sto\sinclude\sa\ssmall\s(12\sbyte)\sheader\sat\sthe\sstart\sof\sthe\sfile. +D 2010-04-16T13:59:31 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c 40e0c5780e7a33556f421142562edbf77f8e6639 +F src/log.c 11f683a3429319fb2731aa357717bf9117cdcba4 F src/log.h a2654af46ce7b5732f4d5a731abfdd180f0a06d9 F src/main.c c0e7192bad5b90544508b241eb2487ac661de890 F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a @@ -757,7 +757,7 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test ff3c101453044e1f9930d0421a164e09619a678c +F test/wal.test bb1fa35fc03353a1a154f583f01e5093e25ba001 F test/walcrash.test 45cfbab30bb7cbe0b2e9d5cabe90dbcad10cb89b F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 F test/walthread.test 58cd64b06f186251f09f64e4918fb74a7e52c963 @@ -804,7 +804,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P ecd828f96909895535d7dc744e5a8530e234e04d -R 6491b6f312e37bd3163c20e935da6a05 +P 67d2a89ec2d593a077eb19a6ea2b06cb1c2e9ba8 +R 2c35134540abe427220a5914099a1c09 U dan -Z 46ab8b741bc2f3b7a6a7fee62de40d58 +Z a25fb37e86e611cc50a9ddc963124e5a diff --git a/manifest.uuid b/manifest.uuid index c682fa9a6a..e6f912dc82 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -67d2a89ec2d593a077eb19a6ea2b06cb1c2e9ba8 \ No newline at end of file +9865d14d6041874bc1239ce7a061d5c75f2d33c9 \ No newline at end of file diff --git a/src/log.c b/src/log.c index 67bf9c0841..73b44863d0 100644 --- a/src/log.c +++ b/src/log.c @@ -4,6 +4,35 @@ ** "journal_mode=wal" mode. */ +/* +** LOG FILE FORMAT +** +** A log file consists of a header followed by zero or more log frames. +** The log header is 12 bytes in size and consists of the following three +** big-endian 32-bit unsigned integer values: +** +** 0: Database page size, +** 4: Randomly selected salt value 1, +** 8: Randomly selected salt value 2. +** +** Immediately following the log header are zero or more log frames. Each +** frame itself consists of a 16-byte header followed by a bytes +** of page data. The header is broken into 4 big-endian 32-bit unsigned +** integer values, as follows: +** +** 0: Page number. +** 4: For commit records, the size of the database image in pages +** after the commit. For all other records, zero. +** 8: Checksum value 1. +** 12: Checksum value 2. +*/ + +/* +** LOG SUMMARY FORMAT +** +** TODO. +*/ + #include "log.h" #include @@ -39,8 +68,20 @@ struct LogSummaryHdr { #define LOGSUMMARY_FRAME_OFFSET \ (LOGSUMMARY_HDR_NFIELD + LOG_CKSM_BYTES/sizeof(u32)) + + /* Size of frame header */ -#define LOG_FRAME_HDRSIZE 20 +#define LOG_FRAME_HDRSIZE 16 +#define LOG_HDRSIZE 12 + +/* +** Return the offset of frame iFrame in the log file, assuming a database +** page size of pgsz bytes. The offset returned is to the start of the +** log frame-header. +*/ +#define logFrameOffset(iFrame, pgsz) ( \ + LOG_HDRSIZE + ((iFrame)-1)*((pgsz)+LOG_FRAME_HDRSIZE) \ +) /* ** There is one instance of this structure for each log-summary object @@ -126,6 +167,7 @@ struct LogIterator { }; + /* ** List of all LogSummary objects created by this process. Protected by ** static mutex LOG_SUMMARY_MUTEX. TODO: Should have a dedicated mutex @@ -291,7 +333,6 @@ static int logSummaryUnmap(LogSummary *pSummary, int isTruncate){ return rc; } - static void logSummaryWriteHdr(LogSummary *pSummary, LogSummaryHdr *pHdr){ u32 *aData = pSummary->aData; memcpy(aData, pHdr, sizeof(LogSummaryHdr)); @@ -321,17 +362,16 @@ static void logEncodeFrame( u8 *aData, /* Pointer to page data (for checksum) */ u8 *aFrame /* OUT: Write encoded frame here */ ){ - assert( LOG_FRAME_HDRSIZE==20 ); + assert( LOG_FRAME_HDRSIZE==16 ); - sqlite3Put4byte(&aFrame[0], nData); - sqlite3Put4byte(&aFrame[4], iPage); - sqlite3Put4byte(&aFrame[8], nTruncate); + sqlite3Put4byte(&aFrame[0], iPage); + sqlite3Put4byte(&aFrame[4], nTruncate); - logChecksumBytes(aFrame, 12, aCksum); + logChecksumBytes(aFrame, 8, aCksum); logChecksumBytes(aData, nData, aCksum); - sqlite3Put4byte(&aFrame[12], aCksum[0]); - sqlite3Put4byte(&aFrame[16], aCksum[1]); + sqlite3Put4byte(&aFrame[8], aCksum[0]); + sqlite3Put4byte(&aFrame[12], aCksum[1]); } /* @@ -346,20 +386,20 @@ static int logDecodeFrame( u8 *aData, /* Pointer to page data (for checksum) */ u8 *aFrame /* Frame data */ ){ - assert( LOG_FRAME_HDRSIZE==20 ); + assert( LOG_FRAME_HDRSIZE==16 ); - logChecksumBytes(aFrame, 12, aCksum); + logChecksumBytes(aFrame, 8, aCksum); logChecksumBytes(aData, nData, aCksum); - if( aCksum[0]!=sqlite3Get4byte(&aFrame[12]) - || aCksum[1]!=sqlite3Get4byte(&aFrame[16]) + if( aCksum[0]!=sqlite3Get4byte(&aFrame[8]) + || aCksum[1]!=sqlite3Get4byte(&aFrame[12]) ){ /* Checksum failed. */ return 0; } - *piPage = sqlite3Get4byte(&aFrame[4]); - *pnTruncate = sqlite3Get4byte(&aFrame[8]); + *piPage = sqlite3Get4byte(&aFrame[0]); + *pnTruncate = sqlite3Get4byte(&aFrame[4]); return 1; } @@ -486,12 +526,12 @@ static int logSummaryRecover(LogSummary *pSummary, sqlite3_file *pFd){ int iFrame; /* Index of last frame read */ i64 iOffset; /* Next offset to read from log file */ int nPgsz; /* Page size according to the log */ - u32 aCksum[2] = {2, 3}; /* Running checksum */ + u32 aCksum[2]; /* Running checksum */ /* Read in the first frame header in the file (to determine the ** database page size). */ - rc = sqlite3OsRead(pFd, aBuf, LOG_FRAME_HDRSIZE, 0); + rc = sqlite3OsRead(pFd, aBuf, LOG_HDRSIZE, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -503,6 +543,8 @@ static int logSummaryRecover(LogSummary *pSummary, sqlite3_file *pFd){ if( nPgsz&(nPgsz-1) || nPgsz>SQLITE_MAX_PAGE_SIZE ){ goto finished; } + aCksum[0] = sqlite3Get4byte(&aBuf[4]); + aCksum[1] = sqlite3Get4byte(&aBuf[8]); /* Malloc a buffer to read frames into. */ nFrame = nPgsz + LOG_FRAME_HDRSIZE; @@ -514,8 +556,7 @@ static int logSummaryRecover(LogSummary *pSummary, sqlite3_file *pFd){ /* Read all frames from the log file. */ iFrame = 0; - iOffset = 0; - for(iOffset=0; (iOffset+nFrame)<=nSize; iOffset+=nFrame){ + for(iOffset=LOG_HDRSIZE; (iOffset+nFrame)<=nSize; iOffset+=nFrame){ u32 pgno; /* Database page number for frame */ u32 nTruncate; /* dbsize field from frame header */ int isValid; /* True if this frame is valid */ @@ -823,7 +864,7 @@ static int logCheckpoint( /* Iterate through the contents of the log, copying data to the db file. */ while( 0==logIteratorNext(pIter, &iDbpage, &iFrame) ){ rc = sqlite3OsRead(pLog->pFd, zBuf, pgsz, - (iFrame-1) * (pgsz+LOG_FRAME_HDRSIZE) + LOG_FRAME_HDRSIZE + logFrameOffset(iFrame, pgsz) + LOG_FRAME_HDRSIZE ); if( rc!=SQLITE_OK ) goto out; rc = sqlite3OsWrite(pFd, zBuf, pgsz, (iDbpage-1)*pgsz); @@ -1319,8 +1360,7 @@ int sqlite3LogRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut){ ** required page. Read and return data from the log file. */ if( iRead ){ - i64 iOffset = (iRead-1) * (pLog->hdr.pgsz+LOG_FRAME_HDRSIZE); - iOffset += LOG_FRAME_HDRSIZE; + i64 iOffset = logFrameOffset(iRead, pLog->hdr.pgsz) + LOG_FRAME_HDRSIZE; *pInLog = 1; return sqlite3OsRead(pLog->pFd, pOut, pLog->hdr.pgsz, iOffset); } @@ -1401,45 +1441,43 @@ int sqlite3LogFrames( int isCommit, /* True if this is a commit */ int isSync /* True to sync the log file */ ){ - /* Each frame has a 20 byte header, as follows: - ** - ** + Pseudo-random salt (4 bytes) - ** + Page number (4 bytes) - ** + New database size, or 0 if not a commit frame (4 bytes) - ** + Checksum (CHECKSUM_BYTES bytes); - ** - ** The checksum is computed based on the following: - ** - ** + The previous checksum, or {2, 3} for the first frame in the log. - ** + The non-checksum fields of the frame header, and - ** + The frame contents (page data). - ** - ** This format must also be understood by the code in logSummaryRecover(). - ** The size of the frame header is used by LogRead() and LogCheckpoint(). - */ int rc; /* Used to catch return codes */ u32 iFrame; /* Next frame address */ - u8 aFrame[LOG_FRAME_HDRSIZE]; + u8 aFrame[LOG_FRAME_HDRSIZE]; /* Buffer to assemble frame-header in */ PgHdr *p; /* Iterator to run through pList with. */ - u32 aCksum[2]; - + u32 aCksum[2]; /* Checksums */ PgHdr *pLast; /* Last frame in list */ int nLast = 0; /* Number of extra copies of last page */ - assert( LOG_FRAME_HDRSIZE==(4 * 3 + LOG_CKSM_BYTES) ); + assert( LOG_FRAME_HDRSIZE==(4 * 2 + LOG_CKSM_BYTES) ); assert( pList ); + /* If this is the first frame written into the log, write the log + ** header to the start of the log file. See comments at the top of + ** this file for a description of the log-header format. + */ + assert( LOG_FRAME_HDRSIZE>=LOG_HDRSIZE ); + iFrame = pLog->hdr.iLastPg; + if( iFrame==0 ){ + sqlite3Put4byte(aFrame, nPgsz); + sqlite3_randomness(8, &aFrame[4]); + pLog->hdr.iCheck1 = sqlite3Get4byte(&aFrame[4]); + pLog->hdr.iCheck2 = sqlite3Get4byte(&aFrame[8]); + rc = sqlite3OsWrite(pLog->pFd, aFrame, LOG_HDRSIZE, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + } + aCksum[0] = pLog->hdr.iCheck1; aCksum[1] = pLog->hdr.iCheck2; /* Write the log file. */ - iFrame = pLog->hdr.iLastPg; for(p=pList; p; p=p->pDirty){ u32 nDbsize; /* Db-size field for frame header */ i64 iOffset; /* Write offset in log file */ - iFrame++; - iOffset = (iFrame-1) * (nPgsz+sizeof(aFrame)); + iOffset = logFrameOffset(++iFrame, nPgsz); /* Populate and write the frame header */ nDbsize = (isCommit && p->pDirty==0) ? nTruncate : 0; diff --git a/test/wal.test b/test/wal.test index 79b3b52d50..a41b263c1b 100644 --- a/test/wal.test +++ b/test/wal.test @@ -28,6 +28,10 @@ proc sqlite3_wal {args} { [lindex $args 0] eval { PRAGMA journal_mode = wal } } +proc log_file_size {nFrame pgsz} { + expr {12 + ($pgsz+16)*$nFrame} +} + # # These are 'warm-body' tests used while developing the WAL code. They # serve to prove that a few really simple cases work: @@ -58,7 +62,7 @@ do_test wal-1.1 { do_test wal-1.2 { # There are now two pages in the log. file size test.db-wal -} [expr (20+1024)*2] +} [log_file_size 2 1024] do_test wal-1.3 { execsql { SELECT * FROM sqlite_master } @@ -208,11 +212,11 @@ do_test wal-7.1 { INSERT INTO t1 VALUES(1, 2); } list [file size test.db] [file size test.db-wal] -} [list 0 [expr (1024+20)*3]] +} [list 0 [log_file_size 3 1024]] do_test wal-7.2 { execsql { PRAGMA checkpoint } list [file size test.db] [file size test.db-wal] -} [list 2048 [expr (1024+20)*3]] +} [list 2048 [log_file_size 3 1024]] # Execute some transactions in auto-vacuum mode to test database file # truncation. @@ -547,12 +551,12 @@ do_test wal-11.1 { } {0 3} do_test wal-11.2 { execsql { PRAGMA checkpoint } - list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] -} {3 3} + list [expr [file size test.db]/1024] [file size test.db-wal] +} [list 3 [log_file_size 3 1024]] do_test wal-11.3 { execsql { INSERT INTO t1 VALUES( randomblob(900) ) } - list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] -} {3 4} + list [expr [file size test.db]/1024] [file size test.db-wal] +} [list 3 [log_file_size 4 1024]] do_test wal-11.4 { execsql { @@ -562,8 +566,8 @@ do_test wal-11.4 { INSERT INTO t1 SELECT randomblob(900) FROM t1; -- 8 INSERT INTO t1 SELECT randomblob(900) FROM t1; -- 16 } - list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] -} {3 33} + list [expr [file size test.db]/1024] [file size test.db-wal] +} [list 3 [log_file_size 33 1024]] do_test wal-11.5 { execsql { SELECT count(*) FROM t1; @@ -572,8 +576,8 @@ do_test wal-11.5 { } {16 ok} do_test wal-11.6 { execsql COMMIT - list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] -} {3 42} + list [expr [file size test.db]/1024] [file size test.db-wal] +} [list 3 [log_file_size 42 1024]] do_test wal-11.7 { execsql { SELECT count(*) FROM t1; @@ -582,12 +586,12 @@ do_test wal-11.7 { } {16 ok} do_test wal-11.8 { execsql { PRAGMA checkpoint } - list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] -} {37 42} + list [expr [file size test.db]/1024] [file size test.db-wal] +} [list 37 [log_file_size 42 1024]] do_test wal-11.9 { db close sqlite3_wal db test.db - list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] + list [expr [file size test.db]/1024] [file size test.db-wal] } {37 0} do_test wal-11.10 { @@ -597,8 +601,8 @@ do_test wal-11.10 { INSERT INTO t1 SELECT randomblob(900) FROM t1; -- 32 SELECT count(*) FROM t1; } - list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] -} {37 38} + list [expr [file size test.db]/1024] [file size test.db-wal] +} [list 37 [log_file_size 40 1024]] do_test wal-11.11 { execsql { SELECT count(*) FROM t1; @@ -607,8 +611,8 @@ do_test wal-11.11 { } } {32 16} do_test wal-11.12 { - list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] -} {37 38} + list [expr [file size test.db]/1024] [file size test.db-wal] +} [list 37 [log_file_size 40 1024]] do_test wal-11.13 { execsql { INSERT INTO t1 VALUES( randomblob(900) ); @@ -617,12 +621,14 @@ do_test wal-11.13 { } } {17 ok} do_test wal-11.14 { - list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] -} {37 38} + list [expr [file size test.db]/1024] [file size test.db-wal] +} [list 37 [log_file_size 40 1024]] #------------------------------------------------------------------------- -# This block of tests, wal-12.*, tests a problem... +# This block of tests, wal-12.*, tests the fix for a problem that +# could occur if a log that is a prefix of an older log is written +# into a reused log file. # reopen_db do_test wal-12.1 { @@ -632,8 +638,8 @@ do_test wal-12.1 { CREATE TABLE t2(x, y); INSERT INTO t1 VALUES('A', 1); } - list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] -} {0 5} + list [expr [file size test.db]/1024] [file size test.db-wal] +} [list 0 [log_file_size 5 1024]] do_test wal-12.2 { db close sqlite3_wal db test.db From 3de777fd8cb66d710c02783e9786c756fbd91db8 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 17 Apr 2010 12:31:37 +0000 Subject: [PATCH 19/89] Enhancements to wal-mode locking scheme. FossilOrigin-Name: 8549c286497f3d2cd118be1334fce00d6f8a26c4 --- manifest | 18 +- manifest.uuid | 2 +- src/log.c | 482 ++++++++++++++++++++++++-------------------- src/pager.c | 95 ++++++--- test/wal.test | 13 +- test/walthread.test | 6 +- 6 files changed, 356 insertions(+), 260 deletions(-) diff --git a/manifest b/manifest index ba84322467..dacfee46f5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\slog\sfile\sformat\sto\sinclude\sa\ssmall\s(12\sbyte)\sheader\sat\sthe\sstart\sof\sthe\sfile. -D 2010-04-16T13:59:31 +C Enhancements\sto\swal-mode\slocking\sscheme. +D 2010-04-17T12:31:37 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c 11f683a3429319fb2731aa357717bf9117cdcba4 +F src/log.c 6ac96c84ca4564f156de564ceddb3c1a2319ac6e F src/log.h a2654af46ce7b5732f4d5a731abfdd180f0a06d9 F src/main.c c0e7192bad5b90544508b241eb2487ac661de890 F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a @@ -154,7 +154,7 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053 -F src/pager.c 35c7e3b5bbad76f04e7143d2d4676a269a8ba9fc +F src/pager.c 674d6558a618ca7714ba28741e6dc681692bffe2 F src/pager.h ce5d076f3860a5f2d7460c582cd68383343b33cf F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf @@ -757,10 +757,10 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test bb1fa35fc03353a1a154f583f01e5093e25ba001 +F test/wal.test 5fa3cdf2e93e79b0891af6cf7fab8ef9e07a23c6 F test/walcrash.test 45cfbab30bb7cbe0b2e9d5cabe90dbcad10cb89b F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 -F test/walthread.test 58cd64b06f186251f09f64e4918fb74a7e52c963 +F test/walthread.test 27e44ee6fd02f1f494a24f999c97086af3ab739d F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 F test/where2.test 45eacc126aabb37959a387aa83e59ce1f1f03820 F test/where3.test aa44a9b29e8c9f3d7bb94a3bb3a95b31627d520d @@ -804,7 +804,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 67d2a89ec2d593a077eb19a6ea2b06cb1c2e9ba8 -R 2c35134540abe427220a5914099a1c09 +P 9865d14d6041874bc1239ce7a061d5c75f2d33c9 +R f6bb150cf1f3c0e2bfeac5bd76d233a7 U dan -Z a25fb37e86e611cc50a9ddc963124e5a +Z ea6a1c06d626e2f5d8edb2ab802e8ea3 diff --git a/manifest.uuid b/manifest.uuid index e6f912dc82..8375084062 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9865d14d6041874bc1239ce7a061d5c75f2d33c9 \ No newline at end of file +8549c286497f3d2cd118be1334fce00d6f8a26c4 \ No newline at end of file diff --git a/src/log.c b/src/log.c index 73b44863d0..f0b3a73ebf 100644 --- a/src/log.c +++ b/src/log.c @@ -11,19 +11,19 @@ ** The log header is 12 bytes in size and consists of the following three ** big-endian 32-bit unsigned integer values: ** -** 0: Database page size, -** 4: Randomly selected salt value 1, -** 8: Randomly selected salt value 2. +** 0: Database page size, +** 4: Randomly selected salt value 1, +** 8: Randomly selected salt value 2. ** ** Immediately following the log header are zero or more log frames. Each ** frame itself consists of a 16-byte header followed by a bytes ** of page data. The header is broken into 4 big-endian 32-bit unsigned ** integer values, as follows: ** -** 0: Page number. -** 4: For commit records, the size of the database image in pages +** 0: Page number. +** 4: For commit records, the size of the database image in pages ** after the commit. For all other records, zero. -** 8: Checksum value 1. +** 8: Checksum value 1. ** 12: Checksum value 2. */ @@ -106,13 +106,19 @@ struct LogSummary { /* ** The four lockable regions associated with each log-summary. A connection -** may take either a SHARED or EXCLUSIVE lock on each. +** may take either a SHARED or EXCLUSIVE lock on each. An ORed combination +** of the following bitmasks is passed as the second argument to the +** logLockRegion() function. */ #define LOG_REGION_A 0x01 #define LOG_REGION_B 0x02 #define LOG_REGION_C 0x04 #define LOG_REGION_D 0x08 +#define LOG_LOCK_MUTEX 12 +#define LOG_LOCK_DMH 13 +#define LOG_LOCK_REGION 14 + /* ** A single instance of this structure is allocated as part of each ** connection to a database log. All structures associated with the @@ -316,14 +322,19 @@ static int logSummaryMap(LogSummary *pSummary, int nByte){ ** Regardless of the value of isTruncate, close the file-descriptor ** opened on the log-summary file. */ -static int logSummaryUnmap(LogSummary *pSummary, int isTruncate){ +static int logSummaryUnmap(LogSummary *pSummary, int isUnlink){ int rc = SQLITE_OK; if( pSummary->aData ){ assert( pSummary->fd>0 ); munmap(pSummary->aData, pSummary->nData); pSummary->aData = 0; - if( isTruncate ){ - rc = (ftruncate(pSummary->fd, 0) ? SQLITE_IOERR : SQLITE_OK); + if( isUnlink ){ + char *zFile = sqlite3_mprintf("%s-summary", pSummary->zPath); + if( !zFile ){ + rc = SQLITE_NOMEM; + } + unlink(zFile); + sqlite3_free(zFile); } } if( pSummary->fd>0 ){ @@ -589,12 +600,197 @@ finished: return rc; } +/* +** Values for the third parameter to logLockRegion(). +*/ +#define LOG_UNLOCK 0 +#define LOG_RDLOCK 1 +#define LOG_WRLOCK 2 +#define LOG_WRLOCKW 3 + +static int logLockFd(LogSummary *pSummary, int iStart, int nByte, int op){ + int aType[4] = { + F_UNLCK, /* LOG_UNLOCK */ + F_RDLCK, /* LOG_RDLOCK */ + F_WRLCK, /* LOG_WRLOCK */ + F_WRLCK /* LOG_WRLOCKW */ + }; + int aOp[4] = { + F_SETLK, /* LOG_UNLOCK */ + F_SETLK, /* LOG_RDLOCK */ + F_SETLK, /* LOG_WRLOCK */ + F_SETLKW /* LOG_WRLOCKW */ + }; + + struct flock f; /* Locking operation */ + int rc; /* Value returned by fcntl() */ + + assert( ArraySize(aType)==ArraySize(aOp) ); + assert( op>=0 && opfd, aOp[op], &f); + return (rc==0) ? SQLITE_OK : SQLITE_BUSY; +} + +static int logLockRegion(Log *pLog, u32 mRegion, int op){ + LogSummary *pSummary = pLog->pSummary; + LogLock *p; /* Used to iterate through in-process locks */ + u32 mOther; /* Locks held by other connections */ + u32 mNew; /* New mask for pLog */ + + assert( + /* Writer lock operations */ + (op==LOG_WRLOCK && mRegion==(LOG_REGION_C|LOG_REGION_D)) + || (op==LOG_UNLOCK && mRegion==(LOG_REGION_C|LOG_REGION_D)) + + /* Normal reader lock operations */ + || (op==LOG_RDLOCK && mRegion==(LOG_REGION_A|LOG_REGION_B)) + || (op==LOG_UNLOCK && mRegion==(LOG_REGION_A)) + || (op==LOG_UNLOCK && mRegion==(LOG_REGION_B)) + + /* Region D reader lock operations */ + || (op==LOG_RDLOCK && mRegion==(LOG_REGION_D)) + || (op==LOG_RDLOCK && mRegion==(LOG_REGION_A)) + || (op==LOG_UNLOCK && mRegion==(LOG_REGION_D)) + + /* Checkpointer lock operations */ + || (op==LOG_WRLOCK && mRegion==(LOG_REGION_B|LOG_REGION_C)) + || (op==LOG_WRLOCK && mRegion==(LOG_REGION_A)) + || (op==LOG_UNLOCK && mRegion==(LOG_REGION_B|LOG_REGION_C)) + || (op==LOG_UNLOCK && mRegion==(LOG_REGION_A|LOG_REGION_B|LOG_REGION_C)) + ); + + /* Assert that a connection never tries to go from an EXCLUSIVE to a + ** SHARED lock on a region. Moving from SHARED to EXCLUSIVE sometimes + ** happens though (when a region D reader upgrades to a writer). + */ + assert( op!=LOG_RDLOCK || 0==(pLog->lock.mLock & (mRegion<<8)) ); + + sqlite3_mutex_enter(pSummary->mutex); + + /* Calculate a mask of logs held by all connections in this process apart + ** from this one. The least significant byte of the mask contains a mask + ** of the SHARED logs held. The next least significant byte of the mask + ** indicates the EXCLUSIVE locks held. For example, to test if some other + ** connection is holding a SHARED lock on region A, or an EXCLUSIVE lock + ** on region C, do: + ** + ** hasSharedOnA = (mOther & (LOG_REGION_A<<0)); + ** hasExclusiveOnC = (mOther & (LOG_REGION_C<<8)); + ** + ** In all masks, if the bit in the EXCLUSIVE byte mask is set, so is the + ** corresponding bit in the SHARED mask. + */ + mOther = 0; + for(p=pSummary->pLock; p; p=p->pNext){ + assert( (p->mLock & (p->mLock<<8))==(p->mLock&0x0000FF00) ); + if( p!=&pLog->lock ){ + mOther |= p->mLock; + } + } + + /* If this call is to lock a region (not to unlock one), test if locks held + ** by any other connection in this process prevent the new locks from + ** begin granted. If so, exit the summary mutex and return SQLITE_BUSY. + */ + if( op && (mOther & (mRegion << (op==LOG_RDLOCK ? 8 : 0))) ){ + sqlite3_mutex_leave(pSummary->mutex); + return SQLITE_BUSY; + } + + /* Figure out the new log mask for this connection. */ + switch( op ){ + case LOG_UNLOCK: + mNew = (pLog->lock.mLock & ~(mRegion|(mRegion<<8))); + break; + case LOG_RDLOCK: + mNew = (pLog->lock.mLock | mRegion); + break; + default: + assert( op==LOG_WRLOCK ); + mNew = (pLog->lock.mLock | (mRegion<<8) | mRegion); + break; + } + + /* Now modify the locks held on the log-summary file descriptor. This + ** file descriptor is shared by all log connections in this process. + ** Therefore: + ** + ** + If one or more log connections in this process hold a SHARED lock + ** on a region, the file-descriptor should hold a SHARED lock on + ** the file region. + ** + ** + If a log connection in this process holds an EXCLUSIVE lock on a + ** region, the file-descriptor should also hold an EXCLUSIVE lock on + ** the region in question. + ** + ** If this is an LOG_UNLOCK operation, only regions for which no other + ** connection holds a lock should actually be unlocked. And if this + ** is a LOG_RDLOCK operation and other connections already hold all + ** the required SHARED locks, then no system call is required. + */ + if( op==LOG_UNLOCK ){ + mRegion = (mRegion & ~mOther); + } + if( (op==LOG_WRLOCK) + || (op==LOG_UNLOCK && mRegion) + || (op==LOG_RDLOCK && (mOther&mRegion)!=mRegion) + ){ + struct LockMap { + int iStart; /* Byte offset to start locking operation */ + int iLen; /* Length field for locking operation */ + } aMap[] = { + /* 0000 */ {0, 0}, /* 0001 */ {4+LOG_LOCK_REGION, 1}, + /* 0010 */ {3+LOG_LOCK_REGION, 1}, /* 0011 */ {3+LOG_LOCK_REGION, 2}, + /* 0100 */ {2+LOG_LOCK_REGION, 1}, /* 0101 */ {0, 0}, + /* 0110 */ {2+LOG_LOCK_REGION, 2}, /* 0111 */ {2+LOG_LOCK_REGION, 3}, + /* 1000 */ {1+LOG_LOCK_REGION, 1}, /* 1001 */ {0, 0}, + /* 1010 */ {0, 0}, /* 1011 */ {0, 0}, + /* 1100 */ {1+LOG_LOCK_REGION, 2}, /* 1101 */ {0, 0}, + /* 1110 */ {0, 0}, /* 1111 */ {0, 0} + }; + int rc; /* Return code of logLockFd() */ + + assert( mRegionmutex); + return rc; + } + } + + pLog->lock.mLock = mNew; + sqlite3_mutex_leave(pSummary->mutex); + return SQLITE_OK; +} + +static int logLockDMH(LogSummary *pSummary, int eLock){ + assert( eLock==LOG_RDLOCK || eLock==LOG_WRLOCK ); + return logLockFd(pSummary, LOG_LOCK_DMH, 1, eLock); +} + +static int logLockMutex(LogSummary *pSummary, int eLock){ + assert( eLock==LOG_WRLOCKW || eLock==LOG_UNLOCK ); + logLockFd(pSummary, LOG_LOCK_MUTEX, 1, eLock); + return SQLITE_OK; +} + + /* ** This function intializes the connection to the log-summary identified ** by struct pSummary. */ -static int logSummaryInit(LogSummary *pSummary, sqlite3_file *pFd){ +static int logSummaryInit( + LogSummary *pSummary, /* Log summary object to initialize */ + sqlite3_file *pFd /* File descriptor open on log file */ +){ int rc; /* Return Code */ char *zFile; /* File name for summary file */ @@ -614,36 +810,35 @@ static int logSummaryInit(LogSummary *pSummary, sqlite3_file *pFd){ return SQLITE_IOERR; } - /* Grab an exclusive lock the summary file. Then mmap() it. TODO: This - ** code needs to be enhanced to support a growable mapping. For now, just - ** make the mapping very large to start with. + /* Grab an exclusive lock the summary file. Then mmap() it. + ** + ** TODO: This code needs to be enhanced to support a growable mapping. + ** For now, just make the mapping very large to start with. The + ** pages should not be allocated until they are first accessed anyhow, + ** so using a large mapping consumes no more resources than a smaller + ** one would. */ - rc = logSummaryLock(pSummary); + assert( sqlite3_mutex_held(pSummary->mutex) ); + rc = logLockMutex(pSummary, LOG_WRLOCKW); if( rc!=SQLITE_OK ) return rc; rc = logSummaryMap(pSummary, 512*1024); if( rc!=SQLITE_OK ) goto out; - /* Grab a SHARED lock on the log file. Then try to upgrade to an EXCLUSIVE - ** lock. If successful, then this is the first (and only) connection to - ** the database. In this case assume the contents of the log-summary - ** cannot be trusted. Zero the log-summary header to make sure. - ** - ** The SHARED lock on the log file is not released until the connection - ** to the database is closed. + /* Try to obtain an EXCLUSIVE lock on the dead-mans-hand region. If this + ** is possible, the contents of the log-summary file (if any) may not + ** be trusted. Zero the log-summary header before continuing. */ - rc = sqlite3OsLock(pFd, SQLITE_LOCK_SHARED); - if( rc!=SQLITE_OK ) goto out; - rc = sqlite3OsLock(pFd, SQLITE_LOCK_EXCLUSIVE); + rc = logLockDMH(pSummary, LOG_WRLOCK); if( rc==SQLITE_OK ){ - /* This is the first and only connection. */ memset(pSummary->aData, 0, (LOGSUMMARY_HDR_NFIELD+2)*sizeof(u32) ); - rc = sqlite3OsUnlock(pFd, SQLITE_LOCK_SHARED); - }else if( rc==SQLITE_BUSY ){ - rc = SQLITE_OK; + } + rc = logLockDMH(pSummary, LOG_RDLOCK); + if( rc!=SQLITE_OK ){ + return SQLITE_IOERR; } out: - logSummaryUnlock(pSummary); + logLockMutex(pSummary, LOG_UNLOCK); return rc; } @@ -652,6 +847,12 @@ static int logSummaryInit(LogSummary *pSummary, sqlite3_file *pFd){ ** database file does not actually have to exist. zDb is used only to ** figure out the name of the log file to open. If the log file does not ** exist it is created by this call. +** +** A SHARED lock should be held on the database file when this function +** is called. The purpose of this SHARED lock is to prevent any other +** client from unlinking the log or log-summary file. If another process +** were to do this just after this client opened one of these files, the +** system would be badly broken. */ int sqlite3LogOpen( sqlite3_vfs *pVfs, /* vfs module to open log file with */ @@ -666,11 +867,10 @@ int sqlite3LogOpen( char *zWal = 0; /* Path to WAL file */ int nWal; /* Length of zWal in bytes */ - /* Zero output variables */ assert( zDb ); - *ppLog = 0; /* Allocate an instance of struct Log to return. */ + *ppLog = 0; pRet = (Log *)sqlite3MallocZero(sizeof(Log) + pVfs->szOsFile); if( !pRet ) goto out; pRet->pVfs = pVfs; @@ -726,15 +926,12 @@ int sqlite3LogOpen( /* Object pSummary is shared between all connections to the database made ** by this process. So at this point it may or may not be connected to - ** the log-summary. If it is not, connect it. Otherwise, just take the - ** SHARED lock on the log file. + ** the log-summary. If it is not, connect it. */ sqlite3_mutex_enter(pSummary->mutex); mutex = pSummary->mutex; if( pSummary->fd<0 ){ rc = logSummaryInit(pSummary, pRet->pFd); - }else{ - rc = sqlite3OsLock(pRet->pFd, SQLITE_LOCK_SHARED); } pRet->lock.pNext = pSummary->pLock; @@ -940,45 +1137,43 @@ int sqlite3LogClose( **/ pSummary->nRef--; if( pSummary->nRef==0 ){ + int rc; LogSummary **pp; - - rc = logSummaryLock(pSummary); - if( rc==SQLITE_OK ){ - int isTruncate = 0; - int rc2 = sqlite3OsLock(pLog->pFd, SQLITE_LOCK_EXCLUSIVE); - if( rc2==SQLITE_OK ){ - /* This is the last connection to the database (including other - ** processes). Do three things: - ** - ** 1. Checkpoint the db. - ** 2. Truncate the log file to zero bytes. - ** 3. Truncate the log-summary file to zero bytes. - */ - rc2 = logCheckpoint(pLog, pFd, zBuf); - if( rc2==SQLITE_OK ){ - rc2 = sqlite3OsTruncate(pLog->pFd, 0); - } - isTruncate = 1; - }else if( rc2==SQLITE_BUSY ){ - rc2 = SQLITE_OK; - } - logSummaryUnmap(pSummary, isTruncate); - sqlite3OsUnlock(pLog->pFd, SQLITE_LOCK_NONE); - rc = logSummaryUnlock(pSummary); - if( rc2!=SQLITE_OK ) rc = rc2; - } - - /* Remove the LogSummary object from the global list. Then free the - ** mutex and the object itself. - */ for(pp=&pLogSummary; *pp!=pSummary; pp=&(*pp)->pNext); *pp = (*pp)->pNext; + + sqlite3_mutex_leave(mutex); + + rc = sqlite3OsLock(pFd, SQLITE_LOCK_EXCLUSIVE); + if( rc==SQLITE_OK ){ + + /* This is the last connection to the database (including other + ** processes). Do three things: + ** + ** 1. Checkpoint the db. + ** 2. Truncate the log file. + ** 3. Unlink the log-summary file. + */ + rc = logCheckpoint(pLog, pFd, zBuf); + if( rc==SQLITE_OK ){ + rc = sqlite3OsDelete(pLog->pVfs, pSummary->zPath, 0); + } + + logSummaryUnmap(pSummary, 1); + }else{ + if( rc==SQLITE_BUSY ){ + rc = SQLITE_OK; + } + logSummaryUnmap(pSummary, 0); + } + sqlite3OsUnlock(pFd, SQLITE_LOCK_NONE); + sqlite3_mutex_free(pSummary->mutex); sqlite3_free(pSummary); + }else{ + sqlite3_mutex_leave(mutex); } - sqlite3_mutex_leave(mutex); - /* Close the connection to the log file and free the Log handle. */ sqlite3OsClose(pLog->pFd); sqlite3_free(pLog); @@ -1012,7 +1207,7 @@ static int logEnterMutex(Log *pLog){ int rc; sqlite3_mutex_enter(pSummary->mutex); - rc = logSummaryLock(pSummary); + rc = logLockMutex(pSummary, LOG_WRLOCKW); if( rc!=SQLITE_OK ){ sqlite3_mutex_leave(pSummary->mutex); } @@ -1020,157 +1215,10 @@ static int logEnterMutex(Log *pLog){ } static void logLeaveMutex(Log *pLog){ LogSummary *pSummary = pLog->pSummary; - logSummaryUnlock(pSummary); + logLockMutex(pSummary, LOG_UNLOCK); sqlite3_mutex_leave(pSummary->mutex); } -/* -** Values for the second parameter to logLockRegion(). -*/ -#define LOG_UNLOCK 0 -#define LOG_RDLOCK 1 -#define LOG_WRLOCK 2 - -static int logLockRegion(Log *pLog, u32 mRegion, int op){ - LogSummary *pSummary = pLog->pSummary; - LogLock *p; /* Used to iterate through in-process locks */ - u32 mOther; /* Locks held by other connections */ - u32 mNew; /* New mask for pLog */ - - assert( - /* Writer lock operations */ - (op==LOG_WRLOCK && mRegion==(LOG_REGION_C|LOG_REGION_D)) - || (op==LOG_UNLOCK && mRegion==(LOG_REGION_C|LOG_REGION_D)) - - /* Normal reader lock operations */ - || (op==LOG_RDLOCK && mRegion==(LOG_REGION_A|LOG_REGION_B)) - || (op==LOG_UNLOCK && mRegion==(LOG_REGION_A)) - || (op==LOG_UNLOCK && mRegion==(LOG_REGION_B)) - - /* Region D reader lock operations */ - || (op==LOG_RDLOCK && mRegion==(LOG_REGION_D)) - || (op==LOG_RDLOCK && mRegion==(LOG_REGION_A)) - || (op==LOG_UNLOCK && mRegion==(LOG_REGION_D)) - - /* Checkpointer lock operations */ - || (op==LOG_WRLOCK && mRegion==(LOG_REGION_B|LOG_REGION_C)) - || (op==LOG_WRLOCK && mRegion==(LOG_REGION_A)) - || (op==LOG_UNLOCK && mRegion==(LOG_REGION_B|LOG_REGION_C)) - || (op==LOG_UNLOCK && mRegion==(LOG_REGION_A|LOG_REGION_B|LOG_REGION_C)) - ); - - /* Assert that a connection never tries to go from an EXCLUSIVE to a - ** SHARED lock on a region. Moving from SHARED to EXCLUSIVE sometimes - ** happens though (when a region D reader upgrades to a writer). - */ - assert( op!=LOG_RDLOCK || 0==(pLog->lock.mLock & (mRegion<<8)) ); - - sqlite3_mutex_enter(pSummary->mutex); - - /* Calculate a mask of logs held by all connections in this process apart - ** from this one. The least significant byte of the mask contains a mask - ** of the SHARED logs held. The next least significant byte of the mask - ** indicates the EXCLUSIVE locks held. For example, to test if some other - ** connection is holding a SHARED lock on region A, or an EXCLUSIVE lock - ** on region C, do: - ** - ** hasSharedOnA = (mOther & (LOG_REGION_A<<0)); - ** hasExclusiveOnC = (mOther & (LOG_REGION_C<<8)); - ** - ** In all masks, if the bit in the EXCLUSIVE byte mask is set, so is the - ** corresponding bit in the SHARED mask. - */ - mOther = 0; - for(p=pSummary->pLock; p; p=p->pNext){ - assert( (p->mLock & (p->mLock<<8))==(p->mLock&0x0000FF00) ); - if( p!=&pLog->lock ){ - mOther |= p->mLock; - } - } - - /* If this call is to lock a region (not to unlock one), test if locks held - ** by any other connection in this process prevent the new locks from - ** begin granted. If so, exit the summary mutex and return SQLITE_BUSY. - */ - if( op && (mOther & (mRegion << (op==LOG_RDLOCK ? 8 : 0))) ){ - sqlite3_mutex_leave(pSummary->mutex); - return SQLITE_BUSY; - } - - /* Figure out the new log mask for this connection. */ - switch( op ){ - case LOG_UNLOCK: - mNew = (pLog->lock.mLock & ~(mRegion|(mRegion<<8))); - break; - case LOG_RDLOCK: - mNew = (pLog->lock.mLock | mRegion); - break; - default: - assert( op==LOG_WRLOCK ); - mNew = (pLog->lock.mLock | (mRegion<<8) | mRegion); - break; - } - - /* Now modify the locks held on the log-summary file descriptor. This - ** file descriptor is shared by all log connections in this process. - ** Therefore: - ** - ** + If one or more log connections in this process hold a SHARED lock - ** on a region, the file-descriptor should hold a SHARED lock on - ** the file region. - ** - ** + If a log connection in this process holds an EXCLUSIVE lock on a - ** region, the file-descriptor should also hold an EXCLUSIVE lock on - ** the region in question. - ** - ** If this is an LOG_UNLOCK operation, only regions for which no other - ** connection holds a lock should actually be unlocked. And if this - ** is a LOG_RDLOCK operation and other connections already hold all - ** the required SHARED locks, then no system call is required. - */ - if( op==LOG_UNLOCK ){ - mRegion = (mRegion & ~mOther); - } - if( (op==LOG_WRLOCK) - || (op==LOG_UNLOCK && mRegion) - || (op==LOG_RDLOCK && (mOther&mRegion)!=mRegion) - ){ - struct LockMap { - int iStart; /* Byte offset to start locking operation */ - int iLen; /* Length field for locking operation */ - } aMap[] = { - /* 0000 */ {0, 0}, /* 0001 */ {4, 1}, - /* 0010 */ {3, 1}, /* 0011 */ {3, 2}, - /* 0100 */ {2, 1}, /* 0101 */ {0, 0}, - /* 0110 */ {2, 2}, /* 0111 */ {2, 3}, - /* 1000 */ {1, 1}, /* 1001 */ {0, 0}, - /* 1010 */ {0, 0}, /* 1011 */ {0, 0}, - /* 1100 */ {1, 2}, /* 1101 */ {0, 0}, - /* 1110 */ {0, 0}, /* 1111 */ {0, 0} - }; - int rc; /* Return code of fcntl() */ - struct flock f; /* Locking operation */ - - assert( mRegionfd, F_SETLK, &f); - if( rc!=0 ){ - sqlite3_mutex_leave(pSummary->mutex); - return SQLITE_BUSY; - } - } - - pLog->lock.mLock = mNew; - sqlite3_mutex_leave(pSummary->mutex); - return SQLITE_OK; -} - /* ** Try to read the log-summary header. Attempt to verify the header ** checksum. If the checksum can be verified, copy the log-summary diff --git a/src/pager.c b/src/pager.c index 3b16b86dd8..c2ed8601aa 100644 --- a/src/pager.c +++ b/src/pager.c @@ -491,6 +491,7 @@ static int assert_pager_state(Pager *pPager){ } #endif + /* ** Return true if it is necessary to write page *pPg into the sub-journal. ** A page needs to be written into the sub-journal if there exists one @@ -1187,19 +1188,6 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ return rc; } -/* -** Open a connection to the write-ahead log file for pager pPager. -*/ -static int pagerOpenLog(Pager *pPager){ - if( !pPager->pLog ){ - int rc; /* Return code from LogOpen() */ - - rc = sqlite3LogOpen(pPager->pVfs, pPager->zFilename, &pPager->pLog); - if( rc!=SQLITE_OK ) return rc; - } - return SQLITE_OK; -} - /* ** Return true if this pager uses a write-ahead log instead of the usual ** rollback journal. Otherwise false. @@ -1241,8 +1229,9 @@ static void pager_unlock(Pager *pPager){ if( pagerUseLog(pPager) ){ sqlite3LogCloseSnapshot(pPager->pLog); + }else{ + rc = osUnlock(pPager->fd, NO_LOCK); } - rc = osUnlock(pPager->fd, NO_LOCK); if( rc ){ pPager->errCode = rc; } @@ -3733,6 +3722,54 @@ static int hasHotJournal(Pager *pPager, int *pExists){ return rc; } +/* +** Open a connection to the write-ahead log file for pager pPager. If +** the log connection is already open, this function is a no-op. +*/ +static int pagerOpenLog(Pager *pPager){ + if( !pPager->pLog ){ + int rc; /* Return code */ + + /* Before opening the log file, obtain a SHARED lock on the database + ** file. This lock will not be released until after the log file + ** connection has been closed. The purpose of this lock is to stop + ** any other process from unlinking the log or log-summary files while + ** this connection still has them open. An EXCLUSIVE lock on the + ** database file is required to unlink either of those two files. + */ + assert( pPager->state==PAGER_UNLOCK ); + rc = pager_wait_on_lock(pPager, SHARED_LOCK); + if( rc!=SQLITE_OK ){ + assert( pPager->state==PAGER_UNLOCK ); + return pager_error(pPager, rc); + } + assert( pPager->state>=SHARED_LOCK ); + + /* Open the connection to the log file. If this operation fails, + ** (e.g. due to malloc() failure), unlock the database file and + ** return an error code. + */ + rc = sqlite3LogOpen(pPager->pVfs, pPager->zFilename, &pPager->pLog); + if( rc!=SQLITE_OK ){ + osUnlock(pPager->fd, SQLITE_LOCK_NONE); + pPager->state = PAGER_UNLOCK; + return rc; + } + }else{ + /* If the log file was already open, check that the pager is still holding + ** the required SHARED lock on the database file. + */ +#ifdef SQLITE_DEBUG + int locktype; + sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_LOCKSTATE, &locktype); + assert( locktype==SQLITE_LOCK_SHARED ); +#endif + } + + return SQLITE_OK; +} + + /* ** This function is called to obtain a shared lock on the database file. ** It is illegal to call sqlite3PagerAcquire() until after this function @@ -3786,17 +3823,25 @@ int sqlite3PagerSharedLock(Pager *pPager){ pager_reset(pPager); } - if( pagerUseLog(pPager) ){ - int changed = 0; + if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){ + int changed = 0; /* True if the cache must be flushed */ + + /* Open the log file, if it is not already open. */ + rc = pagerOpenLog(pPager); + if( rc!=SQLITE_OK ){ + return rc; + } + + /* Open a log snapshot to read from. */ rc = sqlite3LogOpenSnapshot(pPager->pLog, &changed); if( rc==SQLITE_OK ){ + int dummy; if( changed ){ pager_reset(pPager); assert( pPager->errCode || pPager->dbSizeValid==0 ); } - pPager->state = PAGER_SHARED; /* TODO: Is this right? */ - rc = sqlite3PagerPagecount(pPager, &changed); + rc = sqlite3PagerPagecount(pPager, &dummy); } }else if( pPager->state==PAGER_UNLOCK || isErrorReset ){ sqlite3_vfs * const pVfs = pPager->pVfs; @@ -5611,15 +5656,13 @@ int sqlite3PagerJournalMode(Pager *pPager, int eMode){ sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); } - if( eMode==PAGER_JOURNALMODE_WAL ){ - int rc = pagerOpenLog(pPager); - if( rc!=SQLITE_OK ){ - /* TODO: The error code should not just get dropped here. Change - ** this to set a flag to force the log to be opened the first time - ** it is actually required. */ - return (int)pPager->journalMode; - } + /* Switching into WAL mode can only take place when no + ** locks are held on the database file. + */ + if( eMode==PAGER_JOURNALMODE_WAL && pPager->state!=PAGER_UNLOCK ){ + return (int)pPager->journalMode; } + pPager->journalMode = (u8)eMode; } return (int)pPager->journalMode; diff --git a/test/wal.test b/test/wal.test index a41b263c1b..5484d1eafa 100644 --- a/test/wal.test +++ b/test/wal.test @@ -32,6 +32,10 @@ proc log_file_size {nFrame pgsz} { expr {12 + ($pgsz+16)*$nFrame} } +proc log_deleted {logfile} { + return [expr [file exists $logfile]==0] +} + # # These are 'warm-body' tests used while developing the WAL code. They # serve to prove that a few really simple cases work: @@ -179,7 +183,6 @@ do_test wal-5.5 { } {1 2 3 4} db close - foreach sector {512 4096} { sqlite3_simulate_device -sectorsize $sector foreach pgsz {512 1024 2048 4096} { @@ -198,8 +201,8 @@ foreach sector {512 4096} { } [expr $pgsz*2] do_test wal-6.$sector.$pgsz.2 { - file size test.db-wal - } {0} + log_deleted test.db-wal + } {1} } } @@ -591,8 +594,8 @@ do_test wal-11.8 { do_test wal-11.9 { db close sqlite3_wal db test.db - list [expr [file size test.db]/1024] [file size test.db-wal] -} {37 0} + list [expr [file size test.db]/1024] [log_deleted test.db-wal] +} {37 1} do_test wal-11.10 { execsql { diff --git a/test/walthread.test b/test/walthread.test index fbd6c653cc..08219a7f28 100644 --- a/test/walthread.test +++ b/test/walthread.test @@ -21,12 +21,14 @@ if {[run_thread_tests]==0} { finish_test ; return } do_test walthread-1.1 { execsql { PRAGMA journal_mode = WAL; + PRAGMA lock_status; CREATE TABLE t1(x PRIMARY KEY); + PRAGMA lock_status; INSERT INTO t1 VALUES(randomblob(100)); INSERT INTO t1 VALUES(randomblob(100)); INSERT INTO t1 SELECT md5sum(x) FROM t1; } -} {wal} +} {wal main unlocked temp closed main shared temp closed} do_test walthread-1.2 { execsql { SELECT (SELECT count(*) FROM t1), ( @@ -41,7 +43,7 @@ do_test walthread-1.3 { } {ok} do_test walthread-1.4 { execsql { PRAGMA lock_status } -} {main unlocked temp unknown} +} {main shared temp unknown} #-------------------------------------------------------------------------- # Start N threads. Each thread performs both read and write transactions. From 6703239d2f024880fcbfc849282abf5e40633573 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 17 Apr 2010 15:42:43 +0000 Subject: [PATCH 20/89] In synchronous=normal mode, do not sync the log after every transaction. In synchronous=full mode, sync the log and add any extra frames required to avoid blast-radius related problems after each transaction. FossilOrigin-Name: 9bc9b6847303d0324543a9ded8dd0473490122d8 --- manifest | 16 ++++++------ manifest.uuid | 2 +- src/log.c | 8 +++--- src/pager.c | 3 ++- test/wal.test | 70 ++++++++++++++++++++++++++++----------------------- 5 files changed, 54 insertions(+), 45 deletions(-) diff --git a/manifest b/manifest index dacfee46f5..41f3141c14 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhancements\sto\swal-mode\slocking\sscheme. -D 2010-04-17T12:31:37 +C In\ssynchronous=normal\smode,\sdo\snot\ssync\sthe\slog\safter\severy\stransaction.\sIn\ssynchronous=full\smode,\ssync\sthe\slog\sand\sadd\sany\sextra\sframes\srequired\sto\savoid\sblast-radius\srelated\sproblems\safter\seach\stransaction. +D 2010-04-17T15:42:43 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c 6ac96c84ca4564f156de564ceddb3c1a2319ac6e +F src/log.c a72baea84cecef9a4e45308b1504e6fe69c8284e F src/log.h a2654af46ce7b5732f4d5a731abfdd180f0a06d9 F src/main.c c0e7192bad5b90544508b241eb2487ac661de890 F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a @@ -154,7 +154,7 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053 -F src/pager.c 674d6558a618ca7714ba28741e6dc681692bffe2 +F src/pager.c 751ada65b9a4aa0b31c36ffa3f6548200a55ca16 F src/pager.h ce5d076f3860a5f2d7460c582cd68383343b33cf F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf @@ -757,7 +757,7 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test 5fa3cdf2e93e79b0891af6cf7fab8ef9e07a23c6 +F test/wal.test a56ff378f58b145fd3bf38c277fbfe792cd47bdd F test/walcrash.test 45cfbab30bb7cbe0b2e9d5cabe90dbcad10cb89b F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 F test/walthread.test 27e44ee6fd02f1f494a24f999c97086af3ab739d @@ -804,7 +804,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 9865d14d6041874bc1239ce7a061d5c75f2d33c9 -R f6bb150cf1f3c0e2bfeac5bd76d233a7 +P 8549c286497f3d2cd118be1334fce00d6f8a26c4 +R 6ac8dd31103bf297ef819af9a9f02a12 U dan -Z ea6a1c06d626e2f5d8edb2ab802e8ea3 +Z 35f390be152b1baf460a096be51e37de diff --git a/manifest.uuid b/manifest.uuid index 8375084062..d5b1aacc94 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8549c286497f3d2cd118be1334fce00d6f8a26c4 \ No newline at end of file +9bc9b6847303d0324543a9ded8dd0473490122d8 \ No newline at end of file diff --git a/src/log.c b/src/log.c index f0b3a73ebf..4253d659af 100644 --- a/src/log.c +++ b/src/log.c @@ -881,7 +881,7 @@ int sqlite3LogOpen( zWal = sqlite3_mprintf("%s-wal", zDb); if( !zWal ) goto out; logNormalizePath(zWal); - flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_MAIN_DB); + flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_MAIN_JOURNAL); nWal = sqlite3Strlen30(zWal); /* Enter the mutex that protects the linked-list of LogSummary structures */ @@ -1545,9 +1545,10 @@ int sqlite3LogFrames( /* Sync the log file if the 'isSync' flag was specified. */ if( isSync ){ -#if 0 i64 iSegment = sqlite3OsSectorSize(pLog->pFd); - i64 iOffset = iFrame * (nPgsz+sizeof(aFrame)); + i64 iOffset = logFrameOffset(iFrame+1, nPgsz); + + assert( isCommit ); if( iSegmentpFd, pLog->sync_flags); if( rc!=SQLITE_OK ){ diff --git a/src/pager.c b/src/pager.c index c2ed8601aa..68d561400f 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3764,6 +3764,7 @@ static int pagerOpenLog(Pager *pPager){ sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_LOCKSTATE, &locktype); assert( locktype==SQLITE_LOCK_SHARED ); #endif + pPager->state = PAGER_SHARED; } return SQLITE_OK; @@ -4880,7 +4881,7 @@ int sqlite3PagerCommitPhaseOne( PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); if( pList ){ rc = sqlite3LogFrames(pPager->pLog, pPager->pageSize, pList, - pPager->dbSize, 1, 1 + pPager->dbSize, 1, pPager->fullSync ); } sqlite3PcacheCleanAll(pPager->pPCache); diff --git a/test/wal.test b/test/wal.test index 5484d1eafa..fb21d820f9 100644 --- a/test/wal.test +++ b/test/wal.test @@ -23,9 +23,17 @@ proc reopen_db {} { sqlite3_wal db test.db } +set ::blobcnt 0 +proc blob {nByte} { + incr ::blobcnt + return [string range [string repeat "${::blobcnt}x" $nByte] 1 $nByte] +} + proc sqlite3_wal {args} { eval sqlite3 $args [lindex $args 0] eval { PRAGMA journal_mode = wal } + [lindex $args 0] eval { PRAGMA synchronous = normal } + [lindex $args 0] function blob blob } proc log_file_size {nFrame pgsz} { @@ -49,6 +57,7 @@ proc log_deleted {logfile} { # do_test wal-0.1 { + execsql { PRAGMA synchronous = normal } execsql { PRAGMA journal_mode = wal } } {wal} @@ -235,13 +244,13 @@ do_test wal-8.2 { execsql { PRAGMA page_size = 1024; CREATE TABLE t1(x); - INSERT INTO t1 VALUES(randomblob(900)); - INSERT INTO t1 VALUES(randomblob(900)); - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 32 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 64 */ + INSERT INTO t1 VALUES(blob(900)); + INSERT INTO t1 VALUES(blob(900)); + INSERT INTO t1 SELECT blob(900) FROM t1; /* 4 */ + INSERT INTO t1 SELECT blob(900) FROM t1; /* 8 */ + INSERT INTO t1 SELECT blob(900) FROM t1; /* 16 */ + INSERT INTO t1 SELECT blob(900) FROM t1; /* 32 */ + INSERT INTO t1 SELECT blob(900) FROM t1; /* 64 */ PRAGMA checkpoint; } file size test.db @@ -262,15 +271,15 @@ do_test wal-9.1 { execsql { PRAGMA page_size = 1024; CREATE TABLE t1(x PRIMARY KEY); - INSERT INTO t1 VALUES(randomblob(900)); - INSERT INTO t1 VALUES(randomblob(900)); - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 32 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 64 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 128 */ - INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 256 */ + INSERT INTO t1 VALUES(blob(900)); + INSERT INTO t1 VALUES(blob(900)); + INSERT INTO t1 SELECT blob(900) FROM t1; /* 4 */ + INSERT INTO t1 SELECT blob(900) FROM t1; /* 8 */ + INSERT INTO t1 SELECT blob(900) FROM t1; /* 16 */ + INSERT INTO t1 SELECT blob(900) FROM t1; /* 32 */ + INSERT INTO t1 SELECT blob(900) FROM t1; /* 64 */ + INSERT INTO t1 SELECT blob(900) FROM t1; /* 128 */ + INSERT INTO t1 SELECT blob(900) FROM t1; /* 256 */ } file size test.db } 0 @@ -557,20 +566,20 @@ do_test wal-11.2 { list [expr [file size test.db]/1024] [file size test.db-wal] } [list 3 [log_file_size 3 1024]] do_test wal-11.3 { - execsql { INSERT INTO t1 VALUES( randomblob(900) ) } + execsql { INSERT INTO t1 VALUES( blob(900) ) } list [expr [file size test.db]/1024] [file size test.db-wal] } [list 3 [log_file_size 4 1024]] do_test wal-11.4 { execsql { BEGIN; - INSERT INTO t1 SELECT randomblob(900) FROM t1; -- 2 - INSERT INTO t1 SELECT randomblob(900) FROM t1; -- 4 - INSERT INTO t1 SELECT randomblob(900) FROM t1; -- 8 - INSERT INTO t1 SELECT randomblob(900) FROM t1; -- 16 + INSERT INTO t1 SELECT blob(900) FROM t1; -- 2 + INSERT INTO t1 SELECT blob(900) FROM t1; -- 4 + INSERT INTO t1 SELECT blob(900) FROM t1; -- 8 + INSERT INTO t1 SELECT blob(900) FROM t1; -- 16 } list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 3 [log_file_size 33 1024]] +} [list 3 [log_file_size 32 1024]] do_test wal-11.5 { execsql { SELECT count(*) FROM t1; @@ -580,7 +589,7 @@ do_test wal-11.5 { do_test wal-11.6 { execsql COMMIT list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 3 [log_file_size 42 1024]] +} [list 3 [log_file_size 41 1024]] do_test wal-11.7 { execsql { SELECT count(*) FROM t1; @@ -590,22 +599,21 @@ do_test wal-11.7 { do_test wal-11.8 { execsql { PRAGMA checkpoint } list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [log_file_size 42 1024]] +} [list 37 [log_file_size 41 1024]] do_test wal-11.9 { db close - sqlite3_wal db test.db list [expr [file size test.db]/1024] [log_deleted test.db-wal] } {37 1} - +sqlite3_wal db test.db do_test wal-11.10 { execsql { PRAGMA cache_size = 10; BEGIN; - INSERT INTO t1 SELECT randomblob(900) FROM t1; -- 32 + INSERT INTO t1 SELECT blob(900) FROM t1; -- 32 SELECT count(*) FROM t1; } list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [log_file_size 40 1024]] +} [list 37 [log_file_size 35 1024]] do_test wal-11.11 { execsql { SELECT count(*) FROM t1; @@ -615,17 +623,17 @@ do_test wal-11.11 { } {32 16} do_test wal-11.12 { list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [log_file_size 40 1024]] +} [list 37 [log_file_size 35 1024]] do_test wal-11.13 { execsql { - INSERT INTO t1 VALUES( randomblob(900) ); + INSERT INTO t1 VALUES( blob(900) ); SELECT count(*) FROM t1; PRAGMA integrity_check; } } {17 ok} do_test wal-11.14 { list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [log_file_size 40 1024]] +} [list 37 [log_file_size 35 1024]] #------------------------------------------------------------------------- From c511878955599619f021a91ddfc05e22d8cbf668 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 17 Apr 2010 17:34:41 +0000 Subject: [PATCH 21/89] Do not sync any files in wal mode if "PRAGMA synchronous=off" is set. If files are synced, pass either SQLITE_SYNC_FULL or SQLITE_SYNC_NORMAL to the xSync() callback as configured by "PRAGMA fullfsync". FossilOrigin-Name: 0ae91b0008b242a47385fc1f295c6b645483ee22 --- manifest | 16 +++++------ manifest.uuid | 2 +- src/log.c | 74 +++++++++++---------------------------------------- src/log.h | 18 +++---------- src/pager.c | 7 +++-- 5 files changed, 34 insertions(+), 83 deletions(-) diff --git a/manifest b/manifest index 2d93f8ada9..628fe3b0e0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\swith\strunk\scommit\s[3e646e3f4c]. -D 2010-04-17T15:45:35 +C Do\snot\ssync\sany\sfiles\sin\swal\smode\sif\s"PRAGMA\ssynchronous=off"\sis\sset.\sIf\sfiles\sare\ssynced,\spass\seither\sSQLITE_SYNC_FULL\sor\sSQLITE_SYNC_NORMAL\sto\sthe\sxSync()\scallback\sas\sconfigured\sby\s"PRAGMA\sfullfsync". +D 2010-04-17T17:34:42 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,8 +131,8 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c a72baea84cecef9a4e45308b1504e6fe69c8284e -F src/log.h a2654af46ce7b5732f4d5a731abfdd180f0a06d9 +F src/log.c 8bcd490de703a12563c939a59cb4e187a0c67fe3 +F src/log.h 3cfec02156f5db3d233352897a47601930a29955 F src/main.c c0e7192bad5b90544508b241eb2487ac661de890 F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -154,7 +154,7 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053 -F src/pager.c 751ada65b9a4aa0b31c36ffa3f6548200a55ca16 +F src/pager.c 344eddf3c59eed4b4aca31abbf762d0e301ae44d F src/pager.h ce5d076f3860a5f2d7460c582cd68383343b33cf F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf @@ -805,7 +805,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 9bc9b6847303d0324543a9ded8dd0473490122d8 3e646e3f4cd0ca288e444561e951cecfdaee2ab5 -R 9ec1fc417b85c6217c6e7a04071a1912 +P 43463970f5885fb116588695146f2a56cb22804a +R 0e49b7b76893a9b984350c6fdab0ea9a U dan -Z 5c9ba544c6cd36a35ee164445a4a1f25 +Z ad3f9c0ef28635a13cd67b8f240d7d22 diff --git a/manifest.uuid b/manifest.uuid index d195514289..ed4b8c00cd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -43463970f5885fb116588695146f2a56cb22804a \ No newline at end of file +0ae91b0008b242a47385fc1f295c6b645483ee22 \ No newline at end of file diff --git a/src/log.c b/src/log.c index 4253d659af..a6d7f35440 100644 --- a/src/log.c +++ b/src/log.c @@ -140,7 +140,6 @@ struct Log { LogSummary *pSummary; /* Log file summary data */ sqlite3_vfs *pVfs; /* The VFS used to create pFd */ sqlite3_file *pFd; /* File handle for log file */ - int sync_flags; /* Flags to use with OsSync() */ int isLocked; /* Non-zero if a snapshot is held open */ int isWriteLocked; /* True if this is the writer connection */ LogSummaryHdr hdr; /* Log summary header for current snapshot */ @@ -242,42 +241,6 @@ static void logNormalizePath(char *zPath){ z[j] = 0; } -/* -** Lock the summary file pSummary->fd. -*/ -static int logSummaryLock(LogSummary *pSummary){ - int rc; - struct flock f; - memset(&f, 0, sizeof(f)); - f.l_type = F_WRLCK; - f.l_whence = SEEK_SET; - f.l_start = 0; - f.l_len = 1; - rc = fcntl(pSummary->fd, F_SETLKW, &f); - if( rc!=0 ){ - return SQLITE_IOERR; - } - return SQLITE_OK; -} - -/* -** Unlock the summary file pSummary->fd. -*/ -static int logSummaryUnlock(LogSummary *pSummary){ - int rc; - struct flock f; - memset(&f, 0, sizeof(f)); - f.l_type = F_UNLCK; - f.l_whence = SEEK_SET; - f.l_start = 0; - f.l_len = 1; - rc = fcntl(pSummary->fd, F_SETLK, &f); - if( rc!=0 ){ - return SQLITE_IOERR; - } - return SQLITE_OK; -} - /* ** Memory map the first nByte bytes of the summary file opened with ** pSummary->fd at pSummary->aData. If the summary file is smaller than @@ -875,7 +838,6 @@ int sqlite3LogOpen( if( !pRet ) goto out; pRet->pVfs = pVfs; pRet->pFd = (sqlite3_file *)&pRet[1]; - pRet->sync_flags = SQLITE_SYNC_NORMAL; /* Normalize the path name. */ zWal = sqlite3_mprintf("%s-wal", zDb); @@ -1038,6 +1000,7 @@ static void logIteratorFree(LogIterator *p){ static int logCheckpoint( Log *pLog, /* Log connection */ sqlite3_file *pFd, /* File descriptor open on db file */ + int sync_flags, /* Flags for OsSync() (or 0) */ u8 *zBuf /* Temporary buffer to use */ ){ int rc; /* Return code */ @@ -1055,8 +1018,10 @@ static int logCheckpoint( if( !pIter ) return SQLITE_NOMEM; /* Sync the log file to disk */ - rc = sqlite3OsSync(pLog->pFd, pLog->sync_flags); - if( rc!=SQLITE_OK ) goto out; + if( sync_flags ){ + rc = sqlite3OsSync(pLog->pFd, sync_flags); + if( rc!=SQLITE_OK ) goto out; + } /* Iterate through the contents of the log, copying data to the db file. */ while( 0==logIteratorNext(pIter, &iDbpage, &iFrame) ){ @@ -1073,8 +1038,10 @@ static int logCheckpoint( if( rc!=SQLITE_OK ) goto out; /* Sync the database file. If successful, update the log-summary. */ - rc = sqlite3OsSync(pFd, pLog->sync_flags); - if( rc!=SQLITE_OK ) goto out; + if( sync_flags ){ + rc = sqlite3OsSync(pFd, sync_flags); + if( rc!=SQLITE_OK ) goto out; + } pLog->hdr.iLastPg = 0; pLog->hdr.iCheck1 = 2; pLog->hdr.iCheck2 = 3; @@ -1111,6 +1078,7 @@ static int logCheckpoint( int sqlite3LogClose( Log *pLog, /* Log to close */ sqlite3_file *pFd, /* Database file */ + int sync_flags, /* Flags to pass to OsSync() (or 0) */ u8 *zBuf /* Buffer of at least page-size bytes */ ){ int rc = SQLITE_OK; @@ -1154,7 +1122,7 @@ int sqlite3LogClose( ** 2. Truncate the log file. ** 3. Unlink the log-summary file. */ - rc = logCheckpoint(pLog, pFd, zBuf); + rc = logCheckpoint(pLog, pFd, sync_flags, zBuf); if( rc==SQLITE_OK ){ rc = sqlite3OsDelete(pLog->pVfs, pSummary->zPath, 0); } @@ -1181,17 +1149,6 @@ int sqlite3LogClose( return rc; } -/* -** Set the flags to pass to the sqlite3OsSync() function when syncing -** the log file. -*/ -#if 0 -void sqlite3LogSetSyncflags(Log *pLog, int sync_flags){ - assert( sync_flags==SQLITE_SYNC_NORMAL || sync_flags==SQLITE_SYNC_FULL ); - pLog->sync_flags = sync_flags; -} -#endif - /* ** Enter and leave the log-summary mutex. In this context, entering the ** log-summary mutex means: @@ -1487,7 +1444,7 @@ int sqlite3LogFrames( PgHdr *pList, /* List of dirty pages to write */ Pgno nTruncate, /* Database size after this commit */ int isCommit, /* True if this is a commit */ - int isSync /* True to sync the log file */ + int sync_flags /* Flags to pass to OsSync() (or 0) */ ){ int rc; /* Used to catch return codes */ u32 iFrame; /* Next frame address */ @@ -1544,7 +1501,7 @@ int sqlite3LogFrames( } /* Sync the log file if the 'isSync' flag was specified. */ - if( isSync ){ + if( sync_flags ){ i64 iSegment = sqlite3OsSectorSize(pLog->pFd); i64 iOffset = logFrameOffset(iFrame+1, nPgsz); @@ -1570,7 +1527,7 @@ int sqlite3LogFrames( iOffset += nPgsz; } - rc = sqlite3OsSync(pLog->pFd, pLog->sync_flags); + rc = sqlite3OsSync(pLog->pFd, sync_flags); if( rc!=SQLITE_OK ){ return rc; } @@ -1623,6 +1580,7 @@ int sqlite3LogFrames( int sqlite3LogCheckpoint( Log *pLog, /* Log connection */ sqlite3_file *pFd, /* File descriptor open on db file */ + int sync_flags, /* Flags to sync db file with (or 0) */ u8 *zBuf, /* Temporary buffer to use */ int (*xBusyHandler)(void *), /* Pointer to busy-handler function */ void *pBusyHandlerArg /* Argument to pass to xBusyHandler */ @@ -1649,7 +1607,7 @@ int sqlite3LogCheckpoint( /* Copy data from the log to the database file. */ rc = logSummaryReadHdr(pLog, 0); if( rc==SQLITE_OK ){ - rc = logCheckpoint(pLog, pFd, zBuf); + rc = logCheckpoint(pLog, pFd, sync_flags, zBuf); } /* Release the locks. */ diff --git a/src/log.h b/src/log.h index 816f9354eb..5da5fb1051 100644 --- a/src/log.h +++ b/src/log.h @@ -19,25 +19,14 @@ #include "sqliteInt.h" -/* Flags that may be set in the 'flags' argument to sqlite3LogWrite(): */ -#define LOG_MASK_COMMIT 0x08 -#define LOG_MASK_MASTERJOURNAL 0x10 -#define LOG_MASK_TRUNCATE 0x20 - - -#define LOG_TRUNCATE_BIT 0x80000000 - /* Connection to a log file. There is one object of this type for each pager. */ typedef struct Log Log; /* Open and close a connection to a log file. */ int sqlite3LogOpen(sqlite3_vfs*, const char *zDb, Log **ppLog); -int sqlite3LogClose(Log *pLog, sqlite3_file *pFd, u8 *zBuf); +int sqlite3LogClose(Log *pLog, sqlite3_file *pFd, int sync_flags, u8 *zBuf); -/* Configure the log connection. */ -void sqlite3LogSetSyncflags(Log *, int sync_flags); - -/* Used by readers to open (lock) and close (unlock) a database snapshot. */ +/* Used by readers to open (lock) and close (unlock) a snapshot. */ int sqlite3LogOpenSnapshot(Log *pLog, int *); void sqlite3LogCloseSnapshot(Log *pLog); @@ -48,13 +37,14 @@ void sqlite3LogMaxpgno(Log *pLog, Pgno *pPgno); /* Obtain or release the WRITER lock. */ int sqlite3LogWriteLock(Log *pLog, int op); -/* Write a segment to the log. */ +/* Write a frame or frames to the log. */ int sqlite3LogFrames(Log *pLog, int, PgHdr *, Pgno, int, int); /* Copy pages from the log to the database file */ int sqlite3LogCheckpoint( Log *pLog, /* Log connection */ sqlite3_file *pFd, /* File descriptor open on db file */ + int sync_flags, /* Flags to sync db file with (or 0) */ u8 *zBuf, /* Temporary buffer to use */ int (*xBusyHandler)(void *), /* Pointer to busy-handler function */ void *pBusyHandlerArg /* Argument to pass to xBusyHandler */ diff --git a/src/pager.c b/src/pager.c index 68d561400f..5fb259ff73 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2871,7 +2871,9 @@ int sqlite3PagerClose(Pager *pPager){ sqlite3BeginBenignMalloc(); pPager->errCode = 0; pPager->exclusiveMode = 0; - sqlite3LogClose(pPager->pLog, pPager->fd, pTmp); + sqlite3LogClose(pPager->pLog, pPager->fd, + (pPager->noSync ? 0 : pPager->sync_flags), pTmp + ); pPager->pLog = 0; pager_reset(pPager); if( MEMDB ){ @@ -4881,7 +4883,7 @@ int sqlite3PagerCommitPhaseOne( PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); if( pList ){ rc = sqlite3LogFrames(pPager->pLog, pPager->pageSize, pList, - pPager->dbSize, 1, pPager->fullSync + pPager->dbSize, 1, (pPager->fullSync ? pPager->sync_flags : 0) ); } sqlite3PcacheCleanAll(pPager->pPCache); @@ -5700,6 +5702,7 @@ int sqlite3PagerCheckpoint(Pager *pPager){ if( pPager->pLog ){ u8 *zBuf = (u8 *)pPager->pTmpSpace; rc = sqlite3LogCheckpoint(pPager->pLog, pPager->fd, + (pPager->noSync ? 0 : pPager->sync_flags), zBuf, pPager->xBusyHandler, pPager->pBusyHandlerArg ); } From 54934f4689ee181d366987cfe746dc2ed79c35be Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 17 Apr 2010 18:50:27 +0000 Subject: [PATCH 22/89] Add some comments regarding file-locks to log.c. FossilOrigin-Name: 9d51c3b754f0b94fea5ef3d669ad583b93b2b024 --- manifest | 12 +++---- manifest.uuid | 2 +- src/log.c | 91 ++++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 94 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 628fe3b0e0..e3133fe899 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\ssync\sany\sfiles\sin\swal\smode\sif\s"PRAGMA\ssynchronous=off"\sis\sset.\sIf\sfiles\sare\ssynced,\spass\seither\sSQLITE_SYNC_FULL\sor\sSQLITE_SYNC_NORMAL\sto\sthe\sxSync()\scallback\sas\sconfigured\sby\s"PRAGMA\sfullfsync". -D 2010-04-17T17:34:42 +C Add\ssome\scomments\sregarding\sfile-locks\sto\slog.c. +D 2010-04-17T18:50:28 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c 8bcd490de703a12563c939a59cb4e187a0c67fe3 +F src/log.c fd6631f740b0cf2c8720477a036ffee8ca705c92 F src/log.h 3cfec02156f5db3d233352897a47601930a29955 F src/main.c c0e7192bad5b90544508b241eb2487ac661de890 F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a @@ -805,7 +805,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 43463970f5885fb116588695146f2a56cb22804a -R 0e49b7b76893a9b984350c6fdab0ea9a +P 0ae91b0008b242a47385fc1f295c6b645483ee22 +R a205bea3ab4fa24118ce751726c3c719 U dan -Z ad3f9c0ef28635a13cd67b8f240d7d22 +Z c979f9f88d66076be76b02e7ab3fa216 diff --git a/manifest.uuid b/manifest.uuid index ed4b8c00cd..26af5e1525 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0ae91b0008b242a47385fc1f295c6b645483ee22 \ No newline at end of file +9d51c3b754f0b94fea5ef3d669ad583b93b2b024 \ No newline at end of file diff --git a/src/log.c b/src/log.c index a6d7f35440..81f808aa6c 100644 --- a/src/log.c +++ b/src/log.c @@ -103,6 +103,93 @@ struct LogSummary { u32 *aData; /* File body */ }; +/* +** This module uses three different types of file-locks. All are taken +** on the log-summary file. The three types of locks are as follows: +** +** MUTEX: The MUTEX lock is used as a robust inter-process mutex. It +** is held while the log-summary header is modified, and +** sometimes when it is read. It is also held while a new client +** obtains the DMH lock (see below), and while log recovery is +** being run. +** +** DMH: The DMH (Dead Mans Hand mechanism) lock is used to ensure +** that log-recovery is always run following a system restart. +** When it first opens a log-summary file, a process takes a +** SHARED lock on the DMH region. This lock is not released until +** the log-summary file is closed. +** +** The process then attempts to upgrade to an EXCLUSIVE lock. If +** successful, then the contents of the log-summary file are deemed +** suspect and the log-summary header zeroed. This forces the +** first process that reads the log-summary file to run log +** recovery. After zeroing the log-summary header, the process +** downgrades to a SHARED lock on the DMH region. +** +** If the attempt to obtain the EXCLUSIVE lock fails, then the +** process concludes that some other process is already using the +** log-summary file, and it can therefore be trusted. +** +** The procedure described in the previous three paragraphs (taking +** a SHARED lock and then upgrading to an EXCLUSIVE lock to check +** if the process is the only one to have an open connection to the +** log file) is protected by holding the MUTEX lock. This avoids the +** race condition wherein the first two clients connect almost +** simultaneously following a system restart and each prevents +** the other from obtaining the EXCLUSIVE lock. +** +** +** REGION: There are 4 different region locks, regions A, B, C and D. +** Various EXCLUSIVE and SHARED locks on these regions are obtained +** when a client reads, writes or checkpoints the database. +** +** To obtain a reader lock: +** +** 1. Attempt a SHARED lock on regions A and B. +** 2. If step 1 is successful, drop the lock on region B. Or, if +** it is unsuccessful, attempt a SHARED lock on region D. +** 3. Repeat the above until the lock attempt in step 1 or 2 is +** successful. +** +** The reader lock is released when the read transaction is finished. +** +** To obtain a writer lock: +** +** 1. Take (wait for) an EXCLUSIVE lock on regions C and D. +** +** The locks are released after the write transaction is finished +** and, if any frames were committed to the log, the log-summary +** file updated. +** +** To obtain a checkpointer lock: +** +** 1. Take (wait for) an EXCLUSIVE lock on regions B and C. +** 2. Take (wait for) an EXCLUSIVE lock on region A. +** +** Step 1 waits until any existing writer has finished. And forces +** all new readers to become "region D" readers. +** +** Step 2 causes the checkpointer to wait until all existing region A +** readers have finished their transactions. Once the exclusive lock +** on region A has been obtained, only "region D" readers exist. +** These readers are operating on the snapshot at the head of the +** log. As such, the log can be safely copied into the database file +** without interfering with the readers. +** +** Once the checkpoint has finished and the log-summary header +** updated (to indicate the log contents can now be ignored), all +** locks are released. +** +** However, there may still exist region D readers using data in +** the body of the log file, so the log file itself cannot be +** truncated or overwritten until all region D readers have finished. +** That requirement is satisfied, because writers (the clients that +** write to the log file) require an exclusive lock on region D. +** Which they cannot get until all region D readers have finished. +*/ +#define LOG_LOCK_MUTEX 12 +#define LOG_LOCK_DMH 13 +#define LOG_LOCK_REGION 14 /* ** The four lockable regions associated with each log-summary. A connection @@ -115,10 +202,6 @@ struct LogSummary { #define LOG_REGION_C 0x04 #define LOG_REGION_D 0x08 -#define LOG_LOCK_MUTEX 12 -#define LOG_LOCK_DMH 13 -#define LOG_LOCK_REGION 14 - /* ** A single instance of this structure is allocated as part of each ** connection to a database log. All structures associated with the From 8d22a174115152a0a6a1d55ee7e7a855398b62eb Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 19 Apr 2010 18:03:51 +0000 Subject: [PATCH 23/89] Add the sqlite3_log_hook() interface for scheduling checkpoints. FossilOrigin-Name: 9bda601455705475075e33bfa85687bce34b15ff --- manifest | 33 ++++++++++---------- manifest.uuid | 2 +- src/log.c | 74 +++++++++++++++++++++++++++++++-------------- src/log.h | 5 ++- src/main.c | 18 +++++++++++ src/pager.c | 6 +++- src/pager.h | 2 ++ src/sqlite.h.in | 34 +++++++++++++++++++++ src/sqliteInt.h | 2 ++ src/tclsqlite.c | 59 ++++++++++++++++++++++++++++-------- src/vdbeapi.c | 25 +++++++++++++++ src/vdbeaux.c | 2 +- test/tclsqlite.test | 2 +- test/walhook.test | 73 ++++++++++++++++++++++++++++++++++++++++++++ 14 files changed, 281 insertions(+), 56 deletions(-) create mode 100644 test/walhook.test diff --git a/manifest b/manifest index e3133fe899..a4e1d8ff29 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssome\scomments\sregarding\sfile-locks\sto\slog.c. -D 2010-04-17T18:50:28 +C Add\sthe\ssqlite3_log_hook()\sinterface\sfor\sscheduling\scheckpoints. +D 2010-04-19T18:03:52 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,9 +131,9 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c fd6631f740b0cf2c8720477a036ffee8ca705c92 -F src/log.h 3cfec02156f5db3d233352897a47601930a29955 -F src/main.c c0e7192bad5b90544508b241eb2487ac661de890 +F src/log.c a5bbeb05c456ee7ee9d4f3dba5e72bae2d9b6530 +F src/log.h b8c45a6348d9ef57c5205a08c611d57d07ee9feb +F src/main.c 867de6aa444abd97771b2b70472f448d65c1c77e F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 89d4ea8d5cdd55635cbaa48ad53132af6294cbb2 @@ -154,8 +154,8 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053 -F src/pager.c 344eddf3c59eed4b4aca31abbf762d0e301ae44d -F src/pager.h ce5d076f3860a5f2d7460c582cd68383343b33cf +F src/pager.c f9ad51dfa617131240ae77545c0083118324fc81 +F src/pager.h aeaee9ceb4a4a6d15c2163ef4e11a7590fd54b59 F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 @@ -168,13 +168,13 @@ F src/resolve.c ac5f1a713cd1ae77f08b83cc69581e11bf5ae6f9 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c b7c9a40bc1567bceff52ad4b73108734ee4bf268 F src/shell.c c40427c7245535a04a9cb4a417b6cc05c022e6a4 -F src/sqlite.h.in dc98616304e3e776008655671d81e3ad3028ada7 +F src/sqlite.h.in b9b0365613d4a99a0e6bc0ae6dbc73cc4a7e7cb7 F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89 -F src/sqliteInt.h bd7ff54663bdd5b57e0eb8b49aca5a3a3c60119a +F src/sqliteInt.h 6ed0ed77103d00006d6a534c0239baf6decadb3f F src/sqliteLimit.h 3afab2291762b5d09ae20c18feb8e9fa935a60a6 F src/status.c 4df6fe7dce2d256130b905847c6c60055882bdbe F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e -F src/tclsqlite.c ebf7a964f8311bc52266cf75afedae98cd743df3 +F src/tclsqlite.c e27e0457109d72c7404df08af1726ff6ddc3d92c F src/test1.c 8b4c246c41e75c3ff033edb2e8c2cf15820861ae F src/test2.c b6b43413d495addd039a88b87d65c839f86b18cb F src/test3.c 4c21700c73a890a47fc685c1097bfb661346ac94 @@ -217,8 +217,8 @@ F src/vacuum.c b1d542c8919d4d11119f78069e1906a1ad07e0ee F src/vdbe.c 2e2aaa765de667dd15e0462cf853efd1b2f97998 F src/vdbe.h 471f6a3dcec4817ca33596fe7f6654d56c0e75f3 F src/vdbeInt.h 19ebc8c2a2e938340051ee65af3f377fb99102d1 -F src/vdbeapi.c 11bcc381e81e797fcf3e81fa6a14ec16a04801cc -F src/vdbeaux.c 3028b2d50df39697e21263685349b5ff1333a29b +F src/vdbeapi.c 466044df5bc916f778833e927165fd02cdef6086 +F src/vdbeaux.c be6ca4bb330c7730b358dd627d6d632ca93482c9 F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 @@ -599,7 +599,7 @@ F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/sync.test ded6b39d8d8ca3c0c5518516c6371b3316d3e3a3 F test/table.test bf102a5669c4db7a41330802f24a4a81a4204f83 F test/tableapi.test 7262a8cbaa9965d429f1cbd2747edc185fa56516 -F test/tclsqlite.test bf4227eb236a4c097aa7974a2bf7d3225acf34be +F test/tclsqlite.test 7378a4e7474d9ffdb7c930086112b829e5955fb0 F test/tempdb.test 1bf52da28a9c24e29717362a87722dff08feb72b F test/temptable.test f42121a0d29a62f00f93274464164177ab1cc24a F test/temptrigger.test b0273db072ce5f37cf19140ceb1f0d524bbe9f05 @@ -760,6 +760,7 @@ F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d F test/wal.test a56ff378f58b145fd3bf38c277fbfe792cd47bdd F test/walcrash.test 45cfbab30bb7cbe0b2e9d5cabe90dbcad10cb89b +F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47 F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 F test/walthread.test 27e44ee6fd02f1f494a24f999c97086af3ab739d F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 @@ -805,7 +806,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 0ae91b0008b242a47385fc1f295c6b645483ee22 -R a205bea3ab4fa24118ce751726c3c719 +P 9d51c3b754f0b94fea5ef3d669ad583b93b2b024 +R 8eb45c9f56fe0d743248bffdfb606bc1 U dan -Z c979f9f88d66076be76b02e7ab3fa216 +Z e82f118f370901abf724c6ac84b89351 diff --git a/manifest.uuid b/manifest.uuid index 26af5e1525..aa29f9aa71 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9d51c3b754f0b94fea5ef3d669ad583b93b2b024 \ No newline at end of file +9bda601455705475075e33bfa85687bce34b15ff \ No newline at end of file diff --git a/src/log.c b/src/log.c index 81f808aa6c..67029d3f8e 100644 --- a/src/log.c +++ b/src/log.c @@ -202,6 +202,14 @@ struct LogSummary { #define LOG_REGION_C 0x04 #define LOG_REGION_D 0x08 +/* +** Values for the third parameter to logLockRegion(). +*/ +#define LOG_UNLOCK 0 /* Unlock a range of bytes */ +#define LOG_RDLOCK 1 /* Put a SHARED lock on a range of bytes */ +#define LOG_WRLOCK 2 /* Put an EXCLUSIVE lock on a byte-range */ +#define LOG_WRLOCKW 3 /* Block on EXCLUSIVE lock on a byte-range */ + /* ** A single instance of this structure is allocated as part of each ** connection to a database log. All structures associated with the @@ -225,6 +233,7 @@ struct Log { sqlite3_file *pFd; /* File handle for log file */ int isLocked; /* Non-zero if a snapshot is held open */ int isWriteLocked; /* True if this is the writer connection */ + u32 iCallback; /* Value to pass to log callback (or 0) */ LogSummaryHdr hdr; /* Log summary header for current snapshot */ LogLock lock; /* Lock held by this connection (if any) */ }; @@ -647,29 +656,29 @@ finished: } /* -** Values for the third parameter to logLockRegion(). +** Place, modify or remove a lock on the log-summary file associated +** with pSummary. */ -#define LOG_UNLOCK 0 -#define LOG_RDLOCK 1 -#define LOG_WRLOCK 2 -#define LOG_WRLOCKW 3 - -static int logLockFd(LogSummary *pSummary, int iStart, int nByte, int op){ +static int logLockFd( + LogSummary *pSummary, /* The log-summary object to lock */ + int iStart, /* First byte to lock */ + int nByte, /* Number of bytes to lock */ + int op /* LOG_UNLOCK, RDLOCK, WRLOCK or WRLOCKW */ +){ int aType[4] = { - F_UNLCK, /* LOG_UNLOCK */ - F_RDLCK, /* LOG_RDLOCK */ - F_WRLCK, /* LOG_WRLOCK */ - F_WRLCK /* LOG_WRLOCKW */ + F_UNLCK, /* LOG_UNLOCK */ + F_RDLCK, /* LOG_RDLOCK */ + F_WRLCK, /* LOG_WRLOCK */ + F_WRLCK /* LOG_WRLOCKW */ }; int aOp[4] = { - F_SETLK, /* LOG_UNLOCK */ - F_SETLK, /* LOG_RDLOCK */ - F_SETLK, /* LOG_WRLOCK */ - F_SETLKW /* LOG_WRLOCKW */ + F_SETLK, /* LOG_UNLOCK */ + F_SETLK, /* LOG_RDLOCK */ + F_SETLK, /* LOG_WRLOCK */ + F_SETLKW /* LOG_WRLOCKW */ }; - - struct flock f; /* Locking operation */ - int rc; /* Value returned by fcntl() */ + struct flock f; /* Locking operation */ + int rc; /* Value returned by fcntl() */ assert( ArraySize(aType)==ArraySize(aOp) ); assert( op>=0 && opmutex) ); assert( eLock==LOG_RDLOCK || eLock==LOG_WRLOCK ); return logLockFd(pSummary, LOG_LOCK_DMH, 1, eLock); } +/* +** Lock (or unlock) the MUTEX region. It is always locked using an +** EXCLUSIVE, blocking lock. +*/ static int logLockMutex(LogSummary *pSummary, int eLock){ + assert( sqlite3_mutex_held(pSummary->mutex) ); assert( eLock==LOG_WRLOCKW || eLock==LOG_UNLOCK ); logLockFd(pSummary, LOG_LOCK_MUTEX, 1, eLock); return SQLITE_OK; } - - /* ** This function intializes the connection to the log-summary identified ** by struct pSummary. @@ -880,7 +898,7 @@ static int logSummaryInit( } rc = logLockDMH(pSummary, LOG_RDLOCK); if( rc!=SQLITE_OK ){ - return SQLITE_IOERR; + rc = SQLITE_IOERR; } out: @@ -1461,7 +1479,7 @@ int sqlite3LogRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut){ /* ** Set *pPgno to the size of the database file (or zero, if unknown). */ -void sqlite3LogMaxpgno(Log *pLog, Pgno *pPgno){ +void sqlite3LogDbsize(Log *pLog, Pgno *pPgno){ assert( pLog->isLocked ); *pPgno = pLog->hdr.nPage; } @@ -1646,9 +1664,10 @@ int sqlite3LogFrames( if( isCommit && SQLITE_OK==(rc = logEnterMutex(pLog)) ){ logSummaryWriteHdr(pLog->pSummary, &pLog->hdr); logLeaveMutex(pLog); + pLog->iCallback = iFrame; } - return SQLITE_OK; + return rc; } /* @@ -1698,3 +1717,12 @@ int sqlite3LogCheckpoint( return rc; } +int sqlite3LogCallback(Log *pLog){ + u32 ret = 0; + if( pLog ){ + ret = pLog->iCallback; + pLog->iCallback = 0; + } + return (int)ret; +} + diff --git a/src/log.h b/src/log.h index 5da5fb1051..af698ebf5f 100644 --- a/src/log.h +++ b/src/log.h @@ -32,7 +32,7 @@ void sqlite3LogCloseSnapshot(Log *pLog); /* Read a page from the log, if it is present. */ int sqlite3LogRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut); -void sqlite3LogMaxpgno(Log *pLog, Pgno *pPgno); +void sqlite3LogDbsize(Log *pLog, Pgno *pPgno); /* Obtain or release the WRITER lock. */ int sqlite3LogWriteLock(Log *pLog, int op); @@ -50,4 +50,7 @@ int sqlite3LogCheckpoint( void *pBusyHandlerArg /* Argument to pass to xBusyHandler */ ); +/* Return the value to pass to a log callback. Or 0 for no callback. */ +int sqlite3LogCallback(Log *pLog); + #endif /* _LOG_H_ */ diff --git a/src/main.c b/src/main.c index cc03d50ed3..80c15963a8 100644 --- a/src/main.c +++ b/src/main.c @@ -1186,6 +1186,24 @@ void *sqlite3_rollback_hook( return pRet; } +/* +** Register a callback to be invoked each time a transaction is written +** into the write-ahead-log by this database connection. +*/ +void *sqlite3_log_hook( + sqlite3 *db, /* Attach the hook to this db handle */ + int(*xCallback)(void *, sqlite3*, const char*, int), + void *pArg /* First argument passed to xCallback() */ +){ + void *pRet; + sqlite3_mutex_enter(db->mutex); + pRet = db->pLogArg; + db->xLogCallback = xCallback; + db->pLogArg = pArg; + sqlite3_mutex_leave(db->mutex); + return pRet; +} + /* ** This function returns true if main-memory should be used instead of ** a temporary file for transient pager files and statement journals. diff --git a/src/pager.c b/src/pager.c index 5fb259ff73..d46350afd2 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2686,7 +2686,7 @@ int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ i64 n = 0; /* File size in bytes returned by OsFileSize() */ if( pagerUseLog(pPager) ){ - sqlite3LogMaxpgno(pPager->pLog, &nPage); + sqlite3LogDbsize(pPager->pLog, &nPage); } if( nPage==0 ){ @@ -5709,4 +5709,8 @@ int sqlite3PagerCheckpoint(Pager *pPager){ return rc; } +int sqlite3PagerLogCallback(Pager *pPager){ + return sqlite3LogCallback(pPager->pLog); +} + #endif /* SQLITE_OMIT_DISKIO */ diff --git a/src/pager.h b/src/pager.h index 1e14d2ea60..17d61a500f 100644 --- a/src/pager.h +++ b/src/pager.h @@ -133,7 +133,9 @@ int sqlite3PagerRollback(Pager*); int sqlite3PagerOpenSavepoint(Pager *pPager, int n); int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); int sqlite3PagerSharedLock(Pager *pPager); + int sqlite3PagerCheckpoint(Pager *pPager); +int sqlite3PagerLogCallback(Pager *pPager); /* Functions used to query pager state and configuration. */ u8 sqlite3PagerIsreadonly(Pager*); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index c1363031ea..97c97aab09 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5726,6 +5726,40 @@ int sqlite3_strnicmp(const char *, const char *, int); */ void sqlite3_log(int iErrCode, const char *zFormat, ...); +/* +** Experimental WAL callback interface. +** +** The [sqlite3_log_hook()] function is used to register a callback that +** will be invoked each time a database connection commits data to a +** write-ahead-log (i.e. whenever a transaction is committed in +** journal_mode=WAL mode). +** +** The callback is invoked by SQLite after the commit has taken place and +** the associated write-lock on the database released, so the implementation +** may read, write or checkpoint the database as required. +** +** The first parameter passed to the callback function when it is invoked +** is a copy of the third parameter passed to sqlite3_log_hook() when +** registering the callback. The second is a copy of the database handle. +** The third parameter is the name of the database that was written to - +** either "main" or the name of an ATTACHed database. The fourth parameter +** is the number of pages currently in the log file, including those that +** were just committed. +** +** If an invocation of the callback function returns non-zero, then a +** checkpoint is automatically run on the database. If zero is returned, +** no special action is taken. +** +** A single database handle may have at most a single log callback +** registered at one time. Calling [sqlite3_log_hook()] replaces any +** previously registered log callback. +*/ +void *sqlite3_log_hook( + sqlite3*, + int(*)(void *,sqlite3*,const char*,int), + void* +); + /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 6007b8af99..1dfc86da24 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -823,6 +823,8 @@ struct sqlite3 { void (*xRollbackCallback)(void*); /* Invoked at every commit. */ void *pUpdateArg; void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64); + int (*xLogCallback)(void *, sqlite3 *, const char *, u32); + void *pLogArg; void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*); void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*); void *pCollNeededArg; diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 02d67c4798..8582ce34ee 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -123,6 +123,7 @@ struct SqliteDb { SqlFunc *pFunc; /* List of SQL functions */ Tcl_Obj *pUpdateHook; /* Update hook script (if any) */ Tcl_Obj *pRollbackHook; /* Rollback hook script (if any) */ + Tcl_Obj *pLogHook; /* WAL hook script (if any) */ Tcl_Obj *pUnlockNotify; /* Unlock notify script (if any) */ SqlCollate *pCollate; /* List of SQL collation functions */ int rc; /* Return code of most recent sqlite3_exec() */ @@ -485,6 +486,9 @@ static void DbDeleteCmd(void *db){ if( pDb->pRollbackHook ){ Tcl_DecrRefCount(pDb->pRollbackHook); } + if( pDb->pLogHook ){ + Tcl_DecrRefCount(pDb->pLogHook); + } if( pDb->pCollateNeeded ){ Tcl_DecrRefCount(pDb->pCollateNeeded); } @@ -589,6 +593,32 @@ static void DbRollbackHandler(void *clientData){ } } +static int DbLogHandler( + void *clientData, + sqlite3 *db, + const char *zDb, + int nEntry +){ + int ret = 0; + Tcl_Obj *p; + SqliteDb *pDb = (SqliteDb*)clientData; + Tcl_Interp *interp = pDb->interp; + assert(pDb->pLogHook); + + p = Tcl_DuplicateObj(pDb->pLogHook); + Tcl_IncrRefCount(p); + Tcl_ListObjAppendElement(interp, p, Tcl_NewStringObj(zDb, -1)); + Tcl_ListObjAppendElement(interp, p, Tcl_NewIntObj(nEntry)); + if( TCL_OK!=Tcl_EvalObjEx(interp, p, 0) + || TCL_OK!=Tcl_GetIntFromObj(interp, Tcl_GetObjResult(interp), &ret) + ){ + Tcl_BackgroundError(interp); + } + Tcl_DecrRefCount(p); + + return ret; +} + #if defined(SQLITE_TEST) && defined(SQLITE_ENABLE_UNLOCK_NOTIFY) static void setTestUnlockNotifyVars(Tcl_Interp *interp, int iArg, int nArg){ char zBuf[64]; @@ -1540,12 +1570,12 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ "complete", "copy", "enable_load_extension", "errorcode", "eval", "exists", "function", "incrblob", "interrupt", - "last_insert_rowid", "nullvalue", "onecolumn", - "profile", "progress", "rekey", - "restore", "rollback_hook", "status", - "timeout", "total_changes", "trace", - "transaction", "unlock_notify", "update_hook", - "version", 0 + "last_insert_rowid", "log_hook", "nullvalue", + "onecolumn", "profile", "progress", + "rekey", "restore", "rollback_hook", + "status", "timeout", "total_changes", + "trace", "transaction", "unlock_notify", + "update_hook", "version", 0 }; enum DB_enum { DB_AUTHORIZER, DB_BACKUP, DB_BUSY, @@ -1554,12 +1584,12 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ DB_COMPLETE, DB_COPY, DB_ENABLE_LOAD_EXTENSION, DB_ERRORCODE, DB_EVAL, DB_EXISTS, DB_FUNCTION, DB_INCRBLOB, DB_INTERRUPT, - DB_LAST_INSERT_ROWID, DB_NULLVALUE, DB_ONECOLUMN, - DB_PROFILE, DB_PROGRESS, DB_REKEY, - DB_RESTORE, DB_ROLLBACK_HOOK, DB_STATUS, - DB_TIMEOUT, DB_TOTAL_CHANGES, DB_TRACE, - DB_TRANSACTION, DB_UNLOCK_NOTIFY, DB_UPDATE_HOOK, - DB_VERSION, + DB_LAST_INSERT_ROWID, DB_LOG_HOOK, DB_NULLVALUE, + DB_ONECOLUMN, DB_PROFILE, DB_PROGRESS, + DB_REKEY, DB_RESTORE, DB_ROLLBACK_HOOK, + DB_STATUS, DB_TIMEOUT, DB_TOTAL_CHANGES, + DB_TRACE, DB_TRANSACTION, DB_UNLOCK_NOTIFY, + DB_UPDATE_HOOK, DB_VERSION }; /* don't leave trailing commas on DB_enum, it confuses the AIX xlc compiler */ @@ -2730,9 +2760,11 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ } /* + ** $db log_hook ?script? ** $db update_hook ?script? ** $db rollback_hook ?script? */ + case DB_LOG_HOOK: case DB_UPDATE_HOOK: case DB_ROLLBACK_HOOK: { @@ -2742,6 +2774,8 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ Tcl_Obj **ppHook; if( choice==DB_UPDATE_HOOK ){ ppHook = &pDb->pUpdateHook; + }else if( choice==DB_LOG_HOOK ){ + ppHook = &pDb->pLogHook; }else{ ppHook = &pDb->pRollbackHook; } @@ -2767,6 +2801,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ sqlite3_update_hook(pDb->db, (pDb->pUpdateHook?DbUpdateHandler:0), pDb); sqlite3_rollback_hook(pDb->db,(pDb->pRollbackHook?DbRollbackHandler:0),pDb); + sqlite3_log_hook(pDb->db,(pDb->pLogHook?DbLogHandler:0),pDb); break; } diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 29a2a429bc..1c938ee5d2 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -306,6 +306,23 @@ void sqlite3_result_error_nomem(sqlite3_context *pCtx){ pCtx->s.db->mallocFailed = 1; } +static int doLogCallbacks(sqlite3 *db){ + int i; + int rc = SQLITE_OK; + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + int nEntry = sqlite3PagerLogCallback(sqlite3BtreePager(pBt)); + if( db->xLogCallback && nEntry>0 && rc==SQLITE_OK + && db->xLogCallback(db->pLogArg, db, db->aDb[i].zName, nEntry) + ){ + rc = sqlite3PagerCheckpoint(sqlite3BtreePager(pBt)); + } + } + } + return rc; +} + /* ** Execute the statement pStmt, either until a row of data is ready, the ** statement is completely executed or an error occurs. @@ -387,6 +404,14 @@ static int sqlite3Step(Vdbe *p){ } #endif + if( rc==SQLITE_DONE ){ + assert( p->rc==SQLITE_OK ); + p->rc = doLogCallbacks(db); + if( p->rc!=SQLITE_OK ){ + rc = SQLITE_ERROR; + } + } + db->errCode = rc; if( SQLITE_NOMEM==sqlite3ApiExit(p->db, p->rc) ){ p->rc = SQLITE_NOMEM; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index a5746f869a..7f00f397a8 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1651,7 +1651,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ ** one database file has an open write transaction, a master journal ** file is required for an atomic commit. */ - for(i=0; inDb; i++){ + for(i=0; inDb; i++){ Btree *pBt = db->aDb[i].pBt; if( sqlite3BtreeIsInTrans(pBt) ){ needXcommit = 1; diff --git a/test/tclsqlite.test b/test/tclsqlite.test index 6bae7f204e..e04841a062 100644 --- a/test/tclsqlite.test +++ b/test/tclsqlite.test @@ -35,7 +35,7 @@ do_test tcl-1.1 { do_test tcl-1.2 { set v [catch {db bogus} msg] lappend v $msg -} {1 {bad option "bogus": must be authorizer, backup, busy, cache, changes, close, collate, collation_needed, commit_hook, complete, copy, enable_load_extension, errorcode, eval, exists, function, incrblob, interrupt, last_insert_rowid, nullvalue, onecolumn, profile, progress, rekey, restore, rollback_hook, status, timeout, total_changes, trace, transaction, unlock_notify, update_hook, or version}} +} {1 {bad option "bogus": must be authorizer, backup, busy, cache, changes, close, collate, collation_needed, commit_hook, complete, copy, enable_load_extension, errorcode, eval, exists, function, incrblob, interrupt, last_insert_rowid, log_hook, nullvalue, onecolumn, profile, progress, rekey, restore, rollback_hook, status, timeout, total_changes, trace, transaction, unlock_notify, update_hook, or version}} do_test tcl-1.2.1 { set v [catch {db cache bogus} msg] lappend v $msg diff --git a/test/walhook.test b/test/walhook.test new file mode 100644 index 0000000000..6f2813b618 --- /dev/null +++ b/test/walhook.test @@ -0,0 +1,73 @@ +# 2010 April 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +proc sqlite3_wal {args} { + eval sqlite3 $args + [lindex $args 0] eval { + PRAGMA journal_mode = wal; + PRAGMA synchronous = normal; + PRAGMA page_size = 1024; + } +} +sqlite3_wal db test.db +db log_hook log_hook + +set ::log_hook [list] +proc log_hook {zDb nEntry} { + lappend ::log_hook $zDb $nEntry + return 0 +} + +do_test walhook-1.1 { + execsql { CREATE TABLE t1(i PRIMARY KEY, j) } + set ::log_hook +} {main 3} +do_test walhook-1.2 { + set ::log_hook [list] + execsql { INSERT INTO t1 VALUES(1, 'one') } + set ::log_hook +} {main 5} +do_test walhook-1.3 { + proc log_hook {args} { return 1 } + execsql { INSERT INTO t1 VALUES(2, 'two') } + file size test.db +} [expr 3*1024] + +do_test walhook-1.4 { + proc log_hook {zDb nEntry} { + execsql { PRAGMA checkpoint } + return 0 + } + execsql { CREATE TABLE t2(a, b) } + file size test.db +} [expr 4*1024] + +do_test walhook-1.5 { + sqlite3_wal db2 test.db + proc log_hook {zDb nEntry} { + execsql { PRAGMA checkpoint } db2 + return 0 + } + execsql { CREATE TABLE t3(a PRIMARY KEY, b) } + file size test.db +} [expr 6*1024] + +catch { db2 close } +catch { db close } +finish_test + From e04dc88be5436c47e49d46217ab1ce4dd2e27e11 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 20 Apr 2010 18:53:15 +0000 Subject: [PATCH 24/89] Use the read and write version fields of the database header to mark a database as operating in wal-mode. FossilOrigin-Name: 96bef18c1411c3e0348295886f105e1646c46320 --- manifest | 25 ++++---- manifest.uuid | 2 +- src/btree.c | 39 +++++++++++- src/btree.h | 2 + src/pager.c | 159 +++++++++++++++++++++++++++------------------- src/pager.h | 2 + src/pragma.c | 77 +++++++++++++--------- src/vdbe.c | 81 +++++++++++++++++++++++ test/wal.test | 43 ++++++++----- test/walmode.test | 25 ++++++++ 10 files changed, 327 insertions(+), 128 deletions(-) create mode 100644 test/walmode.test diff --git a/manifest b/manifest index a4e1d8ff29..f648a1e977 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssqlite3_log_hook()\sinterface\sfor\sscheduling\scheckpoints. -D 2010-04-19T18:03:52 +C Use\sthe\sread\sand\swrite\sversion\sfields\sof\sthe\sdatabase\sheader\sto\smark\sa\sdatabase\sas\soperating\sin\swal-mode. +D 2010-04-20T18:53:15 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -109,8 +109,8 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c e86634da8c48357a759694c9c7c471125cd8d5a8 F src/bitvec.c 06ad2c36a9c3819c0b9cbffec7b15f58d5d834e0 F src/btmutex.c 96a12f50f7a17475155971a241d85ec5171573ff -F src/btree.c 01559397cbd4a5aa62a822e8ca9ac94b6db14743 -F src/btree.h ad6cff92286f9b02ec32f0b97136e9a544249f37 +F src/btree.c 013cf019484e309e909a667654612b38a02dc2b6 +F src/btree.h dd83041eda10c17daf023257c1fc883b5f71f85a F src/btreeInt.h 22447d259639271774a931cbf66aa55112846681 F src/build.c 11100b66fb97638d2d874c1d34d8db90650bb1d7 F src/callback.c 908f3e0172c3d4058f4ca0acd42c637c52e9669f @@ -154,13 +154,13 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053 -F src/pager.c f9ad51dfa617131240ae77545c0083118324fc81 -F src/pager.h aeaee9ceb4a4a6d15c2163ef4e11a7590fd54b59 +F src/pager.c 1dcf76bead8f1b51809bd8189ef9b02f2c94dc7f +F src/pager.h cee4487ab4f0911dd9f22a40e3cd55afdb7ef444 F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 F src/pcache1.c 6dc1871ce8ead9187161c370a58cd06c84221f76 -F src/pragma.c f12cb58a8aa0d80cfed282ef87a285ed71beb793 +F src/pragma.c bae3ce82ca6d0ac5d3ca3e46b286856d1f0ddd70 F src/prepare.c fd1398cb1da54385ba5bd68d93928f10d10a1d9c F src/printf.c 5f5b65a83e63f2096a541a340722a509fa0240a7 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -214,7 +214,7 @@ F src/update.c c0dc6b75ad28b76b619042d934f337b02acee208 F src/utf.c 1baeeac91707a4df97ccc6141ec0f808278af685 F src/util.c 32aebf04c10e51ad3977a928b7416bed671b620b F src/vacuum.c b1d542c8919d4d11119f78069e1906a1ad07e0ee -F src/vdbe.c 2e2aaa765de667dd15e0462cf853efd1b2f97998 +F src/vdbe.c 304851b32c5f918a7ce92e4e9f94fbde0b604b21 F src/vdbe.h 471f6a3dcec4817ca33596fe7f6654d56c0e75f3 F src/vdbeInt.h 19ebc8c2a2e938340051ee65af3f377fb99102d1 F src/vdbeapi.c 466044df5bc916f778833e927165fd02cdef6086 @@ -758,9 +758,10 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test a56ff378f58b145fd3bf38c277fbfe792cd47bdd +F test/wal.test 273c0006e75c99cb34d0659851e7103fd4748c20 F test/walcrash.test 45cfbab30bb7cbe0b2e9d5cabe90dbcad10cb89b F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47 +F test/walmode.test cd6ee20f08af2d81fddc049f8d7e387a807f067e F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 F test/walthread.test 27e44ee6fd02f1f494a24f999c97086af3ab739d F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 @@ -806,7 +807,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 9d51c3b754f0b94fea5ef3d669ad583b93b2b024 -R 8eb45c9f56fe0d743248bffdfb606bc1 +P 9bda601455705475075e33bfa85687bce34b15ff +R b30f3db6dd942a33d471d74ec2e33c70 U dan -Z e82f118f370901abf724c6ac84b89351 +Z d82da3172ce11d2f9fc95cc996cdb180 diff --git a/manifest.uuid b/manifest.uuid index aa29f9aa71..8af34827b8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9bda601455705475075e33bfa85687bce34b15ff \ No newline at end of file +96bef18c1411c3e0348295886f105e1646c46320 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index c8810543a1..069c2d0327 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2261,13 +2261,25 @@ static int lockBtree(BtShared *pBt){ if( memcmp(page1, zMagicHeader, 16)!=0 ){ goto page1_init_failed; } - if( page1[18]>1 ){ + if( page1[18]>2 ){ pBt->readOnly = 1; } - if( page1[19]>1 ){ + if( page1[19]>2 ){ goto page1_init_failed; } + /* If the write version is set to 2, turn on write-ahead logging mode. */ + if( page1[19]==2 ){ + int isOpen = 0; + rc = sqlite3PagerOpenLog(pBt->pPager, &isOpen); + if( rc!=SQLITE_OK ){ + goto page1_init_failed; + }else if( isOpen==0 ){ + releasePage(pPage1); + return SQLITE_OK; + } + } + /* The maximum embedded fraction must be exactly 25%. And the minimum ** embedded fraction must be 12.5% for both leaf-data and non-leaf-data. ** The original design allowed these amounts to vary, but as of @@ -7963,3 +7975,26 @@ void sqlite3BtreeCacheOverflow(BtCursor *pCur){ pCur->isIncrblobHandle = 1; } #endif + +/* +** Set both the "read version" (single byte at byte offset 18) and +** "write version" (single byte at byte offset 19) fields in the database +** header to iVersion. +*/ +int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ + BtShared *pBt = pBtree->pBt; + int rc; /* Return code */ + + assert( pBtree->inTrans==TRANS_WRITE ); + assert( pBt->pPage1 ); + assert( iVersion==1 || iVersion==2 ); + + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + if( rc==SQLITE_OK ){ + u8 *aData = pBt->pPage1->aData; + aData[18] = (u8)iVersion; + aData[19] = (u8)iVersion; + } + + return rc; +} diff --git a/src/btree.h b/src/btree.h index aba06c965e..584b463384 100644 --- a/src/btree.h +++ b/src/btree.h @@ -186,6 +186,8 @@ int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*); void sqlite3BtreeCacheOverflow(BtCursor *); void sqlite3BtreeClearCursor(BtCursor *); +int sqlite3BtreeSetVersion(Btree *pBt, int iVersion); + #ifndef NDEBUG int sqlite3BtreeCursorIsValid(BtCursor*); #endif diff --git a/src/pager.c b/src/pager.c index d46350afd2..9d1fd0479c 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1405,9 +1405,11 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ /* This branch may be executed with Pager.journalMode==MEMORY if ** a hot-journal was just rolled back. In this case the journal ** file should be closed and deleted. If this connection writes to - ** the database file, it will do so using an in-memory journal. */ + ** the database file, it will do so using an in-memory journal. + */ assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE || pPager->journalMode==PAGER_JOURNALMODE_MEMORY + || pPager->journalMode==PAGER_JOURNALMODE_WAL ); sqlite3OsClose(pPager->jfd); if( !pPager->tempFile ){ @@ -3724,54 +3726,6 @@ static int hasHotJournal(Pager *pPager, int *pExists){ return rc; } -/* -** Open a connection to the write-ahead log file for pager pPager. If -** the log connection is already open, this function is a no-op. -*/ -static int pagerOpenLog(Pager *pPager){ - if( !pPager->pLog ){ - int rc; /* Return code */ - - /* Before opening the log file, obtain a SHARED lock on the database - ** file. This lock will not be released until after the log file - ** connection has been closed. The purpose of this lock is to stop - ** any other process from unlinking the log or log-summary files while - ** this connection still has them open. An EXCLUSIVE lock on the - ** database file is required to unlink either of those two files. - */ - assert( pPager->state==PAGER_UNLOCK ); - rc = pager_wait_on_lock(pPager, SHARED_LOCK); - if( rc!=SQLITE_OK ){ - assert( pPager->state==PAGER_UNLOCK ); - return pager_error(pPager, rc); - } - assert( pPager->state>=SHARED_LOCK ); - - /* Open the connection to the log file. If this operation fails, - ** (e.g. due to malloc() failure), unlock the database file and - ** return an error code. - */ - rc = sqlite3LogOpen(pPager->pVfs, pPager->zFilename, &pPager->pLog); - if( rc!=SQLITE_OK ){ - osUnlock(pPager->fd, SQLITE_LOCK_NONE); - pPager->state = PAGER_UNLOCK; - return rc; - } - }else{ - /* If the log file was already open, check that the pager is still holding - ** the required SHARED lock on the database file. - */ -#ifdef SQLITE_DEBUG - int locktype; - sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_LOCKSTATE, &locktype); - assert( locktype==SQLITE_LOCK_SHARED ); -#endif - pPager->state = PAGER_SHARED; - } - - return SQLITE_OK; -} - /* ** This function is called to obtain a shared lock on the database file. @@ -3826,16 +3780,9 @@ int sqlite3PagerSharedLock(Pager *pPager){ pager_reset(pPager); } - - if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){ + if( pagerUseLog(pPager) ){ int changed = 0; /* True if the cache must be flushed */ - /* Open the log file, if it is not already open. */ - rc = pagerOpenLog(pPager); - if( rc!=SQLITE_OK ){ - return rc; - } - /* Open a log snapshot to read from. */ rc = sqlite3LogOpenSnapshot(pPager->pLog, &changed); if( rc==SQLITE_OK ){ @@ -3846,6 +3793,8 @@ int sqlite3PagerSharedLock(Pager *pPager){ } rc = sqlite3PagerPagecount(pPager, &dummy); } + pPager->state = PAGER_SHARED; + }else if( pPager->state==PAGER_UNLOCK || isErrorReset ){ sqlite3_vfs * const pVfs = pPager->pVfs; int isHotJournal = 0; @@ -4002,6 +3951,10 @@ int sqlite3PagerSharedLock(Pager *pPager){ } } assert( pPager->exclusiveMode || pPager->state==PAGER_SHARED ); + + if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){ + pPager->journalMode = PAGER_JOURNALMODE_DELETE; + } } failed: @@ -4384,6 +4337,7 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ if( rc==SQLITE_OK ){ pPager->dbOrigSize = pPager->dbSize; pPager->state = PAGER_RESERVED; + pPager->journalOff = 0; } }else{ /* Obtain a RESERVED lock on the database file. If the exFlag parameter @@ -4485,7 +4439,7 @@ static int pager_write(PgHdr *pPg){ } if( !isOpen(pPager->jfd) && pPager->journalMode!=PAGER_JOURNALMODE_OFF - && pPager->journalMode!=PAGER_JOURNALMODE_WAL + && !pagerUseLog(pPager) ){ assert( pPager->useJournal ); rc = pager_open_journal(pPager); @@ -5621,7 +5575,7 @@ int sqlite3PagerLockingMode(Pager *pPager, int eMode){ ** PAGER_JOURNALMODE_WAL ** ** If the parameter is not _QUERY, then the journal_mode is set to the -** value specified if the change is allowed. The change is disallowed +** value specified if the change is allowed. The change may be disallowed ** for the following reasons: ** ** * An in-memory database can only have its journal_mode set to _OFF @@ -5640,7 +5594,12 @@ int sqlite3PagerJournalMode(Pager *pPager, int eMode){ || eMode==PAGER_JOURNALMODE_WAL || eMode==PAGER_JOURNALMODE_MEMORY ); assert( PAGER_JOURNALMODE_QUERY<0 ); - if( eMode>=0 + + if( eMode==PAGER_JOURNALMODE_WAL + && pPager->journalMode==PAGER_JOURNALMODE_DELETE + ){ + pPager->journalMode = PAGER_JOURNALMODE_WAL; + }else if( eMode>=0 && (pPager->tempFile==0 || eMode!=PAGER_JOURNALMODE_WAL) && (!MEMDB || eMode==PAGER_JOURNALMODE_MEMORY||eMode==PAGER_JOURNALMODE_OFF) && !pPager->dbModified @@ -5659,13 +5618,6 @@ int sqlite3PagerJournalMode(Pager *pPager, int eMode){ sqlite3OsDelete(pPager->pVfs, pPager->zJournal, 0); } - /* Switching into WAL mode can only take place when no - ** locks are held on the database file. - */ - if( eMode==PAGER_JOURNALMODE_WAL && pPager->state!=PAGER_UNLOCK ){ - return (int)pPager->journalMode; - } - pPager->journalMode = (u8)eMode; } return (int)pPager->journalMode; @@ -5713,4 +5665,77 @@ int sqlite3PagerLogCallback(Pager *pPager){ return sqlite3LogCallback(pPager->pLog); } +/* +** Open a connection to the write-ahead log file for pager pPager. If +** the log connection is already open, this function is a no-op. +** +** The caller must be holding a SHARED lock on the database file to call +** this function. +*/ +int sqlite3PagerOpenLog(Pager *pPager, int *pisOpen){ + int rc = SQLITE_OK; /* Return code */ + +#ifdef SQLITE_DEBUG + int locktype; + sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_LOCKSTATE, &locktype); + assert( locktype==SQLITE_LOCK_SHARED ); + assert( pPager->state>=PAGER_SHARED ); +#endif + + if( !pPager->pLog ){ + + /* Open the connection to the log file. If this operation fails, + ** (e.g. due to malloc() failure), unlock the database file and + ** return an error code. + */ + rc = sqlite3LogOpen(pPager->pVfs, pPager->zFilename, &pPager->pLog); + if( rc==SQLITE_OK ){ + pPager->journalMode = PAGER_JOURNALMODE_WAL; + } + }else{ + *pisOpen = 1; + } + + return rc; +} + + +/* +** This function is called to close the connection to the log file prior +** to switching from WAL to rollback mode. +** +** Before closing the log file, this function attempts to take an +** EXCLUSIVE lock on the database file. If this cannot be obtained, an +** error (SQLITE_BUSY) is returned and the log connection is not closed. +** If successful, the EXCLUSIVE lock is not released before returning. +*/ +int sqlite3PagerCloseLog(Pager *pPager){ + int rc = SQLITE_OK; + if( pPager->pLog ){ + + /* Try to obtain an EXCLUSIVE lock on the database file. */ + rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + + /* If the EXCLUSIVE lock was obtained, checkpoint and close the log. */ + if( rc==SQLITE_OK ){ + rc = sqlite3LogClose(pPager->pLog, pPager->fd, + (pPager->noSync ? 0 : pPager->sync_flags), + (u8*)pPager->pTmpSpace + ); + pPager->pLog = 0; + } + + /* Make sure the EXCLUSIVE lock has not been lost somehow */ +#ifdef SQLITE_DEBUG + { + int locktype; + sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_LOCKSTATE, &locktype); + assert( locktype==SQLITE_LOCK_EXCLUSIVE ); + } +#endif + + } + return rc; +} + #endif /* SQLITE_OMIT_DISKIO */ diff --git a/src/pager.h b/src/pager.h index 17d61a500f..7b760c9b1a 100644 --- a/src/pager.h +++ b/src/pager.h @@ -136,6 +136,8 @@ int sqlite3PagerSharedLock(Pager *pPager); int sqlite3PagerCheckpoint(Pager *pPager); int sqlite3PagerLogCallback(Pager *pPager); +int sqlite3PagerOpenLog(Pager *pPager, int *pisOpen); +int sqlite3PagerCloseLog(Pager *pPager); /* Functions used to query pager state and configuration. */ u8 sqlite3PagerIsreadonly(Pager*); diff --git a/src/pragma.c b/src/pragma.c index 137ff510d9..1dfc90d2b6 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -257,6 +257,28 @@ static const char *actionName(u8 action){ } #endif + +/* +** Parameter eMode must be one of the PAGER_JOURNALMODE_XXX constants +** defined in pager.h. This function returns the associated lowercase +** journal-mode name. +*/ +const char *sqlite3JournalModename(int eMode){ + static char * const azModeName[] = { + "delete", "persist", "off", "truncate", "memory", "wal" + }; + assert( PAGER_JOURNALMODE_DELETE==0 ); + assert( PAGER_JOURNALMODE_PERSIST==1 ); + assert( PAGER_JOURNALMODE_OFF==2 ); + assert( PAGER_JOURNALMODE_TRUNCATE==3 ); + assert( PAGER_JOURNALMODE_MEMORY==4 ); + assert( PAGER_JOURNALMODE_WAL==5 ); + assert( eMode>=0 && eMode<=ArraySize(azModeName) ); + + if( eMode==ArraySize(azModeName) ) return 0; + return azModeName[eMode]; +} + /* ** Process a pragma statement. ** @@ -513,18 +535,21 @@ void sqlite3Pragma( ** PRAGMA [database.]journal_mode = (delete|persist|off|truncate|memory) */ if( sqlite3StrICmp(zLeft,"journal_mode")==0 ){ - int eMode; - static char * const azModeName[] = { - "delete", "persist", "off", "truncate", "memory", "wal" - }; + int eMode; /* One of the PAGER_JOURNALMODE_XXX symbols */ + + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "journal_mode", SQLITE_STATIC); if( zRight==0 ){ eMode = PAGER_JOURNALMODE_QUERY; }else{ + const char *zMode; int n = sqlite3Strlen30(zRight); - eMode = sizeof(azModeName)/sizeof(azModeName[0]) - 1; - while( eMode>=0 && sqlite3StrNICmp(zRight, azModeName[eMode], n)!=0 ){ - eMode--; + for(eMode=0; (zMode = sqlite3JournalModename(eMode)); eMode++){ + if( sqlite3StrNICmp(zRight, zMode, n)==0 ) break; + } + if( !zMode ){ + eMode = PAGER_JOURNALMODE_QUERY; } } if( pId2->n==0 && eMode==PAGER_JOURNALMODE_QUERY ){ @@ -533,40 +558,32 @@ void sqlite3Pragma( ** the journal-mode of the main database). */ eMode = db->dfltJournalMode; + sqlite3VdbeAddOp2(v, OP_String8, 0, 1); + sqlite3VdbeChangeP4(v, -1, sqlite3JournalModename(eMode), P4_STATIC); }else{ - Pager *pPager; - if( pId2->n==0 ){ + int ii; + + if( pId2->n==0 && eMode!=PAGER_JOURNALMODE_WAL ){ /* This indicates that no database name was specified as part ** of the PRAGMA command. In this case the journal-mode must be ** set on all attached databases, as well as the main db file. ** ** Also, the sqlite3.dfltJournalMode variable is set so that ** any subsequently attached databases also use the specified - ** journal mode. + ** journal mode. Except, the default journal mode is never set + ** to WAL. */ - int ii; - assert(pDb==&db->aDb[0]); - for(ii=1; iinDb; ii++){ - if( db->aDb[ii].pBt ){ - pPager = sqlite3BtreePager(db->aDb[ii].pBt); - sqlite3PagerJournalMode(pPager, eMode); - } - } db->dfltJournalMode = (u8)eMode; } - pPager = sqlite3BtreePager(pDb->pBt); - eMode = sqlite3PagerJournalMode(pPager, eMode); + + for(ii=db->nDb-1; ii>=0; ii--){ + if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ + sqlite3VdbeUsesBtree(v, ii); + sqlite3VdbeAddOp3(v, OP_JournalMode, ii, 1, eMode); + } + } } - assert( eMode==PAGER_JOURNALMODE_DELETE - || eMode==PAGER_JOURNALMODE_TRUNCATE - || eMode==PAGER_JOURNALMODE_PERSIST - || eMode==PAGER_JOURNALMODE_OFF - || eMode==PAGER_JOURNALMODE_WAL - || eMode==PAGER_JOURNALMODE_MEMORY ); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "journal_mode", SQLITE_STATIC); - sqlite3VdbeAddOp4(v, OP_String8, 0, 1, 0, - azModeName[eMode], P4_STATIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); }else diff --git a/src/vdbe.c b/src/vdbe.c index 42562cee09..cc53c9bff1 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5187,6 +5187,9 @@ case OP_AggFinal: { } /* Opcode: Checkpoint P1 * * * * +** +** Checkpoint database P1. This is a no-op if P1 is not currently in +** WAL mode. */ case OP_Checkpoint: { Btree *pBt; /* Btree to checkpoint */ @@ -5198,6 +5201,84 @@ case OP_Checkpoint: { break; }; +/* Opcode: JournalMode P1 P2 P3 * * +** +** Change the journal mode of database P1 to P3. P3 must be one of the +** PAGER_JOURNALMODE_XXX values. If changing between the various rollback +** modes (delete, truncate, persist, off and memory), this is a simple +** operation. No IO is required. +** +** If changing into or out of WAL mode the procedure is more complicated. +** +** Write a string containing the final journal-mode to register P2. +*/ +case OP_JournalMode: { + Btree *pBt; /* Btree to change journal mode of */ + Pager *pPager; /* Pager associated with pBt */ + int eNew = pOp->p3; /* New journal mode */ + + assert( eNew==PAGER_JOURNALMODE_DELETE + || eNew==PAGER_JOURNALMODE_TRUNCATE + || eNew==PAGER_JOURNALMODE_PERSIST + || eNew==PAGER_JOURNALMODE_OFF + || eNew==PAGER_JOURNALMODE_MEMORY + || eNew==PAGER_JOURNALMODE_WAL + || eNew==PAGER_JOURNALMODE_QUERY + ); + assert( pOp->p1>=0 && pOp->p1nDb ); + assert( (p->btreeMask & (1<p1))!=0 ); + + pBt = db->aDb[pOp->p1].pBt; + pPager = sqlite3BtreePager(pBt); + + if( eNew!=PAGER_JOURNALMODE_QUERY ){ + int eOld = sqlite3PagerJournalMode(pPager, PAGER_JOURNALMODE_QUERY); + if( (eNew!=eOld) + && (eOld==PAGER_JOURNALMODE_WAL || eNew==PAGER_JOURNALMODE_WAL) + ){ + if( !db->autoCommit || db->activeVdbeCnt>1 ){ + rc = SQLITE_ERROR; + sqlite3SetString(&p->zErrMsg, db, + "cannot change %s wal mode from within a transaction", + (eNew==PAGER_JOURNALMODE_WAL ? "into" : "out of") + ); + }else{ + + /* If leaving WAL mode, close the log file. If successful, the call to + ** PagerCloseLog() checkpoints and deletes the write-ahead-log file. + ** An EXCLUSIVE lock is still held on the database file after returning. + */ + if( eOld==PAGER_JOURNALMODE_WAL ){ + rc = sqlite3PagerCloseLog(pPager); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + sqlite3PagerJournalMode(pPager, eNew); + }else{ + sqlite3PagerJournalMode(pPager, PAGER_JOURNALMODE_DELETE); + } + + /* Open a transaction on the database file. Regardless of the journal + ** mode, this transaction always uses a rollback journal. + */ + assert( sqlite3BtreeIsInTrans(pBt)==0 ); + rc = sqlite3BtreeBeginTrans(pBt, 2); + assert( rc==SQLITE_OK || eOld!=PAGER_JOURNALMODE_WAL ); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + rc = sqlite3BtreeSetVersion(pBt, (eNew==PAGER_JOURNALMODE_WAL ? 2 : 1)); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + } + } + } + + eNew = sqlite3PagerJournalMode(pPager, eNew); + pOut = &aMem[pOp->p2]; + pOut->flags = MEM_Str|MEM_Static|MEM_Term; + pOut->z = sqlite3JournalModename(eNew); + pOut->n = sqlite3Strlen30(pOut->z); + pOut->enc = SQLITE_UTF8; + sqlite3VdbeChangeEncoding(pOut, encoding); + break; +}; + #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) /* Opcode: Vacuum * * * * * ** diff --git a/test/wal.test b/test/wal.test index fb21d820f9..bb538b120f 100644 --- a/test/wal.test +++ b/test/wal.test @@ -31,6 +31,7 @@ proc blob {nByte} { proc sqlite3_wal {args} { eval sqlite3 $args + [lindex $args 0] eval { PRAGMA page_size = 1024 } [lindex $args 0] eval { PRAGMA journal_mode = wal } [lindex $args 0] eval { PRAGMA synchronous = normal } [lindex $args 0] function blob blob @@ -60,14 +61,19 @@ do_test wal-0.1 { execsql { PRAGMA synchronous = normal } execsql { PRAGMA journal_mode = wal } } {wal} +do_test wal-0.2 { + file size test.db +} {1024} do_test wal-1.0 { execsql { BEGIN; CREATE TABLE t1(a, b); } - list [file exists test.db-journal] [file exists test.db-wal] -} {0 1} + list [file exists test.db-journal] \ + [file exists test.db-wal] \ + [file size test.db] +} {0 1 1024} do_test wal-1.1 { execsql COMMIT list [file exists test.db-journal] [file exists test.db-wal] @@ -197,9 +203,10 @@ foreach sector {512 4096} { foreach pgsz {512 1024 2048 4096} { file delete -force test.db test.db-wal do_test wal-6.$sector.$pgsz.1 { - sqlite3_wal db test.db -vfs devsym + sqlite3 db test.db -vfs devsym execsql " - PRAGMA page_size = $pgsz ; + PRAGMA page_size = $pgsz; + PRAGMA journal_mode = wal; " execsql " CREATE TABLE t1(a, b); @@ -224,7 +231,7 @@ do_test wal-7.1 { INSERT INTO t1 VALUES(1, 2); } list [file size test.db] [file size test.db-wal] -} [list 0 [log_file_size 3 1024]] +} [list 1024 [log_file_size 3 1024]] do_test wal-7.2 { execsql { PRAGMA checkpoint } list [file size test.db] [file size test.db-wal] @@ -235,11 +242,17 @@ do_test wal-7.2 { # do_test wal-8.1 { reopen_db + catch { db close } + file delete -force test.db test.db-wal + + sqlite3 db test.db + db function blob blob execsql { PRAGMA auto_vacuum = 1; + PRAGMA journal_mode = wal; PRAGMA auto_vacuum; } -} {1} +} {wal 1} do_test wal-8.2 { execsql { PRAGMA page_size = 1024; @@ -269,7 +282,6 @@ do_test wal-8.3 { do_test wal-9.1 { reopen_db execsql { - PRAGMA page_size = 1024; CREATE TABLE t1(x PRIMARY KEY); INSERT INTO t1 VALUES(blob(900)); INSERT INTO t1 VALUES(blob(900)); @@ -282,7 +294,7 @@ do_test wal-9.1 { INSERT INTO t1 SELECT blob(900) FROM t1; /* 256 */ } file size test.db -} 0 +} 1024 do_test wal-9.2 { sqlite3_wal db2 test.db execsql {PRAGMA integrity_check } db2 @@ -560,7 +572,7 @@ do_test wal-11.1 { CREATE TABLE t1(x PRIMARY KEY); } list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] -} {0 3} +} {1 3} do_test wal-11.2 { execsql { PRAGMA checkpoint } list [expr [file size test.db]/1024] [file size test.db-wal] @@ -613,7 +625,7 @@ do_test wal-11.10 { SELECT count(*) FROM t1; } list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [log_file_size 35 1024]] +} [list 37 [log_file_size 37 1024]] do_test wal-11.11 { execsql { SELECT count(*) FROM t1; @@ -623,7 +635,7 @@ do_test wal-11.11 { } {32 16} do_test wal-11.12 { list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [log_file_size 35 1024]] +} [list 37 [log_file_size 37 1024]] do_test wal-11.13 { execsql { INSERT INTO t1 VALUES( blob(900) ); @@ -633,7 +645,7 @@ do_test wal-11.13 { } {17 ok} do_test wal-11.14 { list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [log_file_size 35 1024]] +} [list 37 [log_file_size 37 1024]] #------------------------------------------------------------------------- @@ -650,11 +662,12 @@ do_test wal-12.1 { INSERT INTO t1 VALUES('A', 1); } list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 0 [log_file_size 5 1024]] +} [list 1 [log_file_size 5 1024]] do_test wal-12.2 { db close - sqlite3_wal db test.db + sqlite3 db test.db execsql { + PRAGMA synchronous = normal; UPDATE t1 SET y = 0 WHERE x = 'A'; } list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] @@ -668,7 +681,6 @@ do_test wal-12.4 { file copy -force test.db test2.db file copy -force test.db-wal test2.db-wal sqlite3_wal db2 test2.db -breakpoint execsql { SELECT * FROM t2 } db2 } {B 1} db2 close @@ -695,6 +707,5 @@ do_test wal-12.4 { } {B 2} db2 close - finish_test diff --git a/test/walmode.test b/test/walmode.test new file mode 100644 index 0000000000..1a3af3a705 --- /dev/null +++ b/test/walmode.test @@ -0,0 +1,25 @@ +# 2010 April 19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_test walmode-1.1 { + execsql { + PRAGMA journal_mode = wal; + } +} {wal} + +finish_test From 28e5386f79f6f37fce75b31de84e47297ad53212 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 21 Apr 2010 06:19:12 +0000 Subject: [PATCH 25/89] Minor changes to test cases to account for the fact that databases with read/write versions of 2 are now understood. FossilOrigin-Name: 278ed41e1d4bbff9544cb2cf4cf52dbbcf0c17df --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/sqliteInt.h | 3 ++- test/corruptA.test | 2 +- test/rdonly.test | 6 +++--- 5 files changed, 15 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index f648a1e977..be9552df7d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sthe\sread\sand\swrite\sversion\sfields\sof\sthe\sdatabase\sheader\sto\smark\sa\sdatabase\sas\soperating\sin\swal-mode. -D 2010-04-20T18:53:15 +C Minor\schanges\sto\stest\scases\sto\saccount\sfor\sthe\sfact\sthat\sdatabases\swith\sread/write\sversions\sof\s2\sare\snow\sunderstood. +D 2010-04-21T06:19:13 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -170,7 +170,7 @@ F src/select.c b7c9a40bc1567bceff52ad4b73108734ee4bf268 F src/shell.c c40427c7245535a04a9cb4a417b6cc05c022e6a4 F src/sqlite.h.in b9b0365613d4a99a0e6bc0ae6dbc73cc4a7e7cb7 F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89 -F src/sqliteInt.h 6ed0ed77103d00006d6a534c0239baf6decadb3f +F src/sqliteInt.h 8da48f67766bafb916b551482c51d8aad1be2331 F src/sqliteLimit.h 3afab2291762b5d09ae20c18feb8e9fa935a60a6 F src/status.c 4df6fe7dce2d256130b905847c6c60055882bdbe F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -305,7 +305,7 @@ F test/corrupt6.test e69b877d478224deab7b66844566258cecacd25e F test/corrupt7.test 1eb2214f29474fa6b155aa3da8a7d46bf52089e1 F test/corrupt8.test 9992ef7f67cefc576b92373f6bf5ab8775280f51 F test/corrupt9.test 4aa1cb1ef091cb0e13e89a819c72911631b5176a -F test/corruptA.test 99e95620b980161cb3e79f06a884a4bb8ae265ff +F test/corruptA.test a2fb8d8037e5105d21dc094c3ef44864934325dd F test/corruptB.test d88b500f7994cb2eb9646410c263111e9831f889 F test/corruptC.test 691ed070baef5e1345939caadf270a52837a5064 F test/corruptD.test 3ae6e2dc6e2226c6935a8a40d4b5ee3eba75f8c0 @@ -542,7 +542,7 @@ F test/quick.test 6f202befe1cfae0b63df96b3120a8022ab11f574 F test/quote.test 215897dbe8de1a6f701265836d6601cc6ed103e6 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test 1084050991e9ba22c1c10edd8d84673b501cc25a -F test/rdonly.test bd054831f8a3078e765a0657e247182486f0cb47 +F test/rdonly.test 329558d8f3f7679e7e00f5f1d7cdd5dde78ba382 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 F test/rollback.test 73355ad4492ff9a3a31e61c7e5eb5e01a1de94ca F test/rowhash.test 97f56043ba11f0679920416c0cdbc72e5272267b @@ -807,7 +807,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 9bda601455705475075e33bfa85687bce34b15ff -R b30f3db6dd942a33d471d74ec2e33c70 +P 96bef18c1411c3e0348295886f105e1646c46320 +R 9d271c34e1ffdb3d856b3b8726b0e8d7 U dan -Z d82da3172ce11d2f9fc95cc996cdb180 +Z 58b9eaab75f6eeb69a49f445ee7bfb3c diff --git a/manifest.uuid b/manifest.uuid index 8af34827b8..3378df2d22 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -96bef18c1411c3e0348295886f105e1646c46320 \ No newline at end of file +278ed41e1d4bbff9544cb2cf4cf52dbbcf0c17df \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 1dfc86da24..a93f56b677 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -823,7 +823,7 @@ struct sqlite3 { void (*xRollbackCallback)(void*); /* Invoked at every commit. */ void *pUpdateArg; void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64); - int (*xLogCallback)(void *, sqlite3 *, const char *, u32); + int (*xLogCallback)(void *, sqlite3 *, const char *, int); void *pLogArg; void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*); void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*); @@ -2992,6 +2992,7 @@ void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*); CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *); int sqlite3TempInMemory(const sqlite3*); VTable *sqlite3GetVTable(sqlite3*, Table*); +const char *sqlite3JournalModename(int); /* Declarations for functions in fkey.c. All of these are replaced by ** no-op macros if OMIT_FOREIGN_KEY is defined. In this case no foreign diff --git a/test/corruptA.test b/test/corruptA.test index fa3257f3df..5d5e6f49a2 100644 --- a/test/corruptA.test +++ b/test/corruptA.test @@ -39,7 +39,7 @@ file copy -force test.db test.db-template do_test corruptA-2.1 { file copy -force test.db-template test.db - hexio_write test.db 19 02 ;# the read format number + hexio_write test.db 19 03 ;# the read format number sqlite3 db test.db catchsql {SELECT * FROM t1} } {1 {file is encrypted or is not a database}} diff --git a/test/rdonly.test b/test/rdonly.test index a975cef97b..cd611aeee5 100644 --- a/test/rdonly.test +++ b/test/rdonly.test @@ -29,7 +29,7 @@ do_test rdonly-1.1 { } } {1} -# Changes the write version from 1 to 2. Verify that the database +# Changes the write version from 1 to 3. Verify that the database # can be read but not written. # do_test rdonly-1.2 { @@ -37,7 +37,7 @@ do_test rdonly-1.2 { hexio_get_int [hexio_read test.db 18 1] } 1 do_test rdonly-1.3 { - hexio_write test.db 18 02 + hexio_write test.db 18 03 sqlite3 db test.db execsql { SELECT * FROM t1; @@ -68,7 +68,7 @@ do_test rdonly-1.5 { # the database is read-only until after it is locked. # do_test rdonly-1.6 { - hexio_write test.db 18 02 ; # write-version + hexio_write test.db 18 03 ; # write-version hexio_write test.db 24 11223344 ; # change-counter catchsql { INSERT INTO t1 VALUES(2); From a470aeb4ac0de99fc8d34d1a4006fbc9ebb402c9 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 21 Apr 2010 11:43:38 +0000 Subject: [PATCH 26/89] If, after obtaining a SHARED lock, there exists a *-wal file in the file-system, use WAL mode. This is necessary to recover from a crash that damages the first page of the database file. FossilOrigin-Name: 33cabf271b8f4dda508a610bf59964273fe2cb84 --- manifest | 18 ++++---- manifest.uuid | 2 +- src/btree.c | 9 +++- src/pager.c | 109 ++++++++++++++++++++++++++++++++------------- test/wal.test | 2 - test/walcrash.test | 57 +++++++++++++++++------- 6 files changed, 138 insertions(+), 59 deletions(-) diff --git a/manifest b/manifest index be9552df7d..324d0ff215 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\schanges\sto\stest\scases\sto\saccount\sfor\sthe\sfact\sthat\sdatabases\swith\sread/write\sversions\sof\s2\sare\snow\sunderstood. -D 2010-04-21T06:19:13 +C If,\safter\sobtaining\sa\sSHARED\slock,\sthere\sexists\sa\s*-wal\sfile\sin\sthe\sfile-system,\suse\sWAL\smode.\sThis\sis\snecessary\sto\srecover\sfrom\sa\scrash\sthat\sdamages\sthe\sfirst\spage\sof\sthe\sdatabase\sfile. +D 2010-04-21T11:43:38 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -109,7 +109,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c e86634da8c48357a759694c9c7c471125cd8d5a8 F src/bitvec.c 06ad2c36a9c3819c0b9cbffec7b15f58d5d834e0 F src/btmutex.c 96a12f50f7a17475155971a241d85ec5171573ff -F src/btree.c 013cf019484e309e909a667654612b38a02dc2b6 +F src/btree.c e6ef1020c35db638f9ebbc91b77bc6493d9be2a1 F src/btree.h dd83041eda10c17daf023257c1fc883b5f71f85a F src/btreeInt.h 22447d259639271774a931cbf66aa55112846681 F src/build.c 11100b66fb97638d2d874c1d34d8db90650bb1d7 @@ -154,7 +154,7 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053 -F src/pager.c 1dcf76bead8f1b51809bd8189ef9b02f2c94dc7f +F src/pager.c 4ff50f4e72ca7fff6c1e40e67db78377a53879fe F src/pager.h cee4487ab4f0911dd9f22a40e3cd55afdb7ef444 F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf @@ -758,8 +758,8 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test 273c0006e75c99cb34d0659851e7103fd4748c20 -F test/walcrash.test 45cfbab30bb7cbe0b2e9d5cabe90dbcad10cb89b +F test/wal.test 9ee4025785972fa509250c1f20c5b9456472a789 +F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47 F test/walmode.test cd6ee20f08af2d81fddc049f8d7e387a807f067e F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 @@ -807,7 +807,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 96bef18c1411c3e0348295886f105e1646c46320 -R 9d271c34e1ffdb3d856b3b8726b0e8d7 +P 278ed41e1d4bbff9544cb2cf4cf52dbbcf0c17df +R aeea7589488764dd7471605e974b33ab U dan -Z 58b9eaab75f6eeb69a49f445ee7bfb3c +Z eeb703c307b6c73f6f34fd39a65a4afd diff --git a/manifest.uuid b/manifest.uuid index 3378df2d22..f51f56c3a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -278ed41e1d4bbff9544cb2cf4cf52dbbcf0c17df \ No newline at end of file +33cabf271b8f4dda508a610bf59964273fe2cb84 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 069c2d0327..2c387ffabf 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2268,7 +2268,14 @@ static int lockBtree(BtShared *pBt){ goto page1_init_failed; } - /* If the write version is set to 2, turn on write-ahead logging mode. */ + /* If the write version is set to 2, this database should be accessed + ** in WAL mode. If the log is not already open, open it now. Then + ** return SQLITE_OK and return without populating BtShared.pPage1. + ** The caller detects this and calls this function again. This is + ** required as the version of page 1 currently in the page1 buffer + ** may not be the latest version - there may be a newer one in the log + ** file. + */ if( page1[19]==2 ){ int isOpen = 0; rc = sqlite3PagerOpenLog(pBt->pPager, &isOpen); diff --git a/src/pager.c b/src/pager.c index 9d1fd0479c..48e67fdb03 100644 --- a/src/pager.c +++ b/src/pager.c @@ -491,6 +491,16 @@ static int assert_pager_state(Pager *pPager){ } #endif +#ifndef NDEBUG +static void assert_file_lock(Pager *pPager, int eLock){ + int locktype; + sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_LOCKSTATE, &locktype); + assert( locktype==eLock ); +} +#else +# define assert_file_lock(x,y) +#endif + /* ** Return true if it is necessary to write page *pPg into the sub-journal. @@ -3726,6 +3736,59 @@ static int hasHotJournal(Pager *pPager, int *pExists){ return rc; } +/* +** Check if the *-wal file that corresponds to the database opened by pPager +** exists. Assuming no error occurs, set *pExists to 1 if the file exists, +** or 0 otherwise and return SQLITE_OK. If an IO or OOM error occurs, return +** an SQLite error code. +** +** The caller must hold a SHARED lock on the database file to call this +** function. +*/ +static int pagerHasWAL(Pager *pPager, int *pExists){ + int rc; /* Return code */ + + /* Check that a SHARED lock is held on the database file. Because an + ** EXCLUSIVE lock on the db file is required to delete a WAL, this + ** ensures there is no race condition between the xAccess() below and + ** an xDelete() being executed by some other connection. + */ + assert_file_lock(pPager, SQLITE_LOCK_SHARED); + + if( !pPager->tempFile ){ + char *zLog = sqlite3_mprintf("%s-wal", pPager->zFilename); + if( !zLog ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3OsAccess(pPager->pVfs, zLog, SQLITE_ACCESS_EXISTS, pExists); + sqlite3_free(zLog); + } + }else{ + rc = SQLITE_OK; + *pExists = 0; + } + return rc; +} + +static int pagerOpenSnapshot(Pager *pPager){ + int rc; + int changed; + + assert( pagerUseLog(pPager) ); + + rc = sqlite3LogOpenSnapshot(pPager->pLog, &changed); + if( rc==SQLITE_OK ){ + int dummy; + if( changed ){ + pager_reset(pPager); + assert( pPager->errCode || pPager->dbSizeValid==0 ); + } + rc = sqlite3PagerPagecount(pPager, &dummy); + } + pPager->state = PAGER_SHARED; + + return rc; +} /* ** This function is called to obtain a shared lock on the database file. @@ -3781,23 +3844,11 @@ int sqlite3PagerSharedLock(Pager *pPager){ } if( pagerUseLog(pPager) ){ - int changed = 0; /* True if the cache must be flushed */ - - /* Open a log snapshot to read from. */ - rc = sqlite3LogOpenSnapshot(pPager->pLog, &changed); - if( rc==SQLITE_OK ){ - int dummy; - if( changed ){ - pager_reset(pPager); - assert( pPager->errCode || pPager->dbSizeValid==0 ); - } - rc = sqlite3PagerPagecount(pPager, &dummy); - } - pPager->state = PAGER_SHARED; - + rc = pagerOpenSnapshot(pPager); }else if( pPager->state==PAGER_UNLOCK || isErrorReset ){ sqlite3_vfs * const pVfs = pPager->pVfs; int isHotJournal = 0; + int isWal = 0; assert( !MEMDB ); assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); if( pPager->noReadlock ){ @@ -3952,7 +4003,17 @@ int sqlite3PagerSharedLock(Pager *pPager){ } assert( pPager->exclusiveMode || pPager->state==PAGER_SHARED ); - if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){ + rc = pagerHasWAL(pPager, &isWal); + if( rc!=SQLITE_OK ){ + goto failed; + } + if( isWal ){ + pager_reset(pPager); + rc = sqlite3PagerOpenLog(pPager, 0); + if( rc==SQLITE_OK ){ + rc = pagerOpenSnapshot(pPager); + } + }else if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){ pPager->journalMode = PAGER_JOURNALMODE_DELETE; } } @@ -5675,13 +5736,7 @@ int sqlite3PagerLogCallback(Pager *pPager){ int sqlite3PagerOpenLog(Pager *pPager, int *pisOpen){ int rc = SQLITE_OK; /* Return code */ -#ifdef SQLITE_DEBUG - int locktype; - sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_LOCKSTATE, &locktype); - assert( locktype==SQLITE_LOCK_SHARED ); - assert( pPager->state>=PAGER_SHARED ); -#endif - + assert_file_lock(pPager, SQLITE_LOCK_SHARED); if( !pPager->pLog ){ /* Open the connection to the log file. If this operation fails, @@ -5725,15 +5780,7 @@ int sqlite3PagerCloseLog(Pager *pPager){ pPager->pLog = 0; } - /* Make sure the EXCLUSIVE lock has not been lost somehow */ -#ifdef SQLITE_DEBUG - { - int locktype; - sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_LOCKSTATE, &locktype); - assert( locktype==SQLITE_LOCK_EXCLUSIVE ); - } -#endif - + assert_file_lock(pPager, SQLITE_LOCK_EXCLUSIVE); } return rc; } diff --git a/test/wal.test b/test/wal.test index bb538b120f..2c73904634 100644 --- a/test/wal.test +++ b/test/wal.test @@ -685,7 +685,6 @@ do_test wal-12.4 { } {B 1} db2 close -file copy -force test.db-wal A do_test wal-12.5 { execsql { PRAGMA checkpoint; @@ -697,7 +696,6 @@ do_test wal-12.5 { SELECT * FROM t2; } } {B 2} -file copy -force test.db-wal B do_test wal-12.4 { file copy -force test.db test2.db diff --git a/test/walcrash.test b/test/walcrash.test index dd4c572941..ed2b292861 100644 --- a/test/walcrash.test +++ b/test/walcrash.test @@ -32,11 +32,6 @@ db close set seed 0 set REPEATS 100 -proc sqlite3_wal {args} { - eval sqlite3 $args - [lindex $args 0] eval { PRAGMA journal_mode = wal } -} - # walcrash-1.* # for {set i 1} {$i < $REPEATS} {incr i} { @@ -51,7 +46,7 @@ for {set i 1} {$i < $REPEATS} {incr i} { } } {1 {child process exited abnormally}} do_test walcrash-1.$i.2 { - sqlite3_wal db test.db + sqlite3 db test.db execsql { SELECT sum(a)==max(b) FROM t1 } } {1} integrity_check walcrash-1.$i.3 @@ -59,17 +54,18 @@ for {set i 1} {$i < $REPEATS} {incr i} { do_test walcrash-1.$i.4 { crashsql -delay 2 -file test.db-wal -seed [incr seed] { - PRAGMA journal_mode = WAL; - PRAGMA journal_mode = WAL; INSERT INTO t1 VALUES(4, (SELECT sum(a) FROM t1) + 4); INSERT INTO t1 VALUES(5, (SELECT sum(a) FROM t1) + 5); } } {1 {child process exited abnormally}} do_test walcrash-1.$i.5 { - sqlite3_wal db test.db + sqlite3 db test.db execsql { SELECT sum(a)==max(b) FROM t1 } } {1} integrity_check walcrash-1.$i.6 + do_test walcrash-1.$i.5 { + execsql { PRAGMA main.journal_mode } + } {wal} db close } @@ -87,7 +83,7 @@ for {set i 1} {$i < $REPEATS} {incr i} { } } {1 {child process exited abnormally}} do_test walcrash-2.$i.2 { - sqlite3_wal db test.db + sqlite3 db test.db execsql { SELECT sum(a)==max(b) FROM t1 } } {1} integrity_check walcrash-2.$i.3 @@ -95,16 +91,18 @@ for {set i 1} {$i < $REPEATS} {incr i} { do_test walcrash-2.$i.4 { crashsql -delay 2 -file test.db-wal -seed [incr seed] { - PRAGMA journal_mode = WAL; INSERT INTO t1 VALUES(6, (SELECT sum(a) FROM t1) + 6); INSERT INTO t1 VALUES(7, (SELECT sum(a) FROM t1) + 7); } } {1 {child process exited abnormally}} do_test walcrash-2.$i.5 { - sqlite3_wal db test.db + sqlite3 db test.db execsql { SELECT sum(a)==max(b) FROM t1 } } {1} integrity_check walcrash-2.$i.6 + do_test walcrash-2.$i.6 { + execsql { PRAGMA main.journal_mode } + } {wal} db close } @@ -157,12 +155,13 @@ for {set i 1} {$i < $REPEATS} {incr i} { } {1 {child process exited abnormally}} do_test walcrash-4.$i.2 { - sqlite3_wal db test.db + sqlite3 db test.db execsql { SELECT * FROM t1 WHERE a = 1; } } {1 2} do_test walcrash-4.$i.3 { execsql { PRAGMA main.integrity_check } } {ok} + do_test walcrash-4.$i.4 { execsql { PRAGMA main.journal_mode } } {wal} db close } @@ -199,10 +198,11 @@ for {set i 1} {$i < $REPEATS} {incr i} { } {1 {child process exited abnormally}} do_test walcrash-5.$i.2 { - sqlite3_wal db test.db + sqlite3 db test.db execsql { SELECT count(*)==33 OR count(*)==34 FROM t1 WHERE x != 1 } } {1} do_test walcrash-5.$i.3 { execsql { PRAGMA main.integrity_check } } {ok} + do_test walcrash-5.$i.4 { execsql { PRAGMA main.journal_mode } } {wal} db close } @@ -239,10 +239,37 @@ for {set i 1} {$i < $REPEATS} {incr i} { } {1 {child process exited abnormally}} do_test walcrash-6.$i.2 { - sqlite3_wal db test.db + sqlite3 db test.db execsql { SELECT count(*)==34 OR count(*)==35 FROM t1 WHERE x != 1 } } {1} do_test walcrash-6.$i.3 { execsql { PRAGMA main.integrity_check } } {ok} + do_test walcrash-6.$i.4 { execsql { PRAGMA main.journal_mode } } {wal} + + db close +} + +for {set i 1} {$i < $REPEATS} {incr i} { + file delete -force test.db test.db-wal + + do_test walcrash-7.$i.1 { + crashsql -delay 3 -file test.db -seed [incr seed] -blocksize 512 { + PRAGMA journal_mode = wal; + BEGIN; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + COMMIT; + PRAGMA checkpoint; + CREATE INDEX i1 ON t1(a); + PRAGMA checkpoint; + } + } {1 {child process exited abnormally}} + + do_test walcrash-7.$i.2 { + sqlite3 db test.db + execsql { SELECT b FROM t1 WHERE a = 1 } + } {2} + do_test walcrash-7.$i.3 { execsql { PRAGMA main.integrity_check } } {ok} + do_test walcrash-7.$i.4 { execsql { PRAGMA main.journal_mode } } {wal} db close } From b9780027438f426ca2d9650a1305fe6811c6f57e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 21 Apr 2010 18:37:57 +0000 Subject: [PATCH 27/89] Tests for (and changes to) the code to switch between WAL and rollback modes. FossilOrigin-Name: 9f4f933f2c6596064fcfc7fb5add87e8c5b57448 --- manifest | 24 +++++++-------- manifest.uuid | 2 +- src/btree.c | 25 ++++++++++++---- src/btreeInt.h | 1 + src/log.c | 1 - src/pager.c | 27 +++++------------ src/vdbe.c | 5 +--- test/wal.test | 6 ++-- test/walmode.test | 76 +++++++++++++++++++++++++++++++++++++++++++++-- 9 files changed, 117 insertions(+), 50 deletions(-) diff --git a/manifest b/manifest index 324d0ff215..8f9be85819 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If,\safter\sobtaining\sa\sSHARED\slock,\sthere\sexists\sa\s*-wal\sfile\sin\sthe\sfile-system,\suse\sWAL\smode.\sThis\sis\snecessary\sto\srecover\sfrom\sa\scrash\sthat\sdamages\sthe\sfirst\spage\sof\sthe\sdatabase\sfile. -D 2010-04-21T11:43:38 +C Tests\sfor\s(and\schanges\sto)\sthe\scode\sto\sswitch\sbetween\sWAL\sand\srollback\smodes. +D 2010-04-21T18:37:57 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -109,9 +109,9 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c e86634da8c48357a759694c9c7c471125cd8d5a8 F src/bitvec.c 06ad2c36a9c3819c0b9cbffec7b15f58d5d834e0 F src/btmutex.c 96a12f50f7a17475155971a241d85ec5171573ff -F src/btree.c e6ef1020c35db638f9ebbc91b77bc6493d9be2a1 +F src/btree.c fcfff21fb259fc5f09d8ae65478c3240553f4cd9 F src/btree.h dd83041eda10c17daf023257c1fc883b5f71f85a -F src/btreeInt.h 22447d259639271774a931cbf66aa55112846681 +F src/btreeInt.h b0c87f6725b06a0aa194a6d25d54b16ce9d6e291 F src/build.c 11100b66fb97638d2d874c1d34d8db90650bb1d7 F src/callback.c 908f3e0172c3d4058f4ca0acd42c637c52e9669f F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac @@ -131,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c a5bbeb05c456ee7ee9d4f3dba5e72bae2d9b6530 +F src/log.c edafa806982a281205e71c439e99766e924de938 F src/log.h b8c45a6348d9ef57c5205a08c611d57d07ee9feb F src/main.c 867de6aa444abd97771b2b70472f448d65c1c77e F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a @@ -154,7 +154,7 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053 -F src/pager.c 4ff50f4e72ca7fff6c1e40e67db78377a53879fe +F src/pager.c f20f9090456b2f4aa3c5174e390a3cfd61bd3cc1 F src/pager.h cee4487ab4f0911dd9f22a40e3cd55afdb7ef444 F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf @@ -214,7 +214,7 @@ F src/update.c c0dc6b75ad28b76b619042d934f337b02acee208 F src/utf.c 1baeeac91707a4df97ccc6141ec0f808278af685 F src/util.c 32aebf04c10e51ad3977a928b7416bed671b620b F src/vacuum.c b1d542c8919d4d11119f78069e1906a1ad07e0ee -F src/vdbe.c 304851b32c5f918a7ce92e4e9f94fbde0b604b21 +F src/vdbe.c 4d99d175264ef5b2ac57b19396f2b78199539e22 F src/vdbe.h 471f6a3dcec4817ca33596fe7f6654d56c0e75f3 F src/vdbeInt.h 19ebc8c2a2e938340051ee65af3f377fb99102d1 F src/vdbeapi.c 466044df5bc916f778833e927165fd02cdef6086 @@ -758,10 +758,10 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test 9ee4025785972fa509250c1f20c5b9456472a789 +F test/wal.test fd0e5914599b205172dae33c765368a75feb2462 F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47 -F test/walmode.test cd6ee20f08af2d81fddc049f8d7e387a807f067e +F test/walmode.test 004da14c0d6be8314bba076d2d89c16458b6f849 F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 F test/walthread.test 27e44ee6fd02f1f494a24f999c97086af3ab739d F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 @@ -807,7 +807,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 278ed41e1d4bbff9544cb2cf4cf52dbbcf0c17df -R aeea7589488764dd7471605e974b33ab +P 33cabf271b8f4dda508a610bf59964273fe2cb84 +R ce8d5df20a1a383d4fb608ef1b16a4c8 U dan -Z eeb703c307b6c73f6f34fd39a65a4afd +Z 9951e081ef8379216185ea5ef99b745d diff --git a/manifest.uuid b/manifest.uuid index f51f56c3a0..8514b695f3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -33cabf271b8f4dda508a610bf59964273fe2cb84 \ No newline at end of file +9f4f933f2c6596064fcfc7fb5add87e8c5b57448 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 2c387ffabf..71de1ef681 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2276,7 +2276,7 @@ static int lockBtree(BtShared *pBt){ ** may not be the latest version - there may be a newer one in the log ** file. */ - if( page1[19]==2 ){ + if( page1[19]==2 && pBt->doNotUseWAL==0 ){ int isOpen = 0; rc = sqlite3PagerOpenLog(pBt->pPager, &isOpen); if( rc!=SQLITE_OK ){ @@ -7992,16 +7992,29 @@ int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ BtShared *pBt = pBtree->pBt; int rc; /* Return code */ - assert( pBtree->inTrans==TRANS_WRITE ); - assert( pBt->pPage1 ); + assert( pBtree->inTrans==TRANS_NONE ); assert( iVersion==1 || iVersion==2 ); - rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + /* If setting the version fields to 1, do not automatically open the + ** WAL connection, even if the version fields are currently set to 2. + */ + pBt->doNotUseWAL = (iVersion==1); + + rc = sqlite3BtreeBeginTrans(pBtree, 0); if( rc==SQLITE_OK ){ u8 *aData = pBt->pPage1->aData; - aData[18] = (u8)iVersion; - aData[19] = (u8)iVersion; + if( aData[18]!=(u8)iVersion || aData[19]!=(u8)iVersion ){ + rc = sqlite3BtreeBeginTrans(pBtree, 1); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); + if( rc==SQLITE_OK ){ + aData[18] = (u8)iVersion; + aData[19] = (u8)iVersion; + } + } + } } + pBt->doNotUseWAL = 0; return rc; } diff --git a/src/btreeInt.h b/src/btreeInt.h index cd40713ead..8b68d7b316 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -420,6 +420,7 @@ struct BtShared { u16 maxLeaf; /* Maximum local payload in a LEAFDATA table */ u16 minLeaf; /* Minimum local payload in a LEAFDATA table */ u8 inTransaction; /* Transaction state */ + u8 doNotUseWAL; /* If true, do not open write-ahead-log file */ int nTransaction; /* Number of open transactions (read + write) */ u32 nPage; /* Number of pages in the database */ void *pSchema; /* Pointer to space allocated by sqlite3BtreeSchema() */ diff --git a/src/log.c b/src/log.c index 67029d3f8e..89b60568b5 100644 --- a/src/log.c +++ b/src/log.c @@ -1235,7 +1235,6 @@ int sqlite3LogClose( } logSummaryUnmap(pSummary, 0); } - sqlite3OsUnlock(pFd, SQLITE_LOCK_NONE); sqlite3_mutex_free(pSummary->mutex); sqlite3_free(pSummary); diff --git a/src/pager.c b/src/pager.c index 48e67fdb03..7e96049d3b 100644 --- a/src/pager.c +++ b/src/pager.c @@ -491,17 +491,6 @@ static int assert_pager_state(Pager *pPager){ } #endif -#ifndef NDEBUG -static void assert_file_lock(Pager *pPager, int eLock){ - int locktype; - sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_LOCKSTATE, &locktype); - assert( locktype==eLock ); -} -#else -# define assert_file_lock(x,y) -#endif - - /* ** Return true if it is necessary to write page *pPg into the sub-journal. ** A page needs to be written into the sub-journal if there exists one @@ -3008,10 +2997,10 @@ static int syncJournal(Pager *pPager){ */ i64 iNextHdrOffset; u8 aMagic[8]; - u8 zHeader[sizeof(aJournalMagic)+4]; + u8 zHeader[sizeof(aJournalMagic)+4]; - memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic)); - put32bits(&zHeader[sizeof(aJournalMagic)], pPager->nRec); + memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic)); + put32bits(&zHeader[sizeof(aJournalMagic)], pPager->nRec); iNextHdrOffset = journalHdrOffset(pPager); rc = sqlite3OsRead(pPager->jfd, aMagic, 8, iNextHdrOffset); @@ -3043,7 +3032,7 @@ static int syncJournal(Pager *pPager){ IOTRACE(("JHDR %p %lld\n", pPager, pPager->journalHdr)); rc = sqlite3OsWrite( pPager->jfd, zHeader, sizeof(zHeader), pPager->journalHdr - ); + ); if( rc!=SQLITE_OK ) return rc; } if( 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){ @@ -3753,7 +3742,7 @@ static int pagerHasWAL(Pager *pPager, int *pExists){ ** ensures there is no race condition between the xAccess() below and ** an xDelete() being executed by some other connection. */ - assert_file_lock(pPager, SQLITE_LOCK_SHARED); + assert( pPager->state>=PAGER_SHARED ); if( !pPager->tempFile ){ char *zLog = sqlite3_mprintf("%s-wal", pPager->zFilename); @@ -5736,7 +5725,7 @@ int sqlite3PagerLogCallback(Pager *pPager){ int sqlite3PagerOpenLog(Pager *pPager, int *pisOpen){ int rc = SQLITE_OK; /* Return code */ - assert_file_lock(pPager, SQLITE_LOCK_SHARED); + assert( pPager->state>=PAGER_SHARED ); if( !pPager->pLog ){ /* Open the connection to the log file. If this operation fails, @@ -5769,7 +5758,7 @@ int sqlite3PagerCloseLog(Pager *pPager){ if( pPager->pLog ){ /* Try to obtain an EXCLUSIVE lock on the database file. */ - rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); + rc = sqlite3OsLock(pPager->fd, SQLITE_LOCK_EXCLUSIVE); /* If the EXCLUSIVE lock was obtained, checkpoint and close the log. */ if( rc==SQLITE_OK ){ @@ -5779,8 +5768,6 @@ int sqlite3PagerCloseLog(Pager *pPager){ ); pPager->pLog = 0; } - - assert_file_lock(pPager, SQLITE_LOCK_EXCLUSIVE); } return rc; } diff --git a/src/vdbe.c b/src/vdbe.c index cc53c9bff1..5e2316678f 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5260,9 +5260,6 @@ case OP_JournalMode: { ** mode, this transaction always uses a rollback journal. */ assert( sqlite3BtreeIsInTrans(pBt)==0 ); - rc = sqlite3BtreeBeginTrans(pBt, 2); - assert( rc==SQLITE_OK || eOld!=PAGER_JOURNALMODE_WAL ); - if( rc!=SQLITE_OK ) goto abort_due_to_error; rc = sqlite3BtreeSetVersion(pBt, (eNew==PAGER_JOURNALMODE_WAL ? 2 : 1)); if( rc!=SQLITE_OK ) goto abort_due_to_error; } @@ -5272,7 +5269,7 @@ case OP_JournalMode: { eNew = sqlite3PagerJournalMode(pPager, eNew); pOut = &aMem[pOp->p2]; pOut->flags = MEM_Str|MEM_Static|MEM_Term; - pOut->z = sqlite3JournalModename(eNew); + pOut->z = (char *)sqlite3JournalModename(eNew); pOut->n = sqlite3Strlen30(pOut->z); pOut->enc = SQLITE_UTF8; sqlite3VdbeChangeEncoding(pOut, encoding); diff --git a/test/wal.test b/test/wal.test index 2c73904634..f0268e19a2 100644 --- a/test/wal.test +++ b/test/wal.test @@ -625,7 +625,7 @@ do_test wal-11.10 { SELECT count(*) FROM t1; } list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [log_file_size 37 1024]] +} [list 37 [log_file_size 35 1024]] do_test wal-11.11 { execsql { SELECT count(*) FROM t1; @@ -635,7 +635,7 @@ do_test wal-11.11 { } {32 16} do_test wal-11.12 { list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [log_file_size 37 1024]] +} [list 37 [log_file_size 35 1024]] do_test wal-11.13 { execsql { INSERT INTO t1 VALUES( blob(900) ); @@ -645,7 +645,7 @@ do_test wal-11.13 { } {17 ok} do_test wal-11.14 { list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [log_file_size 37 1024]] +} [list 37 [log_file_size 35 1024]] #------------------------------------------------------------------------- diff --git a/test/walmode.test b/test/walmode.test index 1a3af3a705..8d3b882f01 100644 --- a/test/walmode.test +++ b/test/walmode.test @@ -17,9 +17,79 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl do_test walmode-1.1 { - execsql { - PRAGMA journal_mode = wal; - } + set sqlite_sync_count 0 + execsql { PRAGMA page_size = 1024 } + execsql { PRAGMA journal_mode = wal } } {wal} +do_test walmode-1.2 { + file size test.db +} {1024} +do_test walmode-1.3 { + set sqlite_sync_count +} {4} +do_test walmode-1.4 { + file exists test.db-wal +} {0} +do_test walmode-1.5 { + execsql { CREATE TABLE t1(a, b) } + file size test.db +} {1024} +do_test walmode-1.6 { + file exists test.db-wal +} {1} +do_test walmode-1.7 { + db close + file exists test.db-wal +} {0} + +# There is now a database file with the read and write versions set to 2 +# in the file system. This file should default to WAL mode. +# +do_test walmode-2.1 { + sqlite3 db test.db + file exists test.db-wal +} {0} +do_test walmode-2.2 { + execsql { SELECT * FROM sqlite_master } + file exists test.db-wal +} {1} +do_test walmode-2.3 { + db close + file exists test.db-wal +} {0} + +# If the first statement executed is "PRAGMA journal_mode = wal", and +# the file is already configured for WAL (read and write versions set +# to 2), then there should be no need to write the database. The +# statement should cause the client to connect to the log file. +# +set sqlite_sync_count 0 +do_test walmode-3.1 { + sqlite3 db test.db + execsql { PRAGMA journal_mode = wal } +} {wal} +do_test walmode-3.2 { + list $sqlite_sync_count [file exists test.db-wal] [file size test.db-wal] +} {0 1 0} + +do_test walmode-4.1 { + execsql { INSERT INTO t1 VALUES(1, 2) } + execsql { PRAGMA journal_mode = persist } +} {persist} +do_test walmode-4.2 { + list [file exists test.db-journal] [file exists test.db-wal] +} {1 0} +do_test walmode-4.3 { + execsql { SELECT * FROM t1 } +} {1 2} +do_test walmode-4.4 { + db close + sqlite3 db test.db + execsql { SELECT * FROM t1 } +} {1 2} +do_test walmode-4.5 { + list [file exists test.db-journal] [file exists test.db-wal] +} {1 0} finish_test + From ede6eb8d1df9c4751ecb656c337e4bb7a9a534e9 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 22 Apr 2010 06:27:04 +0000 Subject: [PATCH 28/89] Further tests and changes related to switching between WAL and rollback modes. FossilOrigin-Name: 1236318477787a612f02cc98caf2931bd2e99d94 --- manifest | 18 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 2 +- src/pager.c | 36 +++++++++++++++++++++----------- src/vdbe.c | 9 ++++---- test/walmode.test | 52 +++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 92 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index 8f9be85819..8b101354cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Tests\sfor\s(and\schanges\sto)\sthe\scode\sto\sswitch\sbetween\sWAL\sand\srollback\smodes. -D 2010-04-21T18:37:57 +C Further\stests\sand\schanges\srelated\sto\sswitching\sbetween\sWAL\sand\srollback\smodes. +D 2010-04-22T06:27:05 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -109,7 +109,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c e86634da8c48357a759694c9c7c471125cd8d5a8 F src/bitvec.c 06ad2c36a9c3819c0b9cbffec7b15f58d5d834e0 F src/btmutex.c 96a12f50f7a17475155971a241d85ec5171573ff -F src/btree.c fcfff21fb259fc5f09d8ae65478c3240553f4cd9 +F src/btree.c f87ed73ef82bccb1e5dd63d6a4919272d3354c8a F src/btree.h dd83041eda10c17daf023257c1fc883b5f71f85a F src/btreeInt.h b0c87f6725b06a0aa194a6d25d54b16ce9d6e291 F src/build.c 11100b66fb97638d2d874c1d34d8db90650bb1d7 @@ -154,7 +154,7 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053 -F src/pager.c f20f9090456b2f4aa3c5174e390a3cfd61bd3cc1 +F src/pager.c 06fb9b8b7c5283c62f5258e098d6d1649dde7635 F src/pager.h cee4487ab4f0911dd9f22a40e3cd55afdb7ef444 F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf @@ -214,7 +214,7 @@ F src/update.c c0dc6b75ad28b76b619042d934f337b02acee208 F src/utf.c 1baeeac91707a4df97ccc6141ec0f808278af685 F src/util.c 32aebf04c10e51ad3977a928b7416bed671b620b F src/vacuum.c b1d542c8919d4d11119f78069e1906a1ad07e0ee -F src/vdbe.c 4d99d175264ef5b2ac57b19396f2b78199539e22 +F src/vdbe.c 09e88e8b01b413271b5ccd8c902c424cd563e342 F src/vdbe.h 471f6a3dcec4817ca33596fe7f6654d56c0e75f3 F src/vdbeInt.h 19ebc8c2a2e938340051ee65af3f377fb99102d1 F src/vdbeapi.c 466044df5bc916f778833e927165fd02cdef6086 @@ -761,7 +761,7 @@ F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d F test/wal.test fd0e5914599b205172dae33c765368a75feb2462 F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47 -F test/walmode.test 004da14c0d6be8314bba076d2d89c16458b6f849 +F test/walmode.test c2f4e30ad64910b2d8faf6cf4e940b3f201b41df F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 F test/walthread.test 27e44ee6fd02f1f494a24f999c97086af3ab739d F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 @@ -807,7 +807,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 33cabf271b8f4dda508a610bf59964273fe2cb84 -R ce8d5df20a1a383d4fb608ef1b16a4c8 +P 9f4f933f2c6596064fcfc7fb5add87e8c5b57448 +R cdccc9135123b3edc2fcb17c83fbd857 U dan -Z 9951e081ef8379216185ea5ef99b745d +Z 891f9ffc6892beb06734f78c0f8e5711 diff --git a/manifest.uuid b/manifest.uuid index 8514b695f3..0aa3e70b8f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9f4f933f2c6596064fcfc7fb5add87e8c5b57448 \ No newline at end of file +1236318477787a612f02cc98caf2931bd2e99d94 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 71de1ef681..2fd3fedc1e 100644 --- a/src/btree.c +++ b/src/btree.c @@ -8004,7 +8004,7 @@ int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ if( rc==SQLITE_OK ){ u8 *aData = pBt->pPage1->aData; if( aData[18]!=(u8)iVersion || aData[19]!=(u8)iVersion ){ - rc = sqlite3BtreeBeginTrans(pBtree, 1); + rc = sqlite3BtreeBeginTrans(pBtree, 2); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); if( rc==SQLITE_OK ){ diff --git a/src/pager.c b/src/pager.c index 7e96049d3b..157413e9b0 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3732,18 +3732,13 @@ static int hasHotJournal(Pager *pPager, int *pExists){ ** an SQLite error code. ** ** The caller must hold a SHARED lock on the database file to call this -** function. +** function. Because an EXCLUSIVE lock on the db file is required to delete +** a WAL, this ensures there is no race condition between the xAccess() +** below and an xDelete() being executed by some other connection. */ static int pagerHasWAL(Pager *pPager, int *pExists){ int rc; /* Return code */ - /* Check that a SHARED lock is held on the database file. Because an - ** EXCLUSIVE lock on the db file is required to delete a WAL, this - ** ensures there is no race condition between the xAccess() below and - ** an xDelete() being executed by some other connection. - */ - assert( pPager->state>=PAGER_SHARED ); - if( !pPager->tempFile ){ char *zLog = sqlite3_mprintf("%s-wal", pPager->zFilename); if( !zLog ){ @@ -5755,12 +5750,29 @@ int sqlite3PagerOpenLog(Pager *pPager, int *pisOpen){ */ int sqlite3PagerCloseLog(Pager *pPager){ int rc = SQLITE_OK; - if( pPager->pLog ){ - /* Try to obtain an EXCLUSIVE lock on the database file. */ + assert( pPager->journalMode==PAGER_JOURNALMODE_WAL ); + + /* If the log file is not already open, but does exist in the file-system, + ** it may need to be checkpointed before the connection can switch to + ** rollback mode. Open it now so this can happen. + */ + if( !pPager->pLog ){ + int logexists = 0; + rc = sqlite3OsLock(pPager->fd, SQLITE_LOCK_SHARED); + if( rc==SQLITE_OK ){ + rc = pagerHasWAL(pPager, &logexists); + } + if( rc==SQLITE_OK && logexists ){ + rc = sqlite3LogOpen(pPager->pVfs, pPager->zFilename, &pPager->pLog); + } + } + + /* Checkpoint and close the log. Because an EXCLUSIVE lock is held on + ** the database file, the log and log-summary files will be deleted. + */ + if( rc==SQLITE_OK && pPager->pLog ){ rc = sqlite3OsLock(pPager->fd, SQLITE_LOCK_EXCLUSIVE); - - /* If the EXCLUSIVE lock was obtained, checkpoint and close the log. */ if( rc==SQLITE_OK ){ rc = sqlite3LogClose(pPager->pLog, pPager->fd, (pPager->noSync ? 0 : pPager->sync_flags), diff --git a/src/vdbe.c b/src/vdbe.c index 5e2316678f..2af5c7384c 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5244,11 +5244,12 @@ case OP_JournalMode: { ); }else{ - /* If leaving WAL mode, close the log file. If successful, the call to - ** PagerCloseLog() checkpoints and deletes the write-ahead-log file. - ** An EXCLUSIVE lock is still held on the database file after returning. - */ if( eOld==PAGER_JOURNALMODE_WAL ){ + /* If leaving WAL mode, close the log file. If successful, the call + ** to PagerCloseLog() checkpoints and deletes the write-ahead-log + ** file. An EXCLUSIVE lock may still be held on the database file + ** after a successful return. + */ rc = sqlite3PagerCloseLog(pPager); if( rc!=SQLITE_OK ) goto abort_due_to_error; sqlite3PagerJournalMode(pPager, eNew); diff --git a/test/walmode.test b/test/walmode.test index 8d3b882f01..91b5b92049 100644 --- a/test/walmode.test +++ b/test/walmode.test @@ -72,6 +72,8 @@ do_test walmode-3.2 { list $sqlite_sync_count [file exists test.db-wal] [file size test.db-wal] } {0 1 0} +# Test that changing back to journal_mode=persist works. +# do_test walmode-4.1 { execsql { INSERT INTO t1 VALUES(1, 2) } execsql { PRAGMA journal_mode = persist } @@ -91,5 +93,55 @@ do_test walmode-4.5 { list [file exists test.db-journal] [file exists test.db-wal] } {1 0} +# Test that nothing goes wrong if a connection is prevented from changing +# from WAL to rollback mode because a second connection has the database +# open. Or from rollback to WAL. +# +do_test walmode-4.1 { + sqlite3 db2 test.db + execsql { PRAGMA main.journal_mode } db2 +} {delete} +do_test walmode-4.2 { + execsql { PRAGMA main.journal_mode = wal } db +} {wal} +do_test walmode-4.3 { + execsql { SELECT * FROM t1 } db2 +} {1 2} +do_test walmode-4.4 { + catchsql { PRAGMA journal_mode = delete } db +} {1 {database is locked}} +do_test walmode-4.5 { + execsql { PRAGMA main.journal_mode } db +} {wal} +do_test walmode-4.6 { + db2 close + execsql { PRAGMA journal_mode = delete } db +} {delete} +do_test walmode-4.7 { + execsql { PRAGMA main.journal_mode } db +} {delete} +do_test walmode-4.8 { + list [file exists test.db-journal] [file exists test.db-wal] +} {0 0} +do_test walmode-4.9 { + sqlite3 db2 test.db + execsql { + BEGIN; + SELECT * FROM t1; + } db2 +} {1 2} +do_test walmode-4.11 { + execsql { PRAGMA main.journal_mode } db +} {delete} +do_test walmode-4.10 { + catchsql { PRAGMA main.journal_mode = wal } db +} {1 {database is locked}} +do_test walmode-4.11 { + execsql { PRAGMA main.journal_mode } db +} {delete} + + +catch { db close } +catch { db2 close } finish_test From ce4f05f3749f6912a5c5967061d84a480df1237e Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 22 Apr 2010 19:14:13 +0000 Subject: [PATCH 29/89] Create a version of the log checksummer that works on big-endian platforms. Remove the 512KB size limit on the log-summary. FossilOrigin-Name: 5d6d4423d1def39bd2424703120aa985085c3f8e --- manifest | 16 +++---- manifest.uuid | 2 +- src/log.c | 118 ++++++++++++++++++++++++++++++++------------------ src/pager.c | 8 ++-- test/wal.test | 39 +++++++++++++++-- 5 files changed, 124 insertions(+), 59 deletions(-) diff --git a/manifest b/manifest index 8b101354cd..b451c49511 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\stests\sand\schanges\srelated\sto\sswitching\sbetween\sWAL\sand\srollback\smodes. -D 2010-04-22T06:27:05 +C Create\sa\sversion\sof\sthe\slog\schecksummer\sthat\sworks\son\sbig-endian\splatforms.\sRemove\sthe\s512KB\ssize\slimit\son\sthe\slog-summary. +D 2010-04-22T19:14:14 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c edafa806982a281205e71c439e99766e924de938 +F src/log.c da486aab7abfab8edbc20eb49e2f68e9d3a902b6 F src/log.h b8c45a6348d9ef57c5205a08c611d57d07ee9feb F src/main.c 867de6aa444abd97771b2b70472f448d65c1c77e F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a @@ -154,7 +154,7 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053 -F src/pager.c 06fb9b8b7c5283c62f5258e098d6d1649dde7635 +F src/pager.c d83d2ea6d2025554e079f891854b133924305e19 F src/pager.h cee4487ab4f0911dd9f22a40e3cd55afdb7ef444 F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf @@ -758,7 +758,7 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test fd0e5914599b205172dae33c765368a75feb2462 +F test/wal.test fcb5ec1fbec08c1165b6974f126056f2b4cead49 F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47 F test/walmode.test c2f4e30ad64910b2d8faf6cf4e940b3f201b41df @@ -807,7 +807,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 9f4f933f2c6596064fcfc7fb5add87e8c5b57448 -R cdccc9135123b3edc2fcb17c83fbd857 +P 1236318477787a612f02cc98caf2931bd2e99d94 +R 2eba2ad72744173c34347863efb93bd2 U dan -Z 891f9ffc6892beb06734f78c0f8e5711 +Z b18c4f1fb26d4b385c95cfa4edf72422 diff --git a/manifest.uuid b/manifest.uuid index 0aa3e70b8f..815632c84e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1236318477787a612f02cc98caf2931bd2e99d94 \ No newline at end of file +5d6d4423d1def39bd2424703120aa985085c3f8e \ No newline at end of file diff --git a/src/log.c b/src/log.c index 89b60568b5..f32fdb221d 100644 --- a/src/log.c +++ b/src/log.c @@ -83,6 +83,15 @@ struct LogSummaryHdr { LOG_HDRSIZE + ((iFrame)-1)*((pgsz)+LOG_FRAME_HDRSIZE) \ ) +/* +** If using mmap() to access a shared (or otherwise) log-summary file, then +** the mapping size is incremented in units of the following size. +** +** A 64 KB log-summary mapping corresponds to a log file containing over +** 13000 frames, so the mapping size does not need to be increased often. +*/ +#define LOGSUMMARY_MMAP_INCREMENT (64*1024) + /* ** There is one instance of this structure for each log-summary object ** that this process has a connection to. They are stored in a linked @@ -99,6 +108,7 @@ struct LogSummary { char *zPath; /* Path to associated WAL file */ LogLock *pLock; /* Linked list of locks on this object */ LogSummary *pNext; /* Next in global list */ + int nData; /* Size of aData allocation/mapping */ u32 *aData; /* File body */ }; @@ -288,10 +298,22 @@ static void logChecksumBytes(u8 *aByte, int nByte, u32 *aCksum){ assert( LOG_CKSM_BYTES==2*sizeof(u32) ); assert( (nByte&0x00000003)==0 ); - do { - sum1 += (*a32++); - sum2 += sum1; - } while( a32>24); aCksum[1] = sum2 + (sum2>>24); @@ -333,42 +355,6 @@ static void logNormalizePath(char *zPath){ z[j] = 0; } -/* -** Memory map the first nByte bytes of the summary file opened with -** pSummary->fd at pSummary->aData. If the summary file is smaller than -** nByte bytes in size when this function is called, ftruncate() is -** used to expand it before it is mapped. -** -** It is assumed that an exclusive lock is held on the summary file -** by the caller (to protect the ftruncate()). -*/ -static int logSummaryMap(LogSummary *pSummary, int nByte){ - struct stat sStat; - int rc; - int fd = pSummary->fd; - void *pMap; - - assert( pSummary->aData==0 ); - - /* If the file is less than nByte bytes in size, cause it to grow. */ - rc = fstat(fd, &sStat); - if( rc!=0 ) return SQLITE_IOERR; - if( sStat.st_sizeaData = (u32 *)pMap; - pSummary->nData = nByte; - - return SQLITE_OK; -} - /* ** Unmap the log-summary mapping and close the file-descriptor. If ** the isTruncate argument is non-zero, truncate the log-summary file @@ -523,6 +509,44 @@ static void logMergesort8( } +/* +** Memory map the first nByte bytes of the summary file opened with +** pSummary->fd at pSummary->aData. If the summary file is smaller than +** nByte bytes in size when this function is called, ftruncate() is +** used to expand it before it is mapped. +** +** It is assumed that an exclusive lock is held on the summary file +** by the caller (to protect the ftruncate()). +*/ +static int logSummaryMap(LogSummary *pSummary, int nByte){ + struct stat sStat; + int rc; + int fd = pSummary->fd; + void *pMap; + + assert( pSummary->aData==0 ); + + /* If the file is less than nByte bytes in size, cause it to grow. */ + rc = fstat(fd, &sStat); + if( rc!=0 ) return SQLITE_IOERR; + if( sStat.st_sizeaData = (u32 *)pMap; + pSummary->nData = nByte/4; + + return SQLITE_OK; +} + /* ** Return the index in the LogSummary.aData array that corresponds to ** frame iFrame. The log-summary file consists of a header, followed by @@ -543,6 +567,16 @@ static int logSummaryEntry(u32 iFrame){ static void logSummaryAppend(LogSummary *pSummary, u32 iFrame, u32 iPage){ u32 iSlot = logSummaryEntry(iFrame); + if( (iSlot+128)>=pSummary->nData ){ + int nByte = pSummary->nData*4 + LOGSUMMARY_MMAP_INCREMENT; + + sqlite3_mutex_enter(pSummary->mutex); + munmap(pSummary->aData, pSummary->nData*4); + pSummary->aData = 0; + logSummaryMap(pSummary, nByte); + sqlite3_mutex_leave(pSummary->mutex); + } + /* Set the log-summary entry itself */ pSummary->aData[iSlot] = iPage; @@ -606,7 +640,7 @@ static int logSummaryRecover(LogSummary *pSummary, sqlite3_file *pFd){ ** SQLITE_MAX_PAGE_SIZE, conclude that the log file contains no valid data. */ nPgsz = sqlite3Get4byte(&aBuf[0]); - if( nPgsz&(nPgsz-1) || nPgsz>SQLITE_MAX_PAGE_SIZE ){ + if( nPgsz&(nPgsz-1) || nPgsz>SQLITE_MAX_PAGE_SIZE || nPgsz<512 ){ goto finished; } aCksum[0] = sqlite3Get4byte(&aBuf[4]); @@ -885,7 +919,7 @@ static int logSummaryInit( assert( sqlite3_mutex_held(pSummary->mutex) ); rc = logLockMutex(pSummary, LOG_WRLOCKW); if( rc!=SQLITE_OK ) return rc; - rc = logSummaryMap(pSummary, 512*1024); + rc = logSummaryMap(pSummary, LOGSUMMARY_MMAP_INCREMENT); if( rc!=SQLITE_OK ) goto out; /* Try to obtain an EXCLUSIVE lock on the dead-mans-hand region. If this diff --git a/src/pager.c b/src/pager.c index 157413e9b0..81f0968f54 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3755,8 +3755,8 @@ static int pagerHasWAL(Pager *pPager, int *pExists){ } static int pagerOpenSnapshot(Pager *pPager){ - int rc; - int changed; + int rc; /* Return code */ + int changed = 0; /* True if cache must be reset */ assert( pagerUseLog(pPager) ); @@ -3764,8 +3764,8 @@ static int pagerOpenSnapshot(Pager *pPager){ if( rc==SQLITE_OK ){ int dummy; if( changed ){ - pager_reset(pPager); - assert( pPager->errCode || pPager->dbSizeValid==0 ); + pager_reset(pPager); + assert( pPager->errCode || pPager->dbSizeValid==0 ); } rc = sqlite3PagerPagecount(pPager, &dummy); } diff --git a/test/wal.test b/test/wal.test index f0268e19a2..f69c94020b 100644 --- a/test/wal.test +++ b/test/wal.test @@ -676,7 +676,6 @@ do_test wal-12.3 { execsql { INSERT INTO t2 VALUES('B', 1) } list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] } {3 2} - do_test wal-12.4 { file copy -force test.db test2.db file copy -force test.db-wal test2.db-wal @@ -684,7 +683,6 @@ do_test wal-12.4 { execsql { SELECT * FROM t2 } db2 } {B 1} db2 close - do_test wal-12.5 { execsql { PRAGMA checkpoint; @@ -696,14 +694,47 @@ do_test wal-12.5 { SELECT * FROM t2; } } {B 2} - -do_test wal-12.4 { +do_test wal-12.6 { file copy -force test.db test2.db file copy -force test.db-wal test2.db-wal sqlite3_wal db2 test2.db execsql { SELECT * FROM t2 } db2 } {B 2} db2 close +db close + +#------------------------------------------------------------------------- +# Test large log summaries. +# +do_test wal-13.1 { + list [file exists test.db] [file exists test.db-wal] +} {1 0} +do_test wal-13.2 { + set fd [open test.db-wal w] + seek $fd [expr 200*1024*1024] + puts $fd "" + close $fd + sqlite3 db test.db + execsql { SELECT * FROM t2 } +} {B 2} +do_test wal-13.3 { + db close + file exists test.db-wal +} {0} +do_test wal-13.4 { + sqlite3 db test.db + execsql { SELECT count(*) FROM t2 } +} {1} +do_test wal-13.5 { + for {set i 0} {$i < 15} {incr i} { + execsql { INSERT INTO t2 SELECT randomblob(400), randomblob(400) FROM t2 } + } + execsql { SELECT count(*) FROM t2 } +} [expr int(pow(2, 15))] +do_test wal-13.6 { + file size test.db-wal-summary +} [expr 192*1024] + finish_test From d5c99c393941cf67ce88bebc5df512d25ccc4c85 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 23 Apr 2010 11:44:40 +0000 Subject: [PATCH 30/89] Add very simple test cases for backup and VACUUM of WAL databases. More to come. FossilOrigin-Name: 1077d8130b8ed5716ad73f78382270909d347963 --- manifest | 11 ++++--- manifest.uuid | 2 +- test/walbak.test | 86 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 test/walbak.test diff --git a/manifest b/manifest index b451c49511..49b2e5bcc4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Create\sa\sversion\sof\sthe\slog\schecksummer\sthat\sworks\son\sbig-endian\splatforms.\sRemove\sthe\s512KB\ssize\slimit\son\sthe\slog-summary. -D 2010-04-22T19:14:14 +C Add\svery\ssimple\stest\scases\sfor\sbackup\sand\sVACUUM\sof\sWAL\sdatabases.\sMore\sto\scome. +D 2010-04-23T11:44:41 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -759,6 +759,7 @@ F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d F test/wal.test fcb5ec1fbec08c1165b6974f126056f2b4cead49 +F test/walbak.test 3aecf824ee6433bd34673336623c4990aa1346ba F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47 F test/walmode.test c2f4e30ad64910b2d8faf6cf4e940b3f201b41df @@ -807,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 1236318477787a612f02cc98caf2931bd2e99d94 -R 2eba2ad72744173c34347863efb93bd2 +P 5d6d4423d1def39bd2424703120aa985085c3f8e +R 7813072bd3855cfe2a094b15eff01c69 U dan -Z b18c4f1fb26d4b385c95cfa4edf72422 +Z f898bd1b5727c5103088c594c0575550 diff --git a/manifest.uuid b/manifest.uuid index 815632c84e..b5381319c7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5d6d4423d1def39bd2424703120aa985085c3f8e \ No newline at end of file +1077d8130b8ed5716ad73f78382270909d347963 \ No newline at end of file diff --git a/test/walbak.test b/test/walbak.test new file mode 100644 index 0000000000..f4998ac7f1 --- /dev/null +++ b/test/walbak.test @@ -0,0 +1,86 @@ +# 2010 April 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +proc log_file_size {nFrame pgsz} { + expr {12 + ($pgsz+16)*$nFrame} +} + +# Make sure a simple backup from a WAL database works. +# +do_test walbak-1.0 { + execsql { + PRAGMA synchronous = NORMAL; + PRAGMA page_size = 1024; + PRAGMA auto_vacuum = 0; + PRAGMA journal_mode = wal; + BEGIN; + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES('I', 'one'); + COMMIT; + } +} {wal} +do_test walbak-1.1 { + file delete -force bak.db bak.db-journal bak.db-wal + db backup bak.db + file size bak.db +} [expr 3*1024] +do_test walbak-1.2 { + sqlite3 db2 bak.db + execsql { + SELECT * FROM t1; + PRAGMA main.journal_mode; + } db2 +} {I one wal} +do_test walbak-1.3 { + execsql { PRAGMA integrity_check } db2 +} {ok} +db2 close + +# Try a VACUUM on a WAL database. +# +do_test walbak-2.1 { + execsql { + VACUUM; + PRAGMA main.journal_mode; + } +} {wal} +do_test walbak-2.2 { + list [file size test.db] [file size test.db-wal] +} [list 1024 [log_file_size 6 1024]] +do_test walbak-2.3 { + execsql { PRAGMA checkpoint } + list [file size test.db] [file size test.db-wal] +} [list [expr 3*1024] [log_file_size 6 1024]] +do_test walbak-2.4 { + execsql { + CREATE TABLE t2(a, b); + INSERT INTO t2 SELECT * FROM t1; + DROP TABLE t1; + } + list [file size test.db] [file size test.db-wal] +} [list [expr 3*1024] [log_file_size 6 1024]] +do_test walbak-2.5 { + execsql { VACUUM } + list [file size test.db] [file size test.db-wal] +} [list [expr 3*1024] [log_file_size 8 1024]] +do_test walbak-2.6 { + execsql { PRAGMA checkpoint } + list [file size test.db] [file size test.db-wal] +} [list [expr 2*1024] [log_file_size 8 1024]] + +finish_test From 3306c4a945de71d1313b2e090d1e3bd2a09821cd Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 23 Apr 2010 19:15:00 +0000 Subject: [PATCH 31/89] Fixes and tests for backup of a WAL database. FossilOrigin-Name: 480d12db4c0ebcc37598f7620d39193875eab15b --- manifest | 20 ++++----- manifest.uuid | 2 +- src/log.c | 8 ++++ src/log.h | 3 ++ src/pager.c | 55 ++++++++++++++++++++++-- test/backup.test | 3 ++ test/walbak.test | 110 ++++++++++++++++++++++++++++++++++++++++++++--- 7 files changed, 181 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 49b2e5bcc4..2f6f6317d8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\svery\ssimple\stest\scases\sfor\sbackup\sand\sVACUUM\sof\sWAL\sdatabases.\sMore\sto\scome. -D 2010-04-23T11:44:41 +C Fixes\sand\stests\sfor\sbackup\sof\sa\sWAL\sdatabase. +D 2010-04-23T19:15:00 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,8 +131,8 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c da486aab7abfab8edbc20eb49e2f68e9d3a902b6 -F src/log.h b8c45a6348d9ef57c5205a08c611d57d07ee9feb +F src/log.c d9fdaad6b0b5ace54153b85f5bfcf1b9d1abac67 +F src/log.h bc44b0eec723648c8aa0a05ab78e1b76a4032e02 F src/main.c 867de6aa444abd97771b2b70472f448d65c1c77e F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -154,7 +154,7 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053 -F src/pager.c d83d2ea6d2025554e079f891854b133924305e19 +F src/pager.c 485a34834a96863fd709f4d01f886ae70a9689ea F src/pager.h cee4487ab4f0911dd9f22a40e3cd55afdb7ef444 F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf @@ -253,7 +253,7 @@ F test/autoindex1.test ffb06a246e2c1f89cfbe3d93eca513c9e78d4063 F test/autovacuum.test 25f891bc343a8bf5d9229e2e9ddab9f31a9ab5ec F test/autovacuum_ioerr2.test 598b0663074d3673a9c1bc9a16e80971313bafe6 F test/avtrans.test 1e901d8102706b63534dbd2bdd4d8f16c4082650 -F test/backup.test 3549ea8f541a08205c0eb813b21e81ea8301f6ed +F test/backup.test b1e874fa9b01de9dd5137a8371d060b76a435162 F test/backup2.test 159419073d9769fdb1780ed7e5b391a046f898d5 F test/backup_ioerr.test 1f012e692f42c0442ae652443258f70e9f20fa38 F test/backup_malloc.test 1e063c6d75143d0d6e0ae77971dd690070369387 @@ -759,7 +759,7 @@ F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d F test/wal.test fcb5ec1fbec08c1165b6974f126056f2b4cead49 -F test/walbak.test 3aecf824ee6433bd34673336623c4990aa1346ba +F test/walbak.test f6fde9a5f59d0c697cb1f4af7876178c2f69a7ba F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47 F test/walmode.test c2f4e30ad64910b2d8faf6cf4e940b3f201b41df @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 5d6d4423d1def39bd2424703120aa985085c3f8e -R 7813072bd3855cfe2a094b15eff01c69 +P 1077d8130b8ed5716ad73f78382270909d347963 +R 0d1581e70f8ee05579afbc11de72b135 U dan -Z f898bd1b5727c5103088c594c0575550 +Z c6fc40f98d5658443e299ebf167b2a61 diff --git a/manifest.uuid b/manifest.uuid index b5381319c7..4ef5c84110 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1077d8130b8ed5716ad73f78382270909d347963 \ No newline at end of file +480d12db4c0ebcc37598f7620d39193875eab15b \ No newline at end of file diff --git a/src/log.c b/src/log.c index f32fdb221d..c1b734f8fe 100644 --- a/src/log.c +++ b/src/log.c @@ -1568,6 +1568,14 @@ int sqlite3LogWriteLock(Log *pLog, int op){ return SQLITE_OK; } +/* +** Return true if data has been written but not committed to the log file. +*/ +int sqlite3LogDirty(Log *pLog){ + assert( pLog->isWriteLocked ); + return( pLog->hdr.iLastPg!=((LogSummaryHdr*)pLog->pSummary->aData)->iLastPg ); +} + /* ** Write a set of frames to the log. The caller must hold at least a ** RESERVED lock on the database file. diff --git a/src/log.h b/src/log.h index af698ebf5f..02201c837e 100644 --- a/src/log.h +++ b/src/log.h @@ -37,6 +37,9 @@ void sqlite3LogDbsize(Log *pLog, Pgno *pPgno); /* Obtain or release the WRITER lock. */ int sqlite3LogWriteLock(Log *pLog, int op); +/* Return true if data has been written but not committed to the log file. */ +int sqlite3LogDirty(Log *pLog); + /* Write a frame or frames to the log. */ int sqlite3LogFrames(Log *pLog, int, PgHdr *, Pgno, int, int); diff --git a/src/pager.c b/src/pager.c index 81f0968f54..2db8b9336c 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2234,9 +2234,30 @@ static int readDbPage(PgHdr *pPg){ return rc; } +/* +** This function is called when a transaction on a WAL database is rolled +** back. For each dirty page in the cache, do one of the following: +** +** * If the page has no outstanding references, simply discard it. +** * Otherwise, if the page has one or more outstanding references, +** reload the original content from the database (or log file). +*/ static int pagerRollbackLog(Pager *pPager){ int rc = SQLITE_OK; PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); + + /* Normally, if a transaction is rolled back, any backup processes are + ** updated as data is copied out of the rollback journal and into the + ** database. This is not generally possible with a WAL database, as + ** rollback involves simply truncating the log file. Therefore, if one + ** or more frames have already been written to the log (and therefore + ** also copied into the backup databases) as part of this transaction, + ** the backups must be restarted. + */ + if( sqlite3LogDirty(pPager->pLog) ){ + sqlite3BackupRestart(pPager->pBackup); + } + pPager->dbSize = pPager->dbOrigSize; while( pList && rc==SQLITE_OK ){ PgHdr *pNext = pList->pDirty; @@ -3218,6 +3239,33 @@ static int subjournalPage(PgHdr *pPg){ return rc; } +/* +** This function is a wrapper around sqlite3LogFrames(). As well as logging +** the contents of the list of pages headed by pList (connected by pDirty), +** this function notifies any active backup processes that the pages have +** changed. +*/ +static int pagerLogFrames( + Pager *pPager, /* Pager object */ + PgHdr *pList, /* List of frames to log */ + Pgno nTruncate, /* Database size after this commit */ + int isCommit, /* True if this is a commit */ + int sync_flags /* Flags to pass to OsSync() (or 0) */ +){ + int rc; /* Return code */ + + assert( pPager->pLog ); + rc = sqlite3LogFrames(pPager->pLog, + pPager->pageSize, pList, nTruncate, isCommit, sync_flags + ); + if( rc==SQLITE_OK && pPager->pBackup ){ + PgHdr *p; + for(p=pList; p; p=p->pDirty){ + sqlite3BackupUpdate(pPager->pBackup, p->pgno, (u8 *)p->pData); + } + } + return rc; +} /* ** This function is called by the pcache layer when it has reached some @@ -3248,7 +3296,7 @@ static int pagerStress(void *p, PgHdr *pPg){ pPg->pDirty = 0; if( pagerUseLog(pPager) ){ /* Write a single frame for this page to the log. */ - rc = sqlite3LogFrames(pPager->pLog, pPager->pageSize, pPg, 0, 0, 0); + rc = pagerLogFrames(pPager, pPg, 0, 0, 0); }else{ /* The doNotSync flag is set by the sqlite3PagerWrite() function while it ** is journalling a set of two or more database pages that are stored @@ -4881,8 +4929,8 @@ int sqlite3PagerCommitPhaseOne( if( pagerUseLog(pPager) ){ PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); if( pList ){ - rc = sqlite3LogFrames(pPager->pLog, pPager->pageSize, pList, - pPager->dbSize, 1, (pPager->fullSync ? pPager->sync_flags : 0) + rc = pagerLogFrames(pPager, pList, pPager->dbSize, 1, + (pPager->fullSync ? pPager->sync_flags : 0) ); } sqlite3PcacheCleanAll(pPager->pPCache); @@ -5119,6 +5167,7 @@ int sqlite3PagerRollback(Pager *pPager){ PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager))); if( pagerUseLog(pPager) ){ int rc2; + rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1); rc2 = pager_end_transaction(pPager, pPager->setMaster); if( rc==SQLITE_OK ) rc = rc2; diff --git a/test/backup.test b/test/backup.test index ec30adb361..ca8de8ecca 100644 --- a/test/backup.test +++ b/test/backup.test @@ -38,6 +38,8 @@ source $testdir/tester.tcl # # backup-9.*: Test that passing a negative argument to backup_step() is # interpreted as "copy the whole file". +# +# backup-10.*: Test writing the source database mid backup. # proc data_checksum {db file} { $db one "SELECT md5sum(a, b) FROM ${file}.t1" } @@ -905,6 +907,7 @@ ifcapable memorymanage { } +#----------------------------------------------------------------------- # Test that if the database is written to via the same database handle being # used as the source by a backup operation: # diff --git a/test/walbak.test b/test/walbak.test index f4998ac7f1..11a7040cdc 100644 --- a/test/walbak.test +++ b/test/walbak.test @@ -20,6 +20,12 @@ proc log_file_size {nFrame pgsz} { expr {12 + ($pgsz+16)*$nFrame} } +# Test organization: +# +# walback-1.*: Simple tests. +# walback-2.*: Test backups when the source db is modified mid-backup. +# + # Make sure a simple backup from a WAL database works. # do_test walbak-1.0 { @@ -53,20 +59,20 @@ db2 close # Try a VACUUM on a WAL database. # -do_test walbak-2.1 { +do_test walbak-1.4 { execsql { VACUUM; PRAGMA main.journal_mode; } } {wal} -do_test walbak-2.2 { +do_test walbak-1.5 { list [file size test.db] [file size test.db-wal] } [list 1024 [log_file_size 6 1024]] -do_test walbak-2.3 { +do_test walbak-1.6 { execsql { PRAGMA checkpoint } list [file size test.db] [file size test.db-wal] } [list [expr 3*1024] [log_file_size 6 1024]] -do_test walbak-2.4 { +do_test walbak-1.7 { execsql { CREATE TABLE t2(a, b); INSERT INTO t2 SELECT * FROM t1; @@ -74,13 +80,105 @@ do_test walbak-2.4 { } list [file size test.db] [file size test.db-wal] } [list [expr 3*1024] [log_file_size 6 1024]] -do_test walbak-2.5 { +do_test walbak-1.8 { execsql { VACUUM } list [file size test.db] [file size test.db-wal] } [list [expr 3*1024] [log_file_size 8 1024]] -do_test walbak-2.6 { +do_test walbak-1.9 { execsql { PRAGMA checkpoint } list [file size test.db] [file size test.db-wal] } [list [expr 2*1024] [log_file_size 8 1024]] +#------------------------------------------------------------------------- +# Backups when the source db is modified mid-backup. +# +proc sig {{db db}} { + $db eval { + PRAGMA integrity_check; + SELECT md5sum(a, b) FROM t1; + } +} +db close +file delete test.db +sqlite3 db test.db +do_test walbak-2.1 { + execsql { PRAGMA journal_mode = WAL } + execsql { + CREATE TABLE t1(a PRIMARY KEY, b); + BEGIN; + INSERT INTO t1 VALUES(randomblob(500), randomblob(500)); + INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /* 2 */ + INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /* 4 */ + INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /* 8 */ + INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /* 16 */ + INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /* 32 */ + INSERT INTO t1 SELECT randomblob(500), randomblob(500) FROM t1; /* 64 */ + COMMIT; + } +} {} +do_test walbak-2.2 { + db backup abc.db + sqlite3 db2 abc.db + string compare [sig db] [sig db2] +} {0} + +do_test walbak-2.3 { + sqlite3_backup B db2 main db main + B step 50 + execsql { UPDATE t1 SET b = randomblob(500) } + list [B step 1000] [B finish] +} {SQLITE_DONE SQLITE_OK} +do_test walbak-2.4 { + string compare [sig db] [sig db2] +} {0} + +do_test walbak-2.5 { + db close + sqlite3 db test.db + execsql { PRAGMA cache_size = 10 } + sqlite3_backup B db2 main db main + B step 50 + execsql { + BEGIN; + UPDATE t1 SET b = randomblob(500); + } + expr [file size test.db-wal] > 10*1024 +} {1} +do_test walbak-2.6 { + B step 1000 +} {SQLITE_BUSY} +do_test walbak-2.7 { + execsql COMMIT + list [B step 1000] [B finish] +} {SQLITE_DONE SQLITE_OK} +do_test walbak-2.8 { + string compare [sig db] [sig db2] +} {0} + +do_test walbak-2.9 { + db close + sqlite3 db test.db + execsql { PRAGMA cache_size = 10 } + sqlite3_backup B db2 main db main + B step 50 + execsql { + BEGIN; + UPDATE t1 SET b = randomblob(500); + } + expr [file size test.db-wal] > 10*1024 +} {1} +do_test walbak-2.10 { + B step 1000 +} {SQLITE_BUSY} +do_test walbak-2.11 { + execsql ROLLBACK +set sigB [sig db] + list [B step 1000] [B finish] +} {SQLITE_DONE SQLITE_OK} +do_test walbak-2.12 { + string compare [sig db] [sig db2] +} {0} +db2 close + finish_test + From ff207012e6b7326d66eaaf4ca32acaa00472da08 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 24 Apr 2010 04:49:15 +0000 Subject: [PATCH 32/89] Reserve some space at the start of the log-summary file to apply locks to. FossilOrigin-Name: a031aa1fa01f062b087154833738f011f7bc4eb5 --- manifest | 12 +++---- manifest.uuid | 2 +- src/log.c | 95 +++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 76 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 2f6f6317d8..ffb0444ea2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sand\stests\sfor\sbackup\sof\sa\sWAL\sdatabase. -D 2010-04-23T19:15:00 +C Reserve\ssome\sspace\sat\sthe\sstart\sof\sthe\slog-summary\sfile\sto\sapply\slocks\sto. +D 2010-04-24T04:49:15 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c d9fdaad6b0b5ace54153b85f5bfcf1b9d1abac67 +F src/log.c 7569a2f6541c86127914d50e9fd6e65c17111bc6 F src/log.h bc44b0eec723648c8aa0a05ab78e1b76a4032e02 F src/main.c 867de6aa444abd97771b2b70472f448d65c1c77e F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 1077d8130b8ed5716ad73f78382270909d347963 -R 0d1581e70f8ee05579afbc11de72b135 +P 480d12db4c0ebcc37598f7620d39193875eab15b +R f27ff96a8474356c5e71b7eed104ff79 U dan -Z c6fc40f98d5658443e299ebf167b2a61 +Z 9b426b4ef3d6b692bab78cb4477e0b43 diff --git a/manifest.uuid b/manifest.uuid index 4ef5c84110..43145bd9a2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -480d12db4c0ebcc37598f7620d39193875eab15b \ No newline at end of file +a031aa1fa01f062b087154833738f011f7bc4eb5 \ No newline at end of file diff --git a/src/log.c b/src/log.c index c1b734f8fe..3837d7e170 100644 --- a/src/log.c +++ b/src/log.c @@ -28,9 +28,33 @@ */ /* -** LOG SUMMARY FORMAT +** LOG SUMMARY FILE FORMAT ** -** TODO. +** The log-summary file consists of a header region, followed by an +** region that contains no useful data (used to apply byte-range locks +** to), followed by the data region. +** +** The contents of both the header and data region are specified in terms +** of 1, 2 and 4 byte unsigned integers. All integers are stored in +** machine-endian order. +** +** A log-summary file is essentially a shadow-pager map. It contains a +** mapping from database page number to the set of locations in the log +** file that contain versions of the database page. When a database +** client needs to read a page of data, it first queries the log-summary +** file to determine if the required version of the page is stored in +** the log. If so, it is read from the log file. If not, it is read from +** the database file. +** +** Whenever a transaction is appended to the log or a checkpoint transfers +** data from the log file into the database file, the log-summary is +** updated accordingly. +** +** The fields in the log-summary file header are described in the comment +** directly above the definition of struct LogSummaryHdr (see below). +** Immediately following the fields in the LogSummaryHdr structure is +** an 8 byte checksum based on the contents of the header. This field is +** not the same as the iCheck1 and iCheck2 fields of the LogSummaryHdr. */ #include "log.h" @@ -65,14 +89,19 @@ struct LogSummaryHdr { /* Size of serialized LogSummaryHdr object. */ #define LOGSUMMARY_HDR_NFIELD (sizeof(LogSummaryHdr) / sizeof(u32)) -#define LOGSUMMARY_FRAME_OFFSET \ - (LOGSUMMARY_HDR_NFIELD + LOG_CKSM_BYTES/sizeof(u32)) +/* A block of 16 bytes beginning at LOGSUMMARY_LOCK_OFFSET is reserved +** for locks. Since some systems only feature mandatory file-locks, we +** do not read or write data from the region of the file on which locks +** are applied. +*/ +#define LOGSUMMARY_LOCK_OFFSET ((sizeof(LogSummaryHdr))+2*sizeof(u32)) +#define LOGSUMMARY_LOCK_RESERVED 16 - - -/* Size of frame header */ +/* Size of header before each frame in log file */ #define LOG_FRAME_HDRSIZE 16 -#define LOG_HDRSIZE 12 + +/* Size of log header */ +#define LOG_HDRSIZE 12 /* ** Return the offset of frame iFrame in the log file, assuming a database @@ -80,7 +109,7 @@ struct LogSummaryHdr { ** log frame-header. */ #define logFrameOffset(iFrame, pgsz) ( \ - LOG_HDRSIZE + ((iFrame)-1)*((pgsz)+LOG_FRAME_HDRSIZE) \ + LOG_HDRSIZE + ((iFrame)-1)*((pgsz)+LOG_FRAME_HDRSIZE) \ ) /* @@ -197,9 +226,9 @@ struct LogSummary { ** write to the log file) require an exclusive lock on region D. ** Which they cannot get until all region D readers have finished. */ -#define LOG_LOCK_MUTEX 12 -#define LOG_LOCK_DMH 13 -#define LOG_LOCK_REGION 14 +#define LOG_LOCK_MUTEX (LOGSUMMARY_LOCK_OFFSET) +#define LOG_LOCK_DMH (LOG_LOCK_MUTEX+1) +#define LOG_LOCK_REGION (LOG_LOCK_DMH+1) /* ** The four lockable regions associated with each log-summary. A connection @@ -386,13 +415,13 @@ static int logSummaryUnmap(LogSummary *pSummary, int isUnlink){ } static void logSummaryWriteHdr(LogSummary *pSummary, LogSummaryHdr *pHdr){ - u32 *aData = pSummary->aData; - memcpy(aData, pHdr, sizeof(LogSummaryHdr)); - aData[LOGSUMMARY_HDR_NFIELD] = 1; - aData[LOGSUMMARY_HDR_NFIELD+1] = 1; - logChecksumBytes( - (u8 *)aData, sizeof(LogSummaryHdr), &aData[LOGSUMMARY_HDR_NFIELD] - ); + u32 *aHdr = pSummary->aData; /* Write header here */ + u32 *aCksum = &aHdr[LOGSUMMARY_HDR_NFIELD]; /* Write header cksum here */ + + assert( LOGSUMMARY_HDR_NFIELD==sizeof(LogSummaryHdr)/4 ); + memcpy(aHdr, pHdr, sizeof(LogSummaryHdr)); + aCksum[0] = aCksum[1] = 1; + logChecksumBytes((u8 *)aHdr, sizeof(LogSummaryHdr), aCksum); } /* @@ -553,7 +582,11 @@ static int logSummaryMap(LogSummary *pSummary, int nByte){ ** alternating "map" and "index" blocks. */ static int logSummaryEntry(u32 iFrame){ - return ((((iFrame-1)>>8)<<6) + iFrame-1 + 2 + LOGSUMMARY_HDR_NFIELD); + return ( + (LOGSUMMARY_LOCK_OFFSET+LOGSUMMARY_LOCK_RESERVED)/sizeof(u32) + + (((iFrame-1)>>8)<<6) /* Indexes that occur before iFrame */ + + iFrame-1 /* Db page numbers that occur before iFrame */ + ); } @@ -692,6 +725,9 @@ finished: /* ** Place, modify or remove a lock on the log-summary file associated ** with pSummary. +** +** The locked byte-range should be inside the region dedicated to +** locking. This region of the log-summary file is never read or written. */ static int logLockFd( LogSummary *pSummary, /* The log-summary object to lock */ @@ -716,6 +752,13 @@ static int logLockFd( assert( ArraySize(aType)==ArraySize(aOp) ); assert( op>=0 && op0 ); + assert( iStart>=LOGSUMMARY_LOCK_OFFSET + && iStart+nByte<=LOGSUMMARY_LOCK_OFFSET+LOGSUMMARY_LOCK_RESERVED + ); +#if defined(SQLITE_DEBUG) && defined(SQLITE_OS_UNIX) + if( pSummary->aData ) memset(&((u8*)pSummary->aData)[iStart], op, nByte); +#endif memset(&f, 0, sizeof(f)); f.l_type = aType[op]; @@ -834,13 +877,13 @@ static int logLockRegion(Log *pLog, u32 mRegion, int op){ int iStart; /* Byte offset to start locking operation */ int iLen; /* Length field for locking operation */ } aMap[] = { - /* 0000 */ {0, 0}, /* 0001 */ {4+LOG_LOCK_REGION, 1}, - /* 0010 */ {3+LOG_LOCK_REGION, 1}, /* 0011 */ {3+LOG_LOCK_REGION, 2}, - /* 0100 */ {2+LOG_LOCK_REGION, 1}, /* 0101 */ {0, 0}, - /* 0110 */ {2+LOG_LOCK_REGION, 2}, /* 0111 */ {2+LOG_LOCK_REGION, 3}, - /* 1000 */ {1+LOG_LOCK_REGION, 1}, /* 1001 */ {0, 0}, + /* 0000 */ {0, 0}, /* 0001 */ {3+LOG_LOCK_REGION, 1}, + /* 0010 */ {2+LOG_LOCK_REGION, 1}, /* 0011 */ {2+LOG_LOCK_REGION, 2}, + /* 0100 */ {1+LOG_LOCK_REGION, 1}, /* 0101 */ {0, 0}, + /* 0110 */ {1+LOG_LOCK_REGION, 2}, /* 0111 */ {1+LOG_LOCK_REGION, 3}, + /* 1000 */ {0+LOG_LOCK_REGION, 1}, /* 1001 */ {0, 0}, /* 1010 */ {0, 0}, /* 1011 */ {0, 0}, - /* 1100 */ {1+LOG_LOCK_REGION, 2}, /* 1101 */ {0, 0}, + /* 1100 */ {0+LOG_LOCK_REGION, 2}, /* 1101 */ {0, 0}, /* 1110 */ {0, 0}, /* 1111 */ {0, 0} }; int rc; /* Return code of logLockFd() */ From 74d6cd887f8cd63b7b3f9e92b372aaad8df17c69 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 24 Apr 2010 18:44:05 +0000 Subject: [PATCH 33/89] Fix bugs in WAL mode rollback. FossilOrigin-Name: 31215969f59be536fe87431bb9fbfa7d13027e35 --- manifest | 18 +++++++-------- manifest.uuid | 2 +- src/log.c | 27 ++++++++++++++++++++++ src/log.h | 3 +++ src/pager.c | 62 ++++++++++++++++++++++++++++++++++++++------------- test/wal.test | 55 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 141 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index e0cccadbc3..d67960a052 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\swith\s[0291ed974d].\nMerge\swith\s[0291ed974d].\nMerge\swith\s[0291ed974d]. -D 2010-04-24T14:33:08 +C Fix\sbugs\sin\sWAL\smode\srollback. +D 2010-04-24T18:44:05 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,8 +131,8 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c 7569a2f6541c86127914d50e9fd6e65c17111bc6 -F src/log.h bc44b0eec723648c8aa0a05ab78e1b76a4032e02 +F src/log.c 6097555f746e9992b3bb7a257d2e60ced69e3a84 +F src/log.h 06eacedac8b5a720cb88aa3c9e5d886ba7b394fe F src/main.c 867de6aa444abd97771b2b70472f448d65c1c77e F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -154,7 +154,7 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1 F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053 -F src/pager.c 485a34834a96863fd709f4d01f886ae70a9689ea +F src/pager.c 9da40bb50d2baa8f7cbb18579f147b7623856211 F src/pager.h cee4487ab4f0911dd9f22a40e3cd55afdb7ef444 F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf @@ -758,7 +758,7 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test fcb5ec1fbec08c1165b6974f126056f2b4cead49 +F test/wal.test 8dcfd1ce874aead6199fa715ba8e35cfbb3ef766 F test/walbak.test f6fde9a5f59d0c697cb1f4af7876178c2f69a7ba F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47 @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 1e793d3a6d46ec5d744e077b10362e7cfa118eef 0291ed974d5bf1e344e2c38422530cc961b897da -R 2a96ead5dd8a150336ca23e8e3a9aa5f +P a352f6285e33a806fbe4475e720e763fdc5bb47d +R a372a014eac56d1df8e79bcff4ecc6e7 U dan -Z 95ff3026ac52c375d39478f793178d69 +Z 3d8f3676c5fc9fd427fe6aa0aa15e428 diff --git a/manifest.uuid b/manifest.uuid index 9d7f64dc51..b8874609f2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a352f6285e33a806fbe4475e720e763fdc5bb47d \ No newline at end of file +31215969f59be536fe87431bb9fbfa7d13027e35 \ No newline at end of file diff --git a/src/log.c b/src/log.c index 3837d7e170..d9637de024 100644 --- a/src/log.c +++ b/src/log.c @@ -1611,6 +1611,33 @@ int sqlite3LogWriteLock(Log *pLog, int op){ return SQLITE_OK; } +/* +** The log handle passed to this function must be holding the write-lock. +** +** If any data has been written (but not committed) to the log file, this +** function moves the write-pointer back to the start of the transaction. +** +** Additionally, the callback function is invoked for each frame written +** to the log since the start of the transaction. If the callback returns +** other than SQLITE_OK, it is not invoked again and the error code is +** returned to the caller. +** +** Otherwise, if the callback function does not return an error, this +** function returns SQLITE_OK. +*/ +int sqlite3LogUndo(Log *pLog, int (*xUndo)(void *, Pgno), void *pUndoCtx){ + int rc = SQLITE_OK; + Pgno iMax = pLog->hdr.iLastPg; + Pgno iFrame; + + assert( pLog->isWriteLocked ); + logSummaryReadHdr(pLog, 0); + for(iFrame=pLog->hdr.iLastPg+1; iFrame<=iMax && rc==SQLITE_OK; iFrame++){ + rc = xUndo(pUndoCtx, pLog->pSummary->aData[logSummaryEntry(iFrame)]); + } + return rc; +} + /* ** Return true if data has been written but not committed to the log file. */ diff --git a/src/log.h b/src/log.h index 02201c837e..c0699ba6fb 100644 --- a/src/log.h +++ b/src/log.h @@ -37,6 +37,9 @@ void sqlite3LogDbsize(Log *pLog, Pgno *pPgno); /* Obtain or release the WRITER lock. */ int sqlite3LogWriteLock(Log *pLog, int op); +/* Undo any frames written (but not committed) to the log */ +int sqlite3LogUndo(Log *pLog, int (*xUndo)(void *, Pgno), void *pUndoCtx); + /* Return true if data has been written but not committed to the log file. */ int sqlite3LogDirty(Log *pLog); diff --git a/src/pager.c b/src/pager.c index 2db8b9336c..18b8960da7 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2235,16 +2235,44 @@ static int readDbPage(PgHdr *pPg){ } /* -** This function is called when a transaction on a WAL database is rolled -** back. For each dirty page in the cache, do one of the following: +** This function is invoked once for each page that has already been +** written into the log file when a WAL transaction is rolled back. +** Parameter iPg is the page number of said page. The pCtx argument +** is actually a pointer to the Pager structure. ** -** * If the page has no outstanding references, simply discard it. -** * Otherwise, if the page has one or more outstanding references, -** reload the original content from the database (or log file). +** If page iPg is present in the cache, and has no outstanding references, +** it is discarded. Otherwise, if there are one or more outstanding +** references, the page content is reloaded from the database. If the +** attempt to reload content from the database is required and fails, +** return an SQLite error code. Otherwise, SQLITE_OK. +*/ +static int pagerUndoCallback(void *pCtx, Pgno iPg){ + int rc = SQLITE_OK; + Pager *pPager = (Pager *)pCtx; + PgHdr *pPg; + + pPg = sqlite3PagerLookup(pPager, iPg); + if( pPg ){ + if( sqlite3PcachePageRefcount(pPg)==1 ){ + sqlite3PcacheDrop(pPg); + }else{ + rc = readDbPage(pPg); + if( rc==SQLITE_OK ){ + pPager->xReiniter(pPg); + } + sqlite3PagerUnref(pPg); + } + } + + return rc; +} + +/* +** This function is called to rollback a transaction on a WAL database. */ static int pagerRollbackLog(Pager *pPager){ - int rc = SQLITE_OK; - PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); + int rc; /* Return Code */ + PgHdr *pList; /* List of dirty pages to revert */ /* Normally, if a transaction is rolled back, any backup processes are ** updated as data is copied out of the rollback journal and into the @@ -2258,20 +2286,22 @@ static int pagerRollbackLog(Pager *pPager){ sqlite3BackupRestart(pPager->pBackup); } + /* For all pages in the cache that are currently dirty or have already + ** been written (but not committed) to the log file, do one of the + ** following: + ** + ** + Discard the cached page (if refcount==0), or + ** + Reload page content from the database (if refcount>0). + */ pPager->dbSize = pPager->dbOrigSize; + rc = sqlite3LogUndo(pPager->pLog, pagerUndoCallback, (void *)pPager); + pList = sqlite3PcacheDirtyList(pPager->pPCache); while( pList && rc==SQLITE_OK ){ PgHdr *pNext = pList->pDirty; - if( sqlite3PcachePageRefcount(pList)==0 ){ - sqlite3PagerLookup(pPager, pList->pgno); - sqlite3PcacheDrop(pList); - }else{ - rc = readDbPage(pList); - if( rc==SQLITE_OK ){ - pPager->xReiniter(pList); - } - } + rc = pagerUndoCallback((void *)pPager, pList->pgno); pList = pNext; } + return rc; } diff --git a/test/wal.test b/test/wal.test index f69c94020b..d1a609ae56 100644 --- a/test/wal.test +++ b/test/wal.test @@ -139,6 +139,10 @@ do_test wal-3.3 { } {1 2 3 4 5 6 7 8 9 10 11 12 13 14} db2 close +#------------------------------------------------------------------------- +# The following tests, wal-4.*, test that savepoints work with WAL +# databases. +# do_test wal-4.1 { execsql { DELETE FROM t1; @@ -162,6 +166,57 @@ do_test wal-4.3 { } } {a b} +do_test wal-4.4 { + db close + sqlite3 db test.db + db func blob blob + list [execsql { SELECT * FROM t1 }] [file size test.db-wal] +} {{a b} 0} +do_test wal-4.5 { + execsql { PRAGMA cache_size = 10 } + execsql { + CREATE TABLE t2(a, b); + INSERT INTO t2 VALUES(blob(400), blob(400)); + SAVEPOINT tr; + INSERT INTO t2 SELECT blob(400), blob(400) FROM t2; /* 2 */ + INSERT INTO t2 SELECT blob(400), blob(400) FROM t2; /* 4 */ + INSERT INTO t2 SELECT blob(400), blob(400) FROM t2; /* 8 */ + INSERT INTO t2 SELECT blob(400), blob(400) FROM t2; /* 16 */ + INSERT INTO t2 SELECT blob(400), blob(400) FROM t2; /* 32 */ + INSERT INTO t1 SELECT blob(400), blob(400) FROM t1; /* 2 */ + INSERT INTO t1 SELECT blob(400), blob(400) FROM t1; /* 4 */ + INSERT INTO t1 SELECT blob(400), blob(400) FROM t1; /* 8 */ + INSERT INTO t1 SELECT blob(400), blob(400) FROM t1; /* 16 */ + INSERT INTO t1 SELECT blob(400), blob(400) FROM t1; /* 32 */ + SELECT count(*) FROM t2; + } +} {32} +do_test wal-4.6 { + execsql { ROLLBACK TO tr } +} {} +do_test wal-4.7 { + set logsize [file size test.db-wal] + execsql { + INSERT INTO t1 VALUES('x', 'y'); + RELEASE tr; + } + expr { $logsize == [file size test.db-wal] } +} {1} +do_test wal-4.8 { + execsql { SELECT count(*) FROM t2 } +} {1} +do_test wal-4.9 { + file copy -force test.db test2.db + file copy -force test.db-wal test2.db-wal + sqlite3 db2 test2.db + execsql { SELECT count(*) FROM t2 ; SELECT count(*) FROM t1 } db2 +} {1 2} +do_test wal-4.10 { + execsql { PRAGMA integrity_check } db2 +} {ok} +db2 close + +reopen_db do_test wal-5.1 { execsql { CREATE TEMP TABLE t2(a, b); From 56d95913ebec66dd9d192905f1d08159ae68e32b Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 24 Apr 2010 19:07:29 +0000 Subject: [PATCH 34/89] Add comment explaining checksum mechanism. FossilOrigin-Name: 3e9ef5153ebf0543ad0f75a7561f73d22171da53 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/log.c | 17 ++++++++++++++--- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index d67960a052..ed63519a6f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sbugs\sin\sWAL\smode\srollback. -D 2010-04-24T18:44:05 +C Add\scomment\sexplaining\schecksum\smechanism. +D 2010-04-24T19:07:29 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c 6097555f746e9992b3bb7a257d2e60ced69e3a84 +F src/log.c 538da9c42f6da426e82b114f37c27cf730fcba34 F src/log.h 06eacedac8b5a720cb88aa3c9e5d886ba7b394fe F src/main.c 867de6aa444abd97771b2b70472f448d65c1c77e F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P a352f6285e33a806fbe4475e720e763fdc5bb47d -R a372a014eac56d1df8e79bcff4ecc6e7 +P 31215969f59be536fe87431bb9fbfa7d13027e35 +R 1d48ccc5eb3586f62961518732858aa1 U dan -Z 3d8f3676c5fc9fd427fe6aa0aa15e428 +Z d82402bdae4dea648cb0a1aad66c722d diff --git a/manifest.uuid b/manifest.uuid index b8874609f2..aa217ce025 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -31215969f59be536fe87431bb9fbfa7d13027e35 \ No newline at end of file +3e9ef5153ebf0543ad0f75a7561f73d22171da53 \ No newline at end of file diff --git a/src/log.c b/src/log.c index d9637de024..c0357e87de 100644 --- a/src/log.c +++ b/src/log.c @@ -316,15 +316,26 @@ static LogSummary *pLogSummary = 0; ** Generate an 8 byte checksum based on the data in array aByte[] and the ** initial values of aCksum[0] and aCksum[1]. The checksum is written into ** aCksum[] before returning. +** +** The range of bytes to checksum is treated as an array of 32-bit +** little-endian unsigned integers. For each integer X in the array, from +** start to finish, do the following: +** +** aCksum[0] += X; +** aCksum[1] += aCksum[0]; +** +** For the calculation above, use 64-bit unsigned accumulators. Before +** returning, truncate the values to 32-bits as follows: +** +** aCksum[0] = (u32)(aCksum[0] + (aCksum[0]>>24)); +** aCksum[1] = (u32)(aCksum[1] + (aCksum[1]>>24)); */ -#define LOG_CKSM_BYTES 8 static void logChecksumBytes(u8 *aByte, int nByte, u32 *aCksum){ u64 sum1 = aCksum[0]; u64 sum2 = aCksum[1]; u32 *a32 = (u32 *)aByte; u32 *aEnd = (u32 *)&aByte[nByte]; - assert( LOG_CKSM_BYTES==2*sizeof(u32) ); assert( (nByte&0x00000003)==0 ); if( SQLITE_LITTLEENDIAN ){ @@ -1666,7 +1677,7 @@ int sqlite3LogFrames( PgHdr *pLast; /* Last frame in list */ int nLast = 0; /* Number of extra copies of last page */ - assert( LOG_FRAME_HDRSIZE==(4 * 2 + LOG_CKSM_BYTES) ); + assert( LOG_FRAME_HDRSIZE==(4 * 2 + 2*sizeof(u32)) ); assert( pList ); /* If this is the first frame written into the log, write the log From f2424c521bea90fe4c333690a1a7d71554e864fd Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 26 Apr 2010 00:04:55 +0000 Subject: [PATCH 35/89] Begin moving WAL-specific I/O into the VFS. This checkin contains VFS infrastructure but it is untested and is not yet hooked up to the WAL. The version number is bumped to 3.7.0 because of the VFS extension. FossilOrigin-Name: f5e615c28c7035a7e6d896790b51cf9bc7371d5f --- VERSION | 2 +- manifest | 44 +++++++----- manifest.uuid | 2 +- src/os_os2.c | 9 +++ src/os_unix.c | 166 ++++++++++++++++++++++++++++++++++++++++++++- src/os_win.c | 9 +++ src/sqlite.h.in | 39 ++++++++++- src/test6.c | 11 ++- src/test_demovfs.c | 11 ++- src/test_devsym.c | 11 ++- src/test_journal.c | 11 ++- src/test_onefile.c | 11 ++- src/test_osinst.c | 11 ++- 13 files changed, 307 insertions(+), 30 deletions(-) diff --git a/VERSION b/VERSION index 2d6744e051..7c69a55dbb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.6.23 +3.7.0 diff --git a/manifest b/manifest index ed63519a6f..49edfccca3 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,14 @@ -C Add\scomment\sexplaining\schecksum\smechanism. -D 2010-04-24T19:07:29 +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 + +C Begin\smoving\sWAL-specific\sI/O\sinto\sthe\sVFS.\s\sThis\scheckin\scontains\sVFS\ninfrastructure\sbut\sit\sis\suntested\sand\sis\snot\syet\shooked\sup\sto\sthe\sWAL.\nThe\sversion\snumber\sis\sbumped\sto\s3.7.0\sbecause\sof\sthe\sVFS\sextension. +D 2010-04-26T00:04:55 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 F Makefile.vxworks ab005d301296c40e021ccd0133ce49ca811e319f F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 -F VERSION 3e18b3dd7290883913c25d66c0ddb409ce318654 +F VERSION 4dce4379514b12d6bc5c30f7d1f64582ccb4f412 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 17dc593f791f874d2c23a0f9360850ded0286531 F art/2005osaward.gif 0d1851b2a7c1c9d0ccce545f3e14bca42d7fd248 @@ -151,9 +154,9 @@ F src/notify.c cbfa66a836da3a51567209636e6a94059c137930 F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 -F src/os_os2.c 75a8c7b9a00a2cf1a65f9fa4afbc27d46634bb2f -F src/os_unix.c 5bf0015cebe2f21635da2af983c348eb88b3b4c1 -F src/os_win.c 1c7453c2df4dab26d90ff6f91272aea18bcf7053 +F src/os_os2.c 082884dc2a20a2f1fddc404f3606ec2abe907b56 +F src/os_unix.c 1acc854259b045cf029c6f95828a5f4b84f361c7 +F src/os_win.c 607b40e40662c6c74ff2400ff03e7f42521d5c75 F src/pager.c 9da40bb50d2baa8f7cbb18579f147b7623856211 F src/pager.h cee4487ab4f0911dd9f22a40e3cd55afdb7ef444 F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e @@ -168,7 +171,7 @@ F src/resolve.c ac5f1a713cd1ae77f08b83cc69581e11bf5ae6f9 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c b7c9a40bc1567bceff52ad4b73108734ee4bf268 F src/shell.c c40427c7245535a04a9cb4a417b6cc05c022e6a4 -F src/sqlite.h.in b9b0365613d4a99a0e6bc0ae6dbc73cc4a7e7cb7 +F src/sqlite.h.in 96b0bed7825da354119a9742a83033087c537133 F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89 F src/sqliteInt.h de668d16e73ef456b940ab3f60b2ac8842957853 F src/sqliteLimit.h 3afab2291762b5d09ae20c18feb8e9fa935a60a6 @@ -180,7 +183,7 @@ F src/test2.c b6b43413d495addd039a88b87d65c839f86b18cb F src/test3.c 4c21700c73a890a47fc685c1097bfb661346ac94 F src/test4.c ad03bb987ddedce928f4258c1e7fa4109a73497d F src/test5.c cc55900118fa4add8ec9cf69fc4225a4662f76b1 -F src/test6.c a8ece4284d0e34477f349ac05655db73c48e0926 +F src/test6.c d23fc84954751735bcb1050d4606b00952eb7809 F src/test7.c 3f2d63e4ccf97f8c2cf1a7fa0a3c8e2e2a354e6e F src/test8.c f959db9a22d882013b64c92753fa793b2ce3bdea F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 @@ -189,19 +192,19 @@ F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de F src/test_btree.c 47cd771250f09cdc6e12dda5bc71bc0b3abc96e2 F src/test_config.c 8c408fbffbe8082d1d3dc55044ddfd5580f3c9b9 -F src/test_demovfs.c 301a047287021940136a3a52d2366dc51f8072d5 -F src/test_devsym.c 3adf69c716e5c90bf5b7a1afe817e3ecac1adf6f +F src/test_demovfs.c 68f8e6cfd6b337612e26b7027f97bf7ea02d4e04 +F src/test_devsym.c 76f47435769168fb665828f5cc24a4a0a130c302 F src/test_func.c 13b582345fb1185a93e46c53310fae8547dcce20 F src/test_hexio.c 1237f000ec7a491009b1233f5c626ea71bce1ea2 F src/test_init.c 5d624ffd0409d424cf9adbfe1f056b200270077c F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 -F src/test_journal.c adc0ce3840ed19b49feb1d583b2212f560ef7866 +F src/test_journal.c 6f2db1b847929791242e32323721c48c7d2d6e0c F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 2842c922b8e8d992aba722214952204ca025b411 F src/test_mutex.c ce06b59aca168cd8c520b77159a24352a7469bd3 -F src/test_onefile.c 06da7e085dce42924cf062b91763dd4bb84c6101 -F src/test_osinst.c 90fb03d396f39956897dfb4bd0e62c6711db1cca +F src/test_onefile.c 1d098e7446d89087def7d20e0cdb77a0713ed1bf +F src/test_osinst.c 79a9b9fdb87fccd7fdc2f523bfbb91f964bafa68 F src/test_pcache.c 7bf828972ac0d2403f5cfa4cd14da41f8ebe73d8 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c bbba05c144b5fc4b52ff650a4328027b3fa5fcc6 @@ -808,7 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 31215969f59be536fe87431bb9fbfa7d13027e35 -R 1d48ccc5eb3586f62961518732858aa1 -U dan -Z d82402bdae4dea648cb0a1aad66c722d +P 3e9ef5153ebf0543ad0f75a7561f73d22171da53 +R d0d00cdfe34b5188edd49187f65dd4f0 +U drh +Z a55984eccf1af95745f2d144e70469ac +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.6 (GNU/Linux) + +iD8DBQFL1NiqoxKgR168RlERAkSGAKCMwxXVaj6pJFUojxgq8jkWD5q0bgCfXZsd +BjC57GjZtQKSf+QHQLj1hP0= +=LyL/ +-----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index aa217ce025..019749c7c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3e9ef5153ebf0543ad0f75a7561f73d22171da53 \ No newline at end of file +f5e615c28c7035a7e6d896790b51cf9bc7371d5f \ No newline at end of file diff --git a/src/os_os2.c b/src/os_os2.c index 572b6a3c44..04b6dce0e9 100644 --- a/src/os_os2.c +++ b/src/os_os2.c @@ -1112,6 +1112,15 @@ int sqlite3_os_init(void){ os2Sleep, /* xSleep */ os2CurrentTime, /* xCurrentTime */ os2GetLastError /* xGetLastError */ + 0, /* xShmOpen */ + 0, /* xShmSize */ + 0, /* xShmPush */ + 0, /* xShmPull */ + 0, /* xShmLock */ + 0, /* xShmClose */ + 0, /* xShmDelete */ + 0, /* xRename */ + 0, /* xCurrentTimeInt64 */ }; sqlite3_vfs_register(&os2Vfs, 1); initUconvObjects(); diff --git a/src/os_unix.c b/src/os_unix.c index 80ce9e0b00..b2cf9be76d 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -119,6 +119,7 @@ #include #include #include +#include #if SQLITE_ENABLE_LOCKING_STYLE # include @@ -4561,6 +4562,158 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ return 0; } +/* +** Structure used internally by this VFS to record the state of an +** open shared memory segment. +*/ +struct unixShm { + sqlite3_vfs *pVfs; /* VFS that opened this shared-memory segment */ + int size; /* Size of the shared memory area */ + char *pBuf; /* Pointer to the beginning */ + unixFile fd; /* The open file descriptor */ +}; + +/* +** Close a shared-memory segment +*/ +static int unixShmClose(sqlite3_shm *pSharedMem){ + struct unixShm *p = (struct unixShm*)pSharedMem; + if( p && p->pVfs ){ + if( p->pBuf ){ + munmap(p->pBuf, p->size); + } + if( p->fd.pMethod ){ + p->fd.pMethod->xClose((sqlite3_file*)&p->fd); + } + memset(p, 0, sizeof(*p)); + sqlite3_free(p); + } + return SQLITE_OK; +} + +/* +** Size increment by which shared memory grows +*/ +#define SQLITE_UNIX_SHM_INCR 4096 + +/* +** Open a shared-memory area. This implementation uses mmapped files. +*/ +static int unixShmOpen( + sqlite3_vfs *pVfs, /* The VFS */ + const char *zName, /* Name of file to mmap */ + sqlite3_shm **pShm /* Write the unixShm object created here */ +){ + struct unixShm *p = 0; + int rc; + int outFlags; + struct stat sStat; + + p = sqlite3_malloc( sizeof(*p) ); + *pShm = (sqlite3_shm*)p; + if( p==0 ) return SQLITE_NOMEM; + memset(p, 0, sizeof(*p)); + p->pVfs = pVfs; + rc = pVfs->xOpen(pVfs, zName, (sqlite3_file*)&p->fd, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_MAIN_JOURNAL, + &outFlags); + if( rc!=SQLITE_OK ) goto shm_open_err; + + rc = fstat(p->fd.h, &sStat); + if( rc!=0 ) goto shm_open_err; + if( sStat.st_sizefd.h, SQLITE_UNIX_SHM_INCR); + if( rc!=0 ) goto shm_open_err; + p->size = SQLITE_UNIX_SHM_INCR; + }else{ + p->size = sStat.st_size; + } + + /* Map the file. */ + p->pBuf = mmap(0, p->size, PROT_READ|PROT_WRITE, MAP_SHARED, p->fd.h, 0); + if( p->pBuf==MAP_FAILED ){ + rc = SQLITE_IOERR; + goto shm_open_err; + } + return SQLITE_OK; + +shm_open_err: + unixShmClose((sqlite3_shm*)p); + *pShm = 0; + return rc; +} + +/* +** Query and/or changes the size of a shared-memory segment. +** The reqSize parameter is the new size of the segment, or -1 to +** do just a query. The size of the segment after resizing is +** written into pNewSize. The start of the shared memory buffer +** is stored in **ppBuffer. +*/ +static int unixShmSize( + sqlite3_shm *pSharedMem, /* Pointer returned by unixShmOpen() */ + int reqSize, /* Requested size. -1 for query only */ + int *pNewSize, /* Write new size here */ + char **ppBuf /* Write new buffer origin here */ +){ + struct unixShm *p = (struct unixShm*)pSharedMem; + int rc = SQLITE_OK; + + if( reqSize>=0 ){ + reqSize = (reqSize + SQLITE_UNIX_SHM_INCR - 1)/SQLITE_UNIX_SHM_INCR; + reqSize *= SQLITE_UNIX_SHM_INCR; + if( reqSize!=p->size ){ + munmap(p->pBuf, p->size); + rc = ftruncate(p->fd.h, reqSize); + if( rc ){ + p->pBuf = 0; + p->size = 0; + }else{ + p->pBuf = mmap(0, reqSize, PROT_READ|PROT_WRITE, MAP_SHARED, p->fd.h,0); + p->size = p->pBuf ? reqSize : 0; + } + } + } + *pNewSize = p->size; + *ppBuf = p->pBuf; + return rc; +} + +/* +** Create or release a lock on shared memory. +*/ +static int unixShmLock( + sqlite3_shm *pSharedMem, /* Pointer from unixShmOpen() */ + int lockType, /* _RDLK, _WRLK, or _UNLK, possibly ORed _BLOCK */ + int ofst, /* Start of lock region */ + int nByte /* Size of lock region in bytes */ +){ + struct unixShm *p = (struct unixShm*)pSharedMem; + struct flock f; + int op; + int rc; + + f.l_whence = SEEK_SET; + f.l_start = ofst; + f.l_len = nByte; + switch( lockType & 0x07 ){ + case SQLITE_SHM_RDLK: f.l_type = F_RDLCK; break; + case SQLITE_SHM_WRLK: f.l_type = F_WRLCK; break; + case SQLITE_SHM_UNLK: f.l_type = F_UNLCK; break; + } + op = (lockType & 0x08)!=0 ? F_SETLKW : F_SETLK; + rc = fcntl(p->fd.h, op, &f); + return (rc==0) ? SQLITE_OK : SQLITE_BUSY; +} + +/* +** Delete a shared-memory segment from the system. +*/ +static int unixShmDelete(sqlite3_vfs *pVfs, const char *zName){ + return pVfs->xDelete(pVfs, zName, 0); +} + + /* ************************ End of sqlite3_vfs methods *************************** ******************************************************************************/ @@ -5761,7 +5914,7 @@ int sqlite3_os_init(void){ ** that filesystem time. */ #define UNIXVFS(VFSNAME, FINDER) { \ - 1, /* iVersion */ \ + 2, /* iVersion */ \ sizeof(unixFile), /* szOsFile */ \ MAX_PATHNAME, /* mxPathname */ \ 0, /* pNext */ \ @@ -5778,7 +5931,16 @@ int sqlite3_os_init(void){ unixRandomness, /* xRandomness */ \ unixSleep, /* xSleep */ \ unixCurrentTime, /* xCurrentTime */ \ - unixGetLastError /* xGetLastError */ \ + unixGetLastError, /* xGetLastError */ \ + unixShmOpen, /* xShmOpen */ \ + unixShmSize, /* xShmSize */ \ + 0, /* xShmPush */ \ + 0, /* xShmPull */ \ + unixShmLock, /* xShmLock */ \ + unixShmClose, /* xShmClose */ \ + unixShmDelete, /* xShmDelete */ \ + 0, /* xRename */ \ + 0, /* xCurrentTimeInt64 */ \ } /* diff --git a/src/os_win.c b/src/os_win.c index 4721eadff8..31c3cffce3 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1913,6 +1913,15 @@ int sqlite3_os_init(void){ winSleep, /* xSleep */ winCurrentTime, /* xCurrentTime */ winGetLastError /* xGetLastError */ + 0, /* xShmOpen */ + 0, /* xShmSize */ + 0, /* xShmPush */ + 0, /* xShmPull */ + 0, /* xShmLock */ + 0, /* xShmClose */ + 0, /* xShmDelete */ + 0, /* xRename */ + 0, /* xCurrentTimeInt64 */ }; sqlite3_vfs_register(&winVfs, 1); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 97c97aab09..f673781e00 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -818,8 +818,9 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** */ typedef struct sqlite3_vfs sqlite3_vfs; +typedef struct sqlite3_shm sqlite3_shm; struct sqlite3_vfs { - int iVersion; /* Structure version number */ + int iVersion; /* Structure version number (currently 2) */ int szOsFile; /* Size of subclassed sqlite3_file */ int mxPathname; /* Maximum file pathname length */ sqlite3_vfs *pNext; /* Next registered VFS */ @@ -838,8 +839,24 @@ struct sqlite3_vfs { int (*xSleep)(sqlite3_vfs*, int microseconds); int (*xCurrentTime)(sqlite3_vfs*, double*); int (*xGetLastError)(sqlite3_vfs*, int, char *); - /* New fields may be appended in figure versions. The iVersion - ** value will increment whenever this happens. */ + /* + ** The methods above are in version 1 of the sqlite_vfs object + ** definition. Those that follow are added in version 2 or later + */ + int (*xShmOpen)(sqlite3_vfs*, const char *zName, sqlite3_shm**); + int (*xShmSize)(sqlite3_shm*, int reqSize, int *pNewSize, char **); + int (*xShmPush)(sqlite3_shm*); + int (*xShmPull)(sqlite3_shm*); + int (*xShmLock)(sqlite3_shm*, int lockType, int ofst, int nByte); + int (*xShmClose)(sqlite3_shm*); + int (*xShmDelete)(sqlite3_vfs*, const char *zName); + int (*xRename)(sqlite3_vfs*, const char *zOld, const char *zNew, int dirSync); + int (*xCurrentTimeInt64)(sqlite3_vfs*, sqlite3_int64*); + /* + ** The methods above are in versions 1 and 2 of the sqlite_vfs object. + ** New fields may be appended in figure versions. The iVersion + ** value will increment whenever this happens. + */ }; /* @@ -859,6 +876,22 @@ struct sqlite3_vfs { #define SQLITE_ACCESS_READWRITE 1 #define SQLITE_ACCESS_READ 2 +/* +** CAPI3REF: Flags for the xShmLock VFS method +** +** These integer constants can be used as the second parameter to +** the xShmLock method of an [sqlite3_vfs] object. They determine +** the specific locking action. Exactly one of the first three +** values must be used ini the lockType parameter. The fourth +** value (SQLITE_SHM_BLOCK) can optionally be ORed into the lockType +** parameter to cause the thread to block until the lock becomes +** available. +*/ +#define SQLITE_SHM_RDLK 0x01 +#define SQLITE_SHM_WRLK 0x02 +#define SQLITE_SHM_UNLK 0x04 +#define SQLITE_SHM_BLOCK 0x08 + /* ** CAPI3REF: Initialize The SQLite Library ** diff --git a/src/test6.c b/src/test6.c index b6fa3aadf0..a6e9954517 100644 --- a/src/test6.c +++ b/src/test6.c @@ -781,7 +781,16 @@ static int crashEnableCmd( cfDlClose, /* xDlClose */ cfRandomness, /* xRandomness */ cfSleep, /* xSleep */ - cfCurrentTime /* xCurrentTime */ + cfCurrentTime, /* xCurrentTime */ + 0, /* xShmOpen */ + 0, /* xShmSize */ + 0, /* xShmPush */ + 0, /* xShmPull */ + 0, /* xShmLock */ + 0, /* xShmClose */ + 0, /* xShmDelete */ + 0, /* xRename */ + 0 /* xCurrentTimeInt64 */ }; if( objc!=2 ){ diff --git a/src/test_demovfs.c b/src/test_demovfs.c index 1039d0222c..d91a389dbb 100644 --- a/src/test_demovfs.c +++ b/src/test_demovfs.c @@ -623,7 +623,16 @@ sqlite3_vfs *sqlite3_demovfs(void){ demoDlClose, /* xDlClose */ demoRandomness, /* xRandomness */ demoSleep, /* xSleep */ - demoCurrentTime /* xCurrentTime */ + demoCurrentTime, /* xCurrentTime */ + 0, /* xShmOpen */ + 0, /* xShmSize */ + 0, /* xShmPush */ + 0, /* xShmPull */ + 0, /* xShmLock */ + 0, /* xShmClose */ + 0, /* xShmDelete */ + 0, /* xRename */ + 0 /* xCurrentTimeInt64 */ }; return &demovfs; } diff --git a/src/test_devsym.c b/src/test_devsym.c index 9617e63af3..4249573aa1 100644 --- a/src/test_devsym.c +++ b/src/test_devsym.c @@ -92,7 +92,16 @@ static sqlite3_vfs devsym_vfs = { #endif /* SQLITE_OMIT_LOAD_EXTENSION */ devsymRandomness, /* xRandomness */ devsymSleep, /* xSleep */ - devsymCurrentTime /* xCurrentTime */ + devsymCurrentTime, /* xCurrentTime */ + 0, /* xShmOpen */ + 0, /* xShmSize */ + 0, /* xShmPush */ + 0, /* xShmPull */ + 0, /* xShmLock */ + 0, /* xShmClose */ + 0, /* xShmDelete */ + 0, /* xRename */ + 0 /* xCurrentTimeInt64 */ }; static sqlite3_io_methods devsym_io_methods = { diff --git a/src/test_journal.c b/src/test_journal.c index f89f3a4c96..fcfe778d1b 100644 --- a/src/test_journal.c +++ b/src/test_journal.c @@ -179,7 +179,16 @@ static sqlite3_vfs jt_vfs = { jtDlClose, /* xDlClose */ jtRandomness, /* xRandomness */ jtSleep, /* xSleep */ - jtCurrentTime /* xCurrentTime */ + jtCurrentTime, /* xCurrentTime */ + 0, /* xShmOpen */ + 0, /* xShmSize */ + 0, /* xShmPush */ + 0, /* xShmPull */ + 0, /* xShmLock */ + 0, /* xShmClose */ + 0, /* xShmDelete */ + 0, /* xRename */ + 0 /* xCurrentTimeInt64 */ }; static sqlite3_io_methods jt_io_methods = { diff --git a/src/test_onefile.c b/src/test_onefile.c index 818bd1775e..fe02e016e7 100644 --- a/src/test_onefile.c +++ b/src/test_onefile.c @@ -198,7 +198,16 @@ static fs_vfs_t fs_vfs = { fsDlClose, /* xDlClose */ fsRandomness, /* xRandomness */ fsSleep, /* xSleep */ - fsCurrentTime /* xCurrentTime */ + fsCurrentTime, /* xCurrentTime */ + 0, /* xShmOpen */ + 0, /* xShmSize */ + 0, /* xShmPush */ + 0, /* xShmPull */ + 0, /* xShmLock */ + 0, /* xShmClose */ + 0, /* xShmDelete */ + 0, /* xRename */ + 0 /* xCurrentTimeInt64 */ }, 0, /* pFileList */ 0 /* pParent */ diff --git a/src/test_osinst.c b/src/test_osinst.c index 283afe60e4..c6d16aa67b 100644 --- a/src/test_osinst.c +++ b/src/test_osinst.c @@ -203,7 +203,16 @@ static sqlite3_vfs inst_vfs = { instDlClose, /* xDlClose */ instRandomness, /* xRandomness */ instSleep, /* xSleep */ - instCurrentTime /* xCurrentTime */ + instCurrentTime, /* xCurrentTime */ + 0, /* xShmOpen */ + 0, /* xShmSize */ + 0, /* xShmPush */ + 0, /* xShmPull */ + 0, /* xShmLock */ + 0, /* xShmClose */ + 0, /* xShmDelete */ + 0, /* xRename */ + 0 /* xCurrentTimeInt64 */ }; static sqlite3_io_methods inst_io_methods = { From c438efd68a2bcbe79ecf159b561b67c57fa5202c Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 26 Apr 2010 00:19:45 +0000 Subject: [PATCH 36/89] Change the names of the log.c and log.h source files to wal.c and wal.h. FossilOrigin-Name: 56fe5d7624f840417152bcc63efbe21a5f557920 --- Makefile.in | 5 +++++ main.mk | 10 +++++----- manifest | 28 ++++++++++++++-------------- manifest.uuid | 2 +- src/pager.c | 34 +++++++++++++++++----------------- src/{log.c => wal.c} | 29 ++++++++++++++--------------- src/{log.h => wal.h} | 30 +++++++++++++++--------------- tool/mksqlite3c.tcl | 4 ++-- 8 files changed, 73 insertions(+), 69 deletions(-) rename src/{log.c => wal.c} (99%) rename src/{log.h => wal.h} (72%) diff --git a/Makefile.in b/Makefile.in index e954ccb7d5..a1387bda42 100644 --- a/Makefile.in +++ b/Makefile.in @@ -276,6 +276,8 @@ SRC = \ $(TOP)/src/vdbetrace.c \ $(TOP)/src/vdbeInt.h \ $(TOP)/src/vtab.c \ + $(TOP)/src/wal.c \ + $(TOP)/src/wal.h \ $(TOP)/src/walker.c \ $(TOP)/src/where.c @@ -742,6 +744,9 @@ vdbetrace.lo: $(TOP)/src/vdbetrace.c $(HDR) vtab.lo: $(TOP)/src/vtab.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/vtab.c +wal.lo: $(TOP)/src/wal.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/wal.c + walker.lo: $(TOP)/src/walker.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/walker.c diff --git a/main.mk b/main.mk index 89f1c6ada9..353a3cef6b 100644 --- a/main.mk +++ b/main.mk @@ -56,7 +56,7 @@ LIBOBJ+= alter.o analyze.o attach.o auth.o \ fts3.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o fts3_write.o \ func.o global.o hash.o \ - icu.o insert.o journal.o legacy.o loadext.o log.o \ + icu.o insert.o journal.o legacy.o loadext.o \ main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \ memjournal.o \ mutex.o mutex_noop.o mutex_os2.o mutex_unix.o mutex_w32.o \ @@ -66,7 +66,7 @@ LIBOBJ+= alter.o analyze.o attach.o auth.o \ table.o tokenize.o trigger.o \ update.o util.o vacuum.o \ vdbe.o vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o vdbetrace.o \ - walker.o where.o utf.o vtab.o + wal.o walker.o where.o utf.o vtab.o @@ -101,8 +101,6 @@ SRC = \ $(TOP)/src/journal.c \ $(TOP)/src/legacy.c \ $(TOP)/src/loadext.c \ - $(TOP)/src/log.c \ - $(TOP)/src/log.h \ $(TOP)/src/main.c \ $(TOP)/src/malloc.c \ $(TOP)/src/mem0.c \ @@ -160,6 +158,8 @@ SRC = \ $(TOP)/src/vdbetrace.c \ $(TOP)/src/vdbeInt.h \ $(TOP)/src/vtab.c \ + $(TOP)/src/wal.c \ + $(TOP)/src/wal.h \ $(TOP)/src/walker.c \ $(TOP)/src/where.c @@ -257,7 +257,7 @@ TESTSRC = \ TESTSRC2 = \ $(TOP)/src/attach.c $(TOP)/src/backup.c $(TOP)/src/btree.c \ $(TOP)/src/build.c $(TOP)/src/date.c \ - $(TOP)/src/expr.c $(TOP)/src/func.c $(TOP)/src/insert.c $(TOP)/src/log.c \ + $(TOP)/src/expr.c $(TOP)/src/func.c $(TOP)/src/insert.c $(TOP)/src/wal.c \ $(TOP)/src/mem5.c $(TOP)/src/os.c \ $(TOP)/src/os_os2.c $(TOP)/src/os_unix.c $(TOP)/src/os_win.c \ $(TOP)/src/pager.c $(TOP)/src/pragma.c $(TOP)/src/prepare.c \ diff --git a/manifest b/manifest index 49edfccca3..a3bb80a173 100644 --- a/manifest +++ b/manifest @@ -1,10 +1,10 @@ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 -C Begin\smoving\sWAL-specific\sI/O\sinto\sthe\sVFS.\s\sThis\scheckin\scontains\sVFS\ninfrastructure\sbut\sit\sis\suntested\sand\sis\snot\syet\shooked\sup\sto\sthe\sWAL.\nThe\sversion\snumber\sis\sbumped\sto\s3.7.0\sbecause\sof\sthe\sVFS\sextension. -D 2010-04-26T00:04:55 +C Change\sthe\snames\sof\sthe\slog.c\sand\slog.h\ssource\sfiles\sto\swal.c\sand\swal.h. +D 2010-04-26T00:19:45 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 -F Makefile.in 4f2f967b7e58a35bb74fb7ec8ae90e0f4ca7868b +F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 F Makefile.vxworks ab005d301296c40e021ccd0133ce49ca811e319f F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 @@ -92,7 +92,7 @@ F ext/rtree/tkt3363.test 2bf324f7908084a5f463de3109db9c6e607feb1b F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk f12991ace528dd01d018420988ff053350ae81f8 +F main.mk b681194d6bf557b80ca363306e9c172f81f15b1d F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -134,8 +134,6 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/log.c 538da9c42f6da426e82b114f37c27cf730fcba34 -F src/log.h 06eacedac8b5a720cb88aa3c9e5d886ba7b394fe F src/main.c 867de6aa444abd97771b2b70472f448d65c1c77e F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -157,7 +155,7 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 082884dc2a20a2f1fddc404f3606ec2abe907b56 F src/os_unix.c 1acc854259b045cf029c6f95828a5f4b84f361c7 F src/os_win.c 607b40e40662c6c74ff2400ff03e7f42521d5c75 -F src/pager.c 9da40bb50d2baa8f7cbb18579f147b7623856211 +F src/pager.c 5924538fb99f62774419bfae882debb0d65bc3c8 F src/pager.h cee4487ab4f0911dd9f22a40e3cd55afdb7ef444 F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf @@ -226,6 +224,8 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda +F src/wal.c df5283ae70cce52852fd788479bee22994fccbe5 w src/log.c +F src/wal.h 1a34bb71642b46af5e6d51c2f69ae0f55268df81 w src/log.h F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -791,7 +791,7 @@ F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/mkkeywordhash.c d2e6b4a5965e23afb80fbe74bb54648cd371f309 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 -F tool/mksqlite3c.tcl 25ec827588893857eba2d24a645ace1bb7cdab73 +F tool/mksqlite3c.tcl 42b6ba8b997a5f5a98f7f1df946c21c173eef586 F tool/mksqlite3h.tcl eb100dce83f24b501b325b340f8b5eb8e5106b3b F tool/mksqlite3internalh.tcl 7b43894e21bcb1bb39e11547ce7e38a063357e87 F tool/omittest.tcl 27d6f6e3b1e95aeb26a1c140e6eb57771c6d794a @@ -811,14 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 3e9ef5153ebf0543ad0f75a7561f73d22171da53 -R d0d00cdfe34b5188edd49187f65dd4f0 +P f5e615c28c7035a7e6d896790b51cf9bc7371d5f +R 993bceae438ad1d8ab0904d8daf11104 U drh -Z a55984eccf1af95745f2d144e70469ac +Z e29cadabf74dbcaa3b3703c56732da89 -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.6 (GNU/Linux) -iD8DBQFL1NiqoxKgR168RlERAkSGAKCMwxXVaj6pJFUojxgq8jkWD5q0bgCfXZsd -BjC57GjZtQKSf+QHQLj1hP0= -=LyL/ +iD8DBQFL1NwloxKgR168RlERAks8AJ9QVBJiBku5alys49Firj8NBRPvlwCfSHT7 +JCRELzIJ9CW5GkQjtcejMJU= +=4Ljy -----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index 019749c7c1..e99c944438 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f5e615c28c7035a7e6d896790b51cf9bc7371d5f \ No newline at end of file +56fe5d7624f840417152bcc63efbe21a5f557920 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 18b8960da7..171b584573 100644 --- a/src/pager.c +++ b/src/pager.c @@ -20,7 +20,7 @@ */ #ifndef SQLITE_OMIT_DISKIO #include "sqliteInt.h" -#include "log.h" +#include "wal.h" /* ******************** NOTES ON THE DESIGN OF THE PAGER ************************ @@ -1227,7 +1227,7 @@ static void pager_unlock(Pager *pPager){ pPager->dbSizeValid = 0; if( pagerUseLog(pPager) ){ - sqlite3LogCloseSnapshot(pPager->pLog); + sqlite3WalCloseSnapshot(pPager->pLog); }else{ rc = osUnlock(pPager->fd, NO_LOCK); } @@ -1426,7 +1426,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ sqlite3PcacheCleanAll(pPager->pPCache); if( pagerUseLog(pPager) ){ - rc2 = sqlite3LogWriteLock(pPager->pLog, 0); + rc2 = sqlite3WalWriteLock(pPager->pLog, 0); pPager->state = PAGER_SHARED; }else if( !pPager->exclusiveMode ){ rc2 = osUnlock(pPager->fd, SHARED_LOCK); @@ -2193,7 +2193,7 @@ static int readDbPage(PgHdr *pPg){ if( pagerUseLog(pPager) ){ /* Try to pull the page from the write-ahead log. */ - rc = sqlite3LogRead(pPager->pLog, pgno, &isInLog, pPg->pData); + rc = sqlite3WalRead(pPager->pLog, pgno, &isInLog, pPg->pData); } if( rc==SQLITE_OK && !isInLog ){ iOffset = (pgno-1)*(i64)pPager->pageSize; @@ -2282,7 +2282,7 @@ static int pagerRollbackLog(Pager *pPager){ ** also copied into the backup databases) as part of this transaction, ** the backups must be restarted. */ - if( sqlite3LogDirty(pPager->pLog) ){ + if( sqlite3WalDirty(pPager->pLog) ){ sqlite3BackupRestart(pPager->pBackup); } @@ -2294,7 +2294,7 @@ static int pagerRollbackLog(Pager *pPager){ ** + Reload page content from the database (if refcount>0). */ pPager->dbSize = pPager->dbOrigSize; - rc = sqlite3LogUndo(pPager->pLog, pagerUndoCallback, (void *)pPager); + rc = sqlite3WalUndo(pPager->pLog, pagerUndoCallback, (void *)pPager); pList = sqlite3PcacheDirtyList(pPager->pPCache); while( pList && rc==SQLITE_OK ){ PgHdr *pNext = pList->pDirty; @@ -2738,7 +2738,7 @@ int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ i64 n = 0; /* File size in bytes returned by OsFileSize() */ if( pagerUseLog(pPager) ){ - sqlite3LogDbsize(pPager->pLog, &nPage); + sqlite3WalDbsize(pPager->pLog, &nPage); } if( nPage==0 ){ @@ -2923,7 +2923,7 @@ int sqlite3PagerClose(Pager *pPager){ sqlite3BeginBenignMalloc(); pPager->errCode = 0; pPager->exclusiveMode = 0; - sqlite3LogClose(pPager->pLog, pPager->fd, + sqlite3WalClose(pPager->pLog, pPager->fd, (pPager->noSync ? 0 : pPager->sync_flags), pTmp ); pPager->pLog = 0; @@ -3270,7 +3270,7 @@ static int subjournalPage(PgHdr *pPg){ } /* -** This function is a wrapper around sqlite3LogFrames(). As well as logging +** This function is a wrapper around sqlite3WalFrames(). As well as logging ** the contents of the list of pages headed by pList (connected by pDirty), ** this function notifies any active backup processes that the pages have ** changed. @@ -3285,7 +3285,7 @@ static int pagerLogFrames( int rc; /* Return code */ assert( pPager->pLog ); - rc = sqlite3LogFrames(pPager->pLog, + rc = sqlite3WalFrames(pPager->pLog, pPager->pageSize, pList, nTruncate, isCommit, sync_flags ); if( rc==SQLITE_OK && pPager->pBackup ){ @@ -3838,7 +3838,7 @@ static int pagerOpenSnapshot(Pager *pPager){ assert( pagerUseLog(pPager) ); - rc = sqlite3LogOpenSnapshot(pPager->pLog, &changed); + rc = sqlite3WalOpenSnapshot(pPager->pLog, &changed); if( rc==SQLITE_OK ){ int dummy; if( changed ){ @@ -4456,7 +4456,7 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ ** The busy-handler is not invoked if another connection already ** holds the write-lock. If possible, the upper layer will call it. */ - rc = sqlite3LogWriteLock(pPager->pLog, 1); + rc = sqlite3WalWriteLock(pPager->pLog, 1); if( rc==SQLITE_OK ){ pPager->dbOrigSize = pPager->dbSize; pPager->state = PAGER_RESERVED; @@ -5777,7 +5777,7 @@ int sqlite3PagerCheckpoint(Pager *pPager){ int rc = SQLITE_OK; if( pPager->pLog ){ u8 *zBuf = (u8 *)pPager->pTmpSpace; - rc = sqlite3LogCheckpoint(pPager->pLog, pPager->fd, + rc = sqlite3WalCheckpoint(pPager->pLog, pPager->fd, (pPager->noSync ? 0 : pPager->sync_flags), zBuf, pPager->xBusyHandler, pPager->pBusyHandlerArg ); @@ -5786,7 +5786,7 @@ int sqlite3PagerCheckpoint(Pager *pPager){ } int sqlite3PagerLogCallback(Pager *pPager){ - return sqlite3LogCallback(pPager->pLog); + return sqlite3WalCallback(pPager->pLog); } /* @@ -5806,7 +5806,7 @@ int sqlite3PagerOpenLog(Pager *pPager, int *pisOpen){ ** (e.g. due to malloc() failure), unlock the database file and ** return an error code. */ - rc = sqlite3LogOpen(pPager->pVfs, pPager->zFilename, &pPager->pLog); + rc = sqlite3WalOpen(pPager->pVfs, pPager->zFilename, &pPager->pLog); if( rc==SQLITE_OK ){ pPager->journalMode = PAGER_JOURNALMODE_WAL; } @@ -5843,7 +5843,7 @@ int sqlite3PagerCloseLog(Pager *pPager){ rc = pagerHasWAL(pPager, &logexists); } if( rc==SQLITE_OK && logexists ){ - rc = sqlite3LogOpen(pPager->pVfs, pPager->zFilename, &pPager->pLog); + rc = sqlite3WalOpen(pPager->pVfs, pPager->zFilename, &pPager->pLog); } } @@ -5853,7 +5853,7 @@ int sqlite3PagerCloseLog(Pager *pPager){ if( rc==SQLITE_OK && pPager->pLog ){ rc = sqlite3OsLock(pPager->fd, SQLITE_LOCK_EXCLUSIVE); if( rc==SQLITE_OK ){ - rc = sqlite3LogClose(pPager->pLog, pPager->fd, + rc = sqlite3WalClose(pPager->pLog, pPager->fd, (pPager->noSync ? 0 : pPager->sync_flags), (u8*)pPager->pTmpSpace ); diff --git a/src/log.c b/src/wal.c similarity index 99% rename from src/log.c rename to src/wal.c index c0357e87de..f0239bdb3a 100644 --- a/src/log.c +++ b/src/wal.c @@ -57,7 +57,7 @@ ** not the same as the iCheck1 and iCheck2 fields of the LogSummaryHdr. */ -#include "log.h" +#include "wal.h" #include #include @@ -1006,7 +1006,7 @@ static int logSummaryInit( ** were to do this just after this client opened one of these files, the ** system would be badly broken. */ -int sqlite3LogOpen( +int sqlite3WalOpen( sqlite3_vfs *pVfs, /* vfs module to open log file with */ const char *zDb, /* Name of database file */ Log **ppLog /* OUT: Allocated Log handle */ @@ -1264,7 +1264,7 @@ static int logCheckpoint( /* ** Close a connection to a log file. */ -int sqlite3LogClose( +int sqlite3WalClose( Log *pLog, /* Log to close */ sqlite3_file *pFd, /* Database file */ int sync_flags, /* Flags to pass to OsSync() (or 0) */ @@ -1444,7 +1444,7 @@ int logSummaryReadHdr(Log *pLog, int *pChanged){ ** is left unmodified. This is used by the pager layer to determine whether ** or not any cached pages may be safely reused. */ -int sqlite3LogOpenSnapshot(Log *pLog, int *pChanged){ +int sqlite3WalOpenSnapshot(Log *pLog, int *pChanged){ int rc = SQLITE_OK; if( pLog->isLocked==0 ){ int nAttempt; @@ -1480,7 +1480,7 @@ int sqlite3LogOpenSnapshot(Log *pLog, int *pChanged){ rc = logSummaryReadHdr(pLog, pChanged); if( rc!=SQLITE_OK ){ /* An error occured while attempting log recovery. */ - sqlite3LogCloseSnapshot(pLog); + sqlite3WalCloseSnapshot(pLog); } } return rc; @@ -1489,7 +1489,7 @@ int sqlite3LogOpenSnapshot(Log *pLog, int *pChanged){ /* ** Unlock the current snapshot. */ -void sqlite3LogCloseSnapshot(Log *pLog){ +void sqlite3WalCloseSnapshot(Log *pLog){ if( pLog->isLocked ){ assert( pLog->isLocked==LOG_REGION_A || pLog->isLocked==LOG_REGION_D ); logLockRegion(pLog, pLog->isLocked, LOG_UNLOCK); @@ -1500,7 +1500,7 @@ void sqlite3LogCloseSnapshot(Log *pLog){ /* ** Read a page from the log, if it is present. */ -int sqlite3LogRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut){ +int sqlite3WalRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut){ u32 iRead = 0; u32 *aData = pLog->pSummary->aData; int iFrame = (pLog->hdr.iLastPg & 0xFFFFFF00); @@ -1566,7 +1566,7 @@ int sqlite3LogRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut){ /* ** Set *pPgno to the size of the database file (or zero, if unknown). */ -void sqlite3LogDbsize(Log *pLog, Pgno *pPgno){ +void sqlite3WalDbsize(Log *pLog, Pgno *pPgno){ assert( pLog->isLocked ); *pPgno = pLog->hdr.nPage; } @@ -1576,7 +1576,7 @@ void sqlite3LogDbsize(Log *pLog, Pgno *pPgno){ ** Otherwise, if the caller is operating on a snapshot that has already ** been overwritten by another writer, SQLITE_BUSY is returned. */ -int sqlite3LogWriteLock(Log *pLog, int op){ +int sqlite3WalWriteLock(Log *pLog, int op){ assert( pLog->isLocked ); if( op ){ @@ -1636,7 +1636,7 @@ int sqlite3LogWriteLock(Log *pLog, int op){ ** Otherwise, if the callback function does not return an error, this ** function returns SQLITE_OK. */ -int sqlite3LogUndo(Log *pLog, int (*xUndo)(void *, Pgno), void *pUndoCtx){ +int sqlite3WalUndo(Log *pLog, int (*xUndo)(void *, Pgno), void *pUndoCtx){ int rc = SQLITE_OK; Pgno iMax = pLog->hdr.iLastPg; Pgno iFrame; @@ -1652,7 +1652,7 @@ int sqlite3LogUndo(Log *pLog, int (*xUndo)(void *, Pgno), void *pUndoCtx){ /* ** Return true if data has been written but not committed to the log file. */ -int sqlite3LogDirty(Log *pLog){ +int sqlite3WalDirty(Log *pLog){ assert( pLog->isWriteLocked ); return( pLog->hdr.iLastPg!=((LogSummaryHdr*)pLog->pSummary->aData)->iLastPg ); } @@ -1661,7 +1661,7 @@ int sqlite3LogDirty(Log *pLog){ ** Write a set of frames to the log. The caller must hold at least a ** RESERVED lock on the database file. */ -int sqlite3LogFrames( +int sqlite3WalFrames( Log *pLog, /* Log handle to write to */ int nPgsz, /* Database page-size in bytes */ PgHdr *pList, /* List of dirty pages to write */ @@ -1801,7 +1801,7 @@ int sqlite3LogFrames( ** 4. Zero the log-summary header (so new readers will ignore the log). ** 5. Drop the locks obtained in steps 1 and 2. */ -int sqlite3LogCheckpoint( +int sqlite3WalCheckpoint( Log *pLog, /* Log connection */ sqlite3_file *pFd, /* File descriptor open on db file */ int sync_flags, /* Flags to sync db file with (or 0) */ @@ -1839,7 +1839,7 @@ int sqlite3LogCheckpoint( return rc; } -int sqlite3LogCallback(Log *pLog){ +int sqlite3WalCallback(Log *pLog){ u32 ret = 0; if( pLog ){ ret = pLog->iCallback; @@ -1847,4 +1847,3 @@ int sqlite3LogCallback(Log *pLog){ } return (int)ret; } - diff --git a/src/log.h b/src/wal.h similarity index 72% rename from src/log.h rename to src/wal.h index c0699ba6fb..c53e41d5ea 100644 --- a/src/log.h +++ b/src/wal.h @@ -14,8 +14,8 @@ ** the implementation of each function in log.c for further details. */ -#ifndef _LOG_H_ -#define _LOG_H_ +#ifndef _WAL_H_ +#define _WAL_H_ #include "sqliteInt.h" @@ -23,31 +23,31 @@ typedef struct Log Log; /* Open and close a connection to a log file. */ -int sqlite3LogOpen(sqlite3_vfs*, const char *zDb, Log **ppLog); -int sqlite3LogClose(Log *pLog, sqlite3_file *pFd, int sync_flags, u8 *zBuf); +int sqlite3WalOpen(sqlite3_vfs*, const char *zDb, Log **ppLog); +int sqlite3WalClose(Log *pLog, sqlite3_file *pFd, int sync_flags, u8 *zBuf); /* Used by readers to open (lock) and close (unlock) a snapshot. */ -int sqlite3LogOpenSnapshot(Log *pLog, int *); -void sqlite3LogCloseSnapshot(Log *pLog); +int sqlite3WalOpenSnapshot(Log *pLog, int *); +void sqlite3WalCloseSnapshot(Log *pLog); /* Read a page from the log, if it is present. */ -int sqlite3LogRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut); -void sqlite3LogDbsize(Log *pLog, Pgno *pPgno); +int sqlite3WalRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut); +void sqlite3WalDbsize(Log *pLog, Pgno *pPgno); /* Obtain or release the WRITER lock. */ -int sqlite3LogWriteLock(Log *pLog, int op); +int sqlite3WalWriteLock(Log *pLog, int op); /* Undo any frames written (but not committed) to the log */ -int sqlite3LogUndo(Log *pLog, int (*xUndo)(void *, Pgno), void *pUndoCtx); +int sqlite3WalUndo(Log *pLog, int (*xUndo)(void *, Pgno), void *pUndoCtx); /* Return true if data has been written but not committed to the log file. */ -int sqlite3LogDirty(Log *pLog); +int sqlite3WalDirty(Log *pLog); /* Write a frame or frames to the log. */ -int sqlite3LogFrames(Log *pLog, int, PgHdr *, Pgno, int, int); +int sqlite3WalFrames(Log *pLog, int, PgHdr *, Pgno, int, int); /* Copy pages from the log to the database file */ -int sqlite3LogCheckpoint( +int sqlite3WalCheckpoint( Log *pLog, /* Log connection */ sqlite3_file *pFd, /* File descriptor open on db file */ int sync_flags, /* Flags to sync db file with (or 0) */ @@ -57,6 +57,6 @@ int sqlite3LogCheckpoint( ); /* Return the value to pass to a log callback. Or 0 for no callback. */ -int sqlite3LogCallback(Log *pLog); +int sqlite3WalCallback(Log *pLog); -#endif /* _LOG_H_ */ +#endif /* _WAL_H_ */ diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 12e4a5ce00..686b9dd5f8 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -93,7 +93,6 @@ foreach hdr { hash.h hwtime.h keywordhash.h - log.h mutex.h opcodes.h os_common.h @@ -110,6 +109,7 @@ foreach hdr { sqliteLimit.h vdbe.h vdbeInt.h + wal.h } { set available_hdr($hdr) 1 } @@ -244,7 +244,7 @@ foreach file { pcache.c pcache1.c rowset.c - log.c + wal.c pager.c btmutex.c From cd11fb28c38c64ceb0a0c2b37ca9f4c8c592bab3 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 26 Apr 2010 10:40:52 +0000 Subject: [PATCH 37/89] Add mutexes to fix a race condition in wal.c. This isn't a very good fix. FossilOrigin-Name: 3d159939cc2beb18c4ca0c8e9a99a75d4107e6e4 --- manifest | 32 +++++++++++--------------------- manifest.uuid | 2 +- src/test1.c | 3 +++ src/wal.c | 30 +++++++++++++++++++++++++----- test/wal.test | 5 +++-- test/walthread.test | 8 ++++++-- 6 files changed, 49 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index a3bb80a173..991bc73f00 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,5 @@ ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA1 - -C Change\sthe\snames\sof\sthe\slog.c\sand\slog.h\ssource\sfiles\sto\swal.c\sand\swal.h. -D 2010-04-26T00:19:45 +C Add\smutexes\sto\sfix\sa\srace\scondition\sin\swal.c.\sThis\sisn't\sa\svery\sgood\sfix. +D 2010-04-26T10:40:52 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -176,7 +173,7 @@ F src/sqliteLimit.h 3afab2291762b5d09ae20c18feb8e9fa935a60a6 F src/status.c 4df6fe7dce2d256130b905847c6c60055882bdbe F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c e27e0457109d72c7404df08af1726ff6ddc3d92c -F src/test1.c 8b4c246c41e75c3ff033edb2e8c2cf15820861ae +F src/test1.c 64b5b8135080b94370e8100e5066bb394f5c3122 F src/test2.c b6b43413d495addd039a88b87d65c839f86b18cb F src/test3.c 4c21700c73a890a47fc685c1097bfb661346ac94 F src/test4.c ad03bb987ddedce928f4258c1e7fa4109a73497d @@ -224,8 +221,8 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c df5283ae70cce52852fd788479bee22994fccbe5 w src/log.c -F src/wal.h 1a34bb71642b46af5e6d51c2f69ae0f55268df81 w src/log.h +F src/wal.c f93ce8968d6e86378aa2d465f592027e02929a7a +F src/wal.h 1a34bb71642b46af5e6d51c2f69ae0f55268df81 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -761,13 +758,13 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test 8dcfd1ce874aead6199fa715ba8e35cfbb3ef766 +F test/wal.test 7d44c2ee327e86a7c26052795789dd2d472bd1d4 F test/walbak.test f6fde9a5f59d0c697cb1f4af7876178c2f69a7ba F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47 F test/walmode.test c2f4e30ad64910b2d8faf6cf4e940b3f201b41df F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 -F test/walthread.test 27e44ee6fd02f1f494a24f999c97086af3ab739d +F test/walthread.test a7962d8b899366cc71f8381d9aeab8ae9e1b544d F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 F test/where2.test 45eacc126aabb37959a387aa83e59ce1f1f03820 F test/where3.test aa44a9b29e8c9f3d7bb94a3bb3a95b31627d520d @@ -811,14 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P f5e615c28c7035a7e6d896790b51cf9bc7371d5f -R 993bceae438ad1d8ab0904d8daf11104 -U drh -Z e29cadabf74dbcaa3b3703c56732da89 ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.6 (GNU/Linux) - -iD8DBQFL1NwloxKgR168RlERAks8AJ9QVBJiBku5alys49Firj8NBRPvlwCfSHT7 -JCRELzIJ9CW5GkQjtcejMJU= -=4Ljy ------END PGP SIGNATURE----- +P 56fe5d7624f840417152bcc63efbe21a5f557920 +R 262915592f11e9609982ee307c6e8694 +U dan +Z 883881034efc398d53801b8849d7e631 diff --git a/manifest.uuid b/manifest.uuid index e99c944438..ed29de1623 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -56fe5d7624f840417152bcc63efbe21a5f557920 \ No newline at end of file +3d159939cc2beb18c4ca0c8e9a99a75d4107e6e4 \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index b5fda952c0..6f20cf9a90 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5113,6 +5113,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ #ifdef SQLITE_TEST extern char sqlite3_query_plan[]; static char *query_plan = sqlite3_query_plan; + extern int sqlite3_walsummary_mmap_incr; /* In wal.c */ #ifdef SQLITE_ENABLE_FTS3 extern int sqlite3_fts3_enable_parentheses; #endif @@ -5172,6 +5173,8 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ #ifdef SQLITE_TEST Tcl_LinkVar(interp, "sqlite_query_plan", (char*)&query_plan, TCL_LINK_STRING|TCL_LINK_READ_ONLY); + Tcl_LinkVar(interp, "sqlite_walsummary_mmap_incr", + (char*)&sqlite3_walsummary_mmap_incr, TCL_LINK_INT); #endif #ifdef SQLITE_DEBUG Tcl_LinkVar(interp, "sqlite_addop_trace", diff --git a/src/wal.c b/src/wal.c index f0239bdb3a..0797822f8a 100644 --- a/src/wal.c +++ b/src/wal.c @@ -119,7 +119,12 @@ struct LogSummaryHdr { ** A 64 KB log-summary mapping corresponds to a log file containing over ** 13000 frames, so the mapping size does not need to be increased often. */ -#define LOGSUMMARY_MMAP_INCREMENT (64*1024) +#ifdef SQLITE_TEST +int sqlite3_walsummary_mmap_incr = 128; +# define LOGSUMMARY_MMAP_INCREMENT sqlite3_walsummary_mmap_incr +#else +# define LOGSUMMARY_MMAP_INCREMENT (64*1024) +#endif /* ** There is one instance of this structure for each log-summary object @@ -611,9 +616,10 @@ static int logSummaryEntry(u32 iFrame){ static void logSummaryAppend(LogSummary *pSummary, u32 iFrame, u32 iPage){ u32 iSlot = logSummaryEntry(iFrame); - if( (iSlot+128)>=pSummary->nData ){ + while( (iSlot+128)>=pSummary->nData ){ int nByte = pSummary->nData*4 + LOGSUMMARY_MMAP_INCREMENT; + /* Unmap and remap the log-summary file. */ sqlite3_mutex_enter(pSummary->mutex); munmap(pSummary->aData, pSummary->nData*4); pSummary->aData = 0; @@ -1377,8 +1383,11 @@ int logSummaryTryHdr(Log *pLog, int *pChanged){ u32 aCksum[2] = {1, 1}; u32 aHdr[LOGSUMMARY_HDR_NFIELD+2]; - /* First try to read the header without a lock. Verify the checksum - ** before returning. This will almost always work. + /* Read the header. The caller may or may not have locked the log-summary + ** file, meaning it is possible that an inconsistent snapshot is read + ** from the file. If this happens, return SQLITE_ERROR. The caller will + ** retry. Or, if the caller has already locked the file and the header + ** still looks inconsistent, it will run recovery. */ memcpy(aHdr, pLog->pSummary->aData, sizeof(aHdr)); logChecksumBytes((u8*)aHdr, sizeof(u32)*LOGSUMMARY_HDR_NFIELD, aCksum); @@ -1410,10 +1419,16 @@ int logSummaryReadHdr(Log *pLog, int *pChanged){ /* First try to read the header without a lock. Verify the checksum ** before returning. This will almost always work. + ** + ** TODO: Doing this causes a race-condition with the code that resizes + ** the mapping. Unless Log.pSummary->mutex is held, it is possible that + ** LogSummary.aData is invalid. */ +#if 0 if( SQLITE_OK==logSummaryTryHdr(pLog, pChanged) ){ return SQLITE_OK; } +#endif /* If the first attempt to read the header failed, lock the log-summary ** file and try again. If the header checksum verification fails this @@ -1502,11 +1517,14 @@ void sqlite3WalCloseSnapshot(Log *pLog){ */ int sqlite3WalRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut){ u32 iRead = 0; - u32 *aData = pLog->pSummary->aData; + u32 *aData; int iFrame = (pLog->hdr.iLastPg & 0xFFFFFF00); assert( pLog->isLocked ); + sqlite3_mutex_enter(pLog->pSummary->mutex); + aData = pLog->pSummary->aData; + /* Do a linear search of the unindexed block of page-numbers (if any) ** at the end of the log-summary. An alternative to this would be to ** build an index in private memory each time a read transaction is @@ -1549,6 +1567,8 @@ int sqlite3WalRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut){ } assert( iRead==0 || aData[logSummaryEntry(iRead)]==pgno ); + sqlite3_mutex_leave(pLog->pSummary->mutex); + /* If iRead is non-zero, then it is the log frame number that contains the ** required page. Read and return data from the log file. */ diff --git a/test/wal.test b/test/wal.test index d1a609ae56..1398ffd3ac 100644 --- a/test/wal.test +++ b/test/wal.test @@ -787,8 +787,9 @@ do_test wal-13.5 { execsql { SELECT count(*) FROM t2 } } [expr int(pow(2, 15))] do_test wal-13.6 { - file size test.db-wal-summary -} [expr 192*1024] + set sz [file size test.db-wal-summary] + expr {$sz<=(3*64*1024) && $sz>(2*64*1024)} +} {1} finish_test diff --git a/test/walthread.test b/test/walthread.test index 08219a7f28..23eb1d6a0e 100644 --- a/test/walthread.test +++ b/test/walthread.test @@ -10,7 +10,7 @@ #*********************************************************************** # This file implements regression tests for SQLite library. The # focus of this file is testing the operation of the library in -# "PRAGMA journal_mode=WAL" mode. +# "PRAGMA journal_mode=WAL" mode with multiple threads. # set testdir [file dirname $argv0] @@ -18,6 +18,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl if {[run_thread_tests]==0} { finish_test ; return } +set sqlite_walsummary_mmap_incr 64 + +#-------------------------------------------------------------------------- +# Initialize the database used for the multi-thread test. +# do_test walthread-1.1 { execsql { PRAGMA journal_mode = WAL; @@ -69,7 +74,6 @@ do_test walthread-1.4 { # 1) Execute "CHECKPOINT main 32 -1 1" # 2) Sleep for 500 ms. # - set thread_program { proc rest {ms} { set ::rest 0 From 71cb518fca40486b3468426625e077284a48ae8f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 26 Apr 2010 12:39:03 +0000 Subject: [PATCH 38/89] Add the "wal" permutation to run existing test files savepoint.test and savepoint2.test in WAL mode. FossilOrigin-Name: 205e5d8ac08f7d7853427b4cd235fca125155d2d --- manifest | 18 +- manifest.uuid | 2 +- test/permutations.test | 7 + test/savepoint.test | 429 +++++++++++++++++++++++------------------ test/savepoint2.test | 6 + test/tester.tcl | 36 ++++ 6 files changed, 296 insertions(+), 202 deletions(-) diff --git a/manifest b/manifest index 991bc73f00..3e8c4278ae 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smutexes\sto\sfix\sa\srace\scondition\sin\swal.c.\sThis\sisn't\sa\svery\sgood\sfix. -D 2010-04-26T10:40:52 +C Add\sthe\s"wal"\spermutation\sto\srun\sexisting\stest\sfiles\ssavepoint.test\sand\ssavepoint2.test\sin\sWAL\smode. +D 2010-04-26T12:39:04 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -532,7 +532,7 @@ F test/pageropt.test 3ee6578891baaca967f0bd349e4abfa736229e1a F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb F test/pcache.test eebc4420b37cb07733ae9b6e99c9da7c40dd6d58 F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16 -F test/permutations.test 6117bae41935a787cc92faee81e91f813580eccb +F test/permutations.test e9a9070862e3c675f0844815054cfd2ed4ed9c66 F test/pragma.test 6960f9efbce476f70ba9ee2171daf5042f9e3d8a F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea @@ -548,8 +548,8 @@ F test/rollback.test 73355ad4492ff9a3a31e61c7e5eb5e01a1de94ca F test/rowhash.test 97f56043ba11f0679920416c0cdbc72e5272267b F test/rowid.test e58e0acef38b527ed1b0b70d3ada588f804af287 F test/rtree.test 55466a200af3591946c5da77ad5dbfbc1e5e05f9 -F test/savepoint.test f2ede4b643ad87ead36c041c72d774a1f5c8a564 -F test/savepoint2.test 427c8b20f43d3edf17a290c6788ae9e2703ac51c +F test/savepoint.test 500ef5afb383107fd3b7d055cdeb015d36d1bc76 +F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint3.test e328085853b14898d78ceea00dfe7db18bb6a9ec F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd @@ -603,7 +603,7 @@ F test/tclsqlite.test 7378a4e7474d9ffdb7c930086112b829e5955fb0 F test/tempdb.test 1bf52da28a9c24e29717362a87722dff08feb72b F test/temptable.test f42121a0d29a62f00f93274464164177ab1cc24a F test/temptrigger.test b0273db072ce5f37cf19140ceb1f0d524bbe9f05 -F test/tester.tcl 49d76f12940160d623da104f995530fc6ee8f46f +F test/tester.tcl 17082172a11795f9bc3e843aadb463ff76050c68 F test/thread001.test a3e6a7254d1cb057836cb3145b60c10bf5b7e60f F test/thread002.test afd20095e6e845b405df4f2c920cb93301ca69db F test/thread003.test b824d4f52b870ae39fc5bae4d8070eca73085dca @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 56fe5d7624f840417152bcc63efbe21a5f557920 -R 262915592f11e9609982ee307c6e8694 +P 3d159939cc2beb18c4ca0c8e9a99a75d4107e6e4 +R 9cdc568754b75f48338d755d9a93a206 U dan -Z 883881034efc398d53801b8849d7e631 +Z e32fb56774989323e02682dddebb9008 diff --git a/manifest.uuid b/manifest.uuid index ed29de1623..a461064513 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3d159939cc2beb18c4ca0c8e9a99a75d4107e6e4 \ No newline at end of file +205e5d8ac08f7d7853427b4cd235fca125155d2d \ No newline at end of file diff --git a/test/permutations.test b/test/permutations.test index ada6503a79..f11bb661e5 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -756,6 +756,13 @@ if {[info commands register_demovfs] != ""} { } } +run_tests "wal" -description { + Run tests with journal_mode=WAL +} -include { + savepoint.test + savepoint2.test +} + # End of tests ############################################################################# diff --git a/test/savepoint.test b/test/savepoint.test index 71037d6f6a..bd38530836 100644 --- a/test/savepoint.test +++ b/test/savepoint.test @@ -14,13 +14,13 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl - #---------------------------------------------------------------------- # The following tests - savepoint-1.* - test that the SAVEPOINT, RELEASE # and ROLLBACK TO comands are correctly parsed, and that the auto-commit # flag is correctly set and unset as a result. # do_test savepoint-1.1 { + wal_set_journal_mode execsql { SAVEPOINT sp1; RELEASE sp1; @@ -93,6 +93,7 @@ do_test savepoint-1.5 { do_test savepoint-1.6 { execsql COMMIT } {} +wal_check_journal_mode savepoint-1.7 #------------------------------------------------------------------------ # These tests - savepoint-2.* - test rollbacks and releases of savepoints @@ -175,36 +176,42 @@ do_test savepoint-2.11 { } execsql { SELECT * FROM t1 } } {} +wal_check_journal_mode savepoint-2.12 #------------------------------------------------------------------------ # This block of tests - savepoint-3.* - test that when a transaction # savepoint is rolled back, locks are not released from database files. # And that when a transaction savepoint is released, they are released. +# +# These tests do not work in WAL mode. WAL mode does not take RESERVED +# locks on the database file. # -do_test savepoint-3.1 { - execsql { SAVEPOINT "transaction" } - execsql { PRAGMA lock_status } -} {main unlocked temp closed} - -do_test savepoint-3.2 { - execsql { INSERT INTO t1 VALUES(1, 2, 3) } - execsql { PRAGMA lock_status } -} {main reserved temp closed} - -do_test savepoint-3.3 { - execsql { ROLLBACK TO "transaction" } - execsql { PRAGMA lock_status } -} {main reserved temp closed} - -do_test savepoint-3.4 { - execsql { INSERT INTO t1 VALUES(1, 2, 3) } - execsql { PRAGMA lock_status } -} {main reserved temp closed} - -do_test savepoint-3.5 { - execsql { RELEASE "transaction" } - execsql { PRAGMA lock_status } -} {main unlocked temp closed} +if {[wal_is_wal_mode]==0} { + do_test savepoint-3.1 { + execsql { SAVEPOINT "transaction" } + execsql { PRAGMA lock_status } + } {main unlocked temp closed} + + do_test savepoint-3.2 { + execsql { INSERT INTO t1 VALUES(1, 2, 3) } + execsql { PRAGMA lock_status } + } {main reserved temp closed} + + do_test savepoint-3.3 { + execsql { ROLLBACK TO "transaction" } + execsql { PRAGMA lock_status } + } {main reserved temp closed} + + do_test savepoint-3.4 { + execsql { INSERT INTO t1 VALUES(1, 2, 3) } + execsql { PRAGMA lock_status } + } {main reserved temp closed} + + do_test savepoint-3.5 { + execsql { RELEASE "transaction" } + execsql { PRAGMA lock_status } + } {main unlocked temp closed} +} #------------------------------------------------------------------------ # Test that savepoints that include schema modifications are handled @@ -264,6 +271,7 @@ do_test savepoint-4.7 { do_test savepoint-4.8 { execsql COMMIT } {} +wal_check_journal_mode savepoint-4.9 #------------------------------------------------------------------------ # Test some logic errors to do with the savepoint feature. @@ -312,6 +320,18 @@ ifcapable incrblob { execsql {release abc} } {} + # Rollback mode: + # + # Open a savepoint transaction and insert a row into the database. Then, + # using a second database handle, open a read-only transaction on the + # database file. Check that the savepoint transaction cannot be committed + # until after the read-only transaction has been closed. + # + # WAL mode: + # + # As above, except that the savepoint transaction can be successfully + # committed before the read-only transaction has been closed. + # do_test savepoint-5.4.1 { execsql { SAVEPOINT main; @@ -320,17 +340,28 @@ ifcapable incrblob { } {} do_test savepoint-5.4.2 { sqlite3 db2 test.db - execsql { BEGIN ; SELECT * FROM blobs } db2 - catchsql { RELEASE main } - } {1 {database is locked}} - do_test savepoint-5.4.3 { - db2 close - catchsql { RELEASE main } - } {0 {}} - do_test savepoint-5.4.4 { + execsql { BEGIN ; SELECT count(*) FROM blobs } db2 + } {1} + if {[wal_is_wal_mode]} { + do_test savepoint-5.4.3 { catchsql "RELEASE main" } {0 {}} + do_test savepoint-5.4.4 { db2 close } {} + } else { + do_test savepoint-5.4.3 { + catchsql { RELEASE main } + } {1 {database is locked}} + do_test savepoint-5.4.4 { + db2 close + catchsql { RELEASE main } + } {0 {}} + } + do_test savepoint-5.4.5 { execsql { SELECT x FROM blobs WHERE rowid = 2 } } {{another blob}} + do_test savepoint-5.4.6 { + execsql { SELECT count(*) FROM blobs } + } {2} } +wal_check_journal_mode savepoint-5.5 #------------------------------------------------------------------------- # The following tests, savepoint-6.*, test an incr-vacuum inside of a @@ -342,8 +373,9 @@ ifcapable {autovacuum && pragma} { sqlite3 db test.db do_test savepoint-6.1 { - execsql { - PRAGMA auto_vacuum = incremental; + execsql { PRAGMA auto_vacuum = incremental } + wal_set_journal_mode + execsql { CREATE TABLE t1(a, b, c); CREATE INDEX i1 ON t1(a, b); BEGIN; @@ -376,6 +408,8 @@ ifcapable {autovacuum && pragma} { } {} integrity_check savepoint-6.4 + + wal_check_journal_mode savepoint-6.5 } #------------------------------------------------------------------------- @@ -387,8 +421,9 @@ file delete -force test.db sqlite3 db test.db do_test savepoint-7.1 { + execsql { PRAGMA auto_vacuum = incremental } + wal_set_journal_mode execsql { - PRAGMA auto_vacuum = incremental; PRAGMA cache_size = 10; BEGIN; CREATE TABLE t1(a PRIMARY KEY, b); @@ -449,13 +484,15 @@ do_test savepoint-7.3.2 { } execsql { PRAGMA integrity_check } } {ok} +wal_check_journal_mode savepoint-7.3.3 do_test savepoint-7.4.1 { db close file delete -force test.db sqlite3 db test.db + execsql { PRAGMA auto_vacuum = incremental } + wal_set_journal_mode execsql { - PRAGMA auto_vacuum = incremental; CREATE TABLE t1(a, b, PRIMARY KEY(a, b)); INSERT INTO t1 VALUES(randstr(1000,1000), randstr(1000,1000)); BEGIN; @@ -497,6 +534,7 @@ do_test savepoint-7.5.2 { DROP TABLE t5; } } {} +wal_check_journal_mode savepoint-7.5.3 # Test oddly named and quoted savepoints. # @@ -598,120 +636,122 @@ do_test savepoint-10.1.3 { set templockstate [lindex [db eval {PRAGMA lock_status}] 3] -do_test savepoint-10.2.1 { - file delete -force test3.db - file delete -force test2.db - execsql { - ATTACH 'test2.db' AS aux1; - ATTACH 'test3.db' AS aux2; - DROP TABLE t1; - CREATE TABLE main.t1(x, y); - CREATE TABLE aux1.t2(x, y); - CREATE TABLE aux2.t3(x, y); - SELECT name FROM sqlite_master - UNION ALL - SELECT name FROM aux1.sqlite_master - UNION ALL - SELECT name FROM aux2.sqlite_master; - } -} {t1 t2 t3} -do_test savepoint-10.2.2 { - execsql { PRAGMA lock_status } -} [list main unlocked temp $templockstate aux1 unlocked aux2 unlocked] - -do_test savepoint-10.2.3 { - execsql { - SAVEPOINT one; - INSERT INTO t1 VALUES(1, 2); - PRAGMA lock_status; - } -} [list main reserved temp $templockstate aux1 unlocked aux2 unlocked] -do_test savepoint-10.2.4 { - execsql { - INSERT INTO t3 VALUES(3, 4); - PRAGMA lock_status; - } -} [list main reserved temp $templockstate aux1 unlocked aux2 reserved] -do_test savepoint-10.2.5 { - execsql { - SAVEPOINT two; - INSERT INTO t2 VALUES(5, 6); - PRAGMA lock_status; - } -} [list main reserved temp $templockstate aux1 reserved aux2 reserved] -do_test savepoint-10.2.6 { - execsql { SELECT * FROM t2 } -} {5 6} -do_test savepoint-10.2.7 { - execsql { ROLLBACK TO two } - execsql { SELECT * FROM t2 } -} {} -do_test savepoint-10.2.8 { - execsql { PRAGMA lock_status } -} [list main reserved temp $templockstate aux1 reserved aux2 reserved] -do_test savepoint-10.2.9 { - execsql { SELECT 'a', * FROM t1 UNION ALL SELECT 'b', * FROM t3 } -} {a 1 2 b 3 4} -do_test savepoint-10.2.9 { - execsql { - INSERT INTO t2 VALUES(5, 6); - RELEASE one; - } - execsql { - SELECT * FROM t1; - SELECT * FROM t2; - SELECT * FROM t3; - } -} {1 2 5 6 3 4} -do_test savepoint-10.2.9 { - execsql { PRAGMA lock_status } -} [list main unlocked temp $templockstate aux1 unlocked aux2 unlocked] - -do_test savepoint-10.2.10 { - execsql { - SAVEPOINT one; - INSERT INTO t1 VALUES('a', 'b'); +if {[wal_is_wal_mode]==0} { + do_test savepoint-10.2.1 { + file delete -force test3.db + file delete -force test2.db + execsql { + ATTACH 'test2.db' AS aux1; + ATTACH 'test3.db' AS aux2; + DROP TABLE t1; + CREATE TABLE main.t1(x, y); + CREATE TABLE aux1.t2(x, y); + CREATE TABLE aux2.t3(x, y); + SELECT name FROM sqlite_master + UNION ALL + SELECT name FROM aux1.sqlite_master + UNION ALL + SELECT name FROM aux2.sqlite_master; + } + } {t1 t2 t3} + do_test savepoint-10.2.2 { + execsql { PRAGMA lock_status } + } [list main unlocked temp $templockstate aux1 unlocked aux2 unlocked] + + do_test savepoint-10.2.3 { + execsql { + SAVEPOINT one; + INSERT INTO t1 VALUES(1, 2); + PRAGMA lock_status; + } + } [list main reserved temp $templockstate aux1 unlocked aux2 unlocked] + do_test savepoint-10.2.4 { + execsql { + INSERT INTO t3 VALUES(3, 4); + PRAGMA lock_status; + } + } [list main reserved temp $templockstate aux1 unlocked aux2 reserved] + do_test savepoint-10.2.5 { + execsql { SAVEPOINT two; - INSERT INTO t2 VALUES('c', 'd'); - SAVEPOINT three; - INSERT INTO t3 VALUES('e', 'f'); - } - execsql { - SELECT * FROM t1; - SELECT * FROM t2; - SELECT * FROM t3; - } -} {1 2 a b 5 6 c d 3 4 e f} -do_test savepoint-10.2.11 { - execsql { ROLLBACK TO two } - execsql { - SELECT * FROM t1; - SELECT * FROM t2; - SELECT * FROM t3; - } -} {1 2 a b 5 6 3 4} -do_test savepoint-10.2.12 { - execsql { - INSERT INTO t3 VALUES('g', 'h'); - ROLLBACK TO two; - } - execsql { - SELECT * FROM t1; - SELECT * FROM t2; - SELECT * FROM t3; - } -} {1 2 a b 5 6 3 4} -do_test savepoint-10.2.13 { - execsql { ROLLBACK } - execsql { - SELECT * FROM t1; - SELECT * FROM t2; - SELECT * FROM t3; - } -} {1 2 5 6 3 4} -do_test savepoint-10.2.14 { - execsql { PRAGMA lock_status } -} [list main unlocked temp $templockstate aux1 unlocked aux2 unlocked] + INSERT INTO t2 VALUES(5, 6); + PRAGMA lock_status; + } + } [list main reserved temp $templockstate aux1 reserved aux2 reserved] + do_test savepoint-10.2.6 { + execsql { SELECT * FROM t2 } + } {5 6} + do_test savepoint-10.2.7 { + execsql { ROLLBACK TO two } + execsql { SELECT * FROM t2 } + } {} + do_test savepoint-10.2.8 { + execsql { PRAGMA lock_status } + } [list main reserved temp $templockstate aux1 reserved aux2 reserved] + do_test savepoint-10.2.9 { + execsql { SELECT 'a', * FROM t1 UNION ALL SELECT 'b', * FROM t3 } + } {a 1 2 b 3 4} + do_test savepoint-10.2.9 { + execsql { + INSERT INTO t2 VALUES(5, 6); + RELEASE one; + } + execsql { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + } + } {1 2 5 6 3 4} + do_test savepoint-10.2.9 { + execsql { PRAGMA lock_status } + } [list main unlocked temp $templockstate aux1 unlocked aux2 unlocked] + + do_test savepoint-10.2.10 { + execsql { + SAVEPOINT one; + INSERT INTO t1 VALUES('a', 'b'); + SAVEPOINT two; + INSERT INTO t2 VALUES('c', 'd'); + SAVEPOINT three; + INSERT INTO t3 VALUES('e', 'f'); + } + execsql { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + } + } {1 2 a b 5 6 c d 3 4 e f} + do_test savepoint-10.2.11 { + execsql { ROLLBACK TO two } + execsql { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + } + } {1 2 a b 5 6 3 4} + do_test savepoint-10.2.12 { + execsql { + INSERT INTO t3 VALUES('g', 'h'); + ROLLBACK TO two; + } + execsql { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + } + } {1 2 a b 5 6 3 4} + do_test savepoint-10.2.13 { + execsql { ROLLBACK } + execsql { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + } + } {1 2 5 6 3 4} + do_test savepoint-10.2.14 { + execsql { PRAGMA lock_status } + } [list main unlocked temp $templockstate aux1 unlocked aux2 unlocked] +} #------------------------------------------------------------------------- # The following tests - savepoint-11.* - test the interaction of @@ -722,8 +762,9 @@ do_test savepoint-11.1 { db close file delete -force test.db sqlite3 db test.db + execsql { PRAGMA auto_vacuum = full; } + wal_set_journal_mode execsql { - PRAGMA auto_vacuum = full; CREATE TABLE t1(a, b, UNIQUE(a, b)); INSERT INTO t1 VALUES(1, randstr(1000,1000)); INSERT INTO t1 VALUES(2, randstr(1000,1000)); @@ -751,10 +792,10 @@ do_test savepoint-11.6 { integrity_check savepoint-11.7 do_test savepoint-11.8 { execsql { ROLLBACK } + execsql { PRAGMA checkpoint } file size test.db } {8192} - do_test savepoint-11.9 { execsql { DROP TABLE IF EXISTS t1; @@ -782,6 +823,7 @@ do_test savepoint-11.11 { do_test savepoint-11.12 { execsql {SELECT * FROM t2} } {1 2 3 4} +wal_check_journal_mode savepoint-11.13 #------------------------------------------------------------------------- # The following tests - savepoint-12.* - test the interaction of @@ -815,50 +857,53 @@ do_test savepoint-12.3 { do_test savepoint-12.4 { execsql { SAVEPOINT one } } {} +wal_check_journal_mode savepoint-12.5 #------------------------------------------------------------------------- # The following tests - savepoint-13.* - test the interaction of # savepoints and "journal_mode = off". # -do_test savepoint-13.1 { - db close - catch {file delete -force test.db} - sqlite3 db test.db - execsql { - BEGIN; - CREATE TABLE t1(a PRIMARY KEY, b); - INSERT INTO t1 VALUES(1, 2); - COMMIT; - PRAGMA journal_mode = off; - } -} {off} -do_test savepoint-13.2 { - execsql { - BEGIN; - INSERT INTO t1 VALUES(3, 4); - INSERT INTO t1 SELECT a+4,b+4 FROM t1; - COMMIT; - } -} {} -do_test savepoint-13.3 { - execsql { - BEGIN; - INSERT INTO t1 VALUES(9, 10); - SAVEPOINT s1; - INSERT INTO t1 VALUES(11, 12); - COMMIT; - } -} {} -do_test savepoint-13.4 { - execsql { - BEGIN; - INSERT INTO t1 VALUES(13, 14); - SAVEPOINT s1; - INSERT INTO t1 VALUES(15, 16); - ROLLBACK TO s1; - ROLLBACK; - SELECT * FROM t1; - } -} {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16} +if {[wal_is_wal_mode]==0} { + do_test savepoint-13.1 { + db close + catch {file delete -force test.db} + sqlite3 db test.db + execsql { + BEGIN; + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 2); + COMMIT; + PRAGMA journal_mode = off; + } + } {off} + do_test savepoint-13.2 { + execsql { + BEGIN; + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 SELECT a+4,b+4 FROM t1; + COMMIT; + } + } {} + do_test savepoint-13.3 { + execsql { + BEGIN; + INSERT INTO t1 VALUES(9, 10); + SAVEPOINT s1; + INSERT INTO t1 VALUES(11, 12); + COMMIT; + } + } {} + do_test savepoint-13.4 { + execsql { + BEGIN; + INSERT INTO t1 VALUES(13, 14); + SAVEPOINT s1; + INSERT INTO t1 VALUES(15, 16); + ROLLBACK TO s1; + ROLLBACK; + SELECT * FROM t1; + } + } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16} +} finish_test diff --git a/test/savepoint2.test b/test/savepoint2.test index 10765a3166..be1bdbd424 100644 --- a/test/savepoint2.test +++ b/test/savepoint2.test @@ -14,6 +14,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl + # Tests in this file are quite similar to those run by trans.test and # avtrans.test. # @@ -23,6 +24,7 @@ proc signature {} { } do_test savepoint2-1 { + wal_set_journal_mode execsql { PRAGMA cache_size=10; BEGIN; @@ -42,6 +44,7 @@ do_test savepoint2-1 { SELECT count(*) FROM t3; } } {1024} +wal_check_journal_mode savepoint2-1.1 unset -nocomplain ::sig unset -nocomplain SQL @@ -140,6 +143,9 @@ for {set ii 2} {$ii < ($iterations+2)} {incr ii} { sqlite3_get_autocommit db } {1} integrity_check savepoint2-$ii.6.1 + + # Check that the connection is still running in WAL mode. + wal_check_journal_mode savepoint2-$ii.7 } unset -nocomplain ::sig diff --git a/test/tester.tcl b/test/tester.tcl index 44798a7098..9cf4a9b59e 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -983,6 +983,42 @@ proc drop_all_tables {{db db}} { } } +#------------------------------------------------------------------------- +# If a test script is executed with global variable +# $::permutations_test_prefix set to "wal", then the tests are run +# in WAL mode. Otherwise, they should be run in rollback mode. The +# following Tcl procs are used to make this less intrusive: +# +# wal_set_journal_mode ?DB? +# +# If running a WAL test, execute "PRAGMA journal_mode = wal" using +# connection handle DB. Otherwise, this command is a no-op. +# +# wal_check_journal_mode TESTNAME ?DB? +# +# If running a WAL test, execute a tests case that fails if the main +# database for connection handle DB is not currently a WAL database. +# Otherwise (if not running a WAL permutation) this is a no-op. +# +# wal_is_wal_mode +# +# Returns true if this test should be run in WAL mode. False otherwise. +# +proc wal_is_wal_mode {} { + expr { [catch {set ::permutations_test_prefix} v]==0 && $v == "wal" } +} +proc wal_set_journal_mode {{db db}} { + if { [wal_is_wal_mode] } { + $db eval "PRAGMA journal_mode = WAL" + } +} +proc wal_check_journal_mode {testname {db db}} { + if { [wal_is_wal_mode] } { + $db eval { SELECT * FROM sqlite_master } + do_test $testname [list $db eval "PRAGMA main.journal_mode"] {wal} + } +} + # If the library is compiled with the SQLITE_DEFAULT_AUTOVACUUM macro set # to non-zero, then set the global variable $AUTOVACUUM to 1. From 4cd78b4d8943308b64c18001fa5e066d467334e8 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 26 Apr 2010 16:57:10 +0000 Subject: [PATCH 39/89] Fixes for problems with small caches and SAVEPOINT rollback in WAL mode. FossilOrigin-Name: 6a944f028d4a070bef29e1fbc6fbef481ebcd34c --- manifest | 22 ++++++------ manifest.uuid | 2 +- src/pager.c | 25 +++++++++++-- src/wal.c | 25 +++++++++++-- src/wal.h | 3 ++ test/permutations.test | 1 + test/savepoint6.test | 9 +++-- test/wal.test | 82 ++++++++++++++++++++++++++++++++++++------ 8 files changed, 141 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index 3e8c4278ae..9451cdd82f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"wal"\spermutation\sto\srun\sexisting\stest\sfiles\ssavepoint.test\sand\ssavepoint2.test\sin\sWAL\smode. -D 2010-04-26T12:39:04 +C Fixes\sfor\sproblems\swith\ssmall\scaches\sand\sSAVEPOINT\srollback\sin\sWAL\smode. +D 2010-04-26T16:57:10 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -152,7 +152,7 @@ F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 082884dc2a20a2f1fddc404f3606ec2abe907b56 F src/os_unix.c 1acc854259b045cf029c6f95828a5f4b84f361c7 F src/os_win.c 607b40e40662c6c74ff2400ff03e7f42521d5c75 -F src/pager.c 5924538fb99f62774419bfae882debb0d65bc3c8 +F src/pager.c b4a41030860229e80295fa1f37addab24d21799c F src/pager.h cee4487ab4f0911dd9f22a40e3cd55afdb7ef444 F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf @@ -221,8 +221,8 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c f93ce8968d6e86378aa2d465f592027e02929a7a -F src/wal.h 1a34bb71642b46af5e6d51c2f69ae0f55268df81 +F src/wal.c ce03bdd759c7b949188c0b75c46777fe0f7f105e +F src/wal.h 812101dd76610401fbcd44114e7e8b7ce0224645 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -532,7 +532,7 @@ F test/pageropt.test 3ee6578891baaca967f0bd349e4abfa736229e1a F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb F test/pcache.test eebc4420b37cb07733ae9b6e99c9da7c40dd6d58 F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16 -F test/permutations.test e9a9070862e3c675f0844815054cfd2ed4ed9c66 +F test/permutations.test 0730dc6c028b620a15c38a0a7d07238da088ecd3 F test/pragma.test 6960f9efbce476f70ba9ee2171daf5042f9e3d8a F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea @@ -553,7 +553,7 @@ F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint3.test e328085853b14898d78ceea00dfe7db18bb6a9ec F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd -F test/savepoint6.test 2df1d093e59e78d688c64eb20e0457aaea7d08f9 +F test/savepoint6.test 8e871226bcd20241c0ee7da28166f5c525d7cba9 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/securedel.test 328d2921c0ca49bdd3352e516b0377fc07143254 @@ -758,7 +758,7 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test 7d44c2ee327e86a7c26052795789dd2d472bd1d4 +F test/wal.test b5d5fe1b3171eefb25355758e0dc689096bc1f87 F test/walbak.test f6fde9a5f59d0c697cb1f4af7876178c2f69a7ba F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47 @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 3d159939cc2beb18c4ca0c8e9a99a75d4107e6e4 -R 9cdc568754b75f48338d755d9a93a206 +P 205e5d8ac08f7d7853427b4cd235fca125155d2d +R f3447c8eaaa2559454ba677c09b3ac47 U dan -Z e32fb56774989323e02682dddebb9008 +Z 1f4eebc47ae413fc016108c1a59ad1db diff --git a/manifest.uuid b/manifest.uuid index a461064513..c549155246 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -205e5d8ac08f7d7853427b4cd235fca125155d2d \ No newline at end of file +6a944f028d4a070bef29e1fbc6fbef481ebcd34c \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 171b584573..bd704fa7c7 100644 --- a/src/pager.c +++ b/src/pager.c @@ -221,6 +221,7 @@ struct PagerSavepoint { Bitvec *pInSavepoint; /* Set of pages in this savepoint */ Pgno nOrig; /* Original number of pages in file */ Pgno iSubRec; /* Index of first record in sub-journal */ + u32 iFrame; /* Last frame in WAL when savepoint opened */ }; /* @@ -1539,6 +1540,7 @@ static int pager_playback_one_page( aData = pPager->pTmpSpace; assert( aData ); /* Temp storage must have already been allocated */ + assert( pagerUseLog(pPager)==0 || (!isMainJrnl && isSavepnt) ); /* Read the page number and page data from the journal or sub-journal ** file. Return an error code to the caller if an IO error occurs. @@ -1608,7 +1610,11 @@ static int pager_playback_one_page( ** is possible to fail a statement on a database that does not yet exist. ** Do not attempt to write if database file has never been opened. */ - pPg = pager_lookup(pPager, pgno); + if( pagerUseLog(pPager) ){ + pPg = 0; + }else{ + pPg = pager_lookup(pPager, pgno); + } assert( pPg || !MEMDB ); PAGERTRACE(("PLAYBACK %d page %d hash(%08x) %s\n", PAGERID(pPager), pgno, pager_datahash(pPager->pageSize, (u8*)aData), @@ -1625,6 +1631,7 @@ static int pager_playback_one_page( ){ i64 ofst = (pgno-1)*(i64)pPager->pageSize; testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); + assert( !pagerUseLog(pPager) ); rc = sqlite3OsWrite(pPager->fd, (u8*)aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; @@ -1689,6 +1696,7 @@ static int pager_playback_one_page( ** segment is synced. If a crash occurs during or following this, ** database corruption may ensue. */ + assert( !pagerUseLog(pPager) ); sqlite3PcacheMakeClean(pPg); } #ifdef SQLITE_CHECK_PAGES @@ -2428,6 +2436,10 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ if( pSavepoint ){ u32 ii; /* Loop counter */ i64 offset = pSavepoint->iSubRec*(4+pPager->pageSize); + + if( pagerUseLog(pPager) ){ + rc = sqlite3WalSavepointUndo(pPager->pLog, pSavepoint->iFrame); + } for(ii=pSavepoint->iSubRec; rc==SQLITE_OK && iinSubRec; ii++){ assert( offset==ii*(4+pPager->pageSize) ); rc = pager_playback_one_page(pPager, &offset, pDone, 0, 1); @@ -2439,6 +2451,7 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ if( rc==SQLITE_OK ){ pPager->journalOff = szJ; } + return rc; } @@ -3326,7 +3339,12 @@ static int pagerStress(void *p, PgHdr *pPg){ pPg->pDirty = 0; if( pagerUseLog(pPager) ){ /* Write a single frame for this page to the log. */ - rc = pagerLogFrames(pPager, pPg, 0, 0, 0); + if( subjRequiresPage(pPg) ){ + rc = subjournalPage(pPg); + } + if( rc==SQLITE_OK ){ + rc = pagerLogFrames(pPager, pPg, 0, 0, 0); + } }else{ /* The doNotSync flag is set by the sqlite3PagerWrite() function while it ** is journalling a set of two or more database pages that are stored @@ -5342,6 +5360,9 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ if( !aNew[ii].pInSavepoint ){ return SQLITE_NOMEM; } + if( pagerUseLog(pPager) ){ + aNew[ii].iFrame = sqlite3WalSavepoint(pPager->pLog); + } } /* Open the sub-journal, if it is not already opened. */ diff --git a/src/wal.c b/src/wal.c index 0797822f8a..8c6878a1a4 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1669,6 +1669,27 @@ int sqlite3WalUndo(Log *pLog, int (*xUndo)(void *, Pgno), void *pUndoCtx){ return rc; } +u32 sqlite3WalSavepoint(Log *pLog){ + assert( pLog->isWriteLocked ); + return pLog->hdr.iLastPg; +} + +int sqlite3WalSavepointUndo(Log *pLog, u32 iFrame){ + int rc = SQLITE_OK; + u8 aCksum[8]; + assert( pLog->isWriteLocked ); + + pLog->hdr.iLastPg = iFrame; + if( iFrame>0 ){ + i64 iOffset = logFrameOffset(iFrame, pLog->hdr.pgsz) + sizeof(u32)*2; + rc = sqlite3OsRead(pLog->pFd, aCksum, sizeof(aCksum), iOffset); + pLog->hdr.iCheck1 = sqlite3Get4byte(&aCksum[0]); + pLog->hdr.iCheck2 = sqlite3Get4byte(&aCksum[4]); + } + + return rc; +} + /* ** Return true if data has been written but not committed to the log file. */ @@ -1678,8 +1699,8 @@ int sqlite3WalDirty(Log *pLog){ } /* -** Write a set of frames to the log. The caller must hold at least a -** RESERVED lock on the database file. +** Write a set of frames to the log. The caller must hold the write-lock +** on the log file (obtained using sqlite3WalWriteLock()). */ int sqlite3WalFrames( Log *pLog, /* Log handle to write to */ diff --git a/src/wal.h b/src/wal.h index c53e41d5ea..64067ce09b 100644 --- a/src/wal.h +++ b/src/wal.h @@ -40,6 +40,9 @@ int sqlite3WalWriteLock(Log *pLog, int op); /* Undo any frames written (but not committed) to the log */ int sqlite3WalUndo(Log *pLog, int (*xUndo)(void *, Pgno), void *pUndoCtx); +u32 sqlite3WalSavepoint(Log *pLog); +int sqlite3WalSavepointUndo(Log *pLog, u32 iFrame); + /* Return true if data has been written but not committed to the log file. */ int sqlite3WalDirty(Log *pLog); diff --git a/test/permutations.test b/test/permutations.test index f11bb661e5..e4c47a3650 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -761,6 +761,7 @@ run_tests "wal" -description { } -include { savepoint.test savepoint2.test + savepoint6.test } # End of tests diff --git a/test/savepoint6.test b/test/savepoint6.test index 33758b8648..ae866e215a 100644 --- a/test/savepoint6.test +++ b/test/savepoint6.test @@ -222,6 +222,7 @@ foreach zSetup [list { set testname normal sqlite3 db test.db } { + if {[wal_is_wal_mode]} continue set testname tempdb sqlite3 db "" } { @@ -241,10 +242,12 @@ foreach zSetup [list { unset -nocomplain ::aEntry catch { db close } - file delete -force test.db + file delete -force test.db test.db-wal test.db-journal eval $zSetup sql $DATABASE_SCHEMA + wal_set_journal_mode + do_test savepoint6-$testname.setup { savepoint one insert_rows [random_integers 100 1000] @@ -252,7 +255,7 @@ foreach zSetup [list { checkdb } {ok} - for {set i 0} {$i < 1000} {incr i} { + for {set i 0} {$i < 50} {incr i} { do_test savepoint6-$testname.$i.1 { savepoint_op checkdb @@ -264,6 +267,8 @@ foreach zSetup [list { checkdb } {ok} } + + wal_check_journal_mode savepoint6-$testname.walok } unset -nocomplain ::lSavepoint diff --git a/test/wal.test b/test/wal.test index 1398ffd3ac..d533f9e0c7 100644 --- a/test/wal.test +++ b/test/wal.test @@ -166,13 +166,13 @@ do_test wal-4.3 { } } {a b} -do_test wal-4.4 { +do_test wal-4.4.1 { db close sqlite3 db test.db db func blob blob list [execsql { SELECT * FROM t1 }] [file size test.db-wal] } {{a b} 0} -do_test wal-4.5 { +do_test wal-4.4.2 { execsql { PRAGMA cache_size = 10 } execsql { CREATE TABLE t2(a, b); @@ -191,10 +191,10 @@ do_test wal-4.5 { SELECT count(*) FROM t2; } } {32} -do_test wal-4.6 { +do_test wal-4.4.3 { execsql { ROLLBACK TO tr } } {} -do_test wal-4.7 { +do_test wal-4.4.4 { set logsize [file size test.db-wal] execsql { INSERT INTO t1 VALUES('x', 'y'); @@ -202,20 +202,82 @@ do_test wal-4.7 { } expr { $logsize == [file size test.db-wal] } } {1} -do_test wal-4.8 { +do_test wal-4.4.5 { execsql { SELECT count(*) FROM t2 } } {1} -do_test wal-4.9 { +do_test wal-4.4.6 { file copy -force test.db test2.db file copy -force test.db-wal test2.db-wal sqlite3 db2 test2.db execsql { SELECT count(*) FROM t2 ; SELECT count(*) FROM t1 } db2 } {1 2} -do_test wal-4.10 { +do_test wal-4.4.7 { execsql { PRAGMA integrity_check } db2 } {ok} db2 close +do_test wal-4.5.1 { + reopen_db + db func blob blob + execsql { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES('a', 'b'); + } + sqlite3 db test.db + db func blob blob + list [execsql { SELECT * FROM t1 }] [file size test.db-wal] +} {{a b} 0} +do_test wal-4.5.2 { + execsql { PRAGMA cache_size = 10 } + execsql { + CREATE TABLE t2(a, b); + BEGIN; + INSERT INTO t2 VALUES(blob(400), blob(400)); + SAVEPOINT tr; + INSERT INTO t2 SELECT blob(400), blob(400) FROM t2; /* 2 */ + INSERT INTO t2 SELECT blob(400), blob(400) FROM t2; /* 4 */ + INSERT INTO t2 SELECT blob(400), blob(400) FROM t2; /* 8 */ + INSERT INTO t2 SELECT blob(400), blob(400) FROM t2; /* 16 */ + INSERT INTO t2 SELECT blob(400), blob(400) FROM t2; /* 32 */ + INSERT INTO t1 SELECT blob(400), blob(400) FROM t1; /* 2 */ + INSERT INTO t1 SELECT blob(400), blob(400) FROM t1; /* 4 */ + INSERT INTO t1 SELECT blob(400), blob(400) FROM t1; /* 8 */ + INSERT INTO t1 SELECT blob(400), blob(400) FROM t1; /* 16 */ + INSERT INTO t1 SELECT blob(400), blob(400) FROM t1; /* 32 */ + SELECT count(*) FROM t2; + } +} {32} +do_test wal-4.5.3 { +breakpoint + execsql { ROLLBACK TO tr } +} {} +do_test wal-4.5.4 { + set logsize [file size test.db-wal] +breakpoint + execsql { + INSERT INTO t1 VALUES('x', 'y'); + RELEASE tr; + COMMIT; + } + expr { $logsize == [file size test.db-wal] } +} {1} +do_test wal-4.5.5 { + execsql { SELECT count(*) FROM t2 ; SELECT count(*) FROM t1 } +} {1 2} +do_test wal-4.5.6 { + file copy -force test.db test2.db + file copy -force test.db-wal test2.db-wal + sqlite3 db2 test2.db +breakpoint + execsql { SELECT count(*) FROM t2 ; SELECT count(*) FROM t1 } db2 +} {1 2} +do_test wal-4.5.7 { + execsql { PRAGMA integrity_check } db2 +} {ok} +db2 close + + reopen_db do_test wal-5.1 { execsql { @@ -680,7 +742,7 @@ do_test wal-11.10 { SELECT count(*) FROM t1; } list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [log_file_size 35 1024]] +} [list 37 [log_file_size 37 1024]] do_test wal-11.11 { execsql { SELECT count(*) FROM t1; @@ -690,7 +752,7 @@ do_test wal-11.11 { } {32 16} do_test wal-11.12 { list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [log_file_size 35 1024]] +} [list 37 [log_file_size 37 1024]] do_test wal-11.13 { execsql { INSERT INTO t1 VALUES( blob(900) ); @@ -700,7 +762,7 @@ do_test wal-11.13 { } {17 ok} do_test wal-11.14 { list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [log_file_size 35 1024]] +} [list 37 [log_file_size 37 1024]] #------------------------------------------------------------------------- From e180c296ea55a4a34814010761965b2c5d056862 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 26 Apr 2010 17:42:56 +0000 Subject: [PATCH 40/89] Do not attempt to set journal_mode=wal on :memory: or temp file databases. FossilOrigin-Name: 30d0134454734fd9335896372c30a903e9adcc64 --- manifest | 16 ++++---- manifest.uuid | 2 +- src/vdbe.c | 6 +++ test/savepoint6.test | 4 +- test/walmode.test | 94 +++++++++++++++++++++++++++++++++++++++++++- 5 files changed, 110 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 9451cdd82f..133446facb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sfor\sproblems\swith\ssmall\scaches\sand\sSAVEPOINT\srollback\sin\sWAL\smode. -D 2010-04-26T16:57:10 +C Do\snot\sattempt\sto\sset\sjournal_mode=wal\son\s:memory:\sor\stemp\sfile\sdatabases. +D 2010-04-26T17:42:56 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -212,7 +212,7 @@ F src/update.c c0dc6b75ad28b76b619042d934f337b02acee208 F src/utf.c 1baeeac91707a4df97ccc6141ec0f808278af685 F src/util.c 32aebf04c10e51ad3977a928b7416bed671b620b F src/vacuum.c 8e7d9a6fd076774bb6e211cad335c7337948a164 -F src/vdbe.c 09e88e8b01b413271b5ccd8c902c424cd563e342 +F src/vdbe.c da7ee09fdb5ee6ecfac11b74086834ff6583cd23 F src/vdbe.h 471f6a3dcec4817ca33596fe7f6654d56c0e75f3 F src/vdbeInt.h 19ebc8c2a2e938340051ee65af3f377fb99102d1 F src/vdbeapi.c 466044df5bc916f778833e927165fd02cdef6086 @@ -553,7 +553,7 @@ F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint3.test e328085853b14898d78ceea00dfe7db18bb6a9ec F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd -F test/savepoint6.test 8e871226bcd20241c0ee7da28166f5c525d7cba9 +F test/savepoint6.test 30f5c068d7aceacbf9bd6a7df1c99f34b2fc2723 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/securedel.test 328d2921c0ca49bdd3352e516b0377fc07143254 @@ -762,7 +762,7 @@ F test/wal.test b5d5fe1b3171eefb25355758e0dc689096bc1f87 F test/walbak.test f6fde9a5f59d0c697cb1f4af7876178c2f69a7ba F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47 -F test/walmode.test c2f4e30ad64910b2d8faf6cf4e940b3f201b41df +F test/walmode.test 40119078da084e6a7403ba57485d5a86ee0e2646 F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 F test/walthread.test a7962d8b899366cc71f8381d9aeab8ae9e1b544d F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 205e5d8ac08f7d7853427b4cd235fca125155d2d -R f3447c8eaaa2559454ba677c09b3ac47 +P 6a944f028d4a070bef29e1fbc6fbef481ebcd34c +R 74f903b2272910b858a0c5c9e03f5db8 U dan -Z 1f4eebc47ae413fc016108c1a59ad1db +Z 9dd751e835cbcac366a5e64c32910528 diff --git a/manifest.uuid b/manifest.uuid index c549155246..59b15965e5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6a944f028d4a070bef29e1fbc6fbef481ebcd34c \ No newline at end of file +30d0134454734fd9335896372c30a903e9adcc64 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 2af5c7384c..a8eddf857f 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5231,6 +5231,12 @@ case OP_JournalMode: { pBt = db->aDb[pOp->p1].pBt; pPager = sqlite3BtreePager(pBt); + if( 0==sqlite3Strlen30(sqlite3BtreeGetFilename(pBt)) + && eNew==PAGER_JOURNALMODE_WAL + ){ + eNew = PAGER_JOURNALMODE_QUERY; + } + if( eNew!=PAGER_JOURNALMODE_QUERY ){ int eOld = sqlite3PagerJournalMode(pPager, PAGER_JOURNALMODE_QUERY); if( (eNew!=eOld) diff --git a/test/savepoint6.test b/test/savepoint6.test index ae866e215a..2b2bcc8e06 100644 --- a/test/savepoint6.test +++ b/test/savepoint6.test @@ -26,6 +26,8 @@ set DATABASE_SCHEMA { CREATE INDEX i2 ON t1(y); } +set ::ITERATIONS 1000 + #-------------------------------------------------------------------------- # In memory database state. # @@ -255,7 +257,7 @@ foreach zSetup [list { checkdb } {ok} - for {set i 0} {$i < 50} {incr i} { + for {set i 0} {$i < $::ITERATIONS} {incr i} { do_test savepoint6-$testname.$i.1 { savepoint_op checkdb diff --git a/test/walmode.test b/test/walmode.test index 91b5b92049..3af72d91c0 100644 --- a/test/walmode.test +++ b/test/walmode.test @@ -139,9 +139,99 @@ do_test walmode-4.10 { do_test walmode-4.11 { execsql { PRAGMA main.journal_mode } db } {delete} - - catch { db close } catch { db2 close } + +# Test that it is not possible to change a temporary or in-memory database +# to WAL mode. WAL mode is for persistent file-backed databases only. +# +# walmode-5.1.*: Try to set journal_mode=WAL on [sqlite3 db :memory:] database. +# walmode-5.2.*: Try to set journal_mode=WAL on [sqlite3 db ""] database. +# walmode-5.3.*: Try to set temp.journal_mode=WAL. +# +do_test walmode-5.1.1 { + sqlite3 db :memory: + execsql { PRAGMA main.journal_mode } +} {memory} +do_test walmode-5.1.2 { + execsql { PRAGMA main.journal_mode = wal } +} {memory} +do_test walmode-5.1.3 { + execsql { + BEGIN; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + COMMIT; + SELECT * FROM t1; + PRAGMA main.journal_mode; + } +} {1 2 memory} +do_test walmode-5.1.4 { + execsql { PRAGMA main.journal_mode = wal } +} {memory} +do_test walmode-5.1.5 { + execsql { + INSERT INTO t1 VALUES(3, 4); + SELECT * FROM t1; + PRAGMA main.journal_mode; + } +} {1 2 3 4 memory} + +do_test walmode-5.2.1 { + sqlite3 db "" + execsql { PRAGMA main.journal_mode } +} {delete} +do_test walmode-5.2.2 { + execsql { PRAGMA main.journal_mode = wal } +} {delete} +do_test walmode-5.2.3 { + execsql { + BEGIN; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + COMMIT; + SELECT * FROM t1; + PRAGMA main.journal_mode; + } +} {1 2 delete} +do_test walmode-5.2.4 { + execsql { PRAGMA main.journal_mode = wal } +} {delete} +do_test walmode-5.2.5 { + execsql { + INSERT INTO t1 VALUES(3, 4); + SELECT * FROM t1; + PRAGMA main.journal_mode; + } +} {1 2 3 4 delete} + +do_test walmode-5.3.1 { + sqlite3 db test.db + execsql { PRAGMA temp.journal_mode } +} {delete} +do_test walmode-5.3.2 { + execsql { PRAGMA temp.journal_mode = wal } +} {delete} +do_test walmode-5.3.3 { + execsql { + BEGIN; + CREATE TEMP TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + COMMIT; + SELECT * FROM t1; + PRAGMA temp.journal_mode; + } +} {1 2 delete} +do_test walmode-5.3.4 { + execsql { PRAGMA temp.journal_mode = wal } +} {delete} +do_test walmode-5.3.5 { + execsql { + INSERT INTO t1 VALUES(3, 4); + SELECT * FROM t1; + PRAGMA temp.journal_mode; + } +} {1 2 3 4 delete} + finish_test From 31f98fc8e41ae7f8bf0ced0f8572cfcd692e648b Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 27 Apr 2010 05:42:32 +0000 Subject: [PATCH 41/89] Fix a problem that occurs when one process causes the log-summary file to grow and then a second process attempts to read the database. FossilOrigin-Name: b51a5f8bc660616aa264025dd7ad4bdab458814b --- manifest | 34 ++++++++-------------- manifest.uuid | 2 +- src/wal.c | 46 +++++++++++++++++++++++------ test/lock2.test | 1 - test/lock_common.tcl | 1 + test/permutations.test | 4 +++ test/savepoint6.test | 6 ++-- test/wal.test | 66 ++++++++++++++++++++++++++++++++---------- 8 files changed, 110 insertions(+), 50 deletions(-) diff --git a/manifest b/manifest index 74ed04bf37..d538e7c77b 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,5 @@ ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA1 - -C Merge\sin\srecent\schanges\sfrom\sthe\strunk -D 2010-04-27T01:56:22 +C Fix\sa\sproblem\sthat\soccurs\swhen\sone\sprocess\scauses\sthe\slog-summary\sfile\sto\sgrow\sand\sthen\sa\ssecond\sprocess\sattempts\sto\sread\sthe\sdatabase. +D 2010-04-27T05:42:32 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -224,7 +221,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c ce03bdd759c7b949188c0b75c46777fe0f7f105e +F src/wal.c 26d2a7028cc4af995e657431e2dbcea0aac63784 F src/wal.h 812101dd76610401fbcd44114e7e8b7ce0224645 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 @@ -476,13 +473,13 @@ F test/limit.test 2db7b3b34fb925b8e847d583d2eb67531d0ce67e F test/loadext.test 0393ce12d9616aa87597dd0ec88181de181f6db0 F test/loadext2.test 0bcaeb4d81cd5b6e883fdfea3c1bdbe1f173cbca F test/lock.test 842e80b6be816c79525a20b098cca066989feed7 -F test/lock2.test ec208a5f394d92affaf599fde3f374361657d0ff +F test/lock2.test 64ba8f1c7709abf91287d564984a6b8c5209e755 F test/lock3.test f271375930711ae044080f4fe6d6eda930870d00 F test/lock4.test f4f36271aa5ae1da449646bf43c7341f6b2b4c4e F test/lock5.test 6b1f78f09ad1522843dad571b76b321e6f439bf7 F test/lock6.test 862aa71e97b288d6b3f92ba3313f51bd0b003776 F test/lock7.test 64006c84c1c616657e237c7ad6532b765611cf64 -F test/lock_common.tcl 58aa21f38c28223cc1107b5b2c9d7d61aa428e79 +F test/lock_common.tcl ebc5b9a238d6d9f08a8433e52bdbc67d1478e774 F test/lookaside.test 1dd350dc6dff015c47c07fcc5a727a72fc5bae02 F test/main.test 2be2352ac77ac5b238c6337a5469aeeef57677e6 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 @@ -535,7 +532,7 @@ F test/pageropt.test 3ee6578891baaca967f0bd349e4abfa736229e1a F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb F test/pcache.test eebc4420b37cb07733ae9b6e99c9da7c40dd6d58 F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16 -F test/permutations.test 0730dc6c028b620a15c38a0a7d07238da088ecd3 +F test/permutations.test 894a6df3c4394194f84feea8520580dad1b71280 F test/pragma.test 6960f9efbce476f70ba9ee2171daf5042f9e3d8a F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea @@ -556,7 +553,7 @@ F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint3.test e328085853b14898d78ceea00dfe7db18bb6a9ec F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd -F test/savepoint6.test 30f5c068d7aceacbf9bd6a7df1c99f34b2fc2723 +F test/savepoint6.test bdc653c5c08328c84a74471b5b23fd483b919dfe F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/securedel.test 328d2921c0ca49bdd3352e516b0377fc07143254 @@ -761,7 +758,7 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test b5d5fe1b3171eefb25355758e0dc689096bc1f87 +F test/wal.test fbf7a911b35eb9b09a84eee4020e9e4c8b148d6f F test/walbak.test f6fde9a5f59d0c697cb1f4af7876178c2f69a7ba F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47 @@ -811,14 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 30d0134454734fd9335896372c30a903e9adcc64 ca9d86baf70f210d331ce93102177c8005c494cb -R b0a15978d17250dbff5ca26f38578573 -U drh -Z 50883e064ff8d31e35ff6084caafc75f ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.6 (GNU/Linux) - -iD8DBQFL1kRKoxKgR168RlERAhGtAJ0ZNNVfZUQ2rD2DR5znbRsfpO0FrQCeMGjZ -bgsVZnQebWJklqCayCQA1Cc= -=MFJQ ------END PGP SIGNATURE----- +P 7a0ac682c3bffcb345321fe97434563350ac90f9 +R 8f311e98676985c9f340b1e7e337c8e4 +U dan +Z 70973fdb159f8a0ed7dee7d674e22de0 diff --git a/manifest.uuid b/manifest.uuid index e84e36275c..f9a7784eff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7a0ac682c3bffcb345321fe97434563350ac90f9 \ No newline at end of file +b51a5f8bc660616aa264025dd7ad4bdab458814b \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 8c6878a1a4..14ee78d9f1 100644 --- a/src/wal.c +++ b/src/wal.c @@ -592,6 +592,22 @@ static int logSummaryMap(LogSummary *pSummary, int nByte){ return SQLITE_OK; } +/* +** The log-summary file is already mapped to pSummary->aData[], but the +** mapping needs to be resized. Unmap and remap the file so that the mapping +** is at least nByte bytes in size, or the size of the entire file if it +** is larger than nByte bytes. +*/ +static int logSummaryRemap(LogSummary *pSummary, int nByte){ + int rc; + sqlite3_mutex_enter(pSummary->mutex); + munmap(pSummary->aData, pSummary->nData*4); + pSummary->aData = 0; + rc = logSummaryMap(pSummary, nByte); + sqlite3_mutex_leave(pSummary->mutex); + return rc; +} + /* ** Return the index in the LogSummary.aData array that corresponds to ** frame iFrame. The log-summary file consists of a header, followed by @@ -613,18 +629,18 @@ static int logSummaryEntry(u32 iFrame){ ** the previous call), but that restriction is not enforced or asserted ** here. */ -static void logSummaryAppend(LogSummary *pSummary, u32 iFrame, u32 iPage){ +static int logSummaryAppend(LogSummary *pSummary, u32 iFrame, u32 iPage){ u32 iSlot = logSummaryEntry(iFrame); while( (iSlot+128)>=pSummary->nData ){ + int rc; int nByte = pSummary->nData*4 + LOGSUMMARY_MMAP_INCREMENT; /* Unmap and remap the log-summary file. */ - sqlite3_mutex_enter(pSummary->mutex); - munmap(pSummary->aData, pSummary->nData*4); - pSummary->aData = 0; - logSummaryMap(pSummary, nByte); - sqlite3_mutex_leave(pSummary->mutex); + rc = logSummaryRemap(pSummary, nByte); + if( rc!=SQLITE_OK ){ + return rc; + } } /* Set the log-summary entry itself */ @@ -649,6 +665,8 @@ static void logSummaryAppend(LogSummary *pSummary, u32 iFrame, u32 iPage){ logMergesort8(aFrame, aTmp, aIndex, &nIndex); memset(&aIndex[nIndex], aIndex[nIndex-1], 256-nIndex); } + + return SQLITE_OK; } @@ -1496,6 +1514,16 @@ int sqlite3WalOpenSnapshot(Log *pLog, int *pChanged){ if( rc!=SQLITE_OK ){ /* An error occured while attempting log recovery. */ sqlite3WalCloseSnapshot(pLog); + }else{ + /* Check if the mapping needs to grow. */ + LogSummary *pSummary = pLog->pSummary; + + if( pLog->hdr.iLastPg + && logSummaryEntry(pLog->hdr.iLastPg)>=pSummary->nData + ){ + rc = logSummaryRemap(pSummary, 0); + assert( rc || logSummaryEntry(pLog->hdr.iLastPg)nData ); + } } } return rc; @@ -1516,20 +1544,20 @@ void sqlite3WalCloseSnapshot(Log *pLog){ ** Read a page from the log, if it is present. */ int sqlite3WalRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut){ + LogSummary *pSummary = pLog->pSummary; u32 iRead = 0; u32 *aData; int iFrame = (pLog->hdr.iLastPg & 0xFFFFFF00); assert( pLog->isLocked ); - - sqlite3_mutex_enter(pLog->pSummary->mutex); - aData = pLog->pSummary->aData; + sqlite3_mutex_enter(pSummary->mutex); /* Do a linear search of the unindexed block of page-numbers (if any) ** at the end of the log-summary. An alternative to this would be to ** build an index in private memory each time a read transaction is ** opened on a new snapshot. */ + aData = pSummary->aData; if( pLog->hdr.iLastPg ){ u32 *pi = &aData[logSummaryEntry(pLog->hdr.iLastPg)]; u32 *piStop = pi - (pLog->hdr.iLastPg & 0xFF); diff --git a/test/lock2.test b/test/lock2.test index 63319535d1..e4c5204714 100644 --- a/test/lock2.test +++ b/test/lock2.test @@ -34,7 +34,6 @@ source $testdir/lock_common.tcl # do_test lock2-1.1 { set ::tf1 [launch_testfixture] - testfixture $::tf1 "sqlite3_test_control_pending_byte $::sqlite_pending_byte" testfixture $::tf1 { sqlite3 db test.db -key xyzzy db eval {select * from sqlite_master} diff --git a/test/lock_common.tcl b/test/lock_common.tcl index 31c04e8535..a17de8a3da 100644 --- a/test/lock_common.tcl +++ b/test/lock_common.tcl @@ -24,6 +24,7 @@ proc launch_testfixture {} { } set chan [open "|$prg tf_main.tcl" r+] fconfigure $chan -buffering line + testfixture $chan "sqlite3_test_control_pending_byte $::sqlite_pending_byte" return $chan } diff --git a/test/permutations.test b/test/permutations.test index e4c47a3650..f51cc84422 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -758,6 +758,10 @@ if {[info commands register_demovfs] != ""} { run_tests "wal" -description { Run tests with journal_mode=WAL +} -initialize { + set ::savepoint6_iterations 100 +} -shutdown { + unset -nocomplain ::savepoint6_iterations } -include { savepoint.test savepoint2.test diff --git a/test/savepoint6.test b/test/savepoint6.test index 2b2bcc8e06..7f60769c63 100644 --- a/test/savepoint6.test +++ b/test/savepoint6.test @@ -26,7 +26,9 @@ set DATABASE_SCHEMA { CREATE INDEX i2 ON t1(y); } -set ::ITERATIONS 1000 +if {0==[info exists ::savepoint6_iterations]} { + set ::savepoint6_iterations 1000 +} #-------------------------------------------------------------------------- # In memory database state. @@ -257,7 +259,7 @@ foreach zSetup [list { checkdb } {ok} - for {set i 0} {$i < $::ITERATIONS} {incr i} { + for {set i 0} {$i < $::savepoint6_iterations} {incr i} { do_test savepoint6-$testname.$i.1 { savepoint_op checkdb diff --git a/test/wal.test b/test/wal.test index d533f9e0c7..30586d5cb3 100644 --- a/test/wal.test +++ b/test/wal.test @@ -19,7 +19,7 @@ source $testdir/lock_common.tcl proc reopen_db {} { catch { db close } - file delete -force test.db test.db-wal + file delete -force test.db test.db-wal test.db-wal-summary sqlite3_wal db test.db } @@ -249,12 +249,10 @@ do_test wal-4.5.2 { } } {32} do_test wal-4.5.3 { -breakpoint execsql { ROLLBACK TO tr } } {} do_test wal-4.5.4 { set logsize [file size test.db-wal] -breakpoint execsql { INSERT INTO t1 VALUES('x', 'y'); RELEASE tr; @@ -269,7 +267,6 @@ do_test wal-4.5.6 { file copy -force test.db test2.db file copy -force test.db-wal test2.db-wal sqlite3 db2 test2.db -breakpoint execsql { SELECT count(*) FROM t2 ; SELECT count(*) FROM t1 } db2 } {1 2} do_test wal-4.5.7 { @@ -823,10 +820,11 @@ db close #------------------------------------------------------------------------- # Test large log summaries. # -do_test wal-13.1 { +set sqlite_walsummary_mmap_incr 64 +do_test wal-13.1.1 { list [file exists test.db] [file exists test.db-wal] } {1 0} -do_test wal-13.2 { +do_test wal-13.1.2 { set fd [open test.db-wal w] seek $fd [expr 200*1024*1024] puts $fd "" @@ -834,25 +832,63 @@ do_test wal-13.2 { sqlite3 db test.db execsql { SELECT * FROM t2 } } {B 2} -do_test wal-13.3 { +do_test wal-13.1.3 { db close file exists test.db-wal } {0} -do_test wal-13.4 { +do_test wal-13.1.4 { sqlite3 db test.db execsql { SELECT count(*) FROM t2 } } {1} -do_test wal-13.5 { - for {set i 0} {$i < 15} {incr i} { +do_test wal-13.1.5 { + for {set i 0} {$i < 6} {incr i} { execsql { INSERT INTO t2 SELECT randomblob(400), randomblob(400) FROM t2 } } execsql { SELECT count(*) FROM t2 } -} [expr int(pow(2, 15))] -do_test wal-13.6 { - set sz [file size test.db-wal-summary] - expr {$sz<=(3*64*1024) && $sz>(2*64*1024)} -} {1} +} [expr int(pow(2, 6))] +do_test wal-13.1.6 { + file size test.db-wal +} [log_file_size 80 1024] +foreach code [list { + set tn 2 + proc buddy {tcl} { uplevel #0 $tcl } +} { + set tn 3 + set ::buddy [launch_testfixture] + proc buddy {tcl} { testfixture $::buddy $tcl } +}] { + + eval $code + reopen_db + + do_test wal-13.$tn.0 { + buddy { sqlite3 db2 test.db } + execsql { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(x); + INSERT INTO t1 SELECT randomblob(400); + } + execsql { SELECT count(*) FROM t1 } + } {1} + + for {set ii 1} {$ii<16} {incr ii} { + do_test wal-13.$tn.$ii.a { + buddy { db2 eval { INSERT INTO t1 SELECT randomblob(400) FROM t1 } } + buddy { db2 eval { SELECT count(*) FROM t1 } } + } [expr (1<<$ii)] + do_test wal-13.$tn.$ii.b { + db eval { SELECT count(*) FROM t1 } + } [expr (1<<$ii)] + do_test wal-13.$tn.$ii.c { + db eval { SELECT count(*) FROM t1 } + } [expr (1<<$ii)] + } + + catch { db2 close } + catch { close $::buddy } + db close +} finish_test From e106de6317e46efdde1b0d38c2b66f16768706ce Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 27 Apr 2010 06:49:34 +0000 Subject: [PATCH 42/89] Run trans.test and avtrans.test as part of the "wal" permutation. FossilOrigin-Name: c8893310510148f7e71ede5691606b6fc1c3df4d --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- test/avtrans.test | 9 +++++++-- test/permutations.test | 5 ++--- test/trans.test | 12 +++++++++++- 5 files changed, 29 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index d538e7c77b..d9cabb3bc1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sthat\soccurs\swhen\sone\sprocess\scauses\sthe\slog-summary\sfile\sto\sgrow\sand\sthen\sa\ssecond\sprocess\sattempts\sto\sread\sthe\sdatabase. -D 2010-04-27T05:42:32 +C Run\strans.test\sand\savtrans.test\sas\spart\sof\sthe\s"wal"\spermutation. +D 2010-04-27T06:49:34 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -252,7 +252,7 @@ F test/autoinc.test 85ef3180a737e6580086a018c09c6f1a52759b46 F test/autoindex1.test ffb06a246e2c1f89cfbe3d93eca513c9e78d4063 F test/autovacuum.test 25f891bc343a8bf5d9229e2e9ddab9f31a9ab5ec F test/autovacuum_ioerr2.test 598b0663074d3673a9c1bc9a16e80971313bafe6 -F test/avtrans.test 1e901d8102706b63534dbd2bdd4d8f16c4082650 +F test/avtrans.test 7a26ffc6a355b7fbff2a082f56180ede863b126c F test/backup.test b1e874fa9b01de9dd5137a8371d060b76a435162 F test/backup2.test 159419073d9769fdb1780ed7e5b391a046f898d5 F test/backup_ioerr.test 1f012e692f42c0442ae652443258f70e9f20fa38 @@ -532,7 +532,7 @@ F test/pageropt.test 3ee6578891baaca967f0bd349e4abfa736229e1a F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb F test/pcache.test eebc4420b37cb07733ae9b6e99c9da7c40dd6d58 F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16 -F test/permutations.test 894a6df3c4394194f84feea8520580dad1b71280 +F test/permutations.test 7d9064978f0ca90d0ec827a7ada9e8e2615b1143 F test/pragma.test 6960f9efbce476f70ba9ee2171daf5042f9e3d8a F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea @@ -712,7 +712,7 @@ F test/tkt3997.test a335fa41ca3985660a139df7b734a26ef53284bd F test/tkt4018.test 7c2c9ba4df489c676a0a7a0e809a1fb9b2185bd1 F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7 F test/trace.test 4b36a41a3e9c7842151af6da5998f5080cdad9e5 -F test/trans.test d887cb07630dc39879a322d958ad8b006137485c +F test/trans.test f19bd2c8b5ccc89295454a42b35319b1381b9a3d F test/trans2.test d5337e61de45e66b1fcbf9db833fa8c82e624b22 F test/trans3.test d728abaa318ca364dc370e06576aa7e5fbed7e97 F test/trigger1.test 2e18561f85e448bb633c9c9de792e9bbf7b2dd3e @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 7a0ac682c3bffcb345321fe97434563350ac90f9 -R 8f311e98676985c9f340b1e7e337c8e4 +P b51a5f8bc660616aa264025dd7ad4bdab458814b +R 25f06e3e98c7ea6b1eb255e6fd74be59 U dan -Z 70973fdb159f8a0ed7dee7d674e22de0 +Z 9cceaabbad6baa9da4fc7d1b5b1a7fe5 diff --git a/manifest.uuid b/manifest.uuid index f9a7784eff..dc61bc6fbd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b51a5f8bc660616aa264025dd7ad4bdab458814b \ No newline at end of file +c8893310510148f7e71ede5691606b6fc1c3df4d \ No newline at end of file diff --git a/test/avtrans.test b/test/avtrans.test index 328e028c09..4a3514d2b9 100644 --- a/test/avtrans.test +++ b/test/avtrans.test @@ -22,8 +22,9 @@ source $testdir/tester.tcl # Create several tables to work with. # do_test avtrans-1.0 { - execsql { - PRAGMA auto_vacuum=ON; + execsql { PRAGMA auto_vacuum=ON } + wal_set_journal_mode + execsql { CREATE TABLE one(a int PRIMARY KEY, b text); INSERT INTO one VALUES(1,'one'); INSERT INTO one VALUES(2,'two'); @@ -48,6 +49,7 @@ do_test avtrans-1.10 { execsql {SELECT b FROM two ORDER BY a} altdb } {I V X} integrity_check avtrans-1.11 +wal_check_journal_mode avtrans-1.12 # Basic transactions # @@ -84,6 +86,7 @@ do_test avtrans-2.10 { } } {1 2 3 1 5 10} integrity_check avtrans-2.11 +wal_check_journal_mode avtrans-2.12 # Check the locking behavior # @@ -913,9 +916,11 @@ for {set i 2} {$i<=$limit} {incr i} { } {1} } } + wal_check_journal_mode avtrans-9.$i-6.$cnt } set ::pager_old_format 0 } integrity_check avtrans-10.1 +wal_check_journal_mode avtrans-10.2 finish_test diff --git a/test/permutations.test b/test/permutations.test index f51cc84422..daf2c72bcb 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -763,9 +763,8 @@ run_tests "wal" -description { } -shutdown { unset -nocomplain ::savepoint6_iterations } -include { - savepoint.test - savepoint2.test - savepoint6.test + savepoint.test savepoint2.test savepoint6.test + trans.test avtrans.test } # End of tests diff --git a/test/trans.test b/test/trans.test index 1965941f8a..cace1218a6 100644 --- a/test/trans.test +++ b/test/trans.test @@ -19,6 +19,7 @@ source $testdir/tester.tcl # Create several tables to work with. # +wal_set_journal_mode do_test trans-1.0 { execsql { CREATE TABLE one(a int PRIMARY KEY, b text); @@ -46,6 +47,7 @@ do_test trans-1.10 { execsql {SELECT b FROM two ORDER BY a} altdb } {I V X} integrity_check trans-1.11 +wal_check_journal_mode trans-1.12 # Basic transactions # @@ -82,6 +84,7 @@ do_test trans-2.10 { } } {1 2 3 1 5 10} integrity_check trans-2.11 +wal_check_journal_mode trans-2.12 # Check the locking behavior # @@ -162,6 +165,7 @@ do_test trans-3.14 { lappend v $msg } {0 {1 2 3 4}} integrity_check trans-3.15 +wal_check_journal_mode trans-3.16 do_test trans-4.1 { set v [catch {execsql { @@ -228,6 +232,8 @@ do_test trans-4.11 { lappend v $msg } {0 {1 2 3 4}} integrity_check trans-4.12 +wal_check_journal_mode trans-4.13 +wal_check_journal_mode trans-4.14 altdb do_test trans-4.98 { altdb close execsql { @@ -775,6 +781,7 @@ do_test trans-7.14 { execsql {SELECT md5sum(type,name,tbl_name,rootpage,sql) FROM sqlite_master} } $checksum2 integrity_check trans-7.15 +wal_check_journal_mode trans-7.16 # Arrange for another process to begin modifying the database but abort # and die in the middle of the modification. Then have this process read @@ -824,7 +831,7 @@ do_test trans-8.5 { execsql {SELECT md5sum(type,name,tbl_name,rootpage,sql) FROM sqlite_master} } $checksum2 integrity_check trans-8.6 - +wal_check_journal_mode trans-8.7 # In the following sequence of tests, compute the MD5 sum of the content # of a table, make lots of modifications to that table, then do a rollback. @@ -854,6 +861,7 @@ do_test trans-9.1 { SELECT count(*) FROM t3; } } {1024} +wal_check_journal_mode trans-9.1.1 # The following procedure computes a "signature" for table "t3". If # T3 changes in any way, the signature should change. @@ -940,6 +948,8 @@ for {set i 2} {$i<=$limit} {incr i} { } } } + + wal_check_journal_mode trans-9.$i.6-$cnt set ::pager_old_format 0 } From af75c8695bd53883e57affaf6eac7c50743d4f2a Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 27 Apr 2010 11:49:27 +0000 Subject: [PATCH 43/89] Add the xShmRelease() method to the VFS. The os_unix.c implementation of the shared memory logic is still non-functional. FossilOrigin-Name: ed715b47c5f7657fbf901805981867898054b14d --- manifest | 40 +++++++++++++++++++++++++--------------- manifest.uuid | 2 +- src/os_os2.c | 11 +---------- src/os_unix.c | 18 ++++++++++++++++-- src/os_win.c | 11 +---------- src/sqlite.h.in | 3 ++- src/test6.c | 9 --------- src/test_demovfs.c | 9 --------- src/test_devsym.c | 9 --------- src/test_journal.c | 9 --------- src/test_osinst.c | 9 --------- 11 files changed, 46 insertions(+), 84 deletions(-) diff --git a/manifest b/manifest index d9cabb3bc1..68088c71ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,8 @@ -C Run\strans.test\sand\savtrans.test\sas\spart\sof\sthe\s"wal"\spermutation. -D 2010-04-27T06:49:34 +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 + +C Add\sthe\sxShmRelease()\smethod\sto\sthe\sVFS.\s\sThe\sos_unix.c\simplementation\sof\nthe\sshared\smemory\slogic\sis\sstill\snon-functional. +D 2010-04-27T11:49:27 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -149,9 +152,9 @@ F src/notify.c cbfa66a836da3a51567209636e6a94059c137930 F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 -F src/os_os2.c 082884dc2a20a2f1fddc404f3606ec2abe907b56 -F src/os_unix.c 1acc854259b045cf029c6f95828a5f4b84f361c7 -F src/os_win.c 607b40e40662c6c74ff2400ff03e7f42521d5c75 +F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 +F src/os_unix.c 332ff9185aedcbd25568a06ddbdb51f5496b9756 +F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 F src/pager.c b4a41030860229e80295fa1f37addab24d21799c F src/pager.h cee4487ab4f0911dd9f22a40e3cd55afdb7ef444 F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e @@ -166,7 +169,7 @@ F src/resolve.c ac5f1a713cd1ae77f08b83cc69581e11bf5ae6f9 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c c03d8a0565febcde8c6a12c5d77d065fddae889b F src/shell.c c40427c7245535a04a9cb4a417b6cc05c022e6a4 -F src/sqlite.h.in 96b0bed7825da354119a9742a83033087c537133 +F src/sqlite.h.in f9399ee58862fd146eff9725dda83b5c82d04a93 F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89 F src/sqliteInt.h 700a2df7b8dfe57c3b8d83c52ff40928e026220c F src/sqliteLimit.h 3afab2291762b5d09ae20c18feb8e9fa935a60a6 @@ -178,7 +181,7 @@ F src/test2.c b6b43413d495addd039a88b87d65c839f86b18cb F src/test3.c 4c21700c73a890a47fc685c1097bfb661346ac94 F src/test4.c ad03bb987ddedce928f4258c1e7fa4109a73497d F src/test5.c cc55900118fa4add8ec9cf69fc4225a4662f76b1 -F src/test6.c d23fc84954751735bcb1050d4606b00952eb7809 +F src/test6.c 49b39daf8e88ed6baab25c524ca5635304f6f315 F src/test7.c 3f2d63e4ccf97f8c2cf1a7fa0a3c8e2e2a354e6e F src/test8.c f959db9a22d882013b64c92753fa793b2ce3bdea F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 @@ -187,19 +190,19 @@ F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de F src/test_btree.c 47cd771250f09cdc6e12dda5bc71bc0b3abc96e2 F src/test_config.c 8c408fbffbe8082d1d3dc55044ddfd5580f3c9b9 -F src/test_demovfs.c 68f8e6cfd6b337612e26b7027f97bf7ea02d4e04 -F src/test_devsym.c 76f47435769168fb665828f5cc24a4a0a130c302 +F src/test_demovfs.c da81a5f7785bb352bda7911c332a983ec4f17f27 +F src/test_devsym.c 361d069bf74d4044bf53eb29dc0fa3709ed98baf F src/test_func.c 13b582345fb1185a93e46c53310fae8547dcce20 F src/test_hexio.c 1237f000ec7a491009b1233f5c626ea71bce1ea2 F src/test_init.c 5d624ffd0409d424cf9adbfe1f056b200270077c F src/test_intarray.c d879bbf8e4ce085ab966d1f3c896a7c8b4f5fc99 F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 -F src/test_journal.c 6f2db1b847929791242e32323721c48c7d2d6e0c +F src/test_journal.c 0537fcf9d4ecb394ec8b27d887a004ecaab85f16 F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 2842c922b8e8d992aba722214952204ca025b411 F src/test_mutex.c ce06b59aca168cd8c520b77159a24352a7469bd3 F src/test_onefile.c 1d098e7446d89087def7d20e0cdb77a0713ed1bf -F src/test_osinst.c 79a9b9fdb87fccd7fdc2f523bfbb91f964bafa68 +F src/test_osinst.c f5d1a4ee8b80fc58d1430c56146de748584013a9 F src/test_pcache.c 7bf828972ac0d2403f5cfa4cd14da41f8ebe73d8 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c bbba05c144b5fc4b52ff650a4328027b3fa5fcc6 @@ -808,7 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P b51a5f8bc660616aa264025dd7ad4bdab458814b -R 25f06e3e98c7ea6b1eb255e6fd74be59 -U dan -Z 9cceaabbad6baa9da4fc7d1b5b1a7fe5 +P c8893310510148f7e71ede5691606b6fc1c3df4d +R fb13be4dd0d753be93180ef3cbf1d215 +U drh +Z 6f72fe2227e763bb3a1c4d06bd5a901c +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.6 (GNU/Linux) + +iD8DBQFL1s9KoxKgR168RlERAqqPAJ0SGEtAEImz4CyMiguEDpkB2AslLQCfd9o1 +tWw2lhvU6jis7pTQTOVY1gE= +=4LgW +-----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index dc61bc6fbd..2c39d62be3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c8893310510148f7e71ede5691606b6fc1c3df4d \ No newline at end of file +ed715b47c5f7657fbf901805981867898054b14d \ No newline at end of file diff --git a/src/os_os2.c b/src/os_os2.c index 04b6dce0e9..1c55e4be66 100644 --- a/src/os_os2.c +++ b/src/os_os2.c @@ -1111,16 +1111,7 @@ int sqlite3_os_init(void){ os2Randomness, /* xRandomness */ os2Sleep, /* xSleep */ os2CurrentTime, /* xCurrentTime */ - os2GetLastError /* xGetLastError */ - 0, /* xShmOpen */ - 0, /* xShmSize */ - 0, /* xShmPush */ - 0, /* xShmPull */ - 0, /* xShmLock */ - 0, /* xShmClose */ - 0, /* xShmDelete */ - 0, /* xRename */ - 0, /* xCurrentTimeInt64 */ + os2GetLastError, /* xGetLastError */ }; sqlite3_vfs_register(&os2Vfs, 1); initUconvObjects(); diff --git a/src/os_unix.c b/src/os_unix.c index b2cf9be76d..d603caa0d0 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4647,8 +4647,13 @@ shm_open_err: ** Query and/or changes the size of a shared-memory segment. ** The reqSize parameter is the new size of the segment, or -1 to ** do just a query. The size of the segment after resizing is -** written into pNewSize. The start of the shared memory buffer -** is stored in **ppBuffer. +** written into pNewSize. A writer lock is held on the shared memory +** segment while resizing it. +** +** If ppBuffer is not NULL, the a reader lock is acquired no the shared +** memory segment and *ppBuffer is made to point to the start of the +** shared memory segment. xShmRelease() must be called to release the +** lock. */ static int unixShmSize( sqlite3_shm *pSharedMem, /* Pointer returned by unixShmOpen() */ @@ -4679,6 +4684,14 @@ static int unixShmSize( return rc; } +/* +** Release the lock held on the shared memory segment to that other +** threads are free to resize it if necessary. +*/ +static int unixShmRelease(sqlite3_shm *pSharedMem){ + return SQLITE_OK; +} + /* ** Create or release a lock on shared memory. */ @@ -5934,6 +5947,7 @@ int sqlite3_os_init(void){ unixGetLastError, /* xGetLastError */ \ unixShmOpen, /* xShmOpen */ \ unixShmSize, /* xShmSize */ \ + unixShmRelease, /* xShmRelease */ \ 0, /* xShmPush */ \ 0, /* xShmPull */ \ unixShmLock, /* xShmLock */ \ diff --git a/src/os_win.c b/src/os_win.c index 31c3cffce3..cfcf49e362 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1912,16 +1912,7 @@ int sqlite3_os_init(void){ winRandomness, /* xRandomness */ winSleep, /* xSleep */ winCurrentTime, /* xCurrentTime */ - winGetLastError /* xGetLastError */ - 0, /* xShmOpen */ - 0, /* xShmSize */ - 0, /* xShmPush */ - 0, /* xShmPull */ - 0, /* xShmLock */ - 0, /* xShmClose */ - 0, /* xShmDelete */ - 0, /* xRename */ - 0, /* xCurrentTimeInt64 */ + winGetLastError, /* xGetLastError */ }; sqlite3_vfs_register(&winVfs, 1); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index f673781e00..7bc7509447 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -844,7 +844,8 @@ struct sqlite3_vfs { ** definition. Those that follow are added in version 2 or later */ int (*xShmOpen)(sqlite3_vfs*, const char *zName, sqlite3_shm**); - int (*xShmSize)(sqlite3_shm*, int reqSize, int *pNewSize, char **); + int (*xShmSize)(sqlite3_shm*, int reqSize, int *pNewSize, char**); + int (*xShmRelease)(sqlite3_shm*); int (*xShmPush)(sqlite3_shm*); int (*xShmPull)(sqlite3_shm*); int (*xShmLock)(sqlite3_shm*, int lockType, int ofst, int nByte); diff --git a/src/test6.c b/src/test6.c index a6e9954517..3da9502fd7 100644 --- a/src/test6.c +++ b/src/test6.c @@ -782,15 +782,6 @@ static int crashEnableCmd( cfRandomness, /* xRandomness */ cfSleep, /* xSleep */ cfCurrentTime, /* xCurrentTime */ - 0, /* xShmOpen */ - 0, /* xShmSize */ - 0, /* xShmPush */ - 0, /* xShmPull */ - 0, /* xShmLock */ - 0, /* xShmClose */ - 0, /* xShmDelete */ - 0, /* xRename */ - 0 /* xCurrentTimeInt64 */ }; if( objc!=2 ){ diff --git a/src/test_demovfs.c b/src/test_demovfs.c index d91a389dbb..7cdae80670 100644 --- a/src/test_demovfs.c +++ b/src/test_demovfs.c @@ -624,15 +624,6 @@ sqlite3_vfs *sqlite3_demovfs(void){ demoRandomness, /* xRandomness */ demoSleep, /* xSleep */ demoCurrentTime, /* xCurrentTime */ - 0, /* xShmOpen */ - 0, /* xShmSize */ - 0, /* xShmPush */ - 0, /* xShmPull */ - 0, /* xShmLock */ - 0, /* xShmClose */ - 0, /* xShmDelete */ - 0, /* xRename */ - 0 /* xCurrentTimeInt64 */ }; return &demovfs; } diff --git a/src/test_devsym.c b/src/test_devsym.c index 4249573aa1..9f71538044 100644 --- a/src/test_devsym.c +++ b/src/test_devsym.c @@ -93,15 +93,6 @@ static sqlite3_vfs devsym_vfs = { devsymRandomness, /* xRandomness */ devsymSleep, /* xSleep */ devsymCurrentTime, /* xCurrentTime */ - 0, /* xShmOpen */ - 0, /* xShmSize */ - 0, /* xShmPush */ - 0, /* xShmPull */ - 0, /* xShmLock */ - 0, /* xShmClose */ - 0, /* xShmDelete */ - 0, /* xRename */ - 0 /* xCurrentTimeInt64 */ }; static sqlite3_io_methods devsym_io_methods = { diff --git a/src/test_journal.c b/src/test_journal.c index fcfe778d1b..020b41575d 100644 --- a/src/test_journal.c +++ b/src/test_journal.c @@ -180,15 +180,6 @@ static sqlite3_vfs jt_vfs = { jtRandomness, /* xRandomness */ jtSleep, /* xSleep */ jtCurrentTime, /* xCurrentTime */ - 0, /* xShmOpen */ - 0, /* xShmSize */ - 0, /* xShmPush */ - 0, /* xShmPull */ - 0, /* xShmLock */ - 0, /* xShmClose */ - 0, /* xShmDelete */ - 0, /* xRename */ - 0 /* xCurrentTimeInt64 */ }; static sqlite3_io_methods jt_io_methods = { diff --git a/src/test_osinst.c b/src/test_osinst.c index c6d16aa67b..365da38d86 100644 --- a/src/test_osinst.c +++ b/src/test_osinst.c @@ -204,15 +204,6 @@ static sqlite3_vfs inst_vfs = { instRandomness, /* xRandomness */ instSleep, /* xSleep */ instCurrentTime, /* xCurrentTime */ - 0, /* xShmOpen */ - 0, /* xShmSize */ - 0, /* xShmPush */ - 0, /* xShmPull */ - 0, /* xShmLock */ - 0, /* xShmClose */ - 0, /* xShmDelete */ - 0, /* xRename */ - 0 /* xCurrentTimeInt64 */ }; static sqlite3_io_methods inst_io_methods = { From 8b5444bfae0e0c8d35b6d58da509401e922592b0 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 27 Apr 2010 14:37:47 +0000 Subject: [PATCH 44/89] Fix a problem with identifying a corrupt database in WAL mode. FossilOrigin-Name: c8658da44671e9c43c90ee4857bbe82c55d40f25 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index d9cabb3bc1..5adacab52a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Run\strans.test\sand\savtrans.test\sas\spart\sof\sthe\s"wal"\spermutation. -D 2010-04-27T06:49:34 +C Fix\sa\sproblem\swith\sidentifying\sa\scorrupt\sdatabase\sin\sWAL\smode. +D 2010-04-27T14:37:48 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -109,7 +109,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c e86634da8c48357a759694c9c7c471125cd8d5a8 F src/bitvec.c 06ad2c36a9c3819c0b9cbffec7b15f58d5d834e0 F src/btmutex.c 96a12f50f7a17475155971a241d85ec5171573ff -F src/btree.c f87ed73ef82bccb1e5dd63d6a4919272d3354c8a +F src/btree.c dd26f3b5725ac3cbc765857a76523ac10dbfe761 F src/btree.h dd83041eda10c17daf023257c1fc883b5f71f85a F src/btreeInt.h b0c87f6725b06a0aa194a6d25d54b16ce9d6e291 F src/build.c 11100b66fb97638d2d874c1d34d8db90650bb1d7 @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P b51a5f8bc660616aa264025dd7ad4bdab458814b -R 25f06e3e98c7ea6b1eb255e6fd74be59 +P c8893310510148f7e71ede5691606b6fc1c3df4d +R f19cde4cc4d379e9990f915e41f267c6 U dan -Z 9cceaabbad6baa9da4fc7d1b5b1a7fe5 +Z cdf65bfac6e74e44b7b865cbe4482fc0 diff --git a/manifest.uuid b/manifest.uuid index dc61bc6fbd..6b66b26ebb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c8893310510148f7e71ede5691606b6fc1c3df4d \ No newline at end of file +c8658da44671e9c43c90ee4857bbe82c55d40f25 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 2fd3fedc1e..62a8410764 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2285,6 +2285,7 @@ static int lockBtree(BtShared *pBt){ releasePage(pPage1); return SQLITE_OK; } + rc = SQLITE_NOTADB; } /* The maximum embedded fraction must be exactly 25%. And the minimum From 8b348af860fd2eb123278886edce39bf5fcc0a79 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 27 Apr 2010 18:43:16 +0000 Subject: [PATCH 45/89] Add a test to check that a checkpointer releases its locks before returning, even if it has been prevented by a conflicting lock from checkpointing the database. Add a test to check that a checkpointer releases its locks before returning, even if it has been prevented by a conflicting lock from checkpointing the database. FossilOrigin-Name: be44349d2b790fb470fcc9fb6a12e8b1076dc645 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/wal.test | 28 ++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 5adacab52a..3b7f145e8b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sidentifying\sa\scorrupt\sdatabase\sin\sWAL\smode. -D 2010-04-27T14:37:48 +C Add\sa\stest\sto\scheck\sthat\sa\scheckpointer\sreleases\sits\slocks\sbefore\sreturning,\seven\sif\sit\shas\sbeen\sprevented\sby\sa\sconflicting\slock\sfrom\scheckpointing\sthe\sdatabase.\nAdd\sa\stest\sto\scheck\sthat\sa\scheckpointer\sreleases\sits\slocks\sbefore\sreturning,\seven\sif\sit\shas\sbeen\sprevented\sby\sa\sconflicting\slock\sfrom\scheckpointing\sthe\sdatabase. +D 2010-04-27T18:43:17 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -758,7 +758,7 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test fbf7a911b35eb9b09a84eee4020e9e4c8b148d6f +F test/wal.test bc99b1656b721fddd2dfeea0b0e77b147e40e6da F test/walbak.test f6fde9a5f59d0c697cb1f4af7876178c2f69a7ba F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47 @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P c8893310510148f7e71ede5691606b6fc1c3df4d -R f19cde4cc4d379e9990f915e41f267c6 +P c8658da44671e9c43c90ee4857bbe82c55d40f25 +R a5a18287a7b1df82858844e5aea7a99d U dan -Z cdf65bfac6e74e44b7b865cbe4482fc0 +Z 295004ae7ba64e99842190f4e410241f diff --git a/manifest.uuid b/manifest.uuid index 6b66b26ebb..baa627f5aa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c8658da44671e9c43c90ee4857bbe82c55d40f25 \ No newline at end of file +be44349d2b790fb470fcc9fb6a12e8b1076dc645 \ No newline at end of file diff --git a/test/wal.test b/test/wal.test index 30586d5cb3..ca4d606d23 100644 --- a/test/wal.test +++ b/test/wal.test @@ -666,6 +666,34 @@ foreach code [list { execsql { SELECT * FROM t1 } } {1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22} + # Test that if a checkpointer cannot obtain the required locks, it + # releases all locks before returning a busy error. + # + do_test wal-10.$tn.35 { + execsql { + DELETE FROM t1; + INSERT INTO t1 VALUES('a', 'b'); + INSERT INTO t1 VALUES('c', 'd'); + } + sql2 { + BEGIN; + SELECT * FROM t1; + } + } {a b c d} + proc busyhandler x { return 1 } + db busy busyhandler + do_test wal-10.$tn.36 { + catchsql { PRAGMA checkpoint } + } {1 {database is locked}} + do_test wal-10.$tn.36 { + sql3 { INSERT INTO t1 VALUES('e', 'f') } + sql2 { SELECT * FROM t1 } + } {a b c d} + do_test wal-10.$tn.37 { + sql2 COMMIT + execsql { PRAGMA checkpoint } + } {} + catch { db close } catch { code2 { db2 close } } catch { code3 { db3 close } } From 833bf968e36e14588c4868f0e0710432dcf2d814 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Apr 2010 14:42:19 +0000 Subject: [PATCH 46/89] Rename the sqlite3_log_hook() to sqlite3_wal_hook(). Added comments to wal.h. FossilOrigin-Name: bbc385111b19071e20fe963fab814262c815b3e9 --- manifest | 34 ++++++++++++++++++++++------------ manifest.uuid | 2 +- src/main.c | 2 +- src/sqlite.h.in | 10 +++++----- src/tclsqlite.c | 32 ++++++++++++++++---------------- src/wal.h | 30 +++++++++++++++++++++++++----- test/tclsqlite.test | 2 +- test/walhook.test | 21 ++++++++++----------- 8 files changed, 81 insertions(+), 52 deletions(-) diff --git a/manifest b/manifest index b87e29226b..fec9e7d250 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,8 @@ -C Merge\stwo\s"wal"\sleaves. -D 2010-04-27T18:49:54 +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 + +C Rename\sthe\ssqlite3_log_hook()\sto\ssqlite3_wal_hook().\s\sAdded\scomments\sto\nwal.h. +D 2010-04-28T14:42:19 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,7 +134,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/main.c 867de6aa444abd97771b2b70472f448d65c1c77e +F src/main.c 569561a238a0bb6d55affbaa224ddc885c91df2b F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 89d4ea8d5cdd55635cbaa48ad53132af6294cbb2 @@ -166,13 +169,13 @@ F src/resolve.c ac5f1a713cd1ae77f08b83cc69581e11bf5ae6f9 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c c03d8a0565febcde8c6a12c5d77d065fddae889b F src/shell.c c40427c7245535a04a9cb4a417b6cc05c022e6a4 -F src/sqlite.h.in f9399ee58862fd146eff9725dda83b5c82d04a93 +F src/sqlite.h.in 9529106a7608b57bbe9e290cdddb203bd383f888 F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89 F src/sqliteInt.h 700a2df7b8dfe57c3b8d83c52ff40928e026220c F src/sqliteLimit.h 3afab2291762b5d09ae20c18feb8e9fa935a60a6 F src/status.c 4df6fe7dce2d256130b905847c6c60055882bdbe F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e -F src/tclsqlite.c e27e0457109d72c7404df08af1726ff6ddc3d92c +F src/tclsqlite.c 4de81521174fedacd8393ea7b70b730ce17f8eae F src/test1.c 64b5b8135080b94370e8100e5066bb394f5c3122 F src/test2.c b6b43413d495addd039a88b87d65c839f86b18cb F src/test3.c 4c21700c73a890a47fc685c1097bfb661346ac94 @@ -222,7 +225,7 @@ F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda F src/wal.c 1576b36d55220d42598a5fb48a06e083efecea75 -F src/wal.h 812101dd76610401fbcd44114e7e8b7ce0224645 +F src/wal.h c60781e78e394af07ece3b64a11192eb442241c1 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -599,7 +602,7 @@ F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/sync.test ded6b39d8d8ca3c0c5518516c6371b3316d3e3a3 F test/table.test bf102a5669c4db7a41330802f24a4a81a4204f83 F test/tableapi.test 7262a8cbaa9965d429f1cbd2747edc185fa56516 -F test/tclsqlite.test 7378a4e7474d9ffdb7c930086112b829e5955fb0 +F test/tclsqlite.test 013133fa83128569c6fb8a7a48dc7c4507e6ff1c F test/tempdb.test 1bf52da28a9c24e29717362a87722dff08feb72b F test/temptable.test f42121a0d29a62f00f93274464164177ab1cc24a F test/temptrigger.test b0273db072ce5f37cf19140ceb1f0d524bbe9f05 @@ -761,7 +764,7 @@ F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d F test/wal.test bc99b1656b721fddd2dfeea0b0e77b147e40e6da F test/walbak.test f6fde9a5f59d0c697cb1f4af7876178c2f69a7ba F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 -F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47 +F test/walhook.test 287a69d662939604f2e0452dace2cec8ef634d5e F test/walmode.test 40119078da084e6a7403ba57485d5a86ee0e2646 F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 F test/walthread.test a7962d8b899366cc71f8381d9aeab8ae9e1b544d @@ -808,7 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P be44349d2b790fb470fcc9fb6a12e8b1076dc645 ed715b47c5f7657fbf901805981867898054b14d -R 543123f4859d04253152521c95f2472b -U dan -Z 67c1d58e24d1fcb9816b2a39774cf9f7 +P 8c2d43babd61fe2225d5c402174253f412604092 +R d2467063cf37b05312bd03bb938b0ac5 +U drh +Z 7c95869a158e03c8e63f0877555a0133 +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.6 (GNU/Linux) + +iD8DBQFL2ElOoxKgR168RlERAmaCAJ0cROgoBzwR39dDYWGzjUDfWHeq+ACfdKsQ +SQEr/7S7IaJ1k5L/fD6AU0g= +=eywR +-----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index e7f2fb29f8..9760418781 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8c2d43babd61fe2225d5c402174253f412604092 \ No newline at end of file +bbc385111b19071e20fe963fab814262c815b3e9 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 80c15963a8..3e7c1257c2 100644 --- a/src/main.c +++ b/src/main.c @@ -1190,7 +1190,7 @@ void *sqlite3_rollback_hook( ** Register a callback to be invoked each time a transaction is written ** into the write-ahead-log by this database connection. */ -void *sqlite3_log_hook( +void *sqlite3_wal_hook( sqlite3 *db, /* Attach the hook to this db handle */ int(*xCallback)(void *, sqlite3*, const char*, int), void *pArg /* First argument passed to xCallback() */ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 7bc7509447..225007780b 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5761,9 +5761,9 @@ int sqlite3_strnicmp(const char *, const char *, int); void sqlite3_log(int iErrCode, const char *zFormat, ...); /* -** Experimental WAL callback interface. +** CAPI3REF: Write-Ahead Log Commit Hook ** -** The [sqlite3_log_hook()] function is used to register a callback that +** The [sqlite3_wal_hook()] function is used to register a callback that ** will be invoked each time a database connection commits data to a ** write-ahead-log (i.e. whenever a transaction is committed in ** journal_mode=WAL mode). @@ -5773,7 +5773,7 @@ void sqlite3_log(int iErrCode, const char *zFormat, ...); ** may read, write or checkpoint the database as required. ** ** The first parameter passed to the callback function when it is invoked -** is a copy of the third parameter passed to sqlite3_log_hook() when +** is a copy of the third parameter passed to sqlite3_wal_hook() when ** registering the callback. The second is a copy of the database handle. ** The third parameter is the name of the database that was written to - ** either "main" or the name of an ATTACHed database. The fourth parameter @@ -5785,10 +5785,10 @@ void sqlite3_log(int iErrCode, const char *zFormat, ...); ** no special action is taken. ** ** A single database handle may have at most a single log callback -** registered at one time. Calling [sqlite3_log_hook()] replaces any +** registered at one time. Calling [sqlite3_wal_hook()] replaces any ** previously registered log callback. */ -void *sqlite3_log_hook( +void *sqlite3_wal_hook( sqlite3*, int(*)(void *,sqlite3*,const char*,int), void* diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 8582ce34ee..4ef51d8566 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -1570,12 +1570,12 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ "complete", "copy", "enable_load_extension", "errorcode", "eval", "exists", "function", "incrblob", "interrupt", - "last_insert_rowid", "log_hook", "nullvalue", - "onecolumn", "profile", "progress", - "rekey", "restore", "rollback_hook", - "status", "timeout", "total_changes", - "trace", "transaction", "unlock_notify", - "update_hook", "version", 0 + "last_insert_rowid", "nullvalue", "onecolumn", + "profile", "progress", "rekey", + "restore", "rollback_hook", "status", + "timeout", "total_changes", "trace", + "transaction", "unlock_notify", "update_hook", + "version", "wal_hook", 0 }; enum DB_enum { DB_AUTHORIZER, DB_BACKUP, DB_BUSY, @@ -1584,12 +1584,12 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ DB_COMPLETE, DB_COPY, DB_ENABLE_LOAD_EXTENSION, DB_ERRORCODE, DB_EVAL, DB_EXISTS, DB_FUNCTION, DB_INCRBLOB, DB_INTERRUPT, - DB_LAST_INSERT_ROWID, DB_LOG_HOOK, DB_NULLVALUE, - DB_ONECOLUMN, DB_PROFILE, DB_PROGRESS, - DB_REKEY, DB_RESTORE, DB_ROLLBACK_HOOK, - DB_STATUS, DB_TIMEOUT, DB_TOTAL_CHANGES, - DB_TRACE, DB_TRANSACTION, DB_UNLOCK_NOTIFY, - DB_UPDATE_HOOK, DB_VERSION + DB_LAST_INSERT_ROWID, DB_NULLVALUE, DB_ONECOLUMN, + DB_PROFILE, DB_PROGRESS, DB_REKEY, + DB_RESTORE, DB_ROLLBACK_HOOK, DB_STATUS, + DB_TIMEOUT, DB_TOTAL_CHANGES, DB_TRACE, + DB_TRANSACTION, DB_UNLOCK_NOTIFY, DB_UPDATE_HOOK, + DB_VERSION, DB_WAL_HOOK }; /* don't leave trailing commas on DB_enum, it confuses the AIX xlc compiler */ @@ -2760,11 +2760,11 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ } /* - ** $db log_hook ?script? + ** $db wal_hook ?script? ** $db update_hook ?script? ** $db rollback_hook ?script? */ - case DB_LOG_HOOK: + case DB_WAL_HOOK: case DB_UPDATE_HOOK: case DB_ROLLBACK_HOOK: { @@ -2774,7 +2774,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ Tcl_Obj **ppHook; if( choice==DB_UPDATE_HOOK ){ ppHook = &pDb->pUpdateHook; - }else if( choice==DB_LOG_HOOK ){ + }else if( choice==DB_WAL_HOOK ){ ppHook = &pDb->pLogHook; }else{ ppHook = &pDb->pRollbackHook; @@ -2801,7 +2801,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ sqlite3_update_hook(pDb->db, (pDb->pUpdateHook?DbUpdateHandler:0), pDb); sqlite3_rollback_hook(pDb->db,(pDb->pRollbackHook?DbRollbackHandler:0),pDb); - sqlite3_log_hook(pDb->db,(pDb->pLogHook?DbLogHandler:0),pDb); + sqlite3_wal_hook(pDb->db,(pDb->pLogHook?DbLogHandler:0),pDb); break; } diff --git a/src/wal.h b/src/wal.h index 64067ce09b..f04344c63a 100644 --- a/src/wal.h +++ b/src/wal.h @@ -19,19 +19,30 @@ #include "sqliteInt.h" -/* Connection to a log file. There is one object of this type for each pager. */ +/* Connection to a write-ahead log (WAL) file. +** There is one object of this type for each pager. +*/ typedef struct Log Log; -/* Open and close a connection to a log file. */ +/* Open and close a connection to a write-ahead log. */ int sqlite3WalOpen(sqlite3_vfs*, const char *zDb, Log **ppLog); int sqlite3WalClose(Log *pLog, sqlite3_file *pFd, int sync_flags, u8 *zBuf); -/* Used by readers to open (lock) and close (unlock) a snapshot. */ +/* Used by readers to open (lock) and close (unlock) a snapshot. A +** snapshot is like a read-transaction. It is the state of the database +** at an instant in time. sqlite3WalOpenSnapshot gets a read lock and +** preserves the current state even if the other threads or processes +** write to or checkpoint the WAL. sqlite3WalCloseSnapshot() closes the +** transaction and releases the lock. +*/ int sqlite3WalOpenSnapshot(Log *pLog, int *); void sqlite3WalCloseSnapshot(Log *pLog); -/* Read a page from the log, if it is present. */ +/* Read a page from the write-ahead log, if it is present. */ int sqlite3WalRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut); + +/* Return the size of the database as it existed at the beginning +** of the snapshot */ void sqlite3WalDbsize(Log *pLog, Pgno *pPgno); /* Obtain or release the WRITER lock. */ @@ -40,7 +51,12 @@ int sqlite3WalWriteLock(Log *pLog, int op); /* Undo any frames written (but not committed) to the log */ int sqlite3WalUndo(Log *pLog, int (*xUndo)(void *, Pgno), void *pUndoCtx); +/* Return an integer that records the current (uncommitted) write +** position in the WAL */ u32 sqlite3WalSavepoint(Log *pLog); + +/* Move the write position of the WAL back to iFrame. Called in +** response to a ROLLBACK TO command. */ int sqlite3WalSavepointUndo(Log *pLog, u32 iFrame); /* Return true if data has been written but not committed to the log file. */ @@ -59,7 +75,11 @@ int sqlite3WalCheckpoint( void *pBusyHandlerArg /* Argument to pass to xBusyHandler */ ); -/* Return the value to pass to a log callback. Or 0 for no callback. */ +/* Return the value to pass to a sqlite3_wal_hook callback, the +** number of frames in the WAL at the point of the last commit since +** sqlite3WalCallback() was called. If no commits have occurred since +** the last call, then return 0. +*/ int sqlite3WalCallback(Log *pLog); #endif /* _WAL_H_ */ diff --git a/test/tclsqlite.test b/test/tclsqlite.test index e04841a062..a6511c8be1 100644 --- a/test/tclsqlite.test +++ b/test/tclsqlite.test @@ -35,7 +35,7 @@ do_test tcl-1.1 { do_test tcl-1.2 { set v [catch {db bogus} msg] lappend v $msg -} {1 {bad option "bogus": must be authorizer, backup, busy, cache, changes, close, collate, collation_needed, commit_hook, complete, copy, enable_load_extension, errorcode, eval, exists, function, incrblob, interrupt, last_insert_rowid, log_hook, nullvalue, onecolumn, profile, progress, rekey, restore, rollback_hook, status, timeout, total_changes, trace, transaction, unlock_notify, update_hook, or version}} +} {1 {bad option "bogus": must be authorizer, backup, busy, cache, changes, close, collate, collation_needed, commit_hook, complete, copy, enable_load_extension, errorcode, eval, exists, function, incrblob, interrupt, last_insert_rowid, nullvalue, onecolumn, profile, progress, rekey, restore, rollback_hook, status, timeout, total_changes, trace, transaction, unlock_notify, update_hook, version, or wal_hook}} do_test tcl-1.2.1 { set v [catch {db cache bogus} msg] lappend v $msg diff --git a/test/walhook.test b/test/walhook.test index 6f2813b618..a33b43a005 100644 --- a/test/walhook.test +++ b/test/walhook.test @@ -25,31 +25,31 @@ proc sqlite3_wal {args} { } } sqlite3_wal db test.db -db log_hook log_hook +db wal_hook wal_hook -set ::log_hook [list] -proc log_hook {zDb nEntry} { - lappend ::log_hook $zDb $nEntry +set ::wal_hook [list] +proc wal_hook {zDb nEntry} { + lappend ::wal_hook $zDb $nEntry return 0 } do_test walhook-1.1 { execsql { CREATE TABLE t1(i PRIMARY KEY, j) } - set ::log_hook + set ::wal_hook } {main 3} do_test walhook-1.2 { - set ::log_hook [list] + set ::wal_hook [list] execsql { INSERT INTO t1 VALUES(1, 'one') } - set ::log_hook + set ::wal_hook } {main 5} do_test walhook-1.3 { - proc log_hook {args} { return 1 } + proc wal_hook {args} { return 1 } execsql { INSERT INTO t1 VALUES(2, 'two') } file size test.db } [expr 3*1024] do_test walhook-1.4 { - proc log_hook {zDb nEntry} { + proc wal_hook {zDb nEntry} { execsql { PRAGMA checkpoint } return 0 } @@ -59,7 +59,7 @@ do_test walhook-1.4 { do_test walhook-1.5 { sqlite3_wal db2 test.db - proc log_hook {zDb nEntry} { + proc wal_hook {zDb nEntry} { execsql { PRAGMA checkpoint } db2 return 0 } @@ -70,4 +70,3 @@ do_test walhook-1.5 { catch { db2 close } catch { db close } finish_test - From 9beb1584bd2e1ea3dfdf3ff08ae78e96f1c1e37d Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Apr 2010 17:21:33 +0000 Subject: [PATCH 47/89] Changes to the interface design for the xShmLock method of the VFS. FossilOrigin-Name: 348409de26eafe12f5cb1236e8e167a4183d4051 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/os_unix.c | 23 ++++------------------- src/sqlite.h.in | 26 ++++++++++++++------------ 4 files changed, 29 insertions(+), 42 deletions(-) diff --git a/manifest b/manifest index fec9e7d250..e87f3103a8 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 -C Rename\sthe\ssqlite3_log_hook()\sto\ssqlite3_wal_hook().\s\sAdded\scomments\sto\nwal.h. -D 2010-04-28T14:42:19 +C Changes\sto\sthe\sinterface\sdesign\sfor\sthe\sxShmLock\smethod\sof\sthe\sVFS. +D 2010-04-28T17:21:33 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -153,7 +153,7 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 -F src/os_unix.c 332ff9185aedcbd25568a06ddbdb51f5496b9756 +F src/os_unix.c c8b7d1e0f4315e08cf54324ca706e1a1a80f5fc2 F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 F src/pager.c b4a41030860229e80295fa1f37addab24d21799c F src/pager.h cee4487ab4f0911dd9f22a40e3cd55afdb7ef444 @@ -169,7 +169,7 @@ F src/resolve.c ac5f1a713cd1ae77f08b83cc69581e11bf5ae6f9 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c c03d8a0565febcde8c6a12c5d77d065fddae889b F src/shell.c c40427c7245535a04a9cb4a417b6cc05c022e6a4 -F src/sqlite.h.in 9529106a7608b57bbe9e290cdddb203bd383f888 +F src/sqlite.h.in caf60df0991a14e22cce8243b9caa1c1dbd09d42 F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89 F src/sqliteInt.h 700a2df7b8dfe57c3b8d83c52ff40928e026220c F src/sqliteLimit.h 3afab2291762b5d09ae20c18feb8e9fa935a60a6 @@ -811,14 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 8c2d43babd61fe2225d5c402174253f412604092 -R d2467063cf37b05312bd03bb938b0ac5 +P bbc385111b19071e20fe963fab814262c815b3e9 +R 8707e56d2016564dc168df33cbfc8185 U drh -Z 7c95869a158e03c8e63f0877555a0133 +Z 76427138e798af79dc3cb019d4766cfc -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.6 (GNU/Linux) -iD8DBQFL2ElOoxKgR168RlERAmaCAJ0cROgoBzwR39dDYWGzjUDfWHeq+ACfdKsQ -SQEr/7S7IaJ1k5L/fD6AU0g= -=eywR +iD8DBQFL2G6foxKgR168RlERAj6qAJ4tUzbbIOEurzGhDqNlV4AMpvLtEgCfaPHT +5y4JLRgmNxCyywDyaIJzD+o= +=+YQ0 -----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index 9760418781..aa96371448 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bbc385111b19071e20fe963fab814262c815b3e9 \ No newline at end of file +348409de26eafe12f5cb1236e8e167a4183d4051 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index d603caa0d0..7e3930a1e6 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4697,26 +4697,11 @@ static int unixShmRelease(sqlite3_shm *pSharedMem){ */ static int unixShmLock( sqlite3_shm *pSharedMem, /* Pointer from unixShmOpen() */ - int lockType, /* _RDLK, _WRLK, or _UNLK, possibly ORed _BLOCK */ - int ofst, /* Start of lock region */ - int nByte /* Size of lock region in bytes */ + int desiredLock, /* The locking state desired */ + int *pGotLock, /* The locking state actually obtained */ + int shouldBlock /* Block for the lock if true and possible */ ){ - struct unixShm *p = (struct unixShm*)pSharedMem; - struct flock f; - int op; - int rc; - - f.l_whence = SEEK_SET; - f.l_start = ofst; - f.l_len = nByte; - switch( lockType & 0x07 ){ - case SQLITE_SHM_RDLK: f.l_type = F_RDLCK; break; - case SQLITE_SHM_WRLK: f.l_type = F_WRLCK; break; - case SQLITE_SHM_UNLK: f.l_type = F_UNLCK; break; - } - op = (lockType & 0x08)!=0 ? F_SETLKW : F_SETLK; - rc = fcntl(p->fd.h, op, &f); - return (rc==0) ? SQLITE_OK : SQLITE_BUSY; + return SQLITE_OK; } /* diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 225007780b..9f6f1bc967 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -848,7 +848,7 @@ struct sqlite3_vfs { int (*xShmRelease)(sqlite3_shm*); int (*xShmPush)(sqlite3_shm*); int (*xShmPull)(sqlite3_shm*); - int (*xShmLock)(sqlite3_shm*, int lockType, int ofst, int nByte); + int (*xShmLock)(sqlite3_shm*, int desiredLock, int *gotLock, int shouldBlock); int (*xShmClose)(sqlite3_shm*); int (*xShmDelete)(sqlite3_vfs*, const char *zName); int (*xRename)(sqlite3_vfs*, const char *zOld, const char *zNew, int dirSync); @@ -880,18 +880,20 @@ struct sqlite3_vfs { /* ** CAPI3REF: Flags for the xShmLock VFS method ** -** These integer constants can be used as the second parameter to -** the xShmLock method of an [sqlite3_vfs] object. They determine -** the specific locking action. Exactly one of the first three -** values must be used ini the lockType parameter. The fourth -** value (SQLITE_SHM_BLOCK) can optionally be ORed into the lockType -** parameter to cause the thread to block until the lock becomes -** available. +** These integer constants define the various locking states that +** an sqlite3_shm object can be in. The SQLITE_SHM_QUERY integer +** is not a valid data - it is a constant pasted to the +** sqlite3_vfs.xShmLock() method for querying the current lock +** state. */ -#define SQLITE_SHM_RDLK 0x01 -#define SQLITE_SHM_WRLK 0x02 -#define SQLITE_SHM_UNLK 0x04 -#define SQLITE_SHM_BLOCK 0x08 +#define SQLITE_SHM_UNLOCK 0 +#define SQLITE_SHM_READ_PREFIX 1 +#define SQLITE_SHM_READ_FULL 2 +#define SQLITE_SHM_WRITE 3 +#define SQLITE_SHM_PENDING 4 +#define SQLITE_SHM_CHECKPOINT 5 +#define SQLITE_SHM_RECOVER 6 +#define SQLITE_SHM_QUERY (-1) /* ** CAPI3REF: Initialize The SQLite Library From 5e0ce87a3f8dd8b66c69651d8a40ebf2c35710ab Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 28 Apr 2010 17:48:44 +0000 Subject: [PATCH 48/89] Change walthread.test so that tests can be run with either multiple threads or multiple processes. FossilOrigin-Name: 25f85f68723e56c18e44b094d85f67b99912dc86 --- manifest | 18 +-- manifest.uuid | 2 +- src/test_thread.c | 2 + src/wal.c | 2 +- test/lock_common.tcl | 25 +++- test/walthread.test | 344 +++++++++++++++++++++++++------------------ 6 files changed, 236 insertions(+), 157 deletions(-) diff --git a/manifest b/manifest index b87e29226b..eccf56fa3e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\stwo\s"wal"\sleaves. -D 2010-04-27T18:49:54 +C Change\swalthread.test\sso\sthat\stests\scan\sbe\srun\swith\seither\smultiple\sthreads\sor\smultiple\sprocesses. +D 2010-04-28T17:48:44 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -204,7 +204,7 @@ F src/test_pcache.c 7bf828972ac0d2403f5cfa4cd14da41f8ebe73d8 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c bbba05c144b5fc4b52ff650a4328027b3fa5fcc6 F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa -F src/test_thread.c 00fed80690ae7f1525483a35861511c48bc579f2 +F src/test_thread.c aa9919c885a1fe53eafc73492f0898ee6c0a0726 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 25ceb0f0a746ea1d0f9553787f3f0a56853cfaeb F src/trigger.c 8927588cb9e6d47f933b53bfe74200fbb504100d @@ -221,7 +221,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c 1576b36d55220d42598a5fb48a06e083efecea75 +F src/wal.c d63318e8e73f9ed1a6f3c277f71022024f38a7c3 F src/wal.h 812101dd76610401fbcd44114e7e8b7ce0224645 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 @@ -479,7 +479,7 @@ F test/lock4.test f4f36271aa5ae1da449646bf43c7341f6b2b4c4e F test/lock5.test 6b1f78f09ad1522843dad571b76b321e6f439bf7 F test/lock6.test 862aa71e97b288d6b3f92ba3313f51bd0b003776 F test/lock7.test 64006c84c1c616657e237c7ad6532b765611cf64 -F test/lock_common.tcl ebc5b9a238d6d9f08a8433e52bdbc67d1478e774 +F test/lock_common.tcl bbc4e15ee5334cc4d01fcac08d7c9de7d8906e55 F test/lookaside.test 1dd350dc6dff015c47c07fcc5a727a72fc5bae02 F test/main.test 2be2352ac77ac5b238c6337a5469aeeef57677e6 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 @@ -764,7 +764,7 @@ F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 76a559e262f0715c470bade4a8d8333035f8ee47 F test/walmode.test 40119078da084e6a7403ba57485d5a86ee0e2646 F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 -F test/walthread.test a7962d8b899366cc71f8381d9aeab8ae9e1b544d +F test/walthread.test 871aeecc5b89133b094bcc00c4fcfa040749c726 F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 F test/where2.test 45eacc126aabb37959a387aa83e59ce1f1f03820 F test/where3.test aa44a9b29e8c9f3d7bb94a3bb3a95b31627d520d @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P be44349d2b790fb470fcc9fb6a12e8b1076dc645 ed715b47c5f7657fbf901805981867898054b14d -R 543123f4859d04253152521c95f2472b +P 8c2d43babd61fe2225d5c402174253f412604092 +R 797336faa9830e538744894acb6ac97d U dan -Z 67c1d58e24d1fcb9816b2a39774cf9f7 +Z 5512df46d9c359853729f7512b676a72 diff --git a/manifest.uuid b/manifest.uuid index e7f2fb29f8..f5bda9e6dc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8c2d43babd61fe2225d5c402174253f412604092 \ No newline at end of file +25f85f68723e56c18e44b094d85f67b99912dc86 \ No newline at end of file diff --git a/src/test_thread.c b/src/test_thread.c index 6a319fabf4..c4ade68ed9 100644 --- a/src/test_thread.c +++ b/src/test_thread.c @@ -58,6 +58,7 @@ static Tcl_ObjCmdProc blocking_step_proc; static Tcl_ObjCmdProc blocking_prepare_v2_proc; #endif int Sqlitetest1_Init(Tcl_Interp *); +int Sqlite3_Init(Tcl_Interp *); /* Functions from test1.c */ void *sqlite3TestTextToPtr(const char *); @@ -124,6 +125,7 @@ static Tcl_ThreadCreateType tclScriptThread(ClientData pSqlThread){ #endif Sqlitetest1_Init(interp); Sqlitetest_mutex_Init(interp); + Sqlite3_Init(interp); rc = Tcl_Eval(interp, p->zScript); pRes = Tcl_GetObjResult(interp); diff --git a/src/wal.c b/src/wal.c index 4710c89a50..c7478d0e82 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1541,7 +1541,7 @@ void sqlite3WalCloseSnapshot(Log *pLog){ pLog->isLocked = 0; } -/* +/* ** Read a page from the log, if it is present. */ int sqlite3WalRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut){ diff --git a/test/lock_common.tcl b/test/lock_common.tcl index a17de8a3da..b2e4184cc4 100644 --- a/test/lock_common.tcl +++ b/test/lock_common.tcl @@ -46,6 +46,27 @@ proc testfixture {chan cmd} { } } +proc testfixture_nb_cb {varname chan} { + set line [gets $chan] + if { $line == "OVER" } { + set $varname $::tfnb($chan) + unset ::tfnb($chan) + close $chan + } else { + append ::tfnb($chan) $line + } +} + +proc testfixture_nb {varname cmd} { + set chan [launch_testfixture] + set ::tfnb($chan) "" + fconfigure $chan -blocking 0 -buffering none + puts $chan $cmd + puts $chan OVER + fileevent $chan readable [list testfixture_nb_cb $varname $chan] + return "" +} + # Write the main loop for the child testfixture processes into file # tf_main.tcl. The parent (this script) interacts with the child processes # via a two way pipe. The parent writes a script to the stdin of the child @@ -61,7 +82,7 @@ puts $f { set line [gets stdin] puts $l "READ $line" if { $line == "OVER" } { - catch {eval $script} result + set rc [catch {eval $script} result] puts $result puts $l "WRITE $result" puts OVER @@ -70,7 +91,7 @@ puts $f { set script "" } else { append script $line - append script " ; " + append script "\n" } } close $l diff --git a/test/walthread.test b/test/walthread.test index 23eb1d6a0e..8e20d96dfb 100644 --- a/test/walthread.test +++ b/test/walthread.test @@ -16,43 +16,158 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +source $testdir/lock_common.tcl if {[run_thread_tests]==0} { finish_test ; return } set sqlite_walsummary_mmap_incr 64 -#-------------------------------------------------------------------------- -# Initialize the database used for the multi-thread test. +# The number of threads to start. And the amount of time to run the test +# for. Respectively. # -do_test walthread-1.1 { - execsql { - PRAGMA journal_mode = WAL; - PRAGMA lock_status; - CREATE TABLE t1(x PRIMARY KEY); - PRAGMA lock_status; - INSERT INTO t1 VALUES(randomblob(100)); - INSERT INTO t1 VALUES(randomblob(100)); - INSERT INTO t1 SELECT md5sum(x) FROM t1; +set NTHREAD 10 +set SECONDS 5 + +# The parameter is the name of a variable in the callers context. The +# variable may or may not exist when this command is invoked. +# +# If the variable does exist, its value is returned. Otherwise, this +# command uses [vwait] to wait until it is set, then returns the value. +# In other words, this is a version of the [set VARNAME] command that +# blocks until a variable exists. +# +proc wait_for_var {varname} { + if {0==[uplevel [list info exists $varname]]} { + uplevel [list vwait $varname] } -} {wal main unlocked temp closed main shared temp closed} -do_test walthread-1.2 { - execsql { - SELECT (SELECT count(*) FROM t1), ( - SELECT md5sum(x) FROM t1 WHERE oid != (SELECT max(oid) FROM t1) - ) == ( - SELECT x FROM t1 WHERE oid = (SELECT max(oid) FROM t1) - ) + uplevel [list set $varname] +} + +proc lshift {lvar} { + upvar $lvar L + set ret [lindex $L 0] + set L [lrange $L 1 end] + return $ret +} + + +#------------------------------------------------------------------------- +# do_thread_test TESTNAME OPTIONS... +# +# where OPTIONS are: +# +# -seconds SECONDS How many seconds to run the test for +# -init SCRIPT Script to run before test. +# -thread NAME COUNT SCRIPT Scripts to run in threads (or processes). +# -processes BOOLEAN True to use processes instead of threads. +# +proc do_thread_test {args} { + set A $args + + set P(testname) [lshift A] + set P(seconds) 5 + set P(init) "" + set P(threads) [list] + set P(processes) 0 + + unset -nocomplain ::done + + while {[llength $A]>0} { + set a [lshift A] + switch -glob -- $a { + -seconds { + set P(seconds) [lshift A] + } + + -init { + set P(init) [lshift A] + } + + -processes { + set P(processes) [lshift A] + } + + -thread { + set name [lshift A] + set count [lshift A] + set prg [lshift A] + lappend P(threads) [list $name $count $prg] + } + + default { + error "Unknown option: $a" + } + } } -} {3 1} -do_test walthread-1.3 { - execsql { PRAGMA integrity_check } -} {ok} -do_test walthread-1.4 { - execsql { PRAGMA lock_status } -} {main shared temp unknown} + + puts "Running $P(testname) for $P(seconds) seconds..." + + catch { db close } + file delete -force test.db test.db-journal test.db-wal + + sqlite3 db test.db + eval $P(init) + db close + + foreach T $P(threads) { + set name [lindex $T 0] + set count [lindex $T 1] + set prg [lindex $T 2] + + for {set i 1} {$i <= $count} {incr i} { + set program [string map [list %TEST% $prg %SECONDS% $P(seconds) %I% $i] { + + set tid %I% + + proc usleep {ms} { + set ::usleep 0 + after $ms {set ::usleep 1} + vwait ::usleep + } + proc busyhandler {n} { usleep 10 ; return 0 } + + sqlite3 db test.db + db busy busyhandler + db eval { SELECT randomblob($tid*5) } + + set ::finished 0 + after [expr %SECONDS% * 1000] {set ::finished 1} + proc tt_continue {} { expr ($::finished==0) } + + set rc [catch { %TEST% } msg] + + db close + list $rc $msg + }] + + if {$P(processes)==0} { + sqlthread spawn ::done($name,$i) $program + } else { + testfixture_nb ::done($name,$i) $program + } + } + } + + set report " Results:" + foreach T $P(threads) { + set name [lindex $T 0] + set count [lindex $T 1] + set prg [lindex $T 2] + + set reslist [list] + for {set i 1} {$i <= $count} {incr i} { + set res [wait_for_var ::done($name,$i)] + lappend reslist [lindex $res 1] + do_test $P(testname).$name.$i [list lindex $res 0] 0 + } + + append report " $name $reslist" + } + puts $report +} #-------------------------------------------------------------------------- -# Start N threads. Each thread performs both read and write transactions. -# Each read transaction consists of: +# Start NTHREAD threads. Each thread performs both read and write +# transactions. Each read transaction consists of: # # 1) Reading the md5sum of all but the last table row, # 2) Running integrity check. @@ -71,132 +186,73 @@ do_test walthread-1.4 { # # Ther is also a single checkpointer thread. It runs the following loop: # -# 1) Execute "CHECKPOINT main 32 -1 1" +# 1) Execute "PRAGMA checkpoint" # 2) Sleep for 500 ms. # -set thread_program { - proc rest {ms} { - set ::rest 0 - after $ms {set ::rest 1} - vwait ::rest - } - proc dosql {DB sql} { - set res "" - set stmt [sqlite3_prepare_v2 $DB $sql -1 dummy_tail] - set rc [sqlite3_step $stmt] - if {$rc eq "SQLITE_ROW"} { - set res [sqlite3_column_text $stmt 0] +foreach {mode name} { + 0 walthread-1-threads + 1 walthread-1-processes +} { + do_thread_test $name -processes $mode -seconds $SECONDS -init { + execsql { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(x PRIMARY KEY); + PRAGMA lock_status; + INSERT INTO t1 VALUES(randomblob(100)); + INSERT INTO t1 VALUES(randomblob(100)); + INSERT INTO t1 SELECT md5sum(x) FROM t1; } - set rc [sqlite3_finalize $stmt] - - if {$rc ne "SQLITE_OK"} { - error "$rc: [sqlite3_errmsg $DB]" + } -thread main $NTHREAD { + + proc read_transaction {} { + set results [db eval { + BEGIN; + PRAGMA integrity_check; + SELECT md5sum(x) FROM t1 WHERE rowid != (SELECT max(rowid) FROM t1); + SELECT x FROM t1 WHERE rowid = (SELECT max(rowid) FROM t1); + SELECT md5sum(x) FROM t1 WHERE rowid != (SELECT max(rowid) FROM t1); + COMMIT; + }] + + if {[llength $results]!=4 + || [lindex $results 0] != "ok" + || [lindex $results 1] != [lindex $results 2] + || [lindex $results 2] != [lindex $results 3] + } { + error "Failed read transaction: $results" + } } - return $res - } - - proc read_transaction {DB} { - dosql $DB BEGIN - - set md5_1 [dosql $DB { - SELECT md5sum(x) FROM t1 WHERE rowid != (SELECT max(rowid) FROM t1) - }] - set check [dosql $DB { PRAGMA integrity_check }] - set md5_2 [dosql $DB { - SELECT x FROM t1 WHERE rowid = (SELECT max(rowid) FROM t1) - }] - set md5_3 [dosql $DB { - SELECT md5sum(x) FROM t1 WHERE rowid != (SELECT max(rowid) FROM t1) - }] - - dosql $DB COMMIT - - if {$check ne "ok" - || $md5_1 ne $md5_2 - || $md5_2 ne $md5_3 - } { - error "Failed read transaction $check $md5_1 $md5_2 $md5_3" + + proc write_transaction {} { + db eval { + BEGIN; + INSERT INTO t1 VALUES(randomblob(100)); + INSERT INTO t1 VALUES(randomblob(100)); + INSERT INTO t1 SELECT md5sum(x) FROM t1; + COMMIT; + } } - } - - proc write_transaction {DB} { - dosql $DB BEGIN - dosql $DB "INSERT INTO t1 VALUES(randomblob(100))" - dosql $DB "INSERT INTO t1 VALUES(randomblob(100))" - dosql $DB "INSERT INTO t1 SELECT md5sum(x) FROM t1" - dosql $DB COMMIT - } - - proc checkpointer {DB} { - while { !$::finished } { - dosql $DB "PRAGMA checkpoint" - rest 1000 + + set nRun 0 + while {[tt_continue]} { + read_transaction + write_transaction + usleep 1 + incr nRun } - } - - proc worker {DB N} { - set j 0 - while { !$::finished } { - for {set i 0} {$i < $N} {incr i} { read_transaction $DB } - write_transaction $DB - rest 1 + set nRun + + } -thread ckpt 1 { + set nRun 0 + while {[tt_continue]} { + db eval "PRAGMA checkpoint" + usleep 500 + incr nRun } + set nRun } - - set ::finished 0 - after [expr $seconds*1000] {set ::finished 1} - - set ::DB [sqlthread open test.db] - dosql $::DB { PRAGMA journal_mode = WAL } - - - set rc [catch { - if {$role eq "worker"} { worker $DB $N } - if {$role eq "checkpointer"} { checkpointer $DB } - } msg] - - sqlite3_close $::DB - - if {$rc==0} { set msg OK } - set msg } -set NTHREAD 6 -set SECONDS 30 - -#set prg "set N $NTHREAD ; set seconds $SECONDS" -set prg "set N 1 ; set seconds $SECONDS" - -array unset finished -for {set i 0} {$i < $NTHREAD} {incr i} { - thread_spawn finished($i) {set role worker} $prg $thread_program -} -thread_spawn finished(C) {set role checkpointer} $prg $thread_program -#set finished(C) 1 - -puts "... test runs for approximately $SECONDS seconds ..." -for {set i 0} {$i < $::NTHREAD} {incr i} { - if {![info exists finished($i)]} { - vwait finished($i) - } - do_test walthread-2.$i { - set ::finished($i) - } OK -} -do_test walthread-2.C { - if {![info exists finished(C)]} { vwait finished(C) } - set ::finished(C) -} OK - -set logsize 0 - -set rows [execsql { SELECT count(*) FROM t1 }] -catch { set logsize [expr [file size test.db-wal] / 1024] } -set dbsize [expr [file size test.db] / 1024] - -puts "rows=$rows db=${dbsize}K log=${logsize}K" - finish_test - From e45d442692bb68790dbb9a42966c76184135e397 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 28 Apr 2010 18:17:23 +0000 Subject: [PATCH 49/89] Add a test to walthread.test for changing between WAL and rollback modes. FossilOrigin-Name: da229e44bd4a5d512261da05958d560808c9889f --- manifest | 12 ++++---- manifest.uuid | 2 +- test/walthread.test | 67 ++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 70 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 23a34d1f78..8d7772b612 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\stwo\s"wal"\nleaves. -D 2010-04-28T17:49:57 +C Add\sa\stest\sto\swalthread.test\sfor\schanging\sbetween\sWAL\sand\srollback\smodes. +D 2010-04-28T18:17:23 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -764,7 +764,7 @@ F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 287a69d662939604f2e0452dace2cec8ef634d5e F test/walmode.test 40119078da084e6a7403ba57485d5a86ee0e2646 F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 -F test/walthread.test 871aeecc5b89133b094bcc00c4fcfa040749c726 +F test/walthread.test ce1d0fc9905cf8a7cf1a61dbcafcbd3434166236 F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 F test/where2.test 45eacc126aabb37959a387aa83e59ce1f1f03820 F test/where3.test aa44a9b29e8c9f3d7bb94a3bb3a95b31627d520d @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 25f85f68723e56c18e44b094d85f67b99912dc86 348409de26eafe12f5cb1236e8e167a4183d4051 -R 39134c2ac17dc2dba6c744dfce94f1bb +P 13d2d5a66e9eaa81aa6314354201ee1fbd2b3824 +R 03ea219fce1817690b6cb9bee0a76d8a U dan -Z 83204dbbb275bd731c15521fddcf192e +Z e37b4277b60d879edbf1e6cb6d434b3c diff --git a/manifest.uuid b/manifest.uuid index 6103307f91..c424e5c417 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -13d2d5a66e9eaa81aa6314354201ee1fbd2b3824 \ No newline at end of file +da229e44bd4a5d512261da05958d560808c9889f \ No newline at end of file diff --git a/test/walthread.test b/test/walthread.test index 8e20d96dfb..057ba2396f 100644 --- a/test/walthread.test +++ b/test/walthread.test @@ -61,6 +61,9 @@ proc lshift {lvar} { # -processes BOOLEAN True to use processes instead of threads. # proc do_thread_test {args} { + + #if {[string match walthread-2* [lindex $args 0]]==0} return + set A $args set P(testname) [lshift A] @@ -131,11 +134,11 @@ proc do_thread_test {args} { set ::finished 0 after [expr %SECONDS% * 1000] {set ::finished 1} - proc tt_continue {} { expr ($::finished==0) } + proc tt_continue {} { update ; expr ($::finished==0) } set rc [catch { %TEST% } msg] - db close + catch { db close } list $rc $msg }] @@ -184,7 +187,7 @@ proc do_thread_test {args} { # Each of the N threads runs N read transactions followed by a single write # transaction in a loop as fast as possible. # -# Ther is also a single checkpointer thread. It runs the following loop: +# There is also a single checkpointer thread. It runs the following loop: # # 1) Execute "PRAGMA checkpoint" # 2) Sleep for 500 ms. @@ -238,7 +241,6 @@ foreach {mode name} { while {[tt_continue]} { read_transaction write_transaction - usleep 1 incr nRun } set nRun @@ -254,5 +256,62 @@ foreach {mode name} { } } +#-------------------------------------------------------------------------- +# +foreach {mode name} { + 0 walthread-2-threads + 1 walthread-2-processes +} { + do_thread_test $name -processes $mode -seconds $SECONDS -init { + execsql { CREATE TABLE t1(x INTEGER PRIMARY KEY, y UNIQUE) } + } -thread RB 2 { + + db close + set nRun 0 + set nDel 0 + while {[tt_continue]} { + sqlite3 db test.db + db busy busyhandler + db eval { SELECT * FROM sqlite_master } + catch { db eval { PRAGMA journal_mode = DELETE } } + db eval { + BEGIN; + INSERT INTO t1 VALUES(NULL, randomblob(100+$tid)); + } + incr nRun 1 + incr nDel [file exists test.db-journal] + db eval COMMIT + + set ic [db eval {PRAGMA integrity_check}] + if {$ic != "ok"} { error $ic } + db close + } + list $nRun $nDel + + } -thread WAL 2 { + db close + set nRun 0 + set nWal 0 + while {[tt_continue]} { + sqlite3 db test.db + db busy busyhandler + db eval { SELECT * FROM sqlite_master } + catch { db eval { PRAGMA journal_mode = WAL } } + db eval { + BEGIN; + INSERT INTO t1 VALUES(NULL, randomblob(110+$tid)); + } + incr nRun 1 + incr nWal [file exists test.db-wal] + db eval COMMIT + + set ic [db eval {PRAGMA integrity_check}] + if {$ic != "ok"} { error $ic } + db close + } + list $nRun $nWal + } +} + finish_test From b4e3a6f72f38be188f5b13933306df2fd0f491b6 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 29 Apr 2010 08:47:28 +0000 Subject: [PATCH 50/89] Add tests to walthread.test. FossilOrigin-Name: 9e891e7543ea4d7dee76deb2456af940f4cac49b --- manifest | 12 +- manifest.uuid | 2 +- test/walthread.test | 354 +++++++++++++++++++++++++++++--------------- 3 files changed, 240 insertions(+), 128 deletions(-) diff --git a/manifest b/manifest index 8d7772b612..d16501f28f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\sto\swalthread.test\sfor\schanging\sbetween\sWAL\sand\srollback\smodes. -D 2010-04-28T18:17:23 +C Add\stests\sto\swalthread.test. +D 2010-04-29T08:47:28 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -764,7 +764,7 @@ F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 287a69d662939604f2e0452dace2cec8ef634d5e F test/walmode.test 40119078da084e6a7403ba57485d5a86ee0e2646 F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 -F test/walthread.test ce1d0fc9905cf8a7cf1a61dbcafcbd3434166236 +F test/walthread.test 8fdce1721bf01e88a1f83a93ce7ae2e7668bcb26 F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 F test/where2.test 45eacc126aabb37959a387aa83e59ce1f1f03820 F test/where3.test aa44a9b29e8c9f3d7bb94a3bb3a95b31627d520d @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 13d2d5a66e9eaa81aa6314354201ee1fbd2b3824 -R 03ea219fce1817690b6cb9bee0a76d8a +P da229e44bd4a5d512261da05958d560808c9889f +R f40b5481d2db02fd1a8b59b3cbe35412 U dan -Z e37b4277b60d879edbf1e6cb6d434b3c +Z 9a86a68774ea80363d82258dc9f803a1 diff --git a/manifest.uuid b/manifest.uuid index c424e5c417..cfa4e15520 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -da229e44bd4a5d512261da05958d560808c9889f \ No newline at end of file +9e891e7543ea4d7dee76deb2456af940f4cac49b \ No newline at end of file diff --git a/test/walthread.test b/test/walthread.test index 057ba2396f..353b7e42dd 100644 --- a/test/walthread.test +++ b/test/walthread.test @@ -21,11 +21,12 @@ if {[run_thread_tests]==0} { finish_test ; return } set sqlite_walsummary_mmap_incr 64 -# The number of threads to start. And the amount of time to run the test -# for. Respectively. +# How long, in seconds, to run each test for. If a test is set to run for +# 0 seconds, it is omitted entirely. # -set NTHREAD 10 -set SECONDS 5 +set seconds(walthread-1) 0 +set seconds(walthread-2) 0 +set seconds(walthread-3) 20 # The parameter is the name of a variable in the callers context. The # variable may or may not exist when this command is invoked. @@ -42,6 +43,13 @@ proc wait_for_var {varname} { uplevel [list set $varname] } +# The argument is the name of a list variable in the callers context. The +# first element of the list is removed and returned. For example: +# +# set L {a b c} +# set x [lshift L] +# assert { $x == "a" && $L == "b c" } +# proc lshift {lvar} { upvar $lvar L set ret [lindex $L 0] @@ -59,11 +67,10 @@ proc lshift {lvar} { # -init SCRIPT Script to run before test. # -thread NAME COUNT SCRIPT Scripts to run in threads (or processes). # -processes BOOLEAN True to use processes instead of threads. +# -check SCRIPT Script to run after test. # proc do_thread_test {args} { - #if {[string match walthread-2* [lindex $args 0]]==0} return - set A $args set P(testname) [lshift A] @@ -71,6 +78,10 @@ proc do_thread_test {args} { set P(init) "" set P(threads) [list] set P(processes) 0 + set P(check) { + set ic [db eval "PRAGMA integrity_check"] + if {$ic != "ok"} { error $ic } + } unset -nocomplain ::done @@ -89,6 +100,10 @@ proc do_thread_test {args} { set P(processes) [lshift A] } + -check { + set P(check) [lshift A] + } + -thread { set name [lshift A] set count [lshift A] @@ -102,6 +117,11 @@ proc do_thread_test {args} { } } + if {$P(seconds) == 0} { + puts "Skipping $P(testname)" + return + } + puts "Running $P(testname) for $P(seconds) seconds..." catch { db close } @@ -117,23 +137,34 @@ proc do_thread_test {args} { set prg [lindex $T 2] for {set i 1} {$i <= $count} {incr i} { - set program [string map [list %TEST% $prg %SECONDS% $P(seconds) %I% $i] { + set vars " + set E(pid) $i + set E(nthread) $count + set E(seconds) $P(seconds) + " + set program [string map [list %TEST% $prg %VARS% $vars] { - set tid %I% + %VARS% proc usleep {ms} { set ::usleep 0 after $ms {set ::usleep 1} vwait ::usleep } + + proc integrity_check {{db db}} { + set ic [$db eval {PRAGMA integrity_check}] + if {$ic != "ok"} {error $ic} + } + proc busyhandler {n} { usleep 10 ; return 0 } sqlite3 db test.db db busy busyhandler - db eval { SELECT randomblob($tid*5) } + db eval { SELECT randomblob($E(pid)*5) } set ::finished 0 - after [expr %SECONDS% * 1000] {set ::finished 1} + after [expr $E(seconds) * 1000] {set ::finished 1} proc tt_continue {} { update ; expr ($::finished==0) } set rc [catch { %TEST% } msg] @@ -166,10 +197,28 @@ proc do_thread_test {args} { append report " $name $reslist" } puts $report + + sqlite3 db test.db + set res "" + if {[catch $P(check) msg]} { set res $msg } + do_test $P(testname).check [list set {} $res] "" } +# A wrapper around [do_thread_test] which runs the specified test twice. +# Once using processes, once using threads. This command takes the same +# arguments as [do_thread_test], except specifying the -processes switch +# is illegal. +# +proc do_thread_test2 {args} { + set name [lindex $args 0] + if {[lsearch $args -processes]>=0} { error "bad option: -processes"} + uplevel [lreplace $args 0 0 do_thread_test "$name-threads" -processes 0] + uplevel [lreplace $args 0 0 do_thread_test "$name-processes" -processes 1] +} + + #-------------------------------------------------------------------------- -# Start NTHREAD threads. Each thread performs both read and write +# Start 10 threads. Each thread performs both read and write # transactions. Each read transaction consists of: # # 1) Reading the md5sum of all but the last table row, @@ -192,124 +241,187 @@ proc do_thread_test {args} { # 1) Execute "PRAGMA checkpoint" # 2) Sleep for 500 ms. # - -foreach {mode name} { - 0 walthread-1-threads - 1 walthread-1-processes -} { - do_thread_test $name -processes $mode -seconds $SECONDS -init { - execsql { - PRAGMA journal_mode = WAL; - CREATE TABLE t1(x PRIMARY KEY); - PRAGMA lock_status; - INSERT INTO t1 VALUES(randomblob(100)); - INSERT INTO t1 VALUES(randomblob(100)); - INSERT INTO t1 SELECT md5sum(x) FROM t1; - } - } -thread main $NTHREAD { - - proc read_transaction {} { - set results [db eval { - BEGIN; - PRAGMA integrity_check; - SELECT md5sum(x) FROM t1 WHERE rowid != (SELECT max(rowid) FROM t1); - SELECT x FROM t1 WHERE rowid = (SELECT max(rowid) FROM t1); - SELECT md5sum(x) FROM t1 WHERE rowid != (SELECT max(rowid) FROM t1); - COMMIT; - }] - - if {[llength $results]!=4 - || [lindex $results 0] != "ok" - || [lindex $results 1] != [lindex $results 2] - || [lindex $results 2] != [lindex $results 3] - } { - error "Failed read transaction: $results" - } - } - - proc write_transaction {} { - db eval { - BEGIN; - INSERT INTO t1 VALUES(randomblob(100)); - INSERT INTO t1 VALUES(randomblob(100)); - INSERT INTO t1 SELECT md5sum(x) FROM t1; - COMMIT; - } - } - - set nRun 0 - while {[tt_continue]} { - read_transaction - write_transaction - incr nRun - } - set nRun - - } -thread ckpt 1 { - set nRun 0 - while {[tt_continue]} { - db eval "PRAGMA checkpoint" - usleep 500 - incr nRun - } - set nRun +do_thread_test2 walthread-1 -seconds $seconds(walthread-1) -init { + execsql { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(x PRIMARY KEY); + PRAGMA lock_status; + INSERT INTO t1 VALUES(randomblob(100)); + INSERT INTO t1 VALUES(randomblob(100)); + INSERT INTO t1 SELECT md5sum(x) FROM t1; } +} -thread main 10 { + + proc read_transaction {} { + set results [db eval { + BEGIN; + PRAGMA integrity_check; + SELECT md5sum(x) FROM t1 WHERE rowid != (SELECT max(rowid) FROM t1); + SELECT x FROM t1 WHERE rowid = (SELECT max(rowid) FROM t1); + SELECT md5sum(x) FROM t1 WHERE rowid != (SELECT max(rowid) FROM t1); + COMMIT; + }] + + if {[llength $results]!=4 + || [lindex $results 0] != "ok" + || [lindex $results 1] != [lindex $results 2] + || [lindex $results 2] != [lindex $results 3] + } { + error "Failed read transaction: $results" + } + } + + proc write_transaction {} { + db eval { + BEGIN; + INSERT INTO t1 VALUES(randomblob(100)); + INSERT INTO t1 VALUES(randomblob(100)); + INSERT INTO t1 SELECT md5sum(x) FROM t1; + COMMIT; + } + } + + set nRun 0 + while {[tt_continue]} { + read_transaction + write_transaction + incr nRun + } + set nRun + +} -thread ckpt 1 { + set nRun 0 + while {[tt_continue]} { + db eval "PRAGMA checkpoint" + usleep 500 + incr nRun + } + set nRun } #-------------------------------------------------------------------------- +# This test has clients run the following procedure as fast as possible +# in a loop: # -foreach {mode name} { - 0 walthread-2-threads - 1 walthread-2-processes -} { - do_thread_test $name -processes $mode -seconds $SECONDS -init { - execsql { CREATE TABLE t1(x INTEGER PRIMARY KEY, y UNIQUE) } - } -thread RB 2 { +# 1. Open a database handle. +# 2. Execute a read-only transaction on the db. +# 3. Do "PRAGMA journal_mode = XXX", where XXX is one of WAL or DELETE. +# Ignore any SQLITE_BUSY error. +# 4. Execute a write transaction to insert a row into the db. +# 5. Run "PRAGMA integrity_check" +# +# At present, there are 4 clients in total. 2 do "journal_mode = WAL", and +# two do "journal_mode = DELETE". +# +# Each client returns a string of the form "W w, R r", where W is the +# number of write-transactions performed using a WAL journal, and D is +# the number of write-transactions performed using a rollback journal. +# For example, "192 w, 185 r". +# +do_thread_test2 walthread-2 -seconds $seconds(walthread-2) -init { + execsql { CREATE TABLE t1(x INTEGER PRIMARY KEY, y UNIQUE) } +} -thread RB 2 { - db close - set nRun 0 - set nDel 0 - while {[tt_continue]} { - sqlite3 db test.db - db busy busyhandler - db eval { SELECT * FROM sqlite_master } - catch { db eval { PRAGMA journal_mode = DELETE } } - db eval { - BEGIN; - INSERT INTO t1 VALUES(NULL, randomblob(100+$tid)); - } - incr nRun 1 - incr nDel [file exists test.db-journal] - db eval COMMIT - - set ic [db eval {PRAGMA integrity_check}] - if {$ic != "ok"} { error $ic } - db close + db close + set nRun 0 + set nDel 0 + while {[tt_continue]} { + sqlite3 db test.db + db busy busyhandler + db eval { SELECT * FROM sqlite_master } + catch { db eval { PRAGMA journal_mode = DELETE } } + db eval { + BEGIN; + INSERT INTO t1 VALUES(NULL, randomblob(100+$E(pid))); } - list $nRun $nDel - - } -thread WAL 2 { - db close - set nRun 0 - set nWal 0 - while {[tt_continue]} { - sqlite3 db test.db - db busy busyhandler - db eval { SELECT * FROM sqlite_master } - catch { db eval { PRAGMA journal_mode = WAL } } - db eval { - BEGIN; - INSERT INTO t1 VALUES(NULL, randomblob(110+$tid)); - } - incr nRun 1 - incr nWal [file exists test.db-wal] - db eval COMMIT - - set ic [db eval {PRAGMA integrity_check}] - if {$ic != "ok"} { error $ic } - db close + incr nRun 1 + incr nDel [file exists test.db-journal] + if {[file exists test.db-journal] + [file exists test.db-wal] != 1} { + error "File-system looks bad..." } - list $nRun $nWal + db eval COMMIT + + integrity_check + db close + } + list $nRun $nDel + set {} "[expr $nRun-$nDel] w, $nDel r" + +} -thread WAL 2 { + db close + set nRun 0 + set nDel 0 + while {[tt_continue]} { + sqlite3 db test.db + db busy busyhandler + db eval { SELECT * FROM sqlite_master } + catch { db eval { PRAGMA journal_mode = WAL } } + db eval { + BEGIN; + INSERT INTO t1 VALUES(NULL, randomblob(110+$E(pid))); + } + incr nRun 1 + incr nDel [file exists test.db-journal] + if {[file exists test.db-journal] + [file exists test.db-wal] != 1} { + error "File-system looks bad..." + } + db eval COMMIT + + integrity_check + db close + } + set {} "[expr $nRun-$nDel] w, $nDel r" +} + +do_thread_test2 walthread-3 -seconds $seconds(walthread-3) -init { + execsql { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(cnt PRIMARY KEY, sum1, sum2); + CREATE INDEX i1 ON t1(sum1); + CREATE INDEX i2 ON t1(sum2); + INSERT INTO t1 VALUES(0, 0, 0); + } +} -thread t 10 { + + set nextwrite $E(pid) + + proc wal_hook {zDb nEntry} { + if {$nEntry>10} { return 1 } + return 0 + } + db wal_hook wal_hook + + while {[tt_continue]} { + set max 0 + while { $max != ($nextwrite-1) && [tt_continue] } { + set max [db eval { SELECT max(cnt) FROM t1 }] + } + + if {[tt_continue]} { + set sum1 [db eval { SELECT sum(cnt) FROM t1 }] + set sum2 [db eval { SELECT sum(sum1) FROM t1 }] + db eval { INSERT INTO t1 VALUES($nextwrite, $sum1, $sum2) } + incr nextwrite $E(nthread) + integrity_check + } + } + + set {} ok +} -check { + puts " Final db contains [db eval {SELECT count(*) FROM t1}] rows" + puts " Final integrity-check says: [db eval {PRAGMA integrity_check}]" + + # Check that the contents of the database are Ok. + set c 0 + set s1 0 + set s2 0 + db eval { SELECT cnt, sum1, sum2 FROM t1 ORDER BY cnt } { + if {$c != $cnt || $s1 != $sum1 || $s2 != $sum2} { + error "database content is invalid" + } + incr s2 $s1 + incr s1 $c + incr c 1 } } From 31c03907fd41d9be65f74efaf38b69ae98cb8411 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 29 Apr 2010 14:51:33 +0000 Subject: [PATCH 51/89] Fix a but in the WAL checkpoint code causing SQLite to use an inconsistent cache in a subsequent transaction. FossilOrigin-Name: d1cadeed4eea20d8892726cc8c69f4f3f57d0cd4 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/wal.c | 12 +++++++++++- test/wal.test | 45 +++++++++++++++++++++++++++++++++++++++++++++ test/walthread.test | 35 ++++++++++++++++++++++++++++++++--- 5 files changed, 97 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index d16501f28f..52e87c60a9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sto\swalthread.test. -D 2010-04-29T08:47:28 +C Fix\sa\sbut\sin\sthe\sWAL\scheckpoint\scode\scausing\sSQLite\sto\suse\san\sinconsistent\scache\sin\sa\ssubsequent\stransaction. +D 2010-04-29T14:51:33 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -221,7 +221,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c d63318e8e73f9ed1a6f3c277f71022024f38a7c3 +F src/wal.c b1c6868b975a67f6f4dd2cd612eeace4117eb98f F src/wal.h c60781e78e394af07ece3b64a11192eb442241c1 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 @@ -758,13 +758,13 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test bc99b1656b721fddd2dfeea0b0e77b147e40e6da +F test/wal.test d6fa4631310f95ac7d22f74b29c4b197f519d65b F test/walbak.test f6fde9a5f59d0c697cb1f4af7876178c2f69a7ba F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 287a69d662939604f2e0452dace2cec8ef634d5e F test/walmode.test 40119078da084e6a7403ba57485d5a86ee0e2646 F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 -F test/walthread.test 8fdce1721bf01e88a1f83a93ce7ae2e7668bcb26 +F test/walthread.test c9c3b5d19eeb4968254d93f61a7ccd6e8b63f554 F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 F test/where2.test 45eacc126aabb37959a387aa83e59ce1f1f03820 F test/where3.test aa44a9b29e8c9f3d7bb94a3bb3a95b31627d520d @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P da229e44bd4a5d512261da05958d560808c9889f -R f40b5481d2db02fd1a8b59b3cbe35412 +P 9e891e7543ea4d7dee76deb2456af940f4cac49b +R 1d9b135e5479cce531b0c9f4c717d59a U dan -Z 9a86a68774ea80363d82258dc9f803a1 +Z f79bb92cf1fa704536cc5c5230dfad45 diff --git a/manifest.uuid b/manifest.uuid index cfa4e15520..8d47b72c97 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9e891e7543ea4d7dee76deb2456af940f4cac49b \ No newline at end of file +d1cadeed4eea20d8892726cc8c69f4f3f57d0cd4 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index c7478d0e82..8a8f4043d1 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1880,6 +1880,7 @@ int sqlite3WalCheckpoint( void *pBusyHandlerArg /* Argument to pass to xBusyHandler */ ){ int rc; /* Return code */ + int isChanged = 0; /* True if a new wal-index header is loaded */ assert( !pLog->isLocked ); @@ -1899,10 +1900,19 @@ int sqlite3WalCheckpoint( } /* Copy data from the log to the database file. */ - rc = logSummaryReadHdr(pLog, 0); + rc = logSummaryReadHdr(pLog, &isChanged); if( rc==SQLITE_OK ){ rc = logCheckpoint(pLog, pFd, sync_flags, zBuf); } + if( isChanged ){ + /* If a new wal-index header was loaded before the checkpoint was + ** performed, then the pager-cache associated with log pLog is now + ** out of date. So zero the cached wal-index header to ensure that + ** next time the pager opens a snapshot on this database it knows that + ** the cache needs to be reset. + */ + memset(&pLog->hdr, 0, sizeof(LogSummaryHdr)); + } /* Release the locks. */ logLockRegion(pLog, LOG_REGION_A|LOG_REGION_B|LOG_REGION_C, LOG_UNLOCK); diff --git a/test/wal.test b/test/wal.test index ca4d606d23..7b79d0e232 100644 --- a/test/wal.test +++ b/test/wal.test @@ -918,5 +918,50 @@ foreach code [list { db close } +#------------------------------------------------------------------------- +# Check a fun corruption case has been fixed. +# +# The problem was that after performing a checkpoint using a connection +# that had an out-of-date pager-cache, the next time the connection was +# used it did not realize the cache was out-of-date and proceeded to +# operate with an inconsistent cache. Leading to corruption. +# + +catch { db close } +catch { db2 close } +catch { db3 close } +file delete -force test.db test.db-wal +sqlite3 db test.db +sqlite3 db2 test.db + +do_test wal-14 { + execsql { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES(randomblob(10), randomblob(100)); + INSERT INTO t1 SELECT randomblob(10), randomblob(100) FROM t1; + INSERT INTO t1 SELECT randomblob(10), randomblob(100) FROM t1; + INSERT INTO t1 SELECT randomblob(10), randomblob(100) FROM t1; + } + + db2 eval { + INSERT INTO t1 SELECT randomblob(10), randomblob(100); + INSERT INTO t1 SELECT randomblob(10), randomblob(100); + INSERT INTO t1 SELECT randomblob(10), randomblob(100); + INSERT INTO t1 SELECT randomblob(10), randomblob(100); + } + + # After executing the "PRAGMA checkpoint", connection [db] was being + # left with an inconsistent cache. Running the CREATE INDEX statement + # in this state led to database corruption. + catchsql { + PRAGMA checkpoint; + CREATE INDEX i1 on t1(b); + } + + db2 eval { PRAGMA integrity_check } +} {ok} + + finish_test diff --git a/test/walthread.test b/test/walthread.test index 353b7e42dd..efd321df3b 100644 --- a/test/walthread.test +++ b/test/walthread.test @@ -24,9 +24,10 @@ set sqlite_walsummary_mmap_incr 64 # How long, in seconds, to run each test for. If a test is set to run for # 0 seconds, it is omitted entirely. # -set seconds(walthread-1) 0 -set seconds(walthread-2) 0 +set seconds(walthread-1) 20 +set seconds(walthread-2) 20 set seconds(walthread-3) 20 +set seconds(walthread-4) 20 # The parameter is the name of a variable in the callers context. The # variable may or may not exist when this command is invoked. @@ -373,7 +374,7 @@ do_thread_test2 walthread-2 -seconds $seconds(walthread-2) -init { set {} "[expr $nRun-$nDel] w, $nDel r" } -do_thread_test2 walthread-3 -seconds $seconds(walthread-3) -init { +do_thread_test walthread-3 -seconds $seconds(walthread-3) -init { execsql { PRAGMA journal_mode = WAL; CREATE TABLE t1(cnt PRIMARY KEY, sum1, sum2); @@ -425,5 +426,33 @@ do_thread_test2 walthread-3 -seconds $seconds(walthread-3) -init { } } +do_thread_test2 walthread-4 -seconds $seconds(walthread-4) -init { + execsql { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b UNIQUE); + } +} -thread r 1 { + # This connection only ever reads the database. Therefore the + # busy-handler is not required. Disable it to check that this is true. + db busy {} + while {[tt_continue]} integrity_check + set {} ok +} -thread w 1 { + + proc wal_hook {zDb nEntry} { + if {$nEntry>15} { return 1 } + return 0 + } + db wal_hook wal_hook + set row 1 + while {[tt_continue]} { + db eval { REPLACE INTO t1 VALUES($row, randomblob(300)) } + incr row + if {$row == 10} { set row 1 } + } + + set {} ok +} + finish_test From 185cca609b11b4e6c0a4bf4ae310e357b6649242 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 29 Apr 2010 14:58:53 +0000 Subject: [PATCH 52/89] Close all open database connections at the end of wal.test. FossilOrigin-Name: 3cc55a7568daa3796483f632e33478969e381cf5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/wal.test | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 52e87c60a9..a96719324c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbut\sin\sthe\sWAL\scheckpoint\scode\scausing\sSQLite\sto\suse\san\sinconsistent\scache\sin\sa\ssubsequent\stransaction. -D 2010-04-29T14:51:33 +C Close\sall\sopen\sdatabase\sconnections\sat\sthe\send\sof\swal.test. +D 2010-04-29T14:58:53 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -758,7 +758,7 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test d6fa4631310f95ac7d22f74b29c4b197f519d65b +F test/wal.test 67f1b8ec4aa2251db2b4d33f43ea9b1ba1711210 F test/walbak.test f6fde9a5f59d0c697cb1f4af7876178c2f69a7ba F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 287a69d662939604f2e0452dace2cec8ef634d5e @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 9e891e7543ea4d7dee76deb2456af940f4cac49b -R 1d9b135e5479cce531b0c9f4c717d59a +P d1cadeed4eea20d8892726cc8c69f4f3f57d0cd4 +R 606ec50aa0ca99ee2a7fabcf3364b376 U dan -Z f79bb92cf1fa704536cc5c5230dfad45 +Z c947c540d09aca713bd3451e1bec9253 diff --git a/manifest.uuid b/manifest.uuid index 8d47b72c97..32ff2b16a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d1cadeed4eea20d8892726cc8c69f4f3f57d0cd4 \ No newline at end of file +3cc55a7568daa3796483f632e33478969e381cf5 \ No newline at end of file diff --git a/test/wal.test b/test/wal.test index 7b79d0e232..be654c0237 100644 --- a/test/wal.test +++ b/test/wal.test @@ -962,6 +962,7 @@ do_test wal-14 { db2 eval { PRAGMA integrity_check } } {ok} - +catch { db close } +catch { db2 close } finish_test From 7b6944047e7d7e8e1b6c975800d566eaf51a9390 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 29 Apr 2010 15:17:48 +0000 Subject: [PATCH 53/89] Progress towards a VFS that will support WAL. Locking code is in place but is untested. Still no support for the DMS. FossilOrigin-Name: 1bde41cf081570ad257f927b641e752dff4ed014 --- manifest | 28 ++- manifest.uuid | 2 +- src/os_common.h | 2 + src/os_unix.c | 647 +++++++++++++++++++++++++++++++++++++++++++----- src/sqlite.h.in | 4 +- 5 files changed, 612 insertions(+), 71 deletions(-) diff --git a/manifest b/manifest index a96719324c..81762c0b4c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,8 @@ -C Close\sall\sopen\sdatabase\sconnections\sat\sthe\send\sof\swal.test. -D 2010-04-29T14:58:53 +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 + +C Progress\stowards\sa\sVFS\sthat\swill\ssupport\sWAL.\s\sLocking\scode\sis\sin\splace\nbut\sis\suntested.\s\sStill\sno\ssupport\sfor\sthe\sDMS. +D 2010-04-29T15:17:48 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -148,9 +151,9 @@ F src/mutex_w32.c 4cc201c1bfd11d1562810554ff5500e735559d7e F src/notify.c cbfa66a836da3a51567209636e6a94059c137930 F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c -F src/os_common.h 240c88b163b02c21a9f21f87d49678a0aa21ff30 +F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 -F src/os_unix.c c8b7d1e0f4315e08cf54324ca706e1a1a80f5fc2 +F src/os_unix.c 340c503705cdb447753a55771911a92aa41d27c8 F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 F src/pager.c b4a41030860229e80295fa1f37addab24d21799c F src/pager.h cee4487ab4f0911dd9f22a40e3cd55afdb7ef444 @@ -166,7 +169,7 @@ F src/resolve.c ac5f1a713cd1ae77f08b83cc69581e11bf5ae6f9 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c c03d8a0565febcde8c6a12c5d77d065fddae889b F src/shell.c c40427c7245535a04a9cb4a417b6cc05c022e6a4 -F src/sqlite.h.in caf60df0991a14e22cce8243b9caa1c1dbd09d42 +F src/sqlite.h.in 0cc43a0fa5d26cd26d2609b84ecf094a159e9cb9 F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89 F src/sqliteInt.h 700a2df7b8dfe57c3b8d83c52ff40928e026220c F src/sqliteLimit.h 3afab2291762b5d09ae20c18feb8e9fa935a60a6 @@ -808,7 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P d1cadeed4eea20d8892726cc8c69f4f3f57d0cd4 -R 606ec50aa0ca99ee2a7fabcf3364b376 -U dan -Z c947c540d09aca713bd3451e1bec9253 +P 3cc55a7568daa3796483f632e33478969e381cf5 +R 1f2ad602233779dda1e35a2877201d1f +U drh +Z 5bb08c68390b552fdeb1ef93301ae98c +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.6 (GNU/Linux) + +iD8DBQFL2aMhoxKgR168RlERAntLAKCNwNXdF9LrZZqkEN8f3gsvrSxoXACaA2Ku +SvIawhrGV2wraxrir5MNan8= +=Vfpz +-----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index 32ff2b16a0..000aaadb5f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3cc55a7568daa3796483f632e33478969e381cf5 \ No newline at end of file +1bde41cf081570ad257f927b641e752dff4ed014 \ No newline at end of file diff --git a/src/os_common.h b/src/os_common.h index 6a2e2d93be..e6de2b25f7 100644 --- a/src/os_common.h +++ b/src/os_common.h @@ -40,6 +40,7 @@ int sqlite3OSTrace = 0; if(sqlite3OSTrace) sqlite3DebugPrintf(X,Y,Z,A,B,C) #define OSTRACE7(X,Y,Z,A,B,C,D) \ if(sqlite3OSTrace) sqlite3DebugPrintf(X,Y,Z,A,B,C,D) +#define OSTRACE(X) if( sqlite3OSTrace ) sqlite3DebugPrintf X #else #define OSTRACE1(X) #define OSTRACE2(X,Y) @@ -48,6 +49,7 @@ int sqlite3OSTrace = 0; #define OSTRACE5(X,Y,Z,A,B) #define OSTRACE6(X,Y,Z,A,B,C) #define OSTRACE7(X,Y,Z,A,B,C,D) +#define OSTRACE(X) #endif /* diff --git a/src/os_unix.c b/src/os_unix.c index 7e3930a1e6..634f5ee218 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4562,42 +4562,355 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ return 0; } +/* Forward reference */ +typedef struct unixShm unixShm; +typedef struct unixShmFile unixShmFile; + /* -** Structure used internally by this VFS to record the state of an -** open shared memory segment. +** Object used to represent a single file opened and mmapped to provide +** shared memory. When multiple threads all reference the same +** log-summary, each thread has its own unixFile object, but they all +** point to a single instance of this object. In other words, each +** log-summary is opened only once per process. +** +** unixMutexHeld() must be true when creating or destroying +** this object or while reading or writing the following fields: +** +** nRef +** pNext +** +** The following fields are read-only after the object is created: +** +** fid +** zFilename +** +** Either unixShmFile.mutex must be held or unixShmFile.nRef==0 and +** unixMutexHeld() is true when reading or writing any other field +** in this structure. */ -struct unixShm { - sqlite3_vfs *pVfs; /* VFS that opened this shared-memory segment */ - int size; /* Size of the shared memory area */ - char *pBuf; /* Pointer to the beginning */ - unixFile fd; /* The open file descriptor */ +struct unixShmFile { + struct unixFileId fid; /* Unique file identifier */ + sqlite3_mutex *mutex; /* Mutex to access this object */ + sqlite3_mutex *mutexBuf; /* Mutex to access zBuf[] */ + sqlite3_mutex *mutexRecov; /* The RECOVER mutex */ + char *zFilename; /* Name of the file */ + int size; /* Size of the file */ + int h; /* Open file descriptor */ + char *pMMapBuf; /* Where currently mmapped() */ + int nReadPrefix; /* Number of SQLITE_SHM_READ_PREFIX locks */ + int nReadFull; /* Number of SQLITE_SHM_READ_FULL locks */ + int nRef; /* Number of unixShm objects pointing to this */ + unixShm *pFirst; /* All unixShm objects pointing to this */ + unixShmFile *pNext; /* Next in list of all unixShmFile objects */ +#ifdef SQLITE_DEBUG + u8 exclMask; /* Mask of exclusive locks held */ + u8 sharedMask; /* Mask of shared locks held */ + u8 nextShmId; /* Next available unixShm.id value */ +#endif }; /* -** Close a shared-memory segment +** A global array of all unixShmFile objects. +** +** The unixMutexHeld() must be true while reading or writing this list. */ -static int unixShmClose(sqlite3_shm *pSharedMem){ - struct unixShm *p = (struct unixShm*)pSharedMem; - if( p && p->pVfs ){ - if( p->pBuf ){ - munmap(p->pBuf, p->size); - } - if( p->fd.pMethod ){ - p->fd.pMethod->xClose((sqlite3_file*)&p->fd); - } - memset(p, 0, sizeof(*p)); - sqlite3_free(p); - } - return SQLITE_OK; -} +static unixShmFile *unixShmFileList = 0; + +/* +** Structure used internally by this VFS to record the state of an +** open shared memory connection. +** +** unixShm.pFile->mutex must be held while reading or writing the +** unixShm.pNext and unixShm.locks[] elements. +** +** The unixShm.pFile element is initialized when the object is created +** and is read-only thereafter. +*/ +struct unixShm { + unixShmFile *pFile; /* The underlying unixShmFile object */ + unixShm *pNext; /* Next unixShm with the same unixShmFile */ + u8 lockState; /* Current lock state */ + u8 readLock; /* Which of the two read-lock states to use */ + u8 hasMutex; /* True if holding the unixShmFile mutex */ + u8 hasMutexBuf; /* True if holding pFile->mutexBuf */ + u8 hasMutexRecov; /* True if holding pFile->mutexRecov */ + u8 sharedMask; /* Mask of shared locks held */ + u8 exclMask; /* Mask of exclusive locks held */ +#ifdef SQLITE_DEBUG + u8 id; /* Id of this connection with its unixShmFile */ +#endif +}; /* ** Size increment by which shared memory grows */ #define SQLITE_UNIX_SHM_INCR 4096 +/* +** Constants used for locking +*/ +#define UNIX_SHM_BASE 32 /* Byte offset of the first lock byte */ +#define UNIX_SHM_MUTEX 0x01 /* Mask for MUTEX lock */ +#define UNIX_SHM_DMS 0x04 /* Mask for Dead-Man-Switch lock */ +#define UNIX_SHM_A 0x10 /* Mask for region locks... */ +#define UNIX_SHM_B 0x20 +#define UNIX_SHM_C 0x40 +#define UNIX_SHM_D 0x80 + +#ifdef SQLITE_DEBUG +/* +** Return a pointer to a nul-terminated string in static memory that +** describes a locking mask. The string is of the form "MSABCD" with +** each character representing a lock. "M" for MUTEX, "S" for DMS, +** and "A" through "D" for the region locks. If a lock is held, the +** letter is shown. If the lock is not held, the letter is converted +** to ".". +** +** This routine is for debugging purposes only and does not appear +** in a production build. +*/ +static const char *unixShmLockString(u8 mask){ + static char zBuf[48]; + static int iBuf = 0; + char *z; + + z = &zBuf[iBuf]; + iBuf += 8; + if( iBuf>=sizeof(zBuf) ) iBuf = 0; + + z[0] = (mask & UNIX_SHM_MUTEX) ? 'M' : '.'; + z[1] = (mask & UNIX_SHM_DMS) ? 'S' : '.'; + z[2] = (mask & UNIX_SHM_A) ? 'A' : '.'; + z[3] = (mask & UNIX_SHM_B) ? 'B' : '.'; + z[4] = (mask & UNIX_SHM_C) ? 'C' : '.'; + z[5] = (mask & UNIX_SHM_D) ? 'D' : '.'; + z[6] = 0; + return z; +} +#endif /* SQLITE_DEBUG */ + +/* +** Apply posix advisory locks for all bytes identified in lockMask. +** +** lockMask might contain multiple bits but all bits are guaranteed +** to be contiguous. +** +** Locks block if the UNIX_SHM_MUTEX bit is set and are non-blocking +** otherwise. +*/ +static int unixShmSystemLocks( + unixShmFile *pFile, /* Apply locks to this open shared-memory segment */ + int lockType, /* F_UNLCK, F_RDLCK, or F_WRLCK */ + u8 lockMask /* Which bytes to lock or unlock */ +){ + struct flock f; /* The posix advisory locking structure */ + int lockOp; /* The opcode for fcntl() */ + int i; /* Offset into the locking byte range */ + int rc; /* Result code form fcntl() */ + u8 mask; /* Mask of bits in lockMask */ + + /* Initialize the locking parameters */ + memset(&f, 0, sizeof(f)); + f.l_type = lockType; + f.l_whence = SEEK_SET; + if( (lockMask & UNIX_SHM_MUTEX)!=0 && lockType!=F_UNLCK ){ + lockOp = F_SETLKW; + }else{ + lockOp = F_SETLK; + } + + /* Find the first bit in lockMask that is set */ + for(i=0, mask=0x01; mask!=0 && (lockMask&mask)==0; mask <<= 1, i++){} + assert( mask!=0 ); + f.l_start = i+UNIX_SHM_BASE; + f.l_len = 1; + + /* Extend the locking range for each additional bit that is set */ + mask <<= 1; + while( mask!=0 && (lockMask & mask)!=0 ){ + f.l_len++; + } + + /* Verify that all bits set in lockMask are contiguous */ + assert( mask==0 || (lockMask & ~(mask | (mask-1)))==0 ); + + /* Acquire the system-level lock */ + rc = (fcntl(pFile->h, lockOp, &f)==0) ? SQLITE_OK : SQLITE_BUSY; + + /* Update the global lock state and do debug tracing */ +#ifdef SQLITE_DEBUG + OSTRACE(("SHM-LOCK ")); + if( rc==SQLITE_OK ){ + if( lockType==F_UNLCK ){ + OSTRACE(("unlock ok")); + pFile->exclMask &= ~lockMask; + pFile->sharedMask &= ~lockMask; + }else if( lockType==F_RDLCK ){ + OSTRACE(("read-lock ok")); + pFile->exclMask &= ~lockMask; + pFile->sharedMask |= lockMask; + }else{ + assert( lockType==F_WRLCK ); + OSTRACE(("write-lock ok")); + pFile->exclMask |= lockMask; + pFile->sharedMask &= ~lockMask; + } + }else{ + if( lockType==F_UNLCK ){ + OSTRACE(("unlock failed")); + }else if( lockType==F_RDLCK ){ + OSTRACE(("read-lock failed")); + }else{ + assert( lockType==F_WRLCK ); + OSTRACE(("write-lock failed")); + } + } + OSTRACE((" - change requested %s - afterwards %s,%s\n", + unixShmLockString(lockMask), + unixShmLockString(pFile->sharedMask), + unixShmLockString(pFile->exclMask))); +#endif + + return rc; +} + +/* +** For connection p, unlock all of the locks identified by the unlockMask +** parameter. +*/ +static int unixShmUnlock( + unixShmFile *pFile, /* The underlying shared-memory file */ + unixShm *p, /* The connection to be unlocked */ + u8 unlockMask /* Mask of locks to be unlocked */ +){ + int rc; /* Result code */ + unixShm *pX; /* For looping over all sibling connections */ + u8 allMask; /* Union of locks held by connections other than "p" */ + + /* We never try to unlock locks that we do not hold */ + assert( ((p->exclMask|p->sharedMask) & unlockMask)==unlockMask ); + + /* Compute locks held by sibling connections */ + for(pX=pFile->pFirst; pX; pX=pX->pNext){ + if( pX==p ) continue; + assert( (pX->exclMask & unlockMask)==0 ); + allMask |= pX->sharedMask; + } + + /* Unlock the system-level locks */ + if( (unlockMask & allMask)!=unlockMask ){ + rc = unixShmSystemLocks(pFile, F_UNLCK, unlockMask & ~allMask); + }else{ + rc = SQLITE_OK; + } + + /* Undo the local locks */ + if( rc==SQLITE_OK ){ + p->exclMask &= ~unlockMask; + p->sharedMask &= ~unlockMask; + } + return rc; +} + +/* +** Get reader locks for connection p on all locks in the readMask parameter. +*/ +static int unixShmSharedLock( + unixShmFile *pFile, /* The underlying shared-memory file */ + unixShm *p, /* The connection to get the shared locks */ + u8 readMask /* Mask of shared locks to be acquired */ +){ + int rc; /* Result code */ + unixShm *pX; /* For looping over all sibling connections */ + u8 allShared; /* Union of locks held by connections other than "p" */ + + /* Find out which shared locks are already held by sibling connections. + ** If any sibling already holds an exclusive lock, go ahead and return + ** SQLITE_BUSY. + */ + for(pX=pFile->pFirst; pX; pX=pX->pNext){ + if( pX==p ) continue; + if( (pX->exclMask & readMask)!=0 ) return SQLITE_BUSY; + allShared |= pX->sharedMask; + } + + /* Get shared locks at the system level, if necessary */ + if( (~allShared) & readMask ){ + rc = unixShmSystemLocks(pFile, F_RDLCK, readMask); + }else{ + rc = SQLITE_OK; + } + + /* Get the local shared locks */ + if( rc==SQLITE_OK ){ + p->sharedMask |= readMask; + } + return rc; +} + +/* +** For connection p, get an exclusive lock on all locks identified in +** the writeMask parameter. +*/ +static int unixShmExclusiveLock( + unixShmFile *pFile, /* The underlying shared-memory file */ + unixShm *p, /* The connection to get the exclusive locks */ + u8 writeMask /* Mask of exclusive locks to be acquired */ +){ + int rc; /* Result code */ + unixShm *pX; /* For looping over all sibling connections */ + + /* Make sure no sibling connections hold locks that will block this + ** lock. If any do, return SQLITE_BUSY right away. + */ + for(pX=pFile->pFirst; pX; pX=pX->pNext){ + if( pX==p ) continue; + if( (pX->exclMask & writeMask)!=0 ) return SQLITE_BUSY; + if( (pX->sharedMask & writeMask)!=0 ) return SQLITE_BUSY; + } + + /* Get the exclusive locks at the system level. Then if successful + ** also mark the local connection as being locked. + */ + rc = unixShmSystemLocks(pFile, F_WRLCK, writeMask); + if( rc==SQLITE_OK ){ + p->sharedMask &= ~writeMask; + p->exclMask |= writeMask; + } + return rc; +} + +/* +** Purge the unixShmFileList list of all entries with unixShmFile.nRef==0. +** +** This is not a VFS shared-memory method; it is a utility function called +** by VFS shared-memory methods. +*/ +static void unixShmPurge(void){ + unixShmFile **pp; + unixShmFile *p; + assert( unixMutexHeld() ); + pp = &unixShmFileList; + while( (p = *pp)!=0 ){ + if( p->nRef==0 ){ + if( p->mutex ) sqlite3_mutex_free(p->mutex); + if( p->mutexBuf ) sqlite3_mutex_free(p->mutexBuf); + if( p->mutexRecov ) sqlite3_mutex_free(p->mutexRecov); + if( p->h>=0 ) close(p->h); + *pp = p->pNext; + sqlite3_free(p); + }else{ + pp = &p->pNext; + } + } +} + /* ** Open a shared-memory area. This implementation uses mmapped files. +** +** When opening a new shared-memory file, if no other instances of that +** file are currently open, in this process or in other processes, then +** the file must be truncated to zero length or have its header cleared. */ static int unixShmOpen( sqlite3_vfs *pVfs, /* The VFS */ @@ -4605,44 +4918,136 @@ static int unixShmOpen( sqlite3_shm **pShm /* Write the unixShm object created here */ ){ struct unixShm *p = 0; + struct unixShmFile *pFile = 0; int rc; - int outFlags; + struct unixFileId fid; struct stat sStat; p = sqlite3_malloc( sizeof(*p) ); - *pShm = (sqlite3_shm*)p; if( p==0 ) return SQLITE_NOMEM; memset(p, 0, sizeof(*p)); - p->pVfs = pVfs; - rc = pVfs->xOpen(pVfs, zName, (sqlite3_file*)&p->fd, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_MAIN_JOURNAL, - &outFlags); - if( rc!=SQLITE_OK ) goto shm_open_err; - rc = fstat(p->fd.h, &sStat); - if( rc!=0 ) goto shm_open_err; - if( sStat.st_sizefd.h, SQLITE_UNIX_SHM_INCR); - if( rc!=0 ) goto shm_open_err; - p->size = SQLITE_UNIX_SHM_INCR; - }else{ - p->size = sStat.st_size; + unixEnterMutex(); + rc = stat(zName, &sStat); + if( rc==0 ){ + memset(&fid, 0, sizeof(fid)); + fid.dev = sStat.st_dev; + fid.ino = sStat.st_ino; + for(pFile = unixShmFileList; pFile; pFile=pFile->pNext){ + if( memcmp(&pFile->fid, &fid, sizeof(fid))==0 ) break; + } + } + if( pFile==0 ){ + int nName = strlen(zName); + pFile = sqlite3_malloc( sizeof(*pFile) + nName + 1 ); + if( pFile==0 ){ + rc = SQLITE_NOMEM; + goto shm_open_err; + } + memset(pFile, 0, sizeof(pFile)); + pFile->zFilename = (char*)&pFile[1]; + memcpy(pFile->zFilename, zName, nName+1); + pFile->h = -1; + pFile->pNext = unixShmFileList; + unixShmFileList = pFile; + + pFile->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pFile->mutex==0 ){ + rc = SQLITE_NOMEM; + goto shm_open_err; + } + pFile->mutexBuf = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pFile->mutexBuf==0 ){ + rc = SQLITE_NOMEM; + goto shm_open_err; + } + pFile->mutexRecov = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pFile->mutexRecov==0 ){ + rc = SQLITE_NOMEM; + goto shm_open_err; + } + + pFile->h = open(zName, O_CREAT, 0664); + if( pFile->h<0 ){ + rc = SQLITE_CANTOPEN; + goto shm_open_err; + } + + rc = fstat(pFile->h, &sStat); + if( rc ){ + rc = SQLITE_CANTOPEN; + goto shm_open_err; + } + pFile->fid.dev = sStat.st_dev; + pFile->fid.ino = sStat.st_ino; + pFile->size = (int)sStat.st_size; + pFile->size = (pFile->size/SQLITE_UNIX_SHM_INCR)*SQLITE_UNIX_SHM_INCR; + if( pFile->size==0 ){ + pFile->size = SQLITE_UNIX_SHM_INCR; + rc = ftruncate(pFile->h, pFile->size); + if( rc ){ + rc = SQLITE_FULL; + goto shm_open_err; + } + } } - /* Map the file. */ - p->pBuf = mmap(0, p->size, PROT_READ|PROT_WRITE, MAP_SHARED, p->fd.h, 0); - if( p->pBuf==MAP_FAILED ){ - rc = SQLITE_IOERR; - goto shm_open_err; - } + p->pFile = pFile; + p->pNext = pFile->pFirst; +#ifdef SQLITE_DEBUG + p->id = pFile->nextShmId++; +#endif + pFile->pFirst = p; + pFile->nRef++; + *pShm = (sqlite3_shm*)p; + unixLeaveMutex(); return SQLITE_OK; shm_open_err: - unixShmClose((sqlite3_shm*)p); + unixShmPurge(); + sqlite3_free(p); + sqlite3_free(pFile); *pShm = 0; + unixLeaveMutex(); return rc; } +/* +** Close a connectioon to shared-memory. +*/ +static int unixShmClose(sqlite3_shm *pSharedMem){ + unixShm *p; /* The connection to be closed */ + unixShmFile *pFile; /* The underlying shared-memory file */ + unixShm **pp; /* For looping over sibling connections */ + int nRef; /* Number of connections to pFile */ + + p = (struct unixShm*)pSharedMem; + pFile = p->pFile; + + /* Verify that the connection being closed holds no locks */ + assert( p->exclMask==0 ); + assert( p->sharedMask==0 ); + + + /* Remove connection p from the set of connections associated with pFile */ + sqlite3_mutex_enter(pFile->mutex); + for(pp=&pFile->pFirst; (*pp)!=p; pp = &(*pp)->pNext){} + *pp = p->pNext; + pFile->nRef--; + nRef = pFile->nRef; + + /* Free the connection p */ + sqlite3_free(p); + sqlite3_mutex_leave(pFile->mutex); + + /* If pFile->nRef has reached 0, then close the underlying + ** shared-memory file, too */ + if( nRef==0 ){ + unixShmPurge(); + } + return SQLITE_OK; +} + /* ** Query and/or changes the size of a shared-memory segment. ** The reqSize parameter is the new size of the segment, or -1 to @@ -4661,26 +5066,31 @@ static int unixShmSize( int *pNewSize, /* Write new size here */ char **ppBuf /* Write new buffer origin here */ ){ - struct unixShm *p = (struct unixShm*)pSharedMem; + unixShm *p = (unixShm*)pSharedMem; + unixShmFile *pFile = p->pFile; int rc = SQLITE_OK; + sqlite3_mutex_enter(pFile->mutexBuf); + sqlite3_mutex_enter(pFile->mutex); if( reqSize>=0 ){ reqSize = (reqSize + SQLITE_UNIX_SHM_INCR - 1)/SQLITE_UNIX_SHM_INCR; reqSize *= SQLITE_UNIX_SHM_INCR; - if( reqSize!=p->size ){ - munmap(p->pBuf, p->size); - rc = ftruncate(p->fd.h, reqSize); + if( reqSize!=pFile->size ){ + if( pFile->pMMapBuf ) munmap(pFile->pMMapBuf, pFile->size); + rc = ftruncate(pFile->h, reqSize); if( rc ){ - p->pBuf = 0; - p->size = 0; + pFile->pMMapBuf = 0; + pFile->size = 0; }else{ - p->pBuf = mmap(0, reqSize, PROT_READ|PROT_WRITE, MAP_SHARED, p->fd.h,0); - p->size = p->pBuf ? reqSize : 0; + pFile->pMMapBuf = mmap(0, reqSize, PROT_READ|PROT_WRITE, MAP_SHARED, + pFile->h, 0); + pFile->size = pFile->pMMapBuf ? reqSize : 0; } } } - *pNewSize = p->size; - *ppBuf = p->pBuf; + *pNewSize = pFile->size; + *ppBuf = pFile->pMMapBuf; + sqlite3_mutex_leave(pFile->mutex); return rc; } @@ -4689,19 +5099,138 @@ static int unixShmSize( ** threads are free to resize it if necessary. */ static int unixShmRelease(sqlite3_shm *pSharedMem){ + unixShm *p = (unixShm*)pSharedMem; + unixShmFile *pFile = p->pFile; + sqlite3_mutex_leave(pFile->mutexBuf); return SQLITE_OK; } + + /* -** Create or release a lock on shared memory. +** Change the lock state for a shared-memory segment. */ static int unixShmLock( sqlite3_shm *pSharedMem, /* Pointer from unixShmOpen() */ - int desiredLock, /* The locking state desired */ - int *pGotLock, /* The locking state actually obtained */ - int shouldBlock /* Block for the lock if true and possible */ + int desiredLock, /* One of SQLITE_SHM_xxxxx locking states */ + int *pGotLock /* The lock you actually got */ ){ - return SQLITE_OK; + unixShm *p = (unixShm*)pSharedMem; + unixShmFile *pFile = p->pFile; + int rc = SQLITE_PROTOCOL; + + /* Note that SQLITE_SHM_READ_FULL and SQLITE_SHM_PENDING are never + ** directly requested; they are side effects from requesting + ** SQLITE_SHM_READ and SQLITE_SHM_CHECKPOINT, respectively. + */ + assert( desiredLock==SQLITE_SHM_QUERY + || desiredLock==SQLITE_SHM_UNLOCK + || desiredLock==SQLITE_SHM_READ + || desiredLock==SQLITE_SHM_WRITE + || desiredLock==SQLITE_SHM_CHECKPOINT + || desiredLock==SQLITE_SHM_RECOVER ); + + /* Return directly if this is just a lock state query, or if + ** the connection is already in the desired locking state. + */ + if( desiredLock==SQLITE_SHM_QUERY + || desiredLock==p->lockState + || (desiredLock==SQLITE_SHM_READ && p->lockState==SQLITE_SHM_READ_FULL) + ){ + *pGotLock = p->lockState; + return SQLITE_OK; + } + + sqlite3_mutex_enter(pFile->mutex); + switch( desiredLock ){ + case SQLITE_SHM_UNLOCK: { + assert( p->lockState!=SQLITE_SHM_RECOVER ); + unixShmUnlock(pFile, p, UNIX_SHM_A|UNIX_SHM_B|UNIX_SHM_C|UNIX_SHM_D); + rc = SQLITE_OK; + p->lockState = SQLITE_SHM_UNLOCK; + break; + } + case SQLITE_SHM_READ: { + if( p->lockState==SQLITE_SHM_UNLOCK ){ + int nAttempt; + rc = SQLITE_BUSY; + assert( p->lockState==SQLITE_SHM_UNLOCK ); + for(nAttempt=0; nAttempt<5 && rc==SQLITE_BUSY; nAttempt++){ + rc = unixShmSharedLock(pFile, p, UNIX_SHM_A|UNIX_SHM_B); + if( rc==SQLITE_BUSY ){ + rc = unixShmSharedLock(pFile, p, UNIX_SHM_D); + if( rc==SQLITE_OK ){ + p->lockState = p->readLock = SQLITE_SHM_READ_FULL; + } + }else{ + unixShmUnlock(pFile, p, UNIX_SHM_B); + p->lockState = p->readLock = SQLITE_SHM_READ; + } + } + }else if( p->lockState==SQLITE_SHM_WRITE ){ + unixShmUnlock(pFile, p, UNIX_SHM_C|UNIX_SHM_D); + p->lockState = p->readLock; + rc = SQLITE_OK; + }else{ + assert( p->lockState==SQLITE_SHM_RECOVER ); + unixShmUnlock(pFile, p, UNIX_SHM_MUTEX); + sqlite3_mutex_leave(pFile->mutexRecov); + p->lockState = p->readLock; + rc = SQLITE_OK; + } + break; + } + case SQLITE_SHM_WRITE: { + assert( p->lockState==SQLITE_SHM_READ + || p->lockState==SQLITE_SHM_READ_FULL ); + rc = unixShmExclusiveLock(pFile, p, UNIX_SHM_C|UNIX_SHM_D); + if( rc==SQLITE_OK ){ + p->lockState = SQLITE_SHM_WRITE; + } + break; + } + case SQLITE_SHM_CHECKPOINT: { + assert( p->lockState==SQLITE_SHM_UNLOCK + || p->lockState==SQLITE_SHM_PENDING + || p->lockState==SQLITE_SHM_RECOVER ); + if( p->lockState==SQLITE_SHM_RECOVER ){ + unixShmUnlock(pFile, p, UNIX_SHM_MUTEX); + sqlite3_mutex_leave(pFile->mutexRecov); + p->lockState = SQLITE_SHM_CHECKPOINT; + rc = SQLITE_OK; + } + if( p->lockState==SQLITE_SHM_UNLOCK ){ + rc = unixShmExclusiveLock(pFile, p, UNIX_SHM_B|UNIX_SHM_C); + if( rc==SQLITE_OK ){ + p->lockState = SQLITE_SHM_PENDING; + } + } + if( p->lockState==SQLITE_SHM_PENDING ){ + rc = unixShmExclusiveLock(pFile, p, UNIX_SHM_A); + if( rc==SQLITE_OK ){ + p->lockState = SQLITE_SHM_CHECKPOINT; + } + } + break; + } + default: { + assert( desiredLock==SQLITE_SHM_RECOVER ); + assert( p->lockState==SQLITE_SHM_READ + || p->lockState==SQLITE_SHM_READ_FULL + || p->lockState==SQLITE_SHM_CHECKPOINT ); + sqlite3_mutex_leave(pFile->mutex); + sqlite3_mutex_enter(pFile->mutexRecov); + sqlite3_mutex_enter(pFile->mutex); + rc = unixShmExclusiveLock(pFile, p, UNIX_SHM_MUTEX); + if( rc==SQLITE_OK ){ + p->lockState = SQLITE_SHM_RECOVER; + } + break; + } + } + sqlite3_mutex_leave(pFile->mutex); + *pGotLock = p->lockState; + return rc; } /* diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 9f6f1bc967..70e7d26a8c 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -848,7 +848,7 @@ struct sqlite3_vfs { int (*xShmRelease)(sqlite3_shm*); int (*xShmPush)(sqlite3_shm*); int (*xShmPull)(sqlite3_shm*); - int (*xShmLock)(sqlite3_shm*, int desiredLock, int *gotLock, int shouldBlock); + int (*xShmLock)(sqlite3_shm*, int desiredLock, int *gotLock); int (*xShmClose)(sqlite3_shm*); int (*xShmDelete)(sqlite3_vfs*, const char *zName); int (*xRename)(sqlite3_vfs*, const char *zOld, const char *zNew, int dirSync); @@ -887,7 +887,7 @@ struct sqlite3_vfs { ** state. */ #define SQLITE_SHM_UNLOCK 0 -#define SQLITE_SHM_READ_PREFIX 1 +#define SQLITE_SHM_READ 1 #define SQLITE_SHM_READ_FULL 2 #define SQLITE_SHM_WRITE 3 #define SQLITE_SHM_PENDING 4 From 31cbbbad1ccf37ad2f83daf2b34416e643f75d3e Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 29 Apr 2010 16:40:51 +0000 Subject: [PATCH 54/89] Untested implementation of the shared-memory dead-man-switch. FossilOrigin-Name: 706611283ea2575c2942543391026b36061cfc1c --- manifest | 18 ++++++++--------- manifest.uuid | 2 +- src/os_unix.c | 56 ++++++++++++++++++++++++++++++++++++++------------- 3 files changed, 52 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 81762c0b4c..ee0cee40d9 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 -C Progress\stowards\sa\sVFS\sthat\swill\ssupport\sWAL.\s\sLocking\scode\sis\sin\splace\nbut\sis\suntested.\s\sStill\sno\ssupport\sfor\sthe\sDMS. -D 2010-04-29T15:17:48 +C Untested\simplementation\sof\sthe\sshared-memory\sdead-man-switch. +D 2010-04-29T16:40:51 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -153,7 +153,7 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 -F src/os_unix.c 340c503705cdb447753a55771911a92aa41d27c8 +F src/os_unix.c f0e002e74108c8a3f70c63dd91419ab6bcafd095 F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 F src/pager.c b4a41030860229e80295fa1f37addab24d21799c F src/pager.h cee4487ab4f0911dd9f22a40e3cd55afdb7ef444 @@ -811,14 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 3cc55a7568daa3796483f632e33478969e381cf5 -R 1f2ad602233779dda1e35a2877201d1f +P 1bde41cf081570ad257f927b641e752dff4ed014 +R a7ee89d9d5eb7d4f4fd9d69824beab31 U drh -Z 5bb08c68390b552fdeb1ef93301ae98c +Z 15bd57757c5a0ce16c3065e9d45dae67 -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.6 (GNU/Linux) -iD8DBQFL2aMhoxKgR168RlERAntLAKCNwNXdF9LrZZqkEN8f3gsvrSxoXACaA2Ku -SvIawhrGV2wraxrir5MNan8= -=Vfpz +iD4DBQFL2baVoxKgR168RlERArO2AJ9g8Vnb6XlVDltcg68idEyJzFPcYwCYh7Xs +0Mdctk0jP1xbHiTJhHLX+A== +=minX -----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index 000aaadb5f..ca7b008f92 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1bde41cf081570ad257f927b641e752dff4ed014 \ No newline at end of file +706611283ea2575c2942543391026b36061cfc1c \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 634f5ee218..248309664d 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4698,7 +4698,7 @@ static const char *unixShmLockString(u8 mask){ ** Locks block if the UNIX_SHM_MUTEX bit is set and are non-blocking ** otherwise. */ -static int unixShmSystemLocks( +static int unixShmSystemLock( unixShmFile *pFile, /* Apply locks to this open shared-memory segment */ int lockType, /* F_UNLCK, F_RDLCK, or F_WRLCK */ u8 lockMask /* Which bytes to lock or unlock */ @@ -4709,6 +4709,9 @@ static int unixShmSystemLocks( int rc; /* Result code form fcntl() */ u8 mask; /* Mask of bits in lockMask */ + /* Access to the unixShmFile object is serialized by the caller */ + assert( sqlite3_mutex_held(pFile->mutex) ); + /* Initialize the locking parameters */ memset(&f, 0, sizeof(f)); f.l_type = lockType; @@ -4787,6 +4790,9 @@ static int unixShmUnlock( unixShm *pX; /* For looping over all sibling connections */ u8 allMask; /* Union of locks held by connections other than "p" */ + /* Access to the unixShmFile object is serialized by the caller */ + assert( sqlite3_mutex_held(pFile->mutex) ); + /* We never try to unlock locks that we do not hold */ assert( ((p->exclMask|p->sharedMask) & unlockMask)==unlockMask ); @@ -4799,7 +4805,7 @@ static int unixShmUnlock( /* Unlock the system-level locks */ if( (unlockMask & allMask)!=unlockMask ){ - rc = unixShmSystemLocks(pFile, F_UNLCK, unlockMask & ~allMask); + rc = unixShmSystemLock(pFile, F_UNLCK, unlockMask & ~allMask); }else{ rc = SQLITE_OK; } @@ -4824,6 +4830,9 @@ static int unixShmSharedLock( unixShm *pX; /* For looping over all sibling connections */ u8 allShared; /* Union of locks held by connections other than "p" */ + /* Access to the unixShmFile object is serialized by the caller */ + assert( sqlite3_mutex_held(pFile->mutex) ); + /* Find out which shared locks are already held by sibling connections. ** If any sibling already holds an exclusive lock, go ahead and return ** SQLITE_BUSY. @@ -4836,7 +4845,7 @@ static int unixShmSharedLock( /* Get shared locks at the system level, if necessary */ if( (~allShared) & readMask ){ - rc = unixShmSystemLocks(pFile, F_RDLCK, readMask); + rc = unixShmSystemLock(pFile, F_RDLCK, readMask); }else{ rc = SQLITE_OK; } @@ -4860,6 +4869,9 @@ static int unixShmExclusiveLock( int rc; /* Result code */ unixShm *pX; /* For looping over all sibling connections */ + /* Access to the unixShmFile object is serialized by the caller */ + assert( sqlite3_mutex_held(pFile->mutex) ); + /* Make sure no sibling connections hold locks that will block this ** lock. If any do, return SQLITE_BUSY right away. */ @@ -4872,7 +4884,7 @@ static int unixShmExclusiveLock( /* Get the exclusive locks at the system level. Then if successful ** also mark the local connection as being locked. */ - rc = unixShmSystemLocks(pFile, F_WRLCK, writeMask); + rc = unixShmSystemLock(pFile, F_WRLCK, writeMask); if( rc==SQLITE_OK ){ p->sharedMask &= ~writeMask; p->exclMask |= writeMask; @@ -4917,16 +4929,20 @@ static int unixShmOpen( const char *zName, /* Name of file to mmap */ sqlite3_shm **pShm /* Write the unixShm object created here */ ){ - struct unixShm *p = 0; - struct unixShmFile *pFile = 0; - int rc; - struct unixFileId fid; - struct stat sStat; + struct unixShm *p = 0; /* The connection to be opened */ + struct unixShmFile *pFile = 0; /* The underlying mmapped file */ + int rc; /* Result code */ + struct unixFileId fid; /* Unix file identifier */ + struct stat sStat; /* Result from stat() an fstat() */ + /* Allocate space for the new sqlite3_shm object */ p = sqlite3_malloc( sizeof(*p) ); if( p==0 ) return SQLITE_NOMEM; memset(p, 0, sizeof(*p)); + /* Look to see if there is an existing unixShmFile that can be used. + ** If no matching unixShmFile currently exists, create a new one. + */ unixEnterMutex(); rc = stat(zName, &sStat); if( rc==0 ){ @@ -4982,16 +4998,27 @@ static int unixShmOpen( pFile->fid.ino = sStat.st_ino; pFile->size = (int)sStat.st_size; pFile->size = (pFile->size/SQLITE_UNIX_SHM_INCR)*SQLITE_UNIX_SHM_INCR; - if( pFile->size==0 ){ - pFile->size = SQLITE_UNIX_SHM_INCR; - rc = ftruncate(pFile->h, pFile->size); - if( rc ){ - rc = SQLITE_FULL; + + /* Check to see if another process is holding the dead-man switch. + ** If not, truncate the file to zero length. + */ + if( unixShmSystemLock(pFile, F_WRLCK, UNIX_SHM_MUTEX) ){ + rc = SQLITE_IOERR_LOCK; + goto shm_open_err; + } + if( unixShmSystemLock(pFile, F_WRLCK, UNIX_SHM_DMS)==SQLITE_OK ){ + if( ftruncate(pFile->h, 0) ){ + rc = SQLITE_IOERR; goto shm_open_err; } + pFile->size = 0; } + rc = unixShmSystemLock(pFile, F_RDLCK, UNIX_SHM_DMS); + if( rc ) goto shm_open_err; + unixShmSystemLock(pFile, F_UNLCK, UNIX_SHM_MUTEX); } + /* Make the new connection a child of the unixShmFile */ p->pFile = pFile; p->pNext = pFile->pFirst; #ifdef SQLITE_DEBUG @@ -5003,6 +5030,7 @@ static int unixShmOpen( unixLeaveMutex(); return SQLITE_OK; + /* Jump here on any error */ shm_open_err: unixShmPurge(); sqlite3_free(p); From 7ed91f2344f1cb6837e1fba1b83fba5b6ddce887 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 29 Apr 2010 22:34:07 +0000 Subject: [PATCH 55/89] Refactor wal.c to use the VFS. This check-in compiles and links and works ok as long as you leave WAL turned off, but WAL does not work. FossilOrigin-Name: 62db5fa3b61be885b2d94e9b9ce3877b2c588350 --- manifest | 40 +- manifest.uuid | 2 +- src/btree.c | 2 +- src/main.c | 6 +- src/os_unix.c | 7 +- src/pager.c | 140 ++--- src/pager.h | 6 +- src/sqlite.h.in | 2 +- src/sqliteInt.h | 4 +- src/test1.c | 6 +- src/vdbe.c | 4 +- src/vdbeapi.c | 10 +- src/wal.c | 1485 +++++++++++++---------------------------------- src/wal.h | 30 +- 14 files changed, 534 insertions(+), 1210 deletions(-) diff --git a/manifest b/manifest index ee0cee40d9..4e258ea0da 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 -C Untested\simplementation\sof\sthe\sshared-memory\sdead-man-switch. -D 2010-04-29T16:40:51 +C Refactor\swal.c\sto\suse\sthe\sVFS.\s\sThis\scheck-in\scompiles\sand\slinks\sand\sworks\nok\sas\slong\sas\syou\sleave\sWAL\sturned\soff,\sbut\sWAL\sdoes\snot\swork. +D 2010-04-29T22:34:08 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -112,7 +112,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c e86634da8c48357a759694c9c7c471125cd8d5a8 F src/bitvec.c 06ad2c36a9c3819c0b9cbffec7b15f58d5d834e0 F src/btmutex.c 96a12f50f7a17475155971a241d85ec5171573ff -F src/btree.c dd26f3b5725ac3cbc765857a76523ac10dbfe761 +F src/btree.c 2e41e62e80ad4686b3322721982ceb7aa6d8c9f1 F src/btree.h dd83041eda10c17daf023257c1fc883b5f71f85a F src/btreeInt.h b0c87f6725b06a0aa194a6d25d54b16ce9d6e291 F src/build.c 11100b66fb97638d2d874c1d34d8db90650bb1d7 @@ -134,7 +134,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/main.c 569561a238a0bb6d55affbaa224ddc885c91df2b +F src/main.c 28171059466a5dac18160c466b978eaafdbb3594 F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 89d4ea8d5cdd55635cbaa48ad53132af6294cbb2 @@ -153,10 +153,10 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 -F src/os_unix.c f0e002e74108c8a3f70c63dd91419ab6bcafd095 +F src/os_unix.c bf4cc543fcb32dbcb25a88cf817c761ceef14fdd F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 -F src/pager.c b4a41030860229e80295fa1f37addab24d21799c -F src/pager.h cee4487ab4f0911dd9f22a40e3cd55afdb7ef444 +F src/pager.c 07c3522319a449db9dc732da828a09c4c55c41a4 +F src/pager.h 934b598583a9d936bb13c37d62a2fe68ac48781c F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 @@ -169,14 +169,14 @@ F src/resolve.c ac5f1a713cd1ae77f08b83cc69581e11bf5ae6f9 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c c03d8a0565febcde8c6a12c5d77d065fddae889b F src/shell.c c40427c7245535a04a9cb4a417b6cc05c022e6a4 -F src/sqlite.h.in 0cc43a0fa5d26cd26d2609b84ecf094a159e9cb9 +F src/sqlite.h.in a710846e91159742ee3ddcbabb56eb7c7478e3a5 F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89 -F src/sqliteInt.h 700a2df7b8dfe57c3b8d83c52ff40928e026220c +F src/sqliteInt.h 32dfe6294b46914fb567db493994bf902fb0f8c9 F src/sqliteLimit.h 3afab2291762b5d09ae20c18feb8e9fa935a60a6 F src/status.c 4df6fe7dce2d256130b905847c6c60055882bdbe F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 4de81521174fedacd8393ea7b70b730ce17f8eae -F src/test1.c 64b5b8135080b94370e8100e5066bb394f5c3122 +F src/test1.c a449144907face6414f68f6a5663a5b8cadabbd4 F src/test2.c b6b43413d495addd039a88b87d65c839f86b18cb F src/test3.c 4c21700c73a890a47fc685c1097bfb661346ac94 F src/test4.c ad03bb987ddedce928f4258c1e7fa4109a73497d @@ -215,17 +215,17 @@ F src/update.c c0dc6b75ad28b76b619042d934f337b02acee208 F src/utf.c 1baeeac91707a4df97ccc6141ec0f808278af685 F src/util.c 32aebf04c10e51ad3977a928b7416bed671b620b F src/vacuum.c 8e7d9a6fd076774bb6e211cad335c7337948a164 -F src/vdbe.c da7ee09fdb5ee6ecfac11b74086834ff6583cd23 +F src/vdbe.c 21a96eeeb43dedd40072028db0b42e137296a81a F src/vdbe.h 471f6a3dcec4817ca33596fe7f6654d56c0e75f3 F src/vdbeInt.h 19ebc8c2a2e938340051ee65af3f377fb99102d1 -F src/vdbeapi.c 466044df5bc916f778833e927165fd02cdef6086 +F src/vdbeapi.c 85e448c7d1d6dde7226c721cd8da8f9b5286699d F src/vdbeaux.c be6ca4bb330c7730b358dd627d6d632ca93482c9 F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c b1c6868b975a67f6f4dd2cd612eeace4117eb98f -F src/wal.h c60781e78e394af07ece3b64a11192eb442241c1 +F src/wal.c 1c984ba3bba07a9efb4974837d12daef8f0d80c4 +F src/wal.h d0a7048e68a86d3fe4639858d7f592606545d224 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -811,14 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 1bde41cf081570ad257f927b641e752dff4ed014 -R a7ee89d9d5eb7d4f4fd9d69824beab31 +P 706611283ea2575c2942543391026b36061cfc1c +R d5f4a8af6d2162b2c802208b97207bbd U drh -Z 15bd57757c5a0ce16c3065e9d45dae67 +Z eca35982e551d3fde58c618a2bb3d812 -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.6 (GNU/Linux) -iD4DBQFL2baVoxKgR168RlERArO2AJ9g8Vnb6XlVDltcg68idEyJzFPcYwCYh7Xs -0Mdctk0jP1xbHiTJhHLX+A== -=minX +iD8DBQFL2glkoxKgR168RlERAm4BAJ9/6BoUJM0bQ0jccd1czYE2rrEefgCfYnyN +XCr+jA+ZIb6AheCdZkTnnE4= +=Dx/c -----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index ca7b008f92..78f7afdb38 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -706611283ea2575c2942543391026b36061cfc1c \ No newline at end of file +62db5fa3b61be885b2d94e9b9ce3877b2c588350 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 62a8410764..e60754e582 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2278,7 +2278,7 @@ static int lockBtree(BtShared *pBt){ */ if( page1[19]==2 && pBt->doNotUseWAL==0 ){ int isOpen = 0; - rc = sqlite3PagerOpenLog(pBt->pPager, &isOpen); + rc = sqlite3PagerOpenWal(pBt->pPager, &isOpen); if( rc!=SQLITE_OK ){ goto page1_init_failed; }else if( isOpen==0 ){ diff --git a/src/main.c b/src/main.c index 3e7c1257c2..c723a4ef55 100644 --- a/src/main.c +++ b/src/main.c @@ -1197,9 +1197,9 @@ void *sqlite3_wal_hook( ){ void *pRet; sqlite3_mutex_enter(db->mutex); - pRet = db->pLogArg; - db->xLogCallback = xCallback; - db->pLogArg = pArg; + pRet = db->pWalArg; + db->xWalCallback = xCallback; + db->pWalArg = pArg; sqlite3_mutex_leave(db->mutex); return pRet; } diff --git a/src/os_unix.c b/src/os_unix.c index 248309664d..0e525c0732 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5049,6 +5049,7 @@ static int unixShmClose(sqlite3_shm *pSharedMem){ unixShm **pp; /* For looping over sibling connections */ int nRef; /* Number of connections to pFile */ + if( pSharedMem==0 ) return SQLITE_OK; p = (struct unixShm*)pSharedMem; pFile = p->pFile; @@ -5092,7 +5093,7 @@ static int unixShmSize( sqlite3_shm *pSharedMem, /* Pointer returned by unixShmOpen() */ int reqSize, /* Requested size. -1 for query only */ int *pNewSize, /* Write new size here */ - char **ppBuf /* Write new buffer origin here */ + void **ppBuf /* Write new buffer origin here */ ){ unixShm *p = (unixShm*)pSharedMem; unixShmFile *pFile = p->pFile; @@ -5165,7 +5166,7 @@ static int unixShmLock( || desiredLock==p->lockState || (desiredLock==SQLITE_SHM_READ && p->lockState==SQLITE_SHM_READ_FULL) ){ - *pGotLock = p->lockState; + if( pGotLock ) *pGotLock = p->lockState; return SQLITE_OK; } @@ -5257,7 +5258,7 @@ static int unixShmLock( } } sqlite3_mutex_leave(pFile->mutex); - *pGotLock = p->lockState; + if( pGotLock ) *pGotLock = p->lockState; return rc; } diff --git a/src/pager.c b/src/pager.c index bd704fa7c7..0ad245a8c4 100644 --- a/src/pager.c +++ b/src/pager.c @@ -399,7 +399,7 @@ struct Pager { char *pTmpSpace; /* Pager.pageSize bytes of space for tmp use */ PCache *pPCache; /* Pointer to page cache object */ sqlite3_backup *pBackup; /* Pointer to list of ongoing backup processes */ - Log *pLog; /* Log used by "journal_mode=wal" */ + Wal *pWal; /* Write-ahead log used by "journal_mode=wal" */ }; /* @@ -1192,8 +1192,8 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ ** Return true if this pager uses a write-ahead log instead of the usual ** rollback journal. Otherwise false. */ -static int pagerUseLog(Pager *pPager){ - return (pPager->pLog!=0); +static int pagerUseWal(Pager *pPager){ + return (pPager->pWal!=0); } /* @@ -1227,8 +1227,8 @@ static void pager_unlock(Pager *pPager){ */ pPager->dbSizeValid = 0; - if( pagerUseLog(pPager) ){ - sqlite3WalCloseSnapshot(pPager->pLog); + if( pagerUseWal(pPager) ){ + sqlite3WalCloseSnapshot(pPager->pWal); }else{ rc = osUnlock(pPager->fd, NO_LOCK); } @@ -1380,7 +1380,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ assert( isOpen(pPager->jfd) || pPager->pInJournal==0 ); if( isOpen(pPager->jfd) ){ - assert( !pagerUseLog(pPager) ); + assert( !pagerUseWal(pPager) ); /* Finalize the journal file. */ if( sqlite3IsMemJournal(pPager->jfd) ){ @@ -1426,8 +1426,8 @@ static int pager_end_transaction(Pager *pPager, int hasMaster){ pPager->nRec = 0; sqlite3PcacheCleanAll(pPager->pPCache); - if( pagerUseLog(pPager) ){ - rc2 = sqlite3WalWriteLock(pPager->pLog, 0); + if( pagerUseWal(pPager) ){ + rc2 = sqlite3WalWriteLock(pPager->pWal, 0); pPager->state = PAGER_SHARED; }else if( !pPager->exclusiveMode ){ rc2 = osUnlock(pPager->fd, SHARED_LOCK); @@ -1540,7 +1540,7 @@ static int pager_playback_one_page( aData = pPager->pTmpSpace; assert( aData ); /* Temp storage must have already been allocated */ - assert( pagerUseLog(pPager)==0 || (!isMainJrnl && isSavepnt) ); + assert( pagerUseWal(pPager)==0 || (!isMainJrnl && isSavepnt) ); /* Read the page number and page data from the journal or sub-journal ** file. Return an error code to the caller if an IO error occurs. @@ -1610,7 +1610,7 @@ static int pager_playback_one_page( ** is possible to fail a statement on a database that does not yet exist. ** Do not attempt to write if database file has never been opened. */ - if( pagerUseLog(pPager) ){ + if( pagerUseWal(pPager) ){ pPg = 0; }else{ pPg = pager_lookup(pPager, pgno); @@ -1631,7 +1631,7 @@ static int pager_playback_one_page( ){ i64 ofst = (pgno-1)*(i64)pPager->pageSize; testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); - assert( !pagerUseLog(pPager) ); + assert( !pagerUseWal(pPager) ); rc = sqlite3OsWrite(pPager->fd, (u8*)aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; @@ -1696,7 +1696,7 @@ static int pager_playback_one_page( ** segment is synced. If a crash occurs during or following this, ** database corruption may ensue. */ - assert( !pagerUseLog(pPager) ); + assert( !pagerUseWal(pPager) ); sqlite3PcacheMakeClean(pPg); } #ifdef SQLITE_CHECK_PAGES @@ -2188,7 +2188,7 @@ static int readDbPage(PgHdr *pPg){ Pgno pgno = pPg->pgno; /* Page number to read */ int rc = SQLITE_OK; /* Return code */ i64 iOffset; /* Byte offset of file to read from */ - int isInLog = 0; /* True if page is in log file */ + int isInWal = 0; /* True if page is in log file */ assert( pPager->state>=PAGER_SHARED && !MEMDB ); assert( isOpen(pPager->fd) ); @@ -2199,11 +2199,11 @@ static int readDbPage(PgHdr *pPg){ return SQLITE_OK; } - if( pagerUseLog(pPager) ){ + if( pagerUseWal(pPager) ){ /* Try to pull the page from the write-ahead log. */ - rc = sqlite3WalRead(pPager->pLog, pgno, &isInLog, pPg->pData); + rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pPg->pData); } - if( rc==SQLITE_OK && !isInLog ){ + if( rc==SQLITE_OK && !isInWal ){ iOffset = (pgno-1)*(i64)pPager->pageSize; rc = sqlite3OsRead(pPager->fd, pPg->pData, pPager->pageSize, iOffset); if( rc==SQLITE_IOERR_SHORT_READ ){ @@ -2278,7 +2278,7 @@ static int pagerUndoCallback(void *pCtx, Pgno iPg){ /* ** This function is called to rollback a transaction on a WAL database. */ -static int pagerRollbackLog(Pager *pPager){ +static int pagerRollbackWal(Pager *pPager){ int rc; /* Return Code */ PgHdr *pList; /* List of dirty pages to revert */ @@ -2290,7 +2290,7 @@ static int pagerRollbackLog(Pager *pPager){ ** also copied into the backup databases) as part of this transaction, ** the backups must be restarted. */ - if( sqlite3WalDirty(pPager->pLog) ){ + if( sqlite3WalDirty(pPager->pWal) ){ sqlite3BackupRestart(pPager->pBackup); } @@ -2302,7 +2302,7 @@ static int pagerRollbackLog(Pager *pPager){ ** + Reload page content from the database (if refcount>0). */ pPager->dbSize = pPager->dbOrigSize; - rc = sqlite3WalUndo(pPager->pLog, pagerUndoCallback, (void *)pPager); + rc = sqlite3WalUndo(pPager->pWal, pagerUndoCallback, (void *)pPager); pList = sqlite3PcacheDirtyList(pPager->pPCache); while( pList && rc==SQLITE_OK ){ PgHdr *pNext = pList->pDirty; @@ -2370,8 +2370,8 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ */ pPager->dbSize = pSavepoint ? pSavepoint->nOrig : pPager->dbOrigSize; - if( !pSavepoint && pagerUseLog(pPager) ){ - return pagerRollbackLog(pPager); + if( !pSavepoint && pagerUseWal(pPager) ){ + return pagerRollbackWal(pPager); } /* Use pPager->journalOff as the effective size of the main rollback @@ -2380,7 +2380,7 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ ** past pPager->journalOff is off-limits to us. */ szJ = pPager->journalOff; - assert( pagerUseLog(pPager)==0 || szJ==0 ); + assert( pagerUseWal(pPager)==0 || szJ==0 ); /* Begin by rolling back records from the main journal starting at ** PagerSavepoint.iOffset and continuing to the next journal header. @@ -2389,7 +2389,7 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ ** will be skipped automatically. Pages are added to pDone as they ** are played back. */ - if( pSavepoint && !pagerUseLog(pPager) ){ + if( pSavepoint && !pagerUseWal(pPager) ){ iHdrOff = pSavepoint->iHdrOffset ? pSavepoint->iHdrOffset : szJ; pPager->journalOff = pSavepoint->iOffset; while( rc==SQLITE_OK && pPager->journalOffiSubRec*(4+pPager->pageSize); - if( pagerUseLog(pPager) ){ - rc = sqlite3WalSavepointUndo(pPager->pLog, pSavepoint->iFrame); + if( pagerUseWal(pPager) ){ + rc = sqlite3WalSavepointUndo(pPager->pWal, pSavepoint->iFrame); } for(ii=pSavepoint->iSubRec; rc==SQLITE_OK && iinSubRec; ii++){ assert( offset==ii*(4+pPager->pageSize) ); @@ -2750,8 +2750,8 @@ int sqlite3PagerPagecount(Pager *pPager, int *pnPage){ int rc; /* Error returned by OsFileSize() */ i64 n = 0; /* File size in bytes returned by OsFileSize() */ - if( pagerUseLog(pPager) ){ - sqlite3WalDbsize(pPager->pLog, &nPage); + if( pagerUseWal(pPager) ){ + sqlite3WalDbsize(pPager->pWal, &nPage); } if( nPage==0 ){ @@ -2936,10 +2936,10 @@ int sqlite3PagerClose(Pager *pPager){ sqlite3BeginBenignMalloc(); pPager->errCode = 0; pPager->exclusiveMode = 0; - sqlite3WalClose(pPager->pLog, pPager->fd, + sqlite3WalClose(pPager->pWal, pPager->fd, (pPager->noSync ? 0 : pPager->sync_flags), pTmp ); - pPager->pLog = 0; + pPager->pWal = 0; pager_reset(pPager); if( MEMDB ){ pager_unlock(pPager); @@ -3176,7 +3176,7 @@ static int pager_write_pagelist(PgHdr *pList){ ** EXCLUSIVE, it means the database file has been changed and any rollback ** will require a journal playback. */ - assert( !pagerUseLog(pList->pPager) ); + assert( !pagerUseWal(pList->pPager) ); assert( pPager->state>=PAGER_RESERVED ); rc = pager_wait_on_lock(pPager, EXCLUSIVE_LOCK); @@ -3265,7 +3265,7 @@ static int subjournalPage(PgHdr *pPg){ CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM, pData2); PAGERTRACE(("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno)); - assert( pagerUseLog(pPager) + assert( pagerUseWal(pPager) || pageInJournal(pPg) || pPg->pgno>pPager->dbOrigSize ); @@ -3288,7 +3288,7 @@ static int subjournalPage(PgHdr *pPg){ ** this function notifies any active backup processes that the pages have ** changed. */ -static int pagerLogFrames( +static int pagerWalFrames( Pager *pPager, /* Pager object */ PgHdr *pList, /* List of frames to log */ Pgno nTruncate, /* Database size after this commit */ @@ -3297,8 +3297,8 @@ static int pagerLogFrames( ){ int rc; /* Return code */ - assert( pPager->pLog ); - rc = sqlite3WalFrames(pPager->pLog, + assert( pPager->pWal ); + rc = sqlite3WalFrames(pPager->pWal, pPager->pageSize, pList, nTruncate, isCommit, sync_flags ); if( rc==SQLITE_OK && pPager->pBackup ){ @@ -3337,13 +3337,13 @@ static int pagerStress(void *p, PgHdr *pPg){ assert( pPg->flags&PGHDR_DIRTY ); pPg->pDirty = 0; - if( pagerUseLog(pPager) ){ + if( pagerUseWal(pPager) ){ /* Write a single frame for this page to the log. */ if( subjRequiresPage(pPg) ){ rc = subjournalPage(pPg); } if( rc==SQLITE_OK ){ - rc = pagerLogFrames(pPager, pPg, 0, 0, 0); + rc = pagerWalFrames(pPager, pPg, 0, 0, 0); } }else{ /* The doNotSync flag is set by the sqlite3PagerWrite() function while it @@ -3836,12 +3836,12 @@ static int pagerHasWAL(Pager *pPager, int *pExists){ int rc; /* Return code */ if( !pPager->tempFile ){ - char *zLog = sqlite3_mprintf("%s-wal", pPager->zFilename); - if( !zLog ){ + char *zWal = sqlite3_mprintf("%s-wal", pPager->zFilename); + if( !zWal ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3OsAccess(pPager->pVfs, zLog, SQLITE_ACCESS_EXISTS, pExists); - sqlite3_free(zLog); + rc = sqlite3OsAccess(pPager->pVfs, zWal, SQLITE_ACCESS_EXISTS, pExists); + sqlite3_free(zWal); } }else{ rc = SQLITE_OK; @@ -3854,9 +3854,9 @@ static int pagerOpenSnapshot(Pager *pPager){ int rc; /* Return code */ int changed = 0; /* True if cache must be reset */ - assert( pagerUseLog(pPager) ); + assert( pagerUseWal(pPager) ); - rc = sqlite3WalOpenSnapshot(pPager->pLog, &changed); + rc = sqlite3WalOpenSnapshot(pPager->pWal, &changed); if( rc==SQLITE_OK ){ int dummy; if( changed ){ @@ -3923,7 +3923,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ pager_reset(pPager); } - if( pagerUseLog(pPager) ){ + if( pagerUseWal(pPager) ){ rc = pagerOpenSnapshot(pPager); }else if( pPager->state==PAGER_UNLOCK || isErrorReset ){ sqlite3_vfs * const pVfs = pPager->pVfs; @@ -4089,7 +4089,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ } if( isWal ){ pager_reset(pPager); - rc = sqlite3PagerOpenLog(pPager, 0); + rc = sqlite3PagerOpenWal(pPager, 0); if( rc==SQLITE_OK ){ rc = pagerOpenSnapshot(pPager); } @@ -4332,7 +4332,7 @@ void sqlite3PagerUnref(DbPage *pPg){ */ static int openSubJournal(Pager *pPager){ int rc = SQLITE_OK; - if( (pagerUseLog(pPager) || isOpen(pPager->jfd)) && !isOpen(pPager->sjfd) ){ + if( (pagerUseWal(pPager) || isOpen(pPager->jfd)) && !isOpen(pPager->sjfd) ){ if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->subjInMemory ){ sqlite3MemJournalOpen(pPager->sjfd); }else{ @@ -4468,13 +4468,13 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ assert( pPager->pInJournal==0 ); assert( !MEMDB && !pPager->tempFile ); - if( pagerUseLog(pPager) ){ + if( pagerUseWal(pPager) ){ /* Grab the write lock on the log file. If successful, upgrade to ** PAGER_EXCLUSIVE state. Otherwise, return an error code to the caller. ** The busy-handler is not invoked if another connection already ** holds the write-lock. If possible, the upper layer will call it. */ - rc = sqlite3WalWriteLock(pPager->pLog, 1); + rc = sqlite3WalWriteLock(pPager->pWal, 1); if( rc==SQLITE_OK ){ pPager->dbOrigSize = pPager->dbSize; pPager->state = PAGER_RESERVED; @@ -4507,7 +4507,7 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ ** kept open and either was truncated to 0 bytes or its header was ** overwritten with zeros. */ - assert( pagerUseLog(pPager)==0 ); + assert( pagerUseWal(pPager)==0 ); assert( pPager->nRec==0 ); assert( pPager->dbOrigSize==0 ); assert( pPager->pInJournal==0 ); @@ -4562,7 +4562,7 @@ static int pager_write(PgHdr *pPg){ */ sqlite3PcacheMakeDirty(pPg); if( pageInJournal(pPg) && !subjRequiresPage(pPg) ){ - assert( !pagerUseLog(pPager) ); + assert( !pagerUseWal(pPager) ); pPager->dbModified = 1; }else{ @@ -4580,7 +4580,7 @@ static int pager_write(PgHdr *pPg){ } if( !isOpen(pPager->jfd) && pPager->journalMode!=PAGER_JOURNALMODE_OFF - && !pagerUseLog(pPager) + && !pagerUseWal(pPager) ){ assert( pPager->useJournal ); rc = pager_open_journal(pPager); @@ -4593,7 +4593,7 @@ static int pager_write(PgHdr *pPg){ ** the transaction journal if it is not there already. */ if( !pageInJournal(pPg) && isOpen(pPager->jfd) ){ - assert( !pagerUseLog(pPager) ); + assert( !pagerUseWal(pPager) ); if( pPg->pgno<=pPager->dbOrigSize ){ u32 cksum; char *pData2; @@ -4974,10 +4974,10 @@ int sqlite3PagerCommitPhaseOne( */ sqlite3BackupRestart(pPager->pBackup); }else if( pPager->state!=PAGER_SYNCED && pPager->dbModified ){ - if( pagerUseLog(pPager) ){ + if( pagerUseWal(pPager) ){ PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); if( pList ){ - rc = pagerLogFrames(pPager, pList, pPager->dbSize, 1, + rc = pagerWalFrames(pPager, pList, pPager->dbSize, 1, (pPager->fullSync ? pPager->sync_flags : 0) ); } @@ -5213,7 +5213,7 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ int sqlite3PagerRollback(Pager *pPager){ int rc = SQLITE_OK; /* Return code */ PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager))); - if( pagerUseLog(pPager) ){ + if( pagerUseWal(pPager) ){ int rc2; rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1); @@ -5360,8 +5360,8 @@ int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ if( !aNew[ii].pInSavepoint ){ return SQLITE_NOMEM; } - if( pagerUseLog(pPager) ){ - aNew[ii].iFrame = sqlite3WalSavepoint(pPager->pLog); + if( pagerUseWal(pPager) ){ + aNew[ii].iFrame = sqlite3WalSavepoint(pPager->pWal); } } @@ -5440,7 +5440,7 @@ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ ** not yet been opened. In this case there have been no changes to ** the database file, so the playback operation can be skipped. */ - else if( pagerUseLog(pPager) || isOpen(pPager->jfd) ){ + else if( pagerUseWal(pPager) || isOpen(pPager->jfd) ){ PagerSavepoint *pSavepoint = (nNew==0)?0:&pPager->aSavepoint[nNew-1]; rc = pagerPlaybackSavepoint(pPager, pSavepoint); assert(rc!=SQLITE_DONE); @@ -5796,9 +5796,9 @@ sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){ */ int sqlite3PagerCheckpoint(Pager *pPager){ int rc = SQLITE_OK; - if( pPager->pLog ){ + if( pPager->pWal ){ u8 *zBuf = (u8 *)pPager->pTmpSpace; - rc = sqlite3WalCheckpoint(pPager->pLog, pPager->fd, + rc = sqlite3WalCheckpoint(pPager->pWal, pPager->fd, (pPager->noSync ? 0 : pPager->sync_flags), zBuf, pPager->xBusyHandler, pPager->pBusyHandlerArg ); @@ -5806,8 +5806,8 @@ int sqlite3PagerCheckpoint(Pager *pPager){ return rc; } -int sqlite3PagerLogCallback(Pager *pPager){ - return sqlite3WalCallback(pPager->pLog); +int sqlite3PagerWalCallback(Pager *pPager){ + return sqlite3WalCallback(pPager->pWal); } /* @@ -5817,17 +5817,17 @@ int sqlite3PagerLogCallback(Pager *pPager){ ** The caller must be holding a SHARED lock on the database file to call ** this function. */ -int sqlite3PagerOpenLog(Pager *pPager, int *pisOpen){ +int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen){ int rc = SQLITE_OK; /* Return code */ assert( pPager->state>=PAGER_SHARED ); - if( !pPager->pLog ){ + if( !pPager->pWal ){ /* Open the connection to the log file. If this operation fails, ** (e.g. due to malloc() failure), unlock the database file and ** return an error code. */ - rc = sqlite3WalOpen(pPager->pVfs, pPager->zFilename, &pPager->pLog); + rc = sqlite3WalOpen(pPager->pVfs, pPager->zFilename, &pPager->pWal); if( rc==SQLITE_OK ){ pPager->journalMode = PAGER_JOURNALMODE_WAL; } @@ -5848,7 +5848,7 @@ int sqlite3PagerOpenLog(Pager *pPager, int *pisOpen){ ** error (SQLITE_BUSY) is returned and the log connection is not closed. ** If successful, the EXCLUSIVE lock is not released before returning. */ -int sqlite3PagerCloseLog(Pager *pPager){ +int sqlite3PagerCloseWal(Pager *pPager){ int rc = SQLITE_OK; assert( pPager->journalMode==PAGER_JOURNALMODE_WAL ); @@ -5857,28 +5857,28 @@ int sqlite3PagerCloseLog(Pager *pPager){ ** it may need to be checkpointed before the connection can switch to ** rollback mode. Open it now so this can happen. */ - if( !pPager->pLog ){ + if( !pPager->pWal ){ int logexists = 0; rc = sqlite3OsLock(pPager->fd, SQLITE_LOCK_SHARED); if( rc==SQLITE_OK ){ rc = pagerHasWAL(pPager, &logexists); } if( rc==SQLITE_OK && logexists ){ - rc = sqlite3WalOpen(pPager->pVfs, pPager->zFilename, &pPager->pLog); + rc = sqlite3WalOpen(pPager->pVfs, pPager->zFilename, &pPager->pWal); } } /* Checkpoint and close the log. Because an EXCLUSIVE lock is held on ** the database file, the log and log-summary files will be deleted. */ - if( rc==SQLITE_OK && pPager->pLog ){ + if( rc==SQLITE_OK && pPager->pWal ){ rc = sqlite3OsLock(pPager->fd, SQLITE_LOCK_EXCLUSIVE); if( rc==SQLITE_OK ){ - rc = sqlite3WalClose(pPager->pLog, pPager->fd, + rc = sqlite3WalClose(pPager->pWal, pPager->fd, (pPager->noSync ? 0 : pPager->sync_flags), (u8*)pPager->pTmpSpace ); - pPager->pLog = 0; + pPager->pWal = 0; } } return rc; diff --git a/src/pager.h b/src/pager.h index 7b760c9b1a..91b2cb32f9 100644 --- a/src/pager.h +++ b/src/pager.h @@ -135,9 +135,9 @@ int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); int sqlite3PagerSharedLock(Pager *pPager); int sqlite3PagerCheckpoint(Pager *pPager); -int sqlite3PagerLogCallback(Pager *pPager); -int sqlite3PagerOpenLog(Pager *pPager, int *pisOpen); -int sqlite3PagerCloseLog(Pager *pPager); +int sqlite3PagerWalCallback(Pager *pPager); +int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); +int sqlite3PagerCloseWal(Pager *pPager); /* Functions used to query pager state and configuration. */ u8 sqlite3PagerIsreadonly(Pager*); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 70e7d26a8c..35ac5bd008 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -844,7 +844,7 @@ struct sqlite3_vfs { ** definition. Those that follow are added in version 2 or later */ int (*xShmOpen)(sqlite3_vfs*, const char *zName, sqlite3_shm**); - int (*xShmSize)(sqlite3_shm*, int reqSize, int *pNewSize, char**); + int (*xShmSize)(sqlite3_shm*, int reqSize, int *pNewSize, void**); int (*xShmRelease)(sqlite3_shm*); int (*xShmPush)(sqlite3_shm*); int (*xShmPull)(sqlite3_shm*); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 7fbe053403..54effce127 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -823,8 +823,8 @@ struct sqlite3 { void (*xRollbackCallback)(void*); /* Invoked at every commit. */ void *pUpdateArg; void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64); - int (*xLogCallback)(void *, sqlite3 *, const char *, int); - void *pLogArg; + int (*xWalCallback)(void *, sqlite3 *, const char *, int); + void *pWalArg; void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*); void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*); void *pCollNeededArg; diff --git a/src/test1.c b/src/test1.c index 6f20cf9a90..f3eff8c672 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5113,7 +5113,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ #ifdef SQLITE_TEST extern char sqlite3_query_plan[]; static char *query_plan = sqlite3_query_plan; - extern int sqlite3_walsummary_mmap_incr; /* In wal.c */ +/* extern int sqlite3_walsummary_mmap_incr; // In wal.c */ #ifdef SQLITE_ENABLE_FTS3 extern int sqlite3_fts3_enable_parentheses; #endif @@ -5173,8 +5173,8 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ #ifdef SQLITE_TEST Tcl_LinkVar(interp, "sqlite_query_plan", (char*)&query_plan, TCL_LINK_STRING|TCL_LINK_READ_ONLY); - Tcl_LinkVar(interp, "sqlite_walsummary_mmap_incr", - (char*)&sqlite3_walsummary_mmap_incr, TCL_LINK_INT); +/* Tcl_LinkVar(interp, "sqlite_walsummary_mmap_incr", + (char*)&sqlite3_walsummary_mmap_incr, TCL_LINK_INT); */ #endif #ifdef SQLITE_DEBUG Tcl_LinkVar(interp, "sqlite_addop_trace", diff --git a/src/vdbe.c b/src/vdbe.c index a8eddf857f..60e72fadd2 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5252,11 +5252,11 @@ case OP_JournalMode: { if( eOld==PAGER_JOURNALMODE_WAL ){ /* If leaving WAL mode, close the log file. If successful, the call - ** to PagerCloseLog() checkpoints and deletes the write-ahead-log + ** to PagerCloseWal() checkpoints and deletes the write-ahead-log ** file. An EXCLUSIVE lock may still be held on the database file ** after a successful return. */ - rc = sqlite3PagerCloseLog(pPager); + rc = sqlite3PagerCloseWal(pPager); if( rc!=SQLITE_OK ) goto abort_due_to_error; sqlite3PagerJournalMode(pPager, eNew); }else{ diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 1c938ee5d2..12100ce56b 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -306,15 +306,15 @@ void sqlite3_result_error_nomem(sqlite3_context *pCtx){ pCtx->s.db->mallocFailed = 1; } -static int doLogCallbacks(sqlite3 *db){ +static int doWalCallbacks(sqlite3 *db){ int i; int rc = SQLITE_OK; for(i=0; inDb; i++){ Btree *pBt = db->aDb[i].pBt; if( pBt ){ - int nEntry = sqlite3PagerLogCallback(sqlite3BtreePager(pBt)); - if( db->xLogCallback && nEntry>0 && rc==SQLITE_OK - && db->xLogCallback(db->pLogArg, db, db->aDb[i].zName, nEntry) + int nEntry = sqlite3PagerWalCallback(sqlite3BtreePager(pBt)); + if( db->xWalCallback && nEntry>0 && rc==SQLITE_OK + && db->xWalCallback(db->pWalArg, db, db->aDb[i].zName, nEntry) ){ rc = sqlite3PagerCheckpoint(sqlite3BtreePager(pBt)); } @@ -406,7 +406,7 @@ static int sqlite3Step(Vdbe *p){ if( rc==SQLITE_DONE ){ assert( p->rc==SQLITE_OK ); - p->rc = doLogCallbacks(db); + p->rc = doWalCallbacks(db); if( p->rc!=SQLITE_OK ){ rc = SQLITE_ERROR; } diff --git a/src/wal.c b/src/wal.c index 8a8f4043d1..2de53b8b18 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1,22 +1,33 @@ - /* -** This file contains the implementation of a log file used in +** 2010 February 1 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the implementation of a write-ahead log file used in ** "journal_mode=wal" mode. */ +#include "wal.h" /* -** LOG FILE FORMAT +** WRITE-AHEAD LOG (WAL) FILE FORMAT ** -** A log file consists of a header followed by zero or more log frames. -** The log header is 12 bytes in size and consists of the following three +** A wal file consists of a header followed by zero or more "frames". +** The header is 12 bytes in size and consists of the following three ** big-endian 32-bit unsigned integer values: ** ** 0: Database page size, ** 4: Randomly selected salt value 1, ** 8: Randomly selected salt value 2. ** -** Immediately following the log header are zero or more log frames. Each +** Immediately following the header are zero or more frames. Each ** frame itself consists of a 16-byte header followed by a bytes ** of page data. The header is broken into 4 big-endian 32-bit unsigned ** integer values, as follows: @@ -29,56 +40,49 @@ */ /* -** LOG SUMMARY FILE FORMAT +** WAL-INDEX FILE FORMAT ** -** The log-summary file consists of a header region, followed by an -** region that contains no useful data (used to apply byte-range locks +** The wal-index file consists of a 32-byte header region, followed by an +** 8-byte region that contains no useful data (used to apply byte-range locks ** to), followed by the data region. ** ** The contents of both the header and data region are specified in terms ** of 1, 2 and 4 byte unsigned integers. All integers are stored in -** machine-endian order. +** machine-endian order. The wal-index is not a persistent file and +** so it does not need to be portable across archtectures. ** -** A log-summary file is essentially a shadow-pager map. It contains a -** mapping from database page number to the set of locations in the log +** A wal-index file is essentially a shadow-pager map. It contains a +** mapping from database page number to the set of locations in the wal ** file that contain versions of the database page. When a database -** client needs to read a page of data, it first queries the log-summary +** client needs to read a page of data, it first queries the wal-index ** file to determine if the required version of the page is stored in -** the log. If so, it is read from the log file. If not, it is read from -** the database file. +** the wal. If so, the page is read from the wal. If not, the page is +** read from the database file. ** -** Whenever a transaction is appended to the log or a checkpoint transfers -** data from the log file into the database file, the log-summary is +** Whenever a transaction is appended to the wal or a checkpoint transfers +** data from the wal into the database file, the wal-index is ** updated accordingly. ** -** The fields in the log-summary file header are described in the comment -** directly above the definition of struct LogSummaryHdr (see below). -** Immediately following the fields in the LogSummaryHdr structure is +** The fields in the wal-index file header are described in the comment +** directly above the definition of struct WalIndexHdr (see below). +** Immediately following the fields in the WalIndexHdr structure is ** an 8 byte checksum based on the contents of the header. This field is -** not the same as the iCheck1 and iCheck2 fields of the LogSummaryHdr. +** not the same as the iCheck1 and iCheck2 fields of the WalIndexHdr. */ -#include "wal.h" - -#include -#include -#include - -typedef struct LogSummaryHdr LogSummaryHdr; -typedef struct LogSummary LogSummary; -typedef struct LogIterator LogIterator; -typedef struct LogLock LogLock; +/* Object declarations */ +typedef struct WalIndexHdr WalIndexHdr; +typedef struct WalIterator WalIterator; /* -** The following structure may be used to store the same data that -** is stored in the log-summary header. +** The following object stores a copy of the wal-index header. ** ** Member variables iCheck1 and iCheck2 contain the checksum for the -** last frame written to the log, or 2 and 3 respectively if the log +** last frame written to the wal, or 2 and 3 respectively if the log ** is currently empty. */ -struct LogSummaryHdr { +struct WalIndexHdr { u32 iChange; /* Counter incremented each transaction */ u32 pgsz; /* Database page size in bytes */ u32 iLastPg; /* Address of last valid frame in log */ @@ -87,200 +91,46 @@ struct LogSummaryHdr { u32 iCheck2; /* Checkpoint value 2 */ }; -/* Size of serialized LogSummaryHdr object. */ -#define LOGSUMMARY_HDR_NFIELD (sizeof(LogSummaryHdr) / sizeof(u32)) +/* Size of serialized WalIndexHdr object. */ +#define WALINDEX_HDR_NFIELD (sizeof(WalIndexHdr) / sizeof(u32)) -/* A block of 16 bytes beginning at LOGSUMMARY_LOCK_OFFSET is reserved +/* A block of 16 bytes beginning at WALINDEX_LOCK_OFFSET is reserved ** for locks. Since some systems only feature mandatory file-locks, we ** do not read or write data from the region of the file on which locks ** are applied. */ -#define LOGSUMMARY_LOCK_OFFSET ((sizeof(LogSummaryHdr))+2*sizeof(u32)) -#define LOGSUMMARY_LOCK_RESERVED 16 +#define WALINDEX_LOCK_OFFSET ((sizeof(WalIndexHdr))+2*sizeof(u32)) +#define WALINDEX_LOCK_RESERVED 8 -/* Size of header before each frame in log file */ -#define LOG_FRAME_HDRSIZE 16 +/* Size of header before each frame in wal */ +#define WAL_FRAME_HDRSIZE 16 -/* Size of log header */ -#define LOG_HDRSIZE 12 +/* Size of write ahead log header */ +#define WAL_HDRSIZE 12 /* -** Return the offset of frame iFrame in the log file, assuming a database -** page size of pgsz bytes. The offset returned is to the start of the -** log frame-header. +** Return the offset of frame iFrame in the write-ahead log file, +** assuming a database page size of pgsz bytes. The offset returned +** is to the start of the write-ahead log frame-header. */ -#define logFrameOffset(iFrame, pgsz) ( \ - LOG_HDRSIZE + ((iFrame)-1)*((pgsz)+LOG_FRAME_HDRSIZE) \ +#define walFrameOffset(iFrame, pgsz) ( \ + WAL_HDRSIZE + ((iFrame)-1)*((pgsz)+WAL_FRAME_HDRSIZE) \ ) /* -** If using mmap() to access a shared (or otherwise) log-summary file, then -** the mapping size is incremented in units of the following size. -** -** A 64 KB log-summary mapping corresponds to a log file containing over -** 13000 frames, so the mapping size does not need to be increased often. +** An open write-ahead log file is represented by an instance of the +** following object. */ -#ifdef SQLITE_TEST -int sqlite3_walsummary_mmap_incr = 128; -# define LOGSUMMARY_MMAP_INCREMENT sqlite3_walsummary_mmap_incr -#else -# define LOGSUMMARY_MMAP_INCREMENT (64*1024) -#endif - -/* -** There is one instance of this structure for each log-summary object -** that this process has a connection to. They are stored in a linked -** list starting at pLogSummary (global variable). -** -** TODO: LogSummary.fd is a unix file descriptor. Unix APIs are used -** directly in this implementation because the VFS does not support -** the required blocking file-locks. -*/ -struct LogSummary { - sqlite3_mutex *mutex; /* Mutex used to protect this object */ - int nRef; /* Number of pointers to this structure */ - int fd; /* File descriptor open on log-summary */ - char *zPath; /* Path to associated WAL file */ - LogLock *pLock; /* Linked list of locks on this object */ - LogSummary *pNext; /* Next in global list */ - - int nData; /* Size of aData allocation/mapping */ - u32 *aData; /* File body */ -}; - -/* -** This module uses three different types of file-locks. All are taken -** on the log-summary file. The three types of locks are as follows: -** -** MUTEX: The MUTEX lock is used as a robust inter-process mutex. It -** is held while the log-summary header is modified, and -** sometimes when it is read. It is also held while a new client -** obtains the DMH lock (see below), and while log recovery is -** being run. -** -** DMS: The DMS (Dead Mans Switch mechanism) lock is used to ensure -** that log-recovery is always run following a system restart. -** When it first opens a log-summary file, a process takes a -** SHARED lock on the DMH region. This lock is not released until -** the log-summary file is closed. -** -** The process then attempts to upgrade to an EXCLUSIVE lock. If -** successful, then the contents of the log-summary file are deemed -** suspect and the log-summary header zeroed. This forces the -** first process that reads the log-summary file to run log -** recovery. After zeroing the log-summary header, the process -** downgrades to a SHARED lock on the DMH region. -** -** If the attempt to obtain the EXCLUSIVE lock fails, then the -** process concludes that some other process is already using the -** log-summary file, and it can therefore be trusted. -** -** The procedure described in the previous three paragraphs (taking -** a SHARED lock and then upgrading to an EXCLUSIVE lock to check -** if the process is the only one to have an open connection to the -** log file) is protected by holding the MUTEX lock. This avoids the -** race condition wherein the first two clients connect almost -** simultaneously following a system restart and each prevents -** the other from obtaining the EXCLUSIVE lock. -** -** -** REGION: There are 4 different region locks, regions A, B, C and D. -** Various EXCLUSIVE and SHARED locks on these regions are obtained -** when a client reads, writes or checkpoints the database. -** -** To obtain a reader lock: -** -** 1. Attempt a SHARED lock on regions A and B. -** 2. If step 1 is successful, drop the lock on region B. Or, if -** it is unsuccessful, attempt a SHARED lock on region D. -** 3. Repeat the above until the lock attempt in step 1 or 2 is -** successful. -** -** The reader lock is released when the read transaction is finished. -** -** To obtain a writer lock: -** -** 1. Take (wait for) an EXCLUSIVE lock on regions C and D. -** -** The locks are released after the write transaction is finished -** and, if any frames were committed to the log, the log-summary -** file updated. -** -** To obtain a checkpointer lock: -** -** 1. Take (wait for) an EXCLUSIVE lock on regions B and C. -** 2. Take (wait for) an EXCLUSIVE lock on region A. -** -** Step 1 waits until any existing writer has finished. And forces -** all new readers to become "region D" readers. -** -** Step 2 causes the checkpointer to wait until all existing region A -** readers have finished their transactions. Once the exclusive lock -** on region A has been obtained, only "region D" readers exist. -** These readers are operating on the snapshot at the head of the -** log. As such, the log can be safely copied into the database file -** without interfering with the readers. -** -** Once the checkpoint has finished and the log-summary header -** updated (to indicate the log contents can now be ignored), all -** locks are released. -** -** However, there may still exist region D readers using data in -** the body of the log file, so the log file itself cannot be -** truncated or overwritten until all region D readers have finished. -** That requirement is satisfied, because writers (the clients that -** write to the log file) require an exclusive lock on region D. -** Which they cannot get until all region D readers have finished. -*/ -#define LOG_LOCK_MUTEX (LOGSUMMARY_LOCK_OFFSET) -#define LOG_LOCK_DMH (LOG_LOCK_MUTEX+1) -#define LOG_LOCK_REGION (LOG_LOCK_DMH+1) - -/* -** The four lockable regions associated with each log-summary. A connection -** may take either a SHARED or EXCLUSIVE lock on each. An ORed combination -** of the following bitmasks is passed as the second argument to the -** logLockRegion() function. -*/ -#define LOG_REGION_A 0x01 -#define LOG_REGION_B 0x02 -#define LOG_REGION_C 0x04 -#define LOG_REGION_D 0x08 - -/* -** Values for the third parameter to logLockRegion(). -*/ -#define LOG_UNLOCK 0 /* Unlock a range of bytes */ -#define LOG_RDLOCK 1 /* Put a SHARED lock on a range of bytes */ -#define LOG_WRLOCK 2 /* Put an EXCLUSIVE lock on a byte-range */ -#define LOG_WRLOCKW 3 /* Block on EXCLUSIVE lock on a byte-range */ - -/* -** A single instance of this structure is allocated as part of each -** connection to a database log. All structures associated with the -** same log file are linked together into a list using LogLock.pNext -** starting at LogSummary.pLock. -** -** The mLock field of the structure describes the locks (if any) -** currently held by the connection. If a SHARED lock is held on -** any of the four locking regions, then the associated LOG_REGION_X -** bit (see above) is set. If an EXCLUSIVE lock is held on the region, -** then the (LOG_REGION_X << 8) bit is set. -*/ -struct LogLock { - LogLock *pNext; /* Next lock on the same log */ - u32 mLock; /* Mask of locks */ -}; - -struct Log { - LogSummary *pSummary; /* Log file summary data */ - sqlite3_vfs *pVfs; /* The VFS used to create pFd */ - sqlite3_file *pFd; /* File handle for log file */ - int isLocked; /* Non-zero if a snapshot is held open */ - int isWriteLocked; /* True if this is the writer connection */ - u32 iCallback; /* Value to pass to log callback (or 0) */ - LogSummaryHdr hdr; /* Log summary header for current snapshot */ - LogLock lock; /* Lock held by this connection (if any) */ +struct Wal { + sqlite3_vfs *pVfs; /* The VFS used to create pFd */ + sqlite3_file *pFd; /* File handle for WAL file */ + u32 iCallback; /* Value to pass to log callback (or 0) */ + sqlite3_shm *pWIndex; /* The open wal-index file */ + int szWIndex; /* Size of the wal-index */ + u32 *pWiData; /* Pointer to wal-index content in memory */ + u8 lockState; /* SQLITE_SHM_xxxx constant showing lock state */ + u8 readerType; /* SQLITE_SHM_READ or SQLITE_SHM_READ_FULL */ + WalIndexHdr hdr; /* Wal-index for current snapshot */ }; @@ -292,16 +142,16 @@ struct Log { ** ** The internals of this structure are only accessed by: ** -** logIteratorInit() - Create a new iterator, -** logIteratorNext() - Step an iterator, -** logIteratorFree() - Free an iterator. +** walIteratorInit() - Create a new iterator, +** walIteratorNext() - Step an iterator, +** walIteratorFree() - Free an iterator. ** -** This functionality is used by the checkpoint code (see logCheckpoint()). +** This functionality is used by the checkpoint code (see walCheckpoint()). */ -struct LogIterator { - int nSegment; /* Size of LogIterator.aSegment[] array */ +struct WalIterator { + int nSegment; /* Size of WalIterator.aSegment[] array */ int nFinal; /* Elements in segment nSegment-1 */ - struct LogSegment { + struct WalSegment { int iNext; /* Next aIndex index */ u8 *aIndex; /* Pointer to index array */ u32 *aDbPage; /* Pointer to db page array */ @@ -309,15 +159,6 @@ struct LogIterator { }; - -/* -** List of all LogSummary objects created by this process. Protected by -** static mutex LOG_SUMMARY_MUTEX. TODO: Should have a dedicated mutex -** here instead of borrowing the LRU mutex. -*/ -#define LOG_SUMMARY_MUTEX SQLITE_MUTEX_STATIC_LRU -static LogSummary *pLogSummary = 0; - /* ** Generate an 8 byte checksum based on the data in array aByte[] and the ** initial values of aCksum[0] and aCksum[1]. The checksum is written into @@ -336,7 +177,7 @@ static LogSummary *pLogSummary = 0; ** aCksum[0] = (u32)(aCksum[0] + (aCksum[0]>>24)); ** aCksum[1] = (u32)(aCksum[1] + (aCksum[1]>>24)); */ -static void logChecksumBytes(u8 *aByte, int nByte, u32 *aCksum){ +static void walChecksumBytes(u8 *aByte, int nByte, u32 *aCksum){ u64 sum1 = aCksum[0]; u64 sum2 = aCksum[1]; u32 *a32 = (u32 *)aByte; @@ -366,84 +207,42 @@ static void logChecksumBytes(u8 *aByte, int nByte, u32 *aCksum){ } /* -** Argument zPath must be a nul-terminated string containing a path-name. -** This function modifies the string in-place by removing any "./" or "../" -** elements in the path. For example, the following input: +** Attempt to change the lock status. ** -** "/home/user/plans/good/../evil/./world_domination.txt" -** -** is overwritten with the 'normalized' version: -** -** "/home/user/plans/evil/world_domination.txt" +** When changing the lock status to SQLITE_SHM_READ, store the +** type of reader lock (either SQLITE_SHM_READ or SQLITE_SHM_READ_FULL) +** in pWal->readerType. */ -static void logNormalizePath(char *zPath){ - int i, j; - char *z = zPath; - int n = strlen(z); - - while( n>1 && z[n-1]=='/' ){ n--; } - for(i=j=0; i0 && z[j-1]!='/' ){ j--; } - if( j>0 ){ j--; } - i += 2; - continue; - } +static int walSetLock(Wal *pWal, int desiredStatus){ + int rc, got; + if( pWal->lockState==desiredStatus ) return SQLITE_OK; + rc = pWal->pVfs->xShmLock(pWal->pWIndex, desiredStatus, &got); + if( rc==SQLITE_OK ){ + pWal->lockState = desiredStatus; + if( desiredStatus==SQLITE_SHM_READ ){ + pWal->readerType = got; } - z[j++] = z[i]; - } - z[j] = 0; -} - -/* -** Unmap the log-summary mapping and close the file-descriptor. If -** the isTruncate argument is non-zero, truncate the log-summary file -** region to zero bytes. -** -** Regardless of the value of isTruncate, close the file-descriptor -** opened on the log-summary file. -*/ -static int logSummaryUnmap(LogSummary *pSummary, int isUnlink){ - int rc = SQLITE_OK; - if( pSummary->aData ){ - assert( pSummary->fd>0 ); - munmap(pSummary->aData, pSummary->nData); - pSummary->aData = 0; - if( isUnlink ){ - char *zFile = sqlite3_mprintf("%s-summary", pSummary->zPath); - if( !zFile ){ - rc = SQLITE_NOMEM; - } - unlink(zFile); - sqlite3_free(zFile); - } - } - if( pSummary->fd>0 ){ - close(pSummary->fd); - pSummary->fd = -1; } return rc; } -static void logSummaryWriteHdr(LogSummary *pSummary, LogSummaryHdr *pHdr){ - u32 *aHdr = pSummary->aData; /* Write header here */ - u32 *aCksum = &aHdr[LOGSUMMARY_HDR_NFIELD]; /* Write header cksum here */ +/* +** Update the header of the wal-index file. +*/ +static void walIndexWriteHdr(Wal *pWal, WalIndexHdr *pHdr){ + u32 *aHdr = pWal->pWiData; /* Write header here */ + u32 *aCksum = &aHdr[WALINDEX_HDR_NFIELD]; /* Write header cksum here */ - assert( LOGSUMMARY_HDR_NFIELD==sizeof(LogSummaryHdr)/4 ); - memcpy(aHdr, pHdr, sizeof(LogSummaryHdr)); + assert( WALINDEX_HDR_NFIELD==sizeof(WalIndexHdr)/4 ); + assert( aHdr!=0 ); + memcpy(aHdr, pHdr, sizeof(WalIndexHdr)); aCksum[0] = aCksum[1] = 1; - logChecksumBytes((u8 *)aHdr, sizeof(LogSummaryHdr), aCksum); + walChecksumBytes((u8 *)aHdr, sizeof(WalIndexHdr), aCksum); } /* ** This function encodes a single frame header and writes it to a buffer -** supplied by the caller. A log frame-header is made up of a series of +** supplied by the caller. A frame-header is made up of a series of ** 4-byte big-endian integers, as follows: ** ** 0: Database page size in bytes. @@ -452,7 +251,7 @@ static void logSummaryWriteHdr(LogSummary *pSummary, LogSummaryHdr *pHdr){ ** 12: Frame checksum 1. ** 16: Frame checksum 2. */ -static void logEncodeFrame( +static void walEncodeFrame( u32 *aCksum, /* IN/OUT: Checksum values */ u32 iPage, /* Database page number for frame */ u32 nTruncate, /* New db size (or 0 for non-commit frames) */ @@ -460,13 +259,13 @@ static void logEncodeFrame( u8 *aData, /* Pointer to page data (for checksum) */ u8 *aFrame /* OUT: Write encoded frame here */ ){ - assert( LOG_FRAME_HDRSIZE==16 ); + assert( WAL_FRAME_HDRSIZE==16 ); sqlite3Put4byte(&aFrame[0], iPage); sqlite3Put4byte(&aFrame[4], nTruncate); - logChecksumBytes(aFrame, 8, aCksum); - logChecksumBytes(aData, nData, aCksum); + walChecksumBytes(aFrame, 8, aCksum); + walChecksumBytes(aData, nData, aCksum); sqlite3Put4byte(&aFrame[8], aCksum[0]); sqlite3Put4byte(&aFrame[12], aCksum[1]); @@ -476,7 +275,7 @@ static void logEncodeFrame( ** Return 1 and populate *piPage, *pnTruncate and aCksum if the ** frame checksum looks Ok. Otherwise return 0. */ -static int logDecodeFrame( +static int walDecodeFrame( u32 *aCksum, /* IN/OUT: Checksum values */ u32 *piPage, /* OUT: Database page number for frame */ u32 *pnTruncate, /* OUT: New db size (or 0 if not commit) */ @@ -484,10 +283,10 @@ static int logDecodeFrame( u8 *aData, /* Pointer to page data (for checksum) */ u8 *aFrame /* Frame data */ ){ - assert( LOG_FRAME_HDRSIZE==16 ); + assert( WAL_FRAME_HDRSIZE==16 ); - logChecksumBytes(aFrame, 8, aCksum); - logChecksumBytes(aData, nData, aCksum); + walChecksumBytes(aFrame, 8, aCksum); + walChecksumBytes(aData, nData, aCksum); if( aCksum[0]!=sqlite3Get4byte(&aFrame[8]) || aCksum[1]!=sqlite3Get4byte(&aFrame[12]) @@ -501,8 +300,8 @@ static int logDecodeFrame( return 1; } -static void logMergesort8( - Pgno *aContent, /* Pages in log */ +static void walMergesort8( + Pgno *aContent, /* Pages in wal */ u8 *aBuffer, /* Buffer of at least *pnList items to use */ u8 *aList, /* IN/OUT: List to sort */ int *pnList /* IN/OUT: Number of elements in aList[] */ @@ -518,8 +317,8 @@ static void logMergesort8( int iOut = 0; /* Current index in output buffer */ /* TODO: Change to non-recursive version. */ - logMergesort8(aContent, aBuffer, aLeft, &nLeft); - logMergesort8(aContent, aBuffer, aRight, &nRight); + walMergesort8(aContent, aBuffer, aLeft, &nLeft); + walMergesort8(aContent, aBuffer, aRight, &nRight); while( iRightfd at pSummary->aData. If the summary file is smaller than -** nByte bytes in size when this function is called, ftruncate() is -** used to expand it before it is mapped. -** -** It is assumed that an exclusive lock is held on the summary file -** by the caller (to protect the ftruncate()). -*/ -static int logSummaryMap(LogSummary *pSummary, int nByte){ - struct stat sStat; - int rc; - int fd = pSummary->fd; - void *pMap; - - assert( pSummary->aData==0 ); - - /* If the file is less than nByte bytes in size, cause it to grow. */ - rc = fstat(fd, &sStat); - if( rc!=0 ) return SQLITE_IOERR; - if( sStat.st_sizeaData = (u32 *)pMap; - pSummary->nData = nByte/4; - - return SQLITE_OK; -} - -/* -** The log-summary file is already mapped to pSummary->aData[], but the -** mapping needs to be resized. Unmap and remap the file so that the mapping -** is at least nByte bytes in size, or the size of the entire file if it -** is larger than nByte bytes. -*/ -static int logSummaryRemap(LogSummary *pSummary, int nByte){ - int rc; - sqlite3_mutex_enter(pSummary->mutex); - munmap(pSummary->aData, pSummary->nData*4); - pSummary->aData = 0; - rc = logSummaryMap(pSummary, nByte); - sqlite3_mutex_leave(pSummary->mutex); - return rc; -} - -/* -** Return the index in the LogSummary.aData array that corresponds to -** frame iFrame. The log-summary file consists of a header, followed by +** Return the index in the WalIndex.aData array that corresponds to +** frame iFrame. The wal-index file consists of a header, followed by ** alternating "map" and "index" blocks. */ -static int logSummaryEntry(u32 iFrame){ +static int walIndexEntry(u32 iFrame){ return ( - (LOGSUMMARY_LOCK_OFFSET+LOGSUMMARY_LOCK_RESERVED)/sizeof(u32) + (WALINDEX_LOCK_OFFSET+WALINDEX_LOCK_RESERVED)/sizeof(u32) + (((iFrame-1)>>8)<<6) /* Indexes that occur before iFrame */ + iFrame-1 /* Db page numbers that occur before iFrame */ ); } +/* +** Release our reference to the wal-index memory map. +*/ +static void walIndexUnmap(Wal *pWal){ + if( pWal->pWiData ){ + pWal->pVfs->xShmRelease(pWal->pWIndex); + pWal->pWiData = 0; + } +} /* -** Set an entry in the log-summary map to map log frame iFrame to db -** page iPage. Values are always appended to the log-summary (i.e. the +** Map the wal-index file into memory if it isn't already. +*/ +static int walIndexMap(Wal *pWal){ + int rc = SQLITE_OK; + if( pWal->pWiData==0 ){ + rc = pWal->pVfs->xShmSize(pWal->pWIndex, -1, + &pWal->szWIndex, (void**)(char*)&pWal->pWiData); + } + return rc; +} + +/* +** Resize the wal-index file. +*/ +static int walIndexRemap(Wal *pWal, int newSize){ + int rc; + walIndexUnmap(pWal); + rc = pWal->pVfs->xShmSize(pWal->pWIndex, newSize, + &pWal->szWIndex, (void**)(char*)&pWal->pWiData); + return rc; +} + +/* +** Increment by which to increase the wal-index file size. +*/ +#define WALINDEX_MMAP_INCREMENT (64*1024) + +/* +** Set an entry in the wal-index map to map log frame iFrame to db +** page iPage. Values are always appended to the wal-index (i.e. the ** value of iFrame is always exactly one more than the value passed to ** the previous call), but that restriction is not enforced or asserted ** here. */ -static int logSummaryAppend(LogSummary *pSummary, u32 iFrame, u32 iPage){ - u32 iSlot = logSummaryEntry(iFrame); - - while( (iSlot+128)>=pSummary->nData ){ +static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ + u32 iSlot = walIndexEntry(iFrame); + + walIndexMap(pWal); + while( (iSlot+128)>=pWal->szWIndex ){ int rc; - int nByte = pSummary->nData*4 + LOGSUMMARY_MMAP_INCREMENT; + int nByte = pWal->szWIndex*4 + WALINDEX_MMAP_INCREMENT; - /* Unmap and remap the log-summary file. */ - rc = logSummaryRemap(pSummary, nByte); + /* Unmap and remap the wal-index file. */ + rc = walIndexRemap(pWal, nByte); if( rc!=SQLITE_OK ){ return rc; } } - /* Set the log-summary entry itself */ - pSummary->aData[iSlot] = iPage; + /* Set the wal-index entry itself */ + pWal->pWiData[iSlot] = iPage; /* If the frame number is a multiple of 256 (frames are numbered starting ** at 1), build an index of the most recently added 256 frames. @@ -657,13 +440,13 @@ static int logSummaryAppend(LogSummary *pSummary, u32 iFrame, u32 iPage){ u8 *aIndex; /* 256 bytes to build index in */ u8 *aTmp; /* Scratch space to use while sorting */ - aFrame = &pSummary->aData[iSlot-255]; - aIndex = (u8 *)&pSummary->aData[iSlot+1]; + aFrame = &pWal->pWiData[iSlot-255]; + aIndex = (u8 *)&pWal->pWiData[iSlot+1]; aTmp = &aIndex[256]; nIndex = 256; for(i=0; i<256; i++) aIndex[i] = (u8)i; - logMergesort8(aFrame, aTmp, aIndex, &nIndex); + walMergesort8(aFrame, aTmp, aIndex, &nIndex); memset(&aIndex[nIndex], aIndex[nIndex-1], 256-nIndex); } @@ -672,23 +455,24 @@ static int logSummaryAppend(LogSummary *pSummary, u32 iFrame, u32 iPage){ /* -** Recover the log-summary by reading the log file. The caller must hold -** an exclusive lock on the log-summary file. +** Recover the wal-index by reading the write-ahead log file. +** The caller must hold RECOVER lock on the wal-index file. */ -static int logSummaryRecover(LogSummary *pSummary, sqlite3_file *pFd){ +static int walIndexRecover(Wal *pWal){ int rc; /* Return Code */ i64 nSize; /* Size of log file */ - LogSummaryHdr hdr; /* Recovered log-summary header */ + WalIndexHdr hdr; /* Recovered wal-index header */ + assert( pWal->lockState==SQLITE_SHM_RECOVER ); memset(&hdr, 0, sizeof(hdr)); - rc = sqlite3OsFileSize(pFd, &nSize); + rc = sqlite3OsFileSize(pWal->pFd, &nSize); if( rc!=SQLITE_OK ){ return rc; } - if( nSize>LOG_FRAME_HDRSIZE ){ - u8 aBuf[LOG_FRAME_HDRSIZE]; /* Buffer to load first frame header into */ + if( nSize>WAL_FRAME_HDRSIZE ){ + u8 aBuf[WAL_FRAME_HDRSIZE]; /* Buffer to load first frame header into */ u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */ int nFrame; /* Number of bytes at aFrame */ u8 *aData; /* Pointer to data part of aFrame buffer */ @@ -700,7 +484,7 @@ static int logSummaryRecover(LogSummary *pSummary, sqlite3_file *pFd){ /* Read in the first frame header in the file (to determine the ** database page size). */ - rc = sqlite3OsRead(pFd, aBuf, LOG_HDRSIZE, 0); + rc = sqlite3OsRead(pWal->pFd, aBuf, WAL_HDRSIZE, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -716,26 +500,26 @@ static int logSummaryRecover(LogSummary *pSummary, sqlite3_file *pFd){ aCksum[1] = sqlite3Get4byte(&aBuf[8]); /* Malloc a buffer to read frames into. */ - nFrame = nPgsz + LOG_FRAME_HDRSIZE; + nFrame = nPgsz + WAL_FRAME_HDRSIZE; aFrame = (u8 *)sqlite3_malloc(nFrame); if( !aFrame ){ return SQLITE_NOMEM; } - aData = &aFrame[LOG_FRAME_HDRSIZE]; + aData = &aFrame[WAL_FRAME_HDRSIZE]; /* Read all frames from the log file. */ iFrame = 0; - for(iOffset=LOG_HDRSIZE; (iOffset+nFrame)<=nSize; iOffset+=nFrame){ + for(iOffset=WAL_HDRSIZE; (iOffset+nFrame)<=nSize; iOffset+=nFrame){ u32 pgno; /* Database page number for frame */ u32 nTruncate; /* dbsize field from frame header */ int isValid; /* True if this frame is valid */ /* Read and decode the next log frame. */ - rc = sqlite3OsRead(pFd, aFrame, nFrame, iOffset); + rc = sqlite3OsRead(pWal->pFd, aFrame, nFrame, iOffset); if( rc!=SQLITE_OK ) break; - isValid = logDecodeFrame(aCksum, &pgno, &nTruncate, nPgsz, aData, aFrame); + isValid = walDecodeFrame(aCksum, &pgno, &nTruncate, nPgsz, aData, aFrame); if( !isValid ) break; - logSummaryAppend(pSummary, ++iFrame, pgno); + walIndexAppend(pWal, ++iFrame, pgno); /* If nTruncate is non-zero, this is a commit record. */ if( nTruncate ){ @@ -754,268 +538,7 @@ static int logSummaryRecover(LogSummary *pSummary, sqlite3_file *pFd){ } finished: - logSummaryWriteHdr(pSummary, &hdr); - return rc; -} - -/* -** Place, modify or remove a lock on the log-summary file associated -** with pSummary. -** -** The locked byte-range should be inside the region dedicated to -** locking. This region of the log-summary file is never read or written. -*/ -static int logLockFd( - LogSummary *pSummary, /* The log-summary object to lock */ - int iStart, /* First byte to lock */ - int nByte, /* Number of bytes to lock */ - int op /* LOG_UNLOCK, RDLOCK, WRLOCK or WRLOCKW */ -){ - int aType[4] = { - F_UNLCK, /* LOG_UNLOCK */ - F_RDLCK, /* LOG_RDLOCK */ - F_WRLCK, /* LOG_WRLOCK */ - F_WRLCK /* LOG_WRLOCKW */ - }; - int aOp[4] = { - F_SETLK, /* LOG_UNLOCK */ - F_SETLK, /* LOG_RDLOCK */ - F_SETLK, /* LOG_WRLOCK */ - F_SETLKW /* LOG_WRLOCKW */ - }; - struct flock f; /* Locking operation */ - int rc; /* Value returned by fcntl() */ - - assert( ArraySize(aType)==ArraySize(aOp) ); - assert( op>=0 && op0 ); - assert( iStart>=LOGSUMMARY_LOCK_OFFSET - && iStart+nByte<=LOGSUMMARY_LOCK_OFFSET+LOGSUMMARY_LOCK_RESERVED - ); -#if defined(SQLITE_DEBUG) && defined(SQLITE_OS_UNIX) - if( pSummary->aData ) memset(&((u8*)pSummary->aData)[iStart], op, nByte); -#endif - - memset(&f, 0, sizeof(f)); - f.l_type = aType[op]; - f.l_whence = SEEK_SET; - f.l_start = iStart; - f.l_len = nByte; - rc = fcntl(pSummary->fd, aOp[op], &f); - return (rc==0) ? SQLITE_OK : SQLITE_BUSY; -} - -static int logLockRegion(Log *pLog, u32 mRegion, int op){ - LogSummary *pSummary = pLog->pSummary; - LogLock *p; /* Used to iterate through in-process locks */ - u32 mOther; /* Locks held by other connections */ - u32 mNew; /* New mask for pLog */ - - assert( - /* Writer lock operations */ - (op==LOG_WRLOCK && mRegion==(LOG_REGION_C|LOG_REGION_D)) - || (op==LOG_UNLOCK && mRegion==(LOG_REGION_C|LOG_REGION_D)) - - /* Normal reader lock operations */ - || (op==LOG_RDLOCK && mRegion==(LOG_REGION_A|LOG_REGION_B)) - || (op==LOG_UNLOCK && mRegion==(LOG_REGION_A)) - || (op==LOG_UNLOCK && mRegion==(LOG_REGION_B)) - - /* Region D reader lock operations */ - || (op==LOG_RDLOCK && mRegion==(LOG_REGION_D)) - || (op==LOG_RDLOCK && mRegion==(LOG_REGION_A)) - || (op==LOG_UNLOCK && mRegion==(LOG_REGION_D)) - - /* Checkpointer lock operations */ - || (op==LOG_WRLOCK && mRegion==(LOG_REGION_B|LOG_REGION_C)) - || (op==LOG_WRLOCK && mRegion==(LOG_REGION_A)) - || (op==LOG_UNLOCK && mRegion==(LOG_REGION_B|LOG_REGION_C)) - || (op==LOG_UNLOCK && mRegion==(LOG_REGION_A|LOG_REGION_B|LOG_REGION_C)) - ); - - /* Assert that a connection never tries to go from an EXCLUSIVE to a - ** SHARED lock on a region. Moving from SHARED to EXCLUSIVE sometimes - ** happens though (when a region D reader upgrades to a writer). - */ - assert( op!=LOG_RDLOCK || 0==(pLog->lock.mLock & (mRegion<<8)) ); - - sqlite3_mutex_enter(pSummary->mutex); - - /* Calculate a mask of logs held by all connections in this process apart - ** from this one. The least significant byte of the mask contains a mask - ** of the SHARED logs held. The next least significant byte of the mask - ** indicates the EXCLUSIVE locks held. For example, to test if some other - ** connection is holding a SHARED lock on region A, or an EXCLUSIVE lock - ** on region C, do: - ** - ** hasSharedOnA = (mOther & (LOG_REGION_A<<0)); - ** hasExclusiveOnC = (mOther & (LOG_REGION_C<<8)); - ** - ** In all masks, if the bit in the EXCLUSIVE byte mask is set, so is the - ** corresponding bit in the SHARED mask. - */ - mOther = 0; - for(p=pSummary->pLock; p; p=p->pNext){ - assert( (p->mLock & (p->mLock<<8))==(p->mLock&0x0000FF00) ); - if( p!=&pLog->lock ){ - mOther |= p->mLock; - } - } - - /* If this call is to lock a region (not to unlock one), test if locks held - ** by any other connection in this process prevent the new locks from - ** begin granted. If so, exit the summary mutex and return SQLITE_BUSY. - */ - if( op && (mOther & (mRegion << (op==LOG_RDLOCK ? 8 : 0))) ){ - sqlite3_mutex_leave(pSummary->mutex); - return SQLITE_BUSY; - } - - /* Figure out the new log mask for this connection. */ - switch( op ){ - case LOG_UNLOCK: - mNew = (pLog->lock.mLock & ~(mRegion|(mRegion<<8))); - break; - case LOG_RDLOCK: - mNew = (pLog->lock.mLock | mRegion); - break; - default: - assert( op==LOG_WRLOCK ); - mNew = (pLog->lock.mLock | (mRegion<<8) | mRegion); - break; - } - - /* Now modify the locks held on the log-summary file descriptor. This - ** file descriptor is shared by all log connections in this process. - ** Therefore: - ** - ** + If one or more log connections in this process hold a SHARED lock - ** on a region, the file-descriptor should hold a SHARED lock on - ** the file region. - ** - ** + If a log connection in this process holds an EXCLUSIVE lock on a - ** region, the file-descriptor should also hold an EXCLUSIVE lock on - ** the region in question. - ** - ** If this is an LOG_UNLOCK operation, only regions for which no other - ** connection holds a lock should actually be unlocked. And if this - ** is a LOG_RDLOCK operation and other connections already hold all - ** the required SHARED locks, then no system call is required. - */ - if( op==LOG_UNLOCK ){ - mRegion = (mRegion & ~mOther); - } - if( (op==LOG_WRLOCK) - || (op==LOG_UNLOCK && mRegion) - || (op==LOG_RDLOCK && (mOther&mRegion)!=mRegion) - ){ - struct LockMap { - int iStart; /* Byte offset to start locking operation */ - int iLen; /* Length field for locking operation */ - } aMap[] = { - /* 0000 */ {0, 0}, /* 0001 */ {3+LOG_LOCK_REGION, 1}, - /* 0010 */ {2+LOG_LOCK_REGION, 1}, /* 0011 */ {2+LOG_LOCK_REGION, 2}, - /* 0100 */ {1+LOG_LOCK_REGION, 1}, /* 0101 */ {0, 0}, - /* 0110 */ {1+LOG_LOCK_REGION, 2}, /* 0111 */ {1+LOG_LOCK_REGION, 3}, - /* 1000 */ {0+LOG_LOCK_REGION, 1}, /* 1001 */ {0, 0}, - /* 1010 */ {0, 0}, /* 1011 */ {0, 0}, - /* 1100 */ {0+LOG_LOCK_REGION, 2}, /* 1101 */ {0, 0}, - /* 1110 */ {0, 0}, /* 1111 */ {0, 0} - }; - int rc; /* Return code of logLockFd() */ - - assert( mRegionmutex); - return rc; - } - } - - pLog->lock.mLock = mNew; - sqlite3_mutex_leave(pSummary->mutex); - return SQLITE_OK; -} - -/* -** Lock the DMH region, either with an EXCLUSIVE or SHARED lock. This -** function is never called with LOG_UNLOCK - the only way the DMH region -** is every completely unlocked is by by closing the file descriptor. -*/ -static int logLockDMH(LogSummary *pSummary, int eLock){ - assert( sqlite3_mutex_held(pSummary->mutex) ); - assert( eLock==LOG_RDLOCK || eLock==LOG_WRLOCK ); - return logLockFd(pSummary, LOG_LOCK_DMH, 1, eLock); -} - -/* -** Lock (or unlock) the MUTEX region. It is always locked using an -** EXCLUSIVE, blocking lock. -*/ -static int logLockMutex(LogSummary *pSummary, int eLock){ - assert( sqlite3_mutex_held(pSummary->mutex) ); - assert( eLock==LOG_WRLOCKW || eLock==LOG_UNLOCK ); - logLockFd(pSummary, LOG_LOCK_MUTEX, 1, eLock); - return SQLITE_OK; -} - -/* -** This function intializes the connection to the log-summary identified -** by struct pSummary. -*/ -static int logSummaryInit( - LogSummary *pSummary, /* Log summary object to initialize */ - sqlite3_file *pFd /* File descriptor open on log file */ -){ - int rc; /* Return Code */ - char *zFile; /* File name for summary file */ - - assert( pSummary->fd<0 ); - assert( pSummary->aData==0 ); - assert( pSummary->nRef>0 ); - assert( pSummary->zPath ); - - /* Open a file descriptor on the summary file. */ - zFile = sqlite3_mprintf("%s-summary", pSummary->zPath); - if( !zFile ){ - return SQLITE_NOMEM; - } - pSummary->fd = open(zFile, O_RDWR|O_CREAT, S_IWUSR|S_IRUSR); - sqlite3_free(zFile); - if( pSummary->fd<0 ){ - return SQLITE_IOERR; - } - - /* Grab an exclusive lock the summary file. Then mmap() it. - ** - ** TODO: This code needs to be enhanced to support a growable mapping. - ** For now, just make the mapping very large to start with. The - ** pages should not be allocated until they are first accessed anyhow, - ** so using a large mapping consumes no more resources than a smaller - ** one would. - */ - assert( sqlite3_mutex_held(pSummary->mutex) ); - rc = logLockMutex(pSummary, LOG_WRLOCKW); - if( rc!=SQLITE_OK ) return rc; - rc = logSummaryMap(pSummary, LOGSUMMARY_MMAP_INCREMENT); - if( rc!=SQLITE_OK ) goto out; - - /* Try to obtain an EXCLUSIVE lock on the dead-mans-hand region. If this - ** is possible, the contents of the log-summary file (if any) may not - ** be trusted. Zero the log-summary header before continuing. - */ - rc = logLockDMH(pSummary, LOG_WRLOCK); - if( rc==SQLITE_OK ){ - memset(pSummary->aData, 0, (LOGSUMMARY_HDR_NFIELD+2)*sizeof(u32) ); - } - rc = logLockDMH(pSummary, LOG_RDLOCK); - if( rc!=SQLITE_OK ){ - rc = SQLITE_IOERR; - } - - out: - logLockMutex(pSummary, LOG_UNLOCK); + walIndexWriteHdr(pWal, &hdr); return rc; } @@ -1027,112 +550,57 @@ static int logSummaryInit( ** ** A SHARED lock should be held on the database file when this function ** is called. The purpose of this SHARED lock is to prevent any other -** client from unlinking the log or log-summary file. If another process +** client from unlinking the log or wal-index file. If another process ** were to do this just after this client opened one of these files, the ** system would be badly broken. */ int sqlite3WalOpen( - sqlite3_vfs *pVfs, /* vfs module to open log file with */ + sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */ const char *zDb, /* Name of database file */ - Log **ppLog /* OUT: Allocated Log handle */ + Wal **ppWal /* OUT: Allocated Wal handle */ ){ int rc = SQLITE_OK; /* Return Code */ - Log *pRet; /* Object to allocate and return */ - LogSummary *pSummary = 0; /* Summary object */ - sqlite3_mutex *mutex = 0; /* LOG_SUMMARY_MUTEX mutex */ + Wal *pRet; /* Object to allocate and return */ int flags; /* Flags passed to OsOpen() */ char *zWal = 0; /* Path to WAL file */ int nWal; /* Length of zWal in bytes */ assert( zDb ); + if( pVfs->xShmOpen==0 ) return SQLITE_CANTOPEN; - /* Allocate an instance of struct Log to return. */ - *ppLog = 0; - pRet = (Log *)sqlite3MallocZero(sizeof(Log) + pVfs->szOsFile); - if( !pRet ) goto out; + /* Allocate an instance of struct Wal to return. */ + *ppWal = 0; + nWal = strlen(zDb); + pRet = (Wal*)sqlite3MallocZero(sizeof(Wal) + pVfs->szOsFile + nWal+11); + if( !pRet ) goto wal_open_out; pRet->pVfs = pVfs; pRet->pFd = (sqlite3_file *)&pRet[1]; + zWal = pVfs->szOsFile + (char*)pRet->pFd; + sqlite3_snprintf(nWal, zWal, "%s-wal-index", zDb); + rc = pVfs->xShmOpen(pVfs, zWal, &pRet->pWIndex); + if( rc ) goto wal_open_out; - /* Normalize the path name. */ - zWal = sqlite3_mprintf("%s-wal", zDb); - if( !zWal ) goto out; - logNormalizePath(zWal); + /* Open file handle on the write-ahead log file. */ + zWal[nWal-6] = 0; flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_MAIN_JOURNAL); - nWal = sqlite3Strlen30(zWal); + rc = sqlite3OsOpen(pVfs, zWal, pRet->pFd, flags, &flags); - /* Enter the mutex that protects the linked-list of LogSummary structures */ - if( sqlite3GlobalConfig.bCoreMutex ){ - mutex = sqlite3_mutex_alloc(LOG_SUMMARY_MUTEX); - } - sqlite3_mutex_enter(mutex); - - /* Search for an existing log summary object in the linked list. If one - ** cannot be found, allocate and initialize a new object. - */ - for(pSummary=pLogSummary; pSummary; pSummary=pSummary->pNext){ - int nPath = sqlite3Strlen30(pSummary->zPath); - if( nWal==nPath && 0==memcmp(pSummary->zPath, zWal, nPath) ) break; - } - if( !pSummary ){ - int nByte = sizeof(LogSummary) + nWal + 1; - pSummary = (LogSummary *)sqlite3MallocZero(nByte); - if( !pSummary ){ - rc = SQLITE_NOMEM; - goto out; - } - if( sqlite3GlobalConfig.bCoreMutex ){ - pSummary->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_RECURSIVE); - } - pSummary->zPath = (char *)&pSummary[1]; - pSummary->fd = -1; - memcpy(pSummary->zPath, zWal, nWal); - pSummary->pNext = pLogSummary; - pLogSummary = pSummary; - } - pSummary->nRef++; - pRet->pSummary = pSummary; - - /* Exit the mutex protecting the linked-list of LogSummary objects. */ - sqlite3_mutex_leave(mutex); - mutex = 0; - - /* Open file handle on the log file. */ - rc = sqlite3OsOpen(pVfs, pSummary->zPath, pRet->pFd, flags, &flags); - if( rc!=SQLITE_OK ) goto out; - - /* Object pSummary is shared between all connections to the database made - ** by this process. So at this point it may or may not be connected to - ** the log-summary. If it is not, connect it. - */ - sqlite3_mutex_enter(pSummary->mutex); - mutex = pSummary->mutex; - if( pSummary->fd<0 ){ - rc = logSummaryInit(pSummary, pRet->pFd); - } - - pRet->lock.pNext = pSummary->pLock; - pSummary->pLock = &pRet->lock; - - out: - sqlite3_mutex_leave(mutex); - sqlite3_free(zWal); +wal_open_out: if( rc!=SQLITE_OK ){ - assert(0); if( pRet ){ + pVfs->xShmClose(pRet->pWIndex); sqlite3OsClose(pRet->pFd); sqlite3_free(pRet); } - assert( !pSummary || pSummary->nRef==0 ); - sqlite3_free(pSummary); } - *ppLog = pRet; + *ppWal = pRet; return rc; } -static int logIteratorNext( - LogIterator *p, /* Iterator */ - u32 *piPage, /* OUT: Next db page to write */ - u32 *piFrame /* OUT: Log frame to read from */ +static int walIteratorNext( + WalIterator *p, /* Iterator */ + u32 *piPage, /* OUT: Next db page to write */ + u32 *piFrame /* OUT: Wal frame to read from */ ){ u32 iMin = *piPage; u32 iRet = 0xFFFFFFFF; @@ -1140,7 +608,7 @@ static int logIteratorNext( int nBlock = p->nFinal; for(i=p->nSegment-1; i>=0; i--){ - struct LogSegment *pSegment = &p->aSegment[i]; + struct WalSegment *pSegment = &p->aSegment[i]; while( pSegment->iNextaDbPage[pSegment->aIndex[pSegment->iNext]]; if( iPg>iMin ){ @@ -1160,23 +628,25 @@ static int logIteratorNext( return (iRet==0xFFFFFFFF); } -static LogIterator *logIteratorInit(Log *pLog){ - u32 *aData = pLog->pSummary->aData; - LogIterator *p; /* Return value */ +static WalIterator *walIteratorInit(Wal *pWal){ + u32 *aData; /* Content of the wal-index file */ + WalIterator *p; /* Return value */ int nSegment; /* Number of segments to merge */ u32 iLast; /* Last frame in log */ int nByte; /* Number of bytes to allocate */ int i; /* Iterator variable */ int nFinal; /* Number of unindexed entries */ - struct LogSegment *pFinal; /* Final (unindexed) segment */ + struct WalSegment *pFinal; /* Final (unindexed) segment */ u8 *aTmp; /* Temp space used by merge-sort */ - iLast = pLog->hdr.iLastPg; + walIndexMap(pWal); + aData = pWal->pWiData; + iLast = pWal->hdr.iLastPg; nSegment = (iLast >> 8) + 1; nFinal = (iLast & 0x000000FF); - nByte = sizeof(LogIterator) + (nSegment-1)*sizeof(struct LogSegment) + 512; - p = (LogIterator *)sqlite3_malloc(nByte); + nByte = sizeof(WalIterator) + (nSegment-1)*sizeof(struct WalSegment) + 512; + p = (WalIterator *)sqlite3_malloc(nByte); if( p ){ memset(p, 0, nByte); p->nSegment = nSegment; @@ -1184,63 +654,63 @@ static LogIterator *logIteratorInit(Log *pLog){ } for(i=0; iaSegment[i].aDbPage = &aData[logSummaryEntry(i*256+1)]; - p->aSegment[i].aIndex = (u8 *)&aData[logSummaryEntry(i*256+1)+256]; + p->aSegment[i].aDbPage = &aData[walIndexEntry(i*256+1)]; + p->aSegment[i].aIndex = (u8 *)&aData[walIndexEntry(i*256+1)+256]; } pFinal = &p->aSegment[nSegment-1]; - pFinal->aDbPage = &aData[logSummaryEntry((nSegment-1)*256+1)]; + pFinal->aDbPage = &aData[walIndexEntry((nSegment-1)*256+1)]; pFinal->aIndex = (u8 *)&pFinal[1]; aTmp = &pFinal->aIndex[256]; for(i=0; iaIndex[i] = i; } - logMergesort8(pFinal->aDbPage, aTmp, pFinal->aIndex, &nFinal); + walMergesort8(pFinal->aDbPage, aTmp, pFinal->aIndex, &nFinal); p->nFinal = nFinal; return p; } /* -** Free a log iterator allocated by logIteratorInit(). +** Free a log iterator allocated by walIteratorInit(). */ -static void logIteratorFree(LogIterator *p){ +static void walIteratorFree(WalIterator *p){ sqlite3_free(p); } /* ** Checkpoint the contents of the log file. */ -static int logCheckpoint( - Log *pLog, /* Log connection */ +static int walCheckpoint( + Wal *pWal, /* Wal connection */ sqlite3_file *pFd, /* File descriptor open on db file */ int sync_flags, /* Flags for OsSync() (or 0) */ u8 *zBuf /* Temporary buffer to use */ ){ int rc; /* Return code */ - int pgsz = pLog->hdr.pgsz; /* Database page-size */ - LogIterator *pIter = 0; /* Log iterator context */ + int pgsz = pWal->hdr.pgsz; /* Database page-size */ + WalIterator *pIter = 0; /* Wal iterator context */ u32 iDbpage = 0; /* Next database page to write */ - u32 iFrame = 0; /* Log frame containing data for iDbpage */ + u32 iFrame = 0; /* Wal frame containing data for iDbpage */ - if( pLog->hdr.iLastPg==0 ){ + if( pWal->hdr.iLastPg==0 ){ return SQLITE_OK; } /* Allocate the iterator */ - pIter = logIteratorInit(pLog); + pIter = walIteratorInit(pWal); if( !pIter ) return SQLITE_NOMEM; /* Sync the log file to disk */ if( sync_flags ){ - rc = sqlite3OsSync(pLog->pFd, sync_flags); + rc = sqlite3OsSync(pWal->pFd, sync_flags); if( rc!=SQLITE_OK ) goto out; } /* Iterate through the contents of the log, copying data to the db file. */ - while( 0==logIteratorNext(pIter, &iDbpage, &iFrame) ){ - rc = sqlite3OsRead(pLog->pFd, zBuf, pgsz, - logFrameOffset(iFrame, pgsz) + LOG_FRAME_HDRSIZE + while( 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ + rc = sqlite3OsRead(pWal->pFd, zBuf, pgsz, + walFrameOffset(iFrame, pgsz) + WAL_FRAME_HDRSIZE ); if( rc!=SQLITE_OK ) goto out; rc = sqlite3OsWrite(pFd, zBuf, pgsz, (iDbpage-1)*pgsz); @@ -1248,18 +718,18 @@ static int logCheckpoint( } /* Truncate the database file */ - rc = sqlite3OsTruncate(pFd, ((i64)pLog->hdr.nPage*(i64)pgsz)); + rc = sqlite3OsTruncate(pFd, ((i64)pWal->hdr.nPage*(i64)pgsz)); if( rc!=SQLITE_OK ) goto out; - /* Sync the database file. If successful, update the log-summary. */ + /* Sync the database file. If successful, update the wal-index. */ if( sync_flags ){ rc = sqlite3OsSync(pFd, sync_flags); if( rc!=SQLITE_OK ) goto out; } - pLog->hdr.iLastPg = 0; - pLog->hdr.iCheck1 = 2; - pLog->hdr.iCheck2 = 3; - logSummaryWriteHdr(pLog->pSummary, &pLog->hdr); + pWal->hdr.iLastPg = 0; + pWal->hdr.iCheck1 = 2; + pWal->hdr.iCheck2 = 3; + walIndexWriteHdr(pWal, &pWal->hdr); /* TODO: If a crash occurs and the current log is copied into the ** database there is no problem. However, if a crash occurs while @@ -1275,14 +745,14 @@ static int logCheckpoint( ** an unwelcome performance hit. Alternatives are... */ #if 0 - memset(zBuf, 0, LOG_FRAME_HDRSIZE); - rc = sqlite3OsWrite(pLog->pFd, zBuf, LOG_FRAME_HDRSIZE, 0); + memset(zBuf, 0, WAL_FRAME_HDRSIZE); + rc = sqlite3OsWrite(pWal->pFd, zBuf, WAL_FRAME_HDRSIZE, 0); if( rc!=SQLITE_OK ) goto out; - rc = sqlite3OsSync(pLog->pFd, pLog->sync_flags); + rc = sqlite3OsSync(pWal->pFd, pWal->sync_flags); #endif out: - logIteratorFree(pIter); + walIteratorFree(pIter); return rc; } @@ -1290,180 +760,92 @@ static int logCheckpoint( ** Close a connection to a log file. */ int sqlite3WalClose( - Log *pLog, /* Log to close */ + Wal *pWal, /* Wal to close */ sqlite3_file *pFd, /* Database file */ int sync_flags, /* Flags to pass to OsSync() (or 0) */ u8 *zBuf /* Buffer of at least page-size bytes */ ){ int rc = SQLITE_OK; - if( pLog ){ - LogLock **ppL; - LogSummary *pSummary = pLog->pSummary; - sqlite3_mutex *mutex = 0; - - sqlite3_mutex_enter(pSummary->mutex); - for(ppL=&pSummary->pLock; *ppL!=&pLog->lock; ppL=&(*ppL)->pNext); - *ppL = pLog->lock.pNext; - sqlite3_mutex_leave(pSummary->mutex); - - if( sqlite3GlobalConfig.bCoreMutex ){ - mutex = sqlite3_mutex_alloc(LOG_SUMMARY_MUTEX); - } - sqlite3_mutex_enter(mutex); - - /* Decrement the reference count on the log summary. If this is the last - ** reference to the log summary object in this process, the object will - ** be freed. If this is also the last connection to the database, then - ** checkpoint the database and truncate the log and log-summary files - ** to zero bytes in size. - **/ - pSummary->nRef--; - if( pSummary->nRef==0 ){ - int rc; - LogSummary **pp; - for(pp=&pLogSummary; *pp!=pSummary; pp=&(*pp)->pNext); - *pp = (*pp)->pNext; - - sqlite3_mutex_leave(mutex); - - rc = sqlite3OsLock(pFd, SQLITE_LOCK_EXCLUSIVE); - if( rc==SQLITE_OK ){ - - /* This is the last connection to the database (including other - ** processes). Do three things: - ** - ** 1. Checkpoint the db. - ** 2. Truncate the log file. - ** 3. Unlink the log-summary file. - */ - rc = logCheckpoint(pLog, pFd, sync_flags, zBuf); - if( rc==SQLITE_OK ){ - rc = sqlite3OsDelete(pLog->pVfs, pSummary->zPath, 0); - } - - logSummaryUnmap(pSummary, 1); - }else{ - if( rc==SQLITE_BUSY ){ - rc = SQLITE_OK; - } - logSummaryUnmap(pSummary, 0); - } - - sqlite3_mutex_free(pSummary->mutex); - sqlite3_free(pSummary); - }else{ - sqlite3_mutex_leave(mutex); - } - - /* Close the connection to the log file and free the Log handle. */ - sqlite3OsClose(pLog->pFd); - sqlite3_free(pLog); + if( pWal ){ + pWal->pVfs->xShmClose(pWal->pWIndex); + sqlite3OsClose(pWal->pFd); + sqlite3_free(pWal); } return rc; } /* -** Enter and leave the log-summary mutex. In this context, entering the -** log-summary mutex means: -** -** 1. Obtaining mutex pLog->pSummary->mutex, and -** 2. Taking an exclusive lock on the log-summary file. -** -** i.e. this mutex locks out other processes as well as other threads -** hosted in this address space. -*/ -static int logEnterMutex(Log *pLog){ - LogSummary *pSummary = pLog->pSummary; - int rc; - - sqlite3_mutex_enter(pSummary->mutex); - rc = logLockMutex(pSummary, LOG_WRLOCKW); - if( rc!=SQLITE_OK ){ - sqlite3_mutex_leave(pSummary->mutex); - } - return rc; -} -static void logLeaveMutex(Log *pLog){ - LogSummary *pSummary = pLog->pSummary; - logLockMutex(pSummary, LOG_UNLOCK); - sqlite3_mutex_leave(pSummary->mutex); -} - -/* -** Try to read the log-summary header. Attempt to verify the header -** checksum. If the checksum can be verified, copy the log-summary -** header into structure pLog->hdr. If the contents of pLog->hdr are +** Try to read the wal-index header. Attempt to verify the header +** checksum. If the checksum can be verified, copy the wal-index +** header into structure pWal->hdr. If the contents of pWal->hdr are ** modified by this and pChanged is not NULL, set *pChanged to 1. ** Otherwise leave *pChanged unmodified. ** ** If the checksum cannot be verified return SQLITE_ERROR. */ -int logSummaryTryHdr(Log *pLog, int *pChanged){ +int walIndexTryHdr(Wal *pWal, int *pChanged){ u32 aCksum[2] = {1, 1}; - u32 aHdr[LOGSUMMARY_HDR_NFIELD+2]; + u32 aHdr[WALINDEX_HDR_NFIELD+2]; - /* Read the header. The caller may or may not have locked the log-summary + /* Read the header. The caller may or may not have locked the wal-index ** file, meaning it is possible that an inconsistent snapshot is read ** from the file. If this happens, return SQLITE_ERROR. The caller will ** retry. Or, if the caller has already locked the file and the header ** still looks inconsistent, it will run recovery. */ - memcpy(aHdr, pLog->pSummary->aData, sizeof(aHdr)); - logChecksumBytes((u8*)aHdr, sizeof(u32)*LOGSUMMARY_HDR_NFIELD, aCksum); - if( aCksum[0]!=aHdr[LOGSUMMARY_HDR_NFIELD] - || aCksum[1]!=aHdr[LOGSUMMARY_HDR_NFIELD+1] + memcpy(aHdr, pWal->pWiData, sizeof(aHdr)); + walChecksumBytes((u8*)aHdr, sizeof(u32)*WALINDEX_HDR_NFIELD, aCksum); + if( aCksum[0]!=aHdr[WALINDEX_HDR_NFIELD] + || aCksum[1]!=aHdr[WALINDEX_HDR_NFIELD+1] ){ return SQLITE_ERROR; } - if( memcmp(&pLog->hdr, aHdr, sizeof(LogSummaryHdr)) ){ + if( memcmp(&pWal->hdr, aHdr, sizeof(WalIndexHdr)) ){ if( pChanged ){ *pChanged = 1; } - memcpy(&pLog->hdr, aHdr, sizeof(LogSummaryHdr)); + memcpy(&pWal->hdr, aHdr, sizeof(WalIndexHdr)); } return SQLITE_OK; } /* -** Read the log-summary header from the log-summary file into structure -** pLog->hdr. If attempting to verify the header checksum fails, try +** Read the wal-index header from the wal-index file into structure +** pWal->hdr. If attempting to verify the header checksum fails, try ** to recover the log before returning. ** -** If the log-summary header is successfully read, return SQLITE_OK. +** If the wal-index header is successfully read, return SQLITE_OK. ** Otherwise an SQLite error code. */ -int logSummaryReadHdr(Log *pLog, int *pChanged){ +static int walIndexReadHdr(Wal *pWal, int *pChanged){ int rc; + assert( pWal->lockState==SQLITE_SHM_READ ); + walIndexMap(pWal); + /* First try to read the header without a lock. Verify the checksum ** before returning. This will almost always work. - ** - ** TODO: Doing this causes a race-condition with the code that resizes - ** the mapping. Unless Log.pSummary->mutex is held, it is possible that - ** LogSummary.aData is invalid. */ -#if 0 - if( SQLITE_OK==logSummaryTryHdr(pLog, pChanged) ){ + if( SQLITE_OK==walIndexTryHdr(pWal, pChanged) ){ return SQLITE_OK; } -#endif - /* If the first attempt to read the header failed, lock the log-summary + /* If the first attempt to read the header failed, lock the wal-index ** file and try again. If the header checksum verification fails this ** time as well, run log recovery. */ - if( SQLITE_OK==(rc = logEnterMutex(pLog)) ){ - if( SQLITE_OK!=logSummaryTryHdr(pLog, pChanged) ){ + if( SQLITE_OK==(rc = walSetLock(pWal, SQLITE_SHM_RECOVER)) ){ + if( SQLITE_OK!=walIndexTryHdr(pWal, pChanged) ){ if( pChanged ){ *pChanged = 1; } - rc = logSummaryRecover(pLog->pSummary, pLog->pFd); + rc = walIndexRecover(pWal); if( rc==SQLITE_OK ){ - rc = logSummaryTryHdr(pLog, 0); + rc = walIndexTryHdr(pWal, 0); } } - logLeaveMutex(pLog); + walSetLock(pWal, SQLITE_SHM_READ); } return rc; @@ -1473,57 +855,29 @@ int logSummaryReadHdr(Log *pLog, int *pChanged){ ** Lock a snapshot. ** ** If this call obtains a new read-lock and the database contents have been -** modified since the most recent call to LogCloseSnapshot() on this Log +** modified since the most recent call to WalCloseSnapshot() on this Wal ** connection, then *pChanged is set to 1 before returning. Otherwise, it ** is left unmodified. This is used by the pager layer to determine whether ** or not any cached pages may be safely reused. */ -int sqlite3WalOpenSnapshot(Log *pLog, int *pChanged){ - int rc = SQLITE_OK; - if( pLog->isLocked==0 ){ - int nAttempt; +int sqlite3WalOpenSnapshot(Wal *pWal, int *pChanged){ + int rc; - /* Obtain a snapshot-lock on the log-summary file. The procedure - ** for obtaining the snapshot log is: - ** - ** 1. Attempt a SHARED lock on regions A and B. - ** 2a. If step 1 is successful, drop the lock on region B. - ** 2b. If step 1 is unsuccessful, attempt a SHARED lock on region D. - ** 3. Repeat the above until the lock attempt in step 1 or 2b is - ** successful. - ** - ** If neither of the locks can be obtained after 5 tries, presumably - ** something is wrong (i.e. a process not following the locking protocol). - ** Return an error code in this case. - */ - rc = SQLITE_BUSY; - for(nAttempt=0; nAttempt<5 && rc==SQLITE_BUSY; nAttempt++){ - rc = logLockRegion(pLog, LOG_REGION_A|LOG_REGION_B, LOG_RDLOCK); - if( rc==SQLITE_BUSY ){ - rc = logLockRegion(pLog, LOG_REGION_D, LOG_RDLOCK); - if( rc==SQLITE_OK ) pLog->isLocked = LOG_REGION_D; - }else{ - logLockRegion(pLog, LOG_REGION_B, LOG_UNLOCK); - pLog->isLocked = LOG_REGION_A; - } - } - if( rc!=SQLITE_OK ){ - return rc; - } + rc = walSetLock(pWal, SQLITE_SHM_READ); + if( rc==SQLITE_OK ){ + pWal->lockState = SQLITE_SHM_READ; - rc = logSummaryReadHdr(pLog, pChanged); + rc = walIndexReadHdr(pWal, pChanged); if( rc!=SQLITE_OK ){ /* An error occured while attempting log recovery. */ - sqlite3WalCloseSnapshot(pLog); + sqlite3WalCloseSnapshot(pWal); }else{ /* Check if the mapping needs to grow. */ - LogSummary *pSummary = pLog->pSummary; - - if( pLog->hdr.iLastPg - && logSummaryEntry(pLog->hdr.iLastPg)>=pSummary->nData - ){ - rc = logSummaryRemap(pSummary, 0); - assert( rc || logSummaryEntry(pLog->hdr.iLastPg)nData ); + if( pWal->hdr.iLastPg + && walIndexEntry(pWal->hdr.iLastPg)>=pWal->szWIndex + ){ + rc = walIndexRemap(pWal, 0); + assert( rc || walIndexEntry(pWal->hdr.iLastPg)szWIndex ); } } } @@ -1533,41 +887,39 @@ int sqlite3WalOpenSnapshot(Log *pLog, int *pChanged){ /* ** Unlock the current snapshot. */ -void sqlite3WalCloseSnapshot(Log *pLog){ - if( pLog->isLocked ){ - assert( pLog->isLocked==LOG_REGION_A || pLog->isLocked==LOG_REGION_D ); - logLockRegion(pLog, pLog->isLocked, LOG_UNLOCK); +void sqlite3WalCloseSnapshot(Wal *pWal){ + if( pWal->lockState!=SQLITE_SHM_UNLOCK ){ + assert( pWal->lockState==SQLITE_SHM_READ ); + walSetLock(pWal, SQLITE_SHM_UNLOCK); } - pLog->isLocked = 0; } /* ** Read a page from the log, if it is present. */ -int sqlite3WalRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut){ - LogSummary *pSummary = pLog->pSummary; +int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, u8 *pOut){ u32 iRead = 0; u32 *aData; - int iFrame = (pLog->hdr.iLastPg & 0xFFFFFF00); + int iFrame = (pWal->hdr.iLastPg & 0xFFFFFF00); - assert( pLog->isLocked ); - sqlite3_mutex_enter(pSummary->mutex); + assert( pWal->lockState==SQLITE_SHM_READ ); + walIndexMap(pWal); /* Do a linear search of the unindexed block of page-numbers (if any) - ** at the end of the log-summary. An alternative to this would be to + ** at the end of the wal-index. An alternative to this would be to ** build an index in private memory each time a read transaction is ** opened on a new snapshot. */ - aData = pSummary->aData; - if( pLog->hdr.iLastPg ){ - u32 *pi = &aData[logSummaryEntry(pLog->hdr.iLastPg)]; - u32 *piStop = pi - (pLog->hdr.iLastPg & 0xFF); + aData = pWal->pWiData; + if( pWal->hdr.iLastPg ){ + u32 *pi = &aData[walIndexEntry(pWal->hdr.iLastPg)]; + u32 *piStop = pi - (pWal->hdr.iLastPg & 0xFF); while( *pi!=pgno && pi!=piStop ) pi--; if( pi!=piStop ){ iRead = (pi-piStop) + iFrame; } } - assert( iRead==0 || aData[logSummaryEntry(iRead)]==pgno ); + assert( iRead==0 || aData[walIndexEntry(iRead)]==pgno ); while( iRead==0 && iFrame>0 ){ int iLow = 0; @@ -1576,7 +928,7 @@ int sqlite3WalRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut){ u8 *aIndex; iFrame -= 256; - aFrame = &aData[logSummaryEntry(iFrame+1)]; + aFrame = &aData[walIndexEntry(iFrame+1)]; aIndex = (u8 *)&aFrame[256]; while( iLow<=iHigh ){ @@ -1594,20 +946,19 @@ int sqlite3WalRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut){ } } } - assert( iRead==0 || aData[logSummaryEntry(iRead)]==pgno ); - - sqlite3_mutex_leave(pLog->pSummary->mutex); + assert( iRead==0 || aData[walIndexEntry(iRead)]==pgno ); + walIndexUnmap(pWal); /* If iRead is non-zero, then it is the log frame number that contains the ** required page. Read and return data from the log file. */ if( iRead ){ - i64 iOffset = logFrameOffset(iRead, pLog->hdr.pgsz) + LOG_FRAME_HDRSIZE; - *pInLog = 1; - return sqlite3OsRead(pLog->pFd, pOut, pLog->hdr.pgsz, iOffset); + i64 iOffset = walFrameOffset(iRead, pWal->hdr.pgsz) + WAL_FRAME_HDRSIZE; + *pInWal = 1; + return sqlite3OsRead(pWal->pFd, pOut, pWal->hdr.pgsz, iOffset); } - *pInLog = 0; + *pInWal = 0; return SQLITE_OK; } @@ -1615,9 +966,10 @@ int sqlite3WalRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut){ /* ** Set *pPgno to the size of the database file (or zero, if unknown). */ -void sqlite3WalDbsize(Log *pLog, Pgno *pPgno){ - assert( pLog->isLocked ); - *pPgno = pLog->hdr.nPage; +void sqlite3WalDbsize(Wal *pWal, Pgno *pPgno){ + assert( pWal->lockState==SQLITE_SHM_READ + || pWal->lockState==SQLITE_SHM_WRITE ); + *pPgno = pWal->hdr.nPage; } /* @@ -1625,54 +977,19 @@ void sqlite3WalDbsize(Log *pLog, Pgno *pPgno){ ** Otherwise, if the caller is operating on a snapshot that has already ** been overwritten by another writer, SQLITE_BUSY is returned. */ -int sqlite3WalWriteLock(Log *pLog, int op){ - assert( pLog->isLocked ); +int sqlite3WalWriteLock(Wal *pWal, int op){ + int rc; if( op ){ - - /* Obtain the writer lock */ - int rc = logLockRegion(pLog, LOG_REGION_C|LOG_REGION_D, LOG_WRLOCK); - if( rc!=SQLITE_OK ){ - return rc; - } - - /* If this is connection is a region D reader, then the SHARED lock on - ** region D has just been upgraded to EXCLUSIVE. But no lock at all is - ** held on region A. This means that if the write-transaction is committed - ** and this connection downgrades to a reader, it will be left with no - ** lock at all. And so its snapshot could get clobbered by a checkpoint - ** operation. - ** - ** To stop this from happening, grab a SHARED lock on region A now. - ** This should always be successful, as the only time a client holds - ** an EXCLUSIVE lock on region A, it must also be holding an EXCLUSIVE - ** lock on region C (a checkpointer does this). This is not possible, - ** as this connection currently has the EXCLUSIVE lock on region C. - */ - if( pLog->isLocked==LOG_REGION_D ){ - logLockRegion(pLog, LOG_REGION_A, LOG_RDLOCK); - pLog->isLocked = LOG_REGION_A; - } - - /* If this connection is not reading the most recent database snapshot, - ** it is not possible to write to the database. In this case release - ** the write locks and return SQLITE_BUSY. - */ - if( memcmp(&pLog->hdr, pLog->pSummary->aData, sizeof(pLog->hdr)) ){ - logLockRegion(pLog, LOG_REGION_C|LOG_REGION_D, LOG_UNLOCK); - return SQLITE_BUSY; - } - pLog->isWriteLocked = 1; - - }else if( pLog->isWriteLocked ){ - logLockRegion(pLog, LOG_REGION_C|LOG_REGION_D, LOG_UNLOCK); - memcpy(&pLog->hdr, pLog->pSummary->aData, sizeof(pLog->hdr)); - pLog->isWriteLocked = 0; + assert( pWal->lockState == SQLITE_SHM_READ ); + rc = walSetLock(pWal, SQLITE_SHM_WRITE); + }else if( pWal->lockState==SQLITE_SHM_WRITE ){ + rc = walSetLock(pWal, SQLITE_SHM_READ); } - return SQLITE_OK; + return rc; } /* -** The log handle passed to this function must be holding the write-lock. +** The Wal object passed to this function must be holding the write-lock. ** ** If any data has been written (but not committed) to the log file, this ** function moves the write-pointer back to the start of the transaction. @@ -1685,35 +1002,42 @@ int sqlite3WalWriteLock(Log *pLog, int op){ ** Otherwise, if the callback function does not return an error, this ** function returns SQLITE_OK. */ -int sqlite3WalUndo(Log *pLog, int (*xUndo)(void *, Pgno), void *pUndoCtx){ +int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ int rc = SQLITE_OK; - Pgno iMax = pLog->hdr.iLastPg; + Pgno iMax = pWal->hdr.iLastPg; Pgno iFrame; - assert( pLog->isWriteLocked ); - logSummaryReadHdr(pLog, 0); - for(iFrame=pLog->hdr.iLastPg+1; iFrame<=iMax && rc==SQLITE_OK; iFrame++){ - rc = xUndo(pUndoCtx, pLog->pSummary->aData[logSummaryEntry(iFrame)]); + assert( pWal->lockState==SQLITE_SHM_WRITE ); + walIndexReadHdr(pWal, 0); + for(iFrame=pWal->hdr.iLastPg+1; iFrame<=iMax && rc==SQLITE_OK; iFrame++){ + rc = xUndo(pUndoCtx, pWal->pWiData[walIndexEntry(iFrame)]); } + walIndexUnmap(pWal); return rc; } -u32 sqlite3WalSavepoint(Log *pLog){ - assert( pLog->isWriteLocked ); - return pLog->hdr.iLastPg; +/* Return an integer that records the current (uncommitted) write +** position in the WAL +*/ +u32 sqlite3WalSavepoint(Wal *pWal){ + assert( pWal->lockState==SQLITE_SHM_WRITE ); + return pWal->hdr.iLastPg; } -int sqlite3WalSavepointUndo(Log *pLog, u32 iFrame){ +/* Move the write position of the WAL back to iFrame. Called in +** response to a ROLLBACK TO command. +*/ +int sqlite3WalSavepointUndo(Wal *pWal, u32 iFrame){ int rc = SQLITE_OK; u8 aCksum[8]; - assert( pLog->isWriteLocked ); + assert( pWal->lockState==SQLITE_SHM_WRITE ); - pLog->hdr.iLastPg = iFrame; + pWal->hdr.iLastPg = iFrame; if( iFrame>0 ){ - i64 iOffset = logFrameOffset(iFrame, pLog->hdr.pgsz) + sizeof(u32)*2; - rc = sqlite3OsRead(pLog->pFd, aCksum, sizeof(aCksum), iOffset); - pLog->hdr.iCheck1 = sqlite3Get4byte(&aCksum[0]); - pLog->hdr.iCheck2 = sqlite3Get4byte(&aCksum[4]); + i64 iOffset = walFrameOffset(iFrame, pWal->hdr.pgsz) + sizeof(u32)*2; + rc = sqlite3OsRead(pWal->pFd, aCksum, sizeof(aCksum), iOffset); + pWal->hdr.iCheck1 = sqlite3Get4byte(&aCksum[0]); + pWal->hdr.iCheck2 = sqlite3Get4byte(&aCksum[4]); } return rc; @@ -1722,9 +1046,9 @@ int sqlite3WalSavepointUndo(Log *pLog, u32 iFrame){ /* ** Return true if data has been written but not committed to the log file. */ -int sqlite3WalDirty(Log *pLog){ - assert( pLog->isWriteLocked ); - return( pLog->hdr.iLastPg!=((LogSummaryHdr*)pLog->pSummary->aData)->iLastPg ); +int sqlite3WalDirty(Wal *pWal){ + assert( pWal->lockState==SQLITE_SHM_WRITE ); + return( pWal->hdr.iLastPg!=((WalIndexHdr*)pWal->pWiData)->iLastPg ); } /* @@ -1732,7 +1056,7 @@ int sqlite3WalDirty(Log *pLog){ ** on the log file (obtained using sqlite3WalWriteLock()). */ int sqlite3WalFrames( - Log *pLog, /* Log handle to write to */ + Wal *pWal, /* Wal handle to write to */ int nPgsz, /* Database page-size in bytes */ PgHdr *pList, /* List of dirty pages to write */ Pgno nTruncate, /* Database size after this commit */ @@ -1741,52 +1065,53 @@ int sqlite3WalFrames( ){ int rc; /* Used to catch return codes */ u32 iFrame; /* Next frame address */ - u8 aFrame[LOG_FRAME_HDRSIZE]; /* Buffer to assemble frame-header in */ + u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-header in */ PgHdr *p; /* Iterator to run through pList with. */ u32 aCksum[2]; /* Checksums */ PgHdr *pLast; /* Last frame in list */ int nLast = 0; /* Number of extra copies of last page */ - assert( LOG_FRAME_HDRSIZE==(4 * 2 + 2*sizeof(u32)) ); + assert( WAL_FRAME_HDRSIZE==(4 * 2 + 2*sizeof(u32)) ); assert( pList ); + assert( pWal->lockState==SQLITE_SHM_WRITE ); /* If this is the first frame written into the log, write the log ** header to the start of the log file. See comments at the top of ** this file for a description of the log-header format. */ - assert( LOG_FRAME_HDRSIZE>=LOG_HDRSIZE ); - iFrame = pLog->hdr.iLastPg; + assert( WAL_FRAME_HDRSIZE>=WAL_HDRSIZE ); + iFrame = pWal->hdr.iLastPg; if( iFrame==0 ){ sqlite3Put4byte(aFrame, nPgsz); sqlite3_randomness(8, &aFrame[4]); - pLog->hdr.iCheck1 = sqlite3Get4byte(&aFrame[4]); - pLog->hdr.iCheck2 = sqlite3Get4byte(&aFrame[8]); - rc = sqlite3OsWrite(pLog->pFd, aFrame, LOG_HDRSIZE, 0); + pWal->hdr.iCheck1 = sqlite3Get4byte(&aFrame[4]); + pWal->hdr.iCheck2 = sqlite3Get4byte(&aFrame[8]); + rc = sqlite3OsWrite(pWal->pFd, aFrame, WAL_HDRSIZE, 0); if( rc!=SQLITE_OK ){ return rc; } } - aCksum[0] = pLog->hdr.iCheck1; - aCksum[1] = pLog->hdr.iCheck2; + aCksum[0] = pWal->hdr.iCheck1; + aCksum[1] = pWal->hdr.iCheck2; /* Write the log file. */ for(p=pList; p; p=p->pDirty){ u32 nDbsize; /* Db-size field for frame header */ i64 iOffset; /* Write offset in log file */ - iOffset = logFrameOffset(++iFrame, nPgsz); + iOffset = walFrameOffset(++iFrame, nPgsz); /* Populate and write the frame header */ nDbsize = (isCommit && p->pDirty==0) ? nTruncate : 0; - logEncodeFrame(aCksum, p->pgno, nDbsize, nPgsz, p->pData, aFrame); - rc = sqlite3OsWrite(pLog->pFd, aFrame, sizeof(aFrame), iOffset); + walEncodeFrame(aCksum, p->pgno, nDbsize, nPgsz, p->pData, aFrame); + rc = sqlite3OsWrite(pWal->pFd, aFrame, sizeof(aFrame), iOffset); if( rc!=SQLITE_OK ){ return rc; } /* Write the page data */ - rc = sqlite3OsWrite(pLog->pFd, p->pData, nPgsz, iOffset + sizeof(aFrame)); + rc = sqlite3OsWrite(pWal->pFd, p->pData, nPgsz, iOffset + sizeof(aFrame)); if( rc!=SQLITE_OK ){ return rc; } @@ -1795,8 +1120,8 @@ int sqlite3WalFrames( /* Sync the log file if the 'isSync' flag was specified. */ if( sync_flags ){ - i64 iSegment = sqlite3OsSectorSize(pLog->pFd); - i64 iOffset = logFrameOffset(iFrame+1, nPgsz); + i64 iSegment = sqlite3OsSectorSize(pWal->pFd); + i64 iOffset = walFrameOffset(iFrame+1, nPgsz); assert( isCommit ); @@ -1805,14 +1130,14 @@ int sqlite3WalFrames( } iSegment = (((iOffset+iSegment-1)/iSegment) * iSegment); while( iOffsetpgno,nTruncate,nPgsz,pLast->pData,aFrame); - rc = sqlite3OsWrite(pLog->pFd, aFrame, sizeof(aFrame), iOffset); + walEncodeFrame(aCksum,pLast->pgno,nTruncate,nPgsz,pLast->pData,aFrame); + rc = sqlite3OsWrite(pWal->pFd, aFrame, sizeof(aFrame), iOffset); if( rc!=SQLITE_OK ){ return rc; } - iOffset += LOG_FRAME_HDRSIZE; - rc = sqlite3OsWrite(pLog->pFd, pLast->pData, nPgsz, iOffset); + iOffset += WAL_FRAME_HDRSIZE; + rc = sqlite3OsWrite(pWal->pFd, pLast->pData, nPgsz, iOffset); if( rc!=SQLITE_OK ){ return rc; } @@ -1820,44 +1145,44 @@ int sqlite3WalFrames( iOffset += nPgsz; } - rc = sqlite3OsSync(pLog->pFd, sync_flags); + rc = sqlite3OsSync(pWal->pFd, sync_flags); if( rc!=SQLITE_OK ){ return rc; } } /* Append data to the log summary. It is not necessary to lock the - ** log-summary to do this as the RESERVED lock held on the db file + ** wal-index to do this as the RESERVED lock held on the db file ** guarantees that there are no other writers, and no data that may ** be in use by existing readers is being overwritten. */ - iFrame = pLog->hdr.iLastPg; + iFrame = pWal->hdr.iLastPg; for(p=pList; p; p=p->pDirty){ iFrame++; - logSummaryAppend(pLog->pSummary, iFrame, p->pgno); + walIndexAppend(pWal, iFrame, p->pgno); } while( nLast>0 ){ iFrame++; nLast--; - logSummaryAppend(pLog->pSummary, iFrame, pLast->pgno); + walIndexAppend(pWal, iFrame, pLast->pgno); } /* Update the private copy of the header. */ - pLog->hdr.pgsz = nPgsz; - pLog->hdr.iLastPg = iFrame; + pWal->hdr.pgsz = nPgsz; + pWal->hdr.iLastPg = iFrame; if( isCommit ){ - pLog->hdr.iChange++; - pLog->hdr.nPage = nTruncate; + pWal->hdr.iChange++; + pWal->hdr.nPage = nTruncate; } - pLog->hdr.iCheck1 = aCksum[0]; - pLog->hdr.iCheck2 = aCksum[1]; + pWal->hdr.iCheck1 = aCksum[0]; + pWal->hdr.iCheck2 = aCksum[1]; - /* If this is a commit, update the log-summary header too. */ - if( isCommit && SQLITE_OK==(rc = logEnterMutex(pLog)) ){ - logSummaryWriteHdr(pLog->pSummary, &pLog->hdr); - logLeaveMutex(pLog); - pLog->iCallback = iFrame; + /* If this is a commit, update the wal-index header too. */ + if( isCommit ){ + walIndexWriteHdr(pWal, &pWal->hdr); + pWal->iCallback = iFrame; } + walIndexUnmap(pWal); return rc; } @@ -1865,14 +1190,13 @@ int sqlite3WalFrames( /* ** Checkpoint the database: ** -** 1. Wait for an EXCLUSIVE lock on regions B and C. -** 2. Wait for an EXCLUSIVE lock on region A. -** 3. Copy the contents of the log into the database file. -** 4. Zero the log-summary header (so new readers will ignore the log). -** 5. Drop the locks obtained in steps 1 and 2. +** 1. Acquire a CHECKPOINT lock +** 2. Copy the contents of the log into the database file. +** 3. Zero the wal-index header (so new readers will ignore the log). +** 4. Drop the CHECKPOINT lock. */ int sqlite3WalCheckpoint( - Log *pLog, /* Log connection */ + Wal *pWal, /* Wal connection */ sqlite3_file *pFd, /* File descriptor open on db file */ int sync_flags, /* Flags to sync db file with (or 0) */ u8 *zBuf, /* Temporary buffer to use */ @@ -1882,48 +1206,47 @@ int sqlite3WalCheckpoint( int rc; /* Return code */ int isChanged = 0; /* True if a new wal-index header is loaded */ - assert( !pLog->isLocked ); + assert( pWal->lockState==SQLITE_SHM_UNLOCK ); - /* Wait for an EXCLUSIVE lock on regions B and C. */ + /* Get the CHECKPOINT lock */ do { - rc = logLockRegion(pLog, LOG_REGION_B|LOG_REGION_C, LOG_WRLOCK); - }while( rc==SQLITE_BUSY && xBusyHandler(pBusyHandlerArg) ); - if( rc!=SQLITE_OK ) return rc; - - /* Wait for an EXCLUSIVE lock on region A. */ - do { - rc = logLockRegion(pLog, LOG_REGION_A, LOG_WRLOCK); + rc = walSetLock(pWal, SQLITE_SHM_CHECKPOINT); }while( rc==SQLITE_BUSY && xBusyHandler(pBusyHandlerArg) ); if( rc!=SQLITE_OK ){ - logLockRegion(pLog, LOG_REGION_B|LOG_REGION_C, LOG_UNLOCK); + walSetLock(pWal, SQLITE_SHM_UNLOCK); return rc; } /* Copy data from the log to the database file. */ - rc = logSummaryReadHdr(pLog, &isChanged); + rc = walIndexReadHdr(pWal, &isChanged); if( rc==SQLITE_OK ){ - rc = logCheckpoint(pLog, pFd, sync_flags, zBuf); + rc = walCheckpoint(pWal, pFd, sync_flags, zBuf); } if( isChanged ){ /* If a new wal-index header was loaded before the checkpoint was - ** performed, then the pager-cache associated with log pLog is now + ** performed, then the pager-cache associated with log pWal is now ** out of date. So zero the cached wal-index header to ensure that ** next time the pager opens a snapshot on this database it knows that ** the cache needs to be reset. */ - memset(&pLog->hdr, 0, sizeof(LogSummaryHdr)); + memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); } /* Release the locks. */ - logLockRegion(pLog, LOG_REGION_A|LOG_REGION_B|LOG_REGION_C, LOG_UNLOCK); + walSetLock(pWal, SQLITE_SHM_UNLOCK); return rc; } -int sqlite3WalCallback(Log *pLog){ +/* Return the value to pass to a sqlite3_wal_hook callback, the +** number of frames in the WAL at the point of the last commit since +** sqlite3WalCallback() was called. If no commits have occurred since +** the last call, then return 0. +*/ +int sqlite3WalCallback(Wal *pWal){ u32 ret = 0; - if( pLog ){ - ret = pLog->iCallback; - pLog->iCallback = 0; + if( pWal ){ + ret = pWal->iCallback; + pWal->iCallback = 0; } return (int)ret; } diff --git a/src/wal.h b/src/wal.h index f04344c63a..4bab929f01 100644 --- a/src/wal.h +++ b/src/wal.h @@ -22,11 +22,11 @@ /* Connection to a write-ahead log (WAL) file. ** There is one object of this type for each pager. */ -typedef struct Log Log; +typedef struct Wal Wal; /* Open and close a connection to a write-ahead log. */ -int sqlite3WalOpen(sqlite3_vfs*, const char *zDb, Log **ppLog); -int sqlite3WalClose(Log *pLog, sqlite3_file *pFd, int sync_flags, u8 *zBuf); +int sqlite3WalOpen(sqlite3_vfs*, const char *zDb, Wal **ppWal); +int sqlite3WalClose(Wal *pWal, sqlite3_file *pFd, int sync_flags, u8 *zBuf); /* Used by readers to open (lock) and close (unlock) a snapshot. A ** snapshot is like a read-transaction. It is the state of the database @@ -35,39 +35,39 @@ int sqlite3WalClose(Log *pLog, sqlite3_file *pFd, int sync_flags, u8 *zBuf); ** write to or checkpoint the WAL. sqlite3WalCloseSnapshot() closes the ** transaction and releases the lock. */ -int sqlite3WalOpenSnapshot(Log *pLog, int *); -void sqlite3WalCloseSnapshot(Log *pLog); +int sqlite3WalOpenSnapshot(Wal *pWal, int *); +void sqlite3WalCloseSnapshot(Wal *pWal); /* Read a page from the write-ahead log, if it is present. */ -int sqlite3WalRead(Log *pLog, Pgno pgno, int *pInLog, u8 *pOut); +int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, u8 *pOut); /* Return the size of the database as it existed at the beginning ** of the snapshot */ -void sqlite3WalDbsize(Log *pLog, Pgno *pPgno); +void sqlite3WalDbsize(Wal *pWal, Pgno *pPgno); /* Obtain or release the WRITER lock. */ -int sqlite3WalWriteLock(Log *pLog, int op); +int sqlite3WalWriteLock(Wal *pWal, int op); /* Undo any frames written (but not committed) to the log */ -int sqlite3WalUndo(Log *pLog, int (*xUndo)(void *, Pgno), void *pUndoCtx); +int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx); /* Return an integer that records the current (uncommitted) write ** position in the WAL */ -u32 sqlite3WalSavepoint(Log *pLog); +u32 sqlite3WalSavepoint(Wal *pWal); /* Move the write position of the WAL back to iFrame. Called in ** response to a ROLLBACK TO command. */ -int sqlite3WalSavepointUndo(Log *pLog, u32 iFrame); +int sqlite3WalSavepointUndo(Wal *pWal, u32 iFrame); /* Return true if data has been written but not committed to the log file. */ -int sqlite3WalDirty(Log *pLog); +int sqlite3WalDirty(Wal *pWal); /* Write a frame or frames to the log. */ -int sqlite3WalFrames(Log *pLog, int, PgHdr *, Pgno, int, int); +int sqlite3WalFrames(Wal *pWal, int, PgHdr *, Pgno, int, int); /* Copy pages from the log to the database file */ int sqlite3WalCheckpoint( - Log *pLog, /* Log connection */ + Wal *pWal, /* Write-ahead log connection */ sqlite3_file *pFd, /* File descriptor open on db file */ int sync_flags, /* Flags to sync db file with (or 0) */ u8 *zBuf, /* Temporary buffer to use */ @@ -80,6 +80,6 @@ int sqlite3WalCheckpoint( ** sqlite3WalCallback() was called. If no commits have occurred since ** the last call, then return 0. */ -int sqlite3WalCallback(Log *pLog); +int sqlite3WalCallback(Wal *pWal); #endif /* _WAL_H_ */ From 79e6c78ccce39af048334f7fb0ad12bdab82d3af Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 30 Apr 2010 02:13:26 +0000 Subject: [PATCH 56/89] The first 6 WAL tests now work. It's a start. FossilOrigin-Name: a92c1851da10acf51e7f6f086b8a23bd731940b3 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/os_unix.c | 13 ++++++------- src/wal.c | 38 +++++++++++++++++++++++--------------- 4 files changed, 40 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 4e258ea0da..d3df686025 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 -C Refactor\swal.c\sto\suse\sthe\sVFS.\s\sThis\scheck-in\scompiles\sand\slinks\sand\sworks\nok\sas\slong\sas\syou\sleave\sWAL\sturned\soff,\sbut\sWAL\sdoes\snot\swork. -D 2010-04-29T22:34:08 +C The\sfirst\s6\sWAL\stests\snow\swork.\s\sIt's\sa\sstart. +D 2010-04-30T02:13:27 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -153,7 +153,7 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 -F src/os_unix.c bf4cc543fcb32dbcb25a88cf817c761ceef14fdd +F src/os_unix.c d9b2b017216075bba93cd4cb38f23396d3d02eb1 F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 F src/pager.c 07c3522319a449db9dc732da828a09c4c55c41a4 F src/pager.h 934b598583a9d936bb13c37d62a2fe68ac48781c @@ -224,7 +224,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c 1c984ba3bba07a9efb4974837d12daef8f0d80c4 +F src/wal.c d9d76a8189a9ac670cd2d5d5dd0c5019194727d7 F src/wal.h d0a7048e68a86d3fe4639858d7f592606545d224 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 @@ -811,14 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 706611283ea2575c2942543391026b36061cfc1c -R d5f4a8af6d2162b2c802208b97207bbd +P 62db5fa3b61be885b2d94e9b9ce3877b2c588350 +R aef0120cea29fe330250fe68f6819085 U drh -Z eca35982e551d3fde58c618a2bb3d812 +Z 4fcebc28f74cfe3c8cf3e566a4608163 -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.6 (GNU/Linux) -iD8DBQFL2glkoxKgR168RlERAm4BAJ9/6BoUJM0bQ0jccd1czYE2rrEefgCfYnyN -XCr+jA+ZIb6AheCdZkTnnE4= -=Dx/c +iD8DBQFL2jzNoxKgR168RlERAgWeAJ95gf+z42nfTDpmaeXk9ZGzWPlbxwCdFTJp ++xaXsShNpvEvsgzl2lnGlxE= +=76cZ -----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index 78f7afdb38..da28f90e44 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -62db5fa3b61be885b2d94e9b9ce3877b2c588350 \ No newline at end of file +a92c1851da10acf51e7f6f086b8a23bd731940b3 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 0e525c0732..8480b5f447 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4710,7 +4710,7 @@ static int unixShmSystemLock( u8 mask; /* Mask of bits in lockMask */ /* Access to the unixShmFile object is serialized by the caller */ - assert( sqlite3_mutex_held(pFile->mutex) ); + assert( sqlite3_mutex_held(pFile->mutex) || pFile->nRef==0 ); /* Initialize the locking parameters */ memset(&f, 0, sizeof(f)); @@ -4732,13 +4732,15 @@ static int unixShmSystemLock( mask <<= 1; while( mask!=0 && (lockMask & mask)!=0 ){ f.l_len++; + mask <<= 1; } /* Verify that all bits set in lockMask are contiguous */ assert( mask==0 || (lockMask & ~(mask | (mask-1)))==0 ); /* Acquire the system-level lock */ - rc = (fcntl(pFile->h, lockOp, &f)==0) ? SQLITE_OK : SQLITE_BUSY; + rc = fcntl(pFile->h, lockOp, &f); + rc = (rc!=(-1)) ? SQLITE_OK : SQLITE_BUSY; /* Update the global lock state and do debug tracing */ #ifdef SQLITE_DEBUG @@ -4793,9 +4795,6 @@ static int unixShmUnlock( /* Access to the unixShmFile object is serialized by the caller */ assert( sqlite3_mutex_held(pFile->mutex) ); - /* We never try to unlock locks that we do not hold */ - assert( ((p->exclMask|p->sharedMask) & unlockMask)==unlockMask ); - /* Compute locks held by sibling connections */ for(pX=pFile->pFirst; pX; pX=pX->pNext){ if( pX==p ) continue; @@ -4960,7 +4959,7 @@ static int unixShmOpen( rc = SQLITE_NOMEM; goto shm_open_err; } - memset(pFile, 0, sizeof(pFile)); + memset(pFile, 0, sizeof(*pFile)); pFile->zFilename = (char*)&pFile[1]; memcpy(pFile->zFilename, zName, nName+1); pFile->h = -1; @@ -4983,7 +4982,7 @@ static int unixShmOpen( goto shm_open_err; } - pFile->h = open(zName, O_CREAT, 0664); + pFile->h = open(zName, O_RDWR|O_CREAT, 0664); if( pFile->h<0 ){ rc = SQLITE_CANTOPEN; goto shm_open_err; diff --git a/src/wal.c b/src/wal.c index 2de53b8b18..0e44e561f3 100644 --- a/src/wal.c +++ b/src/wal.c @@ -378,25 +378,26 @@ static void walIndexUnmap(Wal *pWal){ } /* -** Map the wal-index file into memory if it isn't already. -*/ -static int walIndexMap(Wal *pWal){ - int rc = SQLITE_OK; - if( pWal->pWiData==0 ){ - rc = pWal->pVfs->xShmSize(pWal->pWIndex, -1, - &pWal->szWIndex, (void**)(char*)&pWal->pWiData); - } - return rc; -} - -/* -** Resize the wal-index file. +** Resize the wal-index file. If newSize is negative, leave the size +** unchanged. */ static int walIndexRemap(Wal *pWal, int newSize){ int rc; walIndexUnmap(pWal); rc = pWal->pVfs->xShmSize(pWal->pWIndex, newSize, &pWal->szWIndex, (void**)(char*)&pWal->pWiData); + if( rc==SQLITE_OK && pWal->pWiData==0 ){ + assert( pWal->szWIndex==0 ); + pWal->pWiData = &pWal->iCallback; + } + return rc; +} + +/* +** Map the wal-index file into memory if it isn't already. +*/ +static int walIndexMap(Wal *pWal){ + int rc = walIndexRemap(pWal, -1); return rc; } @@ -576,12 +577,12 @@ int sqlite3WalOpen( pRet->pVfs = pVfs; pRet->pFd = (sqlite3_file *)&pRet[1]; zWal = pVfs->szOsFile + (char*)pRet->pFd; - sqlite3_snprintf(nWal, zWal, "%s-wal-index", zDb); + sqlite3_snprintf(nWal+11, zWal, "%s-wal-index", zDb); rc = pVfs->xShmOpen(pVfs, zWal, &pRet->pWIndex); if( rc ) goto wal_open_out; /* Open file handle on the write-ahead log file. */ - zWal[nWal-6] = 0; + zWal[nWal+4] = 0; flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_MAIN_JOURNAL); rc = sqlite3OsOpen(pVfs, zWal, pRet->pFd, flags, &flags); @@ -787,11 +788,18 @@ int walIndexTryHdr(Wal *pWal, int *pChanged){ u32 aCksum[2] = {1, 1}; u32 aHdr[WALINDEX_HDR_NFIELD+2]; + if( pWal->szWIndex==0 ){ + int rc = walIndexRemap(pWal, WALINDEX_MMAP_INCREMENT); + if( rc ) return rc; + } + /* Read the header. The caller may or may not have locked the wal-index ** file, meaning it is possible that an inconsistent snapshot is read ** from the file. If this happens, return SQLITE_ERROR. The caller will ** retry. Or, if the caller has already locked the file and the header ** still looks inconsistent, it will run recovery. + ** + ** FIX-ME: It is no longer possible to have not locked the wal-index. */ memcpy(aHdr, pWal->pWiData, sizeof(aHdr)); walChecksumBytes((u8*)aHdr, sizeof(u32)*WALINDEX_HDR_NFIELD, aCksum); From ba51590bcd443671d09f9320be2a218e58e26232 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 30 Apr 2010 09:32:06 +0000 Subject: [PATCH 57/89] Add a missing walIndexUnmap() to sqlite3WalSnapshotOpen(). FossilOrigin-Name: 72b95fde15dae56390dc8d4168fb2757447e289e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 2 +- src/wal.c | 4 ++++ 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index f1b485ae53..552a63398f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\sbugfix\sinto\swal\sbranch. -D 2010-04-30T06:02:31 +C Add\sa\smissing\swalIndexUnmap()\sto\ssqlite3WalSnapshotOpen(). +D 2010-04-30T09:32:06 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -150,7 +150,7 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 -F src/os_unix.c d9b2b017216075bba93cd4cb38f23396d3d02eb1 +F src/os_unix.c 4237d2a6429307307ebd1e7c1e3ccec875468907 F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 F src/pager.c 07c3522319a449db9dc732da828a09c4c55c41a4 F src/pager.h 934b598583a9d936bb13c37d62a2fe68ac48781c @@ -221,7 +221,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c d9d76a8189a9ac670cd2d5d5dd0c5019194727d7 +F src/wal.c b617892b95ac3f82e657e3c047ad6883111f6b8c F src/wal.h d0a7048e68a86d3fe4639858d7f592606545d224 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P a92c1851da10acf51e7f6f086b8a23bd731940b3 f660be615a0c1f4641782a83744533f5d3119218 -R 789557e2ecf4320c154527de09c21ada +P 265e66e6984c32999bf93981e3b1d19887dfe599 +R 7b9c465ab8c7541a3a7b291b0b84a4e5 U dan -Z b95a42f00187a9dd3e8e01985f34fa1c +Z 9561c62faf9368c39d4969e72108b9f2 diff --git a/manifest.uuid b/manifest.uuid index 85d7bc1b3c..8d2667d652 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -265e66e6984c32999bf93981e3b1d19887dfe599 \ No newline at end of file +72b95fde15dae56390dc8d4168fb2757447e289e \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 8480b5f447..fee07cb27e 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5083,7 +5083,7 @@ static int unixShmClose(sqlite3_shm *pSharedMem){ ** written into pNewSize. A writer lock is held on the shared memory ** segment while resizing it. ** -** If ppBuffer is not NULL, the a reader lock is acquired no the shared +** If ppBuffer is not NULL, the a reader lock is acquired on the shared ** memory segment and *ppBuffer is made to point to the start of the ** shared memory segment. xShmRelease() must be called to release the ** lock. diff --git a/src/wal.c b/src/wal.c index 0e44e561f3..9d518c16a0 100644 --- a/src/wal.c +++ b/src/wal.c @@ -889,6 +889,8 @@ int sqlite3WalOpenSnapshot(Wal *pWal, int *pChanged){ } } } + + walIndexUnmap(pWal); return rc; } @@ -1082,6 +1084,7 @@ int sqlite3WalFrames( assert( WAL_FRAME_HDRSIZE==(4 * 2 + 2*sizeof(u32)) ); assert( pList ); assert( pWal->lockState==SQLITE_SHM_WRITE ); + assert( pWal->pWiData==0 ); /* If this is the first frame written into the log, write the log ** header to the start of the log file. See comments at the top of @@ -1158,6 +1161,7 @@ int sqlite3WalFrames( return rc; } } + assert( pWal->pWiData==0 ); /* Append data to the log summary. It is not necessary to lock the ** wal-index to do this as the RESERVED lock held on the db file From 4c97b53425204533345bb58185157d2e6e3eded5 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 30 Apr 2010 09:52:17 +0000 Subject: [PATCH 58/89] Fix a couple of assert() statements in os_unix.c and wal.c. Combine sqlite3WalIsDirty() with sqlite3WalUndo(). FossilOrigin-Name: a8f958be804ee05c4137b3cd110db344713af5f2 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/os_unix.c | 2 +- src/pager.c | 22 ++++++++++------------ src/wal.c | 10 +--------- src/wal.h | 3 --- 6 files changed, 22 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index 552a63398f..6271923420 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\smissing\swalIndexUnmap()\sto\ssqlite3WalSnapshotOpen(). -D 2010-04-30T09:32:06 +C Fix\sa\scouple\sof\sassert()\sstatements\sin\sos_unix.c\sand\swal.c.\sCombine\ssqlite3WalIsDirty()\swith\ssqlite3WalUndo(). +D 2010-04-30T09:52:18 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -150,9 +150,9 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 -F src/os_unix.c 4237d2a6429307307ebd1e7c1e3ccec875468907 +F src/os_unix.c 685b0347383b5d12dee9961b3cc0337ff075617b F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 -F src/pager.c 07c3522319a449db9dc732da828a09c4c55c41a4 +F src/pager.c 434f9751fc2dfc11ade004282deda5f8560bcba2 F src/pager.h 934b598583a9d936bb13c37d62a2fe68ac48781c F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf @@ -221,8 +221,8 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c b617892b95ac3f82e657e3c047ad6883111f6b8c -F src/wal.h d0a7048e68a86d3fe4639858d7f592606545d224 +F src/wal.c 6f0fdfac56f3148b7ff2fcd60f12c2a7bdb5e0de +F src/wal.h c3f347ba8f1cde46d9bcc6fedaf3ed0aa4b53294 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 265e66e6984c32999bf93981e3b1d19887dfe599 -R 7b9c465ab8c7541a3a7b291b0b84a4e5 +P 72b95fde15dae56390dc8d4168fb2757447e289e +R 8864259ed44807eea46d7c4986422a72 U dan -Z 9561c62faf9368c39d4969e72108b9f2 +Z 967d048f88e642230f471eb242428feb diff --git a/manifest.uuid b/manifest.uuid index 8d2667d652..dd868c1211 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -72b95fde15dae56390dc8d4168fb2757447e289e \ No newline at end of file +a8f958be804ee05c4137b3cd110db344713af5f2 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index fee07cb27e..5962d68246 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4798,7 +4798,7 @@ static int unixShmUnlock( /* Compute locks held by sibling connections */ for(pX=pFile->pFirst; pX; pX=pX->pNext){ if( pX==p ) continue; - assert( (pX->exclMask & unlockMask)==0 ); + assert( (pX->exclMask & (p->exclMask|p->sharedMask))==0 ); allMask |= pX->sharedMask; } diff --git a/src/pager.c b/src/pager.c index 0ad245a8c4..ee0c575a59 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2272,6 +2272,16 @@ static int pagerUndoCallback(void *pCtx, Pgno iPg){ } } + /* Normally, if a transaction is rolled back, any backup processes are + ** updated as data is copied out of the rollback journal and into the + ** database. This is not generally possible with a WAL database, as + ** rollback involves simply truncating the log file. Therefore, if one + ** or more frames have already been written to the log (and therefore + ** also copied into the backup databases) as part of this transaction, + ** the backups must be restarted. + */ + sqlite3BackupRestart(pPager->pBackup); + return rc; } @@ -2282,18 +2292,6 @@ static int pagerRollbackWal(Pager *pPager){ int rc; /* Return Code */ PgHdr *pList; /* List of dirty pages to revert */ - /* Normally, if a transaction is rolled back, any backup processes are - ** updated as data is copied out of the rollback journal and into the - ** database. This is not generally possible with a WAL database, as - ** rollback involves simply truncating the log file. Therefore, if one - ** or more frames have already been written to the log (and therefore - ** also copied into the backup databases) as part of this transaction, - ** the backups must be restarted. - */ - if( sqlite3WalDirty(pPager->pWal) ){ - sqlite3BackupRestart(pPager->pBackup); - } - /* For all pages in the cache that are currently dirty or have already ** been written (but not committed) to the log file, do one of the ** following: diff --git a/src/wal.c b/src/wal.c index 9d518c16a0..fa29cb58c7 100644 --- a/src/wal.c +++ b/src/wal.c @@ -829,7 +829,7 @@ int walIndexTryHdr(Wal *pWal, int *pChanged){ static int walIndexReadHdr(Wal *pWal, int *pChanged){ int rc; - assert( pWal->lockState==SQLITE_SHM_READ ); + assert( pWal->lockState>=SQLITE_SHM_READ ); walIndexMap(pWal); /* First try to read the header without a lock. Verify the checksum @@ -1053,14 +1053,6 @@ int sqlite3WalSavepointUndo(Wal *pWal, u32 iFrame){ return rc; } -/* -** Return true if data has been written but not committed to the log file. -*/ -int sqlite3WalDirty(Wal *pWal){ - assert( pWal->lockState==SQLITE_SHM_WRITE ); - return( pWal->hdr.iLastPg!=((WalIndexHdr*)pWal->pWiData)->iLastPg ); -} - /* ** Write a set of frames to the log. The caller must hold the write-lock ** on the log file (obtained using sqlite3WalWriteLock()). diff --git a/src/wal.h b/src/wal.h index 4bab929f01..1c787d8dcd 100644 --- a/src/wal.h +++ b/src/wal.h @@ -59,9 +59,6 @@ u32 sqlite3WalSavepoint(Wal *pWal); ** response to a ROLLBACK TO command. */ int sqlite3WalSavepointUndo(Wal *pWal, u32 iFrame); -/* Return true if data has been written but not committed to the log file. */ -int sqlite3WalDirty(Wal *pWal); - /* Write a frame or frames to the log. */ int sqlite3WalFrames(Wal *pWal, int, PgHdr *, Pgno, int, int); From e93808dde24b8b53b9b2f954f1bf8d2df956b36e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 30 Apr 2010 10:06:09 +0000 Subject: [PATCH 59/89] Add missing mutexes to unixShmClose(). FossilOrigin-Name: a4741cb54dd5e753d48fd05ac9dbe27ee4aa1ec0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 11 ++++++----- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 6271923420..b94ef2bc5a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scouple\sof\sassert()\sstatements\sin\sos_unix.c\sand\swal.c.\sCombine\ssqlite3WalIsDirty()\swith\ssqlite3WalUndo(). -D 2010-04-30T09:52:18 +C Add\smissing\smutexes\sto\sunixShmClose(). +D 2010-04-30T10:06:10 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -150,7 +150,7 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 -F src/os_unix.c 685b0347383b5d12dee9961b3cc0337ff075617b +F src/os_unix.c f5e41bd2f563f85c10a9c9e62bbac8e3b783efc3 F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 F src/pager.c 434f9751fc2dfc11ade004282deda5f8560bcba2 F src/pager.h 934b598583a9d936bb13c37d62a2fe68ac48781c @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 72b95fde15dae56390dc8d4168fb2757447e289e -R 8864259ed44807eea46d7c4986422a72 +P a8f958be804ee05c4137b3cd110db344713af5f2 +R 5f2e996a1ab6f83c7e08f128286e26ff U dan -Z 967d048f88e642230f471eb242428feb +Z 8a044884de878c7767a8936662f5ee57 diff --git a/manifest.uuid b/manifest.uuid index dd868c1211..8fd1613991 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a8f958be804ee05c4137b3cd110db344713af5f2 \ No newline at end of file +a4741cb54dd5e753d48fd05ac9dbe27ee4aa1ec0 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 5962d68246..decf2efc8a 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5046,7 +5046,6 @@ static int unixShmClose(sqlite3_shm *pSharedMem){ unixShm *p; /* The connection to be closed */ unixShmFile *pFile; /* The underlying shared-memory file */ unixShm **pp; /* For looping over sibling connections */ - int nRef; /* Number of connections to pFile */ if( pSharedMem==0 ) return SQLITE_OK; p = (struct unixShm*)pSharedMem; @@ -5056,13 +5055,10 @@ static int unixShmClose(sqlite3_shm *pSharedMem){ assert( p->exclMask==0 ); assert( p->sharedMask==0 ); - /* Remove connection p from the set of connections associated with pFile */ sqlite3_mutex_enter(pFile->mutex); for(pp=&pFile->pFirst; (*pp)!=p; pp = &(*pp)->pNext){} *pp = p->pNext; - pFile->nRef--; - nRef = pFile->nRef; /* Free the connection p */ sqlite3_free(p); @@ -5070,9 +5066,14 @@ static int unixShmClose(sqlite3_shm *pSharedMem){ /* If pFile->nRef has reached 0, then close the underlying ** shared-memory file, too */ - if( nRef==0 ){ + unixEnterMutex(); + assert( pFile->nRef>0 ); + pFile->nRef--; + if( pFile->nRef==0 ){ unixShmPurge(); } + unixLeaveMutex(); + return SQLITE_OK; } From 1bc6171c231ef92411b24dcac4f6833b53e31565 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 30 Apr 2010 10:24:54 +0000 Subject: [PATCH 60/89] Fix an assert() in sqlite3WalRead(). FossilOrigin-Name: 2e7a0050e192dd197d4db92393ab120897c14836 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index b94ef2bc5a..ed5ff95cce 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smissing\smutexes\sto\sunixShmClose(). -D 2010-04-30T10:06:10 +C Fix\san\sassert()\sin\ssqlite3WalRead(). +D 2010-04-30T10:24:54 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -221,7 +221,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c 6f0fdfac56f3148b7ff2fcd60f12c2a7bdb5e0de +F src/wal.c 153dc952b5a759dcd6200562d227c56f8e18974f F src/wal.h c3f347ba8f1cde46d9bcc6fedaf3ed0aa4b53294 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P a8f958be804ee05c4137b3cd110db344713af5f2 -R 5f2e996a1ab6f83c7e08f128286e26ff +P a4741cb54dd5e753d48fd05ac9dbe27ee4aa1ec0 +R 67a224d9b8436f4afd79983ba9c6427a U dan -Z 8a044884de878c7767a8936662f5ee57 +Z 3903311d96c904ad2185b988a7ff9441 diff --git a/manifest.uuid b/manifest.uuid index 8fd1613991..de04fd111d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4741cb54dd5e753d48fd05ac9dbe27ee4aa1ec0 \ No newline at end of file +2e7a0050e192dd197d4db92393ab120897c14836 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index fa29cb58c7..38bbe458cf 100644 --- a/src/wal.c +++ b/src/wal.c @@ -912,7 +912,7 @@ int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, u8 *pOut){ u32 *aData; int iFrame = (pWal->hdr.iLastPg & 0xFFFFFF00); - assert( pWal->lockState==SQLITE_SHM_READ ); + assert( pWal->lockState==SQLITE_SHM_READ||pWal->lockState==SQLITE_SHM_WRITE ); walIndexMap(pWal); /* Do a linear search of the unindexed block of page-numbers (if any) From 87bfb51ef884c59347911133838fe09ffece45d6 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 30 Apr 2010 11:43:28 +0000 Subject: [PATCH 61/89] Add a missing walIndexUnmap() call to the checkpoint code. Change a couple of SQLITE_CANTOPEN constants to SQLITE_CANTOPEN_BKPT. FossilOrigin-Name: 1f9e8c5c27ae2c68bc01d8e2bb2c662be7b3f161 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 4 ++-- src/wal.c | 3 ++- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index ed5ff95cce..2b907e980d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sassert()\sin\ssqlite3WalRead(). -D 2010-04-30T10:24:54 +C Add\sa\smissing\swalIndexUnmap()\scall\sto\sthe\scheckpoint\scode.\sChange\sa\scouple\sof\sSQLITE_CANTOPEN\sconstants\sto\sSQLITE_CANTOPEN_BKPT. +D 2010-04-30T11:43:29 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -150,7 +150,7 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 -F src/os_unix.c f5e41bd2f563f85c10a9c9e62bbac8e3b783efc3 +F src/os_unix.c 7451a717574c9b9809ad7c429345482c29239230 F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 F src/pager.c 434f9751fc2dfc11ade004282deda5f8560bcba2 F src/pager.h 934b598583a9d936bb13c37d62a2fe68ac48781c @@ -221,7 +221,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c 153dc952b5a759dcd6200562d227c56f8e18974f +F src/wal.c 9327e0d4e2b19dcf5614d3bb8cdff25ebbe0e290 F src/wal.h c3f347ba8f1cde46d9bcc6fedaf3ed0aa4b53294 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P a4741cb54dd5e753d48fd05ac9dbe27ee4aa1ec0 -R 67a224d9b8436f4afd79983ba9c6427a +P 2e7a0050e192dd197d4db92393ab120897c14836 +R c2c9e5e0c48b0291d2627d6aeb69855a U dan -Z 3903311d96c904ad2185b988a7ff9441 +Z e7fa4c17c25ce2a31997877294b65d50 diff --git a/manifest.uuid b/manifest.uuid index de04fd111d..4ba7670f10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2e7a0050e192dd197d4db92393ab120897c14836 \ No newline at end of file +1f9e8c5c27ae2c68bc01d8e2bb2c662be7b3f161 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index decf2efc8a..d991ec9d5d 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4984,13 +4984,13 @@ static int unixShmOpen( pFile->h = open(zName, O_RDWR|O_CREAT, 0664); if( pFile->h<0 ){ - rc = SQLITE_CANTOPEN; + rc = SQLITE_CANTOPEN_BKPT; goto shm_open_err; } rc = fstat(pFile->h, &sStat); if( rc ){ - rc = SQLITE_CANTOPEN; + rc = SQLITE_CANTOPEN_BKPT; goto shm_open_err; } pFile->fid.dev = sStat.st_dev; diff --git a/src/wal.c b/src/wal.c index 38bbe458cf..30af69ff6e 100644 --- a/src/wal.c +++ b/src/wal.c @@ -567,7 +567,7 @@ int sqlite3WalOpen( int nWal; /* Length of zWal in bytes */ assert( zDb ); - if( pVfs->xShmOpen==0 ) return SQLITE_CANTOPEN; + if( pVfs->xShmOpen==0 ) return SQLITE_CANTOPEN_BKPT; /* Allocate an instance of struct Wal to return. */ *ppWal = 0; @@ -1237,6 +1237,7 @@ int sqlite3WalCheckpoint( } /* Release the locks. */ + walIndexUnmap(pWal); walSetLock(pWal, SQLITE_SHM_UNLOCK); return rc; } From 5530b764a83f174000cb720b4edf3235c6d989fc Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 30 Apr 2010 14:39:50 +0000 Subject: [PATCH 62/89] Separate the concepts of underlying storage size and mapped size in the VFS shared-memory implementation. FossilOrigin-Name: 4cbe49f13fed288f94ff305bcfd99df907bf7baf --- manifest | 28 ++++++++----- manifest.uuid | 2 +- src/os_unix.c | 102 ++++++++++++++++++++++++++++++++---------------- src/sqlite.h.in | 3 +- src/wal.c | 69 ++++++++++++++++++++------------ 5 files changed, 134 insertions(+), 70 deletions(-) diff --git a/manifest b/manifest index 2b907e980d..63d18faa80 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,8 @@ -C Add\sa\smissing\swalIndexUnmap()\scall\sto\sthe\scheckpoint\scode.\sChange\sa\scouple\sof\sSQLITE_CANTOPEN\sconstants\sto\sSQLITE_CANTOPEN_BKPT. -D 2010-04-30T11:43:29 +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 + +C Separate\sthe\sconcepts\sof\sunderlying\sstorage\ssize\sand\smapped\ssize\sin\sthe\nVFS\sshared-memory\simplementation. +D 2010-04-30T14:39:51 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -150,7 +153,7 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 -F src/os_unix.c 7451a717574c9b9809ad7c429345482c29239230 +F src/os_unix.c b2322d8fb389b4bc7834b1596bd8afac776a2f20 F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 F src/pager.c 434f9751fc2dfc11ade004282deda5f8560bcba2 F src/pager.h 934b598583a9d936bb13c37d62a2fe68ac48781c @@ -166,7 +169,7 @@ F src/resolve.c ac5f1a713cd1ae77f08b83cc69581e11bf5ae6f9 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c c03d8a0565febcde8c6a12c5d77d065fddae889b F src/shell.c c40427c7245535a04a9cb4a417b6cc05c022e6a4 -F src/sqlite.h.in a710846e91159742ee3ddcbabb56eb7c7478e3a5 +F src/sqlite.h.in 6e91727c0c3f9e1365e8fea2b07369a09830f47f F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89 F src/sqliteInt.h 32dfe6294b46914fb567db493994bf902fb0f8c9 F src/sqliteLimit.h 3afab2291762b5d09ae20c18feb8e9fa935a60a6 @@ -221,7 +224,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c 9327e0d4e2b19dcf5614d3bb8cdff25ebbe0e290 +F src/wal.c a4c643582fd9956b46dca187161d466f584b47af F src/wal.h c3f347ba8f1cde46d9bcc6fedaf3ed0aa4b53294 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 @@ -808,7 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 2e7a0050e192dd197d4db92393ab120897c14836 -R c2c9e5e0c48b0291d2627d6aeb69855a -U dan -Z e7fa4c17c25ce2a31997877294b65d50 +P 1f9e8c5c27ae2c68bc01d8e2bb2c662be7b3f161 +R 2fbe76e11b4a2b6e620f52387007559a +U drh +Z 868ea682fbe4fab342b578dc69caee7c +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.6 (GNU/Linux) + +iD8DBQFL2uu6oxKgR168RlERAqI3AJ4jq6FSOAaisShm+qyJakBJL+pVmwCgg1Vd +tQBOoEYMwVK9lQZSibwuRzM= +=5suj +-----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index 4ba7670f10..0313645415 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1f9e8c5c27ae2c68bc01d8e2bb2c662be7b3f161 \ No newline at end of file +4cbe49f13fed288f94ff305bcfd99df907bf7baf \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index d991ec9d5d..006b0d182d 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4594,11 +4594,9 @@ struct unixShmFile { sqlite3_mutex *mutexBuf; /* Mutex to access zBuf[] */ sqlite3_mutex *mutexRecov; /* The RECOVER mutex */ char *zFilename; /* Name of the file */ - int size; /* Size of the file */ int h; /* Open file descriptor */ - char *pMMapBuf; /* Where currently mmapped() */ - int nReadPrefix; /* Number of SQLITE_SHM_READ_PREFIX locks */ - int nReadFull; /* Number of SQLITE_SHM_READ_FULL locks */ + int szMap; /* Size of the mapping of file into memory */ + char *pMMapBuf; /* Where currently mmapped(). NULL if unmapped */ int nRef; /* Number of unixShm objects pointing to this */ unixShm *pFirst; /* All unixShm objects pointing to this */ unixShmFile *pNext; /* Next in list of all unixShmFile objects */ @@ -4995,8 +4993,6 @@ static int unixShmOpen( } pFile->fid.dev = sStat.st_dev; pFile->fid.ino = sStat.st_ino; - pFile->size = (int)sStat.st_size; - pFile->size = (pFile->size/SQLITE_UNIX_SHM_INCR)*SQLITE_UNIX_SHM_INCR; /* Check to see if another process is holding the dead-man switch. ** If not, truncate the file to zero length. @@ -5010,7 +5006,6 @@ static int unixShmOpen( rc = SQLITE_IOERR; goto shm_open_err; } - pFile->size = 0; } rc = unixShmSystemLock(pFile, F_RDLCK, UNIX_SHM_DMS); if( rc ) goto shm_open_err; @@ -5078,22 +5073,64 @@ static int unixShmClose(sqlite3_shm *pSharedMem){ } /* -** Query and/or changes the size of a shared-memory segment. -** The reqSize parameter is the new size of the segment, or -1 to -** do just a query. The size of the segment after resizing is -** written into pNewSize. A writer lock is held on the shared memory -** segment while resizing it. +** Query and/or changes the size of the underlying storage for +** a shared-memory segment. The reqSize parameter is the new size +** of the underlying storage, or -1 to do just a query. The size +** of the underlying storage (after resizing if resizing occurs) is +** written into pNewSize. ** -** If ppBuffer is not NULL, the a reader lock is acquired on the shared -** memory segment and *ppBuffer is made to point to the start of the -** shared memory segment. xShmRelease() must be called to release the -** lock. +** This routine does not (necessarily) change the size of the mapping +** of the underlying storage into memory. Use xShmGet() to change +** the mapping size. +** +** The reqSize parameter is the minimum size requested. The implementation +** is free to expand the storage to some larger amount if it chooses. */ static int unixShmSize( sqlite3_shm *pSharedMem, /* Pointer returned by unixShmOpen() */ int reqSize, /* Requested size. -1 for query only */ - int *pNewSize, /* Write new size here */ - void **ppBuf /* Write new buffer origin here */ + int *pNewSize /* Write new size here */ +){ + unixShm *p = (unixShm*)pSharedMem; + unixShmFile *pFile = p->pFile; + int rc = SQLITE_OK; + struct stat sStat; + + if( reqSize>=0 ){ + reqSize = (reqSize + SQLITE_UNIX_SHM_INCR - 1)/SQLITE_UNIX_SHM_INCR; + reqSize *= SQLITE_UNIX_SHM_INCR; + rc = ftruncate(pFile->h, reqSize); + } + if( fstat(pFile->h, &sStat)==0 ){ + *pNewSize = (int)sStat.st_size; + }else{ + *pNewSize = 0; + rc = SQLITE_IOERR; + } + return rc; +} + + +/* +** Map the shared storage into memory. The minimum size of the +** mapping should be reqMapSize if reqMapSize is positive. If +** reqMapSize is zero or negative, the implementation can choose +** whatever mapping size is convenient. +** +** *ppBuf is made to point to the memory which is a mapping of the +** underlying storage. This segment is locked. unixShmRelease() +** must be called to release the lock. +** +** *pNewMapSize is set to the size of the mapping. +** +** *ppBuf and *pNewMapSize might be NULL and zero if no space has +** yet been allocated to the underlying storage. +*/ +static int unixShmGet( + sqlite3_shm *pSharedMem, /* Pointer returned by unixShmOpen() */ + int reqMapSize, /* Requested size of mapping. -1 means don't care */ + int *pNewMapSize, /* Write new size of mapping here */ + void **ppBuf /* Write mapping buffer origin here */ ){ unixShm *p = (unixShm*)pSharedMem; unixShmFile *pFile = p->pFile; @@ -5101,23 +5138,21 @@ static int unixShmSize( sqlite3_mutex_enter(pFile->mutexBuf); sqlite3_mutex_enter(pFile->mutex); - if( reqSize>=0 ){ - reqSize = (reqSize + SQLITE_UNIX_SHM_INCR - 1)/SQLITE_UNIX_SHM_INCR; - reqSize *= SQLITE_UNIX_SHM_INCR; - if( reqSize!=pFile->size ){ - if( pFile->pMMapBuf ) munmap(pFile->pMMapBuf, pFile->size); - rc = ftruncate(pFile->h, reqSize); - if( rc ){ - pFile->pMMapBuf = 0; - pFile->size = 0; - }else{ - pFile->pMMapBuf = mmap(0, reqSize, PROT_READ|PROT_WRITE, MAP_SHARED, - pFile->h, 0); - pFile->size = pFile->pMMapBuf ? reqSize : 0; - } + if( pFile->szMap==0 || reqMapSize>pFile->szMap ){ + int actualSize; + if( unixShmSize(pSharedMem, -1, &actualSize)==SQLITE_OK + && reqMapSizepMMapBuf ){ + munmap(pFile->pMMapBuf, pFile->szMap); + } + pFile->pMMapBuf = mmap(0, reqMapSize, PROT_READ|PROT_WRITE, MAP_SHARED, + pFile->h, 0); + pFile->szMap = pFile->pMMapBuf ? reqMapSize : 0; } - *pNewSize = pFile->size; + *pNewMapSize = pFile->szMap; *ppBuf = pFile->pMMapBuf; sqlite3_mutex_leave(pFile->mutex); return rc; @@ -6490,6 +6525,7 @@ int sqlite3_os_init(void){ unixGetLastError, /* xGetLastError */ \ unixShmOpen, /* xShmOpen */ \ unixShmSize, /* xShmSize */ \ + unixShmGet, /* xShmGet */ \ unixShmRelease, /* xShmRelease */ \ 0, /* xShmPush */ \ 0, /* xShmPull */ \ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 35ac5bd008..d2ce5396f2 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -844,7 +844,8 @@ struct sqlite3_vfs { ** definition. Those that follow are added in version 2 or later */ int (*xShmOpen)(sqlite3_vfs*, const char *zName, sqlite3_shm**); - int (*xShmSize)(sqlite3_shm*, int reqSize, int *pNewSize, void**); + int (*xShmSize)(sqlite3_shm*, int reqSize, int *pNewSize); + int (*xShmGet)(sqlite3_shm*, int reqMapSize, int *pMapSize, void**); int (*xShmRelease)(sqlite3_shm*); int (*xShmPush)(sqlite3_shm*); int (*xShmPull)(sqlite3_shm*); diff --git a/src/wal.c b/src/wal.c index 30af69ff6e..033d1c5232 100644 --- a/src/wal.c +++ b/src/wal.c @@ -126,7 +126,7 @@ struct Wal { sqlite3_file *pFd; /* File handle for WAL file */ u32 iCallback; /* Value to pass to log callback (or 0) */ sqlite3_shm *pWIndex; /* The open wal-index file */ - int szWIndex; /* Size of the wal-index */ + int szWIndex; /* Size of the wal-index that is mapped in mem */ u32 *pWiData; /* Pointer to wal-index content in memory */ u8 lockState; /* SQLITE_SHM_xxxx constant showing lock state */ u8 readerType; /* SQLITE_SHM_READ or SQLITE_SHM_READ_FULL */ @@ -368,7 +368,8 @@ static int walIndexEntry(u32 iFrame){ } /* -** Release our reference to the wal-index memory map. +** Release our reference to the wal-index memory map, if we are holding +** it. */ static void walIndexUnmap(Wal *pWal){ if( pWal->pWiData ){ @@ -378,26 +379,42 @@ static void walIndexUnmap(Wal *pWal){ } /* -** Resize the wal-index file. If newSize is negative, leave the size -** unchanged. +** Map the wal-index file into memory if it isn't already. +** +** The reqSize parameter is the minimum required size of the mapping. +** A value of -1 means "don't care". The reqSize parameter is ignored +** if the mapping is already held. */ -static int walIndexRemap(Wal *pWal, int newSize){ - int rc; - walIndexUnmap(pWal); - rc = pWal->pVfs->xShmSize(pWal->pWIndex, newSize, - &pWal->szWIndex, (void**)(char*)&pWal->pWiData); - if( rc==SQLITE_OK && pWal->pWiData==0 ){ - assert( pWal->szWIndex==0 ); - pWal->pWiData = &pWal->iCallback; +static int walIndexMap(Wal *pWal, int reqSize){ + int rc = SQLITE_OK; + if( pWal->pWiData==0 ){ + rc = pWal->pVfs->xShmGet(pWal->pWIndex, reqSize, &pWal->szWIndex, + (void**)(char*)&pWal->pWiData); + if( rc==SQLITE_OK && pWal->pWiData==0 ){ + /* Make sure pWal->pWiData is not NULL while we are holding the + ** lock on the mapping. */ + assert( pWal->szWIndex==0 ); + pWal->pWiData = &pWal->iCallback; + } } return rc; } /* -** Map the wal-index file into memory if it isn't already. +** Remap the wal-index so that the mapping covers the full size +** of the underlying file. +** +** If enlargeTo is non-negative, then increase the size of the underlying +** storage to be at least as big as enlargeTo before remapping. */ -static int walIndexMap(Wal *pWal){ - int rc = walIndexRemap(pWal, -1); +static int walIndexRemap(Wal *pWal, int enlargeTo){ + int rc; + int sz; + rc = pWal->pVfs->xShmSize(pWal->pWIndex, enlargeTo, &sz); + if( rc==SQLITE_OK && sz>pWal->szWIndex ){ + walIndexUnmap(pWal); + rc = walIndexMap(pWal, sz); + } return rc; } @@ -416,12 +433,12 @@ static int walIndexMap(Wal *pWal){ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ u32 iSlot = walIndexEntry(iFrame); - walIndexMap(pWal); + walIndexMap(pWal, -1); while( (iSlot+128)>=pWal->szWIndex ){ int rc; int nByte = pWal->szWIndex*4 + WALINDEX_MMAP_INCREMENT; - /* Unmap and remap the wal-index file. */ + /* Enlarge the storage, then remap it. */ rc = walIndexRemap(pWal, nByte); if( rc!=SQLITE_OK ){ return rc; @@ -640,7 +657,7 @@ static WalIterator *walIteratorInit(Wal *pWal){ struct WalSegment *pFinal; /* Final (unindexed) segment */ u8 *aTmp; /* Temp space used by merge-sort */ - walIndexMap(pWal); + walIndexMap(pWal, -1); aData = pWal->pWiData; iLast = pWal->hdr.iLastPg; nSegment = (iLast >> 8) + 1; @@ -789,7 +806,8 @@ int walIndexTryHdr(Wal *pWal, int *pChanged){ u32 aHdr[WALINDEX_HDR_NFIELD+2]; if( pWal->szWIndex==0 ){ - int rc = walIndexRemap(pWal, WALINDEX_MMAP_INCREMENT); + int rc; + rc = walIndexRemap(pWal, WALINDEX_MMAP_INCREMENT); if( rc ) return rc; } @@ -830,7 +848,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ int rc; assert( pWal->lockState>=SQLITE_SHM_READ ); - walIndexMap(pWal); + walIndexMap(pWal, -1); /* First try to read the header without a lock. Verify the checksum ** before returning. This will almost always work. @@ -881,11 +899,10 @@ int sqlite3WalOpenSnapshot(Wal *pWal, int *pChanged){ sqlite3WalCloseSnapshot(pWal); }else{ /* Check if the mapping needs to grow. */ - if( pWal->hdr.iLastPg - && walIndexEntry(pWal->hdr.iLastPg)>=pWal->szWIndex - ){ - rc = walIndexRemap(pWal, 0); - assert( rc || walIndexEntry(pWal->hdr.iLastPg)szWIndex ); + if( pWal->hdr.iLastPg + && walIndexEntry(pWal->hdr.iLastPg)>=pWal->szWIndex + ){ + walIndexRemap(pWal, -1); } } } @@ -913,7 +930,7 @@ int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, u8 *pOut){ int iFrame = (pWal->hdr.iLastPg & 0xFFFFFF00); assert( pWal->lockState==SQLITE_SHM_READ||pWal->lockState==SQLITE_SHM_WRITE ); - walIndexMap(pWal); + walIndexMap(pWal, -1); /* Do a linear search of the unindexed block of page-numbers (if any) ** at the end of the wal-index. An alternative to this would be to From 6045461eee8c9921e1b4155e211e6d944cec41eb Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 30 Apr 2010 15:24:43 +0000 Subject: [PATCH 63/89] If a reader attempts to upgrade to a writer, but is not reading the most recent database snapshot, return SQLITE_BUSY. FossilOrigin-Name: 837d82a92977cbfa0963411daf8160d286a7ed32 --- manifest | 24 +++++++----------------- manifest.uuid | 2 +- src/wal.c | 18 ++++++++++++++++++ 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 63d18faa80..2cf1660e03 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,5 @@ ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA1 - -C Separate\sthe\sconcepts\sof\sunderlying\sstorage\ssize\sand\smapped\ssize\sin\sthe\nVFS\sshared-memory\simplementation. -D 2010-04-30T14:39:51 +C If\sa\sreader\sattempts\sto\supgrade\sto\sa\swriter,\sbut\sis\snot\sreading\sthe\smost\srecent\sdatabase\ssnapshot,\sreturn\sSQLITE_BUSY. +D 2010-04-30T15:24:44 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -224,7 +221,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c a4c643582fd9956b46dca187161d466f584b47af +F src/wal.c 94ebe9e477574baac105975088529f7b141be159 F src/wal.h c3f347ba8f1cde46d9bcc6fedaf3ed0aa4b53294 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 @@ -811,14 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 1f9e8c5c27ae2c68bc01d8e2bb2c662be7b3f161 -R 2fbe76e11b4a2b6e620f52387007559a -U drh -Z 868ea682fbe4fab342b578dc69caee7c ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.6 (GNU/Linux) - -iD8DBQFL2uu6oxKgR168RlERAqI3AJ4jq6FSOAaisShm+qyJakBJL+pVmwCgg1Vd -tQBOoEYMwVK9lQZSibwuRzM= -=5suj ------END PGP SIGNATURE----- +P 4cbe49f13fed288f94ff305bcfd99df907bf7baf +R a532c500516c30c14fdbbd5b70d5c12a +U dan +Z 18b7f313bb88e234cdae10ff0630f7fb diff --git a/manifest.uuid b/manifest.uuid index 0313645415..eff7969501 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4cbe49f13fed288f94ff305bcfd99df907bf7baf \ No newline at end of file +837d82a92977cbfa0963411daf8160d286a7ed32 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 033d1c5232..3bc5bea855 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1009,6 +1009,24 @@ int sqlite3WalWriteLock(Wal *pWal, int op){ if( op ){ assert( pWal->lockState == SQLITE_SHM_READ ); rc = walSetLock(pWal, SQLITE_SHM_WRITE); + + /* If this connection is not reading the most recent database snapshot, + ** it is not possible to write to the database. In this case release + ** the write locks and return SQLITE_BUSY. + */ + if( rc==SQLITE_OK ){ + rc = walIndexMap(pWal, -1); + if( rc==SQLITE_OK + && memcmp(&pWal->hdr, pWal->pWiData, sizeof(WalIndexHdr)) + ){ + rc = SQLITE_BUSY; + } + walIndexUnmap(pWal); + if( rc!=SQLITE_OK ){ + walSetLock(pWal, SQLITE_SHM_READ); + } + } + }else if( pWal->lockState==SQLITE_SHM_WRITE ){ rc = walSetLock(pWal, SQLITE_SHM_READ); } From a7ad518ef02e9e9ce1c3bb190e2895389916d23f Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 30 Apr 2010 15:49:27 +0000 Subject: [PATCH 64/89] When closing a WAL connection, attempt an exclusive lock on the database file. If the lock is obtained, checkpoint the database and delete the wal and wal-index files. FossilOrigin-Name: c05e7dca172719f33e245c08d0c0e8ab47e5a537 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 27 +++++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 2cf1660e03..d2b1cdb977 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sa\sreader\sattempts\sto\supgrade\sto\sa\swriter,\sbut\sis\snot\sreading\sthe\smost\srecent\sdatabase\ssnapshot,\sreturn\sSQLITE_BUSY. -D 2010-04-30T15:24:44 +C When\sclosing\sa\sWAL\sconnection,\sattempt\san\sexclusive\slock\son\sthe\sdatabase\sfile.\sIf\sthe\slock\sis\sobtained,\scheckpoint\sthe\sdatabase\sand\sdelete\sthe\swal\sand\swal-index\sfiles. +D 2010-04-30T15:49:28 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -221,7 +221,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c 94ebe9e477574baac105975088529f7b141be159 +F src/wal.c e4a52d13bff093bc446886cf7b0dfacbc7c2e8a4 F src/wal.h c3f347ba8f1cde46d9bcc6fedaf3ed0aa4b53294 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 4cbe49f13fed288f94ff305bcfd99df907bf7baf -R a532c500516c30c14fdbbd5b70d5c12a +P 837d82a92977cbfa0963411daf8160d286a7ed32 +R 3059f2f53adc88a1128c0bd66d3f0bb8 U dan -Z 18b7f313bb88e234cdae10ff0630f7fb +Z 901b5f2ba561b4eab874813e39c862c1 diff --git a/manifest.uuid b/manifest.uuid index eff7969501..108eb3175d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -837d82a92977cbfa0963411daf8160d286a7ed32 \ No newline at end of file +c05e7dca172719f33e245c08d0c0e8ab47e5a537 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 3bc5bea855..0942a53f78 100644 --- a/src/wal.c +++ b/src/wal.c @@ -785,8 +785,35 @@ int sqlite3WalClose( ){ int rc = SQLITE_OK; if( pWal ){ + int isDelete = 0; /* True to unlink wal and wal-index files */ + + /* If an EXCLUSIVE lock can be obtained on the database file (using the + ** ordinary, rollback-mode locking methods, this guarantees that the + ** connection associated with this log file is the only connection to + ** the database. In this case checkpoint the database and unlink both + ** the wal and wal-index files. + ** + ** The EXCLUSIVE lock is not released before returning. + */ + rc = sqlite3OsLock(pFd, SQLITE_LOCK_EXCLUSIVE); + if( rc==SQLITE_OK ){ + rc = walCheckpoint(pWal, pFd, sync_flags, zBuf); + if( rc==SQLITE_OK ){ + isDelete = 1; + } + walIndexUnmap(pWal); + } + pWal->pVfs->xShmClose(pWal->pWIndex); sqlite3OsClose(pWal->pFd); + if( isDelete ){ + int nWal; + char *zWal = &((char *)pWal->pFd)[pWal->pVfs->szOsFile]; + sqlite3OsDelete(pWal->pVfs, zWal, 0); + nWal = sqlite3Strlen30(zWal); + memcpy(&zWal[nWal], "-index", 7); + pWal->pVfs->xShmDelete(pWal->pVfs, zWal); + } sqlite3_free(pWal); } return rc; From d749084ad01242f87a867e5eb6e6e712bb326234 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 30 Apr 2010 15:54:46 +0000 Subject: [PATCH 65/89] Fix a couple uninitialized variables in the xShmLock method of the unix VFS. Improved debugging logic for xShmLock. FossilOrigin-Name: 69567c5fca9e30b7660e6f56350be929c3890d7e --- manifest | 24 +++++++++++++++++------- manifest.uuid | 2 +- src/os_unix.c | 17 +++++++++++++---- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index d2b1cdb977..a8c76e2fb4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,8 @@ -C When\sclosing\sa\sWAL\sconnection,\sattempt\san\sexclusive\slock\son\sthe\sdatabase\sfile.\sIf\sthe\slock\sis\sobtained,\scheckpoint\sthe\sdatabase\sand\sdelete\sthe\swal\sand\swal-index\sfiles. -D 2010-04-30T15:49:28 +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 + +C Fix\sa\scouple\suninitialized\svariables\sin\sthe\sxShmLock\smethod\sof\sthe\sunix\sVFS.\nImproved\sdebugging\slogic\sfor\sxShmLock. +D 2010-04-30T15:54:47 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -150,7 +153,7 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 -F src/os_unix.c b2322d8fb389b4bc7834b1596bd8afac776a2f20 +F src/os_unix.c 7d10d3a8b4f3e0c7bde874ca7e488ea95b7cae35 F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 F src/pager.c 434f9751fc2dfc11ade004282deda5f8560bcba2 F src/pager.h 934b598583a9d936bb13c37d62a2fe68ac48781c @@ -808,7 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 837d82a92977cbfa0963411daf8160d286a7ed32 -R 3059f2f53adc88a1128c0bd66d3f0bb8 -U dan -Z 901b5f2ba561b4eab874813e39c862c1 +P c05e7dca172719f33e245c08d0c0e8ab47e5a537 +R 948b3a20fa291178488b4491978dd814 +U drh +Z 0b83af1d29a8669f0e45a2c764ca0612 +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.6 (GNU/Linux) + +iD8DBQFL2v1KoxKgR168RlERAtTvAJ9anChtmXUlkLMCUv41sdV2i+tOhgCfd+Pd +kBX8Xzf3zeWR2SKzYU3eb8s= +=dWTT +-----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index 108eb3175d..021b3496c3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c05e7dca172719f33e245c08d0c0e8ab47e5a537 \ No newline at end of file +69567c5fca9e30b7660e6f56350be929c3890d7e \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 006b0d182d..2af4f5a093 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4716,6 +4716,7 @@ static int unixShmSystemLock( f.l_whence = SEEK_SET; if( (lockMask & UNIX_SHM_MUTEX)!=0 && lockType!=F_UNLCK ){ lockOp = F_SETLKW; + OSTRACE(("SHM-LOCK requesting blocking lock\n")); }else{ lockOp = F_SETLK; } @@ -4768,7 +4769,7 @@ static int unixShmSystemLock( OSTRACE(("write-lock failed")); } } - OSTRACE((" - change requested %s - afterwards %s,%s\n", + OSTRACE((" - change requested %s - afterwards %s:%s\n", unixShmLockString(lockMask), unixShmLockString(pFile->sharedMask), unixShmLockString(pFile->exclMask))); @@ -4794,6 +4795,7 @@ static int unixShmUnlock( assert( sqlite3_mutex_held(pFile->mutex) ); /* Compute locks held by sibling connections */ + allMask = 0; for(pX=pFile->pFirst; pX; pX=pX->pNext){ if( pX==p ) continue; assert( (pX->exclMask & (p->exclMask|p->sharedMask))==0 ); @@ -4834,6 +4836,7 @@ static int unixShmSharedLock( ** If any sibling already holds an exclusive lock, go ahead and return ** SQLITE_BUSY. */ + allShared = 0; for(pX=pFile->pFirst; pX; pX=pX->pNext){ if( pX==p ) continue; if( (pX->exclMask & readMask)!=0 ) return SQLITE_BUSY; @@ -5004,12 +5007,13 @@ static int unixShmOpen( if( unixShmSystemLock(pFile, F_WRLCK, UNIX_SHM_DMS)==SQLITE_OK ){ if( ftruncate(pFile->h, 0) ){ rc = SQLITE_IOERR; - goto shm_open_err; } } - rc = unixShmSystemLock(pFile, F_RDLCK, UNIX_SHM_DMS); - if( rc ) goto shm_open_err; + if( rc==SQLITE_OK ){ + rc = unixShmSystemLock(pFile, F_RDLCK, UNIX_SHM_DMS); + } unixShmSystemLock(pFile, F_UNLCK, UNIX_SHM_MUTEX); + if( rc ) goto shm_open_err; } /* Make the new connection a child of the unixShmFile */ @@ -5201,10 +5205,14 @@ static int unixShmLock( || desiredLock==p->lockState || (desiredLock==SQLITE_SHM_READ && p->lockState==SQLITE_SHM_READ_FULL) ){ + OSTRACE(("SHM-LOCK shmid-%d, pid-%d request %d and got %d\n", + p->id, getpid(), desiredLock, p->lockState)); if( pGotLock ) *pGotLock = p->lockState; return SQLITE_OK; } + OSTRACE(("SHM-LOCK shmid-%d, pid-%d request %d->%d\n", + p->id, getpid(), p->lockState, desiredLock)); sqlite3_mutex_enter(pFile->mutex); switch( desiredLock ){ case SQLITE_SHM_UNLOCK: { @@ -5293,6 +5301,7 @@ static int unixShmLock( } } sqlite3_mutex_leave(pFile->mutex); + OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %d\n", p->id,getpid(),p->lockState)); if( pGotLock ) *pGotLock = p->lockState; return rc; } From 49156b2b5eef22e57c0063fc4ad1a17155c39fe3 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 30 Apr 2010 16:12:04 +0000 Subject: [PATCH 66/89] Correctly record the fact that the SHM lock reached PENDING if it did so but failed to reach CHECKPOINT. FossilOrigin-Name: d9250e84ac1fc6590f8670e1d801630650c7846a --- manifest | 18 ++++++++--------- manifest.uuid | 2 +- src/wal.c | 54 ++++----------------------------------------------- 3 files changed, 14 insertions(+), 60 deletions(-) diff --git a/manifest b/manifest index a8c76e2fb4..e197ab314f 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 -C Fix\sa\scouple\suninitialized\svariables\sin\sthe\sxShmLock\smethod\sof\sthe\sunix\sVFS.\nImproved\sdebugging\slogic\sfor\sxShmLock. -D 2010-04-30T15:54:47 +C Correctly\srecord\sthe\sfact\sthat\sthe\sSHM\slock\sreached\sPENDING\sif\sit\sdid\sso\nbut\sfailed\sto\sreach\sCHECKPOINT. +D 2010-04-30T16:12:04 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -224,7 +224,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c e4a52d13bff093bc446886cf7b0dfacbc7c2e8a4 +F src/wal.c f08889611d5da7d2cbe9557748e74f70ab454974 F src/wal.h c3f347ba8f1cde46d9bcc6fedaf3ed0aa4b53294 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 @@ -811,14 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P c05e7dca172719f33e245c08d0c0e8ab47e5a537 -R 948b3a20fa291178488b4491978dd814 +P 69567c5fca9e30b7660e6f56350be929c3890d7e +R f1e5ca7c2966e55a12226da1f0023970 U drh -Z 0b83af1d29a8669f0e45a2c764ca0612 +Z 994a305fc2d5c733b714331360471304 -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.6 (GNU/Linux) -iD8DBQFL2v1KoxKgR168RlERAtTvAJ9anChtmXUlkLMCUv41sdV2i+tOhgCfd+Pd -kBX8Xzf3zeWR2SKzYU3eb8s= -=dWTT +iD8DBQFL2wFboxKgR168RlERAtivAJ421LoVOiKR0QgB26iqLm0H6jPUVQCfaTeL +mXtGX6pDDXUBVGxSGpeJW2k= +=Sf6+ -----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index 021b3496c3..39d508f322 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -69567c5fca9e30b7660e6f56350be929c3890d7e \ No newline at end of file +d9250e84ac1fc6590f8670e1d801630650c7846a \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 0942a53f78..2925023e71 100644 --- a/src/wal.c +++ b/src/wal.c @@ -217,11 +217,10 @@ static int walSetLock(Wal *pWal, int desiredStatus){ int rc, got; if( pWal->lockState==desiredStatus ) return SQLITE_OK; rc = pWal->pVfs->xShmLock(pWal->pWIndex, desiredStatus, &got); - if( rc==SQLITE_OK ){ - pWal->lockState = desiredStatus; - if( desiredStatus==SQLITE_SHM_READ ){ - pWal->readerType = got; - } + pWal->lockState = got; + if( got==SQLITE_SHM_READ_FULL || got==SQLITE_SHM_READ ){ + pWal->readerType = got; + pWal->lockState = SQLITE_SHM_READ; } return rc; } @@ -785,35 +784,8 @@ int sqlite3WalClose( ){ int rc = SQLITE_OK; if( pWal ){ - int isDelete = 0; /* True to unlink wal and wal-index files */ - - /* If an EXCLUSIVE lock can be obtained on the database file (using the - ** ordinary, rollback-mode locking methods, this guarantees that the - ** connection associated with this log file is the only connection to - ** the database. In this case checkpoint the database and unlink both - ** the wal and wal-index files. - ** - ** The EXCLUSIVE lock is not released before returning. - */ - rc = sqlite3OsLock(pFd, SQLITE_LOCK_EXCLUSIVE); - if( rc==SQLITE_OK ){ - rc = walCheckpoint(pWal, pFd, sync_flags, zBuf); - if( rc==SQLITE_OK ){ - isDelete = 1; - } - walIndexUnmap(pWal); - } - pWal->pVfs->xShmClose(pWal->pWIndex); sqlite3OsClose(pWal->pFd); - if( isDelete ){ - int nWal; - char *zWal = &((char *)pWal->pFd)[pWal->pVfs->szOsFile]; - sqlite3OsDelete(pWal->pVfs, zWal, 0); - nWal = sqlite3Strlen30(zWal); - memcpy(&zWal[nWal], "-index", 7); - pWal->pVfs->xShmDelete(pWal->pVfs, zWal); - } sqlite3_free(pWal); } return rc; @@ -1036,24 +1008,6 @@ int sqlite3WalWriteLock(Wal *pWal, int op){ if( op ){ assert( pWal->lockState == SQLITE_SHM_READ ); rc = walSetLock(pWal, SQLITE_SHM_WRITE); - - /* If this connection is not reading the most recent database snapshot, - ** it is not possible to write to the database. In this case release - ** the write locks and return SQLITE_BUSY. - */ - if( rc==SQLITE_OK ){ - rc = walIndexMap(pWal, -1); - if( rc==SQLITE_OK - && memcmp(&pWal->hdr, pWal->pWiData, sizeof(WalIndexHdr)) - ){ - rc = SQLITE_BUSY; - } - walIndexUnmap(pWal); - if( rc!=SQLITE_OK ){ - walSetLock(pWal, SQLITE_SHM_READ); - } - } - }else if( pWal->lockState==SQLITE_SHM_WRITE ){ rc = walSetLock(pWal, SQLITE_SHM_READ); } From 6ddb4bf938221cc7a8932fdcf071b3604d15ea75 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 30 Apr 2010 16:19:39 +0000 Subject: [PATCH 67/89] Add xShmXXX() methods to the test VFS in test_devsym.test. FossilOrigin-Name: 1d20342424b452ea96aaf161de1f98f26a9155a0 --- manifest | 24 ++++++------------- manifest.uuid | 2 +- src/test_devsym.c | 61 ++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 68 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index e197ab314f..c03e51e253 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,5 @@ ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA1 - -C Correctly\srecord\sthe\sfact\sthat\sthe\sSHM\slock\sreached\sPENDING\sif\sit\sdid\sso\nbut\sfailed\sto\sreach\sCHECKPOINT. -D 2010-04-30T16:12:04 +C Add\sxShmXXX()\smethods\sto\sthe\stest\sVFS\sin\stest_devsym.test. +D 2010-04-30T16:19:40 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -191,7 +188,7 @@ F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de F src/test_btree.c 47cd771250f09cdc6e12dda5bc71bc0b3abc96e2 F src/test_config.c 8c408fbffbe8082d1d3dc55044ddfd5580f3c9b9 F src/test_demovfs.c da81a5f7785bb352bda7911c332a983ec4f17f27 -F src/test_devsym.c 361d069bf74d4044bf53eb29dc0fa3709ed98baf +F src/test_devsym.c c05024cd179603181cde66d0ff83ce81e146f6de F src/test_func.c 13b582345fb1185a93e46c53310fae8547dcce20 F src/test_hexio.c 1237f000ec7a491009b1233f5c626ea71bce1ea2 F src/test_init.c 5d624ffd0409d424cf9adbfe1f056b200270077c @@ -811,14 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 69567c5fca9e30b7660e6f56350be929c3890d7e -R f1e5ca7c2966e55a12226da1f0023970 -U drh -Z 994a305fc2d5c733b714331360471304 ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.6 (GNU/Linux) - -iD8DBQFL2wFboxKgR168RlERAtivAJ421LoVOiKR0QgB26iqLm0H6jPUVQCfaTeL -mXtGX6pDDXUBVGxSGpeJW2k= -=Sf6+ ------END PGP SIGNATURE----- +P d9250e84ac1fc6590f8670e1d801630650c7846a +R e706456f30289526a342dac546a83543 +U dan +Z 9513429a159541b5d8980cb57914a083 diff --git a/manifest.uuid b/manifest.uuid index 39d508f322..0bfe58ee67 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d9250e84ac1fc6590f8670e1d801630650c7846a \ No newline at end of file +1d20342424b452ea96aaf161de1f98f26a9155a0 \ No newline at end of file diff --git a/src/test_devsym.c b/src/test_devsym.c index 9f71538044..dff7b98803 100644 --- a/src/test_devsym.c +++ b/src/test_devsym.c @@ -68,8 +68,16 @@ static int devsymRandomness(sqlite3_vfs*, int nByte, char *zOut); static int devsymSleep(sqlite3_vfs*, int microseconds); static int devsymCurrentTime(sqlite3_vfs*, double*); +static int devsymShmOpen(sqlite3_vfs *, const char *, sqlite3_shm **); +static int devsymShmSize(sqlite3_shm *, int , int *); +static int devsymShmGet(sqlite3_shm *, int , int *, void **); +static int devsymShmRelease(sqlite3_shm *); +static int devsymShmLock(sqlite3_shm *, int , int *); +static int devsymShmClose(sqlite3_shm *); +static int devsymShmDelete(sqlite3_vfs *, const char *); + static sqlite3_vfs devsym_vfs = { - 1, /* iVersion */ + 2, /* iVersion */ sizeof(devsym_file), /* szOsFile */ DEVSYM_MAX_PATHNAME, /* mxPathname */ 0, /* pNext */ @@ -93,6 +101,18 @@ static sqlite3_vfs devsym_vfs = { devsymRandomness, /* xRandomness */ devsymSleep, /* xSleep */ devsymCurrentTime, /* xCurrentTime */ + 0, /* xGetLastError */ + devsymShmOpen, + devsymShmSize, + devsymShmGet, + devsymShmRelease, + 0, + 0, + devsymShmLock, + devsymShmClose, + devsymShmDelete, + 0, + 0, }; static sqlite3_io_methods devsym_io_methods = { @@ -333,6 +353,38 @@ static int devsymCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ return sqlite3OsCurrentTime(g.pVfs, pTimeOut); } + +static int devsymShmOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_shm **pp +){ + return g.pVfs->xShmOpen(g.pVfs, zName, pp); +} +static int devsymShmSize(sqlite3_shm *p, int reqSize, int *pNewSize){ + return g.pVfs->xShmSize(p, reqSize, pNewSize); +} +static int devsymShmGet( + sqlite3_shm *p, + int reqMapSize, + int *pMapSize, + void **pp +){ + return g.pVfs->xShmGet(p, reqMapSize, pMapSize, pp); +} +static int devsymShmRelease(sqlite3_shm *p){ + return g.pVfs->xShmRelease(p); +} +static int devsymShmLock(sqlite3_shm *p, int desiredLock, int *gotLock){ + return g.pVfs->xShmLock(p, desiredLock, gotLock); +} +static int devsymShmClose(sqlite3_shm *p){ + return g.pVfs->xShmClose(p); +} +static int devsymShmDelete(sqlite3_vfs *pVfs, const char *zName){ + return g.pVfs->xShmDelete(g.pVfs, zName); +} + /* ** This procedure registers the devsym vfs with SQLite. If the argument is ** true, the devsym vfs becomes the new default vfs. It is the only publicly @@ -342,6 +394,13 @@ void devsym_register(int iDeviceChar, int iSectorSize){ if( g.pVfs==0 ){ g.pVfs = sqlite3_vfs_find(0); devsym_vfs.szOsFile += g.pVfs->szOsFile; + devsym_vfs.xShmOpen = (g.pVfs->xShmOpen ? devsymShmOpen : 0); + devsym_vfs.xShmSize = (g.pVfs->xShmSize ? devsymShmSize : 0); + devsym_vfs.xShmGet = (g.pVfs->xShmGet ? devsymShmGet : 0); + devsym_vfs.xShmRelease = (g.pVfs->xShmRelease ? devsymShmRelease : 0); + devsym_vfs.xShmLock = (g.pVfs->xShmLock ? devsymShmLock : 0); + devsym_vfs.xShmClose = (g.pVfs->xShmClose ? devsymShmClose : 0); + devsym_vfs.xShmDelete = (g.pVfs->xShmDelete ? devsymShmDelete : 0); sqlite3_vfs_register(&devsym_vfs, 0); } if( iDeviceChar>=0 ){ From 30c8629e67373a08c60bf7cc89c39ec7e30809d6 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 30 Apr 2010 16:24:46 +0000 Subject: [PATCH 68/89] Reapply commits [837d82a929] and [c05e7dca17] that were accidentally overwritten. FossilOrigin-Name: 598de52700ba13e3228289220a74fc62073c62d4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index c03e51e253..3855100ef3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sxShmXXX()\smethods\sto\sthe\stest\sVFS\sin\stest_devsym.test. -D 2010-04-30T16:19:40 +C Reapply\scommits\s[837d82a929]\sand\s[c05e7dca17]\sthat\swere\saccidentally\soverwritten. +D 2010-04-30T16:24:46 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -221,7 +221,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c f08889611d5da7d2cbe9557748e74f70ab454974 +F src/wal.c efe21d703cd5f32c18fd87762b6ac20dc47c5785 F src/wal.h c3f347ba8f1cde46d9bcc6fedaf3ed0aa4b53294 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P d9250e84ac1fc6590f8670e1d801630650c7846a -R e706456f30289526a342dac546a83543 +P 1d20342424b452ea96aaf161de1f98f26a9155a0 +R 88f12ad5a621dbb661d799113c2782f7 U dan -Z 9513429a159541b5d8980cb57914a083 +Z 72810ce2ab0ab749da9f7335fd238964 diff --git a/manifest.uuid b/manifest.uuid index 0bfe58ee67..914344034a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1d20342424b452ea96aaf161de1f98f26a9155a0 \ No newline at end of file +598de52700ba13e3228289220a74fc62073c62d4 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 2925023e71..3b57184c81 100644 --- a/src/wal.c +++ b/src/wal.c @@ -784,8 +784,35 @@ int sqlite3WalClose( ){ int rc = SQLITE_OK; if( pWal ){ + int isDelete = 0; /* True to unlink wal and wal-index files */ + + /* If an EXCLUSIVE lock can be obtained on the database file (using the + ** ordinary, rollback-mode locking methods, this guarantees that the + ** connection associated with this log file is the only connection to + ** the database. In this case checkpoint the database and unlink both + ** the wal and wal-index files. + ** + ** The EXCLUSIVE lock is not released before returning. + */ + rc = sqlite3OsLock(pFd, SQLITE_LOCK_EXCLUSIVE); + if( rc==SQLITE_OK ){ + rc = walCheckpoint(pWal, pFd, sync_flags, zBuf); + if( rc==SQLITE_OK ){ + isDelete = 1; + } + walIndexUnmap(pWal); + } + pWal->pVfs->xShmClose(pWal->pWIndex); sqlite3OsClose(pWal->pFd); + if( isDelete ){ + int nWal; + char *zWal = &((char *)pWal->pFd)[pWal->pVfs->szOsFile]; + sqlite3OsDelete(pWal->pVfs, zWal, 0); + nWal = sqlite3Strlen30(zWal); + memcpy(&zWal[nWal], "-index", 7); + pWal->pVfs->xShmDelete(pWal->pVfs, zWal); + } sqlite3_free(pWal); } return rc; @@ -1008,6 +1035,23 @@ int sqlite3WalWriteLock(Wal *pWal, int op){ if( op ){ assert( pWal->lockState == SQLITE_SHM_READ ); rc = walSetLock(pWal, SQLITE_SHM_WRITE); + + /* If this connection is not reading the most recent database snapshot, + ** it is not possible to write to the database. In this case release + ** the write locks and return SQLITE_BUSY. + */ + if( rc==SQLITE_OK ){ + rc = walIndexMap(pWal, -1); + if( rc==SQLITE_OK + && memcmp(&pWal->hdr, pWal->pWiData, sizeof(WalIndexHdr)) + ){ + rc = SQLITE_BUSY; + } + walIndexUnmap(pWal); + if( rc!=SQLITE_OK ){ + walSetLock(pWal, SQLITE_SHM_READ); + } + } }else if( pWal->lockState==SQLITE_SHM_WRITE ){ rc = walSetLock(pWal, SQLITE_SHM_READ); } From 78daa5a7d9ae0629032b459d2fb65c2f6fc5e2a8 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 30 Apr 2010 16:38:59 +0000 Subject: [PATCH 69/89] Fix a bug whereby upgrading from a READ_FULL to a WRITE lock, and then back to a read lock was dropping all WAL locks. FossilOrigin-Name: 55f5af5e174df7a28deeb36faa51138926669897 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 3855100ef3..00988eee33 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reapply\scommits\s[837d82a929]\sand\s[c05e7dca17]\sthat\swere\saccidentally\soverwritten. -D 2010-04-30T16:24:46 +C Fix\sa\sbug\swhereby\supgrading\sfrom\sa\sREAD_FULL\sto\sa\sWRITE\slock,\sand\sthen\sback\sto\sa\sread\slock\swas\sdropping\sall\sWAL\slocks. +D 2010-04-30T16:38:59 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -150,7 +150,7 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 -F src/os_unix.c 7d10d3a8b4f3e0c7bde874ca7e488ea95b7cae35 +F src/os_unix.c d0deee9969cec7364c7de92418888e042a366fcf F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 F src/pager.c 434f9751fc2dfc11ade004282deda5f8560bcba2 F src/pager.h 934b598583a9d936bb13c37d62a2fe68ac48781c @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 1d20342424b452ea96aaf161de1f98f26a9155a0 -R 88f12ad5a621dbb661d799113c2782f7 +P 598de52700ba13e3228289220a74fc62073c62d4 +R 723210e8144121fec5928d0b8c025919 U dan -Z 72810ce2ab0ab749da9f7335fd238964 +Z c28d7cd6bf4885aaa152a2bab954cfdb diff --git a/manifest.uuid b/manifest.uuid index 914344034a..56c1228640 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -598de52700ba13e3228289220a74fc62073c62d4 \ No newline at end of file +55f5af5e174df7a28deeb36faa51138926669897 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 2af4f5a093..6e01ffb6bf 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5240,9 +5240,9 @@ static int unixShmLock( } } }else if( p->lockState==SQLITE_SHM_WRITE ){ + rc = unixShmSharedLock(pFile, p, UNIX_SHM_A); unixShmUnlock(pFile, p, UNIX_SHM_C|UNIX_SHM_D); - p->lockState = p->readLock; - rc = SQLITE_OK; + p->lockState = SQLITE_SHM_READ; }else{ assert( p->lockState==SQLITE_SHM_RECOVER ); unixShmUnlock(pFile, p, UNIX_SHM_MUTEX); From 43a56b812c6faf2069b428c84fb91c368eda4894 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 30 Apr 2010 16:41:53 +0000 Subject: [PATCH 70/89] Fix a problem in the previous commit. Both unixShm.lockState and unixShm.readLock should be set to READ when downgrading from a WRITE to a READ lock FossilOrigin-Name: 90119fecbce06c8db82194dfd9d5045be29b1cda --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 00988eee33..b81cad1a8f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\swhereby\supgrading\sfrom\sa\sREAD_FULL\sto\sa\sWRITE\slock,\sand\sthen\sback\sto\sa\sread\slock\swas\sdropping\sall\sWAL\slocks. -D 2010-04-30T16:38:59 +C Fix\sa\sproblem\sin\sthe\sprevious\scommit.\sBoth\sunixShm.lockState\sand\sunixShm.readLock\sshould\sbe\sset\sto\sREAD\swhen\sdowngrading\sfrom\sa\sWRITE\sto\sa\sREAD\slock +D 2010-04-30T16:41:54 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -150,7 +150,7 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 -F src/os_unix.c d0deee9969cec7364c7de92418888e042a366fcf +F src/os_unix.c 72413d328c45dce9d7d11a8726a660dd501732c7 F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 F src/pager.c 434f9751fc2dfc11ade004282deda5f8560bcba2 F src/pager.h 934b598583a9d936bb13c37d62a2fe68ac48781c @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 598de52700ba13e3228289220a74fc62073c62d4 -R 723210e8144121fec5928d0b8c025919 +P 55f5af5e174df7a28deeb36faa51138926669897 +R 30163f46a964ded4f6fbb73a10520c97 U dan -Z c28d7cd6bf4885aaa152a2bab954cfdb +Z 2d3273731b921855c7e11145f0f666b0 diff --git a/manifest.uuid b/manifest.uuid index 56c1228640..771a318cb1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -55f5af5e174df7a28deeb36faa51138926669897 \ No newline at end of file +90119fecbce06c8db82194dfd9d5045be29b1cda \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 6e01ffb6bf..a85d7a61bf 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5242,7 +5242,7 @@ static int unixShmLock( }else if( p->lockState==SQLITE_SHM_WRITE ){ rc = unixShmSharedLock(pFile, p, UNIX_SHM_A); unixShmUnlock(pFile, p, UNIX_SHM_C|UNIX_SHM_D); - p->lockState = SQLITE_SHM_READ; + p->lockState = p->readLock = SQLITE_SHM_READ; }else{ assert( p->lockState==SQLITE_SHM_RECOVER ); unixShmUnlock(pFile, p, UNIX_SHM_MUTEX); From 400df2ea70f47060dd8c30852931b32d994881da Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 30 Apr 2010 16:48:19 +0000 Subject: [PATCH 71/89] In the debugging output for SHM-LOCK in os_unix.c, use symbolic names for the lock states rather than raw numbers. FossilOrigin-Name: 2afc33de2b2012d034fb0d2057a5a45e304516ca --- manifest | 24 +++++++++++++++++------- manifest.uuid | 2 +- src/os_unix.c | 25 ++++++++++++++++++++----- 3 files changed, 38 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index b81cad1a8f..aff9762597 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,8 @@ -C Fix\sa\sproblem\sin\sthe\sprevious\scommit.\sBoth\sunixShm.lockState\sand\sunixShm.readLock\sshould\sbe\sset\sto\sREAD\swhen\sdowngrading\sfrom\sa\sWRITE\sto\sa\sREAD\slock -D 2010-04-30T16:41:54 +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 + +C In\sthe\sdebugging\soutput\sfor\sSHM-LOCK\sin\sos_unix.c,\suse\ssymbolic\snames\nfor\sthe\slock\sstates\srather\sthan\sraw\snumbers. +D 2010-04-30T16:48:20 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -150,7 +153,7 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 -F src/os_unix.c 72413d328c45dce9d7d11a8726a660dd501732c7 +F src/os_unix.c edbc13a32a6b835b5303a444bd7da9404dc17e58 F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 F src/pager.c 434f9751fc2dfc11ade004282deda5f8560bcba2 F src/pager.h 934b598583a9d936bb13c37d62a2fe68ac48781c @@ -808,7 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 55f5af5e174df7a28deeb36faa51138926669897 -R 30163f46a964ded4f6fbb73a10520c97 -U dan -Z 2d3273731b921855c7e11145f0f666b0 +P 90119fecbce06c8db82194dfd9d5045be29b1cda +R ebae94644ba62e3155e25c91f8d6fd85 +U drh +Z 02eccc20e834aee25fd2cd7428a6b07d +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.6 (GNU/Linux) + +iD8DBQFL2wnXoxKgR168RlERAkXlAJ97RPvDHNYNUAR/6YpO0tH6FowdpgCghlQl +WdLDFQjAGffRMStZhNqyiHA= +=AZcG +-----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index 771a318cb1..0d44bec975 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -90119fecbce06c8db82194dfd9d5045be29b1cda \ No newline at end of file +2afc33de2b2012d034fb0d2057a5a45e304516ca \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index a85d7a61bf..483978673b 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5173,6 +5173,20 @@ static int unixShmRelease(sqlite3_shm *pSharedMem){ return SQLITE_OK; } +/* +** Symbolic names for LOCK states used for debugging. +*/ +#ifdef SQLITE_DEBUG +static const char *azLkName[] = { + "UNLOCK", + "READ", + "READ_FULL", + "WRITE", + "PENDING", + "CHECKPOINT", + "RECOVER" +}; +#endif /* @@ -5205,14 +5219,14 @@ static int unixShmLock( || desiredLock==p->lockState || (desiredLock==SQLITE_SHM_READ && p->lockState==SQLITE_SHM_READ_FULL) ){ - OSTRACE(("SHM-LOCK shmid-%d, pid-%d request %d and got %d\n", - p->id, getpid(), desiredLock, p->lockState)); + OSTRACE(("SHM-LOCK shmid-%d, pid-%d request %s and got %s\n", + p->id, getpid(), azLkName[desiredLock], azLkName[p->lockState])); if( pGotLock ) *pGotLock = p->lockState; return SQLITE_OK; } - OSTRACE(("SHM-LOCK shmid-%d, pid-%d request %d->%d\n", - p->id, getpid(), p->lockState, desiredLock)); + OSTRACE(("SHM-LOCK shmid-%d, pid-%d request %s->%s\n", + p->id, getpid(), azLkName[p->lockState], azLkName[desiredLock])); sqlite3_mutex_enter(pFile->mutex); switch( desiredLock ){ case SQLITE_SHM_UNLOCK: { @@ -5301,7 +5315,8 @@ static int unixShmLock( } } sqlite3_mutex_leave(pFile->mutex); - OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %d\n", p->id,getpid(),p->lockState)); + OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %s\n", + p->id, getpid(), azLkName[p->lockState])); if( pGotLock ) *pGotLock = p->lockState; return rc; } From c9d53dbe5a232b81f33cabff27746a1052264edd Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 30 Apr 2010 16:50:00 +0000 Subject: [PATCH 72/89] Fix a problem with calculating the required size of the wal-index when appending frames. FossilOrigin-Name: 9526b11130f3ed2f5227386a26263c767214603a --- manifest | 24 +++++++----------------- manifest.uuid | 2 +- src/wal.c | 4 ++-- 3 files changed, 10 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index aff9762597..6bab8bba72 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,5 @@ ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA1 - -C In\sthe\sdebugging\soutput\sfor\sSHM-LOCK\sin\sos_unix.c,\suse\ssymbolic\snames\nfor\sthe\slock\sstates\srather\sthan\sraw\snumbers. -D 2010-04-30T16:48:20 +C Fix\sa\sproblem\swith\scalculating\sthe\srequired\ssize\sof\sthe\swal-index\swhen\sappending\sframes. +D 2010-04-30T16:50:00 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -224,7 +221,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c efe21d703cd5f32c18fd87762b6ac20dc47c5785 +F src/wal.c f4bdb4f86fa7e08a69c8955e3bd22b5026143427 F src/wal.h c3f347ba8f1cde46d9bcc6fedaf3ed0aa4b53294 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 @@ -811,14 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 90119fecbce06c8db82194dfd9d5045be29b1cda -R ebae94644ba62e3155e25c91f8d6fd85 -U drh -Z 02eccc20e834aee25fd2cd7428a6b07d ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.6 (GNU/Linux) - -iD8DBQFL2wnXoxKgR168RlERAkXlAJ97RPvDHNYNUAR/6YpO0tH6FowdpgCghlQl -WdLDFQjAGffRMStZhNqyiHA= -=AZcG ------END PGP SIGNATURE----- +P 2afc33de2b2012d034fb0d2057a5a45e304516ca +R 70ef57c4a03e427eca1482f7ccd1ced8 +U dan +Z 0d70ee7035126be286a826f0c20d1cde diff --git a/manifest.uuid b/manifest.uuid index 0d44bec975..8b1c1630d4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2afc33de2b2012d034fb0d2057a5a45e304516ca \ No newline at end of file +9526b11130f3ed2f5227386a26263c767214603a \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 3b57184c81..3ac18c74d6 100644 --- a/src/wal.c +++ b/src/wal.c @@ -433,9 +433,9 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ u32 iSlot = walIndexEntry(iFrame); walIndexMap(pWal, -1); - while( (iSlot+128)>=pWal->szWIndex ){ + while( ((iSlot+128)*sizeof(u32))>=pWal->szWIndex ){ int rc; - int nByte = pWal->szWIndex*4 + WALINDEX_MMAP_INCREMENT; + int nByte = pWal->szWIndex + WALINDEX_MMAP_INCREMENT; /* Enlarge the storage, then remap it. */ rc = walIndexRemap(pWal, nByte); From fe05aa144cc48632ed4af3e90dbaac3b8d0ef317 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 30 Apr 2010 17:05:23 +0000 Subject: [PATCH 73/89] Fix a problem with resizing the wal-index mapping after the mapping has been extended by an external process. FossilOrigin-Name: 79d356fea6008a8adf8ebd4906571375f3cf5213 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 6bab8bba72..f7b6e13c48 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\scalculating\sthe\srequired\ssize\sof\sthe\swal-index\swhen\sappending\sframes. -D 2010-04-30T16:50:00 +C Fix\sa\sproblem\swith\sresizing\sthe\swal-index\smapping\safter\sthe\smapping\shas\sbeen\sextended\sby\san\sexternal\sprocess. +D 2010-04-30T17:05:24 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -221,7 +221,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c f4bdb4f86fa7e08a69c8955e3bd22b5026143427 +F src/wal.c 2115a90d42e1cc14c0f196677609f6c9b32afec7 F src/wal.h c3f347ba8f1cde46d9bcc6fedaf3ed0aa4b53294 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c faadd9c2bf08868e5135192b44e0d753e363a885 @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 2afc33de2b2012d034fb0d2057a5a45e304516ca -R 70ef57c4a03e427eca1482f7ccd1ced8 +P 9526b11130f3ed2f5227386a26263c767214603a +R 91978a8283887836c2a96b7f05bffc8f U dan -Z 0d70ee7035126be286a826f0c20d1cde +Z 2ea44366a0f8d457f9964ceb9f3d8476 diff --git a/manifest.uuid b/manifest.uuid index 8b1c1630d4..b4eca7de53 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9526b11130f3ed2f5227386a26263c767214603a \ No newline at end of file +79d356fea6008a8adf8ebd4906571375f3cf5213 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 3ac18c74d6..d406383a28 100644 --- a/src/wal.c +++ b/src/wal.c @@ -926,7 +926,7 @@ int sqlite3WalOpenSnapshot(Wal *pWal, int *pChanged){ }else{ /* Check if the mapping needs to grow. */ if( pWal->hdr.iLastPg - && walIndexEntry(pWal->hdr.iLastPg)>=pWal->szWIndex + && walIndexEntry(pWal->hdr.iLastPg)*sizeof(u32)>=pWal->szWIndex ){ walIndexRemap(pWal, -1); } From 1b48aa49a4497e418241a46ba110b160dadee4ba Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 30 Apr 2010 17:28:35 +0000 Subject: [PATCH 74/89] Remove some obsolete debugging parameters. FossilOrigin-Name: a012aed498bf41a5f6f3006182b0c4c2ab0fe1bb --- manifest | 26 ++++++++++++++++++-------- manifest.uuid | 2 +- src/test1.c | 3 --- test/wal.test | 5 +++-- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index f7b6e13c48..43c1bf6dbb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,8 @@ -C Fix\sa\sproblem\swith\sresizing\sthe\swal-index\smapping\safter\sthe\smapping\shas\sbeen\sextended\sby\san\sexternal\sprocess. -D 2010-04-30T17:05:24 +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 + +C Remove\ssome\sobsolete\sdebugging\sparameters. +D 2010-04-30T17:28:35 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -173,7 +176,7 @@ F src/sqliteLimit.h 3afab2291762b5d09ae20c18feb8e9fa935a60a6 F src/status.c 4df6fe7dce2d256130b905847c6c60055882bdbe F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 4de81521174fedacd8393ea7b70b730ce17f8eae -F src/test1.c a449144907face6414f68f6a5663a5b8cadabbd4 +F src/test1.c 8b4c246c41e75c3ff033edb2e8c2cf15820861ae F src/test2.c b6b43413d495addd039a88b87d65c839f86b18cb F src/test3.c 4c21700c73a890a47fc685c1097bfb661346ac94 F src/test4.c ad03bb987ddedce928f4258c1e7fa4109a73497d @@ -758,7 +761,7 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test 67f1b8ec4aa2251db2b4d33f43ea9b1ba1711210 +F test/wal.test 830fc2d723fc4c55ef5666141e3a54e0cedd87e5 F test/walbak.test f6fde9a5f59d0c697cb1f4af7876178c2f69a7ba F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 287a69d662939604f2e0452dace2cec8ef634d5e @@ -808,7 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 9526b11130f3ed2f5227386a26263c767214603a -R 91978a8283887836c2a96b7f05bffc8f -U dan -Z 2ea44366a0f8d457f9964ceb9f3d8476 +P 79d356fea6008a8adf8ebd4906571375f3cf5213 +R 7623f491cc3fe25e6cf1447dc278db34 +U drh +Z 4cf1ecd05c61c28ec308c760f4a7a514 +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.6 (GNU/Linux) + +iD8DBQFL2xNGoxKgR168RlERAkqbAKCB34id8nZsSwu8Q+yGhmvcyx0/GACdHh30 +Yn92idGRYRLRmXhKMauGM8c= +=ZLEl +-----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index b4eca7de53..2f3627cc9f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -79d356fea6008a8adf8ebd4906571375f3cf5213 \ No newline at end of file +a012aed498bf41a5f6f3006182b0c4c2ab0fe1bb \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index f3eff8c672..b5fda952c0 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5113,7 +5113,6 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ #ifdef SQLITE_TEST extern char sqlite3_query_plan[]; static char *query_plan = sqlite3_query_plan; -/* extern int sqlite3_walsummary_mmap_incr; // In wal.c */ #ifdef SQLITE_ENABLE_FTS3 extern int sqlite3_fts3_enable_parentheses; #endif @@ -5173,8 +5172,6 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ #ifdef SQLITE_TEST Tcl_LinkVar(interp, "sqlite_query_plan", (char*)&query_plan, TCL_LINK_STRING|TCL_LINK_READ_ONLY); -/* Tcl_LinkVar(interp, "sqlite_walsummary_mmap_incr", - (char*)&sqlite3_walsummary_mmap_incr, TCL_LINK_INT); */ #endif #ifdef SQLITE_DEBUG Tcl_LinkVar(interp, "sqlite_addop_trace", diff --git a/test/wal.test b/test/wal.test index be654c0237..017b47f40e 100644 --- a/test/wal.test +++ b/test/wal.test @@ -848,7 +848,6 @@ db close #------------------------------------------------------------------------- # Test large log summaries. # -set sqlite_walsummary_mmap_incr 64 do_test wal-13.1.1 { list [file exists test.db] [file exists test.db-wal] } {1 0} @@ -911,6 +910,9 @@ foreach code [list { do_test wal-13.$tn.$ii.c { db eval { SELECT count(*) FROM t1 } } [expr (1<<$ii)] + do_test wal-13.$tn.$ii.d { + db eval { PRAGMA integrity_check } + } {ok} } catch { db2 close } @@ -965,4 +967,3 @@ do_test wal-14 { catch { db close } catch { db2 close } finish_test - From 6dea3248aefe879dbb9a186e632fd65b15920299 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 30 Apr 2010 17:47:51 +0000 Subject: [PATCH 75/89] Change the SHM VFS logic in os_unix.c so that it does not hold an exclusive lock n the mapped memory when also holding a CHECKPOINT lock. This improves concurrency between readers and checkpointers. FossilOrigin-Name: 8660cda6f8ef43bd276897ef3b5fc2376b5684dc --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/os_unix.c | 12 +++++++++--- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 43c1bf6dbb..5325f93076 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 -C Remove\ssome\sobsolete\sdebugging\sparameters. -D 2010-04-30T17:28:35 +C Change\sthe\sSHM\sVFS\slogic\sin\sos_unix.c\sso\sthat\sit\sdoes\snot\shold\san\sexclusive\nlock\sn\sthe\smapped\smemory\swhen\salso\sholding\sa\sCHECKPOINT\slock.\s\sThis\nimproves\sconcurrency\sbetween\sreaders\sand\scheckpointers. +D 2010-04-30T17:47:51 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -153,7 +153,7 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 -F src/os_unix.c edbc13a32a6b835b5303a444bd7da9404dc17e58 +F src/os_unix.c 559f9f18ccd85a6d8da0309753f4be124998bddf F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 F src/pager.c 434f9751fc2dfc11ade004282deda5f8560bcba2 F src/pager.h 934b598583a9d936bb13c37d62a2fe68ac48781c @@ -811,14 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 79d356fea6008a8adf8ebd4906571375f3cf5213 -R 7623f491cc3fe25e6cf1447dc278db34 +P a012aed498bf41a5f6f3006182b0c4c2ab0fe1bb +R 8c6bff1c20fef6364d82d75506d94733 U drh -Z 4cf1ecd05c61c28ec308c760f4a7a514 +Z 01ae0928e38536e336a0fa4223ef6f3d -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.6 (GNU/Linux) -iD8DBQFL2xNGoxKgR168RlERAkqbAKCB34id8nZsSwu8Q+yGhmvcyx0/GACdHh30 -Yn92idGRYRLRmXhKMauGM8c= -=ZLEl +iD8DBQFL2xfKoxKgR168RlERAu5rAKCJ9R7soT0Y2Sx2CpYlaWrlm57SuACeIVK9 +g9rON3+p6pxGk+AOwiTbLws= +=wAdF -----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index 2f3627cc9f..74da902bdf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a012aed498bf41a5f6f3006182b0c4c2ab0fe1bb \ No newline at end of file +8660cda6f8ef43bd276897ef3b5fc2376b5684dc \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 483978673b..f37afa1f14 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5140,7 +5140,10 @@ static int unixShmGet( unixShmFile *pFile = p->pFile; int rc = SQLITE_OK; - sqlite3_mutex_enter(pFile->mutexBuf); + if( p->lockState!=SQLITE_SHM_CHECKPOINT ){ + sqlite3_mutex_enter(pFile->mutexBuf); + p->hasMutexBuf = 1; + } sqlite3_mutex_enter(pFile->mutex); if( pFile->szMap==0 || reqMapSize>pFile->szMap ){ int actualSize; @@ -5168,8 +5171,11 @@ static int unixShmGet( */ static int unixShmRelease(sqlite3_shm *pSharedMem){ unixShm *p = (unixShm*)pSharedMem; - unixShmFile *pFile = p->pFile; - sqlite3_mutex_leave(pFile->mutexBuf); + if( p->hasMutexBuf ){ + unixShmFile *pFile = p->pFile; + sqlite3_mutex_leave(pFile->mutexBuf); + p->hasMutexBuf = 0; + } return SQLITE_OK; } From d80b233850c12cd3b3e69bd797e7276a9bf6422f Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 1 May 2010 00:59:37 +0000 Subject: [PATCH 76/89] Do not allow journal_mode=WAL if the underlying VFS does not support xShmOpen. FossilOrigin-Name: d1fcccecdc8e9ac5d0d022914e51c545f4e1b04f --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/vdbe.c | 20 +++++++++++++++----- test/walmode.test | 17 +++++++++++------ 4 files changed, 37 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index 8bd0968857..15a349d662 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 -C Merge\sin\schanges\sfrom\sthe\strunk. -D 2010-04-30T22:28:06 +C Do\snot\sallow\sjournal_mode=WAL\sif\sthe\sunderlying\sVFS\sdoes\snot\ssupport\sxShmOpen. +D 2010-05-01T00:59:38 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -215,7 +215,7 @@ F src/update.c c0dc6b75ad28b76b619042d934f337b02acee208 F src/utf.c 1baeeac91707a4df97ccc6141ec0f808278af685 F src/util.c 32aebf04c10e51ad3977a928b7416bed671b620b F src/vacuum.c 8e7d9a6fd076774bb6e211cad335c7337948a164 -F src/vdbe.c 21a96eeeb43dedd40072028db0b42e137296a81a +F src/vdbe.c 2a5ccf3a760498d16ff3869162d3422f165bfb1d F src/vdbe.h 471f6a3dcec4817ca33596fe7f6654d56c0e75f3 F src/vdbeInt.h 19ebc8c2a2e938340051ee65af3f377fb99102d1 F src/vdbeapi.c 85e448c7d1d6dde7226c721cd8da8f9b5286699d @@ -765,7 +765,7 @@ F test/wal.test 830fc2d723fc4c55ef5666141e3a54e0cedd87e5 F test/walbak.test f6fde9a5f59d0c697cb1f4af7876178c2f69a7ba F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 287a69d662939604f2e0452dace2cec8ef634d5e -F test/walmode.test 40119078da084e6a7403ba57485d5a86ee0e2646 +F test/walmode.test 999be0fbee266ed618de780a6dadfca662d0f763 F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 F test/walthread.test c9c3b5d19eeb4968254d93f61a7ccd6e8b63f554 F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 @@ -811,14 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 8660cda6f8ef43bd276897ef3b5fc2376b5684dc 946e06cd2a2cc209b0786b7e9d28c0746320d646 -R 34e37e2e07b357ad442391f6d6aea47a +P 76bf0eee1fd4003a3f1c39922f8f059611d41dd0 +R 9dfde34976e5276ea286640fcc12ec63 U drh -Z f36c3392b22f93b3ce95c9951a47e7e6 +Z 32ec43d929a1f569787f34775d6f160c -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.6 (GNU/Linux) -iD8DBQFL21l5oxKgR168RlERAqNRAJ906Wol3RyZLztZzmkcpokv0+CtRgCfbeMz -ohrTG6i1s6JqVvmqdQdSCDg= -=50x7 +iD4DBQFL23z+oxKgR168RlERAhO3AJ90Nf9l23uNBBZGi//1U+RT1mFhmQCWLU5N +/Vu6AuAN53A7nGU5H1J5Vg== +=7uDR -----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index 695f751f08..2ccd280d27 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -76bf0eee1fd4003a3f1c39922f8f059611d41dd0 \ No newline at end of file +d1fcccecdc8e9ac5d0d022914e51c545f4e1b04f \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 60e72fadd2..c6a38c5ba6 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5212,11 +5212,15 @@ case OP_Checkpoint: { ** ** Write a string containing the final journal-mode to register P2. */ -case OP_JournalMode: { +case OP_JournalMode: { /* out2-prerelease */ Btree *pBt; /* Btree to change journal mode of */ Pager *pPager; /* Pager associated with pBt */ - int eNew = pOp->p3; /* New journal mode */ + int eNew; /* New journal mode */ + int eOld; /* The old journal mode */ + const sqlite3_vfs *pVfs; /* The VFS of pPager */ + const char *zFilename; /* Name of database file for pPager */ + eNew = pOp->p3; assert( eNew==PAGER_JOURNALMODE_DELETE || eNew==PAGER_JOURNALMODE_TRUNCATE || eNew==PAGER_JOURNALMODE_PERSIST @@ -5230,15 +5234,21 @@ case OP_JournalMode: { pBt = db->aDb[pOp->p1].pBt; pPager = sqlite3BtreePager(pBt); + zFilename = sqlite3PagerFilename(pPager); + pVfs = sqlite3PagerVfs(pPager); - if( 0==sqlite3Strlen30(sqlite3BtreeGetFilename(pBt)) - && eNew==PAGER_JOURNALMODE_WAL + /* Do not allow a transition to journal_mode=WAL for a database + ** in temporary storage or if the VFS does not support xShmOpen. + */ + if( eNew==PAGER_JOURNALMODE_WAL + && (zFilename[0]==0 /* Temp file */ + || pVfs->iVersion<2 || pVfs->xShmOpen==0) /* No xShmOpen support */ ){ eNew = PAGER_JOURNALMODE_QUERY; } if( eNew!=PAGER_JOURNALMODE_QUERY ){ - int eOld = sqlite3PagerJournalMode(pPager, PAGER_JOURNALMODE_QUERY); + eOld = sqlite3PagerJournalMode(pPager, PAGER_JOURNALMODE_QUERY); if( (eNew!=eOld) && (eOld==PAGER_JOURNALMODE_WAL || eNew==PAGER_JOURNALMODE_WAL) ){ diff --git a/test/walmode.test b/test/walmode.test index 3af72d91c0..4df00a350a 100644 --- a/test/walmode.test +++ b/test/walmode.test @@ -153,6 +153,7 @@ do_test walmode-5.1.1 { sqlite3 db :memory: execsql { PRAGMA main.journal_mode } } {memory} +breakpoint do_test walmode-5.1.2 { execsql { PRAGMA main.journal_mode = wal } } {memory} @@ -205,13 +206,18 @@ do_test walmode-5.2.5 { } } {1 2 3 4 delete} +if {$TEMP_STORE>=2} { + set tempJrnlMode memory +} else { + set tempJrnlMode +} do_test walmode-5.3.1 { sqlite3 db test.db execsql { PRAGMA temp.journal_mode } -} {delete} +} $tempJrnlMode do_test walmode-5.3.2 { execsql { PRAGMA temp.journal_mode = wal } -} {delete} +} $tempJrnlMode do_test walmode-5.3.3 { execsql { BEGIN; @@ -221,17 +227,16 @@ do_test walmode-5.3.3 { SELECT * FROM t1; PRAGMA temp.journal_mode; } -} {1 2 delete} +} [list 1 2 $tempJrnlMode] do_test walmode-5.3.4 { execsql { PRAGMA temp.journal_mode = wal } -} {delete} +} $tempJrnlMode do_test walmode-5.3.5 { execsql { INSERT INTO t1 VALUES(3, 4); SELECT * FROM t1; PRAGMA temp.journal_mode; } -} {1 2 3 4 delete} +} [list 1 2 3 4 $tempJrnlMode] finish_test - From bb16ff75dd0b2e840f97e27cb0756faf5f3d20b0 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 1 May 2010 08:30:34 +0000 Subject: [PATCH 77/89] Add test case demonstrating deadlock during recovery of very large log files. No fix yet. FossilOrigin-Name: 63ea318eb19d264667909c70185b8a5cdc4454c0 --- manifest | 24 ++++---------- manifest.uuid | 2 +- test/walthread.test | 79 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 86 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 15a349d662..fca9dd688f 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,5 @@ ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA1 - -C Do\snot\sallow\sjournal_mode=WAL\sif\sthe\sunderlying\sVFS\sdoes\snot\ssupport\sxShmOpen. -D 2010-05-01T00:59:38 +C Add\stest\scase\sdemonstrating\sdeadlock\sduring\srecovery\sof\svery\slarge\slog\sfiles.\sNo\sfix\syet. +D 2010-05-01T08:30:35 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -767,7 +764,7 @@ F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 287a69d662939604f2e0452dace2cec8ef634d5e F test/walmode.test 999be0fbee266ed618de780a6dadfca662d0f763 F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 -F test/walthread.test c9c3b5d19eeb4968254d93f61a7ccd6e8b63f554 +F test/walthread.test 896b2377145e32fab024677d68f609df2f6e9bc6 F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 F test/where2.test 43d4becaf5a5df854e6c21d624a1cb84c6904554 F test/where3.test aa44a9b29e8c9f3d7bb94a3bb3a95b31627d520d @@ -811,14 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 76bf0eee1fd4003a3f1c39922f8f059611d41dd0 -R 9dfde34976e5276ea286640fcc12ec63 -U drh -Z 32ec43d929a1f569787f34775d6f160c ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.6 (GNU/Linux) - -iD4DBQFL23z+oxKgR168RlERAhO3AJ90Nf9l23uNBBZGi//1U+RT1mFhmQCWLU5N -/Vu6AuAN53A7nGU5H1J5Vg== -=7uDR ------END PGP SIGNATURE----- +P d1fcccecdc8e9ac5d0d022914e51c545f4e1b04f +R 034d4495629d0958f2c325287d33624b +U dan +Z 43bb63963c222b2fa1b30185cbd547d7 diff --git a/manifest.uuid b/manifest.uuid index 2ccd280d27..abff647efb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d1fcccecdc8e9ac5d0d022914e51c545f4e1b04f \ No newline at end of file +63ea318eb19d264667909c70185b8a5cdc4454c0 \ No newline at end of file diff --git a/test/walthread.test b/test/walthread.test index efd321df3b..fa42af9c6a 100644 --- a/test/walthread.test +++ b/test/walthread.test @@ -28,6 +28,7 @@ set seconds(walthread-1) 20 set seconds(walthread-2) 20 set seconds(walthread-3) 20 set seconds(walthread-4) 20 +set seconds(walthread-5) 1 # The parameter is the name of a variable in the callers context. The # variable may or may not exist when this command is invoked. @@ -130,7 +131,7 @@ proc do_thread_test {args} { sqlite3 db test.db eval $P(init) - db close + catch { db close } foreach T $P(threads) { set name [lindex $T 0] @@ -454,5 +455,81 @@ do_thread_test2 walthread-4 -seconds $seconds(walthread-4) -init { set {} ok } + +# This test case attempts to provoke a deadlock condition that existed in +# the unix VFS at one point. The problem occurred only while recovering a +# very large wal file (one that requires a wal-index larger than the +# initial default allocation of 64KB). +# +# When recovering a log file, mutexes are usually obtained and released +# as follows: +# +# xShmGet ENTER mutexBuf +# xShmLock(RECOVER) ENTER mutexRecov +# +# xShmLock(READ) LEAVE mutexRecov +# xShmRelease LEAVE mutexBuf +# +# However, if the initial wal-index allocation needs to be enlarged during +# the recovery process, the mutex operations become: +# +# xShmGet ENTER mutexBuf +# xShmLock(RECOVER) ENTER mutexRecov +# +# xShmRelease LEAVE mutexBuf +# xShmSize +# xShmGet ENTER mutexBuf +# +# xShmLock(READ) LEAVE mutexRecov +# xShmRelease LEAVE mutexBuf +# +# If multiple threads attempt the above simultaneously, deadlock can occur. +# +do_thread_test walthread-5 -seconds $seconds(walthread-5) -init { + + proc log_file_size {nFrame pgsz} { + expr {12 + ($pgsz+16)*$nFrame} + } + + execsql { + PRAGMA page_size = 1024; + PRAGMA journal_mode = WAL; + CREATE TABLE t1(x); + BEGIN; + INSERT INTO t1 VALUES(randomblob(900)); + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 2 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 32 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 64 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 128 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 256 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 512 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 1024 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 2048 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 4096 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 8192 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 16384 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 32768 */ + INSERT INTO t1 SELECT randomblob(900) FROM t1; /* 65536 */ + COMMIT; + } + + file copy -force test.db-wal bak.db-wal + file copy -force test.db bak.db + db close + + file copy -force bak.db-wal test.db-wal + file copy -force bak.db test.db + + if {[file size test.db-wal] < [log_file_size [expr 64*1024] 1024]} { + error "Somehow failed to create a large log file" + } + puts "Database with large log file recovered. Now running clients..." +} -thread T 5 { + db eval { SELECT count(*) FROM t1 } +} + finish_test From 7aba8fdef78724c09a89d382cb4373fbedf69d91 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 1 May 2010 11:19:07 +0000 Subject: [PATCH 78/89] Fix the sqlite3_mutex_alloc() interface to return NULL (not segfault) when operating in threadsafe mode. (This is a general-purpose bug fix which really ought to be ported to trunk.) FossilOrigin-Name: 64840a3caf569959c9d5e63bdbc0cf675fda3882 --- manifest | 24 +++++++++++++++++------- manifest.uuid | 2 +- src/mutex.c | 1 + 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index fca9dd688f..43b0d3a294 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,8 @@ -C Add\stest\scase\sdemonstrating\sdeadlock\sduring\srecovery\sof\svery\slarge\slog\sfiles.\sNo\sfix\syet. -D 2010-05-01T08:30:35 +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 + +C Fix\sthe\ssqlite3_mutex_alloc()\sinterface\sto\sreturn\sNULL\s(not\ssegfault)\swhen\noperating\sin\sthreadsafe\smode.\s\s(This\sis\sa\sgeneral-purpose\sbug\sfix\swhich\nreally\sought\sto\sbe\sported\sto\strunk.) +D 2010-05-01T11:19:08 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -139,7 +142,7 @@ F src/mem2.c 2ee7bdacda8299b5a91cff9f7ee3e46573195c38 F src/mem3.c 9b237d911ba9904142a804be727cc6664873f8a3 F src/mem5.c eb7a5cb98915dd7a086fa415ce3a5a0f20d0acff F src/memjournal.c f3be374af30588de297dcf678925b2a4758e4135 -F src/mutex.c 581a272e09098040ca3ef543cb5f3d643eff7d50 +F src/mutex.c c04412513619b3a763f3484bb412336fa3a3fbd9 F src/mutex.h 6fde601e55fa6c3fae768783c439797ab84c87c6 F src/mutex_noop.c 5f58eaa31f2d742cb8957a747f7887ae98f16053 F src/mutex_os2.c 20477db50cf3817c2f1cd3eb61e5c177e50231db @@ -808,7 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P d1fcccecdc8e9ac5d0d022914e51c545f4e1b04f -R 034d4495629d0958f2c325287d33624b -U dan -Z 43bb63963c222b2fa1b30185cbd547d7 +P 63ea318eb19d264667909c70185b8a5cdc4454c0 +R 2100e665be505fa1cf8f47333c26bc0b +U drh +Z 344033a82b2442e0392cbcdded1532fa +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.6 (GNU/Linux) + +iD8DBQFL3A4voxKgR168RlERAlUnAJ0S3R/Dfykt3Oi2UMidrwOa0L6Q6gCfe8MG +dPCr8yJsMbepoEqgAok4Aos= +=yH/M +-----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index abff647efb..3bed7c55fb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -63ea318eb19d264667909c70185b8a5cdc4454c0 \ No newline at end of file +64840a3caf569959c9d5e63bdbc0cf675fda3882 \ No newline at end of file diff --git a/src/mutex.c b/src/mutex.c index 11d498c474..edddaf1011 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -77,6 +77,7 @@ int sqlite3MutexEnd(void){ ** Retrieve a pointer to a static mutex or allocate a new dynamic one. */ sqlite3_mutex *sqlite3_mutex_alloc(int id){ + if( !sqlite3GlobalConfig.bCoreMutex ) return 0; #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif From 057f1ecddb5d0fa7aaebc8257a388b03a9744cbb Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 1 May 2010 11:33:13 +0000 Subject: [PATCH 79/89] Fix a typo in the walmode.test test script introducted yesterday. FossilOrigin-Name: dcabc90ca588281fa7a290972ec8af6816932489 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- test/walmode.test | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 43b0d3a294..173bd4ae6a 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 -C Fix\sthe\ssqlite3_mutex_alloc()\sinterface\sto\sreturn\sNULL\s(not\ssegfault)\swhen\noperating\sin\sthreadsafe\smode.\s\s(This\sis\sa\sgeneral-purpose\sbug\sfix\swhich\nreally\sought\sto\sbe\sported\sto\strunk.) -D 2010-05-01T11:19:08 +C Fix\sa\stypo\sin\sthe\swalmode.test\stest\sscript\sintroducted\syesterday. +D 2010-05-01T11:33:14 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -765,7 +765,7 @@ F test/wal.test 830fc2d723fc4c55ef5666141e3a54e0cedd87e5 F test/walbak.test f6fde9a5f59d0c697cb1f4af7876178c2f69a7ba F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 F test/walhook.test 287a69d662939604f2e0452dace2cec8ef634d5e -F test/walmode.test 999be0fbee266ed618de780a6dadfca662d0f763 +F test/walmode.test 3150edf5e2b36afebc3c6f0dd63b76f91485ac4a F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 F test/walthread.test 896b2377145e32fab024677d68f609df2f6e9bc6 F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 @@ -811,14 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 63ea318eb19d264667909c70185b8a5cdc4454c0 -R 2100e665be505fa1cf8f47333c26bc0b +P 64840a3caf569959c9d5e63bdbc0cf675fda3882 +R 21a564c0ac84f284e21a69521a141263 U drh -Z 344033a82b2442e0392cbcdded1532fa +Z b4b7a879c871fd7cac60b2970ea4e35c -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.6 (GNU/Linux) -iD8DBQFL3A4voxKgR168RlERAlUnAJ0S3R/Dfykt3Oi2UMidrwOa0L6Q6gCfe8MG -dPCr8yJsMbepoEqgAok4Aos= -=yH/M +iD8DBQFL3BF9oxKgR168RlERApFnAJ9WZITqlHLHOJRSxljsAEpGgWWkPwCeKmEF +oOhtURrWyUmr8iL/+kUvfZU= +=vngs -----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index 3bed7c55fb..31a1f9b724 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -64840a3caf569959c9d5e63bdbc0cf675fda3882 \ No newline at end of file +dcabc90ca588281fa7a290972ec8af6816932489 \ No newline at end of file diff --git a/test/walmode.test b/test/walmode.test index 4df00a350a..5b3006cc78 100644 --- a/test/walmode.test +++ b/test/walmode.test @@ -209,7 +209,7 @@ do_test walmode-5.2.5 { if {$TEMP_STORE>=2} { set tempJrnlMode memory } else { - set tempJrnlMode + set tempJrnlMode delete } do_test walmode-5.3.1 { sqlite3 db test.db From 5cf53537a8ec78f1f9b917e2f242ca1924057ddc Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 1 May 2010 16:40:20 +0000 Subject: [PATCH 80/89] Support compile-time option SQLITE_OMIT_WAL, for building without WAL support. FossilOrigin-Name: 9b230c43dfa112e3e1589f9775926807bd36b36e --- manifest | 62 ++++++------- manifest.uuid | 2 +- src/btree.c | 10 ++ src/main.c | 4 + src/os_unix.c | 11 +++ src/pager.c | 219 +++++++++++++++++++++++++------------------- src/pragma.c | 12 ++- src/sqliteInt.h | 2 + src/test_config.c | 6 ++ src/vdbe.c | 5 + src/vdbeapi.c | 8 +- src/wal.c | 4 + src/wal.h | 17 ++++ test/corruptA.test | 4 +- test/rdonly.test | 4 +- test/wal.test | 2 + test/walbak.test | 2 + test/walcrash.test | 2 + test/walhook.test | 2 + test/walmode.test | 19 ++++ test/walslow.test | 2 + test/walthread.test | 1 + 22 files changed, 267 insertions(+), 133 deletions(-) diff --git a/manifest b/manifest index 173bd4ae6a..d6e1251d4a 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,5 @@ ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA1 - -C Fix\sa\stypo\sin\sthe\swalmode.test\stest\sscript\sintroducted\syesterday. -D 2010-05-01T11:33:14 +C Support\scompile-time\soption\sSQLITE_OMIT_WAL,\sfor\sbuilding\swithout\sWAL\ssupport. +D 2010-05-01T16:40:20 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -112,7 +109,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c e86634da8c48357a759694c9c7c471125cd8d5a8 F src/bitvec.c 06ad2c36a9c3819c0b9cbffec7b15f58d5d834e0 F src/btmutex.c 96a12f50f7a17475155971a241d85ec5171573ff -F src/btree.c 2e41e62e80ad4686b3322721982ceb7aa6d8c9f1 +F src/btree.c d0414a5f09b0cacb64bd60b91c5a3720585925aa F src/btree.h dd83041eda10c17daf023257c1fc883b5f71f85a F src/btreeInt.h b0c87f6725b06a0aa194a6d25d54b16ce9d6e291 F src/build.c 11100b66fb97638d2d874c1d34d8db90650bb1d7 @@ -134,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/main.c 28171059466a5dac18160c466b978eaafdbb3594 +F src/main.c b672c4af58f033c54398999dec5db9d4b6d99331 F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 89d4ea8d5cdd55635cbaa48ad53132af6294cbb2 @@ -153,15 +150,15 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 -F src/os_unix.c 559f9f18ccd85a6d8da0309753f4be124998bddf +F src/os_unix.c 27ec47f27987b9f25bc55df5e458f7f380730f66 F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 -F src/pager.c 434f9751fc2dfc11ade004282deda5f8560bcba2 +F src/pager.c e9c7fe979b32a3c5bf4216d8fbe1cf8beff8a1b8 F src/pager.h 934b598583a9d936bb13c37d62a2fe68ac48781c F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 F src/pcache1.c 6dc1871ce8ead9187161c370a58cd06c84221f76 -F src/pragma.c 5ba364af022d9dbdfb39daf4fd26cbf26571fc0e +F src/pragma.c 3d4ba9c35438b296360dc09ebf86c417cf996667 F src/prepare.c fd1398cb1da54385ba5bd68d93928f10d10a1d9c F src/printf.c 5f5b65a83e63f2096a541a340722a509fa0240a7 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -171,7 +168,7 @@ F src/select.c c03d8a0565febcde8c6a12c5d77d065fddae889b F src/shell.c c40427c7245535a04a9cb4a417b6cc05c022e6a4 F src/sqlite.h.in 6e91727c0c3f9e1365e8fea2b07369a09830f47f F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89 -F src/sqliteInt.h 32dfe6294b46914fb567db493994bf902fb0f8c9 +F src/sqliteInt.h 3eb613c8ba7f5159be0acf3fc148d41ec49add86 F src/sqliteLimit.h 3afab2291762b5d09ae20c18feb8e9fa935a60a6 F src/status.c 4df6fe7dce2d256130b905847c6c60055882bdbe F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -189,7 +186,7 @@ F src/test_async.c c1656facbaf43cb2e71b62621e5b9eb080e2621c F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de F src/test_btree.c 47cd771250f09cdc6e12dda5bc71bc0b3abc96e2 -F src/test_config.c 8c408fbffbe8082d1d3dc55044ddfd5580f3c9b9 +F src/test_config.c 6210f501d358bde619ae761f06f123529c6ba24f F src/test_demovfs.c da81a5f7785bb352bda7911c332a983ec4f17f27 F src/test_devsym.c c05024cd179603181cde66d0ff83ce81e146f6de F src/test_func.c 13b582345fb1185a93e46c53310fae8547dcce20 @@ -215,17 +212,17 @@ F src/update.c c0dc6b75ad28b76b619042d934f337b02acee208 F src/utf.c 1baeeac91707a4df97ccc6141ec0f808278af685 F src/util.c 32aebf04c10e51ad3977a928b7416bed671b620b F src/vacuum.c 8e7d9a6fd076774bb6e211cad335c7337948a164 -F src/vdbe.c 2a5ccf3a760498d16ff3869162d3422f165bfb1d +F src/vdbe.c b2e2b70e9518c6cc986e74eca99607700904f4bd F src/vdbe.h 471f6a3dcec4817ca33596fe7f6654d56c0e75f3 F src/vdbeInt.h 19ebc8c2a2e938340051ee65af3f377fb99102d1 -F src/vdbeapi.c 85e448c7d1d6dde7226c721cd8da8f9b5286699d +F src/vdbeapi.c 810abe698db3ccaf67c5a8982f8cd4f31c6c902f F src/vdbeaux.c 395d21a1617553ea9b9f4abe0d302449d8549e1d F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c 2115a90d42e1cc14c0f196677609f6c9b32afec7 -F src/wal.h c3f347ba8f1cde46d9bcc6fedaf3ed0aa4b53294 +F src/wal.c 4a9b7fb028059992dfe51d23f0ced78064459c84 +F src/wal.h d6d4a6809151e30bed5b01dd05cf27858f5a7bc8 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 75fee9e255b62f773fcadd1d1f25b6f63ac7a356 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -308,7 +305,7 @@ F test/corrupt6.test e69b877d478224deab7b66844566258cecacd25e F test/corrupt7.test 1eb2214f29474fa6b155aa3da8a7d46bf52089e1 F test/corrupt8.test 9992ef7f67cefc576b92373f6bf5ab8775280f51 F test/corrupt9.test 4aa1cb1ef091cb0e13e89a819c72911631b5176a -F test/corruptA.test a2fb8d8037e5105d21dc094c3ef44864934325dd +F test/corruptA.test 38b4f81c16099f6d8fa8b37e188fde76b8243994 F test/corruptB.test d88b500f7994cb2eb9646410c263111e9831f889 F test/corruptC.test 691ed070baef5e1345939caadf270a52837a5064 F test/corruptD.test 3ae6e2dc6e2226c6935a8a40d4b5ee3eba75f8c0 @@ -545,7 +542,7 @@ F test/quick.test 6f202befe1cfae0b63df96b3120a8022ab11f574 F test/quote.test 215897dbe8de1a6f701265836d6601cc6ed103e6 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test 1084050991e9ba22c1c10edd8d84673b501cc25a -F test/rdonly.test 329558d8f3f7679e7e00f5f1d7cdd5dde78ba382 +F test/rdonly.test c243623b26c311faab2aee311a26af372e123a65 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 F test/rollback.test 73355ad4492ff9a3a31e61c7e5eb5e01a1de94ca F test/rowhash.test 97f56043ba11f0679920416c0cdbc72e5272267b @@ -761,13 +758,13 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test 830fc2d723fc4c55ef5666141e3a54e0cedd87e5 -F test/walbak.test f6fde9a5f59d0c697cb1f4af7876178c2f69a7ba -F test/walcrash.test f022cee7eb7baa5fb898726120a6a4073dd831d1 -F test/walhook.test 287a69d662939604f2e0452dace2cec8ef634d5e -F test/walmode.test 3150edf5e2b36afebc3c6f0dd63b76f91485ac4a -F test/walslow.test 38076d5fad49e3678027be0f8110e6a32d531dc2 -F test/walthread.test 896b2377145e32fab024677d68f609df2f6e9bc6 +F test/wal.test 4ff92f36495142420ce8c2bb45340dcad714a1e5 +F test/walbak.test fad10a10df78ab67e89a8779ddc3c0d230460693 +F test/walcrash.test 20ebd86c0ea1bfdec97987c3cf667118f5c74a84 +F test/walhook.test 5ee5fd35cdc3763d5674da0dd9974f6f70c07bed +F test/walmode.test bac6f06544a8554588a1543def996bbe2fc41792 +F test/walslow.test 9be52d033e871cb608a18fa4425a6dc64b625377 +F test/walthread.test 16559527eb1f1a6e9eff21cc92f7fa51fb0b5a32 F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 F test/where2.test 43d4becaf5a5df854e6c21d624a1cb84c6904554 F test/where3.test aa44a9b29e8c9f3d7bb94a3bb3a95b31627d520d @@ -811,14 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 64840a3caf569959c9d5e63bdbc0cf675fda3882 -R 21a564c0ac84f284e21a69521a141263 -U drh -Z b4b7a879c871fd7cac60b2970ea4e35c ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.6 (GNU/Linux) - -iD8DBQFL3BF9oxKgR168RlERApFnAJ9WZITqlHLHOJRSxljsAEpGgWWkPwCeKmEF -oOhtURrWyUmr8iL/+kUvfZU= -=vngs ------END PGP SIGNATURE----- +P dcabc90ca588281fa7a290972ec8af6816932489 +R e813455665cf7f3ad3b5157515fc2fe7 +U dan +Z 955de6008d5a15476ae5c3900eb59d71 diff --git a/manifest.uuid b/manifest.uuid index 31a1f9b724..c4a52f55ba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dcabc90ca588281fa7a290972ec8af6816932489 \ No newline at end of file +9b230c43dfa112e3e1589f9775926807bd36b36e \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index e60754e582..eace01fff1 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2261,6 +2261,15 @@ static int lockBtree(BtShared *pBt){ if( memcmp(page1, zMagicHeader, 16)!=0 ){ goto page1_init_failed; } + +#ifdef SQLITE_OMIT_WAL + if( page1[18]>1 ){ + pBt->readOnly = 1; + } + if( page1[19]>1 ){ + goto page1_init_failed; + } +#else if( page1[18]>2 ){ pBt->readOnly = 1; } @@ -2287,6 +2296,7 @@ static int lockBtree(BtShared *pBt){ } rc = SQLITE_NOTADB; } +#endif /* The maximum embedded fraction must be exactly 25%. And the minimum ** embedded fraction must be 12.5% for both leaf-data and non-leaf-data. diff --git a/src/main.c b/src/main.c index c723a4ef55..57ea9736a3 100644 --- a/src/main.c +++ b/src/main.c @@ -1195,6 +1195,7 @@ void *sqlite3_wal_hook( int(*xCallback)(void *, sqlite3*, const char*, int), void *pArg /* First argument passed to xCallback() */ ){ +#ifndef SQLITE_OMIT_WAL void *pRet; sqlite3_mutex_enter(db->mutex); pRet = db->pWalArg; @@ -1202,6 +1203,9 @@ void *sqlite3_wal_hook( db->pWalArg = pArg; sqlite3_mutex_leave(db->mutex); return pRet; +#else + return 0; +#endif } /* diff --git a/src/os_unix.c b/src/os_unix.c index f37afa1f14..b2c22c3652 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4562,6 +4562,8 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ return 0; } +#ifndef SQLITE_OMIT_WAL + /* Forward reference */ typedef struct unixShm unixShm; typedef struct unixShmFile unixShmFile; @@ -5334,6 +5336,15 @@ static int unixShmDelete(sqlite3_vfs *pVfs, const char *zName){ return pVfs->xDelete(pVfs, zName, 0); } +#else +# define unixShmOpen 0 +# define unixShmSize 0 +# define unixShmGet 0 +# define unixShmRelease 0 +# define unixShmLock 0 +# define unixShmClose 0 +# define unixShmDelete 0 +#endif /* #ifndef SQLITE_OMIT_WAL */ /* ************************ End of sqlite3_vfs methods *************************** diff --git a/src/pager.c b/src/pager.c index ee0c575a59..00d07caff0 100644 --- a/src/pager.c +++ b/src/pager.c @@ -399,7 +399,9 @@ struct Pager { char *pTmpSpace; /* Pager.pageSize bytes of space for tmp use */ PCache *pPCache; /* Pointer to page cache object */ sqlite3_backup *pBackup; /* Pointer to list of ongoing backup processes */ +#ifndef SQLITE_OMIT_WAL Wal *pWal; /* Write-ahead log used by "journal_mode=wal" */ +#endif }; /* @@ -1192,9 +1194,16 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ ** Return true if this pager uses a write-ahead log instead of the usual ** rollback journal. Otherwise false. */ +#ifndef SQLITE_OMIT_WAL static int pagerUseWal(Pager *pPager){ return (pPager->pWal!=0); } +#else +# define pagerUseWal(x) 0 +# define pagerRollbackWal(x) 0 +# define pagerWalFrames(v,w,x,y,z) 0 +# define pagerOpenWalIfPresent(z) SQLITE_OK +#endif /* ** Unlock the database file. This function is a no-op if the pager @@ -2242,6 +2251,7 @@ static int readDbPage(PgHdr *pPg){ return rc; } +#ifndef SQLITE_OMIT_WAL /* ** This function is invoked once for each page that has already been ** written into the log file when a WAL transaction is rolled back. @@ -2311,6 +2321,115 @@ static int pagerRollbackWal(Pager *pPager){ return rc; } +/* +** This function is a wrapper around sqlite3WalFrames(). As well as logging +** the contents of the list of pages headed by pList (connected by pDirty), +** this function notifies any active backup processes that the pages have +** changed. +*/ +static int pagerWalFrames( + Pager *pPager, /* Pager object */ + PgHdr *pList, /* List of frames to log */ + Pgno nTruncate, /* Database size after this commit */ + int isCommit, /* True if this is a commit */ + int sync_flags /* Flags to pass to OsSync() (or 0) */ +){ + int rc; /* Return code */ + + assert( pPager->pWal ); + rc = sqlite3WalFrames(pPager->pWal, + pPager->pageSize, pList, nTruncate, isCommit, sync_flags + ); + if( rc==SQLITE_OK && pPager->pBackup ){ + PgHdr *p; + for(p=pList; p; p=p->pDirty){ + sqlite3BackupUpdate(pPager->pBackup, p->pgno, (u8 *)p->pData); + } + } + return rc; +} + +/* +** Open a WAL snapshot on the log file this pager is connected to. +*/ +static int pagerOpenSnapshot(Pager *pPager){ + int rc; /* Return code */ + int changed = 0; /* True if cache must be reset */ + + assert( pagerUseWal(pPager) ); + + rc = sqlite3WalOpenSnapshot(pPager->pWal, &changed); + if( rc==SQLITE_OK ){ + int dummy; + if( changed ){ + pager_reset(pPager); + assert( pPager->errCode || pPager->dbSizeValid==0 ); + } + rc = sqlite3PagerPagecount(pPager, &dummy); + } + pPager->state = PAGER_SHARED; + + return rc; +} + +/* +** Check if the *-wal file that corresponds to the database opened by pPager +** exists. Assuming no error occurs, set *pExists to 1 if the file exists, +** or 0 otherwise and return SQLITE_OK. If an IO or OOM error occurs, return +** an SQLite error code. +*/ +static int pagerHasWAL(Pager *pPager, int *pExists){ + int rc; /* Return code */ + + if( !pPager->tempFile ){ + char *zWal = sqlite3_mprintf("%s-wal", pPager->zFilename); + if( !zWal ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3OsAccess(pPager->pVfs, zWal, SQLITE_ACCESS_EXISTS, pExists); + sqlite3_free(zWal); + } + }else{ + rc = SQLITE_OK; + *pExists = 0; + } + return rc; +} + +/* +** Check if the *-wal file that corresponds to the database opened by pPager +** exists. If it does, open the pager in WAL mode. Otherwise, if no error +** occurs, make sure Pager.journalMode is not set to PAGER_JOURNALMODE_WAL. +** If an IO or OOM error occurs, return an SQLite error code. +** +** If the WAL file is opened, also open a snapshot (read transaction). +** +** The caller must hold a SHARED lock on the database file to call this +** function. Because an EXCLUSIVE lock on the db file is required to delete +** a WAL, this ensures there is no race condition between the xAccess() +** below and an xDelete() being executed by some other connection. +*/ +static int pagerOpenWalIfPresent(Pager *pPager){ + int rc = SQLITE_OK; + if( !pPager->tempFile ){ + int isWal; /* True if WAL file exists */ + rc = pagerHasWAL(pPager, &isWal); + if( rc==SQLITE_OK ){ + if( isWal ){ + pager_reset(pPager); + rc = sqlite3PagerOpenWal(pPager, 0); + if( rc==SQLITE_OK ){ + rc = pagerOpenSnapshot(pPager); + } + }else if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){ + pPager->journalMode = PAGER_JOURNALMODE_DELETE; + } + } + } + return rc; +} +#endif + /* ** Playback savepoint pSavepoint. Or, if pSavepoint==NULL, then playback ** the entire master journal file. The case pSavepoint==NULL occurs when @@ -2934,10 +3053,12 @@ int sqlite3PagerClose(Pager *pPager){ sqlite3BeginBenignMalloc(); pPager->errCode = 0; pPager->exclusiveMode = 0; +#ifndef SQLITE_OMIT_WAL sqlite3WalClose(pPager->pWal, pPager->fd, (pPager->noSync ? 0 : pPager->sync_flags), pTmp ); pPager->pWal = 0; +#endif pager_reset(pPager); if( MEMDB ){ pager_unlock(pPager); @@ -3280,34 +3401,6 @@ static int subjournalPage(PgHdr *pPg){ return rc; } -/* -** This function is a wrapper around sqlite3WalFrames(). As well as logging -** the contents of the list of pages headed by pList (connected by pDirty), -** this function notifies any active backup processes that the pages have -** changed. -*/ -static int pagerWalFrames( - Pager *pPager, /* Pager object */ - PgHdr *pList, /* List of frames to log */ - Pgno nTruncate, /* Database size after this commit */ - int isCommit, /* True if this is a commit */ - int sync_flags /* Flags to pass to OsSync() (or 0) */ -){ - int rc; /* Return code */ - - assert( pPager->pWal ); - rc = sqlite3WalFrames(pPager->pWal, - pPager->pageSize, pList, nTruncate, isCommit, sync_flags - ); - if( rc==SQLITE_OK && pPager->pBackup ){ - PgHdr *p; - for(p=pList; p; p=p->pDirty){ - sqlite3BackupUpdate(pPager->pBackup, p->pgno, (u8 *)p->pData); - } - } - return rc; -} - /* ** This function is called by the pcache layer when it has reached some ** soft memory limit. The first argument is a pointer to a Pager object @@ -3819,55 +3912,6 @@ static int hasHotJournal(Pager *pPager, int *pExists){ return rc; } -/* -** Check if the *-wal file that corresponds to the database opened by pPager -** exists. Assuming no error occurs, set *pExists to 1 if the file exists, -** or 0 otherwise and return SQLITE_OK. If an IO or OOM error occurs, return -** an SQLite error code. -** -** The caller must hold a SHARED lock on the database file to call this -** function. Because an EXCLUSIVE lock on the db file is required to delete -** a WAL, this ensures there is no race condition between the xAccess() -** below and an xDelete() being executed by some other connection. -*/ -static int pagerHasWAL(Pager *pPager, int *pExists){ - int rc; /* Return code */ - - if( !pPager->tempFile ){ - char *zWal = sqlite3_mprintf("%s-wal", pPager->zFilename); - if( !zWal ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3OsAccess(pPager->pVfs, zWal, SQLITE_ACCESS_EXISTS, pExists); - sqlite3_free(zWal); - } - }else{ - rc = SQLITE_OK; - *pExists = 0; - } - return rc; -} - -static int pagerOpenSnapshot(Pager *pPager){ - int rc; /* Return code */ - int changed = 0; /* True if cache must be reset */ - - assert( pagerUseWal(pPager) ); - - rc = sqlite3WalOpenSnapshot(pPager->pWal, &changed); - if( rc==SQLITE_OK ){ - int dummy; - if( changed ){ - pager_reset(pPager); - assert( pPager->errCode || pPager->dbSizeValid==0 ); - } - rc = sqlite3PagerPagecount(pPager, &dummy); - } - pPager->state = PAGER_SHARED; - - return rc; -} - /* ** This function is called to obtain a shared lock on the database file. ** It is illegal to call sqlite3PagerAcquire() until after this function @@ -3926,7 +3970,6 @@ int sqlite3PagerSharedLock(Pager *pPager){ }else if( pPager->state==PAGER_UNLOCK || isErrorReset ){ sqlite3_vfs * const pVfs = pPager->pVfs; int isHotJournal = 0; - int isWal = 0; assert( !MEMDB ); assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); if( pPager->noReadlock ){ @@ -4081,19 +4124,10 @@ int sqlite3PagerSharedLock(Pager *pPager){ } assert( pPager->exclusiveMode || pPager->state==PAGER_SHARED ); - rc = pagerHasWAL(pPager, &isWal); - if( rc!=SQLITE_OK ){ - goto failed; - } - if( isWal ){ - pager_reset(pPager); - rc = sqlite3PagerOpenWal(pPager, 0); - if( rc==SQLITE_OK ){ - rc = pagerOpenSnapshot(pPager); - } - }else if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){ - pPager->journalMode = PAGER_JOURNALMODE_DELETE; - } + /* If there is a WAL file in the file-system, open this database in WAL + ** mode. Otherwise, the following function call is a no-op. + */ + rc = pagerOpenWalIfPresent(pPager); } failed: @@ -5789,6 +5823,7 @@ sqlite3_backup **sqlite3PagerBackupPtr(Pager *pPager){ return &pPager->pBackup; } +#ifndef SQLITE_OMIT_WAL /* ** This function is called when the user invokes "PRAGMA checkpoint". */ @@ -5836,7 +5871,6 @@ int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen){ return rc; } - /* ** This function is called to close the connection to the log file prior ** to switching from WAL to rollback mode. @@ -5881,5 +5915,6 @@ int sqlite3PagerCloseWal(Pager *pPager){ } return rc; } +#endif #endif /* SQLITE_OMIT_DISKIO */ diff --git a/src/pragma.c b/src/pragma.c index da6a6807f5..263b7fc0cb 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -265,7 +265,10 @@ static const char *actionName(u8 action){ */ const char *sqlite3JournalModename(int eMode){ static char * const azModeName[] = { - "delete", "persist", "off", "truncate", "memory", "wal" + "delete", "persist", "off", "truncate", "memory" +#ifndef SQLITE_OMIT_WAL + , "wal" +#endif }; assert( PAGER_JOURNALMODE_DELETE==0 ); assert( PAGER_JOURNALMODE_PERSIST==1 ); @@ -1397,10 +1400,17 @@ void sqlite3Pragma( }else #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ +#ifndef SQLITE_OMIT_WAL + /* + ** PRAGMA [database.]checkpoint + ** + ** Checkpoint the database. + */ if( sqlite3StrICmp(zLeft, "checkpoint")==0 ){ sqlite3VdbeUsesBtree(v, iDb); sqlite3VdbeAddOp3(v, OP_Checkpoint, iDb, 0, 0); }else +#endif #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 54effce127..28268104ca 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -823,8 +823,10 @@ struct sqlite3 { void (*xRollbackCallback)(void*); /* Invoked at every commit. */ void *pUpdateArg; void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64); +#ifndef SQLITE_OMIT_WAL int (*xWalCallback)(void *, sqlite3 *, const char *, int); void *pWalArg; +#endif void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*); void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*); void *pCollNeededArg; diff --git a/src/test_config.c b/src/test_config.c index 8544ee5356..092d6a2228 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -499,6 +499,12 @@ Tcl_SetVar2(interp, "sqlite_options", "long_double", Tcl_SetVar2(interp, "sqlite_options", "vtab", "1", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_OMIT_WAL + Tcl_SetVar2(interp, "sqlite_options", "wal", "0", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "wal", "1", TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_OMIT_WSD Tcl_SetVar2(interp, "sqlite_options", "wsd", "0", TCL_GLOBAL_ONLY); #else diff --git a/src/vdbe.c b/src/vdbe.c index c6a38c5ba6..c040fb8d2e 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5186,6 +5186,7 @@ case OP_AggFinal: { break; } +#ifndef SQLITE_OMIT_WAL /* Opcode: Checkpoint P1 * * * * ** ** Checkpoint database P1. This is a no-op if P1 is not currently in @@ -5200,6 +5201,7 @@ case OP_Checkpoint: { rc = sqlite3PagerCheckpoint(sqlite3BtreePager(pBt)); break; }; +#endif /* Opcode: JournalMode P1 P2 P3 * * ** @@ -5234,6 +5236,8 @@ case OP_JournalMode: { /* out2-prerelease */ pBt = db->aDb[pOp->p1].pBt; pPager = sqlite3BtreePager(pBt); + +#ifndef SQLITE_OMIT_WAL zFilename = sqlite3PagerFilename(pPager); pVfs = sqlite3PagerVfs(pPager); @@ -5282,6 +5286,7 @@ case OP_JournalMode: { /* out2-prerelease */ } } } +#endif /* ifndef SQLITE_OMIT_WAL */ eNew = sqlite3PagerJournalMode(pPager, eNew); pOut = &aMem[pOp->p2]; diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 12100ce56b..5aec7ca0f6 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -306,9 +306,14 @@ void sqlite3_result_error_nomem(sqlite3_context *pCtx){ pCtx->s.db->mallocFailed = 1; } +/* +** This function is called after a transaction has been committed. It +** invokes callbacks registered with sqlite3_wal_hook() as required. +*/ static int doWalCallbacks(sqlite3 *db){ - int i; int rc = SQLITE_OK; +#ifndef SQLITE_OMIT_WAL + int i; for(i=0; inDb; i++){ Btree *pBt = db->aDb[i].pBt; if( pBt ){ @@ -320,6 +325,7 @@ static int doWalCallbacks(sqlite3 *db){ } } } +#endif return rc; } diff --git a/src/wal.c b/src/wal.c index d406383a28..146675ad6b 100644 --- a/src/wal.c +++ b/src/wal.c @@ -13,6 +13,8 @@ ** This file contains the implementation of a write-ahead log file used in ** "journal_mode=wal" mode. */ +#ifndef SQLITE_OMIT_WAL + #include "wal.h" @@ -1271,6 +1273,7 @@ int sqlite3WalCheckpoint( int isChanged = 0; /* True if a new wal-index header is loaded */ assert( pWal->lockState==SQLITE_SHM_UNLOCK ); + assert( pWal->pWiData==0 ); /* Get the CHECKPOINT lock */ do { @@ -1315,3 +1318,4 @@ int sqlite3WalCallback(Wal *pWal){ } return (int)ret; } +#endif /* #ifndef SQLITE_OMIT_WAL */ diff --git a/src/wal.h b/src/wal.h index 1c787d8dcd..fde28d4799 100644 --- a/src/wal.h +++ b/src/wal.h @@ -19,6 +19,22 @@ #include "sqliteInt.h" +#ifdef SQLITE_OMIT_WAL +# define sqlite3WalOpen(x,y,z) 0 +# define sqlite3WalClose(w,x,y,z) 0 +# define sqlite3WalOpenSnapshot(y,z) 0 +# define sqlite3WalCloseSnapshot(z) +# define sqlite3WalRead(w,x,y,z) 0 +# define sqlite3WalDbsize(y,z) +# define sqlite3WalWriteLock(y,z) 0 +# define sqlite3WalUndo(x,y,z) 0 +# define sqlite3WalSavepoint(z) 0 +# define sqlite3WalSavepointUndo(y,z) 0 +# define sqlite3WalFrames(u,v,w,x,y,z) 0 +# define sqlite3WalCheckpoint(u,v,w,x,y,z) 0 +# define sqlite3WalCallback(z) 0 +#else + /* Connection to a write-ahead log (WAL) file. ** There is one object of this type for each pager. */ @@ -79,4 +95,5 @@ int sqlite3WalCheckpoint( */ int sqlite3WalCallback(Wal *pWal); +#endif /* ifndef SQLITE_OMIT_WAL */ #endif /* _WAL_H_ */ diff --git a/test/corruptA.test b/test/corruptA.test index 5d5e6f49a2..58e14da1bd 100644 --- a/test/corruptA.test +++ b/test/corruptA.test @@ -37,9 +37,11 @@ integrity_check corruptA-1.2 db close file copy -force test.db test.db-template +set unreadable_version 02 +ifcapable wal { set unreadable_version 03 } do_test corruptA-2.1 { file copy -force test.db-template test.db - hexio_write test.db 19 03 ;# the read format number + hexio_write test.db 19 $unreadable_version ;# the read format number sqlite3 db test.db catchsql {SELECT * FROM t1} } {1 {file is encrypted or is not a database}} diff --git a/test/rdonly.test b/test/rdonly.test index cd611aeee5..e363497040 100644 --- a/test/rdonly.test +++ b/test/rdonly.test @@ -67,8 +67,10 @@ do_test rdonly-1.5 { # write-version is reloaded). This way, SQLite does not discover that # the database is read-only until after it is locked. # +set ro_version 02 +ifcapable wal { set ro_version 03 } do_test rdonly-1.6 { - hexio_write test.db 18 03 ; # write-version + hexio_write test.db 18 $ro_version ; # write-version hexio_write test.db 24 11223344 ; # change-counter catchsql { INSERT INTO t1 VALUES(2); diff --git a/test/wal.test b/test/wal.test index 017b47f40e..1f032cac32 100644 --- a/test/wal.test +++ b/test/wal.test @@ -17,6 +17,8 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/lock_common.tcl +ifcapable !wal {finish_test ; return } + proc reopen_db {} { catch { db close } file delete -force test.db test.db-wal test.db-wal-summary diff --git a/test/walbak.test b/test/walbak.test index 11a7040cdc..4028aa4d56 100644 --- a/test/walbak.test +++ b/test/walbak.test @@ -16,6 +16,8 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !wal {finish_test ; return } + proc log_file_size {nFrame pgsz} { expr {12 + ($pgsz+16)*$nFrame} } diff --git a/test/walcrash.test b/test/walcrash.test index ed2b292861..dc718b0d29 100644 --- a/test/walcrash.test +++ b/test/walcrash.test @@ -27,6 +27,8 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !wal {finish_test ; return } + db close set seed 0 diff --git a/test/walhook.test b/test/walhook.test index a33b43a005..49f07dda53 100644 --- a/test/walhook.test +++ b/test/walhook.test @@ -16,6 +16,8 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !wal {finish_test ; return } + proc sqlite3_wal {args} { eval sqlite3 $args [lindex $args 0] eval { diff --git a/test/walmode.test b/test/walmode.test index 5b3006cc78..a19a04efc2 100644 --- a/test/walmode.test +++ b/test/walmode.test @@ -16,6 +16,25 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +# If the library was compiled without WAL support, check that the +# "PRAGMA journal_mode=WAL" treats "WAL" as an unrecognized mode. +# +ifcapable !wal { + + do_test walmode-0.1 { + execsql { PRAGMA journal_mode = wal } + } {delete} + do_test walmode-0.2 { + execsql { PRAGMA main.journal_mode = wal } + } {delete} + do_test walmode-0.3 { + execsql { PRAGMA main.journal_mode } + } {delete} + + finish_test + return +} + do_test walmode-1.1 { set sqlite_sync_count 0 execsql { PRAGMA page_size = 1024 } diff --git a/test/walslow.test b/test/walslow.test index 73f93a48df..25e252bbb4 100644 --- a/test/walslow.test +++ b/test/walslow.test @@ -17,6 +17,8 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !wal {finish_test ; return } + proc reopen_db {} { catch { db close } file delete -force test.db test.db-wal diff --git a/test/walthread.test b/test/walthread.test index fa42af9c6a..54f3eb4f7d 100644 --- a/test/walthread.test +++ b/test/walthread.test @@ -18,6 +18,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/lock_common.tcl if {[run_thread_tests]==0} { finish_test ; return } +ifcapable !wal { finish_test ; return } set sqlite_walsummary_mmap_incr 64 From 77b95a379c610a1daa53d2e0151ebcf68df5db7f Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 1 May 2010 17:50:37 +0000 Subject: [PATCH 81/89] Rework mutexes on the SHM implemention for os_unix to avoid a deadlock during WAL recovery. FossilOrigin-Name: 1a0f69bef2c489e81a3d4b910b426972e9ed4054 --- manifest | 24 +++++++++++++++++------- manifest.uuid | 2 +- src/os_unix.c | 44 +++++++++++++++++++++++++++----------------- 3 files changed, 45 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index d6e1251d4a..75db7fed56 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,8 @@ -C Support\scompile-time\soption\sSQLITE_OMIT_WAL,\sfor\sbuilding\swithout\sWAL\ssupport. -D 2010-05-01T16:40:20 +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 + +C Rework\smutexes\son\sthe\sSHM\simplemention\sfor\sos_unix\sto\savoid\sa\sdeadlock\sduring\nWAL\srecovery. +D 2010-05-01T17:50:38 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -150,7 +153,7 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 -F src/os_unix.c 27ec47f27987b9f25bc55df5e458f7f380730f66 +F src/os_unix.c d402146bd3731575ff399d2ebcdc0867034e8451 F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 F src/pager.c e9c7fe979b32a3c5bf4216d8fbe1cf8beff8a1b8 F src/pager.h 934b598583a9d936bb13c37d62a2fe68ac48781c @@ -808,7 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P dcabc90ca588281fa7a290972ec8af6816932489 -R e813455665cf7f3ad3b5157515fc2fe7 -U dan -Z 955de6008d5a15476ae5c3900eb59d71 +P 9b230c43dfa112e3e1589f9775926807bd36b36e +R 70dfb91700ac677788e5556f2fcdd9f6 +U drh +Z aec6f98e0ebcce7f7132104408636399 +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.6 (GNU/Linux) + +iD8DBQFL3GnyoxKgR168RlERAm3bAJwJDWmdlrwEHnjKELzYU8/fifSwAQCeNiz2 +E59QAYMayqNW9OBp4QF0LdQ= +=o6hV +-----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index c4a52f55ba..c88f378a4b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b230c43dfa112e3e1589f9775926807bd36b36e \ No newline at end of file +1a0f69bef2c489e81a3d4b910b426972e9ed4054 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index b2c22c3652..caf22effe6 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4594,7 +4594,6 @@ struct unixShmFile { struct unixFileId fid; /* Unique file identifier */ sqlite3_mutex *mutex; /* Mutex to access this object */ sqlite3_mutex *mutexBuf; /* Mutex to access zBuf[] */ - sqlite3_mutex *mutexRecov; /* The RECOVER mutex */ char *zFilename; /* Name of the file */ int h; /* Open file descriptor */ int szMap; /* Size of the mapping of file into memory */ @@ -4633,7 +4632,6 @@ struct unixShm { u8 readLock; /* Which of the two read-lock states to use */ u8 hasMutex; /* True if holding the unixShmFile mutex */ u8 hasMutexBuf; /* True if holding pFile->mutexBuf */ - u8 hasMutexRecov; /* True if holding pFile->mutexRecov */ u8 sharedMask; /* Mask of shared locks held */ u8 exclMask; /* Mask of exclusive locks held */ #ifdef SQLITE_DEBUG @@ -4909,7 +4907,6 @@ static void unixShmPurge(void){ if( p->nRef==0 ){ if( p->mutex ) sqlite3_mutex_free(p->mutex); if( p->mutexBuf ) sqlite3_mutex_free(p->mutexBuf); - if( p->mutexRecov ) sqlite3_mutex_free(p->mutexRecov); if( p->h>=0 ) close(p->h); *pp = p->pNext; sqlite3_free(p); @@ -4979,11 +4976,6 @@ static int unixShmOpen( rc = SQLITE_NOMEM; goto shm_open_err; } - pFile->mutexRecov = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pFile->mutexRecov==0 ){ - rc = SQLITE_NOMEM; - goto shm_open_err; - } pFile->h = open(zName, O_RDWR|O_CREAT, 0664); if( pFile->h<0 ){ @@ -5124,8 +5116,19 @@ static int unixShmSize( ** whatever mapping size is convenient. ** ** *ppBuf is made to point to the memory which is a mapping of the -** underlying storage. This segment is locked. unixShmRelease() -** must be called to release the lock. +** underlying storage. A mutex is acquired to prevent other threads +** from running while *ppBuf is in use in order to prevent other threads +** remapping *ppBuf out from under this thread. The unixShmRelease() +** call will release the mutex. However, if the lock state is CHECKPOINT, +** the mutex is not acquired because CHECKPOINT will never remap the +** buffer. RECOVER might remap, though, so CHECKPOINT will acquire +** the mutex if and when it promotes to RECOVER. +** +** RECOVER needs to be atomic. The same mutex that prevents *ppBuf from +** being remapped also prevents more than one thread from being in +** RECOVER at a time. But, RECOVER sometimes wants to remap itself. +** To prevent RECOVER from losing its lock while remapping, the +** mutex is not released by unixShmRelease() when in RECOVER. ** ** *pNewMapSize is set to the size of the mapping. ** @@ -5142,7 +5145,7 @@ static int unixShmGet( unixShmFile *pFile = p->pFile; int rc = SQLITE_OK; - if( p->lockState!=SQLITE_SHM_CHECKPOINT ){ + if( p->lockState!=SQLITE_SHM_CHECKPOINT && p->hasMutexBuf==0 ){ sqlite3_mutex_enter(pFile->mutexBuf); p->hasMutexBuf = 1; } @@ -5170,10 +5173,16 @@ static int unixShmGet( /* ** Release the lock held on the shared memory segment to that other ** threads are free to resize it if necessary. +** +** If the lock is not currently held, this routine is a harmless no-op. +** +** If the shared-memory object is in lock state RECOVER, then we do not +** really want to release the lock, so in that case too, this routine +** is a no-op. */ static int unixShmRelease(sqlite3_shm *pSharedMem){ unixShm *p = (unixShm*)pSharedMem; - if( p->hasMutexBuf ){ + if( p->hasMutexBuf && p->lockState!=SQLITE_SHM_RECOVER ){ unixShmFile *pFile = p->pFile; sqlite3_mutex_leave(pFile->mutexBuf); p->hasMutexBuf = 0; @@ -5235,6 +5244,11 @@ static int unixShmLock( OSTRACE(("SHM-LOCK shmid-%d, pid-%d request %s->%s\n", p->id, getpid(), azLkName[p->lockState], azLkName[desiredLock])); + + if( desiredLock==SQLITE_SHM_RECOVER && !p->hasMutexBuf ){ + sqlite3_mutex_enter(pFile->mutexBuf); + p->hasMutexBuf = 1; + } sqlite3_mutex_enter(pFile->mutex); switch( desiredLock ){ case SQLITE_SHM_UNLOCK: { @@ -5268,7 +5282,6 @@ static int unixShmLock( }else{ assert( p->lockState==SQLITE_SHM_RECOVER ); unixShmUnlock(pFile, p, UNIX_SHM_MUTEX); - sqlite3_mutex_leave(pFile->mutexRecov); p->lockState = p->readLock; rc = SQLITE_OK; } @@ -5289,7 +5302,6 @@ static int unixShmLock( || p->lockState==SQLITE_SHM_RECOVER ); if( p->lockState==SQLITE_SHM_RECOVER ){ unixShmUnlock(pFile, p, UNIX_SHM_MUTEX); - sqlite3_mutex_leave(pFile->mutexRecov); p->lockState = SQLITE_SHM_CHECKPOINT; rc = SQLITE_OK; } @@ -5312,9 +5324,7 @@ static int unixShmLock( assert( p->lockState==SQLITE_SHM_READ || p->lockState==SQLITE_SHM_READ_FULL || p->lockState==SQLITE_SHM_CHECKPOINT ); - sqlite3_mutex_leave(pFile->mutex); - sqlite3_mutex_enter(pFile->mutexRecov); - sqlite3_mutex_enter(pFile->mutex); + assert( sqlite3_mutex_held(pFile->mutexBuf) ); rc = unixShmExclusiveLock(pFile, p, UNIX_SHM_MUTEX); if( rc==SQLITE_OK ){ p->lockState = SQLITE_SHM_RECOVER; From bee6f4ebe21126d1811b34dec26a67cde376dae0 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 1 May 2010 17:57:36 +0000 Subject: [PATCH 82/89] Define an invariant to guarantee deadlock-free operation of SHM in os_unix.c and check that invariant with assert() statements. FossilOrigin-Name: 6af2dca75b8139134ea394c1d71aefc6523f02e9 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/os_unix.c | 9 +++++++++ 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 75db7fed56..5a53af92b1 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 -C Rework\smutexes\son\sthe\sSHM\simplemention\sfor\sos_unix\sto\savoid\sa\sdeadlock\sduring\nWAL\srecovery. -D 2010-05-01T17:50:38 +C Define\san\sinvariant\sto\sguarantee\sdeadlock-free\soperation\sof\sSHM\sin\sos_unix.c\nand\scheck\sthat\sinvariant\swith\sassert()\sstatements. +D 2010-05-01T17:57:36 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -153,7 +153,7 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 -F src/os_unix.c d402146bd3731575ff399d2ebcdc0867034e8451 +F src/os_unix.c bb8b65d92a79b4b1311fad7cae8a697dea0b8838 F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 F src/pager.c e9c7fe979b32a3c5bf4216d8fbe1cf8beff8a1b8 F src/pager.h 934b598583a9d936bb13c37d62a2fe68ac48781c @@ -811,14 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 9b230c43dfa112e3e1589f9775926807bd36b36e -R 70dfb91700ac677788e5556f2fcdd9f6 +P 1a0f69bef2c489e81a3d4b910b426972e9ed4054 +R 9a63fdaeb48f811dfa07166dc0124f9e U drh -Z aec6f98e0ebcce7f7132104408636399 +Z 0ec029f9a2c3c35cece8a5840fe4c2a0 -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.6 (GNU/Linux) -iD8DBQFL3GnyoxKgR168RlERAm3bAJwJDWmdlrwEHnjKELzYU8/fifSwAQCeNiz2 -E59QAYMayqNW9OBp4QF0LdQ= -=o6hV +iD8DBQFL3GuToxKgR168RlERAruJAJ93UG+yjpg4IAjppTK82HawrL2+6ACfZrj+ +F2hwX359XH9Tj7T1X7t3awQ= +=0PFi -----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index c88f378a4b..1b197ea0b1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a0f69bef2c489e81a3d4b910b426972e9ed4054 \ No newline at end of file +6af2dca75b8139134ea394c1d71aefc6523f02e9 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index caf22effe6..e15d9d4166 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4589,6 +4589,12 @@ typedef struct unixShmFile unixShmFile; ** Either unixShmFile.mutex must be held or unixShmFile.nRef==0 and ** unixMutexHeld() is true when reading or writing any other field ** in this structure. +** +** To avoid deadlocks, mutex and mutexBuf are always released in the +** reverse order that they are acquired. mutexBuf is always acquired +** first and released last. This invariant is check by asserting +** sqlite3_mutex_notheld() on mutex whenever mutexBuf is acquired or +** released. */ struct unixShmFile { struct unixFileId fid; /* Unique file identifier */ @@ -5146,6 +5152,7 @@ static int unixShmGet( int rc = SQLITE_OK; if( p->lockState!=SQLITE_SHM_CHECKPOINT && p->hasMutexBuf==0 ){ + assert( sqlite3_mutex_notheld(pFile->mutex) ); sqlite3_mutex_enter(pFile->mutexBuf); p->hasMutexBuf = 1; } @@ -5184,6 +5191,7 @@ static int unixShmRelease(sqlite3_shm *pSharedMem){ unixShm *p = (unixShm*)pSharedMem; if( p->hasMutexBuf && p->lockState!=SQLITE_SHM_RECOVER ){ unixShmFile *pFile = p->pFile; + assert( sqlite3_mutex_notheld(pFile->mutex) ); sqlite3_mutex_leave(pFile->mutexBuf); p->hasMutexBuf = 0; } @@ -5246,6 +5254,7 @@ static int unixShmLock( p->id, getpid(), azLkName[p->lockState], azLkName[desiredLock])); if( desiredLock==SQLITE_SHM_RECOVER && !p->hasMutexBuf ){ + assert( sqlite3_mutex_notheld(pFile->mutex) ); sqlite3_mutex_enter(pFile->mutexBuf); p->hasMutexBuf = 1; } From f72a8751c98d2b9c99783a6c3adcff0e2c87090a Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 1 May 2010 18:23:08 +0000 Subject: [PATCH 83/89] Remove the xShmPush and xShmPull methods from the VFS. FossilOrigin-Name: 69e07fdbde5054593de948b76dfac157cf5d4058 --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/os_unix.c | 2 -- src/sqlite.h.in | 2 -- src/test_onefile.c | 2 -- 5 files changed, 12 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 5a53af92b1..0679f724b6 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 -C Define\san\sinvariant\sto\sguarantee\sdeadlock-free\soperation\sof\sSHM\sin\sos_unix.c\nand\scheck\sthat\sinvariant\swith\sassert()\sstatements. -D 2010-05-01T17:57:36 +C Remove\sthe\sxShmPush\sand\sxShmPull\smethods\sfrom\sthe\sVFS. +D 2010-05-01T18:23:09 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -153,7 +153,7 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 -F src/os_unix.c bb8b65d92a79b4b1311fad7cae8a697dea0b8838 +F src/os_unix.c d003631afe10f2cbbf0f4eb4ea72122e0522bbc7 F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 F src/pager.c e9c7fe979b32a3c5bf4216d8fbe1cf8beff8a1b8 F src/pager.h 934b598583a9d936bb13c37d62a2fe68ac48781c @@ -169,7 +169,7 @@ F src/resolve.c ac5f1a713cd1ae77f08b83cc69581e11bf5ae6f9 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c c03d8a0565febcde8c6a12c5d77d065fddae889b F src/shell.c c40427c7245535a04a9cb4a417b6cc05c022e6a4 -F src/sqlite.h.in 6e91727c0c3f9e1365e8fea2b07369a09830f47f +F src/sqlite.h.in fdf272c7bb095633d3f5f916cf7384c4faa505e5 F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89 F src/sqliteInt.h 3eb613c8ba7f5159be0acf3fc148d41ec49add86 F src/sqliteLimit.h 3afab2291762b5d09ae20c18feb8e9fa935a60a6 @@ -201,7 +201,7 @@ F src/test_journal.c 0537fcf9d4ecb394ec8b27d887a004ecaab85f16 F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 2842c922b8e8d992aba722214952204ca025b411 F src/test_mutex.c ce06b59aca168cd8c520b77159a24352a7469bd3 -F src/test_onefile.c 1d098e7446d89087def7d20e0cdb77a0713ed1bf +F src/test_onefile.c d9585f6e2056868f208b0c21378a05b68c9ceae2 F src/test_osinst.c f5d1a4ee8b80fc58d1430c56146de748584013a9 F src/test_pcache.c 7bf828972ac0d2403f5cfa4cd14da41f8ebe73d8 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 @@ -811,14 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 1a0f69bef2c489e81a3d4b910b426972e9ed4054 -R 9a63fdaeb48f811dfa07166dc0124f9e +P 6af2dca75b8139134ea394c1d71aefc6523f02e9 +R 9c99ea4eb3e50a401439b06a40f3fab6 U drh -Z 0ec029f9a2c3c35cece8a5840fe4c2a0 +Z cd290cf28dda9654fd31e687004b2a61 -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.6 (GNU/Linux) -iD8DBQFL3GuToxKgR168RlERAruJAJ93UG+yjpg4IAjppTK82HawrL2+6ACfZrj+ -F2hwX359XH9Tj7T1X7t3awQ= -=0PFi +iD8DBQFL3HGQoxKgR168RlERAlVgAJ49vhn/g4iu17//PQ8uRvfsTGbduwCfWA3t +4QBB9V7wP5NNoershsjx378= +=2IGM -----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index 1b197ea0b1..ee21ebcc71 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6af2dca75b8139134ea394c1d71aefc6523f02e9 \ No newline at end of file +69e07fdbde5054593de948b76dfac157cf5d4058 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index e15d9d4166..314ad9dd77 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -6587,8 +6587,6 @@ int sqlite3_os_init(void){ unixShmSize, /* xShmSize */ \ unixShmGet, /* xShmGet */ \ unixShmRelease, /* xShmRelease */ \ - 0, /* xShmPush */ \ - 0, /* xShmPull */ \ unixShmLock, /* xShmLock */ \ unixShmClose, /* xShmClose */ \ unixShmDelete, /* xShmDelete */ \ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index d2ce5396f2..175141227f 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -847,8 +847,6 @@ struct sqlite3_vfs { int (*xShmSize)(sqlite3_shm*, int reqSize, int *pNewSize); int (*xShmGet)(sqlite3_shm*, int reqMapSize, int *pMapSize, void**); int (*xShmRelease)(sqlite3_shm*); - int (*xShmPush)(sqlite3_shm*); - int (*xShmPull)(sqlite3_shm*); int (*xShmLock)(sqlite3_shm*, int desiredLock, int *gotLock); int (*xShmClose)(sqlite3_shm*); int (*xShmDelete)(sqlite3_vfs*, const char *zName); diff --git a/src/test_onefile.c b/src/test_onefile.c index fe02e016e7..6b39a39ad7 100644 --- a/src/test_onefile.c +++ b/src/test_onefile.c @@ -201,8 +201,6 @@ static fs_vfs_t fs_vfs = { fsCurrentTime, /* xCurrentTime */ 0, /* xShmOpen */ 0, /* xShmSize */ - 0, /* xShmPush */ - 0, /* xShmPull */ 0, /* xShmLock */ 0, /* xShmClose */ 0, /* xShmDelete */ From 2d536e187f6bac7d6756f22ca8d7e327bc64d1df Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 1 May 2010 20:17:30 +0000 Subject: [PATCH 84/89] Change the SHM interface so that it does not take the name of the shared object but rather the name of the WAL file and derives its own name from that. Remove the xShmDelete method from the VFS and replace it with a delete flag on xShmClose. FossilOrigin-Name: 94dea5f9c11a68564220cec76ac3fae2ba90e907 --- manifest | 24 +++++++++--------- manifest.uuid | 2 +- src/os_unix.c | 62 +++++++++++++++++++++++++++-------------------- src/sqlite.h.in | 3 +-- src/test_devsym.c | 12 +++------ src/wal.c | 19 ++++++--------- 6 files changed, 60 insertions(+), 62 deletions(-) diff --git a/manifest b/manifest index 0679f724b6..cd86dfff24 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1 -C Remove\sthe\sxShmPush\sand\sxShmPull\smethods\sfrom\sthe\sVFS. -D 2010-05-01T18:23:09 +C Change\sthe\sSHM\sinterface\sso\sthat\sit\sdoes\snot\stake\sthe\sname\sof\sthe\sshared\nobject\sbut\srather\sthe\sname\sof\sthe\sWAL\sfile\sand\sderives\sits\sown\sname\sfrom\nthat.\s\sRemove\sthe\sxShmDelete\smethod\sfrom\sthe\sVFS\sand\sreplace\sit\swith\sa\ndelete\sflag\son\sxShmClose. +D 2010-05-01T20:17:31 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -153,7 +153,7 @@ F src/os.c 8bc63cf91e9802e2b807198e54e50227fa889306 F src/os.h 534b082c3cb349ad05fa6fa0b06087e022af282c F src/os_common.h 0d6ee583b6ee3185eb9d951f890c6dd03021a08d F src/os_os2.c 8ad77a418630d7dee91d1bb04f79c2096301d3a0 -F src/os_unix.c d003631afe10f2cbbf0f4eb4ea72122e0522bbc7 +F src/os_unix.c 5c821e4eb04093102d901c22f40341bc1d30aef0 F src/os_win.c a8fc01d8483be472e495793c01064fd87e56a5c1 F src/pager.c e9c7fe979b32a3c5bf4216d8fbe1cf8beff8a1b8 F src/pager.h 934b598583a9d936bb13c37d62a2fe68ac48781c @@ -169,7 +169,7 @@ F src/resolve.c ac5f1a713cd1ae77f08b83cc69581e11bf5ae6f9 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c c03d8a0565febcde8c6a12c5d77d065fddae889b F src/shell.c c40427c7245535a04a9cb4a417b6cc05c022e6a4 -F src/sqlite.h.in fdf272c7bb095633d3f5f916cf7384c4faa505e5 +F src/sqlite.h.in 50df638fd31ae342d280582af8fc9d232b588dfe F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89 F src/sqliteInt.h 3eb613c8ba7f5159be0acf3fc148d41ec49add86 F src/sqliteLimit.h 3afab2291762b5d09ae20c18feb8e9fa935a60a6 @@ -191,7 +191,7 @@ F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de F src/test_btree.c 47cd771250f09cdc6e12dda5bc71bc0b3abc96e2 F src/test_config.c 6210f501d358bde619ae761f06f123529c6ba24f F src/test_demovfs.c da81a5f7785bb352bda7911c332a983ec4f17f27 -F src/test_devsym.c c05024cd179603181cde66d0ff83ce81e146f6de +F src/test_devsym.c 84b80966ee2aa72fdd0b42eaa90406478f390b03 F src/test_func.c 13b582345fb1185a93e46c53310fae8547dcce20 F src/test_hexio.c 1237f000ec7a491009b1233f5c626ea71bce1ea2 F src/test_init.c 5d624ffd0409d424cf9adbfe1f056b200270077c @@ -224,7 +224,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c 4a9b7fb028059992dfe51d23f0ced78064459c84 +F src/wal.c dadd533e0103883f72bf541eaf1ef34adb4e3c99 F src/wal.h d6d4a6809151e30bed5b01dd05cf27858f5a7bc8 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 75fee9e255b62f773fcadd1d1f25b6f63ac7a356 @@ -811,14 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 6af2dca75b8139134ea394c1d71aefc6523f02e9 -R 9c99ea4eb3e50a401439b06a40f3fab6 +P 69e07fdbde5054593de948b76dfac157cf5d4058 +R c9705329895ad0e28623e29f2274d0f1 U drh -Z cd290cf28dda9654fd31e687004b2a61 +Z 6445a2aee9bd8de55a45628ec7b23c47 -----BEGIN PGP SIGNATURE----- Version: GnuPG v1.4.6 (GNU/Linux) -iD8DBQFL3HGQoxKgR168RlERAlVgAJ49vhn/g4iu17//PQ8uRvfsTGbduwCfWA3t -4QBB9V7wP5NNoershsjx378= -=2IGM +iD8DBQFL3IxeoxKgR168RlERAtmWAJ9fbAccfEA031KC3xWSzcIzuGnUqQCfVHJO +hXR2b742paRdI5uwLVma/eQ= +=uRJQ -----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index ee21ebcc71..d26027c440 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -69e07fdbde5054593de948b76dfac157cf5d4058 \ No newline at end of file +94dea5f9c11a68564220cec76ac3fae2ba90e907 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 314ad9dd77..986963a184 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4923,7 +4923,15 @@ static void unixShmPurge(void){ } /* -** Open a shared-memory area. This implementation uses mmapped files. +** Open a shared-memory area. This particular implementation uses +** mmapped files. +** +** zName is a filename used to identify the shared-memory area. The +** implementation does not (and perhaps should not) use this name +** directly, but rather use it as a template for finding an appropriate +** name for the shared-memory storage. In this implementation, the +** string "-index" is appended to zName and used as the name of the +** mmapped file. ** ** When opening a new shared-memory file, if no other instances of that ** file are currently open, in this process or in other processes, then @@ -4931,25 +4939,38 @@ static void unixShmPurge(void){ */ static int unixShmOpen( sqlite3_vfs *pVfs, /* The VFS */ - const char *zName, /* Name of file to mmap */ + const char *zName, /* Base name of file to mmap */ sqlite3_shm **pShm /* Write the unixShm object created here */ ){ struct unixShm *p = 0; /* The connection to be opened */ struct unixShmFile *pFile = 0; /* The underlying mmapped file */ int rc; /* Result code */ struct unixFileId fid; /* Unix file identifier */ + struct unixShmFile *pNew; /* Newly allocated pFile */ struct stat sStat; /* Result from stat() an fstat() */ + int nName; /* Size of zName in bytes */ - /* Allocate space for the new sqlite3_shm object */ + /* Allocate space for the new sqlite3_shm object. Also speculatively + ** allocate space for a new unixShmFile and filename. + */ p = sqlite3_malloc( sizeof(*p) ); if( p==0 ) return SQLITE_NOMEM; memset(p, 0, sizeof(*p)); + nName = strlen(zName); + pNew = sqlite3_malloc( sizeof(*pFile) + nName + 10 ); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + goto shm_open_err; + } + memset(pNew, 0, sizeof(*pNew)); + pNew->zFilename = (char*)&pNew[1]; + sqlite3_snprintf(nName+10, pNew->zFilename, "%s-index", zName); /* Look to see if there is an existing unixShmFile that can be used. ** If no matching unixShmFile currently exists, create a new one. */ unixEnterMutex(); - rc = stat(zName, &sStat); + rc = stat(pNew->zFilename, &sStat); if( rc==0 ){ memset(&fid, 0, sizeof(fid)); fid.dev = sStat.st_dev; @@ -4958,16 +4979,11 @@ static int unixShmOpen( if( memcmp(&pFile->fid, &fid, sizeof(fid))==0 ) break; } } - if( pFile==0 ){ - int nName = strlen(zName); - pFile = sqlite3_malloc( sizeof(*pFile) + nName + 1 ); - if( pFile==0 ){ - rc = SQLITE_NOMEM; - goto shm_open_err; - } - memset(pFile, 0, sizeof(*pFile)); - pFile->zFilename = (char*)&pFile[1]; - memcpy(pFile->zFilename, zName, nName+1); + if( pFile ){ + sqlite3_free(pNew); + }else{ + pFile = pNew; + pNew = 0; pFile->h = -1; pFile->pNext = unixShmFileList; unixShmFileList = pFile; @@ -4983,7 +4999,7 @@ static int unixShmOpen( goto shm_open_err; } - pFile->h = open(zName, O_RDWR|O_CREAT, 0664); + pFile->h = open(pFile->zFilename, O_RDWR|O_CREAT, 0664); if( pFile->h<0 ){ rc = SQLITE_CANTOPEN_BKPT; goto shm_open_err; @@ -5033,15 +5049,17 @@ shm_open_err: unixShmPurge(); sqlite3_free(p); sqlite3_free(pFile); + sqlite3_free(pNew); *pShm = 0; unixLeaveMutex(); return rc; } /* -** Close a connectioon to shared-memory. +** Close a connection to shared-memory. Delete the underlying +** storage if deleteFlag is true. */ -static int unixShmClose(sqlite3_shm *pSharedMem){ +static int unixShmClose(sqlite3_shm *pSharedMem, int deleteFlag){ unixShm *p; /* The connection to be closed */ unixShmFile *pFile; /* The underlying shared-memory file */ unixShm **pp; /* For looping over sibling connections */ @@ -5069,6 +5087,7 @@ static int unixShmClose(sqlite3_shm *pSharedMem){ assert( pFile->nRef>0 ); pFile->nRef--; if( pFile->nRef==0 ){ + if( deleteFlag ) unlink(pFile->zFilename); unixShmPurge(); } unixLeaveMutex(); @@ -5348,13 +5367,6 @@ static int unixShmLock( return rc; } -/* -** Delete a shared-memory segment from the system. -*/ -static int unixShmDelete(sqlite3_vfs *pVfs, const char *zName){ - return pVfs->xDelete(pVfs, zName, 0); -} - #else # define unixShmOpen 0 # define unixShmSize 0 @@ -5362,7 +5374,6 @@ static int unixShmDelete(sqlite3_vfs *pVfs, const char *zName){ # define unixShmRelease 0 # define unixShmLock 0 # define unixShmClose 0 -# define unixShmDelete 0 #endif /* #ifndef SQLITE_OMIT_WAL */ /* @@ -6589,7 +6600,6 @@ int sqlite3_os_init(void){ unixShmRelease, /* xShmRelease */ \ unixShmLock, /* xShmLock */ \ unixShmClose, /* xShmClose */ \ - unixShmDelete, /* xShmDelete */ \ 0, /* xRename */ \ 0, /* xCurrentTimeInt64 */ \ } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 175141227f..60984d6e57 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -848,8 +848,7 @@ struct sqlite3_vfs { int (*xShmGet)(sqlite3_shm*, int reqMapSize, int *pMapSize, void**); int (*xShmRelease)(sqlite3_shm*); int (*xShmLock)(sqlite3_shm*, int desiredLock, int *gotLock); - int (*xShmClose)(sqlite3_shm*); - int (*xShmDelete)(sqlite3_vfs*, const char *zName); + int (*xShmClose)(sqlite3_shm*, int deleteFlag); int (*xRename)(sqlite3_vfs*, const char *zOld, const char *zNew, int dirSync); int (*xCurrentTimeInt64)(sqlite3_vfs*, sqlite3_int64*); /* diff --git a/src/test_devsym.c b/src/test_devsym.c index dff7b98803..6420731077 100644 --- a/src/test_devsym.c +++ b/src/test_devsym.c @@ -73,8 +73,7 @@ static int devsymShmSize(sqlite3_shm *, int , int *); static int devsymShmGet(sqlite3_shm *, int , int *, void **); static int devsymShmRelease(sqlite3_shm *); static int devsymShmLock(sqlite3_shm *, int , int *); -static int devsymShmClose(sqlite3_shm *); -static int devsymShmDelete(sqlite3_vfs *, const char *); +static int devsymShmClose(sqlite3_shm *, int); static sqlite3_vfs devsym_vfs = { 2, /* iVersion */ @@ -110,7 +109,6 @@ static sqlite3_vfs devsym_vfs = { 0, devsymShmLock, devsymShmClose, - devsymShmDelete, 0, 0, }; @@ -378,11 +376,8 @@ static int devsymShmRelease(sqlite3_shm *p){ static int devsymShmLock(sqlite3_shm *p, int desiredLock, int *gotLock){ return g.pVfs->xShmLock(p, desiredLock, gotLock); } -static int devsymShmClose(sqlite3_shm *p){ - return g.pVfs->xShmClose(p); -} -static int devsymShmDelete(sqlite3_vfs *pVfs, const char *zName){ - return g.pVfs->xShmDelete(g.pVfs, zName); +static int devsymShmClose(sqlite3_shm *p, int deleteFlag){ + return g.pVfs->xShmClose(p, deleteFlag); } /* @@ -400,7 +395,6 @@ void devsym_register(int iDeviceChar, int iSectorSize){ devsym_vfs.xShmRelease = (g.pVfs->xShmRelease ? devsymShmRelease : 0); devsym_vfs.xShmLock = (g.pVfs->xShmLock ? devsymShmLock : 0); devsym_vfs.xShmClose = (g.pVfs->xShmClose ? devsymShmClose : 0); - devsym_vfs.xShmDelete = (g.pVfs->xShmDelete ? devsymShmDelete : 0); sqlite3_vfs_register(&devsym_vfs, 0); } if( iDeviceChar>=0 ){ diff --git a/src/wal.c b/src/wal.c index 146675ad6b..071521001f 100644 --- a/src/wal.c +++ b/src/wal.c @@ -133,6 +133,7 @@ struct Wal { u8 lockState; /* SQLITE_SHM_xxxx constant showing lock state */ u8 readerType; /* SQLITE_SHM_READ or SQLITE_SHM_READ_FULL */ WalIndexHdr hdr; /* Wal-index for current snapshot */ + char *zName; /* Name of underlying storage */ }; @@ -590,24 +591,23 @@ int sqlite3WalOpen( /* Allocate an instance of struct Wal to return. */ *ppWal = 0; nWal = strlen(zDb); - pRet = (Wal*)sqlite3MallocZero(sizeof(Wal) + pVfs->szOsFile + nWal+11); + pRet = (Wal*)sqlite3MallocZero(sizeof(Wal) + pVfs->szOsFile + nWal+5); if( !pRet ) goto wal_open_out; pRet->pVfs = pVfs; pRet->pFd = (sqlite3_file *)&pRet[1]; - zWal = pVfs->szOsFile + (char*)pRet->pFd; - sqlite3_snprintf(nWal+11, zWal, "%s-wal-index", zDb); + pRet->zName = zWal = pVfs->szOsFile + (char*)pRet->pFd; + sqlite3_snprintf(nWal+5, zWal, "%s-wal", zDb); rc = pVfs->xShmOpen(pVfs, zWal, &pRet->pWIndex); if( rc ) goto wal_open_out; /* Open file handle on the write-ahead log file. */ - zWal[nWal+4] = 0; flags = (SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_MAIN_JOURNAL); rc = sqlite3OsOpen(pVfs, zWal, pRet->pFd, flags, &flags); wal_open_out: if( rc!=SQLITE_OK ){ if( pRet ){ - pVfs->xShmClose(pRet->pWIndex); + pVfs->xShmClose(pRet->pWIndex, 0); sqlite3OsClose(pRet->pFd); sqlite3_free(pRet); } @@ -805,15 +805,10 @@ int sqlite3WalClose( walIndexUnmap(pWal); } - pWal->pVfs->xShmClose(pWal->pWIndex); + pWal->pVfs->xShmClose(pWal->pWIndex, isDelete); sqlite3OsClose(pWal->pFd); if( isDelete ){ - int nWal; - char *zWal = &((char *)pWal->pFd)[pWal->pVfs->szOsFile]; - sqlite3OsDelete(pWal->pVfs, zWal, 0); - nWal = sqlite3Strlen30(zWal); - memcpy(&zWal[nWal], "-index", 7); - pWal->pVfs->xShmDelete(pWal->pVfs, zWal); + sqlite3OsDelete(pWal->pVfs, pWal->zName, 0); } sqlite3_free(pWal); } From 586b9c8a9470c36697f5ec04ef3533c9e26c5c89 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 3 May 2010 08:04:49 +0000 Subject: [PATCH 85/89] Add the sqlite3_wal_checkpoint() and sqlite3_wal_autocheckpoint() APIs. FossilOrigin-Name: 9803196dec85e3aa4105cc477e9cfe98d370e486 --- manifest | 36 +++++--------- manifest.uuid | 2 +- src/main.c | 112 ++++++++++++++++++++++++++++++++++++++++++-- src/pragma.c | 1 - src/sqlite.h.in | 10 ++++ src/sqliteInt.h | 2 + src/vdbe.c | 7 +-- test/backup.test | 1 + test/walthread.test | 24 ---------- 9 files changed, 136 insertions(+), 59 deletions(-) diff --git a/manifest b/manifest index 5a53af92b1..ee782bfe90 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,5 @@ ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA1 - -C Define\san\sinvariant\sto\sguarantee\sdeadlock-free\soperation\sof\sSHM\sin\sos_unix.c\nand\scheck\sthat\sinvariant\swith\sassert()\sstatements. -D 2010-05-01T17:57:36 +C Add\sthe\ssqlite3_wal_checkpoint()\sand\ssqlite3_wal_autocheckpoint()\sAPIs. +D 2010-05-03T08:04:49 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -134,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/main.c b672c4af58f033c54398999dec5db9d4b6d99331 +F src/main.c 7aab969e23fea0140bdc2cc6d2b8978e8d9b0090 F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 89d4ea8d5cdd55635cbaa48ad53132af6294cbb2 @@ -161,7 +158,7 @@ F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 F src/pcache1.c 6dc1871ce8ead9187161c370a58cd06c84221f76 -F src/pragma.c 3d4ba9c35438b296360dc09ebf86c417cf996667 +F src/pragma.c 97c6054d7867b8c5b13023d3dc566274057cd853 F src/prepare.c fd1398cb1da54385ba5bd68d93928f10d10a1d9c F src/printf.c 5f5b65a83e63f2096a541a340722a509fa0240a7 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -169,9 +166,9 @@ F src/resolve.c ac5f1a713cd1ae77f08b83cc69581e11bf5ae6f9 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c c03d8a0565febcde8c6a12c5d77d065fddae889b F src/shell.c c40427c7245535a04a9cb4a417b6cc05c022e6a4 -F src/sqlite.h.in 6e91727c0c3f9e1365e8fea2b07369a09830f47f +F src/sqlite.h.in dcf9cfc4e9c75d00b822d1a0d07f28cb970932bf F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89 -F src/sqliteInt.h 3eb613c8ba7f5159be0acf3fc148d41ec49add86 +F src/sqliteInt.h 1f81585797162269a0f18b77c815d552dad636c2 F src/sqliteLimit.h 3afab2291762b5d09ae20c18feb8e9fa935a60a6 F src/status.c 4df6fe7dce2d256130b905847c6c60055882bdbe F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -215,7 +212,7 @@ F src/update.c c0dc6b75ad28b76b619042d934f337b02acee208 F src/utf.c 1baeeac91707a4df97ccc6141ec0f808278af685 F src/util.c 32aebf04c10e51ad3977a928b7416bed671b620b F src/vacuum.c 8e7d9a6fd076774bb6e211cad335c7337948a164 -F src/vdbe.c b2e2b70e9518c6cc986e74eca99607700904f4bd +F src/vdbe.c 8be37a1b18786b5c026adcb2e9edc93e3a940885 F src/vdbe.h 471f6a3dcec4817ca33596fe7f6654d56c0e75f3 F src/vdbeInt.h 19ebc8c2a2e938340051ee65af3f377fb99102d1 F src/vdbeapi.c 810abe698db3ccaf67c5a8982f8cd4f31c6c902f @@ -256,7 +253,7 @@ F test/autoindex1.test ffb06a246e2c1f89cfbe3d93eca513c9e78d4063 F test/autovacuum.test 25f891bc343a8bf5d9229e2e9ddab9f31a9ab5ec F test/autovacuum_ioerr2.test 598b0663074d3673a9c1bc9a16e80971313bafe6 F test/avtrans.test 7a26ffc6a355b7fbff2a082f56180ede863b126c -F test/backup.test b1e874fa9b01de9dd5137a8371d060b76a435162 +F test/backup.test 5f7f66d067e695a6040be3295a5abc289f823838 F test/backup2.test 159419073d9769fdb1780ed7e5b391a046f898d5 F test/backup_ioerr.test 1f012e692f42c0442ae652443258f70e9f20fa38 F test/backup_malloc.test 1e063c6d75143d0d6e0ae77971dd690070369387 @@ -767,7 +764,7 @@ F test/walcrash.test 20ebd86c0ea1bfdec97987c3cf667118f5c74a84 F test/walhook.test 5ee5fd35cdc3763d5674da0dd9974f6f70c07bed F test/walmode.test bac6f06544a8554588a1543def996bbe2fc41792 F test/walslow.test 9be52d033e871cb608a18fa4425a6dc64b625377 -F test/walthread.test 16559527eb1f1a6e9eff21cc92f7fa51fb0b5a32 +F test/walthread.test 4ef06a22d0d207aecba8cd11add820f6e7e94c43 F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 F test/where2.test 43d4becaf5a5df854e6c21d624a1cb84c6904554 F test/where3.test aa44a9b29e8c9f3d7bb94a3bb3a95b31627d520d @@ -811,14 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 1a0f69bef2c489e81a3d4b910b426972e9ed4054 -R 9a63fdaeb48f811dfa07166dc0124f9e -U drh -Z 0ec029f9a2c3c35cece8a5840fe4c2a0 ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.6 (GNU/Linux) - -iD8DBQFL3GuToxKgR168RlERAruJAJ93UG+yjpg4IAjppTK82HawrL2+6ACfZrj+ -F2hwX359XH9Tj7T1X7t3awQ= -=0PFi ------END PGP SIGNATURE----- +P 6af2dca75b8139134ea394c1d71aefc6523f02e9 +R c94ad925a665396e6feea1ca0a681bf5 +U dan +Z 86d18456fd55c8daa32a6607f2d982a8 diff --git a/manifest.uuid b/manifest.uuid index 1b197ea0b1..c8b15e0043 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6af2dca75b8139134ea394c1d71aefc6523f02e9 \ No newline at end of file +9803196dec85e3aa4105cc477e9cfe98d370e486 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 57ea9736a3..c714e03fff 100644 --- a/src/main.c +++ b/src/main.c @@ -1186,6 +1186,42 @@ void *sqlite3_rollback_hook( return pRet; } +#ifndef SQLITE_OMIT_WAL +/* +** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint(). +** Return non-zero, indicating to the caller that a checkpoint should be run, +** if the number of frames in the log file is greater than +** sqlite3.nDefaultCheckpoint (the value configured by wal_autocheckpoint()). +*/ +static int defaultWalHook(void *p, sqlite3 *db, const char *z, int nFrame){ + UNUSED_PARAMETER(p); + UNUSED_PARAMETER(z); + return ( nFrame>=db->nDefaultCheckpoint ); +} + +/* +** Configure an sqlite3_wal_hook() callback to automatically checkpoint +** a database after committing a transaction if there are nFrame or +** more frames in the log file. Passing zero or a negative value as the +** nFrame parameter disables automatic checkpoints entirely. +** +** The callback registered by this function replaces any existing callback +** registered using sqlite3_wal_hook(). Likewise, registering a callback +** using sqlite3_wal_hook() disables the automatic checkpoint mechanism +** configured by this function. +*/ +int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ + sqlite3_mutex_enter(db->mutex); + if( nFrame>0 ){ + db->nDefaultCheckpoint = nFrame; + sqlite3_wal_hook(db, defaultWalHook, 0); + }else{ + sqlite3_wal_hook(db, 0, 0); + } + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + /* ** Register a callback to be invoked each time a transaction is written ** into the write-ahead-log by this database connection. @@ -1195,7 +1231,6 @@ void *sqlite3_wal_hook( int(*xCallback)(void *, sqlite3*, const char*, int), void *pArg /* First argument passed to xCallback() */ ){ -#ifndef SQLITE_OMIT_WAL void *pRet; sqlite3_mutex_enter(db->mutex); pRet = db->pWalArg; @@ -1203,11 +1238,78 @@ void *sqlite3_wal_hook( db->pWalArg = pArg; sqlite3_mutex_leave(db->mutex); return pRet; -#else - return 0; -#endif } +/* +** Checkpoint database zDb. If zDb is NULL, the main database is checkpointed. +*/ +int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ + int rc; /* Return code */ + int iDb = 0; /* sqlite3.aDb[] index of db to checkpoint */ + + sqlite3_mutex_enter(db->mutex); + if( zDb ){ + iDb = sqlite3FindDbName(db, zDb); + } + if( iDb<0 ){ + rc = SQLITE_ERROR; + }else{ + rc = sqlite3Checkpoint(db, iDb); + } + sqlite3_mutex_leave(db->mutex); + + return rc; +} + +/* +** Run a checkpoint on database iDb. This is a no-op if database iDb is +** not currently open in WAL mode. +** +** If a transaction is open at either the database handle (db) or b-tree +** level, this function returns SQLITE_LOCKED and a checkpoint is not +** attempted. If an error occurs while running the checkpoint, an SQLite +** error code is returned (i.e. SQLITE_IOERR). Otherwise, SQLITE_OK. +** +** The mutex on database handle db should be held by the caller. The mutex +** associated with the specific b-tree being checkpointed is taken by +** this function while the checkpoint is running. +*/ +int sqlite3Checkpoint(sqlite3 *db, int iDb){ + Btree *pBt; /* Btree handle to checkpoint */ + int rc; /* Return code */ + + assert( sqlite3_mutex_held(db->mutex) ); + + pBt = db->aDb[iDb].pBt; + if( sqlite3BtreeIsInReadTrans(pBt) ){ + rc = SQLITE_LOCKED; + }else{ + sqlite3BtreeEnter(pBt); + rc = sqlite3PagerCheckpoint(sqlite3BtreePager(pBt)); + sqlite3BtreeLeave(pBt); + } + + return rc; +} +#else /* ifndef SQLITE_OMIT_WAL */ +/* +** If SQLITE_OMIT_WAL is defined, the following API functions are no-ops: +** +** sqlite3_wal_hook() +** sqlite3_wal_checkpoint() +** sqlite3_wal_autocheckpoint() +*/ +void *sqlite3_wal_hook( + sqlite3 *x, + int(*xCallback)(void *, sqlite3*, const char*, int), + void *pArg +){ + return 0; +} +int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ return SQLITE_OK; } +int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ return SQLITE_OK; } +#endif + /* ** This function returns true if main-memory should be used instead of ** a temporary file for transient pager files and statement journals. @@ -1768,6 +1870,8 @@ static int openDatabase( setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, sqlite3GlobalConfig.nLookaside); + sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_CACHE_SIZE); + opendb_out: if( db ){ assert( db->mutex!=0 || isThreadsafe==0 || sqlite3GlobalConfig.bFullMutex==0 ); diff --git a/src/pragma.c b/src/pragma.c index 263b7fc0cb..7e4c0b3f6b 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1407,7 +1407,6 @@ void sqlite3Pragma( ** Checkpoint the database. */ if( sqlite3StrICmp(zLeft, "checkpoint")==0 ){ - sqlite3VdbeUsesBtree(v, iDb); sqlite3VdbeAddOp3(v, OP_Checkpoint, iDb, 0, 0); }else #endif diff --git a/src/sqlite.h.in b/src/sqlite.h.in index d2ce5396f2..21ad68ead7 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5797,6 +5797,16 @@ void *sqlite3_wal_hook( void* ); +/* +** CAPI3REF: Configure an auto-checkpoint +*/ +int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); + +/* +** CAPI3REF: Checkpoint a database +*/ +int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); + /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 28268104ca..ccc03432af 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -824,6 +824,7 @@ struct sqlite3 { void *pUpdateArg; void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64); #ifndef SQLITE_OMIT_WAL + int nDefaultCheckpoint; /* Value configured by wal_autocheckpoint() */ int (*xWalCallback)(void *, sqlite3 *, const char *, int); void *pWalArg; #endif @@ -2998,6 +2999,7 @@ CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *); int sqlite3TempInMemory(const sqlite3*); VTable *sqlite3GetVTable(sqlite3*, Table*); const char *sqlite3JournalModename(int); +int sqlite3Checkpoint(sqlite3*, int); /* Declarations for functions in fkey.c. All of these are replaced by ** no-op macros if OMIT_FOREIGN_KEY is defined. In this case no foreign diff --git a/src/vdbe.c b/src/vdbe.c index c040fb8d2e..11c723b74b 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5193,12 +5193,7 @@ case OP_AggFinal: { ** WAL mode. */ case OP_Checkpoint: { - Btree *pBt; /* Btree to checkpoint */ - - assert( pOp->p1>=0 && pOp->p1nDb ); - assert( (p->btreeMask & (1<p1))!=0 ); - pBt = db->aDb[pOp->p1].pBt; - rc = sqlite3PagerCheckpoint(sqlite3BtreePager(pBt)); + rc = sqlite3Checkpoint(db, pOp->p1); break; }; #endif diff --git a/test/backup.test b/test/backup.test index ca8de8ecca..97945da4f1 100644 --- a/test/backup.test +++ b/test/backup.test @@ -489,6 +489,7 @@ db2 close # 3) Backing up memory-to-file. # set iTest 0 +file delete -force bak.db-wal foreach {writer file} {db test.db db3 test.db db :memory:} { incr iTest catch { file delete bak.db } diff --git a/test/walthread.test b/test/walthread.test index 54f3eb4f7d..59f38d3e99 100644 --- a/test/walthread.test +++ b/test/walthread.test @@ -462,30 +462,6 @@ do_thread_test2 walthread-4 -seconds $seconds(walthread-4) -init { # very large wal file (one that requires a wal-index larger than the # initial default allocation of 64KB). # -# When recovering a log file, mutexes are usually obtained and released -# as follows: -# -# xShmGet ENTER mutexBuf -# xShmLock(RECOVER) ENTER mutexRecov -# -# xShmLock(READ) LEAVE mutexRecov -# xShmRelease LEAVE mutexBuf -# -# However, if the initial wal-index allocation needs to be enlarged during -# the recovery process, the mutex operations become: -# -# xShmGet ENTER mutexBuf -# xShmLock(RECOVER) ENTER mutexRecov -# -# xShmRelease LEAVE mutexBuf -# xShmSize -# xShmGet ENTER mutexBuf -# -# xShmLock(READ) LEAVE mutexRecov -# xShmRelease LEAVE mutexBuf -# -# If multiple threads attempt the above simultaneously, deadlock can occur. -# do_thread_test walthread-5 -seconds $seconds(walthread-5) -init { proc log_file_size {nFrame pgsz} { From 5a299f9134bb2f3aaa4412739f59be1a25219ca5 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 3 May 2010 11:05:08 +0000 Subject: [PATCH 86/89] Add the "PRAGMA wal_autocheckpoint" command. Rename "PRAGMA checkpoint" to "PRAGMA wal_checkpoint". FossilOrigin-Name: 714e5947264571386f966aa8fcdd5607b5832238 --- manifest | 34 +++++++++++------------ manifest.uuid | 2 +- src/main.c | 9 +++--- src/pragma.c | 21 ++++++++++++-- src/sqliteInt.h | 2 +- src/sqliteLimit.h | 8 ++++++ src/vdbeapi.c | 2 +- test/savepoint.test | 2 +- test/wal.test | 42 ++++++++++++++-------------- test/walbak.test | 4 +-- test/walcrash.test | 8 +++--- test/walhook.test | 67 +++++++++++++++++++++++++++++++++++---------- test/walslow.test | 2 +- test/walthread.test | 4 +-- 14 files changed, 135 insertions(+), 72 deletions(-) diff --git a/manifest b/manifest index e67d5cc39f..bfd14bcdd6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\stwo\swal\sleaves. -D 2010-05-03T08:19:34 +C Add\sthe\s"PRAGMA\swal_autocheckpoint"\scommand.\sRename\s"PRAGMA\scheckpoint"\sto\s"PRAGMA\swal_checkpoint". +D 2010-05-03T11:05:09 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/main.c 7aab969e23fea0140bdc2cc6d2b8978e8d9b0090 +F src/main.c d08f2f34e92e5ad7b5ab605a2d22d24e12567efa F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 89d4ea8d5cdd55635cbaa48ad53132af6294cbb2 @@ -158,7 +158,7 @@ F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 F src/pcache1.c 6dc1871ce8ead9187161c370a58cd06c84221f76 -F src/pragma.c 97c6054d7867b8c5b13023d3dc566274057cd853 +F src/pragma.c 4540555271b98366a408289318c7c1e508d7c8a6 F src/prepare.c fd1398cb1da54385ba5bd68d93928f10d10a1d9c F src/printf.c 5f5b65a83e63f2096a541a340722a509fa0240a7 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -168,8 +168,8 @@ F src/select.c c03d8a0565febcde8c6a12c5d77d065fddae889b F src/shell.c c40427c7245535a04a9cb4a417b6cc05c022e6a4 F src/sqlite.h.in 8097161ad63e2a00e26217da43327d396371c2e5 F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89 -F src/sqliteInt.h 1f81585797162269a0f18b77c815d552dad636c2 -F src/sqliteLimit.h 3afab2291762b5d09ae20c18feb8e9fa935a60a6 +F src/sqliteInt.h cb2b63a9538b9e79fc41b6d5026703600ab84407 +F src/sqliteLimit.h 196e2f83c3b444c4548fc1874f52f84fdbda40f3 F src/status.c 4df6fe7dce2d256130b905847c6c60055882bdbe F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 4de81521174fedacd8393ea7b70b730ce17f8eae @@ -215,7 +215,7 @@ F src/vacuum.c 8e7d9a6fd076774bb6e211cad335c7337948a164 F src/vdbe.c 8be37a1b18786b5c026adcb2e9edc93e3a940885 F src/vdbe.h 471f6a3dcec4817ca33596fe7f6654d56c0e75f3 F src/vdbeInt.h 19ebc8c2a2e938340051ee65af3f377fb99102d1 -F src/vdbeapi.c 810abe698db3ccaf67c5a8982f8cd4f31c6c902f +F src/vdbeapi.c cf1ff7cbcbd44d2082af71b98762407ae01ce009 F src/vdbeaux.c 395d21a1617553ea9b9f4abe0d302449d8549e1d F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 @@ -548,7 +548,7 @@ F test/rollback.test 73355ad4492ff9a3a31e61c7e5eb5e01a1de94ca F test/rowhash.test 97f56043ba11f0679920416c0cdbc72e5272267b F test/rowid.test e58e0acef38b527ed1b0b70d3ada588f804af287 F test/rtree.test 55466a200af3591946c5da77ad5dbfbc1e5e05f9 -F test/savepoint.test 500ef5afb383107fd3b7d055cdeb015d36d1bc76 +F test/savepoint.test 14f78c2c2538764850b38446957048113cc24f75 F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint3.test e328085853b14898d78ceea00dfe7db18bb6a9ec F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 @@ -758,13 +758,13 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test 4ff92f36495142420ce8c2bb45340dcad714a1e5 -F test/walbak.test fad10a10df78ab67e89a8779ddc3c0d230460693 -F test/walcrash.test 20ebd86c0ea1bfdec97987c3cf667118f5c74a84 -F test/walhook.test 5ee5fd35cdc3763d5674da0dd9974f6f70c07bed +F test/wal.test 9f142c65c436b209ffdce0eba37b1b90d2998c1a +F test/walbak.test a0e45187c7d8928df035dfea29b99b016b21ca3c +F test/walcrash.test 63edc6a9e05f645b54d649186a5818fc82953e2e +F test/walhook.test 5f18e0fc8787f1f8889d7a9971af18f334f83786 F test/walmode.test bac6f06544a8554588a1543def996bbe2fc41792 -F test/walslow.test 9be52d033e871cb608a18fa4425a6dc64b625377 -F test/walthread.test 4ef06a22d0d207aecba8cd11add820f6e7e94c43 +F test/walslow.test d21625e2e99e11c032ce949e8a94661576548933 +F test/walthread.test cc3c5302dfd56361def4023b7f29d361f1f1602d F test/where.test de337a3fe0a459ec7c93db16a519657a90552330 F test/where2.test 43d4becaf5a5df854e6c21d624a1cb84c6904554 F test/where3.test aa44a9b29e8c9f3d7bb94a3bb3a95b31627d520d @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 9803196dec85e3aa4105cc477e9cfe98d370e486 94dea5f9c11a68564220cec76ac3fae2ba90e907 -R 788341a84b9fc72cb664689f96e77759 +P 23c0e6c3f333e878fe5a2ae5d61df0d765f437b4 +R f8f9e7aa1d48de791ac4d31549cb2c66 U dan -Z 74b01ac5f69a8df35f22fcc654a97307 +Z 864d996940b21341339cd035e1e7f0cc diff --git a/manifest.uuid b/manifest.uuid index f4ae2cc983..3309d97d6c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -23c0e6c3f333e878fe5a2ae5d61df0d765f437b4 \ No newline at end of file +714e5947264571386f966aa8fcdd5607b5832238 \ No newline at end of file diff --git a/src/main.c b/src/main.c index c714e03fff..f6a343e85f 100644 --- a/src/main.c +++ b/src/main.c @@ -1191,12 +1191,12 @@ void *sqlite3_rollback_hook( ** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint(). ** Return non-zero, indicating to the caller that a checkpoint should be run, ** if the number of frames in the log file is greater than -** sqlite3.nDefaultCheckpoint (the value configured by wal_autocheckpoint()). +** sqlite3.nAutoCheckpoint (the value configured by wal_autocheckpoint()). */ static int defaultWalHook(void *p, sqlite3 *db, const char *z, int nFrame){ UNUSED_PARAMETER(p); UNUSED_PARAMETER(z); - return ( nFrame>=db->nDefaultCheckpoint ); + return ( nFrame>=db->nAutoCheckpoint ); } /* @@ -1213,8 +1213,8 @@ static int defaultWalHook(void *p, sqlite3 *db, const char *z, int nFrame){ int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ sqlite3_mutex_enter(db->mutex); if( nFrame>0 ){ - db->nDefaultCheckpoint = nFrame; sqlite3_wal_hook(db, defaultWalHook, 0); + db->nAutoCheckpoint = nFrame; }else{ sqlite3_wal_hook(db, 0, 0); } @@ -1236,6 +1236,7 @@ void *sqlite3_wal_hook( pRet = db->pWalArg; db->xWalCallback = xCallback; db->pWalArg = pArg; + db->nAutoCheckpoint = 0; sqlite3_mutex_leave(db->mutex); return pRet; } @@ -1870,7 +1871,7 @@ static int openDatabase( setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, sqlite3GlobalConfig.nLookaside); - sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_CACHE_SIZE); + sqlite3_wal_autocheckpoint(db, SQLITE_DEFAULT_WAL_AUTOCHECKPOINT); opendb_out: if( db ){ diff --git a/src/pragma.c b/src/pragma.c index 7e4c0b3f6b..0d2be03daf 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1402,13 +1402,30 @@ void sqlite3Pragma( #ifndef SQLITE_OMIT_WAL /* - ** PRAGMA [database.]checkpoint + ** PRAGMA [database.]wal_checkpoint ** ** Checkpoint the database. */ - if( sqlite3StrICmp(zLeft, "checkpoint")==0 ){ + if( sqlite3StrICmp(zLeft, "wal_checkpoint")==0 ){ + if( sqlite3ReadSchema(pParse) ) goto pragma_out; sqlite3VdbeAddOp3(v, OP_Checkpoint, iDb, 0, 0); }else + + /* + ** PRAGMA wal_autocheckpoint + ** PRAGMA wal_autocheckpoint = N + ** + ** Configure a database connection to automatically checkpoint a database + ** after accumulating N frames in the log. Or query for the current value + ** of N. + */ + if( sqlite3StrICmp(zLeft, "wal_autocheckpoint")==0 ){ + if( zRight ){ + int nAuto = atoi(zRight); + sqlite3_wal_autocheckpoint(db, nAuto); + } + returnSingleInt(pParse, "wal_autocheckpoint", db->nAutoCheckpoint); + }else #endif #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) diff --git a/src/sqliteInt.h b/src/sqliteInt.h index ccc03432af..c216816f6a 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -824,7 +824,7 @@ struct sqlite3 { void *pUpdateArg; void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64); #ifndef SQLITE_OMIT_WAL - int nDefaultCheckpoint; /* Value configured by wal_autocheckpoint() */ + int nAutoCheckpoint; /* Value configured by wal_autocheckpoint() */ int (*xWalCallback)(void *, sqlite3 *, const char *, int); void *pWalArg; #endif diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index faf80d4378..1dc5d675da 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -108,6 +108,14 @@ # define SQLITE_DEFAULT_TEMP_CACHE_SIZE 500 #endif +/* +** The default number of frames to accumulate in the log file before +** checkpointing the database in WAL mode. +*/ +#ifndef SQLITE_DEFAULT_WAL_AUTOCHECKPOINT +# define SQLITE_DEFAULT_WAL_AUTOCHECKPOINT 1000 +#endif + /* ** The maximum number of attached databases. This must be between 0 ** and 30. The upper bound on 30 is because a 32-bit integer bitmap diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 5aec7ca0f6..f16cc1be08 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -321,7 +321,7 @@ static int doWalCallbacks(sqlite3 *db){ if( db->xWalCallback && nEntry>0 && rc==SQLITE_OK && db->xWalCallback(db->pWalArg, db, db->aDb[i].zName, nEntry) ){ - rc = sqlite3PagerCheckpoint(sqlite3BtreePager(pBt)); + rc = sqlite3Checkpoint(db, i); } } } diff --git a/test/savepoint.test b/test/savepoint.test index bd38530836..ab198badde 100644 --- a/test/savepoint.test +++ b/test/savepoint.test @@ -792,7 +792,7 @@ do_test savepoint-11.6 { integrity_check savepoint-11.7 do_test savepoint-11.8 { execsql { ROLLBACK } - execsql { PRAGMA checkpoint } + execsql { PRAGMA wal_checkpoint } file size test.db } {8192} diff --git a/test/wal.test b/test/wal.test index 1f032cac32..e5c1d6cdc1 100644 --- a/test/wal.test +++ b/test/wal.test @@ -349,7 +349,7 @@ do_test wal-7.1 { list [file size test.db] [file size test.db-wal] } [list 1024 [log_file_size 3 1024]] do_test wal-7.2 { - execsql { PRAGMA checkpoint } + execsql { PRAGMA wal_checkpoint } list [file size test.db] [file size test.db-wal] } [list 2048 [log_file_size 3 1024]] @@ -380,14 +380,14 @@ do_test wal-8.2 { INSERT INTO t1 SELECT blob(900) FROM t1; /* 16 */ INSERT INTO t1 SELECT blob(900) FROM t1; /* 32 */ INSERT INTO t1 SELECT blob(900) FROM t1; /* 64 */ - PRAGMA checkpoint; + PRAGMA wal_checkpoint; } file size test.db } [expr 68*1024] do_test wal-8.3 { execsql { DELETE FROM t1 WHERE rowid<54; - PRAGMA checkpoint; + PRAGMA wal_checkpoint; } file size test.db } [expr 14*1024] @@ -426,7 +426,7 @@ do_test wal-9.3 { db3 close do_test wal-9.4 { - execsql { PRAGMA checkpoint } + execsql { PRAGMA wal_checkpoint } db2 close sqlite3_wal db2 test.db execsql {PRAGMA integrity_check } db2 @@ -549,7 +549,7 @@ foreach code [list { sql2 { BEGIN; SELECT * FROM t1 } } {1 2 3 4 5 6 7 8 9 10} do_test wal-10.$tn.12 { - catchsql { PRAGMA checkpoint } + catchsql { PRAGMA wal_checkpoint } } {1 {database is locked}} do_test wal-10.$tn.13 { execsql { INSERT INTO t1 VALUES(11, 12) } @@ -568,7 +568,7 @@ foreach code [list { } db busy busyhandler do_test wal-10.$tn.14 { - execsql { PRAGMA checkpoint } + execsql { PRAGMA wal_checkpoint } } {} # Similar to the test above. Except this time, a new read transaction is @@ -582,7 +582,7 @@ foreach code [list { sql2 { BEGIN; SELECT * FROM t1; } } {1 2 3 4 5 6 7 8 9 10 11 12} do_test wal-10.$tn.16 { - catchsql { PRAGMA checkpoint } + catchsql { PRAGMA wal_checkpoint } } {1 {database is locked}} proc busyhandler x { if {$x==3} { sql3 { BEGIN; SELECT * FROM t1 } } @@ -592,7 +592,7 @@ foreach code [list { } db busy busyhandler do_test wal-10.$tn.17 { - execsql { PRAGMA checkpoint } + execsql { PRAGMA wal_checkpoint } } {} do_test wal-10.$tn.18 { sql3 { SELECT * FROM t1 } @@ -616,13 +616,13 @@ foreach code [list { # (as [db3] still has a snapshot locked). # do_test wal-10.$tn.23 { - execsql { PRAGMA checkpoint } + execsql { PRAGMA wal_checkpoint } } {} do_test wal-10.$tn.24 { sql2 { BEGIN; SELECT * FROM t1; } } {1 2 3 4 5 6 7 8 9 10 11 12 13 14} do_test wal-10.$tn.25 { - execsql { PRAGMA checkpoint } + execsql { PRAGMA wal_checkpoint } } {} do_test wal-10.$tn.26 { catchsql { INSERT INTO t1 VALUES(15, 16) } @@ -641,11 +641,11 @@ foreach code [list { db busy {} do_test wal-10.$tn.29 { execsql { INSERT INTO t1 VALUES(17, 18) } - catchsql { PRAGMA checkpoint } + catchsql { PRAGMA wal_checkpoint } } {1 {database is locked}} do_test wal-10.$tn.30 { code3 { sqlite3_finalize $::STMT } - execsql { PRAGMA checkpoint } + execsql { PRAGMA wal_checkpoint } } {} # At one point, if a reader failed to upgrade to a writer because it @@ -685,7 +685,7 @@ foreach code [list { proc busyhandler x { return 1 } db busy busyhandler do_test wal-10.$tn.36 { - catchsql { PRAGMA checkpoint } + catchsql { PRAGMA wal_checkpoint } } {1 {database is locked}} do_test wal-10.$tn.36 { sql3 { INSERT INTO t1 VALUES('e', 'f') } @@ -693,7 +693,7 @@ foreach code [list { } {a b c d} do_test wal-10.$tn.37 { sql2 COMMIT - execsql { PRAGMA checkpoint } + execsql { PRAGMA wal_checkpoint } } {} catch { db close } @@ -718,7 +718,7 @@ do_test wal-11.1 { list [expr [file size test.db]/1024] [expr [file size test.db-wal]/1044] } {1 3} do_test wal-11.2 { - execsql { PRAGMA checkpoint } + execsql { PRAGMA wal_checkpoint } list [expr [file size test.db]/1024] [file size test.db-wal] } [list 3 [log_file_size 3 1024]] do_test wal-11.3 { @@ -753,7 +753,7 @@ do_test wal-11.7 { } } {16 ok} do_test wal-11.8 { - execsql { PRAGMA checkpoint } + execsql { PRAGMA wal_checkpoint } list [expr [file size test.db]/1024] [file size test.db-wal] } [list 37 [log_file_size 41 1024]] do_test wal-11.9 { @@ -829,11 +829,11 @@ do_test wal-12.4 { db2 close do_test wal-12.5 { execsql { - PRAGMA checkpoint; + PRAGMA wal_checkpoint; UPDATE t2 SET y = 2 WHERE x = 'B'; - PRAGMA checkpoint; + PRAGMA wal_checkpoint; UPDATE t1 SET y = 1 WHERE x = 'A'; - PRAGMA checkpoint; + PRAGMA wal_checkpoint; UPDATE t1 SET y = 0 WHERE x = 'A'; SELECT * FROM t2; } @@ -955,11 +955,11 @@ do_test wal-14 { INSERT INTO t1 SELECT randomblob(10), randomblob(100); } - # After executing the "PRAGMA checkpoint", connection [db] was being + # After executing the "PRAGMA wal_checkpoint", connection [db] was being # left with an inconsistent cache. Running the CREATE INDEX statement # in this state led to database corruption. catchsql { - PRAGMA checkpoint; + PRAGMA wal_checkpoint; CREATE INDEX i1 on t1(b); } diff --git a/test/walbak.test b/test/walbak.test index 4028aa4d56..63c1e2fa31 100644 --- a/test/walbak.test +++ b/test/walbak.test @@ -71,7 +71,7 @@ do_test walbak-1.5 { list [file size test.db] [file size test.db-wal] } [list 1024 [log_file_size 6 1024]] do_test walbak-1.6 { - execsql { PRAGMA checkpoint } + execsql { PRAGMA wal_checkpoint } list [file size test.db] [file size test.db-wal] } [list [expr 3*1024] [log_file_size 6 1024]] do_test walbak-1.7 { @@ -87,7 +87,7 @@ do_test walbak-1.8 { list [file size test.db] [file size test.db-wal] } [list [expr 3*1024] [log_file_size 8 1024]] do_test walbak-1.9 { - execsql { PRAGMA checkpoint } + execsql { PRAGMA wal_checkpoint } list [file size test.db] [file size test.db-wal] } [list [expr 2*1024] [log_file_size 8 1024]] diff --git a/test/walcrash.test b/test/walcrash.test index dc718b0d29..47f454c1e9 100644 --- a/test/walcrash.test +++ b/test/walcrash.test @@ -192,7 +192,7 @@ for {set i 1} {$i < $REPEATS} {incr i} { INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 28 */ INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 32 */ - PRAGMA checkpoint; + PRAGMA wal_checkpoint; INSERT INTO t1 VALUES(randomblob(900)); INSERT INTO t1 VALUES(randomblob(900)); INSERT INTO t1 VALUES(randomblob(900)); @@ -233,7 +233,7 @@ for {set i 1} {$i < $REPEATS} {incr i} { INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 28 */ INSERT INTO t1 SELECT randomblob(900) FROM t1 LIMIT 4; /* 32 */ - PRAGMA checkpoint; + PRAGMA wal_checkpoint; INSERT INTO t1 VALUES(randomblob(900)); INSERT INTO t1 VALUES(randomblob(900)); INSERT INTO t1 VALUES(randomblob(900)); @@ -260,9 +260,9 @@ for {set i 1} {$i < $REPEATS} {incr i} { CREATE TABLE t1(a, b); INSERT INTO t1 VALUES(1, 2); COMMIT; - PRAGMA checkpoint; + PRAGMA wal_checkpoint; CREATE INDEX i1 ON t1(a); - PRAGMA checkpoint; + PRAGMA wal_checkpoint; } } {1 {child process exited abnormally}} diff --git a/test/walhook.test b/test/walhook.test index 49f07dda53..05f543d2c8 100644 --- a/test/walhook.test +++ b/test/walhook.test @@ -11,6 +11,10 @@ # This file implements regression tests for SQLite library. The # focus of this file is testing the operation of the library in # "PRAGMA journal_mode=WAL" mode. +# +# More specifically, this file contains regression tests for the +# sqlite3_wal_hook() mechanism, including the sqlite3_wal_autocheckpoint() +# and "PRAGMA wal_autocheckpoint" convenience interfaces. # set testdir [file dirname $argv0] @@ -18,27 +22,27 @@ source $testdir/tester.tcl ifcapable !wal {finish_test ; return } -proc sqlite3_wal {args} { - eval sqlite3 $args - [lindex $args 0] eval { - PRAGMA journal_mode = wal; - PRAGMA synchronous = normal; - PRAGMA page_size = 1024; - } +proc log_file_size {nFrame pgsz} { + expr {12 + ($pgsz+16)*$nFrame} } -sqlite3_wal db test.db -db wal_hook wal_hook set ::wal_hook [list] proc wal_hook {zDb nEntry} { lappend ::wal_hook $zDb $nEntry return 0 } +db wal_hook wal_hook do_test walhook-1.1 { - execsql { CREATE TABLE t1(i PRIMARY KEY, j) } + execsql { + PRAGMA page_size = 1024; + PRAGMA journal_mode = wal; + PRAGMA synchronous = normal; + CREATE TABLE t1(i PRIMARY KEY, j); + } set ::wal_hook } {main 3} + do_test walhook-1.2 { set ::wal_hook [list] execsql { INSERT INTO t1 VALUES(1, 'one') } @@ -49,10 +53,9 @@ do_test walhook-1.3 { execsql { INSERT INTO t1 VALUES(2, 'two') } file size test.db } [expr 3*1024] - do_test walhook-1.4 { proc wal_hook {zDb nEntry} { - execsql { PRAGMA checkpoint } + execsql { PRAGMA wal_checkpoint } return 0 } execsql { CREATE TABLE t2(a, b) } @@ -60,15 +63,49 @@ do_test walhook-1.4 { } [expr 4*1024] do_test walhook-1.5 { - sqlite3_wal db2 test.db - proc wal_hook {zDb nEntry} { - execsql { PRAGMA checkpoint } db2 + sqlite3 db2 test.db + proc wal_hook {zDb nEntry} { + execsql { PRAGMA wal_checkpoint } db2 return 0 } execsql { CREATE TABLE t3(a PRIMARY KEY, b) } file size test.db } [expr 6*1024] +db2 close +db close +sqlite3 db test.db +do_test walhook-2.1 { + execsql { PRAGMA synchronous = NORMAL } + execsql { PRAGMA wal_autocheckpoint } +} {1000} +do_test walhook-2.2 { + execsql { PRAGMA wal_autocheckpoint = 10} +} {10} +do_test walhook-2.3 { + execsql { PRAGMA wal_autocheckpoint } +} {10} + +# +# The database connection is configured with "PRAGMA wal_autocheckpoint = 10". +# Check that transactions are written to the log file until it contains at +# least 10 frames, then the database is checkpointed. Subsequent transactions +# are written into the start of the log file. +# +foreach {tn sql dbpages logpages} { + 4 "CREATE TABLE t4(x PRIMARY KEY, y)" 6 3 + 5 "INSERT INTO t4 VALUES(1, 'one')" 6 5 + 6 "INSERT INTO t4 VALUES(2, 'two')" 6 7 + 7 "INSERT INTO t4 VALUES(3, 'three')" 6 9 + 8 "INSERT INTO t4 VALUES(4, 'four')" 8 11 + 9 "INSERT INTO t4 VALUES(5, 'five')" 8 11 +} { + do_test walhook-2.$tn { + execsql $sql + list [file size test.db] [file size test.db-wal] + } [list [expr $dbpages*1024] [log_file_size $logpages 1024]] +} + catch { db2 close } catch { db close } finish_test diff --git a/test/walslow.test b/test/walslow.test index 25e252bbb4..d726952804 100644 --- a/test/walslow.test +++ b/test/walslow.test @@ -48,7 +48,7 @@ for {set seed 1} {$seed<10} {incr seed} { } {ok} do_test walslow-1.seed=$seed.$iTest.2 { - execsql "PRAGMA checkpoint;" + execsql "PRAGMA wal_checkpoint;" execsql { PRAGMA integrity_check } } {ok} diff --git a/test/walthread.test b/test/walthread.test index 59f38d3e99..4aa1e2c2c3 100644 --- a/test/walthread.test +++ b/test/walthread.test @@ -241,7 +241,7 @@ proc do_thread_test2 {args} { # # There is also a single checkpointer thread. It runs the following loop: # -# 1) Execute "PRAGMA checkpoint" +# 1) Execute "PRAGMA wal_checkpoint" # 2) Sleep for 500 ms. # do_thread_test2 walthread-1 -seconds $seconds(walthread-1) -init { @@ -295,7 +295,7 @@ do_thread_test2 walthread-1 -seconds $seconds(walthread-1) -init { } -thread ckpt 1 { set nRun 0 while {[tt_continue]} { - db eval "PRAGMA checkpoint" + db eval "PRAGMA wal_checkpoint" usleep 500 incr nRun } From 87c1fe1b69aa55598c3ba5d8a0bb9cf76ed8dda1 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 3 May 2010 12:14:15 +0000 Subject: [PATCH 87/89] Have sqlite3_wal_checkpoint() populate the database handle error message and error code (as returned by sqlite3_errmsg() and sqlite3_errcode()). FossilOrigin-Name: ff234cf574c7ae384ab1ebc79b2171ef0541bc91 --- manifest | 16 +++++----- manifest.uuid | 2 +- src/main.c | 4 ++- src/test1.c | 30 +++++++++++++++++++ test/wal.test | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 124 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index bfd14bcdd6..4bc4e4a91f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"PRAGMA\swal_autocheckpoint"\scommand.\sRename\s"PRAGMA\scheckpoint"\sto\s"PRAGMA\swal_checkpoint". -D 2010-05-03T11:05:09 +C Have\ssqlite3_wal_checkpoint()\spopulate\sthe\sdatabase\shandle\serror\smessage\sand\serror\scode\s(as\sreturned\sby\ssqlite3_errmsg()\sand\ssqlite3_errcode()). +D 2010-05-03T12:14:16 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/main.c d08f2f34e92e5ad7b5ab605a2d22d24e12567efa +F src/main.c 1cc6f391018f2b868d812bf2b1be7c20a5269805 F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 89d4ea8d5cdd55635cbaa48ad53132af6294cbb2 @@ -173,7 +173,7 @@ F src/sqliteLimit.h 196e2f83c3b444c4548fc1874f52f84fdbda40f3 F src/status.c 4df6fe7dce2d256130b905847c6c60055882bdbe F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 4de81521174fedacd8393ea7b70b730ce17f8eae -F src/test1.c 8b4c246c41e75c3ff033edb2e8c2cf15820861ae +F src/test1.c ff95ca772d1df51618f9f1ef7ea432cdf851f97b F src/test2.c b6b43413d495addd039a88b87d65c839f86b18cb F src/test3.c 4c21700c73a890a47fc685c1097bfb661346ac94 F src/test4.c ad03bb987ddedce928f4258c1e7fa4109a73497d @@ -758,7 +758,7 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d -F test/wal.test 9f142c65c436b209ffdce0eba37b1b90d2998c1a +F test/wal.test 0c7c059c194871dd42811c31599cf74f1e1123e4 F test/walbak.test a0e45187c7d8928df035dfea29b99b016b21ca3c F test/walcrash.test 63edc6a9e05f645b54d649186a5818fc82953e2e F test/walhook.test 5f18e0fc8787f1f8889d7a9971af18f334f83786 @@ -808,7 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 23c0e6c3f333e878fe5a2ae5d61df0d765f437b4 -R f8f9e7aa1d48de791ac4d31549cb2c66 +P 714e5947264571386f966aa8fcdd5607b5832238 +R 10b5fe590305e038296dabbf1b136051 U dan -Z 864d996940b21341339cd035e1e7f0cc +Z 5bc5dc751e1eadc239ebcfefeac3d5fa diff --git a/manifest.uuid b/manifest.uuid index 3309d97d6c..170e46abc9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -714e5947264571386f966aa8fcdd5607b5832238 \ No newline at end of file +ff234cf574c7ae384ab1ebc79b2171ef0541bc91 \ No newline at end of file diff --git a/src/main.c b/src/main.c index f6a343e85f..2767878f8e 100644 --- a/src/main.c +++ b/src/main.c @@ -1254,12 +1254,14 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ } if( iDb<0 ){ rc = SQLITE_ERROR; + sqlite3Error(db, SQLITE_ERROR, "unknown database: %s", zDb); }else{ rc = sqlite3Checkpoint(db, iDb); + sqlite3Error(db, rc, 0); } sqlite3_mutex_leave(db->mutex); - return rc; + return sqlite3ApiExit(db, rc); } /* diff --git a/src/test1.c b/src/test1.c index b5fda952c0..0a886e9b44 100644 --- a/src/test1.c +++ b/src/test1.c @@ -4868,6 +4868,35 @@ static int test_unlock_notify( } #endif +/* +** tclcmd: sqlite3_wal_checkpoint db ?NAME? +*/ +static int test_wal_checkpoint( + ClientData clientData, /* Unused */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + char *zDb = 0; + sqlite3 *db; + int rc; + + if( objc!=3 && objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB ?NAME?"); + return TCL_ERROR; + } + + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + if( objc==3 ){ + zDb = Tcl_GetString(objv[2]); + } + rc = sqlite3_wal_checkpoint(db, zDb); + Tcl_SetResult(interp, (char *)t1ErrorName(rc), TCL_STATIC); + return TCL_OK; +} + /* ** tcl_objproc COMMANDNAME ARGS... @@ -5089,6 +5118,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY { "sqlite3_unlock_notify", test_unlock_notify, 0 }, #endif + { "sqlite3_wal_checkpoint", test_wal_checkpoint, 0 }, }; static int bitmask_size = sizeof(Bitmask)*8; int i; diff --git a/test/wal.test b/test/wal.test index e5c1d6cdc1..01e68e9437 100644 --- a/test/wal.test +++ b/test/wal.test @@ -968,4 +968,86 @@ do_test wal-14 { catch { db close } catch { db2 close } + +#------------------------------------------------------------------------- +# The following block of tests - wal-15.* - focus on testing the +# implementation of the sqlite3_wal_checkpoint() interface. +# +file delete -force test.db test.db-wal +sqlite3 db test.db +do_test wal-15.1 { + execsql { + PRAGMA page_size = 1024; + PRAGMA journal_mode = WAL; + } + execsql { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + } +} {} + +# Test that an error is returned if the database name is not recognized +# +do_test wal-15.2.1 { + sqlite3_wal_checkpoint db aux +} {SQLITE_ERROR} +do_test wal-15.2.2 { + sqlite3_errcode db +} {SQLITE_ERROR} +do_test wal-15.2.3 { + sqlite3_errmsg db +} {unknown database: aux} + +# Test that an error is returned if an attempt is made to checkpoint +# if a transaction is open on the database. +# +do_test wal-15.3.1 { + execsql { + BEGIN; + INSERT INTO t1 VALUES(3, 4); + } + sqlite3_wal_checkpoint db main +} {SQLITE_LOCKED} +do_test wal-15.3.2 { + sqlite3_errcode db +} {SQLITE_LOCKED} +do_test wal-15.3.3 { + sqlite3_errmsg db +} {database table is locked} + +# Also test that an error is returned if the db cannot be checkpointed +# because of locks held by another connection. +# +sqlite3 db2 test.db +do_test wal-15.4.1 { + execsql { + BEGIN; + SELECT * FROM t1; + } db2 +} {1 2} +do_test wal-15.4.2 { + execsql { COMMIT } + sqlite3_wal_checkpoint db +} {SQLITE_BUSY} +do_test wal-15.4.3 { + sqlite3_errmsg db +} {database is locked} + +# After [db2] drops its lock, [db] may checkpoint the db. +# +do_test wal-15.4.4 { + execsql { COMMIT } db2 + sqlite3_wal_checkpoint db +} {SQLITE_OK} +do_test wal-15.4.5 { + sqlite3_errmsg db +} {not an error} +do_test wal-15.4.6 { + file size test.db +} [expr 1024*2] + +catch { db2 close } +catch { db close } finish_test + + From b033d8b9dc764aea4d125942420222484711b172 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 May 2010 13:37:30 +0000 Subject: [PATCH 88/89] Make sure the mutex is held while calling sqlite3ApiExit() in sqlite3_wal_checkpoint(). Other cleanup of WAL logic. FossilOrigin-Name: 11a85b821abff1ecb7ec8c37bc7783be9fc4ea6d --- manifest | 28 +++++++++++++++++-------- manifest.uuid | 2 +- src/main.c | 55 ++++++++++++++++++++++++------------------------- src/pragma.c | 4 +++- src/sqliteInt.h | 2 +- 5 files changed, 51 insertions(+), 40 deletions(-) diff --git a/manifest b/manifest index 4bc4e4a91f..8ce641e0de 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,8 @@ -C Have\ssqlite3_wal_checkpoint()\spopulate\sthe\sdatabase\shandle\serror\smessage\sand\serror\scode\s(as\sreturned\sby\ssqlite3_errmsg()\sand\ssqlite3_errcode()). -D 2010-05-03T12:14:16 +-----BEGIN PGP SIGNED MESSAGE----- +Hash: SHA1 + +C Make\ssure\sthe\smutex\sis\sheld\swhile\scalling\ssqlite3ApiExit()\sin\s\nsqlite3_wal_checkpoint().\s\sOther\scleanup\sof\sWAL\slogic. +D 2010-05-03T13:37:30 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -131,7 +134,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/main.c 1cc6f391018f2b868d812bf2b1be7c20a5269805 +F src/main.c 0f66a2b0c2ce2c44ab4a5436e05064d39d8f5cbf F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 89d4ea8d5cdd55635cbaa48ad53132af6294cbb2 @@ -158,7 +161,7 @@ F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e F src/pcache.c ace8f6a5ecd4711cc66a1b23053be7109bd437cf F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050 F src/pcache1.c 6dc1871ce8ead9187161c370a58cd06c84221f76 -F src/pragma.c 4540555271b98366a408289318c7c1e508d7c8a6 +F src/pragma.c 5fa66bc24bd4e842d3edb86a319223b86ba5493c F src/prepare.c fd1398cb1da54385ba5bd68d93928f10d10a1d9c F src/printf.c 5f5b65a83e63f2096a541a340722a509fa0240a7 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -168,7 +171,7 @@ F src/select.c c03d8a0565febcde8c6a12c5d77d065fddae889b F src/shell.c c40427c7245535a04a9cb4a417b6cc05c022e6a4 F src/sqlite.h.in 8097161ad63e2a00e26217da43327d396371c2e5 F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89 -F src/sqliteInt.h cb2b63a9538b9e79fc41b6d5026703600ab84407 +F src/sqliteInt.h e7f1368eb56d8a410e510bcffb9ca5795060c58c F src/sqliteLimit.h 196e2f83c3b444c4548fc1874f52f84fdbda40f3 F src/status.c 4df6fe7dce2d256130b905847c6c60055882bdbe F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -808,7 +811,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 714e5947264571386f966aa8fcdd5607b5832238 -R 10b5fe590305e038296dabbf1b136051 -U dan -Z 5bc5dc751e1eadc239ebcfefeac3d5fa +P ff234cf574c7ae384ab1ebc79b2171ef0541bc91 +R e7545824307d5d5af98e4766ae2a26ad +U drh +Z 9a835b2804bf8190a366f2e988463519 +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.6 (GNU/Linux) + +iD8DBQFL3tGdoxKgR168RlERAkuOAJ96ZgqU19yVJ4CAdOh6a78a7JzGjwCfSnph +d7jKPMvbqcd69IRRUWbjMNM= +=vT7t +-----END PGP SIGNATURE----- diff --git a/manifest.uuid b/manifest.uuid index 170e46abc9..a1b1094a48 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ff234cf574c7ae384ab1ebc79b2171ef0541bc91 \ No newline at end of file +11a85b821abff1ecb7ec8c37bc7783be9fc4ea6d \ No newline at end of file diff --git a/src/main.c b/src/main.c index 2767878f8e..bb0f17c257 100644 --- a/src/main.c +++ b/src/main.c @@ -1191,13 +1191,19 @@ void *sqlite3_rollback_hook( ** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint(). ** Return non-zero, indicating to the caller that a checkpoint should be run, ** if the number of frames in the log file is greater than -** sqlite3.nAutoCheckpoint (the value configured by wal_autocheckpoint()). +** sqlite3.pWalArg cast to an integer (the value configured by +** wal_autocheckpoint()). */ -static int defaultWalHook(void *p, sqlite3 *db, const char *z, int nFrame){ - UNUSED_PARAMETER(p); - UNUSED_PARAMETER(z); - return ( nFrame>=db->nAutoCheckpoint ); +int sqlite3WalDefaultHook( + void *p, /* Argument */ + sqlite3 *db, /* Connection */ + const char *zNotUsed, /* Database */ + int nFrame /* Size of WAL */ +){ + UNUSED_PARAMETER(zNotUsed); + return ( nFrame>=SQLITE_PTR_TO_INT(p)); } +#endif /* SQLITE_OMIT_WAL */ /* ** Configure an sqlite3_wal_hook() callback to automatically checkpoint @@ -1211,14 +1217,15 @@ static int defaultWalHook(void *p, sqlite3 *db, const char *z, int nFrame){ ** configured by this function. */ int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ +#ifndef SQLITE_OMIT_WAL sqlite3_mutex_enter(db->mutex); if( nFrame>0 ){ - sqlite3_wal_hook(db, defaultWalHook, 0); - db->nAutoCheckpoint = nFrame; + sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); }else{ sqlite3_wal_hook(db, 0, 0); } sqlite3_mutex_leave(db->mutex); +#endif return SQLITE_OK; } @@ -1231,20 +1238,27 @@ void *sqlite3_wal_hook( int(*xCallback)(void *, sqlite3*, const char*, int), void *pArg /* First argument passed to xCallback() */ ){ +#ifndef SQLITE_OMIT_WAL void *pRet; sqlite3_mutex_enter(db->mutex); pRet = db->pWalArg; db->xWalCallback = xCallback; db->pWalArg = pArg; - db->nAutoCheckpoint = 0; sqlite3_mutex_leave(db->mutex); return pRet; +#else + return 0; +#endif } + /* ** Checkpoint database zDb. If zDb is NULL, the main database is checkpointed. */ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ +#ifdef SQLITE_OMIT_WAL + return SQLITE_OK; +#else int rc; /* Return code */ int iDb = 0; /* sqlite3.aDb[] index of db to checkpoint */ @@ -1259,11 +1273,13 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ rc = sqlite3Checkpoint(db, iDb); sqlite3Error(db, rc, 0); } + rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); - - return sqlite3ApiExit(db, rc); + return rc; +#endif } +#ifndef SQLITE_OMIT_WAL /* ** Run a checkpoint on database iDb. This is a no-op if database iDb is ** not currently open in WAL mode. @@ -1294,24 +1310,7 @@ int sqlite3Checkpoint(sqlite3 *db, int iDb){ return rc; } -#else /* ifndef SQLITE_OMIT_WAL */ -/* -** If SQLITE_OMIT_WAL is defined, the following API functions are no-ops: -** -** sqlite3_wal_hook() -** sqlite3_wal_checkpoint() -** sqlite3_wal_autocheckpoint() -*/ -void *sqlite3_wal_hook( - sqlite3 *x, - int(*xCallback)(void *, sqlite3*, const char*, int), - void *pArg -){ - return 0; -} -int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ return SQLITE_OK; } -int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ return SQLITE_OK; } -#endif +#endif /* SQLITE_OMIT_WAL */ /* ** This function returns true if main-memory should be used instead of diff --git a/src/pragma.c b/src/pragma.c index 0d2be03daf..65a0550bcb 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1424,7 +1424,9 @@ void sqlite3Pragma( int nAuto = atoi(zRight); sqlite3_wal_autocheckpoint(db, nAuto); } - returnSingleInt(pParse, "wal_autocheckpoint", db->nAutoCheckpoint); + returnSingleInt(pParse, "wal_autocheckpoint", + db->xWalCallback==sqlite3WalDefaultHook ? + SQLITE_PTR_TO_INT(db->pWalArg) : 0); }else #endif diff --git a/src/sqliteInt.h b/src/sqliteInt.h index c216816f6a..842917a9b1 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -824,7 +824,6 @@ struct sqlite3 { void *pUpdateArg; void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64); #ifndef SQLITE_OMIT_WAL - int nAutoCheckpoint; /* Value configured by wal_autocheckpoint() */ int (*xWalCallback)(void *, sqlite3 *, const char *, int); void *pWalArg; #endif @@ -3000,6 +2999,7 @@ int sqlite3TempInMemory(const sqlite3*); VTable *sqlite3GetVTable(sqlite3*, Table*); const char *sqlite3JournalModename(int); int sqlite3Checkpoint(sqlite3*, int); +int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int); /* Declarations for functions in fkey.c. All of these are replaced by ** no-op macros if OMIT_FOREIGN_KEY is defined. In this case no foreign From 6fcff07f6a2d1f5ec7256f44eb66aa3abaaf8b30 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 3 May 2010 14:05:43 +0000 Subject: [PATCH 89/89] If the sqlite3_wal_checkpoint() API is passed a NULL pointer in place of a database name, attempt to checkpoint all attached databases. FossilOrigin-Name: 27a5c09ce8a35039d844d08cfe5698e8b438abfe --- manifest | 24 +++++++----------------- manifest.uuid | 2 +- src/main.c | 38 ++++++++++++++++++++++++-------------- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/manifest b/manifest index 8ce641e0de..cbf0c5ef1a 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,5 @@ ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA1 - -C Make\ssure\sthe\smutex\sis\sheld\swhile\scalling\ssqlite3ApiExit()\sin\s\nsqlite3_wal_checkpoint().\s\sOther\scleanup\sof\sWAL\slogic. -D 2010-05-03T13:37:30 +C If\sthe\ssqlite3_wal_checkpoint()\sAPI\sis\spassed\sa\sNULL\spointer\sin\splace\sof\sa\sdatabase\sname,\sattempt\sto\scheckpoint\sall\sattached\sdatabases. +D 2010-05-03T14:05:43 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in d83a0ffef3dcbfb08b410a6c6dd6c009ec9167fb F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -134,7 +131,7 @@ F src/journal.c b0ea6b70b532961118ab70301c00a33089f9315c F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e F src/loadext.c 1c7a61ce1281041f437333f366a96aa0d29bb581 -F src/main.c 0f66a2b0c2ce2c44ab4a5436e05064d39d8f5cbf +F src/main.c 6bc746f6016e0a3e6a7089078d9feffc0aa24a5e F src/malloc.c a08f16d134f0bfab6b20c3cd142ebf3e58235a6a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 89d4ea8d5cdd55635cbaa48ad53132af6294cbb2 @@ -811,14 +808,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P ff234cf574c7ae384ab1ebc79b2171ef0541bc91 -R e7545824307d5d5af98e4766ae2a26ad -U drh -Z 9a835b2804bf8190a366f2e988463519 ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.6 (GNU/Linux) - -iD8DBQFL3tGdoxKgR168RlERAkuOAJ96ZgqU19yVJ4CAdOh6a78a7JzGjwCfSnph -d7jKPMvbqcd69IRRUWbjMNM= -=vT7t ------END PGP SIGNATURE----- +P 11a85b821abff1ecb7ec8c37bc7783be9fc4ea6d +R 496e930fc5a775a552ace6c40959352e +U dan +Z 225bbd40c88bfb84bfc467a361edc728 diff --git a/manifest.uuid b/manifest.uuid index a1b1094a48..c42491a7d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -11a85b821abff1ecb7ec8c37bc7783be9fc4ea6d \ No newline at end of file +27a5c09ce8a35039d844d08cfe5698e8b438abfe \ No newline at end of file diff --git a/src/main.c b/src/main.c index bb0f17c257..77a790d463 100644 --- a/src/main.c +++ b/src/main.c @@ -1260,7 +1260,7 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ return SQLITE_OK; #else int rc; /* Return code */ - int iDb = 0; /* sqlite3.aDb[] index of db to checkpoint */ + int iDb = SQLITE_MAX_ATTACHED; /* sqlite3.aDb[] index of db to checkpoint */ sqlite3_mutex_enter(db->mutex); if( zDb ){ @@ -1284,28 +1284,38 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ ** Run a checkpoint on database iDb. This is a no-op if database iDb is ** not currently open in WAL mode. ** -** If a transaction is open at either the database handle (db) or b-tree -** level, this function returns SQLITE_LOCKED and a checkpoint is not -** attempted. If an error occurs while running the checkpoint, an SQLite -** error code is returned (i.e. SQLITE_IOERR). Otherwise, SQLITE_OK. +** If a transaction is open on the database being checkpointed, this +** function returns SQLITE_LOCKED and a checkpoint is not attempted. If +** an error occurs while running the checkpoint, an SQLite error code is +** returned (i.e. SQLITE_IOERR). Otherwise, SQLITE_OK. ** ** The mutex on database handle db should be held by the caller. The mutex ** associated with the specific b-tree being checkpointed is taken by ** this function while the checkpoint is running. +** +** If iDb is passed SQLITE_MAX_ATTACHED, then all attached databases are +** checkpointed. If an error is encountered it is returned immediately - +** no attempt is made to checkpoint any remaining databases. */ int sqlite3Checkpoint(sqlite3 *db, int iDb){ - Btree *pBt; /* Btree handle to checkpoint */ - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ + int i; /* Used to iterate through attached dbs */ assert( sqlite3_mutex_held(db->mutex) ); - pBt = db->aDb[iDb].pBt; - if( sqlite3BtreeIsInReadTrans(pBt) ){ - rc = SQLITE_LOCKED; - }else{ - sqlite3BtreeEnter(pBt); - rc = sqlite3PagerCheckpoint(sqlite3BtreePager(pBt)); - sqlite3BtreeLeave(pBt); + for(i=0; inDb && rc==SQLITE_OK; i++){ + if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + if( sqlite3BtreeIsInReadTrans(pBt) ){ + rc = SQLITE_LOCKED; + }else{ + sqlite3BtreeEnter(pBt); + rc = sqlite3PagerCheckpoint(sqlite3BtreePager(pBt)); + sqlite3BtreeLeave(pBt); + } + } + } } return rc;