From b2d3de3bf47b7eaf5bb6bf68de5946895bd943af Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 14 Mar 2013 18:34:37 +0000 Subject: [PATCH 001/204] Use mmap() to read from the database file in rollback mode. This branch is unix only for now. FossilOrigin-Name: 6f21d9cbf5d457e63a7282015a89ae785526cf6d --- manifest | 25 ++++---- manifest.uuid | 2 +- src/btree.c | 31 ++++++++++ src/os_unix.c | 4 ++ src/pager.c | 129 ++++++++++++++++++++++++++++++++++++++++- src/pcache.h | 2 + src/sqlite.h.in | 1 + test/permutations.test | 3 +- 8 files changed, 182 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 42eb32ea0f..a61f43717c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\stests\sfor\sticket\s[4dd95f6943]. -D 2013-03-13T07:02:04.083 +C Use\smmap()\sto\sread\sfrom\sthe\sdatabase\sfile\sin\srollback\smode.\sThis\sbranch\sis\sunix\sonly\sfor\snow. +D 2013-03-14T18:34:37.796 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 746c4dafae8565b3be6fb9ce3bb1fa9f1e67cc59 +F src/btree.c c1a956c6762f2a45188c945e1070daec29f5253f F src/btree.h 3ad7964d6c5b1c7bff569aab6adfa075f8bf06cd F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 375e5df716e03b9343c5e1211be3b24e6d6dff05 @@ -160,13 +160,13 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c f6387eef0cf8f6b808738f4f3aa47e6132af0940 +F src/os_unix.c 2a4cd96aabf413f39cf562baebb27aa9993f6f54 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 582f8da52d0bd4b43d3bdaeba0ea7702c1f23702 +F src/pager.c 4e7e66c2959ee43caf8a7000712d6a4121b0888a F src/pager.h 1109a06578ec5574dc2c74cf8d9f69daf36fe3e0 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 -F src/pcache.h 1b5dcc3dc8103d03e625b177023ee67764fa6b7c +F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 F src/pragma.c 9f0ee3d74a7f33eeeff40a4b014fc3abf8182ce2 F src/prepare.c 78cd7ae8cd0d8de8ef8a8b5352fc5f38693a0852 @@ -176,7 +176,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in f2fa32f440dda59ca47e22889966b2a6eb3b491c +F src/sqlite.h.in 9a5c737a1feb4495d351c56883587d4fda52e81e F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 0f8f05ee4db4ba9120b38f7a3992b325698f6e8a @@ -656,7 +656,7 @@ F test/pageropt.test 9191867ed19a2b3db6c42d1b36b6fbc657cd1ab0 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test 360b92859c0af814b3fe10b68746936389606501 +F test/permutations.test eb49937dca270b2c3f62d4c91fc7034ca905b7f1 F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -1038,7 +1038,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 839aa91faf1db7025d90fa3c65e50efb829b053b -R ad71abb6cd4a6ebc6857153126653bad +P 0b452734faa0839c817f040322e7733e423bfce2 +R d4c281f6a900c9b41816881ffb3c2331 +T *branch * experimental-mmap +T *sym-experimental-mmap * +T -sym-trunk * U dan -Z 69b39b8becc09fc4758d1b797bf5f4f2 +Z f16a594407ec9e95c26319dfbc751443 diff --git a/manifest.uuid b/manifest.uuid index 3d8be2e93e..dbbf6a71bc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0b452734faa0839c817f040322e7733e423bfce2 \ No newline at end of file +6f21d9cbf5d457e63a7282015a89ae785526cf6d \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 07ec3fe52a..038126c3f0 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2564,6 +2564,34 @@ int sqlite3BtreeNewDb(Btree *p){ return rc; } +/* +** If the shared-btree passed as the only argument is holding references +** to mmap pages, replace them with read/write pages. Return SQLITE_OK +** if successful, or an error code otherwise. +*/ +static int btreeSwapOutMmap(BtShared *pBt){ + BtCursor *pCsr; + for(pCsr=pBt->pCursor; pCsr; pCsr=pCsr->pNext){ + int i; + for(i=0; i<=pCsr->iPage; i++){ + MemPage *pPg = pCsr->apPage[i]; + if( pPg->pDbPage->flags & PGHDR_MMAP ){ + int rc; + MemPage *pNew = 0; + rc = btreeGetPage(pBt, pPg->pgno, &pNew, 0); + if( rc==SQLITE_OK && i==pCsr->iPage ){ + pCsr->info.pCell = pNew->aData + (pCsr->info.pCell - pPg->aData); + } + pCsr->apPage[i] = pNew; + releasePage(pPg); + if( rc!=SQLITE_OK ) return rc; + } + } + } + + return SQLITE_OK; +} + /* ** Attempt to start a new transaction. A write-transaction ** is started if the second argument is nonzero, otherwise a read- @@ -2670,6 +2698,9 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ rc = SQLITE_READONLY; }else{ rc = sqlite3PagerBegin(pBt->pPager,wrflag>1,sqlite3TempInMemory(p->db)); + if( rc==SQLITE_OK ){ + rc = btreeSwapOutMmap(pBt); + } if( rc==SQLITE_OK ){ rc = newDatabase(pBt); } diff --git a/src/os_unix.c b/src/os_unix.c index 8f094bdc19..89326783d7 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3623,6 +3623,10 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } + case SQLITE_FCNTL_GETFD: { + *(int*)pArg = pFile->h; + return SQLITE_OK; + } #ifdef SQLITE_DEBUG /* The pager calls this method to signal that it has done ** a rollback and that the database is therefore unchanged and diff --git a/src/pager.c b/src/pager.c index 1d84fa2b7e..c362494093 100644 --- a/src/pager.c +++ b/src/pager.c @@ -655,6 +655,11 @@ struct Pager { PagerSavepoint *aSavepoint; /* Array of active savepoints */ int nSavepoint; /* Number of elements in aSavepoint[] */ char dbFileVers[16]; /* Changes whenever database file changes */ + + void *pMap; /* Memory mapped prefix of database file */ + i64 nMap; /* Size of mapping at pMap in bytes */ + int nMmapOut; /* Number of mmap pages currently outstanding */ + PgHdr *pFree; /* List of free mmap page headers (pDirty) */ /* ** End of the routinely-changing class members ***************************************************************************/ @@ -3803,6 +3808,105 @@ static int pagerSyncHotJournal(Pager *pPager){ return rc; } +#include + +/* +** Unmap any mapping of the database file. +*/ +static int pagerUnmap(Pager *pPager){ + if( pPager->pMap ){ + munmap(pPager->pMap, pPager->nMap); + pPager->pMap = 0; + pPager->nMap = 0; + } + return SQLITE_OK; +} + +static int pagerMap(Pager *pPager){ + int rc; + i64 sz = 0; + + assert( pPager->pMap==0 && pPager->nMap==0 ); + + rc = sqlite3OsFileSize(pPager->fd, &sz); + if( rc==SQLITE_OK && sz>0 ){ + int fd; + rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_GETFD, (void *)&fd); + if( rc==SQLITE_OK ){ + void *pMap = mmap(0, sz, PROT_READ, MAP_SHARED, fd, 0); + if( pMap==MAP_FAILED ){ + assert( 0 ); + return SQLITE_IOERR; + } + pPager->pMap = pMap; + pPager->nMap = sz; + } + } + + return rc; +} + +static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ + int rc; + *ppPage = 0; + + assert( pPager->pWal==0 ); + + if( MEMDB==0 && pPager->tempFile==0 ){ + if( pPager->pMap==0 ){ + rc = pagerMap(pPager); + if( rc!=SQLITE_OK ) return rc; + } + + if( pgno!=1 && pPager->pMap && pPager->nMap>=((i64)pgno*pPager->pageSize) ){ + PgHdr *p; + if( pPager->pFree ){ + p = pPager->pFree; + pPager->pFree = p->pDirty; + p->pDirty = 0; + memset(p->pExtra, 0, pPager->nExtra); + }else{ + p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); + if( p==0 ) return SQLITE_NOMEM; + p->pExtra = (void *)&p[1]; + p->flags = PGHDR_MMAP; + p->nRef = 1; + p->pPager = pPager; + } + + assert( p->pExtra==(void *)&p[1] ); + assert( p->pPage==0 ); + assert( p->flags==PGHDR_MMAP ); + assert( p->pPager==pPager ); + assert( p->nRef==1 ); + + p->pData = &((u8 *)pPager->pMap)[(i64)(pgno-1) * pPager->pageSize]; + p->pgno = pgno; + pPager->nMmapOut++; + *ppPage = p; + } + } + + return SQLITE_OK; +} + +static void pagerReleaseMapPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + pPager->nMmapOut--; + pPg->pDirty = pPager->pFree; + pPager->pFree = pPg; +} + +static void pagerFreeMapHdrs(Pager *pPager){ + PgHdr *p; + PgHdr *pNext; + for(p=pPager->pFree; p; p=pNext){ + pNext = p->pDirty; + sqlite3_free(p); + } +} + + /* ** Shutdown the page cache. Free all memory and close all files. ** @@ -3823,6 +3927,8 @@ int sqlite3PagerClose(Pager *pPager){ assert( assert_pager_state(pPager) ); disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); + pagerUnmap(pPager); + pagerFreeMapHdrs(pPager); /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL @@ -4966,6 +5072,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( memcmp(pPager->dbFileVers, dbFileVers, sizeof(dbFileVers))!=0 ){ pager_reset(pPager); + pagerUnmap(pPager); } } @@ -5007,7 +5114,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** nothing to rollback, so this routine is a no-op. */ static void pagerUnlockIfUnused(Pager *pPager){ - if( (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ + if( (sqlite3PcacheRefCount(pPager->pPCache)==0) && pPager->nMmapOut==0 ){ pagerUnlockAndRollback(pPager); } } @@ -5083,6 +5190,15 @@ int sqlite3PagerAcquire( if( pPager->errCode!=SQLITE_OK ){ rc = pPager->errCode; }else{ + if( pPager->eState==PAGER_READER && pPager->pWal==0 ){ + rc = pagerAcquireMapPage(pPager, pgno, &pPg); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + if( pPg ){ + *ppPage = pPg; + return SQLITE_OK; + } + } + rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, ppPage); } @@ -5196,7 +5312,11 @@ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ void sqlite3PagerUnref(DbPage *pPg){ if( pPg ){ Pager *pPager = pPg->pPager; - sqlite3PcacheRelease(pPg); + if( pPg->flags & PGHDR_MMAP ){ + pagerReleaseMapPage(pPg); + }else{ + sqlite3PcacheRelease(pPg); + } pagerUnlockIfUnused(pPager); } } @@ -5313,6 +5433,8 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ assert( pPager->eState>=PAGER_READER && pPager->eStatesubjInMemory = (u8)subjInMemory; + pagerUnmap(pPager); + if( ALWAYS(pPager->eState==PAGER_READER) ){ assert( pPager->pInJournal==0 ); @@ -5535,6 +5657,9 @@ int sqlite3PagerWrite(DbPage *pDbPage){ assert( pPager->eState!=PAGER_ERROR ); assert( assert_pager_state(pPager) ); + /* There must not be any outstanding mmap pages at this point */ + assert( pPager->nMmapOut==0 ); + if( nPagePerSector>1 ){ Pgno nPageCount; /* Total number of pages in database file */ Pgno pg1; /* First page of the sector pPg is located on. */ diff --git a/src/pcache.h b/src/pcache.h index b9135fd859..f4d4ad71c1 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -53,6 +53,8 @@ struct PgHdr { #define PGHDR_REUSE_UNLIKELY 0x010 /* A hint that reuse is unlikely */ #define PGHDR_DONT_WRITE 0x020 /* Do not write content to disk */ +#define PGHDR_MMAP 0x040 /* This is an mmap page object */ + /* Initialize and shutdown the page cache subsystem */ int sqlite3PcacheInitialize(void); void sqlite3PcacheShutdown(void); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 0373353d7b..0ddb0dd105 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -900,6 +900,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_PRAGMA 14 #define SQLITE_FCNTL_BUSYHANDLER 15 #define SQLITE_FCNTL_TEMPFILENAME 16 +#define SQLITE_FCNTL_GETFD 17 /* ** CAPI3REF: Mutex Handle diff --git a/test/permutations.test b/test/permutations.test index 711d4e57d3..d9014d77fe 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -135,7 +135,8 @@ test_suite "veryquick" -prefix "" -description { This test suite is the same as the "quick" tests, except that some files that test malloc and IO errors are omitted. } -files [ - test_set $allquicktests -exclude *malloc* *ioerr* *fault* + test_set $allquicktests -exclude *malloc* *ioerr* *fault* \ + multiplex* server1.test shared2.test shared6.test ] test_suite "valgrind" -prefix "" -description { From 11dcd119136b3131f7520e1814232e333ce83d1e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 15 Mar 2013 18:29:18 +0000 Subject: [PATCH 002/204] Allow read-only cursors to use mmap pages even if there is an open write transaction. FossilOrigin-Name: b387e2f9d24dccac1fd040e309f6fc7ec1cfffba --- manifest | 19 ++++------ manifest.uuid | 2 +- src/btree.c | 100 +++++++++++++++++++++++++++----------------------- src/pager.c | 73 ++++++++++++++++++++++++++---------- src/pager.h | 6 +++ 5 files changed, 124 insertions(+), 76 deletions(-) diff --git a/manifest b/manifest index a61f43717c..97b08ff77e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\smmap()\sto\sread\sfrom\sthe\sdatabase\sfile\sin\srollback\smode.\sThis\sbranch\sis\sunix\sonly\sfor\snow. -D 2013-03-14T18:34:37.796 +C Allow\sread-only\scursors\sto\suse\smmap\spages\seven\sif\sthere\sis\san\sopen\swrite\stransaction. +D 2013-03-15T18:29:18.146 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c c1a956c6762f2a45188c945e1070daec29f5253f +F src/btree.c 934921ec91456c264f61ce671ca62cb826af977a F src/btree.h 3ad7964d6c5b1c7bff569aab6adfa075f8bf06cd F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 375e5df716e03b9343c5e1211be3b24e6d6dff05 @@ -162,8 +162,8 @@ F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 2a4cd96aabf413f39cf562baebb27aa9993f6f54 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 4e7e66c2959ee43caf8a7000712d6a4121b0888a -F src/pager.h 1109a06578ec5574dc2c74cf8d9f69daf36fe3e0 +F src/pager.c 734578514314a3d923fa1399c9722acbcc4f7e6b +F src/pager.h 81ac95f4fcfe21981f495146f6d7f2fe51afd110 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 @@ -1038,10 +1038,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 0b452734faa0839c817f040322e7733e423bfce2 -R d4c281f6a900c9b41816881ffb3c2331 -T *branch * experimental-mmap -T *sym-experimental-mmap * -T -sym-trunk * +P 6f21d9cbf5d457e63a7282015a89ae785526cf6d +R 94aae53403ccd6ebc7d159fa0d23eb33 U dan -Z f16a594407ec9e95c26319dfbc751443 +Z 133f9be54c97c4ee392fb0706f346b90 diff --git a/manifest.uuid b/manifest.uuid index dbbf6a71bc..a20f3a4f02 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6f21d9cbf5d457e63a7282015a89ae785526cf6d \ No newline at end of file +b387e2f9d24dccac1fd040e309f6fc7ec1cfffba \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 038126c3f0..5104625c35 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1569,13 +1569,17 @@ static int btreeGetPage( BtShared *pBt, /* The btree */ Pgno pgno, /* Number of the page to fetch */ MemPage **ppPage, /* Return the page in this parameter */ - int noContent /* Do not load page content if true */ + int noContent, /* Do not load page content if true */ + int bReadonly /* True if a read-only (mmap) page is ok */ ){ int rc; DbPage *pDbPage; + int flags = (noContent ? PAGER_ACQUIRE_NOCONTENT : 0) + | (bReadonly ? PAGER_ACQUIRE_READONLY : 0); + assert( noContent==0 || bReadonly==0 ); assert( sqlite3_mutex_held(pBt->mutex) ); - rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, noContent); + rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, flags); if( rc ) return rc; *ppPage = btreePageFromDbPage(pDbPage, pgno, pBt); return SQLITE_OK; @@ -1618,9 +1622,10 @@ u32 sqlite3BtreeLastPage(Btree *p){ ** may remain unchanged, or it may be set to an invalid value. */ static int getAndInitPage( - BtShared *pBt, /* The database file */ - Pgno pgno, /* Number of the page to get */ - MemPage **ppPage /* Write the page pointer here */ + BtShared *pBt, /* The database file */ + Pgno pgno, /* Number of the page to get */ + MemPage **ppPage, /* Write the page pointer here */ + int bReadonly /* True if a read-only (mmap) page is ok */ ){ int rc; assert( sqlite3_mutex_held(pBt->mutex) ); @@ -1628,7 +1633,7 @@ static int getAndInitPage( if( pgno>btreePagecount(pBt) ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = btreeGetPage(pBt, pgno, ppPage, 0); + rc = btreeGetPage(pBt, pgno, ppPage, 0, bReadonly); if( rc==SQLITE_OK ){ rc = btreeInitPage(*ppPage); if( rc!=SQLITE_OK ){ @@ -2350,7 +2355,7 @@ static int lockBtree(BtShared *pBt){ assert( pBt->pPage1==0 ); rc = sqlite3PagerSharedLock(pBt->pPager); if( rc!=SQLITE_OK ) return rc; - rc = btreeGetPage(pBt, 1, &pPage1, 0); + rc = btreeGetPage(pBt, 1, &pPage1, 0, 0); if( rc!=SQLITE_OK ) return rc; /* Do some checking to help insure the file we opened really is @@ -2565,31 +2570,30 @@ int sqlite3BtreeNewDb(Btree *p){ } /* -** If the shared-btree passed as the only argument is holding references -** to mmap pages, replace them with read/write pages. Return SQLITE_OK -** if successful, or an error code otherwise. +** Ensure that any root page references held by open cursors are not +** mmap pages. */ static int btreeSwapOutMmap(BtShared *pBt){ - BtCursor *pCsr; - for(pCsr=pBt->pCursor; pCsr; pCsr=pCsr->pNext){ - int i; - for(i=0; i<=pCsr->iPage; i++){ - MemPage *pPg = pCsr->apPage[i]; + int rc = SQLITE_OK; /* Return code */ + BtCursor *pCsr; /* Used to iterate through all open cursors */ + + for(pCsr=pBt->pCursor; pCsr && rc==SQLITE_OK; pCsr=pCsr->pNext){ + if( pCsr->iPage>=0 ){ + MemPage *pPg = pCsr->apPage[0]; if( pPg->pDbPage->flags & PGHDR_MMAP ){ - int rc; MemPage *pNew = 0; - rc = btreeGetPage(pBt, pPg->pgno, &pNew, 0); - if( rc==SQLITE_OK && i==pCsr->iPage ){ + rc = btreeGetPage(pBt, pPg->pgno, &pNew, 0, 0); + if( rc==SQLITE_OK && pCsr->iPage==0 ){ pCsr->info.pCell = pNew->aData + (pCsr->info.pCell - pPg->aData); } - pCsr->apPage[i] = pNew; + pCsr->apPage[0] = pNew; releasePage(pPg); if( rc!=SQLITE_OK ) return rc; } } } - return SQLITE_OK; + return rc; } /* @@ -2940,7 +2944,7 @@ static int relocatePage( ** iPtrPage. */ if( eType!=PTRMAP_ROOTPAGE ){ - rc = btreeGetPage(pBt, iPtrPage, &pPtrPage, 0); + rc = btreeGetPage(pBt, iPtrPage, &pPtrPage, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -3024,7 +3028,7 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ u8 eMode = BTALLOC_ANY; /* Mode parameter for allocateBtreePage() */ Pgno iNear = 0; /* nearby parameter for allocateBtreePage() */ - rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0); + rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -3116,8 +3120,11 @@ int sqlite3BtreeIncrVacuum(Btree *p){ if( nOrig0 ){ - invalidateAllOverflowCache(pBt); - rc = incrVacuumStep(pBt, nFin, nOrig, 0); + rc = saveAllCursors(pBt, 0, 0); + if( rc==SQLITE_OK ){ + invalidateAllOverflowCache(pBt); + rc = incrVacuumStep(pBt, nFin, nOrig, 0); + } if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); put4byte(&pBt->pPage1->aData[28], pBt->nPage); @@ -3438,7 +3445,7 @@ int sqlite3BtreeRollback(Btree *p, int tripCode){ /* The rollback may have destroyed the pPage1->aData value. So ** call btreeGetPage() on page 1 again to make ** sure pPage1->aData is set correctly. */ - if( btreeGetPage(pBt, 1, &pPage1, 0)==SQLITE_OK ){ + if( btreeGetPage(pBt, 1, &pPage1, 0, 0)==SQLITE_OK ){ int nPage = get4byte(28+(u8*)pPage1->aData); testcase( nPage==0 ); if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); @@ -3872,7 +3879,7 @@ static int getOverflowPage( assert( next==0 || rc==SQLITE_DONE ); if( rc==SQLITE_OK ){ - rc = btreeGetPage(pBt, ovfl, &pPage, 0); + rc = btreeGetPage(pBt, ovfl, &pPage, 0, (ppPage==0)); assert( rc==SQLITE_OK || pPage==0 ); if( rc==SQLITE_OK ){ next = get4byte(pPage->aData); @@ -4093,7 +4100,9 @@ static int accessPayload( { DbPage *pDbPage; - rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage); + rc = sqlite3PagerAcquire(pBt->pPager, nextPage, &pDbPage, + (eOp==0 ? PAGER_ACQUIRE_READONLY : 0) + ); if( rc==SQLITE_OK ){ aPayload = sqlite3PagerGetData(pDbPage); nextPage = get4byte(aPayload); @@ -4272,10 +4281,11 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPageiPage>=0 ); if( pCur->iPage>=(BTCURSOR_MAX_DEPTH-1) ){ return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, newPgno, &pNewPage); + rc = getAndInitPage(pBt, newPgno, &pNewPage, (pCur->wrFlag==0)); if( rc ) return rc; pCur->apPage[i+1] = pNewPage; pCur->aiIdx[i+1] = 0; @@ -4392,7 +4402,7 @@ static int moveToRoot(BtCursor *pCur){ pCur->eState = CURSOR_INVALID; return SQLITE_OK; }else{ - rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0]); + rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0], 0); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; @@ -5006,7 +5016,7 @@ static int allocateBtreePage( if( iTrunk>mxPage ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0, 0); } if( rc ){ pTrunk = 0; @@ -5070,7 +5080,7 @@ static int allocateBtreePage( goto end_allocate_page; } testcase( iNewTrunk==mxPage ); - rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0); + rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0, 0); if( rc!=SQLITE_OK ){ goto end_allocate_page; } @@ -5150,7 +5160,7 @@ static int allocateBtreePage( } put4byte(&aData[4], k-1); noContent = !btreeGetHasContent(pBt, *pPgno); - rc = btreeGetPage(pBt, *pPgno, ppPage, noContent); + rc = btreeGetPage(pBt, *pPgno, ppPage, noContent, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ @@ -5198,7 +5208,7 @@ static int allocateBtreePage( MemPage *pPg = 0; TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, pBt->nPage, &pPg, bNoContent); + rc = btreeGetPage(pBt, pBt->nPage, &pPg, bNoContent, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pPg->pDbPage); releasePage(pPg); @@ -5212,7 +5222,7 @@ static int allocateBtreePage( *pPgno = pBt->nPage; assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, *pPgno, ppPage, bNoContent); + rc = btreeGetPage(pBt, *pPgno, ppPage, bNoContent, 0); if( rc ) return rc; rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ @@ -5280,7 +5290,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ /* If the secure_delete option is enabled, then ** always fully overwrite deleted information with zeros. */ - if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0) ) + if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0, 0))!=0) ) || ((rc = sqlite3PagerWrite(pPage->pDbPage))!=0) ){ goto freepage_out; @@ -5307,7 +5317,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ u32 nLeaf; /* Initial number of leaf cells on trunk page */ iTrunk = get4byte(&pPage1->aData[32]); - rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0, 0); if( rc!=SQLITE_OK ){ goto freepage_out; } @@ -5353,7 +5363,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ ** first trunk in the free-list is full. Either way, the page being freed ** will become the new first trunk page in the free-list. */ - if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0)) ){ + if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0, 0)) ){ goto freepage_out; } rc = sqlite3PagerWrite(pPage->pDbPage); @@ -6154,7 +6164,7 @@ static int balance_nonroot( } pgno = get4byte(pRight); while( 1 ){ - rc = getAndInitPage(pBt, pgno, &apOld[i]); + rc = getAndInitPage(pBt, pgno, &apOld[i], 0); if( rc ){ memset(apOld, 0, (i+1)*sizeof(MemPage*)); goto balance_cleanup; @@ -7245,7 +7255,7 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ releasePage(pPageMove); /* Move the page currently at pgnoRoot to pgnoMove. */ - rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -7266,7 +7276,7 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ if( rc!=SQLITE_OK ){ return rc; } - rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -7342,7 +7352,7 @@ static int clearDatabasePage( return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, pgno, &pPage); + rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; for(i=0; inCell; i++){ pCell = findCell(pPage, i); @@ -7444,7 +7454,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return SQLITE_LOCKED_SHAREDCACHE; } - rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); + rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0, 0); if( rc ) return rc; rc = sqlite3BtreeClearTable(p, iTable, 0); if( rc ){ @@ -7479,7 +7489,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ */ MemPage *pMove; releasePage(pPage); - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -7489,7 +7499,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return rc; } pMove = 0; - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0, 0); freePage(pMove, &rc); releasePage(pMove); if( rc!=SQLITE_OK ){ @@ -7901,7 +7911,7 @@ static int checkTreePage( usableSize = pBt->usableSize; if( iPage==0 ) return 0; if( checkRef(pCheck, iPage, zParentContext) ) return 0; - if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){ + if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0, 0))!=0 ){ checkAppendMsg(pCheck, zContext, "unable to get the page. error code=%d", rc); return 0; diff --git a/src/pager.c b/src/pager.c index c362494093..8470e7e35e 100644 --- a/src/pager.c +++ b/src/pager.c @@ -658,6 +658,7 @@ struct Pager { void *pMap; /* Memory mapped prefix of database file */ i64 nMap; /* Size of mapping at pMap in bytes */ + i64 nMapValid; /* Bytes at pMap known to be valid */ int nMmapOut; /* Number of mmap pages currently outstanding */ PgHdr *pFree; /* List of free mmap page headers (pDirty) */ /* @@ -2512,6 +2513,9 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ if( rc==SQLITE_OK && currentSize!=newSize ){ if( currentSize>newSize ){ rc = sqlite3OsTruncate(pPager->fd, newSize); + if( newSizenMapValid ){ + pPager->nMapValid = newSize; + } }else if( (currentSize+szPage)<=newSize ){ char *pTmp = pPager->pTmpSpace; memset(pTmp, 0, szPage); @@ -3818,6 +3822,7 @@ static int pagerUnmap(Pager *pPager){ munmap(pPager->pMap, pPager->nMap); pPager->pMap = 0; pPager->nMap = 0; + pPager->nMapValid = 0; } return SQLITE_OK; } @@ -3839,7 +3844,7 @@ static int pagerMap(Pager *pPager){ return SQLITE_IOERR; } pPager->pMap = pMap; - pPager->nMap = sz; + pPager->nMapValid = pPager->nMap = sz; } } @@ -3858,7 +3863,9 @@ static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ if( rc!=SQLITE_OK ) return rc; } - if( pgno!=1 && pPager->pMap && pPager->nMap>=((i64)pgno*pPager->pageSize) ){ + if( pgno!=1 && pPager->pMap + && pPager->nMapValid>=((i64)pgno*pPager->pageSize) + ){ PgHdr *p; if( pPager->pFree ){ p = pPager->pFree; @@ -5035,9 +5042,11 @@ int sqlite3PagerSharedLock(Pager *pPager){ ); } - if( !pPager->tempFile - && (pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0) - ){ + if( !pPager->tempFile && ( + pPager->pBackup + || sqlite3PcachePagecount(pPager->pPCache)>0 + || pPager->pMap + )){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous ** read or write transaction). Check to see if the database @@ -5072,6 +5081,15 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( memcmp(pPager->dbFileVers, dbFileVers, sizeof(dbFileVers))!=0 ){ pager_reset(pPager); + + /* Unmap the database file. It is possible that external processes + ** may have truncated the database file and then extended it back + ** to its original size while this process was not holding a lock. + ** In this case there may exist a Pager.pMap mapping that appears + ** to be the right size but is not actually valid. Avoid this + ** possibility by unmapping the db here. */ + pagerUnmap(pPager); + }else if( ((i64)nPage*pPager->pageSize)!=pPager->nMap ){ pagerUnmap(pPager); } } @@ -5173,10 +5191,20 @@ int sqlite3PagerAcquire( Pager *pPager, /* The pager open on the database file */ Pgno pgno, /* Page number to fetch */ DbPage **ppPage, /* Write a pointer to the page here */ - int noContent /* Do not bother reading content from disk if true */ + int flags /* PAGER_ACQUIRE_XXX flags */ ){ - int rc; - PgHdr *pPg; + int rc = SQLITE_OK; + PgHdr *pPg = 0; + const int noContent = (flags & PAGER_ACQUIRE_NOCONTENT); + + /* It is acceptable to use a read-only (mmap) page for any page except + ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY + ** flag was specified by the caller. And so long as the db is not a + ** temporary or in-memory database. */ + const int bMmapOk = ( + (pgno!=1 && pPager->pWal==0 && !pPager->tempFile && !MEMDB) + && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY)) + ); assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); @@ -5190,12 +5218,23 @@ int sqlite3PagerAcquire( if( pPager->errCode!=SQLITE_OK ){ rc = pPager->errCode; }else{ - if( pPager->eState==PAGER_READER && pPager->pWal==0 ){ - rc = pagerAcquireMapPage(pPager, pgno, &pPg); - if( rc!=SQLITE_OK ) goto pager_acquire_err; - if( pPg ){ - *ppPage = pPg; - return SQLITE_OK; + + if( bMmapOk ){ + if( pPager->pMap==0 ) rc = pagerMap(pPager); + if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){ + if( pPager->eState>PAGER_READER ){ + (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); + } + if( pPg==0 ){ + rc = pagerAcquireMapPage(pPager, pgno, &pPg); + } + if( pPg ){ + assert( rc==SQLITE_OK ); + *ppPage = pPg; + return SQLITE_OK; + }else if( rc!=SQLITE_OK ){ + goto pager_acquire_err; + } } } @@ -5433,8 +5472,6 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ assert( pPager->eState>=PAGER_READER && pPager->eStatesubjInMemory = (u8)subjInMemory; - pagerUnmap(pPager); - if( ALWAYS(pPager->eState==PAGER_READER) ){ assert( pPager->pInJournal==0 ); @@ -5653,13 +5690,11 @@ int sqlite3PagerWrite(DbPage *pDbPage){ Pager *pPager = pPg->pPager; Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); + assert( (pPg->flags & PGHDR_MMAP)==0 ); assert( pPager->eState>=PAGER_WRITER_LOCKED ); assert( pPager->eState!=PAGER_ERROR ); assert( assert_pager_state(pPager) ); - /* There must not be any outstanding mmap pages at this point */ - assert( pPager->nMmapOut==0 ); - if( nPagePerSector>1 ){ Pgno nPageCount; /* Total number of pages in database file */ Pgno pg1; /* First page of the sector pPg is located on. */ diff --git a/src/pager.h b/src/pager.h index 90f8e6af78..94957dac60 100644 --- a/src/pager.h +++ b/src/pager.h @@ -78,6 +78,12 @@ typedef struct PgHdr DbPage; #define PAGER_JOURNALMODE_MEMORY 4 /* In-memory journal file */ #define PAGER_JOURNALMODE_WAL 5 /* Use write-ahead logging */ +/* +** Flags that make up the mask passed to sqlite3PagerAcquire(). +*/ +#define PAGER_ACQUIRE_NOCONTENT 0x01 /* Do not load data from disk */ +#define PAGER_ACQUIRE_READONLY 0x02 /* Read-only page is acceptable */ + /* ** The remainder of this file contains the declarations of the functions ** that make up the Pager sub-system API. See source code comments for From 7af6547a5afde7c5b48058d948df35857ba22a40 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 15 Mar 2013 19:13:42 +0000 Subject: [PATCH 003/204] Fix a dropped error code in pager.c. FossilOrigin-Name: 022fdc986b33701abfd39621072ac3d9f9f7d43e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 9 ++++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 97b08ff77e..ee60425958 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sread-only\scursors\sto\suse\smmap\spages\seven\sif\sthere\sis\san\sopen\swrite\stransaction. -D 2013-03-15T18:29:18.146 +C Fix\sa\sdropped\serror\scode\sin\spager.c. +D 2013-03-15T19:13:42.183 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 2a4cd96aabf413f39cf562baebb27aa9993f6f54 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 734578514314a3d923fa1399c9722acbcc4f7e6b +F src/pager.c bdbcfe676cda9295572e4ce95a3fae827082f9f0 F src/pager.h 81ac95f4fcfe21981f495146f6d7f2fe51afd110 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1038,7 +1038,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 6f21d9cbf5d457e63a7282015a89ae785526cf6d -R 94aae53403ccd6ebc7d159fa0d23eb33 +P b387e2f9d24dccac1fd040e309f6fc7ec1cfffba +R 53728d59c72e59e92889436c2f9b085b U dan -Z 133f9be54c97c4ee392fb0706f346b90 +Z 3a434ac42175412fea267939fd915765 diff --git a/manifest.uuid b/manifest.uuid index a20f3a4f02..9f9d488fb7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b387e2f9d24dccac1fd040e309f6fc7ec1cfffba \ No newline at end of file +022fdc986b33701abfd39621072ac3d9f9f7d43e \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 8470e7e35e..23e7a67411 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5220,7 +5220,9 @@ int sqlite3PagerAcquire( }else{ if( bMmapOk ){ - if( pPager->pMap==0 ) rc = pagerMap(pPager); + if( pPager->pMap==0 ){ + rc = pagerMap(pPager); + } if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){ if( pPager->eState>PAGER_READER ){ (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); @@ -5232,10 +5234,11 @@ int sqlite3PagerAcquire( assert( rc==SQLITE_OK ); *ppPage = pPg; return SQLITE_OK; - }else if( rc!=SQLITE_OK ){ - goto pager_acquire_err; } } + if( rc!=SQLITE_OK ){ + goto pager_acquire_err; + } } rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, ppPage); From a72014faf50889400af55631d4556a0adcffb593 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 16 Mar 2013 20:19:21 +0000 Subject: [PATCH 004/204] When possible, use memcpy() to and from the mapped region instead of xWrite() and xRead(). FossilOrigin-Name: f8ca5622d99bedca957caa9ad311d798f63b3ce9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 41 +++++++++++++++++++++++++++++++++-------- src/test1.c | 24 ++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index ee60425958..a6549b18c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sdropped\serror\scode\sin\spager.c. -D 2013-03-15T19:13:42.183 +C When\spossible,\suse\smemcpy()\sto\sand\sfrom\sthe\smapped\sregion\sinstead\sof\sxWrite()\sand\sxRead(). +D 2013-03-16T20:19:21.766 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 2a4cd96aabf413f39cf562baebb27aa9993f6f54 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c bdbcfe676cda9295572e4ce95a3fae827082f9f0 +F src/pager.c 495c5344392d5932ea5072f20bfbd8a58cf19d67 F src/pager.h 81ac95f4fcfe21981f495146f6d7f2fe51afd110 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -184,7 +184,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 3213f3101e3b85f047d6e389da5a53d76d3d7540 -F src/test1.c ff3e68eedfbd858c9b89cf03e3db233cd29be1d0 +F src/test1.c 3dac8d76be8852d65ff8b9ce4b50ed08b999ed59 F src/test2.c 4178056dd1e7d70f954ad8a1e3edb71a2a784daf F src/test3.c 3c3c2407fa6ec7a19e24ae23f7cb439d0275a60d F src/test4.c bf9fa9bece01de08e6f5e02314e4af5c13590dfa @@ -1038,7 +1038,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b387e2f9d24dccac1fd040e309f6fc7ec1cfffba -R 53728d59c72e59e92889436c2f9b085b +P 022fdc986b33701abfd39621072ac3d9f9f7d43e +R e9d1b00daf0ae79f0e154b8607fa8ad8 U dan -Z 3a434ac42175412fea267939fd915765 +Z 82853c015b40f8bb1665068f9eea77d4 diff --git a/manifest.uuid b/manifest.uuid index 9f9d488fb7..500dcc6707 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -022fdc986b33701abfd39621072ac3d9f9f7d43e \ No newline at end of file +f8ca5622d99bedca957caa9ad311d798f63b3ce9 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 23e7a67411..d9b9ed5052 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2865,9 +2865,13 @@ static int readDbPage(PgHdr *pPg){ } if( rc==SQLITE_OK && !isInWal ){ i64 iOffset = (pgno-1)*(i64)pPager->pageSize; - rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); - if( rc==SQLITE_IOERR_SHORT_READ ){ - rc = SQLITE_OK; + if( pPager->pMap && pPager->nMapValid>=iOffset+pPager->pageSize ){ + memcpy(pPg->pData, &((u8 *)(pPager->pMap))[iOffset], pPager->pageSize); + }else{ + rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; + } } } @@ -3834,13 +3838,14 @@ static int pagerMap(Pager *pPager){ assert( pPager->pMap==0 && pPager->nMap==0 ); rc = sqlite3OsFileSize(pPager->fd, &sz); + sz = sz & ~(4096-1); + if( rc==SQLITE_OK && sz>0 ){ int fd; rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_GETFD, (void *)&fd); if( rc==SQLITE_OK ){ - void *pMap = mmap(0, sz, PROT_READ, MAP_SHARED, fd, 0); + void *pMap = mmap(0, sz, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); if( pMap==MAP_FAILED ){ - assert( 0 ); return SQLITE_IOERR; } pPager->pMap = pMap; @@ -3851,6 +3856,22 @@ static int pagerMap(Pager *pPager){ return rc; } +static int pagerRemap(Pager *pPager, Pgno nPage){ + i64 sz = (i64)nPage * pPager->pageSize; + sz = sz & ~(4096-1); + + if( pPager->nMap!=sz ){ + void *pMap = mremap(pPager->pMap, pPager->nMap, sz, MREMAP_MAYMOVE); + if( pMap==MAP_FAILED ){ + return SQLITE_IOERR; + } + pPager->pMap = pMap; + pPager->nMapValid = pPager->nMap = sz; + } + + return SQLITE_OK; +} + static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ int rc; *ppPage = 0; @@ -4225,7 +4246,11 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); /* Write out the page data. */ - rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); + if( pPager->nMapValid>=(offset+pPager->pageSize) ){ + memcpy(&((u8 *)(pPager->pMap))[offset], pData, pPager->pageSize); + }else{ + rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); + } /* If page 1 was just written, update Pager.dbFileVers to match ** the value now stored in the database file. If writing this @@ -5089,8 +5114,8 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** to be the right size but is not actually valid. Avoid this ** possibility by unmapping the db here. */ pagerUnmap(pPager); - }else if( ((i64)nPage*pPager->pageSize)!=pPager->nMap ){ - pagerUnmap(pPager); + }else if( pPager->pMap ){ + pagerRemap(pPager, nPage); } } diff --git a/src/test1.c b/src/test1.c index 0bace84420..a89b874a3b 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5844,6 +5844,29 @@ static int test_test_control( return TCL_OK; } +#include +#include + +static int test_getrusage( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + char buf[1024]; + struct rusage r; + memset(&r, 0, sizeof(r)); + getrusage(RUSAGE_SELF, &r); + + sprintf(buf, "ru_utime=%d.%06d ru_stime=%d.%06d ru_minflt=%d ru_majflt=%d", + r.ru_utime.tv_sec, r.ru_utime.tv_usec, + r.ru_stime.tv_sec, r.ru_stime.tv_usec, + r.ru_minflt, r.ru_majflt + ); + Tcl_SetObjResult(interp, Tcl_NewStringObj(buf, -1)); + return TCL_OK; +} + #if SQLITE_OS_WIN /* ** Information passed from the main thread into the windows file locker @@ -6233,6 +6256,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "print_explain_query_plan", test_print_eqp, 0 }, #endif { "sqlite3_test_control", test_test_control }, + { "getrusage", test_getrusage }, }; static int bitmask_size = sizeof(Bitmask)*8; int i; From 5b4b2551513197e36e432ae5b2432042578bdd59 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 18 Mar 2013 11:39:23 +0000 Subject: [PATCH 005/204] Version 3.7.16 FossilOrigin-Name: 66d5f2b76750f3520eb7a495f6247206758f5b90 --- manifest | 13 ++++++++----- manifest.uuid | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/manifest b/manifest index 42eb32ea0f..81e2d81daa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\stests\sfor\sticket\s[4dd95f6943]. -D 2013-03-13T07:02:04.083 +C Version\s3.7.16 +D 2013-03-18T11:39:23.116 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1038,7 +1038,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 839aa91faf1db7025d90fa3c65e50efb829b053b +P 0b452734faa0839c817f040322e7733e423bfce2 R ad71abb6cd4a6ebc6857153126653bad -U dan -Z 69b39b8becc09fc4758d1b797bf5f4f2 +T +bgcolor * #d0c0ff +T +sym-release * +T +sym-version-3.7.16 * +U drh +Z d73f3a9c465960162106ae1e090c862c diff --git a/manifest.uuid b/manifest.uuid index 3d8be2e93e..8d7655aac9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0b452734faa0839c817f040322e7733e423bfce2 \ No newline at end of file +66d5f2b76750f3520eb7a495f6247206758f5b90 \ No newline at end of file From ce3ca25cbcd117d0a238bc028a83c453673dd0fe Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 18 Mar 2013 17:18:18 +0000 Subject: [PATCH 006/204] Clarifications to the documentation for the sqlite3_aggregate_context() API. Also, shorten an over-length source line in sqlite.h.in. FossilOrigin-Name: 4fe2db1d866c80fe51f7fddbf9ce6753fb55b5f6 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/sqlite.h.in | 12 ++++++++---- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 81e2d81daa..ef5a112618 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.7.16 -D 2013-03-18T11:39:23.116 +C Clarifications\sto\sthe\sdocumentation\sfor\sthe\ssqlite3_aggregate_context()\sAPI.\nAlso,\sshorten\san\sover-length\ssource\sline\sin\ssqlite.h.in. +D 2013-03-18T17:18:18.835 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,7 +176,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in f2fa32f440dda59ca47e22889966b2a6eb3b491c +F src/sqlite.h.in e0ace39d7c14dc9e988cf579f75aaf2a7db7ae77 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 0f8f05ee4db4ba9120b38f7a3992b325698f6e8a @@ -1038,10 +1038,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 0b452734faa0839c817f040322e7733e423bfce2 -R ad71abb6cd4a6ebc6857153126653bad -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.7.16 * +P 66d5f2b76750f3520eb7a495f6247206758f5b90 +R 6b3383461c64bf451ee04663b065cc63 U drh -Z d73f3a9c465960162106ae1e090c862c +Z f8cab3c95e26c38e205a97060fe1ac3e diff --git a/manifest.uuid b/manifest.uuid index 8d7655aac9..14df4dedc4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -66d5f2b76750f3520eb7a495f6247206758f5b90 \ No newline at end of file +4fe2db1d866c80fe51f7fddbf9ce6753fb55b5f6 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 0373353d7b..a6f9b0b370 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -3993,7 +3993,8 @@ SQLITE_DEPRECATED int sqlite3_expired(sqlite3_stmt*); SQLITE_DEPRECATED int sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); SQLITE_DEPRECATED int sqlite3_global_recover(void); SQLITE_DEPRECATED void sqlite3_thread_cleanup(void); -SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int),void*,sqlite3_int64); +SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), + void*,sqlite3_int64); #endif /* @@ -4073,14 +4074,17 @@ int sqlite3_value_numeric_type(sqlite3_value*); ** In those cases, sqlite3_aggregate_context() might be called for the ** first time from within xFinal().)^ ** -** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer if N is -** less than or equal to zero or if a memory allocate error occurs. +** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer +** when first called if N is less than or equal to zero or if a memory +** allocate error occurs. ** ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is ** determined by the N parameter on first successful call. Changing the ** value of N in subsequent call to sqlite3_aggregate_context() within ** the same aggregate function instance will not resize the memory -** allocation.)^ +** allocation.)^ Within the xFinal callback, it is customary to set +** N=0 in calls to sqlite3_aggregate_context(C,N) so that no +** pointless memory allocations occur. ** ** ^SQLite automatically frees the memory allocated by ** sqlite3_aggregate_context() when the aggregate query concludes. From 339d6c6d17abc6a1e811a902c9bc435f7c1baf88 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 19 Mar 2013 16:12:40 +0000 Subject: [PATCH 007/204] Bring makefiles and build scripts into alignment with the sessions branch. No changes to code. FossilOrigin-Name: d1f41089aba075eef45fd696599e5d3a74c84d0c --- Makefile.in | 4 +++- Makefile.msc | 2 ++ main.mk | 1 + manifest | 20 ++++++++++---------- manifest.uuid | 2 +- tool/mksqlite3c.tcl | 2 +- tool/mksqlite3h.tcl | 9 +++++++-- 7 files changed, 25 insertions(+), 15 deletions(-) diff --git a/Makefile.in b/Makefile.in index 652928b8cc..1ecaf2bbd5 100644 --- a/Makefile.in +++ b/Makefile.in @@ -519,6 +519,7 @@ sqlite3$(TEXE): $(TOP)/src/shell.c libsqlite3.la sqlite3.h sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl $(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl + cp tsrc/shell.c tsrc/sqlite3ext.h . tclsqlite3.c: sqlite3.c echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c @@ -878,7 +879,8 @@ TESTFIXTURE_FLAGS += -DBUILD_sqlite TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la TESTFIXTURE_SRC1 = sqlite3.c -TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c $(TESTFIXTURE_SRC$(USE_AMALGAMATION)) +TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c +TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION)) testfixture$(TEXE): $(TESTFIXTURE_SRC) $(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \ diff --git a/Makefile.msc b/Makefile.msc index 2fb9687655..df6ed4d46d 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -830,6 +830,8 @@ sqlite3.exe: $(TOP)\src\shell.c libsqlite3.lib $(LIBRESOBJS) sqlite3.h sqlite3.c: .target_source $(TOP)\tool\mksqlite3c.tcl $(TCLSH_CMD) $(TOP)\tool\mksqlite3c.tcl + copy tsrc\shell.c . + copy tsrc\sqlite3ext.h . sqlite3-all.c: sqlite3.c $(TOP)\tool\split-sqlite3c.tcl $(TCLSH_CMD) $(TOP)\tool\split-sqlite3c.tcl diff --git a/main.mk b/main.mk index 2430d3ffe9..d644079f59 100644 --- a/main.mk +++ b/main.mk @@ -385,6 +385,7 @@ target_source: $(SRC) $(TOP)/tool/vdbe-compress.tcl sqlite3.c: target_source $(TOP)/tool/mksqlite3c.tcl tclsh $(TOP)/tool/mksqlite3c.tcl + cp tsrc/shell.c tsrc/sqlite3ext.h . echo '#ifndef USE_SYSTEM_SQLITE' >tclsqlite3.c cat sqlite3.c >>tclsqlite3.c echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c diff --git a/manifest b/manifest index ef5a112618..4e0fb3f191 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Clarifications\sto\sthe\sdocumentation\sfor\sthe\ssqlite3_aggregate_context()\sAPI.\nAlso,\sshorten\san\sover-length\ssource\sline\sin\ssqlite.h.in. -D 2013-03-18T17:18:18.835 +C Bring\smakefiles\sand\sbuild\sscripts\sinto\salignment\swith\sthe\ssessions\sbranch.\nNo\schanges\sto\scode. +D 2013-03-19T16:12:40.639 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 +F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 1bed3bca025ee90441bb0c9f95f36f42b70bd839 +F Makefile.msc 0c1abc21c8deefc88e8a32ad6a07e5f96e158761 F Makefile.vxworks b18ad88e9a8c6a001f5cf4a389116a4f1a7ab45f F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 6d4f66eaebabc42ef8c2a4d2d0caf4ce7ee81137 @@ -103,7 +103,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk 404cd38114d1a94ef6ad5aa01a1c8bb46da36161 +F main.mk c5437f64aad34844d18efe7b2d2adaa36efca49c F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -1009,8 +1009,8 @@ F tool/mkkeywordhash.c bb52064aa614e1426445e4b2b9b00eeecd23cc79 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 8bce31074e4cbe631bb7676526a048335f4c9f02 -F tool/mksqlite3c.tcl 589c7f44e990be1b8443cfe4808dce392b0327fa -F tool/mksqlite3h.tcl 78013ad79a5e492e5f764f3c7a8ef834255061f8 +F tool/mksqlite3c.tcl a570a0b0847f5659482087f70e409ac45ce93b91 +F tool/mksqlite3h.tcl ba24038056f51fde07c0079c41885ab85e2cff12 F tool/mksqlite3internalh.tcl 3dca7bb5374cee003379b8cbac73714f610ef795 F tool/mkvsix.tcl 0be7f7a591f1e83f9199cb82911b66668ca484c9 F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091 @@ -1038,7 +1038,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 66d5f2b76750f3520eb7a495f6247206758f5b90 -R 6b3383461c64bf451ee04663b065cc63 +P 4fe2db1d866c80fe51f7fddbf9ce6753fb55b5f6 +R 7758dd15e3a5f0387c3f17443ec025d8 U drh -Z f8cab3c95e26c38e205a97060fe1ac3e +Z 7c5d1dfbeba3270fc7d5841fae61579f diff --git a/manifest.uuid b/manifest.uuid index 14df4dedc4..ac1d43b9fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4fe2db1d866c80fe51f7fddbf9ce6753fb55b5f6 \ No newline at end of file +d1f41089aba075eef45fd696599e5d3a74c84d0c \ No newline at end of file diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 2c569d7a4e..7c8a7d54ad 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -23,7 +23,7 @@ # # Begin by reading the "sqlite3.h" header file. Extract the version number -# from in this file. The versioon number is needed to generate the header +# from in this file. The version number is needed to generate the header # comment of the amalgamation. # if {[lsearch $argv --nostatic]>=0} { diff --git a/tool/mksqlite3h.tcl b/tool/mksqlite3h.tcl index f68f61a368..a89b9f9beb 100644 --- a/tool/mksqlite3h.tcl +++ b/tool/mksqlite3h.tcl @@ -68,9 +68,14 @@ set declpattern {^ *[a-zA-Z][a-zA-Z_0-9 ]+ \**sqlite3_[_a-zA-Z0-9]+\(} # Force the output to use unix line endings, even on Windows. fconfigure stdout -translation lf -# Process the src/sqlite.h.in ext/rtree/sqlite3rtree.h files. +set filelist [subst { + $TOP/src/sqlite.h.in + $TOP/ext/rtree/sqlite3rtree.h +}] + +# Process the source files. # -foreach file [list $TOP/src/sqlite.h.in $TOP/ext/rtree/sqlite3rtree.h] { +foreach file $filelist { set in [open $file] while {![eof $in]} { From 5d8a137218d43070a2c635900d1ae7d7a7ef0f69 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 19 Mar 2013 19:28:06 +0000 Subject: [PATCH 008/204] Add the sqlite3_io_methods.xMremap() method to the VFS interface. Also "PRAGMA mmap_size". FossilOrigin-Name: 6183f1bd86ceed76d22d9762f3d7eb33262c62d1 --- manifest | 42 +++++----- manifest.uuid | 2 +- src/btree.c | 13 +++ src/btree.h | 1 + src/os.c | 3 + src/os.h | 1 + src/os_unix.c | 41 ++++++++- src/pager.c | 183 ++++++++++++++++++++++++----------------- src/pragma.c | 24 ++++++ src/sqlite.h.in | 21 +++++ src/sqliteInt.h | 1 + src/test1.c | 6 +- test/dbstatus2.test | 1 + test/exclusive2.test | 8 ++ test/func.test | 12 +-- test/incrblob.test | 4 + test/pageropt.test | 2 +- test/permutations.test | 8 ++ 18 files changed, 263 insertions(+), 110 deletions(-) diff --git a/manifest b/manifest index a6549b18c2..45c3ee6fc5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\spossible,\suse\smemcpy()\sto\sand\sfrom\sthe\smapped\sregion\sinstead\sof\sxWrite()\sand\sxRead(). -D 2013-03-16T20:19:21.766 +C Add\sthe\ssqlite3_io_methods.xMremap()\smethod\sto\sthe\sVFS\sinterface.\sAlso\s"PRAGMA\smmap_size". +D 2013-03-19T19:28:06.473 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,8 +121,8 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 934921ec91456c264f61ce671ca62cb826af977a -F src/btree.h 3ad7964d6c5b1c7bff569aab6adfa075f8bf06cd +F src/btree.c d4d551f05a555926a7c0f49c2e263f7ee2b1c59f +F src/btree.h d3259057a38494c4c4358e377032158c762e3d8b F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 375e5df716e03b9343c5e1211be3b24e6d6dff05 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc @@ -157,18 +157,18 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c -F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 +F src/os.c 81a82a736b8a461a656f9b3e401a39768fe73a79 +F src/os.h 4681261aa24a9d2187aaf4cb963880e6cddb1f48 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 2a4cd96aabf413f39cf562baebb27aa9993f6f54 +F src/os_unix.c 0c7b0d076f2ac6279b0b280a26bcae8c89f36f4f F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 495c5344392d5932ea5072f20bfbd8a58cf19d67 +F src/pager.c 78b65bf9685bf21b787ce2a7389c2b96102942dc F src/pager.h 81ac95f4fcfe21981f495146f6d7f2fe51afd110 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c 9f0ee3d74a7f33eeeff40a4b014fc3abf8182ce2 +F src/pragma.c 86c8088ac6a12d3f3be5f7394542651f03fa9a38 F src/prepare.c 78cd7ae8cd0d8de8ef8a8b5352fc5f38693a0852 F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -176,15 +176,15 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in 9a5c737a1feb4495d351c56883587d4fda52e81e +F src/sqlite.h.in fd75f5bcf479b315b1c717fa1d07f018bd919f79 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 -F src/sqliteInt.h 0f8f05ee4db4ba9120b38f7a3992b325698f6e8a +F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 3213f3101e3b85f047d6e389da5a53d76d3d7540 -F src/test1.c 3dac8d76be8852d65ff8b9ce4b50ed08b999ed59 +F src/test1.c 39378e3e14a8162e29dc90d1e05399d12e8a569e F src/test2.c 4178056dd1e7d70f954ad8a1e3edb71a2a784daf F src/test3.c 3c3c2407fa6ec7a19e24ae23f7cb439d0275a60d F src/test4.c bf9fa9bece01de08e6f5e02314e4af5c13590dfa @@ -369,7 +369,7 @@ F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/date.test f3228180c87bbe5d39c9397bf001c0095c3821b9 F test/dbstatus.test 207e5b63fcb7b9c3bb8e1fdf38ebd4654ad0e54b -F test/dbstatus2.test bf7396af964b89e39435babbcdf296ae8fc5f10a +F test/dbstatus2.test 10418e62b3db5dca070f0c3eef3ea13946f339c2 F test/default.test 6faf23ccb300114924353007795aa9a8ec0aa9dc F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa @@ -403,7 +403,7 @@ F test/eqp.test 46aa946dd55c90635327898275d3e533d23a9845 F test/errmsg.test 050717f1c6a5685de9c79f5f9f6b83d7c592f73a F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test a1b324cb21834a490cd052d409d34789cfef57cb -F test/exclusive2.test 372be98f6de44dd78734e364b7b626ea211761a6 +F test/exclusive2.test 881193eccec225cfed9d7f744b65e57f26adee25 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30 F test/expr.test 67c9fd6f8f829e239dc8b0f4a08a73c08b09196d @@ -508,7 +508,7 @@ F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7 F test/fts4unicode.test 25ccad45896f8e50f6a694cff738a35f798cdb40 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d -F test/func.test b058483c17952eff7797b837bbb61e27e6b05606 +F test/func.test b0fc34fdc36897769651975a2b0a606312753643 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a F test/fuzz-oss1.test 4912e528ec9cf2f42134456933659d371c9e0d74 @@ -526,7 +526,7 @@ F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test 64f3cc1acde1b9161ccdd8e5bde3daefdb5b2617 F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 -F test/incrblob.test 34765fa6fb5d8e0f256fc7d6497c04b205398849 +F test/incrblob.test bf210bea512474d4e1d94fbb9b0fcb386cd65dea F test/incrblob2.test edc3a96e557bd61fb39acc8d2edd43371fbbaa19 F test/incrblob3.test aedbb35ea1b6450c33b98f2b6ed98e5020be8dc7 F test/incrblob4.test 09be37d3dd996a31ea6993bba7837ece549414a8 @@ -652,11 +652,11 @@ F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f F test/pagerfault.test 452f2cc23e3bfcfa935f4442aec1da4fe1dc0442 F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 -F test/pageropt.test 9191867ed19a2b3db6c42d1b36b6fbc657cd1ab0 +F test/pageropt.test 290cd59782b1890f02bb33795571facfc5ccac43 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test eb49937dca270b2c3f62d4c91fc7034ca905b7f1 +F test/permutations.test 694f4a2667242bab49cce05c54c2adfcc2727d9e F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -1038,7 +1038,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 022fdc986b33701abfd39621072ac3d9f9f7d43e -R e9d1b00daf0ae79f0e154b8607fa8ad8 +P f8ca5622d99bedca957caa9ad311d798f63b3ce9 +R 1add721bf7e9669d3b4d4c28d41bb13b U dan -Z 82853c015b40f8bb1665068f9eea77d4 +Z 3a3766130d03f0bab6c4936a9c26c7a5 diff --git a/manifest.uuid b/manifest.uuid index 500dcc6707..8d198c909c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8ca5622d99bedca957caa9ad311d798f63b3ce9 \ No newline at end of file +6183f1bd86ceed76d22d9762f3d7eb33262c62d1 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 5104625c35..2b7872fa8c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2130,6 +2130,19 @@ int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ return SQLITE_OK; } +/* +** Change the limit on the amount of the database file that may be +** memory mapped. +*/ +int sqlite3BtreeSetMmapSize(Btree *p, int nMap){ + BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + sqlite3PagerSetMmapsize(pBt->pPager, nMap); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} + /* ** Change the way data is synced to disk in order to increase or decrease ** how well the database resists damage due to OS crashes and power diff --git a/src/btree.h b/src/btree.h index d4c9fe37d7..f2cca2fc71 100644 --- a/src/btree.h +++ b/src/btree.h @@ -63,6 +63,7 @@ int sqlite3BtreeOpen( int sqlite3BtreeClose(Btree*); int sqlite3BtreeSetCacheSize(Btree*,int); +int sqlite3BtreeSetMmapSize(Btree*, int); int sqlite3BtreeSetSafetyLevel(Btree*,int,int,int); int sqlite3BtreeSyncDisabled(Btree*); int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); diff --git a/src/os.c b/src/os.c index b5e918a727..45d8e0c121 100644 --- a/src/os.c +++ b/src/os.c @@ -140,6 +140,9 @@ int sqlite3OsShmMap( DO_OS_MALLOC_TEST(id); return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } +int sqlite3OsMremap(sqlite3_file *id, i64 iOff, i64 nOld, i64 nNew, void **pp){ + return id->pMethods->xMremap(id, iOff, nOld, nNew, pp); +} /* ** The next group of routines are convenience wrappers around the diff --git a/src/os.h b/src/os.h index 1ec7d4ba11..76a60effdb 100644 --- a/src/os.h +++ b/src/os.h @@ -259,6 +259,7 @@ int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); +int sqlite3OsMremap(sqlite3_file *id, i64, i64, i64, void **); /* diff --git a/src/os_unix.c b/src/os_unix.c index 89326783d7..f37f2404ee 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4429,6 +4429,42 @@ static int unixShmUnmap( # define unixShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ +/* +** Map, remap or unmap part of the database file. +*/ +static int unixMremap( + sqlite3_file *fd, /* Main database file */ + sqlite3_int64 iOff, /* Offset to start mapping at */ + sqlite3_int64 nOld, /* Size of old mapping, or zero */ + sqlite3_int64 nNew, /* Size of new mapping, or zero */ + void **ppMap /* IN/OUT: Old/new mappings */ +){ + unixFile *p = (unixFile *)fd; /* The underlying database file */ + int rc = SQLITE_OK; /* Return code */ + void *pNew = 0; /* New mapping */ + + assert( iOff==0 ); + + if( nOld!=0 ){ + void *pOld = *ppMap; + munmap(pOld, nOld); + } + + if( nNew>0 ){ + int flags = PROT_READ; + if( (p->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; + nNew = (nNew+4095) & ~(i64)((1 << 12)-1); + pNew = mmap(0, nNew, flags, MAP_SHARED, p->h, iOff); + if( pNew==MAP_FAILED ){ + pNew = 0; + rc = SQLITE_IOERR; + } + } + + *ppMap = pNew; + return rc; +} + /* ** Here ends the implementation of all sqlite3_file methods. ** @@ -4487,7 +4523,8 @@ static const sqlite3_io_methods METHOD = { \ unixShmMap, /* xShmMap */ \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ - unixShmUnmap /* xShmUnmap */ \ + unixShmUnmap, /* xShmUnmap */ \ + unixMremap, /* xMremap */ \ }; \ static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ @@ -4504,7 +4541,7 @@ static const sqlite3_io_methods *(*const FINDER)(const char*,unixFile *p) \ IOMETHODS( posixIoFinder, /* Finder function name */ posixIoMethods, /* sqlite3_io_methods object name */ - 2, /* shared memory is enabled */ + 3, /* shared memory and mmap are enabled */ unixClose, /* xClose method */ unixLock, /* xLock method */ unixUnlock, /* xUnlock method */ diff --git a/src/pager.c b/src/pager.c index d9b9ed5052..9f7b349382 100644 --- a/src/pager.c +++ b/src/pager.c @@ -659,8 +659,11 @@ struct Pager { void *pMap; /* Memory mapped prefix of database file */ i64 nMap; /* Size of mapping at pMap in bytes */ i64 nMapValid; /* Bytes at pMap known to be valid */ + i64 nMapLimit; /* Maximum permitted mapping size */ + int nMapCfgLimit; /* Configured limit value */ int nMmapOut; /* Number of mmap pages currently outstanding */ PgHdr *pFree; /* List of free mmap page headers (pDirty) */ + int bMapResize; /* Check if the mapping should be resized */ /* ** End of the routinely-changing class members ***************************************************************************/ @@ -3356,6 +3359,35 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ sqlite3PcacheSetCachesize(pPager->pPCache, mxPage); } +/* +** Set Pager.nMapLimit, the maximum permitted mapping size, based on the +** current values of Pager.nMapCfgLimit and Pager.pageSize. +** +** If this connection should not use mmap at all, set nMapLimit to zero. +*/ +static void pagerFixMaplimit(Pager *pPager){ + if( isOpen(pPager->fd)==0 + || pPager->fd->pMethods->iVersion<3 + || pPager->fd->pMethods->xMremap==0 + || pPager->tempFile + || pPager->pWal + ){ + pPager->nMapLimit = 0; + }else if( pPager->nMapCfgLimit<0 ){ + pPager->nMapLimit = (i64)pPager->nMapCfgLimit * -1024; + }else{ + pPager->nMapLimit = (i64)pPager->nMapCfgLimit * pPager->pageSize; + } +} + +/* +** Change the maximum size of any memory mapping made of the database file. +*/ +void sqlite3PagerSetMmapsize(Pager *pPager, int nMap){ + pPager->nMapCfgLimit = nMap; + pagerFixMaplimit(pPager); +} + /* ** Free as much memory as possible from the pager. */ @@ -3591,6 +3623,7 @@ int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ assert( nReserve>=0 && nReserve<1000 ); pPager->nReserve = (i16)nReserve; pagerReportSize(pPager); + pagerFixMaplimit(pPager); } return rc; } @@ -3816,108 +3849,100 @@ static int pagerSyncHotJournal(Pager *pPager){ return rc; } -#include - /* -** Unmap any mapping of the database file. +** Unmap any memory mapping of the database file. */ static int pagerUnmap(Pager *pPager){ if( pPager->pMap ){ - munmap(pPager->pMap, pPager->nMap); - pPager->pMap = 0; + sqlite3OsMremap(pPager->fd, 0, pPager->nMap, 0, &pPager->pMap); pPager->nMap = 0; pPager->nMapValid = 0; } return SQLITE_OK; } +/* +** Create, or recreate, the memory mapping of the database file. +*/ static int pagerMap(Pager *pPager){ - int rc; - i64 sz = 0; + int rc = SQLITE_OK; /* Return code */ + Pgno nPg; /* Size of mapping to request in pages */ + i64 sz; /* Size of mapping to request in bytes */ - assert( pPager->pMap==0 && pPager->nMap==0 ); + assert( pPager->pWal==0 && isOpen(pPager->fd) && pPager->tempFile==0 ); + assert( pPager->pMap==0 || pPager->nMap>0 ); + assert( pPager->eState>=1 ); + assert( pPager->nMmapOut==0 ); + assert( pPager->nMapLimit>0 ); - rc = sqlite3OsFileSize(pPager->fd, &sz); - sz = sz & ~(4096-1); + /* Figure out how large a mapping to request. Set variable sz to this + ** value in bytes. */ + nPg = (pPager->eState==1) ? pPager->dbSize : pPager->dbFileSize; + sz = (i64)nPg * pPager->pageSize; + if( sz>pPager->nMapLimit ) sz = pPager->nMapLimit; - if( rc==SQLITE_OK && sz>0 ){ - int fd; - rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_GETFD, (void *)&fd); + if( sz!=pPager->nMapValid ){ + rc = sqlite3OsMremap(pPager->fd, 0, pPager->nMap, sz, &pPager->pMap); if( rc==SQLITE_OK ){ - void *pMap = mmap(0, sz, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); - if( pMap==MAP_FAILED ){ - return SQLITE_IOERR; - } - pPager->pMap = pMap; - pPager->nMapValid = pPager->nMap = sz; + assert( pPager->pMap!=0 ); + pPager->nMap = sz; + }else{ + assert( pPager->pMap==0 ); + pPager->nMap = 0; } + pPager->nMapValid = pPager->nMap; } + pPager->bMapResize = 0; return rc; } -static int pagerRemap(Pager *pPager, Pgno nPage){ - i64 sz = (i64)nPage * pPager->pageSize; - sz = sz & ~(4096-1); - - if( pPager->nMap!=sz ){ - void *pMap = mremap(pPager->pMap, pPager->nMap, sz, MREMAP_MAYMOVE); - if( pMap==MAP_FAILED ){ - return SQLITE_IOERR; - } - pPager->pMap = pMap; - pPager->nMapValid = pPager->nMap = sz; - } - - return SQLITE_OK; -} - +/* +** Obtain a reference to a memory mapped page object for page number pgno. +** The caller must ensure that page pgno lies within the currently mapped +** region. If successful, set *ppPage to point to the new page reference +** and return SQLITE_OK. Otherwise, return an SQLite error code and set +** *ppPage to zero. +** +** Page references obtained by calling this function should be released +** by calling pagerReleaseMapPage(). +*/ static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ - int rc; - *ppPage = 0; + PgHdr *p; /* Memory mapped page to return */ - assert( pPager->pWal==0 ); - - if( MEMDB==0 && pPager->tempFile==0 ){ - if( pPager->pMap==0 ){ - rc = pagerMap(pPager); - if( rc!=SQLITE_OK ) return rc; - } - - if( pgno!=1 && pPager->pMap - && pPager->nMapValid>=((i64)pgno*pPager->pageSize) - ){ - PgHdr *p; - if( pPager->pFree ){ - p = pPager->pFree; - pPager->pFree = p->pDirty; - p->pDirty = 0; - memset(p->pExtra, 0, pPager->nExtra); - }else{ - p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); - if( p==0 ) return SQLITE_NOMEM; - p->pExtra = (void *)&p[1]; - p->flags = PGHDR_MMAP; - p->nRef = 1; - p->pPager = pPager; - } - - assert( p->pExtra==(void *)&p[1] ); - assert( p->pPage==0 ); - assert( p->flags==PGHDR_MMAP ); - assert( p->pPager==pPager ); - assert( p->nRef==1 ); - - p->pData = &((u8 *)pPager->pMap)[(i64)(pgno-1) * pPager->pageSize]; - p->pgno = pgno; - pPager->nMmapOut++; - *ppPage = p; + if( pPager->pFree ){ + *ppPage = p = pPager->pFree; + pPager->pFree = p->pDirty; + p->pDirty = 0; + memset(p->pExtra, 0, pPager->nExtra); + }else{ + *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); + if( p==0 ){ + return SQLITE_NOMEM; } + p->pExtra = (void *)&p[1]; + p->flags = PGHDR_MMAP; + p->nRef = 1; + p->pPager = pPager; } + assert( p->pExtra==(void *)&p[1] ); + assert( p->pPage==0 ); + assert( p->flags==PGHDR_MMAP ); + assert( p->pPager==pPager ); + assert( p->nRef==1 ); + + p->pData = &((u8 *)pPager->pMap)[(i64)(pgno-1) * pPager->pageSize]; + p->pgno = pgno; + pPager->nMmapOut++; + return SQLITE_OK; } +/* +** Release a reference to page pPg. pPg must have been returned by an +** earlier call to pagerAcquireMapPage(). +*/ static void pagerReleaseMapPage(PgHdr *pPg){ Pager *pPager = pPg->pPager; pPager->nMmapOut--; @@ -3925,6 +3950,9 @@ static void pagerReleaseMapPage(PgHdr *pPg){ pPager->pFree = pPg; } +/* +** Free all PgHdr objects stored in the Pager.pFree list. +*/ static void pagerFreeMapHdrs(Pager *pPager){ PgHdr *p; PgHdr *pNext; @@ -5115,7 +5143,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** possibility by unmapping the db here. */ pagerUnmap(pPager); }else if( pPager->pMap ){ - pagerRemap(pPager, nPage); + pPager->bMapResize = 1; } } @@ -5226,8 +5254,7 @@ int sqlite3PagerAcquire( ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY ** flag was specified by the caller. And so long as the db is not a ** temporary or in-memory database. */ - const int bMmapOk = ( - (pgno!=1 && pPager->pWal==0 && !pPager->tempFile && !MEMDB) + const int bMmapOk = (pPager->nMapLimit>0 && pgno!=1 && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY)) ); @@ -5245,7 +5272,7 @@ int sqlite3PagerAcquire( }else{ if( bMmapOk ){ - if( pPager->pMap==0 ){ + if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){ rc = pagerMap(pPager); } if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){ @@ -7014,6 +7041,7 @@ static int pagerOpenWal(Pager *pPager){ pPager->journalSizeLimit, &pPager->pWal ); } + pagerFixMaplimit(pPager); return rc; } @@ -7104,6 +7132,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ rc = sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; + pagerFixMaplimit(pPager); } } return rc; diff --git a/src/pragma.c b/src/pragma.c index f6dadbd4a8..9b31797d3f 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -744,6 +744,30 @@ void sqlite3Pragma( } }else + /* + ** PRAGMA [database.]mmap_size + ** PRAGMA [database.]mmap_size=N + ** + ** Used to set or query the mapping size limit. The mapping size limit is + ** used to limit the aggregate size of all memory mapped regions of the + ** database file. If this parameter is set to zero, then memory mapping + ** is not used at all. If it is set to a positive value, then it is + ** interpreted as a maximum size in pages. If set to less than zero, then + ** the absolute value is interpreted as a size limit in KB. + ** + ** The default value is zero (do not use memory mapped IO). + */ + if( sqlite3StrICmp(zLeft,"mmap_size")==0 ){ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( !zRight ){ + returnSingleInt(pParse, "mmap_size", pDb->pSchema->mmap_size); + }else{ + int size = sqlite3Atoi(zRight); + pDb->pSchema->mmap_size = size; + sqlite3BtreeSetMmapSize(pDb->pBt, pDb->pSchema->mmap_size); + } + }else + /* ** PRAGMA temp_store ** PRAGMA temp_store = "default"|"memory"|"file" diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 0ddb0dd105..3fa30e01de 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -706,6 +706,24 @@ struct sqlite3_file { ** fails to zero-fill short reads might seem to work. However, ** failure to zero-fill short reads will eventually lead to ** database corruption. +** +** Assuming parameter nNew is non-zero, the xMremap method should attempt +** to memory map a region nNew bytes in size starting at offset iOffset +** of the file. If successful, it should set *ppMap to point to the +** mapping and return SQLITE_OK. If the file is opened for read-write +** access, then the mapping should also be read-write. +** +** If nOld is non-zero, then the initial value of *ppMap points to a +** mapping returned by a previous call to xMremap. The existing mapping +** is nOld bytes in size and starts at offset iOffset of the file. In +** this case the xMremap method is expected to unmap the existing mapping +** and overwrite *ppMap with the pointer to the new mapping. If nOld is +** zero, then the initial value of *ppMap is undefined. +** +** If nNew is zero, then no new mapping should be created. Any old +** mapping must still be unmapped if nOld is non-zero. If the nOld +** parameter is non-zero, then the existing mapping is always unmapped - +** even if an error occurs. */ typedef struct sqlite3_io_methods sqlite3_io_methods; struct sqlite3_io_methods { @@ -728,6 +746,9 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ + int (*xMremap)(sqlite3_file *fd, + sqlite3_int64 iOff, sqlite3_int64 nOld, sqlite3_int64 nNew, void **ppMap); + /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 129c4c5ea5..59483bc87b 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -746,6 +746,7 @@ struct Schema { u8 enc; /* Text encoding used by this database */ u16 flags; /* Flags associated with this schema */ int cache_size; /* Number of pages to use in the cache */ + int mmap_size; /* Number of pages to memory map */ }; /* diff --git a/src/test1.c b/src/test1.c index a89b874a3b..500d60e0fe 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5859,9 +5859,9 @@ static int test_getrusage( getrusage(RUSAGE_SELF, &r); sprintf(buf, "ru_utime=%d.%06d ru_stime=%d.%06d ru_minflt=%d ru_majflt=%d", - r.ru_utime.tv_sec, r.ru_utime.tv_usec, - r.ru_stime.tv_sec, r.ru_stime.tv_usec, - r.ru_minflt, r.ru_majflt + (int)r.ru_utime.tv_sec, (int)r.ru_utime.tv_usec, + (int)r.ru_stime.tv_sec, (int)r.ru_stime.tv_usec, + (int)r.ru_minflt, (int)r.ru_majflt ); Tcl_SetObjResult(interp, Tcl_NewStringObj(buf, -1)); return TCL_OK; diff --git a/test/dbstatus2.test b/test/dbstatus2.test index 18bb0870bd..2541a1a823 100644 --- a/test/dbstatus2.test +++ b/test/dbstatus2.test @@ -40,6 +40,7 @@ proc db_write {db {reset 0}} { do_test 1.1 { db close sqlite3 db test.db + execsql { PRAGMA mmap_size = 0 } expr {[file size test.db] / 1024} } 6 diff --git a/test/exclusive2.test b/test/exclusive2.test index 2208da5101..54203e3d8f 100644 --- a/test/exclusive2.test +++ b/test/exclusive2.test @@ -25,6 +25,14 @@ ifcapable {!pager_pragmas} { return } +# Tests in this file verify that locking_mode=exclusive causes SQLite to +# use cached pages even if the database is changed on disk. This doesn't +# work with mmap. +if {[permutation]=="mmap"} { + finish_test + return +} + # This module does not work right if the cache spills at unexpected # moments. So disable the soft-heap-limit. # diff --git a/test/func.test b/test/func.test index f09ff49805..4ab7688461 100644 --- a/test/func.test +++ b/test/func.test @@ -1273,11 +1273,13 @@ do_test func-29.3 { sqlite3_db_status db CACHE_MISS 1 db eval {SELECT typeof(+x) FROM t29 ORDER BY id} } {integer null real blob text} -do_test func-29.4 { - set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] - if {$x>100} {set x many} - set x -} {many} +if {[permutation] != "mmap"} { + do_test func-29.4 { + set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] + if {$x>100} {set x many} + set x + } {many} +} do_test func-29.5 { db close sqlite3 db test.db diff --git a/test/incrblob.test b/test/incrblob.test index 7cc99dd983..f51f5852d4 100644 --- a/test/incrblob.test +++ b/test/incrblob.test @@ -123,6 +123,7 @@ foreach AutoVacuumMode [list 0 1] { forcedelete test.db test.db-journal sqlite3 db test.db + execsql "PRAGMA mmap_size = 0" execsql "PRAGMA auto_vacuum = $AutoVacuumMode" do_test incrblob-2.$AutoVacuumMode.1 { @@ -149,6 +150,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db + execsql "PRAGMA mmap_size = 0" # Read the last 20 bytes of the blob via a blob handle. set ::blob [db incrblob blobs v 1] @@ -171,6 +173,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db + execsql "PRAGMA mmap_size = 0" # Write the second-to-last 20 bytes of the blob via a blob handle. # @@ -200,6 +203,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db + execsql { PRAGMA mmap_size = 0 } execsql { SELECT i FROM blobs } } {45} diff --git a/test/pageropt.test b/test/pageropt.test index 82311965a5..de39bbaf05 100644 --- a/test/pageropt.test +++ b/test/pageropt.test @@ -92,7 +92,7 @@ do_test pageropt-1.5 { pagercount_sql { SELECT hex(x) FROM t1 } -} [list 6 0 0 $blobcontent] +} [list [expr {[permutation]=="mmap" ? 1 : 6}] 0 0 $blobcontent] do_test pageropt-1.6 { pagercount_sql { SELECT hex(x) FROM t1 diff --git a/test/permutations.test b/test/permutations.test index d9014d77fe..fd05b58e4f 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -134,6 +134,14 @@ test_suite "veryquick" -prefix "" -description { "Very" quick test suite. Runs in less than 5 minutes on a workstation. This test suite is the same as the "quick" tests, except that some files that test malloc and IO errors are omitted. +} -files [ + test_set $allquicktests -exclude *malloc* *ioerr* *fault* +] + +test_suite "mmap" -prefix "mm-" -description { + Similar to veryquick. Except with memory mapping enabled. +} -presql { + pragma mmap_size = -65536; } -files [ test_set $allquicktests -exclude *malloc* *ioerr* *fault* \ multiplex* server1.test shared2.test shared6.test From eecc3983741f501203faf8404eca56332418696e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 20 Mar 2013 10:07:43 +0000 Subject: [PATCH 009/204] Add test file mmap1.test. FossilOrigin-Name: aee1f53a74e636776cbbc11bdd5516432ad50533 --- manifest | 11 ++++--- manifest.uuid | 2 +- test/mmap1.test | 84 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 test/mmap1.test diff --git a/manifest b/manifest index 45c3ee6fc5..1a8ba2228d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssqlite3_io_methods.xMremap()\smethod\sto\sthe\sVFS\sinterface.\sAlso\s"PRAGMA\smmap_size". -D 2013-03-19T19:28:06.473 +C Add\stest\sfile\smmap1.test. +D 2013-03-20T10:07:43.411 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -630,6 +630,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 +F test/mmap1.test 46ff4038b6d300bb9b39d3f89ea1aef3dff9b88f F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 @@ -1038,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f8ca5622d99bedca957caa9ad311d798f63b3ce9 -R 1add721bf7e9669d3b4d4c28d41bb13b +P 6183f1bd86ceed76d22d9762f3d7eb33262c62d1 +R 1501a2a578b96ab1d025c2aefaaa8f35 U dan -Z 3a3766130d03f0bab6c4936a9c26c7a5 +Z a7aa76ce5b116a8f55025f0891a0dfbb diff --git a/manifest.uuid b/manifest.uuid index 8d198c909c..68ed5bb6a8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6183f1bd86ceed76d22d9762f3d7eb33262c62d1 \ No newline at end of file +aee1f53a74e636776cbbc11bdd5516432ad50533 \ No newline at end of file diff --git a/test/mmap1.test b/test/mmap1.test new file mode 100644 index 0000000000..343ad230b1 --- /dev/null +++ b/test/mmap1.test @@ -0,0 +1,84 @@ +# 2013 March 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set testprefix mmap1 + +proc nRead {db} { + set bt [btree_from_db $db] + db_enter $db + array set stats [btree_pager_stats $bt] + db_leave $db + return $stats(read) +} + +foreach {t mmap_size nRead} { + 1 { PRAGMA mmap_size = -65536 } 4 + 2 { PRAGMA mmap_size = -50 } 156 + 3 { PRAGMA mmap_size = 0 } 344 +} { + do_multiclient_test tn { + sql1 $mmap_size + + code2 { + set ::rcnt 0 + proc rblob {n} { + set ::rcnt [expr (($::rcnt << 3) + $::rcnt + 456) & 0xFFFFFFFF] + set str [format %.8x [expr $::rcnt ^ 0xbdf20da3]] + string range [string repeat $str [expr $n/4]] 1 $n + } + db2 func rblob rblob + } + + sql2 { + PRAGMA auto_vacuum = 1; + CREATE TABLE t1(a, b, UNIQUE(a, b)); + INSERT INTO t1 VALUES(rblob(500), rblob(500)); + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 2 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 4 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 8 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 32 + } + do_test $t.$tn.1 { + sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count" + } {32 ok 77} + + # Have connection 2 shrink the file. Check connection 1 can still read it. + sql2 { DELETE FROM t1 WHERE rowid%2; } + do_test $t.$tn.2 { + sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count" + } {16 ok 42} + + # Have connection 2 grow the file. Check connection 1 can still read it. + sql2 { INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1 } + do_test $t.$tn.3 { + sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count" + } {32 ok 79} + + # Have connection 2 grow the file again. Check connection 1 is still ok. + sql2 { INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1 } + do_test $t.$tn.4 { + sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count" + } {64 ok 149} + + # Check that the number of pages read by connection 1 indicates that the + # "PRAGMA mmap_size" command worked. + do_test $t.$tn.5 { nRead db } $nRead + } +} + + +finish_test + From 0e5fba790a07c2d3697b63a81d4e068be32dc707 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 20 Mar 2013 12:04:29 +0000 Subject: [PATCH 010/204] Fix text-to-numeric type casting so that it works correctly on UTF16 strings that contain characters where the LSB is numeric but the MSB is non-zero. Ticket [689137afb6da41] FossilOrigin-Name: 5b22053f918d16f593227a432a5d5b4c195bb0b5 --- manifest | 13 +++++++------ manifest.uuid | 2 +- src/util.c | 38 +++++++++++++++++++++++++++++++------- test/numcast.test | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 test/numcast.test diff --git a/manifest b/manifest index 4e0fb3f191..311e065ca2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Bring\smakefiles\sand\sbuild\sscripts\sinto\salignment\swith\sthe\ssessions\sbranch.\nNo\schanges\sto\scode. -D 2013-03-19T16:12:40.639 +C Fix\stext-to-numeric\stype\scasting\sso\sthat\sit\sworks\scorrectly\son\sUTF16\nstrings\sthat\scontain\scharacters\swhere\sthe\sLSB\sis\snumeric\sbut\sthe\sMSB\nis\snon-zero.\s\sTicket\s[689137afb6da41] +D 2013-03-20T12:04:29.291 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -237,7 +237,7 @@ F src/tokenize.c 1e86210d3976717a19238ea7b047fac481fe8c12 F src/trigger.c cd95ac64efa60e39faf9b5597443192ff27a22fa F src/update.c 28d2d098b43a2c70dae399896ea8a02f622410ef F src/utf.c 8d819e2e5104a430fc2005f018db14347c95a38f -F src/util.c 0af2e515dc0dabacec931bca39525f6c3f1c5455 +F src/util.c 550f2b6a5c0085153a4d00462719fb17ee242792 F src/vacuum.c 2727bdd08847fcb6b2d2da6d14f018910e8645d3 F src/vdbe.c 292f8f7ced59c29c63fe17830cbe5f5a0230cdf0 F src/vdbe.h b52887278cb173e66188da84dfab216bea61119d @@ -641,6 +641,7 @@ F test/notify2.test 9503e51b9a272a5405c205ad61b7623d5a9ca489 F test/notify3.test a86259abbfb923aa27d30f0fc038c88e5251488a F test/notnull.test 2afad748d18fd66d01f66463de73b3e2501fb226 F test/null.test a8b09b8ed87852742343b33441a9240022108993 +F test/numcast.test 5d126f7f581432e86a90d1e35cac625164aec4a1 F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 F test/orderby1.test f33968647da5c546528fe4d2bf86c6a6a2e5a7ae F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 @@ -1038,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 4fe2db1d866c80fe51f7fddbf9ce6753fb55b5f6 -R 7758dd15e3a5f0387c3f17443ec025d8 +P d1f41089aba075eef45fd696599e5d3a74c84d0c +R ccf3378e4accef01a0ea31a61ac073d2 U drh -Z 7c5d1dfbeba3270fc7d5841fae61579f +Z 400c45055ef48421e523d8328d092a1b diff --git a/manifest.uuid b/manifest.uuid index ac1d43b9fa..f861049e0c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d1f41089aba075eef45fd696599e5d3a74c84d0c \ No newline at end of file +5b22053f918d16f593227a432a5d5b4c195bb0b5 \ No newline at end of file diff --git a/src/util.c b/src/util.c index 5cf8ebacb5..937067a06f 100644 --- a/src/util.c +++ b/src/util.c @@ -261,7 +261,7 @@ int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ */ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ #ifndef SQLITE_OMIT_FLOATING_POINT - int incr = (enc==SQLITE_UTF8?1:2); + int incr; const char *zEnd = z + length; /* sign * significand * (10 ^ (esign * exponent)) */ int sign = 1; /* sign of significand */ @@ -272,10 +272,22 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ int eValid = 1; /* True exponent is either not used or is well-formed */ double result; int nDigits = 0; + int nonNum = 0; + assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); *pResult = 0.0; /* Default return value, in case of an error */ - if( enc==SQLITE_UTF16BE ) z++; + if( enc==SQLITE_UTF8 ){ + incr = 1; + }else{ + int i; + incr = 2; + assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + for(i=3-enc; i=zEnd && nDigits>0 && eValid; + return z>=zEnd && nDigits>0 && eValid && nonNum==0; #else return !sqlite3Atoi64(z, pResult, length, enc); #endif /* SQLITE_OMIT_FLOATING_POINT */ @@ -457,21 +469,33 @@ static int compare2pow63(const char *zNum, int incr){ ** signed 64-bit integer, its negative -9223372036854665808 can be. ** ** If zNum is too big for a 64-bit integer and is not -** 9223372036854665808 then return 1. +** 9223372036854665808 or if zNum contains any non-numeric text, +** then return 1. ** ** length is the number of bytes in the string (bytes, not characters). ** The string is not necessarily zero-terminated. The encoding is ** given by enc. */ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ - int incr = (enc==SQLITE_UTF8?1:2); + int incr; u64 u = 0; int neg = 0; /* assume positive */ int i; int c = 0; + int nonNum = 0; const char *zStart; const char *zEnd = zNum + length; - if( enc==SQLITE_UTF16BE ) zNum++; + assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); + if( enc==SQLITE_UTF8 ){ + incr = 1; + }else{ + incr = 2; + assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + for(i=3-enc; i19*incr ){ + if( (c+nonNum!=0 && &zNum[i]19*incr ){ /* zNum is empty or contains non-numeric text or is longer ** than 19 digits (thus guaranteeing that it is too large) */ return 1; diff --git a/test/numcast.test b/test/numcast.test new file mode 100644 index 0000000000..926bbe4b21 --- /dev/null +++ b/test/numcast.test @@ -0,0 +1,46 @@ +# 2013 March 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# This particular file does testing of casting strings into numeric +# values. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +foreach enc {utf8 utf16le utf16be} { + do_test numcast-$enc.0 { + db close + sqlite3 db :memory: + db eval "PRAGMA encoding='$enc'" + set x [db eval {PRAGMA encoding}] + string map {- {}} [string tolower $x] + } $enc + foreach {idx str rval ival} { + 1 12345.0 12345.0 12345 + 2 12345.0e0 12345.0 12345 + 3 -12345.0e0 -12345.0 -12345 + 4 -12345.25 -12345.25 -12345 + 5 { -12345.0} -12345.0 -12345 + 6 { 876xyz} 876.0 876 + 7 { 456ķ89} 456.0 456 + 8 { Ġ 321.5} 0.0 0 + } { + do_test numcast-$enc.$idx.1 { + db eval {SELECT CAST($str AS real)} + } $rval + do_test numcast-$enc.$idx.2 { + db eval {SELECT CAST($str AS integer)} + } $ival + } +} + +finish_test From eb97b29345592cf094b6639a985f9253653a5063 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 20 Mar 2013 14:26:59 +0000 Subject: [PATCH 011/204] When possible, use memory mapping when appending new pages to a database file. FossilOrigin-Name: 14135da3cdbafd699563a29608f32347cda28338 --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/os.c | 11 +++++++++-- src/os.h | 2 +- src/os_unix.c | 33 ++++++++++++++++++++++++++++++--- src/pager.c | 18 +++++++++++++----- src/pager.h | 1 + src/sqlite.h.in | 5 ++++- 8 files changed, 70 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 1a8ba2228d..d519ad3e2a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\sfile\smmap1.test. -D 2013-03-20T10:07:43.411 +C When\spossible,\suse\smemory\smapping\swhen\sappending\snew\spages\sto\sa\sdatabase\sfile. +D 2013-03-20T14:26:59.370 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,13 +157,13 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c 81a82a736b8a461a656f9b3e401a39768fe73a79 -F src/os.h 4681261aa24a9d2187aaf4cb963880e6cddb1f48 +F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 +F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 0c7b0d076f2ac6279b0b280a26bcae8c89f36f4f +F src/os_unix.c aedd47c2145edd49051452afbe82fa6f54a49b2a F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 78b65bf9685bf21b787ce2a7389c2b96102942dc -F src/pager.h 81ac95f4fcfe21981f495146f6d7f2fe51afd110 +F src/pager.c d59af9a70aa2d7222b127351fa3cbe70660e4150 +F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 @@ -176,7 +176,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in fd75f5bcf479b315b1c717fa1d07f018bd919f79 +F src/sqlite.h.in d63c7fb5832287af7e8b903c4a4c30c90414876f F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 6183f1bd86ceed76d22d9762f3d7eb33262c62d1 -R 1501a2a578b96ab1d025c2aefaaa8f35 +P aee1f53a74e636776cbbc11bdd5516432ad50533 +R 041b09b1e247c8054516280d9526a179 U dan -Z a7aa76ce5b116a8f55025f0891a0dfbb +Z 8c309d7c56b08fcba504ca346091b196 diff --git a/manifest.uuid b/manifest.uuid index 68ed5bb6a8..f40916822a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aee1f53a74e636776cbbc11bdd5516432ad50533 \ No newline at end of file +14135da3cdbafd699563a29608f32347cda28338 \ No newline at end of file diff --git a/src/os.c b/src/os.c index 45d8e0c121..38757dcdb1 100644 --- a/src/os.c +++ b/src/os.c @@ -140,8 +140,15 @@ int sqlite3OsShmMap( DO_OS_MALLOC_TEST(id); return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } -int sqlite3OsMremap(sqlite3_file *id, i64 iOff, i64 nOld, i64 nNew, void **pp){ - return id->pMethods->xMremap(id, iOff, nOld, nNew, pp); +int sqlite3OsMremap( + sqlite3_file *id, /* Database file handle */ + int flags, /* SQLITE_MREMAP_XXX flags */ + i64 iOff, /* Offset at which mapping(s) start */ + i64 nOld, /* Size of old mapping */ + i64 nNew, /* Size of requested mapping */ + void **pp /* IN/OUT: Pointer to mapped region */ +){ + return id->pMethods->xMremap(id, flags, iOff, nOld, nNew, pp); } /* diff --git a/src/os.h b/src/os.h index 76a60effdb..f08b92dcf0 100644 --- a/src/os.h +++ b/src/os.h @@ -259,7 +259,7 @@ int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); -int sqlite3OsMremap(sqlite3_file *id, i64, i64, i64, void **); +int sqlite3OsMremap(sqlite3_file *id, int, i64, i64, i64, void **); /* diff --git a/src/os_unix.c b/src/os_unix.c index f37f2404ee..1c26c6badf 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4434,6 +4434,7 @@ static int unixShmUnmap( */ static int unixMremap( sqlite3_file *fd, /* Main database file */ + int flags, /* Mask of SQLITE_MREMAP_XXX flags */ sqlite3_int64 iOff, /* Offset to start mapping at */ sqlite3_int64 nOld, /* Size of old mapping, or zero */ sqlite3_int64 nNew, /* Size of new mapping, or zero */ @@ -4442,8 +4443,35 @@ static int unixMremap( unixFile *p = (unixFile *)fd; /* The underlying database file */ int rc = SQLITE_OK; /* Return code */ void *pNew = 0; /* New mapping */ + i64 nRnd; /* nNew rounded up to 4096 */ assert( iOff==0 ); + nRnd = (nNew+4095) & ~(i64)((1 << 12)-1); + + /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested + ** mapping (nNew bytes) may be greater than the size of the database file. + ** If this is the case, extend the file on disk using ftruncate(). */ + assert( nNew>0 || (flags & SQLITE_MREMAP_EXTEND)==0 ); + if( flags & SQLITE_MREMAP_EXTEND ){ + struct stat statbuf; /* Low-level file information */ + rc = osFstat(p->h, &statbuf); + if( rc==SQLITE_OK && nNew>statbuf.st_size ){ + rc = robust_ftruncate(p->h, nNew); + } + if( rc!=SQLITE_OK ) return rc; + } + +#if defined(_GNU_SOURCE) && defined(__linux__) + if( nRnd && nOld ){ + void *pOld = *ppMap; + *ppMap = pNew = mremap(pOld, nOld, nNew, MREMAP_MAYMOVE); + if( pNew==MAP_FAILED ){ + *ppMap = 0; + return SQLITE_IOERR_MREMAP; + } + return SQLITE_OK; + } +#endif if( nOld!=0 ){ void *pOld = *ppMap; @@ -4453,11 +4481,10 @@ static int unixMremap( if( nNew>0 ){ int flags = PROT_READ; if( (p->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - nNew = (nNew+4095) & ~(i64)((1 << 12)-1); - pNew = mmap(0, nNew, flags, MAP_SHARED, p->h, iOff); + pNew = mmap(0, nRnd, flags, MAP_SHARED, p->h, iOff); if( pNew==MAP_FAILED ){ pNew = 0; - rc = SQLITE_IOERR; + rc = SQLITE_IOERR_MREMAP; } } diff --git a/src/pager.c b/src/pager.c index 9f7b349382..2848f9ad05 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3854,7 +3854,7 @@ static int pagerSyncHotJournal(Pager *pPager){ */ static int pagerUnmap(Pager *pPager){ if( pPager->pMap ){ - sqlite3OsMremap(pPager->fd, 0, pPager->nMap, 0, &pPager->pMap); + sqlite3OsMremap(pPager->fd, 0, 0, pPager->nMap, 0, &pPager->pMap); pPager->nMap = 0; pPager->nMapValid = 0; } @@ -3864,7 +3864,7 @@ static int pagerUnmap(Pager *pPager){ /* ** Create, or recreate, the memory mapping of the database file. */ -static int pagerMap(Pager *pPager){ +static int pagerMap(Pager *pPager, int bExtend){ int rc = SQLITE_OK; /* Return code */ Pgno nPg; /* Size of mapping to request in pages */ i64 sz; /* Size of mapping to request in bytes */ @@ -3882,7 +3882,8 @@ static int pagerMap(Pager *pPager){ if( sz>pPager->nMapLimit ) sz = pPager->nMapLimit; if( sz!=pPager->nMapValid ){ - rc = sqlite3OsMremap(pPager->fd, 0, pPager->nMap, sz, &pPager->pMap); + int flags = (bExtend ? SQLITE_MREMAP_EXTEND : 0); + rc = sqlite3OsMremap(pPager->fd, flags, 0, pPager->nMap, sz, &pPager->pMap); if( rc==SQLITE_OK ){ assert( pPager->pMap!=0 ); pPager->nMap = sz; @@ -4246,10 +4247,17 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ ** file size will be. */ assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); - if( rc==SQLITE_OK && pPager->dbSize>pPager->dbHintSize ){ + if( rc==SQLITE_OK + && (pList->pDirty ? pPager->dbSize : pList->pgno+1)>pPager->dbHintSize + ){ sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); pPager->dbHintSize = pPager->dbSize; + + if( pPager->nMmapOut==0 && pPager->nMapLimit>0 ){ + pPager->dbFileSize = pPager->dbSize; + pagerMap(pPager, 1); + } } while( rc==SQLITE_OK && pList ){ @@ -5273,7 +5281,7 @@ int sqlite3PagerAcquire( if( bMmapOk ){ if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){ - rc = pagerMap(pPager); + rc = pagerMap(pPager, 0); } if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){ if( pPager->eState>PAGER_READER ){ diff --git a/src/pager.h b/src/pager.h index 94957dac60..81ab30c115 100644 --- a/src/pager.h +++ b/src/pager.h @@ -108,6 +108,7 @@ void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); int sqlite3PagerSetPagesize(Pager*, u32*, int); int sqlite3PagerMaxPageCount(Pager*, int); void sqlite3PagerSetCachesize(Pager*, int); +void sqlite3PagerSetMmapsize(Pager *, int); void sqlite3PagerShrink(Pager*); void sqlite3PagerSetSafetyLevel(Pager*,int,int,int); int sqlite3PagerLockingMode(Pager *, int); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 3fa30e01de..c0a2803787 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -470,6 +470,7 @@ int sqlite3_exec( #define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) #define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) #define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) +#define SQLITE_IOERR_MREMAP (SQLITE_IOERR | (24<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) @@ -746,12 +747,14 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ - int (*xMremap)(sqlite3_file *fd, + int (*xMremap)(sqlite3_file *fd, int flags, sqlite3_int64 iOff, sqlite3_int64 nOld, sqlite3_int64 nNew, void **ppMap); /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; +#define SQLITE_MREMAP_EXTEND 0x0001 /* xMremap call may extend file */ + /* ** CAPI3REF: Standard File Control Opcodes ** From d306e1a3a113bd3905c6b661bcb8176b0bb6a844 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 20 Mar 2013 18:25:49 +0000 Subject: [PATCH 012/204] Optimize the xMremap method in os_unix.c some. FossilOrigin-Name: 9529ed88a71fee02fae72dc86f0669bd6856ff92 --- manifest | 12 +++++----- manifest.uuid | 2 +- src/os_unix.c | 61 ++++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 60 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index d519ad3e2a..c9eb696178 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\spossible,\suse\smemory\smapping\swhen\sappending\snew\spages\sto\sa\sdatabase\sfile. -D 2013-03-20T14:26:59.370 +C Optimize\sthe\sxMremap\smethod\sin\sos_unix.c\ssome. +D 2013-03-20T18:25:49.737 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c aedd47c2145edd49051452afbe82fa6f54a49b2a +F src/os_unix.c c96bdc9e912c0f8226a03cd025566b5a43e1c387 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 F src/pager.c d59af9a70aa2d7222b127351fa3cbe70660e4150 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P aee1f53a74e636776cbbc11bdd5516432ad50533 -R 041b09b1e247c8054516280d9526a179 +P 14135da3cdbafd699563a29608f32347cda28338 +R ae7e97a190e1fdc49e8b167e558562e7 U dan -Z 8c309d7c56b08fcba504ca346091b196 +Z a86246ff854d247b1925a564cb8b08dc diff --git a/manifest.uuid b/manifest.uuid index f40916822a..3037557cd4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -14135da3cdbafd699563a29608f32347cda28338 \ No newline at end of file +9529ed88a71fee02fae72dc86f0669bd6856ff92 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 1c26c6badf..7ab61ed0b3 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4429,6 +4429,16 @@ static int unixShmUnmap( # define unixShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ +/* +** Arguments x and y are both integers. Argument y must be a power of 2. +** Round x up to the nearest integer multiple of y. For example: +** +** ROUNDUP(0, 8) -> 0 +** ROUNDUP(13, 8) -> 16 +** ROUNDUP(32, 8) -> 32 +*/ +#define ROUNDUP(x,y) (((x)+y-1)&~(y-1)) + /* ** Map, remap or unmap part of the database file. */ @@ -4443,10 +4453,10 @@ static int unixMremap( unixFile *p = (unixFile *)fd; /* The underlying database file */ int rc = SQLITE_OK; /* Return code */ void *pNew = 0; /* New mapping */ - i64 nRnd; /* nNew rounded up to 4096 */ + i64 nNewRnd; /* nNew rounded up */ + i64 nOldRnd; /* nOld rounded up */ assert( iOff==0 ); - nRnd = (nNew+4095) & ~(i64)((1 << 12)-1); /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested ** mapping (nNew bytes) may be greater than the size of the database file. @@ -4461,10 +4471,43 @@ static int unixMremap( if( rc!=SQLITE_OK ) return rc; } + /* According to some sources, the effect of changing the size of the + ** underlying file on mapped regions that correspond to the added or + ** removed pages is undefined. However, there is reason to believe that + ** on modern platforms like Linux or OSX, things just work. For example, + ** it is possible to create a mapping larger than the file on disk and + ** extend the file on disk later on. + ** + ** Exploit this on OSX to reduce the number of munmap()/mmap() calls + ** if the file size is changing. In this case all mappings are rounded + ** up to the nearest 4MB. And if a new mapping is requested that has the + ** same rounded size as an old mapping, the old mapping can simply be + ** reused as is. + ** + ** It would be possible to do the above on Linux too. However, Linux has + ** the non-standard mremap() call to resize existing mappings, which can + ** be used instead. */ +#if defined(__APPLE__) + nNewRnd = ROUNDUP(nNew, 4096*1024); + nOldRnd = ROUNDUP(nOld, 4096*1024); +#else + nNewRnd = ROUNDUP(nNew, 4096*1); + nOldRnd = ROUNDUP(nOld, 4096*1); +#endif + + /* On OSX or Linux, reuse the old mapping if it is the right size. */ +#if defined(__APPLE__) || defined(__linux__) + if( nNewRnd==nOldRnd ){ + return SQLITE_OK; + } +#endif + + /* On Linux, if there is both an old and new mapping, resize the old + ** mapping using the non-standard mremap() call. */ #if defined(_GNU_SOURCE) && defined(__linux__) - if( nRnd && nOld ){ + if( nNewRnd && nOldRnd ){ void *pOld = *ppMap; - *ppMap = pNew = mremap(pOld, nOld, nNew, MREMAP_MAYMOVE); + *ppMap = pNew = mremap(pOld, nOldRnd, nNewRnd, MREMAP_MAYMOVE); if( pNew==MAP_FAILED ){ *ppMap = 0; return SQLITE_IOERR_MREMAP; @@ -4473,15 +4516,17 @@ static int unixMremap( } #endif - if( nOld!=0 ){ + /* If we get this far, unmap any old mapping. */ + if( nOldRnd!=0 ){ void *pOld = *ppMap; - munmap(pOld, nOld); + munmap(pOld, nOldRnd); } - if( nNew>0 ){ + /* And, if required, use mmap() to create a new mapping. */ + if( nNewRnd>0 ){ int flags = PROT_READ; if( (p->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - pNew = mmap(0, nRnd, flags, MAP_SHARED, p->h, iOff); + pNew = mmap(0, nNewRnd, flags, MAP_SHARED, p->h, iOff); if( pNew==MAP_FAILED ){ pNew = 0; rc = SQLITE_IOERR_MREMAP; From c71b45e61944f6964b384614f3da737eff2c2dd6 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 21 Mar 2013 14:47:47 +0000 Subject: [PATCH 013/204] Do not use the Linux mremap() call. Use the same strategy for xMremap() as on OSX instead. FossilOrigin-Name: 5ed8ad780c991d2ca44003ee84350fb5e95ad58e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 30 ++++++------------------------ 3 files changed, 13 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index c9eb696178..870e5cce8d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Optimize\sthe\sxMremap\smethod\sin\sos_unix.c\ssome. -D 2013-03-20T18:25:49.737 +C Do\snot\suse\sthe\sLinux\smremap()\scall.\sUse\sthe\ssame\sstrategy\sfor\sxMremap()\sas\son\sOSX\sinstead. +D 2013-03-21T14:47:47.427 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c c96bdc9e912c0f8226a03cd025566b5a43e1c387 +F src/os_unix.c be66c31337361a72227638d6f41c7f2031739642 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 F src/pager.c d59af9a70aa2d7222b127351fa3cbe70660e4150 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 14135da3cdbafd699563a29608f32347cda28338 -R ae7e97a190e1fdc49e8b167e558562e7 +P 9529ed88a71fee02fae72dc86f0669bd6856ff92 +R 7e61f02cabbded666ed9e0992a636a37 U dan -Z a86246ff854d247b1925a564cb8b08dc +Z 67f89f7e2101ab4157bb24e7a7ac302a diff --git a/manifest.uuid b/manifest.uuid index 3037557cd4..f10e3bc724 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9529ed88a71fee02fae72dc86f0669bd6856ff92 \ No newline at end of file +5ed8ad780c991d2ca44003ee84350fb5e95ad58e \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 7ab61ed0b3..153cf5f435 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4478,16 +4478,12 @@ static int unixMremap( ** it is possible to create a mapping larger than the file on disk and ** extend the file on disk later on. ** - ** Exploit this on OSX to reduce the number of munmap()/mmap() calls - ** if the file size is changing. In this case all mappings are rounded - ** up to the nearest 4MB. And if a new mapping is requested that has the - ** same rounded size as an old mapping, the old mapping can simply be - ** reused as is. - ** - ** It would be possible to do the above on Linux too. However, Linux has - ** the non-standard mremap() call to resize existing mappings, which can - ** be used instead. */ -#if defined(__APPLE__) + ** Exploit this on Linux and OSX to reduce the number of munmap()/mmap() + ** calls required if the file size is changing. In this case all mappings + ** are rounded up to the nearest 4MB. And if a new mapping is requested + ** that has the same rounded size as an old mapping, the old mapping can + ** be reused as is. */ +#if defined(__APPLE__) || defined(__linux__) nNewRnd = ROUNDUP(nNew, 4096*1024); nOldRnd = ROUNDUP(nOld, 4096*1024); #else @@ -4502,20 +4498,6 @@ static int unixMremap( } #endif - /* On Linux, if there is both an old and new mapping, resize the old - ** mapping using the non-standard mremap() call. */ -#if defined(_GNU_SOURCE) && defined(__linux__) - if( nNewRnd && nOldRnd ){ - void *pOld = *ppMap; - *ppMap = pNew = mremap(pOld, nOldRnd, nNewRnd, MREMAP_MAYMOVE); - if( pNew==MAP_FAILED ){ - *ppMap = 0; - return SQLITE_IOERR_MREMAP; - } - return SQLITE_OK; - } -#endif - /* If we get this far, unmap any old mapping. */ if( nOldRnd!=0 ){ void *pOld = *ppMap; From 5b068f863228ba1f074e2999a7dc1527e34d9612 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 21 Mar 2013 15:57:32 +0000 Subject: [PATCH 014/204] Fix a problem when opening a write-transaction while there exist read-only b-tree cursors in mmap mode. FossilOrigin-Name: 32e0bbb73609ac3ad096a60f1de3095bc79fb0cc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 870e5cce8d..8819c52a55 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\suse\sthe\sLinux\smremap()\scall.\sUse\sthe\ssame\sstrategy\sfor\sxMremap()\sas\son\sOSX\sinstead. -D 2013-03-21T14:47:47.427 +C Fix\sa\sproblem\swhen\sopening\sa\swrite-transaction\swhile\sthere\sexist\sread-only\sb-tree\scursors\sin\smmap\smode. +D 2013-03-21T15:57:32.562 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c d4d551f05a555926a7c0f49c2e263f7ee2b1c59f +F src/btree.c dd4683fa671f0b2b4bc0831036adaa2a5d1806ee F src/btree.h d3259057a38494c4c4358e377032158c762e3d8b F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 375e5df716e03b9343c5e1211be3b24e6d6dff05 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9529ed88a71fee02fae72dc86f0669bd6856ff92 -R 7e61f02cabbded666ed9e0992a636a37 +P 5ed8ad780c991d2ca44003ee84350fb5e95ad58e +R fc9bd174ea58046e3b5bc29693aabeb6 U dan -Z 67f89f7e2101ab4157bb24e7a7ac302a +Z 811da101d49a049638aae5d61fad1590 diff --git a/manifest.uuid b/manifest.uuid index f10e3bc724..95a0f78145 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5ed8ad780c991d2ca44003ee84350fb5e95ad58e \ No newline at end of file +32e0bbb73609ac3ad096a60f1de3095bc79fb0cc \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 2b7872fa8c..0a51ff0301 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2595,7 +2595,7 @@ static int btreeSwapOutMmap(BtShared *pBt){ MemPage *pPg = pCsr->apPage[0]; if( pPg->pDbPage->flags & PGHDR_MMAP ){ MemPage *pNew = 0; - rc = btreeGetPage(pBt, pPg->pgno, &pNew, 0, 0); + rc = getAndInitPage(pBt, pPg->pgno, &pNew, 0); if( rc==SQLITE_OK && pCsr->iPage==0 ){ pCsr->info.pCell = pNew->aData + (pCsr->info.pCell - pPg->aData); } From 7c49427fc25a0fc1738a4f7062abfea45642fbfc Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 21 Mar 2013 20:00:07 +0000 Subject: [PATCH 015/204] Avoid calling xRead() on a part of the database file that is memory mapped. FossilOrigin-Name: c8eac290a7240d69494bd0dad5ed1fdc2505f703 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 8 ++++++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 8819c52a55..b91c073e14 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swhen\sopening\sa\swrite-transaction\swhile\sthere\sexist\sread-only\sb-tree\scursors\sin\smmap\smode. -D 2013-03-21T15:57:32.562 +C Avoid\scalling\sxRead()\son\sa\spart\sof\sthe\sdatabase\sfile\sthat\sis\smemory\smapped. +D 2013-03-21T20:00:07.703 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c be66c31337361a72227638d6f41c7f2031739642 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c d59af9a70aa2d7222b127351fa3cbe70660e4150 +F src/pager.c 5ee2ec6a10aa3d85548e6f0337260f0b2a670899 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5ed8ad780c991d2ca44003ee84350fb5e95ad58e -R fc9bd174ea58046e3b5bc29693aabeb6 +P 32e0bbb73609ac3ad096a60f1de3095bc79fb0cc +R 78d8196d81dc00ab1ac5a50caf4d157f U dan -Z 811da101d49a049638aae5d61fad1590 +Z e76f59323062e94934c6de1e92e548e8 diff --git a/manifest.uuid b/manifest.uuid index 95a0f78145..0a2c6d8ff4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -32e0bbb73609ac3ad096a60f1de3095bc79fb0cc \ No newline at end of file +c8eac290a7240d69494bd0dad5ed1fdc2505f703 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 2848f9ad05..913ae74c46 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5130,9 +5130,13 @@ int sqlite3PagerSharedLock(Pager *pPager){ rc = pagerPagecount(pPager, &nPage); if( rc ) goto failed; - if( nPage>0 ){ + if( nPage>0 || pPager->pMap ){ IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); - rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); + if( pPager->pMap ){ + memcpy(&dbFileVers, &((u8 *)(pPager->pMap))[24], sizeof(dbFileVers)); + }else{ + rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); + } if( rc!=SQLITE_OK ){ goto failed; } From b26e6c14fe6b89b89c0281319743e488d8bfe6ec Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 21 Mar 2013 20:39:55 +0000 Subject: [PATCH 016/204] Fix cases where xRead() was being used to read from a memory mapped part of the database file. FossilOrigin-Name: 5c9e9df27b9f2c46cd55388a858d4e78ee564975 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 29 +++++++++++++++++++++++------ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index b91c073e14..4b2a8a6d5d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\scalling\sxRead()\son\sa\spart\sof\sthe\sdatabase\sfile\sthat\sis\smemory\smapped. -D 2013-03-21T20:00:07.703 +C Fix\scases\swhere\sxRead()\swas\sbeing\sused\sto\sread\sfrom\sa\smemory\smapped\spart\sof\sthe\sdatabase\sfile. +D 2013-03-21T20:39:55.225 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c be66c31337361a72227638d6f41c7f2031739642 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 5ee2ec6a10aa3d85548e6f0337260f0b2a670899 +F src/pager.c 520001015155efee9599c807dfd38e5fff9c6e36 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 32e0bbb73609ac3ad096a60f1de3095bc79fb0cc -R 78d8196d81dc00ab1ac5a50caf4d157f +P c8eac290a7240d69494bd0dad5ed1fdc2505f703 +R 0cdb7517bc23cd336050741c62097e56 U dan -Z e76f59323062e94934c6de1e92e548e8 +Z 0dd43854f08b06a022f11b1a174cdf13 diff --git a/manifest.uuid b/manifest.uuid index 0a2c6d8ff4..50136e7e1b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c8eac290a7240d69494bd0dad5ed1fdc2505f703 \ No newline at end of file +5c9e9df27b9f2c46cd55388a858d4e78ee564975 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 913ae74c46..0ff894bd87 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2087,6 +2087,24 @@ static void pagerReportSize(Pager *pPager){ # define pagerReportSize(X) /* No-op if we do not support a codec */ #endif +/* +** Write nBuf bytes of data from buffer pBuf to offset iOff of the +** database file. If this part of the database file is memory mapped, +** use memcpy() to do so. Otherwise, call sqlite3OsWrite(). +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int pagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff){ + int rc = SQLITE_OK; + if( pPager->nMapValid>=(iOff+nBuf) ){ + memcpy(&((u8 *)(pPager->pMap))[iOff], pBuf, nBuf); + }else{ + rc = sqlite3OsWrite(pPager->fd, pBuf, nBuf, iOff); + } + return rc; +} + /* ** Read a single page from either the journal file (if isMainJrnl==1) or ** from the sub-journal (if isMainJrnl==0) and playback that page. @@ -2261,7 +2279,7 @@ static int pager_playback_one_page( i64 ofst = (pgno-1)*(i64)pPager->pageSize; testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); assert( !pagerUseWal(pPager) ); - rc = sqlite3OsWrite(pPager->fd, (u8*)aData, pPager->pageSize, ofst); + rc = pagerWriteData(pPager, aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } @@ -3853,6 +3871,7 @@ static int pagerSyncHotJournal(Pager *pPager){ ** Unmap any memory mapping of the database file. */ static int pagerUnmap(Pager *pPager){ + assert( pPager->nMmapOut==0 ); if( pPager->pMap ){ sqlite3OsMremap(pPager->fd, 0, 0, pPager->nMap, 0, &pPager->pMap); pPager->nMap = 0; @@ -4282,11 +4301,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); /* Write out the page data. */ - if( pPager->nMapValid>=(offset+pPager->pageSize) ){ - memcpy(&((u8 *)(pPager->pMap))[offset], pData, pPager->pageSize); - }else{ - rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); - } + pagerWriteData(pPager, pData, pPager->pageSize, offset); /* If page 1 was just written, update Pager.dbFileVers to match ** the value now stored in the database file. If writing this @@ -7086,6 +7101,8 @@ int sqlite3PagerOpenWal( assert( pbOpen==0 || *pbOpen==0 ); assert( pbOpen!=0 || (!pPager->tempFile && !pPager->pWal) ); + pagerUnmap(pPager); + if( !pPager->tempFile && !pPager->pWal ){ if( !sqlite3PagerWalSupported(pPager) ) return SQLITE_CANTOPEN; From 48864df97d4a09bb229b0a298bb458db31524432 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 21 Mar 2013 21:20:32 +0000 Subject: [PATCH 017/204] Many spelling fixes in comments. No changes to code. FossilOrigin-Name: 6f6e2d50941e444ebc83604daddcc034137a05b7 --- ext/fts1/ft_hash.h | 2 +- ext/fts1/fts1_hash.h | 2 +- ext/fts2/fts2.c | 4 +- ext/fts2/fts2_hash.h | 2 +- ext/fts2/fts2_tokenizer.c | 2 +- ext/fts2/fts2_tokenizer.h | 2 +- ext/fts3/fts3.c | 10 +-- ext/fts3/fts3_expr.c | 2 +- ext/fts3/fts3_hash.h | 2 +- ext/fts3/fts3_snippet.c | 4 +- ext/fts3/fts3_test.c | 2 +- ext/fts3/fts3_tokenizer.c | 2 +- ext/fts3/fts3_tokenizer.h | 2 +- ext/fts3/fts3_unicode.c | 2 +- ext/icu/README.txt | 2 +- ext/rtree/rtree.c | 2 +- manifest | 124 +++++++++++++++++++------------------- manifest.uuid | 2 +- src/attach.c | 4 +- src/bitvec.c | 2 +- src/btree.c | 6 +- src/build.c | 4 +- src/expr.c | 4 +- src/hash.h | 2 +- src/main.c | 2 +- src/os_unix.c | 12 ++-- src/os_win.c | 8 +-- src/pager.c | 8 +-- src/prepare.c | 6 +- src/select.c | 2 +- src/sqlite.h.in | 4 +- src/tclsqlite.c | 2 +- src/test6.c | 2 +- src/test_multiplex.c | 2 +- src/test_sqllog.c | 2 +- src/update.c | 2 +- src/vdbe.c | 2 +- src/vdbeapi.c | 2 +- src/vdbeaux.c | 8 +-- src/where.c | 4 +- test/backup_ioerr.test | 4 +- test/cache.test | 2 +- test/collate4.test | 2 +- test/crash5.test | 2 +- test/e_createtable.test | 2 +- test/e_fkey.test | 6 +- test/e_select.test | 2 +- test/e_uri.test | 2 +- test/enc2.test | 2 +- test/incrblob.test | 4 +- test/io.test | 2 +- test/malloc.test | 2 +- test/malloc3.test | 4 +- test/notify2.test | 4 +- test/subquery.test | 2 +- test/temptable.test | 2 +- test/tester.tcl | 6 +- test/tkt2409.test | 4 +- 58 files changed, 157 insertions(+), 157 deletions(-) diff --git a/ext/fts1/ft_hash.h b/ext/fts1/ft_hash.h index 93b6dcfb48..95871a4590 100644 --- a/ext/fts1/ft_hash.h +++ b/ext/fts1/ft_hash.h @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This is the header file for the generic hash-table implemenation +** This is the header file for the generic hash-table implementation ** used in SQLite. We've modified it slightly to serve as a standalone ** hash table implementation for the full-text indexing module. ** diff --git a/ext/fts1/fts1_hash.h b/ext/fts1/fts1_hash.h index c31c430f7c..9001152931 100644 --- a/ext/fts1/fts1_hash.h +++ b/ext/fts1/fts1_hash.h @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This is the header file for the generic hash-table implemenation +** This is the header file for the generic hash-table implementation ** used in SQLite. We've modified it slightly to serve as a standalone ** hash table implementation for the full-text indexing module. ** diff --git a/ext/fts2/fts2.c b/ext/fts2/fts2.c index 93e03cd557..f008ce6dc0 100644 --- a/ext/fts2/fts2.c +++ b/ext/fts2/fts2.c @@ -6779,7 +6779,7 @@ void sqlite3Fts2IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); int sqlite3Fts2InitHashTable(sqlite3 *, fts2Hash *, const char *); /* -** Initialise the fts2 extension. If this extension is built as part +** Initialize the fts2 extension. If this extension is built as part ** of the sqlite library, then this function is called directly by ** SQLite. If fts2 is built as a dynamically loadable extension, this ** function is called by the sqlite3_extension_init() entry point. @@ -6797,7 +6797,7 @@ int sqlite3Fts2Init(sqlite3 *db){ sqlite3Fts2IcuTokenizerModule(&pIcu); #endif - /* Allocate and initialise the hash-table used to store tokenizers. */ + /* Allocate and initialize the hash-table used to store tokenizers. */ pHash = sqlite3_malloc(sizeof(fts2Hash)); if( !pHash ){ rc = SQLITE_NOMEM; diff --git a/ext/fts2/fts2_hash.h b/ext/fts2/fts2_hash.h index 571aa2c1c2..02936f18bb 100644 --- a/ext/fts2/fts2_hash.h +++ b/ext/fts2/fts2_hash.h @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This is the header file for the generic hash-table implemenation +** This is the header file for the generic hash-table implementation ** used in SQLite. We've modified it slightly to serve as a standalone ** hash table implementation for the full-text indexing module. ** diff --git a/ext/fts2/fts2_tokenizer.c b/ext/fts2/fts2_tokenizer.c index f8b06633da..a93790c801 100644 --- a/ext/fts2/fts2_tokenizer.c +++ b/ext/fts2/fts2_tokenizer.c @@ -319,7 +319,7 @@ static void intTestFunc( /* ** Set up SQL objects in database db used to access the contents of ** the hash table pointed to by argument pHash. The hash table must -** been initialised to use string keys, and to take a private copy +** been initialized to use string keys, and to take a private copy ** of the key when a value is inserted. i.e. by a call similar to: ** ** sqlite3Fts2HashInit(pHash, FTS2_HASH_STRING, 1); diff --git a/ext/fts2/fts2_tokenizer.h b/ext/fts2/fts2_tokenizer.h index 8c256b2bed..8db2048d6b 100644 --- a/ext/fts2/fts2_tokenizer.h +++ b/ext/fts2/fts2_tokenizer.h @@ -70,7 +70,7 @@ struct sqlite3_tokenizer_module { ** This method should return either SQLITE_OK (0), or an SQLite error ** code. If SQLITE_OK is returned, then *ppTokenizer should be set ** to point at the newly created tokenizer structure. The generic - ** sqlite3_tokenizer.pModule variable should not be initialised by + ** sqlite3_tokenizer.pModule variable should not be initialized by ** this callback. The caller will do so. */ int (*xCreate)( diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 7a53ac37a3..6b3a7b1565 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1571,7 +1571,7 @@ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ }else{ rc = sqlite3_reset(pCsr->pStmt); if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){ - /* If no row was found and no error has occured, then the %_content + /* If no row was found and no error has occurred, then the %_content ** table is missing a row that is present in the full-text index. ** The data structures are corrupt. */ rc = FTS_CORRUPT_VTAB; @@ -2811,7 +2811,7 @@ static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ } /* -** This function retreives the doclist for the specified term (or term +** This function retrieves the doclist for the specified term (or term ** prefix) from the database. */ static int fts3TermSelect( @@ -3562,7 +3562,7 @@ void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule); #endif /* -** Initialise the fts3 extension. If this extension is built as part +** Initialize the fts3 extension. If this extension is built as part ** of the sqlite library, then this function is called directly by ** SQLite. If fts3 is built as a dynamically loadable extension, this ** function is called by the sqlite3_extension_init() entry point. @@ -3596,7 +3596,7 @@ int sqlite3Fts3Init(sqlite3 *db){ sqlite3Fts3SimpleTokenizerModule(&pSimple); sqlite3Fts3PorterTokenizerModule(&pPorter); - /* Allocate and initialise the hash-table used to store tokenizers. */ + /* Allocate and initialize the hash-table used to store tokenizers. */ pHash = sqlite3_malloc(sizeof(Fts3Hash)); if( !pHash ){ rc = SQLITE_NOMEM; @@ -5195,7 +5195,7 @@ int sqlite3Fts3EvalPhraseStats( ** of the current row. ** ** More specifically, the returned buffer contains 1 varint for each -** occurence of the phrase in the column, stored using the normal (delta+2) +** occurrence of the phrase in the column, stored using the normal (delta+2) ** compression and is terminated by either an 0x01 or 0x00 byte. For example, ** if the requested column contains "a b X c d X X" and the position-list ** for 'X' is requested, the buffer returned may contain: diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 7612789de5..04f38483a3 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -106,7 +106,7 @@ struct ParseContext { ** This function is equivalent to the standard isspace() function. ** ** The standard isspace() can be awkward to use safely, because although it -** is defined to accept an argument of type int, its behaviour when passed +** is defined to accept an argument of type int, its behavior when passed ** an integer that falls outside of the range of the unsigned char type ** is undefined (and sometimes, "undefined" means segfault). This wrapper ** is defined to accept an argument of type char, and always returns 0 for diff --git a/ext/fts3/fts3_hash.h b/ext/fts3/fts3_hash.h index 399f51544a..dc3fcf8334 100644 --- a/ext/fts3/fts3_hash.h +++ b/ext/fts3/fts3_hash.h @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This is the header file for the generic hash-table implemenation +** This is the header file for the generic hash-table implementation ** used in SQLite. We've modified it slightly to serve as a standalone ** hash table implementation for the full-text indexing module. ** diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 4bee014dcc..d54a7875ea 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -389,9 +389,9 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ ** is the snippet with the highest score, where scores are calculated ** by adding: ** -** (a) +1 point for each occurence of a matchable phrase in the snippet. +** (a) +1 point for each occurrence of a matchable phrase in the snippet. ** -** (b) +1000 points for the first occurence of each matchable phrase in +** (b) +1000 points for the first occurrence of each matchable phrase in ** the snippet for which the corresponding mCovered bit is not set. ** ** The selected snippet parameters are stored in structure *pFragment before diff --git a/ext/fts3/fts3_test.c b/ext/fts3/fts3_test.c index 4da0b8f13d..75ec6bd01d 100644 --- a/ext/fts3/fts3_test.c +++ b/ext/fts3/fts3_test.c @@ -267,7 +267,7 @@ static int fts3_near_match_cmd( ** ** Whether or not the arguments are present, this command returns a list of ** two integers - the initial chunksize and threshold when the command is -** invoked. This can be used to restore the default behaviour after running +** invoked. This can be used to restore the default behavior after running ** tests. For example: ** ** # Override incr-load settings for testing: diff --git a/ext/fts3/fts3_tokenizer.c b/ext/fts3/fts3_tokenizer.c index 8241be81fc..04f84460e8 100644 --- a/ext/fts3/fts3_tokenizer.c +++ b/ext/fts3/fts3_tokenizer.c @@ -428,7 +428,7 @@ static void intTestFunc( /* ** Set up SQL objects in database db used to access the contents of ** the hash table pointed to by argument pHash. The hash table must -** been initialised to use string keys, and to take a private copy +** been initialized to use string keys, and to take a private copy ** of the key when a value is inserted. i.e. by a call similar to: ** ** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); diff --git a/ext/fts3/fts3_tokenizer.h b/ext/fts3/fts3_tokenizer.h index c91c7ed790..4a40b2b385 100644 --- a/ext/fts3/fts3_tokenizer.h +++ b/ext/fts3/fts3_tokenizer.h @@ -70,7 +70,7 @@ struct sqlite3_tokenizer_module { ** This method should return either SQLITE_OK (0), or an SQLite error ** code. If SQLITE_OK is returned, then *ppTokenizer should be set ** to point at the newly created tokenizer structure. The generic - ** sqlite3_tokenizer.pModule variable should not be initialised by + ** sqlite3_tokenizer.pModule variable should not be initialized by ** this callback. The caller will do so. */ int (*xCreate)( diff --git a/ext/fts3/fts3_unicode.c b/ext/fts3/fts3_unicode.c index 79941edbb8..188358eade 100644 --- a/ext/fts3/fts3_unicode.c +++ b/ext/fts3/fts3_unicode.c @@ -125,7 +125,7 @@ static int unicodeDestroy(sqlite3_tokenizer *pTokenizer){ ** ** If a standalone diacritic mark (one that sqlite3FtsUnicodeIsdiacritic() ** identifies as a diacritic) occurs in the zIn/nIn string it is ignored. -** It is not possible to change the behaviour of the tokenizer with respect +** It is not possible to change the behavior of the tokenizer with respect ** to these codepoints. */ static int unicodeAddExceptions( diff --git a/ext/icu/README.txt b/ext/icu/README.txt index c5cadb57d1..d744f74981 100644 --- a/ext/icu/README.txt +++ b/ext/icu/README.txt @@ -98,7 +98,7 @@ SQLite. Documentation follows. REGEXP This extension uses the ICU defaults for regular expression matching - behaviour. Specifically, this means that: + behavior. Specifically, this means that: * Matching is case-sensitive, * Regular expression comments are not allowed within patterns, and diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 34dde0e968..16a316f400 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -2665,7 +2665,7 @@ static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ RtreeNode *pRoot; /* Root node of rtree structure */ - /* Obtain a reference to the root node to initialise Rtree.iDepth */ + /* Obtain a reference to the root node to initialize Rtree.iDepth */ rc = nodeAcquire(pRtree, 1, 0, &pRoot); /* Obtain a reference to the leaf node that contains the entry diff --git a/manifest b/manifest index 311e065ca2..4437f787e5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stext-to-numeric\stype\scasting\sso\sthat\sit\sworks\scorrectly\son\sUTF16\nstrings\sthat\scontain\scharacters\swhere\sthe\sLSB\sis\snumeric\sbut\sthe\sMSB\nis\snon-zero.\s\sTicket\s[689137afb6da41] -D 2013-03-20T12:04:29.291 +C Many\sspelling\sfixes\sin\scomments.\s\sNo\schanges\sto\scode. +D 2013-03-21T21:20:32.533 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -27,11 +27,11 @@ F ext/async/sqlite3async.c b5a3e30f538a9ffe81538b3063b4d5963f9bb422 F ext/async/sqlite3async.h f489b080af7e72aec0e1ee6f1d98ab6cf2e4dcef F ext/fts1/README.txt 20ac73b006a70bcfd80069bdaf59214b6cf1db5e F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b -F ext/fts1/ft_hash.h 1a35e654a235c2c662d3ca0dfc3138ad60b8b7d5 +F ext/fts1/ft_hash.h 06df7bba40dadd19597aa400a875dbc2fed705ea F ext/fts1/fts1.c 3e7b253e61aab0bb1fea808c7a0ce36c19432acc F ext/fts1/fts1.h 6060b8f62c1d925ea8356cb1a6598073eb9159a6 F ext/fts1/fts1_hash.c 3196cee866edbebb1c0521e21672e6d599965114 -F ext/fts1/fts1_hash.h 957d378355ed29f672cd5add012ce8b088a5e089 +F ext/fts1/fts1_hash.h e7f0d761353996a8175eda351104acfde23afcb0 F ext/fts1/fts1_porter.c b1c7304b8988ba3f764a147cdd32043b4913ea7b F ext/fts1/fts1_tokenizer.h fdea722c38a9f82ed921642981234f666e47919c F ext/fts1/fts1_tokenizer1.c fd00d1fe4dc30dfc5c64cba695ce34f4af20d2fa @@ -41,36 +41,36 @@ F ext/fts1/simple_tokenizer.c 1844d72f7194c3fd3d7e4173053911bf0661b70d F ext/fts1/tokenizer.h 0c53421b832366d20d720d21ea3e1f6e66a36ef9 F ext/fts2/README.tokenizers 21e3684ea5a095b55d70f6878b4ce6af5932dfb7 F ext/fts2/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts2/fts2.c 4ef7d7ecf590da0dd416ac54612c53a7d4175790 +F ext/fts2/fts2.c b48cc0bb657c0a421f4067b79aa0354bd16a046d F ext/fts2/fts2.h da5f76c65163301d1068a971fd32f4119e3c95fa F ext/fts2/fts2_hash.c 2689e42e1107ea67207f725cf69cf8972d00cf93 -F ext/fts2/fts2_hash.h 9a5b1be94664139f93217a0770d7144425cffb3a +F ext/fts2/fts2_hash.h 1824b99dfd8d0225facbdb26a2c87289b2e7dcf8 F ext/fts2/fts2_icu.c 51c5cd3c04954badd329fa738c95fcdb717b5188 F ext/fts2/fts2_porter.c 747056987951f743e955c8479f1df21a565720fe -F ext/fts2/fts2_tokenizer.c 26e993a00b2bd5b6e73c155597361710b12ffe25 -F ext/fts2/fts2_tokenizer.h a7e46462d935a314b2682287f12f27530a3ee08e +F ext/fts2/fts2_tokenizer.c a86d08c9634fabfa237c8f379008de2e11248d36 +F ext/fts2/fts2_tokenizer.h 27a1a99ca2d615cf7e142839b8d79e8751b4529e F ext/fts2/fts2_tokenizer1.c 0123d21078e053bd98fd6186c5c6dc6d67969f2e F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c a867cafae0235324df64c5775cbf352a3f712cb9 +F ext/fts3/fts3.c 0eaedfc6d2eb22563ef1d044dcfed93b70ec79f2 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 1e58825246b56259267382d2f9902774c049460a F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e -F ext/fts3/fts3_expr.c ceefcaf91344abbf6ceb3cadf404eef5be6924e6 +F ext/fts3/fts3_expr.c 6cb4410f87676ae633bd7923bbc78526cb839c4d F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 -F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec +F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5 F ext/fts3/fts3_porter.c a465b49fcb8249a755792f87516eff182efa42b3 -F ext/fts3/fts3_snippet.c f6ebb3536069ceaa27e9f178f4a59379db44ac10 +F ext/fts3/fts3_snippet.c 5fcfcafff46a2a3a63b8e59fcb51987d01c74695 F ext/fts3/fts3_term.c a521f75132f9a495bdca1bdd45949b3191c52763 -F ext/fts3/fts3_test.c 348f7d08cae05285794e23dc4fe8b8fdf66e264a -F ext/fts3/fts3_tokenizer.c 3664bb8836ab7633cf9da786c9d6fd366be5ae2c -F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68 +F ext/fts3/fts3_test.c f9a1a1702db1bfad3e2d0064746eeb808f125489 +F ext/fts3/fts3_tokenizer.c bbdc731bc91338050675c6d1da9ab82147391e16 +F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_unicode.c 49e36e6ba59f79e6bd6a8bfe434570fe48d20559 +F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c a863f05f758af36777dffc2facc898bc73fec896 F ext/fts3/fts3_write.c c2166f7148a4ad8bcdad99a99d647b1091744e6b F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 @@ -79,11 +79,11 @@ F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 7a9bc018e2962abb79563c5a39fe581fcbf2f675 -F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 +F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c ebd07d0f06dc167f1424ff3940a5711a3a039982 +F ext/rtree/rtree.c 757abea591d4ff67c0ff4e8f9776aeda86b18c14 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test e474a2b5eff231496dbd073fe67e5fbaf7f444c9 F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -116,34 +116,34 @@ F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c f8db986c03eb0bfb221523fc9bbb9d0b70de3168 F src/analyze.c d5f895810e8ff9737c9ec7b76abc3dcff5860335 -F src/attach.c ea5247f240e2c08afd608e9beb380814b86655e1 +F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 -F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 +F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 746c4dafae8565b3be6fb9ce3bb1fa9f1e67cc59 +F src/btree.c 3cebaa69db81a528e115b463a5506133a0043710 F src/btree.h 3ad7964d6c5b1c7bff569aab6adfa075f8bf06cd F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 -F src/build.c 375e5df716e03b9343c5e1211be3b24e6d6dff05 +F src/build.c b85db6cb50fc63e44776b741452250626bad491a F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 72a70dcfda75d3a1f81041ce4573e7afddcd8e4e F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 -F src/expr.c d488bb60e54c9305d9fca1fa6fcc7bfbd23b13a2 +F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c e16942bd5c8a868ac53287886464a5ed0e72b179 F src/func.c 48987c025d69399f59a1c2a553cea5da41bf105d F src/global.c e59ecd2c553ad0d4bfbc84ca71231336f8993a7a F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4 -F src/hash.h 2894c932d84d9f892d4b4023a75e501f83050970 +F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 F src/insert.c f7cb141e8ce257cb6b15c497f09e4e23d6055599 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 1422eba4aa2b1fb5f7b3aef574752272477d21e2 -F src/main.c 8d204866d1abf5100503dcd54d3187b88f6846b7 +F src/main.c 379160ec3680e3009aa4978eac47027c3ef27ac5 F src/malloc.c fe085aa851b666b7c375c1ff957643dc20a04bf6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa @@ -160,36 +160,36 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c f6387eef0cf8f6b808738f4f3aa47e6132af0940 -F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 582f8da52d0bd4b43d3bdaeba0ea7702c1f23702 +F src/os_unix.c 21a36fa0b3753609b6606b30d9338d4bb6b24696 +F src/os_win.c 9fe5356f943425ab8431237bd3a4297044928b70 +F src/pager.c 3e9a15939684b0af441325f05335331b15979c9d F src/pager.h 1109a06578ec5574dc2c74cf8d9f69daf36fe3e0 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h 1b5dcc3dc8103d03e625b177023ee67764fa6b7c F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 F src/pragma.c 9f0ee3d74a7f33eeeff40a4b014fc3abf8182ce2 -F src/prepare.c 78cd7ae8cd0d8de8ef8a8b5352fc5f38693a0852 +F src/prepare.c 310eaff1ee5f3c700b3545afb095cfe9346efc3a F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 +F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in e0ace39d7c14dc9e988cf579f75aaf2a7db7ae77 +F src/sqlite.h.in 8d9e83d965f364ff99ebf4b653d0683d05df3900 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 0f8f05ee4db4ba9120b38f7a3992b325698f6e8a F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e -F src/tclsqlite.c 3213f3101e3b85f047d6e389da5a53d76d3d7540 +F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 F src/test1.c ff3e68eedfbd858c9b89cf03e3db233cd29be1d0 F src/test2.c 4178056dd1e7d70f954ad8a1e3edb71a2a784daf F src/test3.c 3c3c2407fa6ec7a19e24ae23f7cb439d0275a60d F src/test4.c bf9fa9bece01de08e6f5e02314e4af5c13590dfa F src/test5.c a6d1ac55ac054d0b2b8f37b5e655b6c92645a013 -F src/test6.c 938794c970ed6810036c8d28450ca28166524bf7 +F src/test6.c a437f76f9874d2563352a7e6cd0d43217663c220 F src/test7.c 2e0781754905c8adc3268d8f0967e7633af58843 F src/test8.c 58ea1d9698f3947e4662107ef98f429e84ae20e0 F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 @@ -210,7 +210,7 @@ F src/test_intarray.h b999bb18d090b8d9d9c49d36ec37ef8f341fe169 F src/test_journal.c f5c0a05b7b3d5930db769b5ee6c3766dc2221a64 F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 6982a357a6a6c24f281b91c89303a5c31075c392 -F src/test_multiplex.c ac0fbc1748e5b86a41a1d7a84654fae0d53a881d +F src/test_multiplex.c 0decc630b683979cb32d4b965efd90b6e55a786e F src/test_multiplex.h 9b63b95f07acedee425fdfe49a47197c9bf5f9d8 F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 0396f220561f3b4eedc450cef26d40c593c69a25 @@ -223,7 +223,7 @@ F src/test_rtree.c aba603c949766c4193f1068b91c787f57274e0d9 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f F src/test_spellfix.c 56dfa6d583ac34f61af0834d7b58d674e7e18e13 -F src/test_sqllog.c 8acb843ddb9928dea8962e31bb09f421a72ffccb +F src/test_sqllog.c bc50e5afeb7fb50e77b4594e42302df9d05446aa F src/test_stat.c d1569c7a4839f13e80187e2c26b2ab4da2d03935 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae @@ -235,15 +235,15 @@ F src/test_wholenumber.c 3d2b9ed1505c40ad5c5ca2ad16ae7a289d6cc251 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 1e86210d3976717a19238ea7b047fac481fe8c12 F src/trigger.c cd95ac64efa60e39faf9b5597443192ff27a22fa -F src/update.c 28d2d098b43a2c70dae399896ea8a02f622410ef +F src/update.c a2a5631d618cbe240fc83725fa9e95c56ae0084c F src/utf.c 8d819e2e5104a430fc2005f018db14347c95a38f F src/util.c 550f2b6a5c0085153a4d00462719fb17ee242792 F src/vacuum.c 2727bdd08847fcb6b2d2da6d14f018910e8645d3 -F src/vdbe.c 292f8f7ced59c29c63fe17830cbe5f5a0230cdf0 +F src/vdbe.c f9622d5b222f7dd929d7f4a4d42278d47d2b24ab F src/vdbe.h b52887278cb173e66188da84dfab216bea61119d F src/vdbeInt.h 396bb03eec560f768d1b86092b00f46c25575d3b -F src/vdbeapi.c 9616986209cc77822aa9f7d91cf9e6880516d557 -F src/vdbeaux.c 735a6905df302a7f3c715a82bd3af06dc7d74ef2 +F src/vdbeapi.c 56f9ad6329d093f54b5dd321a2b4a643f89a6cb6 +F src/vdbeaux.c ecb43014bcd3019e5aa2b5561e5c3a447f007a08 F src/vdbeblob.c 32f2a4899d67f69634ea4dd93e3f651936d732cb F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c c61ca318681c0e7267da8be3abfca8469652a7e9 @@ -252,7 +252,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c bdbbfa7ef4ea04c8d9b09585b45d4717a72f980a +F src/where.c 9a16c0b84bbeb054d11fda96e9e037ae310bd54e F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -293,7 +293,7 @@ F test/backcompat.test ecd841f3a3bfb81518721879cc56a760670e3198 F test/backup.test c9cdd23a495864b9edf75a9fa66f5cb7e10fcf62 F test/backup2.test 34986ef926ea522911a51dfdb2f8e99b7b75ebcf F test/backup4.test 4d90389daeb781fe718816a4fc836ad1b06791d8 -F test/backup_ioerr.test 40d208bc9224b666ee3ed423f49bc9062a36a9d0 +F test/backup_ioerr.test 4c3c7147cee85b024ecf6e150e090c32fdbb5135 F test/backup_malloc.test 7162d604ec2b4683c4b3799a48657fb8b5e2d450 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f F test/badutf2.test f5bc7f2d280670ecd79b9cf4f0f1760c607fe51f @@ -315,7 +315,7 @@ F test/boundary3.test 56ef82096b4329aca2be74fa1e2b0f762ea0eb45 F test/boundary4.tcl 0bb4b1a94f4fc5ae59b79b9a2b7a140c405e2983 F test/boundary4.test 89e02fa66397b8a325d5eb102b5806f961f8ec4b F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 -F test/cache.test f64136b0893c293d0b910ed057b3b711249099a7 +F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de F test/capi2.test e8b18cc61090b6e5e388f54d6b125d711d1b265a F test/capi3.test 56ab450125ead38846cbae7e5b6a216686c3cffa F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 @@ -328,7 +328,7 @@ F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 F test/collate1.test fd02c4d8afc71879c4bb952586389961a21fb0ce F test/collate2.test 04cebe4a033be319d6ddbb3bbc69464e01700b49 F test/collate3.test d28d2cfab2c3a3d4628ae4b2b7afc9965daa3b4c -F test/collate4.test d37682293d3c32223dec2e6afdeaf9de18415248 +F test/collate4.test 031f7265c13308b724ba3c49f41cc04612bd92b1 F test/collate5.test 65d928034d30d2d263a80f6359f7549ee1598ec6 F test/collate6.test 8be65a182abaac8011a622131486dafb8076e907 F test/collate7.test 8ec29d98f3ee4ccebce6e16ce3863fb6b8c7b868 @@ -359,7 +359,7 @@ F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f F test/crash2.test 5b14d4eb58b880e231361d3b609b216acda86651 F test/crash3.test 8f5de9d32ab9ab95475a9efe7f47a940aa889418 F test/crash4.test fe2821baf37168dc59dd733dcf7dba2a401487bc -F test/crash5.test 13b9ca94e048194bca070e26160ce76b406c56be +F test/crash5.test 05dd3aa9dbb751a22d5cdaf22a9c49b6667aa219 F test/crash6.test 4c56f1e40d0291e1110790a99807aa875b1647ba F test/crash7.test 1a194c4900a255258cf94b7fcbfd29536db572df F test/crash8.test 38767cb504bbe491de6be4a7006b154973a2309f @@ -380,23 +380,23 @@ F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 84da1414b2e6887fffd5ed571311b344c5b082ce F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 -F test/e_createtable.test 0a2465736199cb5e084645a8714ee04299b81721 +F test/e_createtable.test d4e17024b1831e7480f5736cf4e02516a5c90927 F test/e_delete.test 89aa84d3d1bd284a0689ede04bce10226a5aeaa5 F test/e_droptrigger.test afd5c4d27dec607f5997a66bf7e2498a082cb235 F test/e_dropview.test 583411e470458c5d76148542cfb5a5fa84c8f93e F test/e_expr.test 5489424d3d9a452ac3701cdf4b680ae31a157894 -F test/e_fkey.test 89a2ff734a33693b997534eff90724573948be7b +F test/e_fkey.test 79a985d95340c6072a884359426ce95cf33e656a F test/e_fts3.test 5c02288842e4f941896fd44afdef564dd5fc1459 F test/e_insert.test d5331cc95e101af2508159fc98b6801631659ffe F test/e_reindex.test dfedfc32c5a282b0596c6537cbcd4217fbb1a216 F test/e_resolve.test dcce9308fb13b934ce29591105d031d3e14fbba6 -F test/e_select.test 07e8d81268ba1ffcaf1dc4bec48956af150c42f9 +F test/e_select.test d5af998a402740d8f0488158d22075df2b6f88fa F test/e_select2.test 5c3d3da19c7b3e90ae444579db2b70098599ab92 F test/e_update.test 161d5dc6a3ed9dd08f5264d13e20735d7a89f00c -F test/e_uri.test bc240fbc6cbbbdff832ee05858432a25961ab36a +F test/e_uri.test 8f2f56b29456a3f846276fa4e0993d4ef8a15b79 F test/e_vacuum.test 331da289ae186656cf5f2eb27f577a89c0c221af F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea -F test/enc2.test 796c59832e2b9a52842f382ffda8f3e989db03ad +F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 F test/eqp.test 46aa946dd55c90635327898275d3e533d23a9845 @@ -526,7 +526,7 @@ F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test 64f3cc1acde1b9161ccdd8e5bde3daefdb5b2617 F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 -F test/incrblob.test 34765fa6fb5d8e0f256fc7d6497c04b205398849 +F test/incrblob.test e7ef2a6094d9b5eb7284c21af2c07644eefffe7d F test/incrblob2.test edc3a96e557bd61fb39acc8d2edd43371fbbaa19 F test/incrblob3.test aedbb35ea1b6450c33b98f2b6ed98e5020be8dc7 F test/incrblob4.test 09be37d3dd996a31ea6993bba7837ece549414a8 @@ -553,7 +553,7 @@ F test/instr.test a34e1d46a9eefb098a7167ef0e730a4a3d82fba0 F test/intarray.test 066b7d7ac38d25bf96f87f1b017bfc687551cdd4 F test/interrupt.test dfe9a67a94b0b2d8f70545ba1a6cca10780d71cc F test/intpkey.test 7af30f6ae852d8d1c2b70e4bf1551946742e92d8 -F test/io.test 36d251507d72e92b965fb2f0801c2f0b56335bcf +F test/io.test a4be25a446a99a0604ceecc039ee7363c56e4185 F test/ioerr.test 40bb2cfcab63fb6aa7424cd97812a84bc16b5fb8 F test/ioerr2.test 9d71166f8466eda510f1af6137bdabaa82b5408d F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd @@ -591,8 +591,8 @@ F test/lock_common.tcl 0c270b121d40959fa2f3add382200c27045b3d95 F test/lookaside.test 93f07bac140c5bb1d49f3892d2684decafdc7af2 F test/main.test 39c4bb8a157f57298ed1659d6df89d9f35aaf2c8 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 -F test/malloc.test bc745155ff4252d4f35ec8316625b0dfe2abc659 -F test/malloc3.test 3e9eb921c4314acf4fb64230868fdb2e1ce60eea +F test/malloc.test fd368e31fe98d4779ed80442f311ed9f03bcd1f7 +F test/malloc3.test e3b32c724b5a124b57cb0ed177f675249ad0c66a F test/malloc4.test 957337613002b7058a85116493a262f679f3a261 F test/malloc5.test a577cbbcc1594c7763b9b3515b3633555751c7f0 F test/malloc6.test 2f039d9821927eacae43e1831f815e157659a151 @@ -637,7 +637,7 @@ F test/mutex1.test 78b2b9bb320e51d156c4efdb71b99b051e7a4b41 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test e9648b9d007c7045242af35e11a984d4b169443a F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf -F test/notify2.test 9503e51b9a272a5405c205ad61b7623d5a9ca489 +F test/notify2.test ce23eb522c9e1fff6443f96376fe67872202061c F test/notify3.test a86259abbfb923aa27d30f0fc038c88e5251488a F test/notnull.test 2afad748d18fd66d01f66463de73b3e2501fb226 F test/null.test a8b09b8ed87852742343b33441a9240022108993 @@ -740,7 +740,7 @@ F test/spellfix.test 52ae2680b1247c52b9e2b2116de3fd26a78e6bd2 F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test be8d477306006ec696bc86757cfb34bec79447ce F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9 -F test/subquery.test d4aea23ac267463d4aa604bf937c3992347b20f7 +F test/subquery.test 869562de9e8c5d8147e0451a2ce5b58cf55ce389 F test/subquery2.test 91e1e364072aeff431d1f9689b15147e421d88c7 F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a @@ -752,9 +752,9 @@ F test/table.test a59d985ca366e39b17b175f387f9d5db5a18d4e2 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c -F test/temptable.test 51edd31c65ed1560dd600b1796e8325df96318e2 +F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptrigger.test 26670ed7a39cf2296a7f0a9e0a1d7bdb7abe936d -F test/tester.tcl 2918ebca150b67ca25b1682f8ecd857af77fab05 +F test/tester.tcl 0d95028653ad6782e17400b7a32a046b87c67a48 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -834,7 +834,7 @@ F test/tkt2285.test cca17be61cf600b397188e77e7143844d2b977e9 F test/tkt2332.test fc955609b958ca86dfa102832243370a0cc84070 F test/tkt2339.test 73bd17818924cd2ac442e5fd9916b58565739450 F test/tkt2391.test ab7a11be7402da8b51a5be603425367aa0684567 -F test/tkt2409.test 464d55beb32e3b12ce2b4bbf9857d063c4c34297 +F test/tkt2409.test be0d60e7d283f639dccea4b0b5e1cd3a4851fb5b F test/tkt2450.test 77ed94863f2049c1420288ddfea2d41e5e0971d6 F test/tkt2565.test 8be666e927cb207aae88188f31c331870878b650 F test/tkt2640.test 28134f5d1e05658ef182520cf0b680fa3de5211b @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d1f41089aba075eef45fd696599e5d3a74c84d0c -R ccf3378e4accef01a0ea31a61ac073d2 -U drh -Z 400c45055ef48421e523d8328d092a1b +P 5b22053f918d16f593227a432a5d5b4c195bb0b5 +R 0ed9a71acb145119af7cea3063285aea +U mistachkin +Z eedfaec177b15430293d5b03ecbc6bec diff --git a/manifest.uuid b/manifest.uuid index f861049e0c..3f2050457f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5b22053f918d16f593227a432a5d5b4c195bb0b5 \ No newline at end of file +6f6e2d50941e444ebc83604daddcc034137a05b7 \ No newline at end of file diff --git a/src/attach.c b/src/attach.c index 6682c91475..b8e12199b5 100644 --- a/src/attach.c +++ b/src/attach.c @@ -109,7 +109,7 @@ static void attachFunc( } } - /* Allocate the new entry in the db->aDb[] array and initialise the schema + /* Allocate the new entry in the db->aDb[] array and initialize the schema ** hash tables. */ if( db->aDb==db->aDbStatic ){ @@ -126,7 +126,7 @@ static void attachFunc( /* Open the database file. If the btree is successfully opened, use ** it to obtain the database schema. At this point the schema may - ** or may not be initialised. + ** or may not be initialized. */ flags = db->openFlags; rc = sqlite3ParseUri(db->pVfs->zName, zFile, &flags, &pVfs, &zPath, &zErr); diff --git a/src/bitvec.c b/src/bitvec.c index 8d805a6fe5..52184aa964 100644 --- a/src/bitvec.c +++ b/src/bitvec.c @@ -72,7 +72,7 @@ /* ** A bitmap is an instance of the following structure. ** -** This bitmap records the existance of zero or more bits +** This bitmap records the existence of zero or more bits ** with values between 1 and iSize, inclusive. ** ** There are three possible representations of the bitmap. diff --git a/src/btree.c b/src/btree.c index 07ec3fe52a..b3549fa697 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2229,7 +2229,7 @@ int sqlite3BtreeGetPageSize(Btree *p){ ** known that the shared b-tree mutex is held, but the mutex on the ** database handle that owns *p is not. In this case if sqlite3BtreeEnter() ** were to be called, it might collide with some other operation on the -** database handle that owns *p, causing undefined behaviour. +** database handle that owns *p, causing undefined behavior. */ int sqlite3BtreeGetReserveNoMutex(Btree *p){ assert( sqlite3_mutex_held(p->pBt->mutex) ); @@ -5509,7 +5509,7 @@ static int fillInCell( ** If this is the first overflow page, then write a partial entry ** to the pointer-map. If we write nothing to this pointer-map slot, ** then the optimistic overflow chain processing in clearCell() - ** may misinterpret the uninitialised values and delete the + ** may misinterpret the uninitialized values and delete the ** wrong pages from the database. */ if( pBt->autoVacuum && rc==SQLITE_OK ){ @@ -6982,7 +6982,7 @@ int sqlite3BtreeInsert( insertCell(pPage, idx, newCell, szNew, 0, 0, &rc); assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 ); - /* If no error has occured and pPage has an overflow cell, call balance() + /* If no error has occurred and pPage has an overflow cell, call balance() ** to redistribute the cells within the tree. Since balance() may move ** the cursor, zero the BtCursor.info.nSize and BtCursor.validNKey ** variables. diff --git a/src/build.c b/src/build.c index 5d063f0726..c4d14d8943 100644 --- a/src/build.c +++ b/src/build.c @@ -2100,7 +2100,7 @@ void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, int isView){ /* Drop all SQLITE_MASTER table and index entries that refer to the ** table. The program name loops through the master table and deletes ** every row that refers to a table of the same name as the one being - ** dropped. Triggers are handled seperately because a trigger can be + ** dropped. Triggers are handled separately because a trigger can be ** created in the temp database that refers to a table in another ** database. */ @@ -2839,7 +2839,7 @@ Index *sqlite3CreateIndex( ** However the ON CONFLICT clauses are different. If both this ** constraint and the previous equivalent constraint have explicit ** ON CONFLICT clauses this is an error. Otherwise, use the - ** explicitly specified behaviour for the index. + ** explicitly specified behavior for the index. */ if( !(pIdx->onError==OE_Default || pIndex->onError==OE_Default) ){ sqlite3ErrorMsg(pParse, diff --git a/src/expr.c b/src/expr.c index ed5451b1c5..ae6a1dec10 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3542,7 +3542,7 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ int r1, r2; assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); - if( NEVER(v==0) ) return; /* Existance of VDBE checked by caller */ + if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ if( NEVER(pExpr==0) ) return; /* No way this can happen */ op = pExpr->op; switch( op ){ @@ -3662,7 +3662,7 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ int r1, r2; assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); - if( NEVER(v==0) ) return; /* Existance of VDBE checked by caller */ + if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ if( pExpr==0 ) return; /* The value of pExpr->op and op are related as follows: diff --git a/src/hash.h b/src/hash.h index 990a2d6e22..82b7c58c71 100644 --- a/src/hash.h +++ b/src/hash.h @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This is the header file for the generic hash-table implemenation +** This is the header file for the generic hash-table implementation ** used in SQLite. */ #ifndef _SQLITE_HASH_H_ diff --git a/src/main.c b/src/main.c index 77aaa6d83b..2fcb236e83 100644 --- a/src/main.c +++ b/src/main.c @@ -885,7 +885,7 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ /* If we reach this point, it means that the database connection has ** closed all sqlite3_stmt and sqlite3_backup objects and has been - ** pased to sqlite3_close (meaning that it is a zombie). Therefore, + ** passed to sqlite3_close (meaning that it is a zombie). Therefore, ** go ahead and free all resources. */ diff --git a/src/os_unix.c b/src/os_unix.c index 8f094bdc19..84b1594ac3 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -337,7 +337,7 @@ static int openDirectory(const char*, int*); ** to all overrideable system calls. */ static struct unix_syscall { - const char *zName; /* Name of the sytem call */ + const char *zName; /* Name of the system call */ sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ sqlite3_syscall_ptr pDefault; /* Default value */ } aSyscall[] = { @@ -1909,7 +1909,7 @@ static int nolockClose(sqlite3_file *id) { /****************************************************************************** ************************* Begin dot-file Locking ****************************** ** -** The dotfile locking implementation uses the existance of separate lock +** The dotfile locking implementation uses the existence of separate lock ** files (really a directory) to control access to the database. This works ** on just about every filesystem imaginable. But there are serious downsides: ** @@ -1924,7 +1924,7 @@ static int nolockClose(sqlite3_file *id) { ** ** Dotfile locking works by creating a subdirectory in the same directory as ** the database and with the same name but with a ".lock" extension added. -** The existance of a lock directory implies an EXCLUSIVE lock. All other +** The existence of a lock directory implies an EXCLUSIVE lock. All other ** lock types (SHARED, RESERVED, PENDING) are mapped into EXCLUSIVE. */ @@ -3413,7 +3413,7 @@ static int unixSync(sqlite3_file *id, int flags){ } /* Also fsync the directory containing the file if the DIRSYNC flag - ** is set. This is a one-time occurrance. Many systems (examples: AIX) + ** is set. This is a one-time occurrence. Many systems (examples: AIX) ** are unable to fsync a directory, so ignore errors on the fsync. */ if( pFile->ctrlFlags & UNIXFILE_DIRSYNC ){ @@ -4787,7 +4787,7 @@ static int fillInUnixFile( unixEnterMutex(); rc = findInodeInfo(pNew, &pNew->pInode); if( rc!=SQLITE_OK ){ - /* If an error occured in findInodeInfo(), close the file descriptor + /* If an error occurred in findInodeInfo(), close the file descriptor ** immediately, before releasing the mutex. findInodeInfo() may fail ** in two scenarios: ** @@ -5425,7 +5425,7 @@ static int unixDelete( } /* -** Test the existance of or access permissions of file zPath. The +** Test the existence of or access permissions of file zPath. The ** test performed depends on the value of flags: ** ** SQLITE_ACCESS_EXISTS: Return 1 if the file exists diff --git a/src/os_win.c b/src/os_win.c index 970a94b5e8..58ba25f302 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -308,7 +308,7 @@ static int sqlite3_os_type = 0; ** to all overrideable system calls. */ static struct win_syscall { - const char *zName; /* Name of the sytem call */ + const char *zName; /* Name of the system call */ sqlite3_syscall_ptr pCurrent; /* Current value of the system call */ sqlite3_syscall_ptr pDefault; /* Default value */ } aSyscall[] = { @@ -2025,7 +2025,7 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ ** containing the lower 32-bits of the new file-offset. Or, if it fails, ** it returns INVALID_SET_FILE_POINTER. However according to MSDN, ** INVALID_SET_FILE_POINTER may also be a valid new offset. So to determine - ** whether an error has actually occured, it is also necessary to call + ** whether an error has actually occurred, it is also necessary to call ** GetLastError(). */ dwRet = osSetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); @@ -2171,7 +2171,7 @@ static int winWrite( int amt, /* Number of bytes to write */ sqlite3_int64 offset /* Offset into the file to begin writing at */ ){ - int rc = 0; /* True if error has occured, else false */ + int rc = 0; /* True if error has occurred, else false */ winFile *pFile = (winFile*)id; /* File handle */ int nRetry = 0; /* Number of retries */ @@ -3993,7 +3993,7 @@ static int winDelete( } /* -** Check the existance and status of a file. +** Check the existence and status of a file. */ static int winAccess( sqlite3_vfs *pVfs, /* Not used on win32 */ diff --git a/src/pager.c b/src/pager.c index 1d84fa2b7e..dfa179d542 100644 --- a/src/pager.c +++ b/src/pager.c @@ -273,7 +273,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** * A write transaction is active. ** * An EXCLUSIVE or greater lock is held on the database file. ** * All writing and syncing of journal and database data has finished. -** If no error occured, all that remains is to finalize the journal to +** If no error occurred, all that remains is to finalize the journal to ** commit the transaction. If an error did occur, the caller will need ** to rollback the transaction. ** @@ -521,7 +521,7 @@ struct PagerSavepoint { ** ** doNotSpill, doNotSyncSpill ** -** These two boolean variables control the behaviour of cache-spills +** These two boolean variables control the behavior of cache-spills ** (calls made by the pcache module to the pagerStress() routine to ** write cached data to the file-system in order to free up memory). ** @@ -1399,7 +1399,7 @@ static int writeJournalHdr(Pager *pPager){ memset(zHeader, 0, sizeof(aJournalMagic)+4); } - /* The random check-hash initialiser */ + /* The random check-hash initializer */ sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit); put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit); /* The initial database size */ @@ -3731,7 +3731,7 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){ ** dirty page were to be discarded from the cache via the pagerStress() ** routine, pagerStress() would not write the current page content to ** the database file. If a savepoint transaction were rolled back after -** this happened, the correct behaviour would be to restore the current +** this happened, the correct behavior would be to restore the current ** content of the page. However, since this content is not present in either ** the database file or the portion of the rollback journal and ** sub-journal rolled back the content could not be restored and the diff --git a/src/prepare.c b/src/prepare.c index 21a12f32cb..26d6c2614f 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -179,7 +179,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ /* zMasterSchema and zInitScript are set to point at the master schema ** and initialisation script appropriate for the database being - ** initialised. zMasterName is the name of the master table. + ** initialized. zMasterName is the name of the master table. */ if( !OMIT_TEMPDB && iDb==1 ){ zMasterSchema = temp_master_schema; @@ -404,7 +404,7 @@ int sqlite3Init(sqlite3 *db, char **pzErrMsg){ } } - /* Once all the other databases have been initialised, load the schema + /* Once all the other databases have been initialized, load the schema ** for the TEMP database. This is loaded last, as the TEMP database ** schema may contain references to objects in other databases. */ @@ -427,7 +427,7 @@ int sqlite3Init(sqlite3 *db, char **pzErrMsg){ } /* -** This routine is a no-op if the database schema is already initialised. +** This routine is a no-op if the database schema is already initialized. ** Otherwise, the schema is loaded. An error code is returned. */ int sqlite3ReadSchema(Parse *pParse){ diff --git a/src/select.c b/src/select.c index 7e2bed434e..c7948096c9 100644 --- a/src/select.c +++ b/src/select.c @@ -4570,7 +4570,7 @@ int sqlite3Select( ** value of x, the only row required). ** ** A special flag must be passed to sqlite3WhereBegin() to slightly - ** modify behaviour as follows: + ** modify behavior as follows: ** ** + If the query is a "SELECT min(x)", then the loop coded by ** where.c should not iterate over any values with a NULL value diff --git a/src/sqlite.h.in b/src/sqlite.h.in index a6f9b0b370..5c89f7e4e9 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2675,7 +2675,7 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** sqlite3_open_v2(). ^Setting the cache parameter to "private" is ** equivalent to setting the SQLITE_OPEN_PRIVATECACHE bit. ** ^If sqlite3_open_v2() is used and the "cache" parameter is present in -** a URI filename, its value overrides any behaviour requested by setting +** a URI filename, its value overrides any behavior requested by setting ** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. ** ** @@ -6378,7 +6378,7 @@ struct sqlite3_pcache_page { ** parameter to help it determined what action to take: ** ** -**
createFlag Behaviour when page is not already in cache +**
createFlag Behavior when page is not already in cache **
0 Do not allocate a new page. Return NULL. **
1 Allocate a new page if it easy and convenient to do so. ** Otherwise return NULL. diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 2777d2f9f8..c0c4fb6499 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -1005,7 +1005,7 @@ static int DbTransPostCmd( /* This is a tricky scenario to handle. The most likely cause of an ** error is that the exec() above was an attempt to commit the ** top-level transaction that returned SQLITE_BUSY. Or, less likely, - ** that an IO-error has occured. In either case, throw a Tcl exception + ** that an IO-error has occurred. In either case, throw a Tcl exception ** and try to rollback the transaction. ** ** But it could also be that the user executed one or more BEGIN, diff --git a/src/test6.c b/src/test6.c index f511be9def..c151ea4298 100644 --- a/src/test6.c +++ b/src/test6.c @@ -87,7 +87,7 @@ typedef struct WriteBuffer WriteBuffer; ** an aligned write() of an integer number of 512 byte regions, then ** option (3) above is never selected. Instead, each 512 byte region ** is either correctly written or left completely untouched. Similar -** logic governs the behaviour if any of the other ATOMICXXX flags +** logic governs the behavior if any of the other ATOMICXXX flags ** is set. ** ** If either the IOCAP_SAFEAPPEND or IOCAP_SEQUENTIAL flags are set diff --git a/src/test_multiplex.c b/src/test_multiplex.c index 23df347ded..18a3a0740f 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -60,7 +60,7 @@ /* ** These should be defined to be the same as the values in -** sqliteInt.h. They are defined seperately here so that +** sqliteInt.h. They are defined separately here so that ** the multiplex VFS shim can be built as a loadable ** module. */ diff --git a/src/test_sqllog.c b/src/test_sqllog.c index 7cb570bcf2..24c675b5e9 100644 --- a/src/test_sqllog.c +++ b/src/test_sqllog.c @@ -32,7 +32,7 @@ ** ** Usually, if the application opens the same database file more than once ** (either by attaching it or by using more than one database handle), only -** a single copy is made. This behaviour may be overridden (so that a +** a single copy is made. This behavior may be overridden (so that a ** separate copy is taken each time the database file is opened or attached) ** by setting the environment variable SQLITE_SQLLOG_REUSE_FILES to 0. ** diff --git a/src/update.c b/src/update.c index 96ba4df83d..1125e5971a 100644 --- a/src/update.c +++ b/src/update.c @@ -458,7 +458,7 @@ void sqlite3Update( /* The row-trigger may have deleted the row being updated. In this ** case, jump to the next row. No updates or AFTER triggers are - ** required. This behaviour - what happens when the row being updated + ** required. This behavior - what happens when the row being updated ** is deleted or renamed by a BEFORE trigger - is left undefined in the ** documentation. */ diff --git a/src/vdbe.c b/src/vdbe.c index b18661c1d3..21c0cf8ec4 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5784,7 +5784,7 @@ case OP_VOpen: { /* Initialize sqlite3_vtab_cursor base class */ pVtabCursor->pVtab = pVtab; - /* Initialise vdbe cursor object */ + /* Initialize vdbe cursor object */ pCur = allocateCursor(p, pOp->p1, 0, -1, 0); if( pCur ){ pCur->pVtabCursor = pVtabCursor; diff --git a/src/vdbeapi.c b/src/vdbeapi.c index ae3ce23902..2387438356 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -445,7 +445,7 @@ end_of_step: assert( p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE ); if( p->isPrepareV2 && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){ /* If this statement was prepared using sqlite3_prepare_v2(), and an - ** error has occured, then return the error code in p->rc to the + ** error has occurred, then return the error code in p->rc to the ** caller. Set the error code in the database handle to the same value. */ rc = sqlite3VdbeTransferError(p); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index cf9237f286..2c4269a59e 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -375,7 +375,7 @@ int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ } sqlite3DbFree(v->db, sIter.apSub); - /* Return true if hasAbort==mayAbort. Or if a malloc failure occured. + /* Return true if hasAbort==mayAbort. Or if a malloc failure occurred. ** If malloc failed, then the while() loop above may not have iterated ** through all opcodes and hasAbort may be set incorrectly. Return ** true for this case to prevent the assert() in the callers frame @@ -2007,7 +2007,7 @@ int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){ /* If p->iStatement is greater than zero, then this Vdbe opened a ** statement transaction that should be closed here. The only exception - ** is that an IO error may have occured, causing an emergency rollback. + ** is that an IO error may have occurred, causing an emergency rollback. ** In this case (db->nStatement==0), and there is nothing to do. */ if( db->nStatement && p->iStatement ){ @@ -2143,7 +2143,7 @@ int sqlite3VdbeHalt(Vdbe *p){ ** ** Even if the statement is read-only, it is important to perform ** a statement or transaction rollback operation. If the error - ** occured while writing to the journal, sub-journal or database + ** occurred while writing to the journal, sub-journal or database ** file as part of an effort to free up cache space (see function ** pagerStress() in pager.c), the rollback is required to restore ** the pager to a consistent state. @@ -2557,7 +2557,7 @@ int sqlite3VdbeCursorMoveto(VdbeCursor *p){ ** the blob of data that it corresponds to. In a table record, all serial ** types are stored at the start of the record, and the blobs of data at ** the end. Hence these functions allow the caller to handle the -** serial-type and data blob seperately. +** serial-type and data blob separately. ** ** The following table describes the various storage classes for data: ** diff --git a/src/where.c b/src/where.c index 25f3d4c54d..9b64fd72fe 100644 --- a/src/where.c +++ b/src/where.c @@ -562,7 +562,7 @@ static int allowedOp(int op){ ** Commute a comparison operator. Expressions of the form "X op Y" ** are converted into "Y op X". ** -** If left/right precendence rules come into play when determining the +** If left/right precedence rules come into play when determining the ** collating ** side of the comparison, it remains associated with the same side after ** the commutation. So "Y collate NOCASE op X" becomes @@ -3628,7 +3628,7 @@ static void bestBtreeIndex(WhereBestIdx *p){ /* If there is no ORDER BY clause and the SQLITE_ReverseOrder flag ** is set, then reverse the order that the index will be scanned ** in. This is used for application testing, to help find cases - ** where application behaviour depends on the (undefined) order that + ** where application behavior depends on the (undefined) order that ** SQLite outputs rows in in the absence of an ORDER BY clause. */ if( !p->pOrderBy && pParse->db->flags & SQLITE_ReverseOrder ){ p->cost.plan.wsFlags |= WHERE_REVERSE; diff --git a/test/backup_ioerr.test b/test/backup_ioerr.test index 313cff3da9..ca3fd3240d 100644 --- a/test/backup_ioerr.test +++ b/test/backup_ioerr.test @@ -115,7 +115,7 @@ proc clear_ioerr_simulation {} { # reported, then the backup process is concluded with a call to # backup_finish(). # -# Test that if an IO error occurs, or if one occured while updating +# Test that if an IO error occurs, or if one occurred while updating # the backup database during step 4, then the conditions listed # under step 3 are all true. # @@ -214,7 +214,7 @@ for {set iError 1} {$bStop == 0} {incr iError} { set rc [catchsql { UPDATE t1 SET b = randstr(1000,1000) WHERE a < 50 } sdb] if {[lindex $rc 0] && $::sqlite_io_error_persist==0} { - # The IO error occured while updating the source database. In this + # The IO error occurred while updating the source database. In this # case the backup should be able to continue. set rc [B step 5000] if { $rc != "SQLITE_IOERR_UNLOCK" } { diff --git a/test/cache.test b/test/cache.test index f81948ba2c..ffc25c460e 100644 --- a/test/cache.test +++ b/test/cache.test @@ -46,7 +46,7 @@ do_test cache-1.2 { # leaked, but would not be reused until the pager-cache was full (i.e. # 2000 pages by default). # -# This tests that once the pager-cache is initialised, it can be locked +# This tests that once the pager-cache is initialized, it can be locked # and unlocked repeatedly without internally allocating any new pages. # set cache_size [pager_cache_size db] diff --git a/test/collate4.test b/test/collate4.test index 6b3a1c7aee..5ae1e7b930 100644 --- a/test/collate4.test +++ b/test/collate4.test @@ -60,7 +60,7 @@ proc cksort {sql} { # # Because these tests also exercise all the different ways indices # can be created, they also serve to verify that indices are correctly -# initialised with user-defined collation sequences when they are +# initialized with user-defined collation sequences when they are # created. # # Tests named collate4-1.1.* use indices with a single column. Tests diff --git a/test/crash5.test b/test/crash5.test index a786712354..83d1647a84 100644 --- a/test/crash5.test +++ b/test/crash5.test @@ -65,7 +65,7 @@ for {set ii 0} {$ii < 10} {incr ii} { # puts "$n $msg ac=[sqlite3_get_autocommit db]" # If the transaction is still active (it may not be if the malloc() - # failure occured in the OS layer), write to the database. Make sure + # failure occurred in the OS layer), write to the database. Make sure # page 4 is among those written. # if {![sqlite3_get_autocommit db]} { diff --git a/test/e_createtable.test b/test/e_createtable.test index 35f7330c4b..351a0f7220 100644 --- a/test/e_createtable.test +++ b/test/e_createtable.test @@ -1257,7 +1257,7 @@ do_createtable_tests 4.4 { # SQLite allows NULL values in a PRIMARY KEY column. # # If the column is an integer primary key, attempting to insert a NULL -# into the column triggers the auto-increment behaviour. Attempting +# into the column triggers the auto-increment behavior. Attempting # to use UPDATE to set an ipk column to a NULL value is an error. # do_createtable_tests 4.5.1 { diff --git a/test/e_fkey.test b/test/e_fkey.test index 01bf534527..001ba6c386 100644 --- a/test/e_fkey.test +++ b/test/e_fkey.test @@ -2741,8 +2741,8 @@ do_test e_fkey-60.6 { } {} #------------------------------------------------------------------------- -# Test that the special behaviours of ALTER and DROP TABLE are only -# activated when foreign keys are enabled. Special behaviours are: +# Test that the special behaviors of ALTER and DROP TABLE are only +# activated when foreign keys are enabled. Special behaviors are: # # 1. ADD COLUMN not allowing a REFERENCES clause with a non-NULL # default value. @@ -2837,7 +2837,7 @@ foreach zMatch [list SIMPLE PARTIAL FULL Simple parTIAL FuLL ] { do_test e_fkey-62.$zMatch.2 { execsql { INSERT INTO p VALUES(1, 2, 3) } - # MATCH SIMPLE behaviour: Allow any child key that contains one or more + # MATCH SIMPLE behavior: Allow any child key that contains one or more # NULL value to be inserted. Non-NULL values do not have to map to any # parent key values, so long as at least one field of the child key is # NULL. diff --git a/test/e_select.test b/test/e_select.test index fb63d051de..ea44aeda9a 100644 --- a/test/e_select.test +++ b/test/e_select.test @@ -1227,7 +1227,7 @@ do_select_tests e_select-5.1 { # the entire set of result rows are returned by the SELECT. # # EVIDENCE-OF: R-47911-02086 If neither ALL or DISTINCT are present, -# then the behaviour is as if ALL were specified. +# then the behavior is as if ALL were specified. # # EVIDENCE-OF: R-14442-41305 If the simple SELECT is a SELECT DISTINCT, # then duplicate rows are removed from the set of result rows before it diff --git a/test/e_uri.test b/test/e_uri.test index f16c0cb12c..ac34ed49aa 100644 --- a/test/e_uri.test +++ b/test/e_uri.test @@ -361,7 +361,7 @@ foreach {tn uri error} " # # EVIDENCE-OF: R-19510-48080 If sqlite3_open_v2() is used and the # "cache" parameter is present in a URI filename, its value overrides -# any behaviour requested by setting SQLITE_OPEN_PRIVATECACHE or +# any behavior requested by setting SQLITE_OPEN_PRIVATECACHE or # SQLITE_OPEN_SHAREDCACHE flag. # set orig [sqlite3_enable_shared_cache] diff --git a/test/enc2.test b/test/enc2.test index 415bc0f720..3eb3aa27c3 100644 --- a/test/enc2.test +++ b/test/enc2.test @@ -32,7 +32,7 @@ ifcapable {!utf16} { # enc2.3.*: Simple tests with a UTF-16BE db. # enc2.4.*: Test that attached databases must have the same text encoding # as the main database. -# enc2.5.*: Test the behaviour of the library when a collation sequence is +# enc2.5.*: Test the behavior of the library when a collation sequence is # not available for the most desirable text encoding. # enc2.6.*: Similar test for user functions. # enc2.7.*: Test that the VerifyCookie opcode protects against assuming the diff --git a/test/incrblob.test b/test/incrblob.test index 7cc99dd983..6634f90f9c 100644 --- a/test/incrblob.test +++ b/test/incrblob.test @@ -505,7 +505,7 @@ if {[permutation] != "memsubsys1"} { sqlite3_soft_heap_limit $cmdlinearg(soft-heap-limit) #----------------------------------------------------------------------- -# The following tests verify the behaviour of the incremental IO +# The following tests verify the behavior of the incremental IO # APIs in the following cases: # # 7.1 A row that containing an open blob is modified. @@ -516,7 +516,7 @@ sqlite3_soft_heap_limit $cmdlinearg(soft-heap-limit) # 7.3 An INCREMENTAL VACUUM moves an overflow page that is part # of an open blob. # -# In the first case above, correct behaviour is for all subsequent +# In the first case above, correct behavior is for all subsequent # read/write operations on the blob-handle to return SQLITE_ABORT. # More accurately, blob-handles are invalidated whenever the table # they belong to is written to. diff --git a/test/io.test b/test/io.test index 9363b0c292..bf4d157275 100644 --- a/test/io.test +++ b/test/io.test @@ -207,7 +207,7 @@ do_test io-2.5.3 { # Changed 2010-03-27: The size of the database is now stored in # bytes 28..31 and so when a page is added to the database, page 1 # is immediately modified and the journal file immediately comes into -# existance. To fix this test, the BEGIN is changed into a a +# existence. To fix this test, the BEGIN is changed into a a # BEGIN IMMEDIATE and the INSERT is omitted. # do_test io-2.6.1 { diff --git a/test/malloc.test b/test/malloc.test index 0d213d7bb9..5d03aa8fe8 100644 --- a/test/malloc.test +++ b/test/malloc.test @@ -842,7 +842,7 @@ do_malloc_test 36 -sqlprep { SELECT test_agg_errmsg16(), group_concat(a) FROM t1 } -# At one point, if an OOM occured immediately after obtaining a shared lock +# At one point, if an OOM occurred immediately after obtaining a shared lock # on the database file, the file remained locked. This test case ensures # that bug has been fixed.i if {[db eval {PRAGMA locking_mode}]!="exclusive"} { diff --git a/test/malloc3.test b/test/malloc3.test index 04f7d32c30..f4a6c3bbe9 100644 --- a/test/malloc3.test +++ b/test/malloc3.test @@ -595,8 +595,8 @@ proc run_test {arglist iRepeat {pcstart 0} {iFailStart 1}} { if {$rc != 0 && $nac && !$ac} { # Before [db eval] the auto-commit flag was clear. Now it - # is set. Since an error occured we assume this was not a - # commit - therefore a rollback occured. Check that the + # is set. Since an error occurred we assume this was not a + # commit - therefore a rollback occurred. Check that the # rollback-hook was invoked. do_test malloc3-rollback_hook_count.$iterid { set ::rollback_hook_count diff --git a/test/notify2.test b/test/notify2.test index 4016b6db11..9e40ed695b 100644 --- a/test/notify2.test +++ b/test/notify2.test @@ -150,9 +150,9 @@ set sql $zSql # Hit some other kind of error. This is a malfunction. error $msg } else { - # No error occured. Check that any SELECT statements in the transaction + # No error occurred. Check that any SELECT statements in the transaction # returned "1". Otherwise, the invariant was false, indicating that - # some malfunction has occured. + # some malfunction has occurred. foreach r $msg { if {$r != 1} { puts "Invariant check failed: $msg" } } } } diff --git a/test/subquery.test b/test/subquery.test index d9d2952fbc..f601d3f80b 100644 --- a/test/subquery.test +++ b/test/subquery.test @@ -421,7 +421,7 @@ do_test subquery-3.5.7 { # and expose bugs to do with re-using statements that have been # passed to sqlite3_reset(). # -# One problem was that VDBE memory cells were not being initialised +# One problem was that VDBE memory cells were not being initialized # to NULL on the second and subsequent executions. # do_test subquery-4.1.1 { diff --git a/test/temptable.test b/test/temptable.test index 5eeb0f5786..6a1e2b9867 100644 --- a/test/temptable.test +++ b/test/temptable.test @@ -150,7 +150,7 @@ do_test temptable-3.4 { # Check for correct name collision processing. A name collision can # occur when process A creates a temporary table T then process B # creates a permanent table also named T. The temp table in process A -# hides the existance of the permanent table. +# hides the existence of the permanent table. # do_test temptable-4.1 { execsql { diff --git a/test/tester.tcl b/test/tester.tcl index f360840789..d6468f7b14 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -123,7 +123,7 @@ if {[info command sqlite_orig]==""} { set res } else { # This command is not opening a new database connection. Pass the - # arguments through to the C implemenation as the are. + # arguments through to the C implementation as the are. # uplevel 1 sqlite_orig $args } @@ -1037,7 +1037,7 @@ proc ifcapable {expr code {else ""} {elsecode ""}} { # boolean, indicating whether or not the process actually crashed or # reported some other error. The second element in the returned list is the # error message. This is "child process exited abnormally" if the crash -# occured. +# occurred. # # crashsql -delay CRASHDELAY -file CRASHFILE ?-blocksize BLOCKSIZE? $sql # @@ -1317,7 +1317,7 @@ proc do_ioerr_test {testname args} { } } - # If an IO error occured, then the checksum of the database should + # If an IO error occurred, then the checksum of the database should # be the same as before the script that caused the IO error was run. # if {$::go && $::sqlite_io_error_hardhit && $::ioerropts(-cksum)} { diff --git a/test/tkt2409.test b/test/tkt2409.test index 9ac65424f1..936f57c98c 100644 --- a/test/tkt2409.test +++ b/test/tkt2409.test @@ -107,7 +107,7 @@ do_test tkt2409-1.2 { integrity_check tkt2409-1.3 # Check that the transaction was rolled back. Because the INSERT -# statement in which the "I/O error" occured did not open a statement +# statement in which the "I/O error" occurred did not open a statement # transaction, SQLite had no choice but to roll back the transaction. # do_test tkt2409-1.4 { @@ -175,7 +175,7 @@ do_test tkt2409-3.2 { integrity_check tkt2409-3.3 # Check that the transaction was rolled back. Because the INSERT -# statement in which the "I/O error" occured did not open a statement +# statement in which the "I/O error" occurred did not open a statement # transaction, SQLite had no choice but to roll back the transaction. # do_test tkt2409-3.4 { From 6101d50471a97423032fbffcb72f5b62a28a90ee Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 22 Mar 2013 08:58:38 +0000 Subject: [PATCH 018/204] Add assert statements to os_unix.c to ensure that any mapped region of the database file is not being read or written using the xRead() or xWrite() methods. FossilOrigin-Name: 765615f9fba7c1765eb741cb98a09a28b464ee55 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 12 ++++++++++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 4b2a8a6d5d..7a0b83d9c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scases\swhere\sxRead()\swas\sbeing\sused\sto\sread\sfrom\sa\smemory\smapped\spart\sof\sthe\sdatabase\sfile. -D 2013-03-21T20:39:55.225 +C Add\sassert\sstatements\sto\sos_unix.c\sto\sensure\sthat\sany\smapped\sregion\sof\sthe\sdatabase\sfile\sis\snot\sbeing\sread\sor\swritten\susing\sthe\sxRead()\sor\sxWrite()\smethods. +D 2013-03-22T08:58:38.319 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c be66c31337361a72227638d6f41c7f2031739642 +F src/os_unix.c 18153e6fc03580e95d28fb39d47a4e8b64b36187 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 F src/pager.c 520001015155efee9599c807dfd38e5fff9c6e36 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c8eac290a7240d69494bd0dad5ed1fdc2505f703 -R 0cdb7517bc23cd336050741c62097e56 +P 5c9e9df27b9f2c46cd55388a858d4e78ee564975 +R eb2eb83f96de836951dbdb14158ccee8 U dan -Z 0dd43854f08b06a022f11b1a174cdf13 +Z 42c0bd3cd106c51e7465866daa07e0a3 diff --git a/manifest.uuid b/manifest.uuid index 50136e7e1b..e0d24b066f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5c9e9df27b9f2c46cd55388a858d4e78ee564975 \ No newline at end of file +765615f9fba7c1765eb741cb98a09a28b464ee55 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 153cf5f435..88f3177021 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -249,6 +249,8 @@ struct unixFile { unsigned char transCntrChng; /* True if the transaction counter changed */ unsigned char dbUpdate; /* True if any part of database file changed */ unsigned char inNormalWrite; /* True if in a normal write operation */ + sqlite3_int64 mmapSize; /* Size of xMremap() */ + void *pMapRegion; /* Area memory mapped */ #endif #ifdef SQLITE_TEST /* In test mode, increase the size of this structure a bit so that @@ -3072,6 +3074,7 @@ static int unixRead( unixFile *pFile = (unixFile *)id; int got; assert( id ); + assert( offset>=pFile->mmapSize ); /* Never read from the mmapped region */ /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ @@ -3154,6 +3157,7 @@ static int unixWrite( int wrote = 0; assert( id ); assert( amt>0 ); + assert( offset>=pFile->mmapSize ); /* Never write into the mmapped region */ /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ @@ -4457,6 +4461,8 @@ static int unixMremap( i64 nOldRnd; /* nOld rounded up */ assert( iOff==0 ); + assert( p->mmapSize==nOld ); + assert( p->pMapRegion==0 || p->pMapRegion==(*ppMap) ); /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested ** mapping (nNew bytes) may be greater than the size of the database file. @@ -4494,6 +4500,7 @@ static int unixMremap( /* On OSX or Linux, reuse the old mapping if it is the right size. */ #if defined(__APPLE__) || defined(__linux__) if( nNewRnd==nOldRnd ){ + VVA_ONLY( p->mmapSize = nNew; ) return SQLITE_OK; } #endif @@ -4502,6 +4509,7 @@ static int unixMremap( if( nOldRnd!=0 ){ void *pOld = *ppMap; munmap(pOld, nOldRnd); + VVA_ONLY( p->mmapSize = 0; p->pMapRegion = 0; ); } /* And, if required, use mmap() to create a new mapping. */ @@ -4511,7 +4519,10 @@ static int unixMremap( pNew = mmap(0, nNewRnd, flags, MAP_SHARED, p->h, iOff); if( pNew==MAP_FAILED ){ pNew = 0; + VVA_ONLY( p->mmapSize = 0; p->pMapRegion = 0; ) rc = SQLITE_IOERR_MREMAP; + }else{ + VVA_ONLY( p->mmapSize = nNew; p->pMapRegion = pNew; ) } } @@ -4846,6 +4857,7 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; + VVA_ONLY( pNew->mmapSize = 0; ) if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; From c00033125d612de5877ea2682cd255dd3199c00c Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 22 Mar 2013 17:46:11 +0000 Subject: [PATCH 019/204] Add a fix for the assert() statements added by the previous commit. FossilOrigin-Name: 19345416ed5e1ab5b0b35993b0b9069c2fb1683b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 10 +++++++++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7a0b83d9c2..1be3bebe19 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sassert\sstatements\sto\sos_unix.c\sto\sensure\sthat\sany\smapped\sregion\sof\sthe\sdatabase\sfile\sis\snot\sbeing\sread\sor\swritten\susing\sthe\sxRead()\sor\sxWrite()\smethods. -D 2013-03-22T08:58:38.319 +C Add\sa\sfix\sfor\sthe\sassert()\sstatements\sadded\sby\sthe\sprevious\scommit. +D 2013-03-22T17:46:11.890 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 18153e6fc03580e95d28fb39d47a4e8b64b36187 +F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 F src/pager.c 520001015155efee9599c807dfd38e5fff9c6e36 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5c9e9df27b9f2c46cd55388a858d4e78ee564975 -R eb2eb83f96de836951dbdb14158ccee8 +P 765615f9fba7c1765eb741cb98a09a28b464ee55 +R 125e508e18383c5f0a5b411a39a78cab U dan -Z 42c0bd3cd106c51e7465866daa07e0a3 +Z 4d1b2a2cebcc3fe5bdd2b5af2205540b diff --git a/manifest.uuid b/manifest.uuid index e0d24b066f..8efe087450 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -765615f9fba7c1765eb741cb98a09a28b464ee55 \ No newline at end of file +19345416ed5e1ab5b0b35993b0b9069c2fb1683b \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 88f3177021..2f03c39670 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3470,6 +3470,14 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ if( pFile->inNormalWrite && nByte==0 ){ pFile->transCntrChng = 1; } + + /* If the file was just truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( nBytemmapSize ){ + pFile->mmapSize = nByte; + } #endif return SQLITE_OK; @@ -4461,7 +4469,7 @@ static int unixMremap( i64 nOldRnd; /* nOld rounded up */ assert( iOff==0 ); - assert( p->mmapSize==nOld ); + /* assert( p->mmapSize==nOld ); */ assert( p->pMapRegion==0 || p->pMapRegion==(*ppMap) ); /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested From 99bd10979abd01877d6c213235c0ad22ab142a8f Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 22 Mar 2013 18:20:14 +0000 Subject: [PATCH 020/204] Allow the database file to be memory mapped in wal mode. FossilOrigin-Name: d190ddabc386bc9654b99e33fb81b2f6e67b54d6 --- manifest | 18 +++++------ manifest.uuid | 2 +- src/pager.c | 88 +++++++++++++++++++++++++++++++++++++++++---------- src/pager.h | 4 +++ src/wal.c | 67 ++++++++++++++++++++------------------- src/wal.h | 7 ++-- 6 files changed, 124 insertions(+), 62 deletions(-) diff --git a/manifest b/manifest index 1be3bebe19..cab9a67764 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sfix\sfor\sthe\sassert()\sstatements\sadded\sby\sthe\sprevious\scommit. -D 2013-03-22T17:46:11.890 +C Allow\sthe\sdatabase\sfile\sto\sbe\smemory\smapped\sin\swal\smode. +D 2013-03-22T18:20:14.143 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,8 +162,8 @@ F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 520001015155efee9599c807dfd38e5fff9c6e36 -F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c +F src/pager.c 12b8ff12519fe529a92884c4cdb99afecb1bea3c +F src/pager.h bbc9170281c9d5d603b2175fdc8ea908e47269a7 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 @@ -249,8 +249,8 @@ F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c c61ca318681c0e7267da8be3abfca8469652a7e9 F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 -F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2 -F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 +F src/wal.c 7fec703bb0770d9faad5190f49897a8bc8eb239f +F src/wal.h d99ce512ac60f9147a0640e9e6fb67dd1057b781 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b F src/where.c bdbbfa7ef4ea04c8d9b09585b45d4717a72f980a F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 765615f9fba7c1765eb741cb98a09a28b464ee55 -R 125e508e18383c5f0a5b411a39a78cab +P 19345416ed5e1ab5b0b35993b0b9069c2fb1683b +R 7243a86b7c03fc5d49450319f9d9b1bf U dan -Z 4d1b2a2cebcc3fe5bdd2b5af2205540b +Z 7b1a30ba76756800b3f32db2afc843bf diff --git a/manifest.uuid b/manifest.uuid index 8efe087450..4f8ecfff16 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -19345416ed5e1ab5b0b35993b0b9069c2fb1683b \ No newline at end of file +d190ddabc386bc9654b99e33fb81b2f6e67b54d6 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 0ff894bd87..660514d508 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2095,7 +2095,7 @@ static void pagerReportSize(Pager *pPager){ ** Return SQLITE_OK if successful, or an SQLite error code if an error ** occurs. */ -static int pagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff){ +int sqlite3PagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff){ int rc = SQLITE_OK; if( pPager->nMapValid>=(iOff+nBuf) ){ memcpy(&((u8 *)(pPager->pMap))[iOff], pBuf, nBuf); @@ -2279,7 +2279,7 @@ static int pager_playback_one_page( i64 ofst = (pgno-1)*(i64)pPager->pageSize; testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); assert( !pagerUseWal(pPager) ); - rc = pagerWriteData(pPager, aData, pPager->pageSize, ofst); + rc = sqlite3PagerWriteData(pPager, aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } @@ -2864,11 +2864,10 @@ end_playback: ** If an IO error occurs, then the IO error is returned to the caller. ** Otherwise, SQLITE_OK is returned. */ -static int readDbPage(PgHdr *pPg){ +static int readDbPage(PgHdr *pPg, u32 iFrame){ Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */ Pgno pgno = pPg->pgno; /* Page number to read */ int rc = SQLITE_OK; /* Return code */ - int isInWal = 0; /* True if page is in log file */ int pgsz = pPager->pageSize; /* Number of bytes to read */ assert( pPager->eState>=PAGER_READER && !MEMDB ); @@ -2880,11 +2879,10 @@ static int readDbPage(PgHdr *pPg){ return SQLITE_OK; } - if( pagerUseWal(pPager) ){ + if( iFrame ){ /* Try to pull the page from the write-ahead log. */ - rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pgsz, pPg->pData); - } - if( rc==SQLITE_OK && !isInWal ){ + rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData); + }else{ i64 iOffset = (pgno-1)*(i64)pPager->pageSize; if( pPager->pMap && pPager->nMapValid>=iOffset+pPager->pageSize ){ memcpy(pPg->pData, &((u8 *)(pPager->pMap))[iOffset], pPager->pageSize); @@ -2972,7 +2970,13 @@ static int pagerUndoCallback(void *pCtx, Pgno iPg){ if( sqlite3PcachePageRefcount(pPg)==1 ){ sqlite3PcacheDrop(pPg); }else{ - rc = readDbPage(pPg); + u32 iFrame = 0; + if( pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); + } + if( rc==SQLITE_OK ){ + rc = readDbPage(pPg, iFrame); + } if( rc==SQLITE_OK ){ pPager->xReiniter(pPg); } @@ -3388,7 +3392,6 @@ static void pagerFixMaplimit(Pager *pPager){ || pPager->fd->pMethods->iVersion<3 || pPager->fd->pMethods->xMremap==0 || pPager->tempFile - || pPager->pWal ){ pPager->nMapLimit = 0; }else if( pPager->nMapCfgLimit<0 ){ @@ -3888,9 +3891,9 @@ static int pagerMap(Pager *pPager, int bExtend){ Pgno nPg; /* Size of mapping to request in pages */ i64 sz; /* Size of mapping to request in bytes */ - assert( pPager->pWal==0 && isOpen(pPager->fd) && pPager->tempFile==0 ); + assert( isOpen(pPager->fd) && pPager->tempFile==0 ); assert( pPager->pMap==0 || pPager->nMap>0 ); - assert( pPager->eState>=1 ); + /* assert( pPager->eState>=1 ); */ assert( pPager->nMmapOut==0 ); assert( pPager->nMapLimit>0 ); @@ -4213,6 +4216,46 @@ static int syncJournal(Pager *pPager, int newHdr){ return SQLITE_OK; } +/* +** This is called by the wal.c module at the start of a checkpoint. If the +** checkpoint runs to completion, it will set the database file size to +** szReq bytes. This function performs two tasks: +** +** * If the file is currently less than szReq bytes in size, an +** xFileControl(SQLITE_FNCTL_SIZE_HINT) is issued to inform the OS +** layer of the expected file size, and +** +** * If mmap is being used, then the mapping is extended to szReq +** bytes in size. +** +** SQLITE_OK is returned if successful, or an error code if an error occurs. +*/ +int sqlite3PagerSetFilesize(Pager *pPager, i64 szReq){ + int rc; + sqlite3_int64 sz; + + assert( pPager->eState==PAGER_OPEN ); + assert( pPager->nMmapOut==0 ); + + rc = sqlite3OsFileSize(pPager->fd, &sz); + if( rc==SQLITE_OK ){ + if( sz>szReq ){ + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &sz); + } + } + + if( rc==SQLITE_OK + && pPager->nMapLimit>0 + && pPager->nMapValidnMapValidnMapLimit + ){ + pPager->dbFileSize = (szReq / pPager->pageSize); + rc = pagerMap(pPager, 1); + } + + return rc; +} + /* ** The argument is the first in a linked list of dirty pages connected ** by the PgHdr.pDirty pointer. This function writes each one of the @@ -4275,7 +4318,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ if( pPager->nMmapOut==0 && pPager->nMapLimit>0 ){ pPager->dbFileSize = pPager->dbSize; - pagerMap(pPager, 1); + rc = pagerMap(pPager, 1); } } @@ -4301,7 +4344,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); /* Write out the page data. */ - pagerWriteData(pPager, pData, pPager->pageSize, offset); + rc = sqlite3PagerWriteData(pPager, pData, pPager->pageSize, offset); /* If page 1 was just written, update Pager.dbFileVers to match ** the value now stored in the database file. If writing this @@ -5275,6 +5318,7 @@ int sqlite3PagerAcquire( ){ int rc = SQLITE_OK; PgHdr *pPg = 0; + u32 iFrame = 0; /* Frame to read from WAL file */ const int noContent = (flags & PAGER_ACQUIRE_NOCONTENT); /* It is acceptable to use a read-only (mmap) page for any page except @@ -5287,6 +5331,7 @@ int sqlite3PagerAcquire( assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); + assert( noContent==0 || bMmapOk==0 ); if( pgno==0 ){ return SQLITE_CORRUPT_BKPT; @@ -5298,7 +5343,12 @@ int sqlite3PagerAcquire( rc = pPager->errCode; }else{ - if( bMmapOk ){ + if( bMmapOk && pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } + + if( iFrame==0 && bMmapOk ){ if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){ rc = pagerMap(pPager, 0); } @@ -5378,9 +5428,13 @@ int sqlite3PagerAcquire( memset(pPg->pData, 0, pPager->pageSize); IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ + if( pagerUseWal(pPager) && bMmapOk==0 ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } assert( pPg->pPager==pPager ); pPager->aStat[PAGER_STAT_MISS]++; - rc = readDbPage(pPg); + rc = readDbPage(pPg, iFrame); if( rc!=SQLITE_OK ){ goto pager_acquire_err; } @@ -7063,7 +7117,7 @@ static int pagerOpenWal(Pager *pPager){ ** (e.g. due to malloc() failure), return an error code. */ if( rc==SQLITE_OK ){ - rc = sqlite3WalOpen(pPager->pVfs, + rc = sqlite3WalOpen(pPager->pVfs, pPager, pPager->fd, pPager->zWal, pPager->exclusiveMode, pPager->journalSizeLimit, &pPager->pWal ); diff --git a/src/pager.h b/src/pager.h index 81ab30c115..970b5035bf 100644 --- a/src/pager.h +++ b/src/pager.h @@ -174,6 +174,10 @@ int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); +int sqlite3PagerSetFilesize(Pager *, i64); + +/* Write data to the database file */ +int sqlite3PagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff); #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); diff --git a/src/wal.c b/src/wal.c index 0d7271bf04..8757003d78 100644 --- a/src/wal.c +++ b/src/wal.c @@ -412,6 +412,7 @@ struct Wal { sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */ sqlite3_file *pDbFd; /* File handle for the database file */ sqlite3_file *pWalFd; /* File handle for WAL file */ + Pager *pPager; /* Pager object */ u32 iCallback; /* Value to pass to log callback (or 0) */ i64 mxWalSize; /* Truncate WAL to this size upon reset */ int nWiData; /* Size of array apWiData */ @@ -1251,6 +1252,7 @@ static void walIndexClose(Wal *pWal, int isDelete){ */ int sqlite3WalOpen( sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */ + Pager *pPager, /* Pager object handle */ sqlite3_file *pDbFd, /* The open database file */ const char *zWalName, /* Name of the WAL file */ int bNoShm, /* True to run in heap-memory mode */ @@ -1291,6 +1293,7 @@ int sqlite3WalOpen( pRet->zWalName = zWalName; pRet->syncHeader = 1; pRet->padToSectorBoundary = 1; + pRet->pPager = pPager; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); /* Open file handle on the write-ahead log file. */ @@ -1727,10 +1730,7 @@ static int walCheckpoint( */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); - rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); - if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); - } + rc = sqlite3PagerSetFilesize(pWal->pPager, nReq); } /* Iterate through the contents of the WAL, copying data to the db file. */ @@ -1744,7 +1744,7 @@ static int walCheckpoint( if( rc!=SQLITE_OK ) break; iOffset = (iDbpage-1)*(i64)szPage; testcase( IS_BIG_INT(iOffset) ); - rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); + rc = sqlite3PagerWriteData(pWal->pPager, zBuf, szPage, iOffset); if( rc!=SQLITE_OK ) break; } @@ -2287,19 +2287,17 @@ void sqlite3WalEndReadTransaction(Wal *pWal){ } /* -** Read a page from the WAL, if it is present in the WAL and if the -** current read transaction is configured to use the WAL. +** Search the wal file for page pgno. If found, set *piRead to the frame that +** contains the page. Otherwise, if pgno is not in the wal file, set *piRead +** to zero. ** -** The *pInWal is set to 1 if the requested page is in the WAL and -** has been loaded. Or *pInWal is set to 0 if the page was not in -** the WAL and needs to be read out of the database. +** Return SQLITE_OK if successful, or an error code if an error occurs. If an +** error does occur, the final value of *piRead is undefined. */ -int sqlite3WalRead( +int sqlite3WalFindFrame( Wal *pWal, /* WAL handle */ Pgno pgno, /* Database page number to read data for */ - int *pInWal, /* OUT: True if data is read from WAL */ - int nOut, /* Size of buffer pOut in bytes */ - u8 *pOut /* Buffer to write page data to */ + u32 *piRead /* OUT: Frame number (or zero) */ ){ u32 iRead = 0; /* If !=0, WAL frame to return data from */ u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ @@ -2315,7 +2313,7 @@ int sqlite3WalRead( ** WAL were empty. */ if( iLast==0 || pWal->readLock==0 ){ - *pInWal = 0; + *piRead = 0; return SQLITE_OK; } @@ -2386,26 +2384,31 @@ int sqlite3WalRead( } #endif - /* If iRead is non-zero, then it is the log frame number that contains the - ** required page. Read and return data from the log file. - */ - if( iRead ){ - int sz; - i64 iOffset; - sz = pWal->hdr.szPage; - sz = (sz&0xfe00) + ((sz&0x0001)<<16); - testcase( sz<=32768 ); - testcase( sz>=65536 ); - iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; - *pInWal = 1; - /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ - return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); - } - - *pInWal = 0; + *piRead = iRead; return SQLITE_OK; } +/* +** Read the contents of frame iRead from the wal file into buffer pOut +** (which is nOut bytes in size). Return SQLITE_OK if successful, or an +** error code otherwise. +*/ +int sqlite3WalReadFrame( + Wal *pWal, /* WAL handle */ + u32 iRead, /* Frame to read */ + int nOut, /* Size of buffer pOut in bytes */ + u8 *pOut /* Buffer to write page data to */ +){ + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (sz&0xfe00) + ((sz&0x0001)<<16); + testcase( sz<=32768 ); + testcase( sz>=65536 ); + iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ + return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); +} /* ** Return the size of the database in pages (or zero, if unknown). diff --git a/src/wal.h b/src/wal.h index a848de1527..96d4ff9adf 100644 --- a/src/wal.h +++ b/src/wal.h @@ -31,7 +31,6 @@ # define sqlite3WalClose(w,x,y,z) 0 # define sqlite3WalBeginReadTransaction(y,z) 0 # define sqlite3WalEndReadTransaction(z) -# define sqlite3WalRead(v,w,x,y,z) 0 # define sqlite3WalDbsize(y) 0 # define sqlite3WalBeginWriteTransaction(y) 0 # define sqlite3WalEndWriteTransaction(x) 0 @@ -54,7 +53,8 @@ typedef struct Wal Wal; /* Open and close a connection to a write-ahead log. */ -int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**); +int sqlite3WalOpen( + sqlite3_vfs*, Pager *, sqlite3_file*, const char *, int, i64, Wal**); int sqlite3WalClose(Wal *pWal, int sync_flags, int, u8 *); /* Set the limiting size of a WAL file. */ @@ -71,7 +71,8 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *); void sqlite3WalEndReadTransaction(Wal *pWal); /* Read a page from the write-ahead log, if it is present. */ -int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, int nOut, u8 *pOut); +int sqlite3WalFindFrame(Wal *, Pgno, u32 *); +int sqlite3WalReadFrame(Wal *, u32, int, u8 *); /* If the WAL is not empty, return the size of the database. */ Pgno sqlite3WalDbsize(Wal *pWal); From aa1d67b1d92564e8360a07ff17c933d513542661 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 22 Mar 2013 19:17:45 +0000 Subject: [PATCH 021/204] Fix a case where the mapped part of a database file may be written during a backup operation. FossilOrigin-Name: c8d67aefff7299dd5c8abeb2a3a52109c0a823ee --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/backup.c | 2 +- test/permutations.test | 2 +- test/speed1p.test | 3 +++ 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index cab9a67764..cb648b797f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sthe\sdatabase\sfile\sto\sbe\smemory\smapped\sin\swal\smode. -D 2013-03-22T18:20:14.143 +C Fix\sa\scase\swhere\sthe\smapped\spart\sof\sa\sdatabase\sfile\smay\sbe\swritten\sduring\sa\sbackup\soperation. +D 2013-03-22T19:17:45.777 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -118,7 +118,7 @@ F src/alter.c f8db986c03eb0bfb221523fc9bbb9d0b70de3168 F src/analyze.c d5f895810e8ff9737c9ec7b76abc3dcff5860335 F src/attach.c ea5247f240e2c08afd608e9beb380814b86655e1 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 -F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 +F src/backup.c 6256400ab8be4a15a2512277d8b214dbd44ff8ce F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c dd4683fa671f0b2b4bc0831036adaa2a5d1806ee @@ -657,7 +657,7 @@ F test/pageropt.test 290cd59782b1890f02bb33795571facfc5ccac43 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test 694f4a2667242bab49cce05c54c2adfcc2727d9e +F test/permutations.test 4381ffc2811884cb4c2063176e4e4a849f236adc F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -730,7 +730,7 @@ F test/softheap1.test c16709a16ad79fa43b32929b2e623d1d117ccf53 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb -F test/speed1p.test c4a469f29f135f4d76c55b1f2a52f36e209466cc +F test/speed1p.test 2577211a9b054ffc973780f5b8708f02a0c7422d F test/speed2.test 53177056baf6556dcbdcf032bbdfc41c1aa74ded F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 19345416ed5e1ab5b0b35993b0b9069c2fb1683b -R 7243a86b7c03fc5d49450319f9d9b1bf +P d190ddabc386bc9654b99e33fb81b2f6e67b54d6 +R d5a852613692dd4e943f0ee647615880 U dan -Z 7b1a30ba76756800b3f32db2afc843bf +Z db771a6d89978ace4423b73d7feafccd diff --git a/manifest.uuid b/manifest.uuid index 4f8ecfff16..caccf2eff1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d190ddabc386bc9654b99e33fb81b2f6e67b54d6 \ No newline at end of file +c8d67aefff7299dd5c8abeb2a3a52109c0a823ee \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 71a8a1a3e7..8939bc67bc 100644 --- a/src/backup.c +++ b/src/backup.c @@ -520,7 +520,7 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); if( rc==SQLITE_OK ){ u8 *zData = sqlite3PagerGetData(pSrcPg); - rc = sqlite3OsWrite(pFile, zData, pgszSrc, iOff); + rc = sqlite3PagerWriteData(pDestPager, zData, pgszSrc, iOff); } sqlite3PagerUnref(pSrcPg); } diff --git a/test/permutations.test b/test/permutations.test index fd05b58e4f..1015066e84 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -144,7 +144,7 @@ test_suite "mmap" -prefix "mm-" -description { pragma mmap_size = -65536; } -files [ test_set $allquicktests -exclude *malloc* *ioerr* *fault* \ - multiplex* server1.test shared2.test shared6.test + multiplex* server1.test shared2.test shared6.test ] test_suite "valgrind" -prefix "" -description { diff --git a/test/speed1p.test b/test/speed1p.test index 915f165354..33be553b00 100644 --- a/test/speed1p.test +++ b/test/speed1p.test @@ -65,6 +65,9 @@ proc number_name {n} { # do_test speed1p-1.0 { execsql { + PRAGMA mmap_size=65536; + PRAGMA journal_mode=wal; + PRAGMA page_size=1024; PRAGMA cache_size=500; PRAGMA locking_mode=EXCLUSIVE; From 23f29ec6284aaa01efdc2abfda257b10139670c8 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 22 Mar 2013 19:56:27 +0000 Subject: [PATCH 022/204] Fix a potential NULL-pointer dereference in btreeSwapOutMmap(). FossilOrigin-Name: e81ccdcd87a1387f7ca08d4d7c899311e2180d32 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index cb648b797f..0a2be06535 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scase\swhere\sthe\smapped\spart\sof\sa\sdatabase\sfile\smay\sbe\swritten\sduring\sa\sbackup\soperation. -D 2013-03-22T19:17:45.777 +C Fix\sa\spotential\sNULL-pointer\sdereference\sin\sbtreeSwapOutMmap(). +D 2013-03-22T19:56:27.789 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 6256400ab8be4a15a2512277d8b214dbd44ff8ce F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c dd4683fa671f0b2b4bc0831036adaa2a5d1806ee +F src/btree.c f5ea8d3d658887b5cae0369ef10a427d7469a768 F src/btree.h d3259057a38494c4c4358e377032158c762e3d8b F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 375e5df716e03b9343c5e1211be3b24e6d6dff05 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d190ddabc386bc9654b99e33fb81b2f6e67b54d6 -R d5a852613692dd4e943f0ee647615880 -U dan -Z db771a6d89978ace4423b73d7feafccd +P c8d67aefff7299dd5c8abeb2a3a52109c0a823ee +R dc31f859ef6cd2199f40f19f84f2b02e +U drh +Z 7f958188506961f9abc2573e8dff7118 diff --git a/manifest.uuid b/manifest.uuid index caccf2eff1..6c200497cb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c8d67aefff7299dd5c8abeb2a3a52109c0a823ee \ No newline at end of file +e81ccdcd87a1387f7ca08d4d7c899311e2180d32 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 0a51ff0301..6a9761e91a 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2593,7 +2593,7 @@ static int btreeSwapOutMmap(BtShared *pBt){ for(pCsr=pBt->pCursor; pCsr && rc==SQLITE_OK; pCsr=pCsr->pNext){ if( pCsr->iPage>=0 ){ MemPage *pPg = pCsr->apPage[0]; - if( pPg->pDbPage->flags & PGHDR_MMAP ){ + if( pPg && pPg->pDbPage->flags & PGHDR_MMAP ){ MemPage *pNew = 0; rc = getAndInitPage(pBt, pPg->pgno, &pNew, 0); if( rc==SQLITE_OK && pCsr->iPage==0 ){ From 7909e54a22a0b466e5990d63a1c01de567deeb8f Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 22 Mar 2013 20:15:31 +0000 Subject: [PATCH 023/204] Update wal mode tests so that they work with the mmap test permutation. FossilOrigin-Name: f7295872a36539d10edaf0c633f935f25cf73657 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- test/permutations.test | 3 +-- test/wal.test | 8 +++++--- test/wal5.test | 10 +++++++++- 5 files changed, 25 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 0a2be06535..868c9e0173 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\spotential\sNULL-pointer\sdereference\sin\sbtreeSwapOutMmap(). -D 2013-03-22T19:56:27.789 +C Update\swal\smode\stests\sso\sthat\sthey\swork\swith\sthe\smmap\stest\spermutation. +D 2013-03-22T20:15:31.365 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -657,7 +657,7 @@ F test/pageropt.test 290cd59782b1890f02bb33795571facfc5ccac43 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test 4381ffc2811884cb4c2063176e4e4a849f236adc +F test/permutations.test cf5d475f024cfd3fbb0255b8e43bf0c6887c6bb4 F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -950,11 +950,11 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 -F test/wal.test a040047d7f2b9f34bc4d597964e5e7c09609c635 +F test/wal.test 5baa9a0e682c0394012aa9a06a3d06cf92655fbf F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c -F test/wal5.test f58ed4b8b542f71c7441da12fbd769d99b362437 +F test/wal5.test 091bf7500e97087f1a87b346c8707e93f42d4f80 F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test b3ee739fe8f7586aaebdc2367f477ebcf3e3b034 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c8d67aefff7299dd5c8abeb2a3a52109c0a823ee -R dc31f859ef6cd2199f40f19f84f2b02e -U drh -Z 7f958188506961f9abc2573e8dff7118 +P e81ccdcd87a1387f7ca08d4d7c899311e2180d32 +R f5bd57e0d254253276d114325fd2ea6d +U dan +Z 0c2e4f926a5fed4f3d574bc2f3c43e20 diff --git a/manifest.uuid b/manifest.uuid index 6c200497cb..a8b241dcad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e81ccdcd87a1387f7ca08d4d7c899311e2180d32 \ No newline at end of file +f7295872a36539d10edaf0c633f935f25cf73657 \ No newline at end of file diff --git a/test/permutations.test b/test/permutations.test index 1015066e84..6e6b5c90f7 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -143,8 +143,7 @@ test_suite "mmap" -prefix "mm-" -description { } -presql { pragma mmap_size = -65536; } -files [ - test_set $allquicktests -exclude *malloc* *ioerr* *fault* \ - multiplex* server1.test shared2.test shared6.test + test_set $allquicktests -exclude *malloc* *ioerr* *fault* ] test_suite "valgrind" -prefix "" -description { diff --git a/test/wal.test b/test/wal.test index 24ce5f86a3..ca7edef952 100644 --- a/test/wal.test +++ b/test/wal.test @@ -727,6 +727,8 @@ do_test wal-11.9 { list [expr [file size test.db]/1024] [log_deleted test.db-wal] } {37 1} sqlite3_wal db test.db +set nWal 37 +if {[permutation]=="mmap"} {set nWal 39} do_test wal-11.10 { execsql { PRAGMA cache_size = 10; @@ -735,7 +737,7 @@ do_test wal-11.10 { SELECT count(*) FROM t1; } list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [wal_file_size 37 1024]] +} [list 37 [wal_file_size $nWal 1024]] do_test wal-11.11 { execsql { SELECT count(*) FROM t1; @@ -745,7 +747,7 @@ do_test wal-11.11 { } {32 16} do_test wal-11.12 { list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [wal_file_size 37 1024]] +} [list 37 [wal_file_size $nWal 1024]] do_test wal-11.13 { execsql { INSERT INTO t1 VALUES( blob(900) ); @@ -755,7 +757,7 @@ do_test wal-11.13 { } {17 ok} do_test wal-11.14 { list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [wal_file_size 37 1024]] +} [list 37 [wal_file_size $nWal 1024]] #------------------------------------------------------------------------- diff --git a/test/wal5.test b/test/wal5.test index 6eceed5e59..d8bf541a4b 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -235,7 +235,15 @@ foreach {testprefix do_wal_checkpoint} { do_test 2.3.$tn.5 { sql1 { INSERT INTO t2 VALUES(3, 4) } } {} do_test 2.3.$tn.6 { file_page_counts } {1 4 1 4} do_test 2.3.$tn.7 { code1 { do_wal_checkpoint db -mode full } } {1 4 3} - do_test 2.3.$tn.8 { file_page_counts } {1 4 2 4} + + # The checkpoint above only writes page 1 of the db file. The other + # page (page 2) is locked by the read-transaction opened by the + # [sql2] commmand above. So normally, the db is 1 page in size here. + # However, in mmap() mode, the db is pre-allocated to 2 pages at the + # start of the checkpoint, even though page 2 cannot be written. + set nDb 1 + if {[permutation]=="mmap"} {set nDb 2} + do_test 2.3.$tn.8 { file_page_counts } [list $nDb 4 2 4] } # Check that checkpoints block on the correct locks. And respond correctly From daf9a5a402fdcd69fce93e84bc61b46dcf9803e2 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 23 Mar 2013 09:56:39 +0000 Subject: [PATCH 024/204] Preliminary changes to support the xMremap VFS method on Windows. FossilOrigin-Name: 75a85a1c6ac59f2d021c28fa2161afb1118deea4 --- manifest | 16 ++-- manifest.uuid | 2 +- src/os_win.c | 201 +++++++++++++++++++++++++++++++++++++++++++++++--- src/test1.c | 4 + 4 files changed, 203 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 868c9e0173..b81d7d1140 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\swal\smode\stests\sso\sthat\sthey\swork\swith\sthe\smmap\stest\spermutation. -D 2013-03-22T20:15:31.365 +C Preliminary\schanges\sto\ssupport\sthe\sxMremap\sVFS\smethod\son\sWindows. +D 2013-03-23T09:56:39.403 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf -F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 +F src/os_win.c 46d2bbad7a65d2867ec0e9afb4aa28478402e9dc F src/pager.c 12b8ff12519fe529a92884c4cdb99afecb1bea3c F src/pager.h bbc9170281c9d5d603b2175fdc8ea908e47269a7 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -184,7 +184,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 3213f3101e3b85f047d6e389da5a53d76d3d7540 -F src/test1.c 39378e3e14a8162e29dc90d1e05399d12e8a569e +F src/test1.c 26e66b839f42c2eed6833f9023e0098f0d863f35 F src/test2.c 4178056dd1e7d70f954ad8a1e3edb71a2a784daf F src/test3.c 3c3c2407fa6ec7a19e24ae23f7cb439d0275a60d F src/test4.c bf9fa9bece01de08e6f5e02314e4af5c13590dfa @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e81ccdcd87a1387f7ca08d4d7c899311e2180d32 -R f5bd57e0d254253276d114325fd2ea6d -U dan -Z 0c2e4f926a5fed4f3d574bc2f3c43e20 +P f7295872a36539d10edaf0c633f935f25cf73657 +R 33ae184619389f0a2e2f45e36bcba41c +U mistachkin +Z 7a5b6171912da74b28162981ae14cd6f diff --git a/manifest.uuid b/manifest.uuid index a8b241dcad..ade491dce4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f7295872a36539d10edaf0c633f935f25cf73657 \ No newline at end of file +75a85a1c6ac59f2d021c28fa2161afb1118deea4 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 970a94b5e8..fd0c8aeb89 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -150,11 +150,15 @@ struct winFile { winceLock local; /* Locks obtained by this instance of winFile */ winceLock *shared; /* Global shared lock memory for the file */ #endif + HANDLE hMap; /* Handle for accessing memory mapping */ + void *pMapRegion; /* Area memory mapped */ + sqlite3_int64 mmapSize; /* Size of xMremap() */ }; /* ** Allowed values for winFile.ctrlFlags */ +#define WINFILE_RDONLY 0x02 /* Connection is read only */ #define WINFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ #define WINFILE_PSOW 0x10 /* SQLITE_IOCAP_POWERSAFE_OVERWRITE */ @@ -2061,6 +2065,9 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ #endif } +/* Forward references to VFS methods */ +static int winUnmap(sqlite3_file *); + /* ** Close a file. ** @@ -2082,6 +2089,10 @@ static int winClose(sqlite3_file *id){ #endif OSTRACE(("CLOSE %d\n", pFile->h)); assert( pFile->h!=NULL && pFile->h!=INVALID_HANDLE_VALUE ); + + rc = winUnmap(id); + if( rc!=SQLITE_OK ) return rc; + do{ rc = osCloseHandle(pFile->h); /* SimulateIOError( rc=0; cnt=MX_CLOSE_ATTEMPT; ); */ @@ -2130,6 +2141,8 @@ static int winRead( int nRetry = 0; /* Number of retrys */ assert( id!=0 ); + assert( amt>0 ); + assert( offset>=pFile->mmapSize ); /* Never read from the mmapped region */ SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); @@ -2176,6 +2189,7 @@ static int winWrite( int nRetry = 0; /* Number of retries */ assert( amt>0 ); + assert( offset>=pFile->mmapSize ); /* Never write into the mmapped region */ assert( pFile ); SimulateIOError(return SQLITE_IOERR_WRITE); SimulateDiskfullError(return SQLITE_FULL); @@ -2249,12 +2263,16 @@ static int winWrite( static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ winFile *pFile = (winFile*)id; /* File handle object */ int rc = SQLITE_OK; /* Return code for this function */ + DWORD lastErrno; assert( pFile ); OSTRACE(("TRUNCATE %d %lld\n", pFile->h, nByte)); SimulateIOError(return SQLITE_IOERR_TRUNCATE); + rc = winUnmap(id); + if( rc!=SQLITE_OK ) return rc; + /* If the user has configured a chunk-size for this file, truncate the ** file so that it consists of an integer number of chunks (i.e. the ** actual file size after the operation may be larger than the requested @@ -2267,11 +2285,20 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ if( seekWinFile(pFile, nByte) ){ rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, - "winTruncate1", pFile->zPath); - }else if( 0==osSetEndOfFile(pFile->h) ){ - pFile->lastErrno = osGetLastError(); + "winTruncate1", pFile->zPath); + }else if( 0==osSetEndOfFile(pFile->h) && + ((lastErrno = osGetLastError())!=ERROR_USER_MAPPED_FILE) ){ + pFile->lastErrno = lastErrno; rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, - "winTruncate2", pFile->zPath); + "winTruncate2", pFile->zPath); + }else{ + /* If the file was just truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( pFile->pMapRegion && nBytemmapSize ){ + pFile->mmapSize = nByte; + } } OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); @@ -3451,6 +3478,154 @@ shmpage_out: # define winShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ +/* +** Arguments x and y are both integers. Argument y must be a power of 2. +** Round x up to the nearest integer multiple of y. For example: +** +** ROUNDUP(0, 8) -> 0 +** ROUNDUP(13, 8) -> 16 +** ROUNDUP(32, 8) -> 32 +*/ +#define ROUNDUP(x,y) (((x)+y-1)&~(y-1)) + +/* +** Cleans up the mapped region of the specified file, if any. +*/ +static int winUnmap( + sqlite3_file *id +){ + winFile *pFile = (winFile*)id; + assert( pFile!=0 ); + if( pFile->pMapRegion ){ + if( !osUnmapViewOfFile(pFile->pMapRegion) ){ + pFile->lastErrno = osGetLastError(); + return winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, + "winUnmap1", pFile->zPath); + } + pFile->pMapRegion = 0; + pFile->mmapSize = 0; + } + if( pFile->hMap!=NULL ){ + if( !osCloseHandle(pFile->hMap) ){ + pFile->lastErrno = osGetLastError(); + return winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, + "winUnmap2", pFile->zPath); + } + pFile->hMap = NULL; + } + return SQLITE_OK; +} + +/* +** Map, remap or unmap part of the database file. +*/ +static int winMremap( + sqlite3_file *id, /* Main database file */ + int flags, /* Mask of SQLITE_MREMAP_XXX flags */ + sqlite3_int64 iOff, /* Offset to start mapping at */ + sqlite3_int64 nOld, /* Size of old mapping, or zero */ + sqlite3_int64 nNew, /* Size of new mapping, or zero */ + void **ppMap /* IN/OUT: Old/new mappings */ +){ + winFile *pFile = (winFile*)id; /* The underlying database file */ + int rc = SQLITE_OK; /* Return code */ + HANDLE hMap = NULL; /* New mapping handle */ + void *pNew = 0; /* New mapping */ + i64 nNewRnd; /* nNew rounded up */ + i64 nOldRnd; /* nOld rounded up */ + + assert( pFile!=0 ); + assert( iOff==0 ); + assert( nOld>=0 ); + assert( nOld==0 || pFile->pMapRegion==(*ppMap) ); + assert( nNew>=0 ); + assert( ppMap ); + assert( pFile->hMap==NULL || pFile->pMapRegion==(*ppMap) ); + assert( pFile->pMapRegion==0 || pFile->pMapRegion==(*ppMap) ); + /* assert( pFile->mmapSize==nOld ); */ + + assert( winSysInfo.dwPageSize>0 ); + nNewRnd = ROUNDUP(nNew, winSysInfo.dwPageSize*1); + assert( nNewRnd>=0 ); + nOldRnd = ROUNDUP(nOld, winSysInfo.dwPageSize*1); + assert( nOldRnd>=0 ); + + if( nNewRnd==nOldRnd ){ + pFile->mmapSize = nNew; + return SQLITE_OK; + } + + /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested + ** mapping (nNew bytes) may be greater than the size of the database file. + ** If this is the case, extend the file on disk using ftruncate(). */ + assert( nNewRnd>0 || (flags & SQLITE_MREMAP_EXTEND)==0 ); + if( flags & SQLITE_MREMAP_EXTEND ){ + sqlite3_int64 oldSz; + rc = winFileSize(id, &oldSz); + if( rc==SQLITE_OK && nNewRnd>oldSz ){ + rc = winTruncate(id, nNewRnd); + } + if( rc!=SQLITE_OK ) return rc; + } + + /* If we get this far, unmap any old mapping. */ + rc = winUnmap(id); + if( rc!=SQLITE_OK ) return rc; + + /* And, if required, create a new mapping. */ + if( nNewRnd>0 ){ + i64 offset = ((iOff / winSysInfo.dwAllocationGranularity) * + (winSysInfo.dwAllocationGranularity)); + DWORD protect = PAGE_READONLY; + DWORD flags = FILE_MAP_READ; + if( (pFile->ctrlFlags & WINFILE_RDONLY)==0 ){ + protect = PAGE_READWRITE; + flags |= FILE_MAP_WRITE; + } +#if SQLITE_OS_WINRT + hMap = osCreateFileMappingFromApp(pFile->h, NULL, protect, nNewRnd, NULL); +#elif defined(SQLITE_WIN32_HAS_WIDE) + hMap = osCreateFileMappingW(pFile->h, NULL, protect, + (DWORD)((nNewRnd>>32) & 0xffffffff), + (DWORD)(nNewRnd & 0xffffffff), NULL); +#elif defined(SQLITE_WIN32_HAS_ANSI) + hMap = osCreateFileMappingA(pFile->h, NULL, protect, + (DWORD)((nNewRnd>>32) & 0xffffffff), + (DWORD)(nNewRnd & 0xffffffff), NULL); +#endif + if( hMap==NULL ){ + pFile->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, + "winMremap1", pFile->zPath); + return rc; + } + assert( (nNewRnd % winSysInfo.dwPageSize)==0 ); +#if SQLITE_OS_WINRT + pNew = osMapViewOfFileFromApp(hMap, flags, offset, nNewRnd); +#else + assert( sizeof(SIZE_T)==sizeof(sqlite3_int64) || nNewRnd<=0xffffffff ); + pNew = osMapViewOfFile(hMap, flags, + (DWORD)((offset>>32) & 0xffffffff), + (DWORD)(offset & 0xffffffff), + (SIZE_T)nNewRnd); +#endif + if( pNew==NULL ){ + osCloseHandle(hMap); + hMap = NULL; + pFile->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, + "winMremap2", pFile->zPath); + } + } + + pFile->hMap = hMap; + pFile->pMapRegion = pNew; + pFile->mmapSize = nNew; + + *ppMap = pNew; + return rc; +} + /* ** Here ends the implementation of all sqlite3_file methods. ** @@ -3462,7 +3637,7 @@ shmpage_out: ** sqlite3_file for win32. */ static const sqlite3_io_methods winIoMethod = { - 2, /* iVersion */ + 3, /* iVersion */ winClose, /* xClose */ winRead, /* xRead */ winWrite, /* xWrite */ @@ -3478,7 +3653,8 @@ static const sqlite3_io_methods winIoMethod = { winShmMap, /* xShmMap */ winShmLock, /* xShmLock */ winShmBarrier, /* xShmBarrier */ - winShmUnmap /* xShmUnmap */ + winShmUnmap, /* xShmUnmap */ + winMremap, /* xMremap */ }; /**************************************************************************** @@ -3654,9 +3830,7 @@ static int winOpen( int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); int isCreate = (flags & SQLITE_OPEN_CREATE); -#ifndef NDEBUG int isReadonly = (flags & SQLITE_OPEN_READONLY); -#endif int isReadWrite = (flags & SQLITE_OPEN_READWRITE); #ifndef NDEBUG @@ -3867,11 +4041,17 @@ static int winOpen( pFile->pMethod = &winIoMethod; pFile->pVfs = pVfs; pFile->h = h; + if( isReadonly ){ + pFile->ctrlFlags |= WINFILE_RDONLY; + } if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pFile->ctrlFlags |= WINFILE_PSOW; } pFile->lastErrno = NO_ERROR; pFile->zPath = zName; + pFile->hMap = NULL; + pFile->pMapRegion = 0; + pFile->mmapSize = 0; OpenCounter(+1); return rc; @@ -4500,7 +4680,6 @@ int sqlite3_os_init(void){ ** correctly. See ticket [bb3a86e890c8e96ab] */ assert( ArraySize(aSyscall)==74 ); -#ifndef SQLITE_OMIT_WAL /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); #if SQLITE_OS_WINRT @@ -4508,8 +4687,8 @@ int sqlite3_os_init(void){ #else osGetSystemInfo(&winSysInfo); #endif - assert(winSysInfo.dwAllocationGranularity > 0); -#endif + assert( winSysInfo.dwAllocationGranularity>0 ); + assert( winSysInfo.dwPageSize>0 ); sqlite3_vfs_register(&winVfs, 1); return SQLITE_OK; diff --git a/src/test1.c b/src/test1.c index 500d60e0fe..fb340f92d0 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5844,6 +5844,7 @@ static int test_test_control( return TCL_OK; } +#if SQLITE_OS_UNIX #include #include @@ -5866,6 +5867,7 @@ static int test_getrusage( Tcl_SetObjResult(interp, Tcl_NewStringObj(buf, -1)); return TCL_OK; } +#endif #if SQLITE_OS_WIN /* @@ -6256,7 +6258,9 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "print_explain_query_plan", test_print_eqp, 0 }, #endif { "sqlite3_test_control", test_test_control }, +#if SQLITE_OS_UNIX { "getrusage", test_getrusage }, +#endif }; static int bitmask_size = sizeof(Bitmask)*8; int i; From 46fce002d9ed09dc1db71678e73eb1336de96f5b Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 23 Mar 2013 10:09:46 +0000 Subject: [PATCH 025/204] Do not unmap the mapped file region in winTruncate as there may be outstanding pointers to it. FossilOrigin-Name: f57a9c91e993f76ce8b923e06e721414980e8e61 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 3 --- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index b81d7d1140..a39a1c5070 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Preliminary\schanges\sto\ssupport\sthe\sxMremap\sVFS\smethod\son\sWindows. -D 2013-03-23T09:56:39.403 +C Do\snot\sunmap\sthe\smapped\sfile\sregion\sin\swinTruncate\sas\sthere\smay\sbe\soutstanding\spointers\sto\sit. +D 2013-03-23T10:09:46.701 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf -F src/os_win.c 46d2bbad7a65d2867ec0e9afb4aa28478402e9dc +F src/os_win.c 8f7de0046d5107be715aa91d7deebbf91d577644 F src/pager.c 12b8ff12519fe529a92884c4cdb99afecb1bea3c F src/pager.h bbc9170281c9d5d603b2175fdc8ea908e47269a7 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f7295872a36539d10edaf0c633f935f25cf73657 -R 33ae184619389f0a2e2f45e36bcba41c +P 75a85a1c6ac59f2d021c28fa2161afb1118deea4 +R 2ac6e41fa14b2fcb69f496caa13f7e10 U mistachkin -Z 7a5b6171912da74b28162981ae14cd6f +Z 3096704ce1da72ac81b0db8561977d4e diff --git a/manifest.uuid b/manifest.uuid index ade491dce4..18b01de7dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -75a85a1c6ac59f2d021c28fa2161afb1118deea4 \ No newline at end of file +f57a9c91e993f76ce8b923e06e721414980e8e61 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index fd0c8aeb89..8d58ab0d44 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2270,9 +2270,6 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ OSTRACE(("TRUNCATE %d %lld\n", pFile->h, nByte)); SimulateIOError(return SQLITE_IOERR_TRUNCATE); - rc = winUnmap(id); - if( rc!=SQLITE_OK ) return rc; - /* If the user has configured a chunk-size for this file, truncate the ** file so that it consists of an integer number of chunks (i.e. the ** actual file size after the operation may be larger than the requested From 2753388e8ae49d71e7b3edc71793950aa002c07d Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 23 Mar 2013 12:15:27 +0000 Subject: [PATCH 026/204] In the winMremap VFS function, unmap the region prior to attempting to truncate the file. FossilOrigin-Name: 8870c4cc6cda07154936644f7df25bd81d6ba38a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a39a1c5070..ada3b7db89 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sunmap\sthe\smapped\sfile\sregion\sin\swinTruncate\sas\sthere\smay\sbe\soutstanding\spointers\sto\sit. -D 2013-03-23T10:09:46.701 +C In\sthe\swinMremap\sVFS\sfunction,\sunmap\sthe\sregion\sprior\sto\sattempting\sto\struncate\sthe\sfile. +D 2013-03-23T12:15:27.020 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf -F src/os_win.c 8f7de0046d5107be715aa91d7deebbf91d577644 +F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac F src/pager.c 12b8ff12519fe529a92884c4cdb99afecb1bea3c F src/pager.h bbc9170281c9d5d603b2175fdc8ea908e47269a7 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 75a85a1c6ac59f2d021c28fa2161afb1118deea4 -R 2ac6e41fa14b2fcb69f496caa13f7e10 +P f57a9c91e993f76ce8b923e06e721414980e8e61 +R a68c203c916fb468751938d0762a7812 U mistachkin -Z 3096704ce1da72ac81b0db8561977d4e +Z c2b92e0c0a8c400cb287f5f47ed82e78 diff --git a/manifest.uuid b/manifest.uuid index 18b01de7dd..0e0a52b9c3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f57a9c91e993f76ce8b923e06e721414980e8e61 \ No newline at end of file +8870c4cc6cda07154936644f7df25bd81d6ba38a \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 8d58ab0d44..8242a33849 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3560,7 +3560,10 @@ static int winMremap( sqlite3_int64 oldSz; rc = winFileSize(id, &oldSz); if( rc==SQLITE_OK && nNewRnd>oldSz ){ - rc = winTruncate(id, nNewRnd); + rc = winUnmap(id); /* Cannot truncate the file with an open mapping. */ + if( rc==SQLITE_OK ){ + rc = winTruncate(id, nNewRnd); + } } if( rc!=SQLITE_OK ) return rc; } From 9d56c6df9aef1e9cb59246b010c55407ea1ab73b Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 23 Mar 2013 14:20:42 +0000 Subject: [PATCH 027/204] Fix a case where a checkpoint operation could write to an invalid part of a memory mapped region. FossilOrigin-Name: 8dbe89d05ce91428c69003f0da79d883fa23e2b5 --- manifest | 18 +++++++-------- manifest.uuid | 2 +- src/pager.c | 16 ++++++------- test/mmap1.test | 57 +++++++++++++++++++++++++++++++++++++++++++---- test/speed1p.test | 3 --- 5 files changed, 71 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index ada3b7db89..cf3732c267 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\swinMremap\sVFS\sfunction,\sunmap\sthe\sregion\sprior\sto\sattempting\sto\struncate\sthe\sfile. -D 2013-03-23T12:15:27.020 +C Fix\sa\scase\swhere\sa\scheckpoint\soperation\scould\swrite\sto\san\sinvalid\spart\sof\sa\smemory\smapped\sregion. +D 2013-03-23T14:20:42.975 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac -F src/pager.c 12b8ff12519fe529a92884c4cdb99afecb1bea3c +F src/pager.c cffcfe753445ef3a3d3830efd5f807fc171e7262 F src/pager.h bbc9170281c9d5d603b2175fdc8ea908e47269a7 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -630,7 +630,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 46ff4038b6d300bb9b39d3f89ea1aef3dff9b88f +F test/mmap1.test e3391ef6c069e5f5733b8339652ce37fd97e3487 F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 @@ -730,7 +730,7 @@ F test/softheap1.test c16709a16ad79fa43b32929b2e623d1d117ccf53 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb -F test/speed1p.test 2577211a9b054ffc973780f5b8708f02a0c7422d +F test/speed1p.test c4a469f29f135f4d76c55b1f2a52f36e209466cc F test/speed2.test 53177056baf6556dcbdcf032bbdfc41c1aa74ded F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f57a9c91e993f76ce8b923e06e721414980e8e61 -R a68c203c916fb468751938d0762a7812 -U mistachkin -Z c2b92e0c0a8c400cb287f5f47ed82e78 +P 8870c4cc6cda07154936644f7df25bd81d6ba38a +R 0b90a49ac5025c5ac7d6b4eb49fe1878 +U dan +Z c9ca2a38d0bbfc316a52f1d1e135baf9 diff --git a/manifest.uuid b/manifest.uuid index 0e0a52b9c3..bd17b56cca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8870c4cc6cda07154936644f7df25bd81d6ba38a \ No newline at end of file +8dbe89d05ce91428c69003f0da79d883fa23e2b5 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 660514d508..09cc4a2375 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4232,7 +4232,7 @@ static int syncJournal(Pager *pPager, int newHdr){ */ int sqlite3PagerSetFilesize(Pager *pPager, i64 szReq){ int rc; - sqlite3_int64 sz; + i64 sz; /* Size of file on disk in bytes */ assert( pPager->eState==PAGER_OPEN ); assert( pPager->nMmapOut==0 ); @@ -4244,13 +4244,13 @@ int sqlite3PagerSetFilesize(Pager *pPager, i64 szReq){ } } - if( rc==SQLITE_OK - && pPager->nMapLimit>0 - && pPager->nMapValidnMapValidnMapLimit - ){ - pPager->dbFileSize = (szReq / pPager->pageSize); - rc = pagerMap(pPager, 1); + + if( rc==SQLITE_OK ){ + i64 szMap = (szReq > pPager->nMapLimit) ? pPager->nMapLimit : szReq; + if( pPager->nMapValid!=pPager->nMap || szMap!=pPager->nMap ){ + pPager->dbFileSize = (szReq / pPager->pageSize); + rc = pagerMap(pPager, 1); + } } return rc; diff --git a/test/mmap1.test b/test/mmap1.test index 343ad230b1..41714e29d8 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -23,13 +23,17 @@ proc nRead {db} { return $stats(read) } -foreach {t mmap_size nRead} { - 1 { PRAGMA mmap_size = -65536 } 4 - 2 { PRAGMA mmap_size = -50 } 156 - 3 { PRAGMA mmap_size = 0 } 344 +foreach {t mmap_size nRead c2init} { + 1.1 { PRAGMA mmap_size = -65536 } 4 {} + 1.2 { PRAGMA mmap_size = -50 } 156 {} + 1.3 { PRAGMA mmap_size = 0 } 344 {} + 1.4 { PRAGMA mmap_size = -65536 } 4 {PRAGMA mmap_size = -65536} + 1.5 { PRAGMA mmap_size = -50 } 156 {PRAGMA mmap_size = -65536} + 1.6 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = -65536} } { do_multiclient_test tn { sql1 $mmap_size + sql2 $c2init code2 { set ::rcnt 0 @@ -79,6 +83,51 @@ foreach {t mmap_size nRead} { } } +set ::rcnt 0 +proc rblob {n} { + set ::rcnt [expr (($::rcnt << 3) + $::rcnt + 456) & 0xFFFFFFFF] + set str [format %.8x [expr $::rcnt ^ 0xbdf20da3]] + string range [string repeat $str [expr $n/4]] 1 $n +} + +reset_db +db func rblob rblob + +do_execsql_test 2.1 { + PRAGMA auto_vacuum = 1; + PRAGMA mmap_size = -65536; + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b, UNIQUE(a, b)); + INSERT INTO t1 VALUES(rblob(500), rblob(500)); + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 2 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 4 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 8 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 32 + PRAGMA wal_checkpoint; +} {wal 0 103 103} + +do_execsql_test 2.2 { + PRAGMA auto_vacuum; + SELECT count(*) FROM t1; +} {1 32} + +do_test 2.3 { + sqlite3 db2 test.db + db2 func rblob rblob + db2 eval { + DELETE FROM t1 WHERE (rowid%4); + PRAGMA wal_checkpoint; + } + db2 eval { + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 + SELECT count(*) FROM t1; + } +} {16} + +do_execsql_test 2.4 { + PRAGMA wal_checkpoint; +} {0 24 24} finish_test diff --git a/test/speed1p.test b/test/speed1p.test index 33be553b00..915f165354 100644 --- a/test/speed1p.test +++ b/test/speed1p.test @@ -65,9 +65,6 @@ proc number_name {n} { # do_test speed1p-1.0 { execsql { - PRAGMA mmap_size=65536; - PRAGMA journal_mode=wal; - PRAGMA page_size=1024; PRAGMA cache_size=500; PRAGMA locking_mode=EXCLUSIVE; From e5586a1a79f3d8e61ee6d263ec680c012ffbabfb Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 23 Mar 2013 17:29:06 +0000 Subject: [PATCH 028/204] Improve a comment in wal.c. No code changes. FossilOrigin-Name: 60b9f5e4dd2af54975ba78437239f0bebd472fd2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 13 +++++++++---- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index cf3732c267..6660b0b860 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scase\swhere\sa\scheckpoint\soperation\scould\swrite\sto\san\sinvalid\spart\sof\sa\smemory\smapped\sregion. -D 2013-03-23T14:20:42.975 +C Improve\sa\scomment\sin\swal.c.\sNo\scode\schanges. +D 2013-03-23T17:29:06.036 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c c61ca318681c0e7267da8be3abfca8469652a7e9 F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 -F src/wal.c 7fec703bb0770d9faad5190f49897a8bc8eb239f +F src/wal.c 923d90992fc3b069571cf3fbd635c54c3e839e55 F src/wal.h d99ce512ac60f9147a0640e9e6fb67dd1057b781 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b F src/where.c bdbbfa7ef4ea04c8d9b09585b45d4717a72f980a @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 8870c4cc6cda07154936644f7df25bd81d6ba38a -R 0b90a49ac5025c5ac7d6b4eb49fe1878 +P 8dbe89d05ce91428c69003f0da79d883fa23e2b5 +R b063a8588944a5f842812de15381707f U dan -Z c9ca2a38d0bbfc316a52f1d1e135baf9 +Z 350cbe536c1fd5aa73b476fafae6db5c diff --git a/manifest.uuid b/manifest.uuid index bd17b56cca..7aea0f37ae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8dbe89d05ce91428c69003f0da79d883fa23e2b5 \ No newline at end of file +60b9f5e4dd2af54975ba78437239f0bebd472fd2 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 8757003d78..051b77f515 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1725,12 +1725,17 @@ static int walCheckpoint( rc = sqlite3OsSync(pWal->pWalFd, sync_flags); } - /* If the database file may grow as a result of this checkpoint, hint - ** about the eventual size of the db file to the VFS layer. - */ + /* If the database file is currently smaller than mxPage pages in size, + ** the call below issues an SQLITE_FCNTL_SIZE_HINT to the OS layer to + ** inform it that it is likely to grow to that size. + ** + ** Additionally, if the pager is using mmap(), then the call to + ** SetFilesize() guarantees that the mapping is not larger than mxPage + ** pages. This makes the sqlite3OsTruncate() call below safe - no pages + ** that are part of the mapped region will be truncated away. */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); - rc = sqlite3PagerSetFilesize(pWal->pPager, nReq); + rc = sqlite3PagerSetFilesize(pWal->pPager, nReq); } /* Iterate through the contents of the WAL, copying data to the db file. */ From f23da96636f9de1a81c939bd7a0cc25ee1d37f49 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 23 Mar 2013 21:00:41 +0000 Subject: [PATCH 029/204] Replace the sqlite3_io_methods.xMremap interface with sqlite3_io_methods.xFetch and xUnfetch. FossilOrigin-Name: 1431be95579160fb70408d43e17fc23c7b69ab4a --- manifest | 28 +++--- manifest.uuid | 2 +- src/backup.c | 2 +- src/os.c | 15 ++-- src/os.h | 3 +- src/os_unix.c | 211 +++++++++++++++++++++++++++------------------ src/pager.c | 221 ++++++++++++------------------------------------ src/pager.h | 3 - src/sqlite.h.in | 30 ++----- src/wal.c | 22 ++--- src/wal.h | 3 +- 11 files changed, 222 insertions(+), 318 deletions(-) diff --git a/manifest b/manifest index 6660b0b860..8d71dc88ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sa\scomment\sin\swal.c.\sNo\scode\schanges. -D 2013-03-23T17:29:06.036 +C Replace\sthe\ssqlite3_io_methods.xMremap\sinterface\swith\ssqlite3_io_methods.xFetch\sand\sxUnfetch. +D 2013-03-23T21:00:41.457 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -118,7 +118,7 @@ F src/alter.c f8db986c03eb0bfb221523fc9bbb9d0b70de3168 F src/analyze.c d5f895810e8ff9737c9ec7b76abc3dcff5860335 F src/attach.c ea5247f240e2c08afd608e9beb380814b86655e1 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 -F src/backup.c 6256400ab8be4a15a2512277d8b214dbd44ff8ce +F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c f5ea8d3d658887b5cae0369ef10a427d7469a768 @@ -157,13 +157,13 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 -F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b +F src/os.c a7ec2ddc43fb2684bfd8cb30db15ea6cce3298e9 +F src/os.h 782980cd0d840b4240a6e29ba9f8ef0ca0d750ca F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf +F src/os_unix.c d8cdf331ad08650fc8397eee79b55548070077b0 F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac -F src/pager.c cffcfe753445ef3a3d3830efd5f807fc171e7262 -F src/pager.h bbc9170281c9d5d603b2175fdc8ea908e47269a7 +F src/pager.c 2e63de7185c2a573d26e5087869a9e48326076f9 +F src/pager.h faf4bed79e80cb6527209dfe5cfb08aa5764b941 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 @@ -176,7 +176,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in d63c7fb5832287af7e8b903c4a4c30c90414876f +F src/sqlite.h.in f41949e1e778043dd9fc23b33b96e7a6e20de4d6 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796 @@ -249,8 +249,8 @@ F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c c61ca318681c0e7267da8be3abfca8469652a7e9 F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 -F src/wal.c 923d90992fc3b069571cf3fbd635c54c3e839e55 -F src/wal.h d99ce512ac60f9147a0640e9e6fb67dd1057b781 +F src/wal.c e84eff498c57ec2d79ca4496a3f4a638af378fb3 +F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b F src/where.c bdbbfa7ef4ea04c8d9b09585b45d4717a72f980a F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 8dbe89d05ce91428c69003f0da79d883fa23e2b5 -R b063a8588944a5f842812de15381707f +P 60b9f5e4dd2af54975ba78437239f0bebd472fd2 +R eee4d88d6400cf65980c5b201532d2c9 U dan -Z 350cbe536c1fd5aa73b476fafae6db5c +Z eabd20cdc7aabbc65591f96e4f6dff04 diff --git a/manifest.uuid b/manifest.uuid index 7aea0f37ae..f378aa2e8b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -60b9f5e4dd2af54975ba78437239f0bebd472fd2 \ No newline at end of file +1431be95579160fb70408d43e17fc23c7b69ab4a \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 8939bc67bc..71a8a1a3e7 100644 --- a/src/backup.c +++ b/src/backup.c @@ -520,7 +520,7 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); if( rc==SQLITE_OK ){ u8 *zData = sqlite3PagerGetData(pSrcPg); - rc = sqlite3PagerWriteData(pDestPager, zData, pgszSrc, iOff); + rc = sqlite3OsWrite(pFile, zData, pgszSrc, iOff); } sqlite3PagerUnref(pSrcPg); } diff --git a/src/os.c b/src/os.c index 38757dcdb1..08e0d42172 100644 --- a/src/os.c +++ b/src/os.c @@ -140,15 +140,12 @@ int sqlite3OsShmMap( DO_OS_MALLOC_TEST(id); return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } -int sqlite3OsMremap( - sqlite3_file *id, /* Database file handle */ - int flags, /* SQLITE_MREMAP_XXX flags */ - i64 iOff, /* Offset at which mapping(s) start */ - i64 nOld, /* Size of old mapping */ - i64 nNew, /* Size of requested mapping */ - void **pp /* IN/OUT: Pointer to mapped region */ -){ - return id->pMethods->xMremap(id, flags, iOff, nOld, nNew, pp); + +int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + return id->pMethods->xFetch(id, iOff, iAmt, pp); +} +int sqlite3OsUnfetch(sqlite3_file *id, void *p){ + return id->pMethods->xUnfetch(id, p); } /* diff --git a/src/os.h b/src/os.h index f08b92dcf0..a7dacfb6ea 100644 --- a/src/os.h +++ b/src/os.h @@ -259,7 +259,8 @@ int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); -int sqlite3OsMremap(sqlite3_file *id, int, i64, i64, i64, void **); +int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); +int sqlite3OsUnfetch(sqlite3_file *, void *); /* diff --git a/src/os_unix.c b/src/os_unix.c index 2f03c39670..334719200a 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -249,9 +249,14 @@ struct unixFile { unsigned char transCntrChng; /* True if the transaction counter changed */ unsigned char dbUpdate; /* True if any part of database file changed */ unsigned char inNormalWrite; /* True if in a normal write operation */ - sqlite3_int64 mmapSize; /* Size of xMremap() */ - void *pMapRegion; /* Area memory mapped */ + #endif + sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ + sqlite3_int64 mmapOrigsize; /* Actual size of mapping at pMapRegion */ + sqlite3_int64 mmapLimit; /* Configured FCNTL_MMAP_SIZE value */ + void *pMapRegion; /* Memory mapped region */ + int nFetchOut; /* Number of outstanding xFetch refs */ + #ifdef SQLITE_TEST /* In test mode, increase the size of this structure a bit so that ** it is larger than the struct CrashFile defined in test6.c. @@ -1805,6 +1810,9 @@ static int unixUnlock(sqlite3_file *id, int eFileLock){ return posixUnlock(id, eFileLock, 0); } +static int unixMapfile(unixFile *pFd, i64 nByte); +static void unixUnmapfile(unixFile *pFd); + /* ** This function performs the parts of the "close file" operation ** common to all locking schemes. It closes the directory and file @@ -1817,6 +1825,7 @@ static int unixUnlock(sqlite3_file *id, int eFileLock){ */ static int closeUnixFile(sqlite3_file *id){ unixFile *pFile = (unixFile*)id; + unixUnmapfile(pFile); if( pFile->h>=0 ){ robust_close(pFile, pFile->h, __LINE__); pFile->h = -1; @@ -3074,7 +3083,6 @@ static int unixRead( unixFile *pFile = (unixFile *)id; int got; assert( id ); - assert( offset>=pFile->mmapSize ); /* Never read from the mmapped region */ /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ @@ -3085,6 +3093,21 @@ static int unixRead( ); #endif + /* Deal with as much of this write request as possible by transfering + ** data to the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } + got = seekAndRead(pFile, offset, pBuf, amt); if( got==amt ){ return SQLITE_OK; @@ -3157,7 +3180,6 @@ static int unixWrite( int wrote = 0; assert( id ); assert( amt>0 ); - assert( offset>=pFile->mmapSize ); /* Never write into the mmapped region */ /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ @@ -3190,6 +3212,21 @@ static int unixWrite( } #endif + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } + while( amt>0 && (wrote = seekAndWrite(pFile, offset, pBuf, amt))>0 ){ amt -= wrote; offset += wrote; @@ -3470,6 +3507,7 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ if( pFile->inNormalWrite && nByte==0 ){ pFile->transCntrChng = 1; } +#endif /* If the file was just truncated to a size smaller than the currently ** mapped region, reduce the effective mapping size as well. SQLite will @@ -3478,7 +3516,6 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ if( nBytemmapSize ){ pFile->mmapSize = nByte; } -#endif return SQLITE_OK; } @@ -3568,6 +3605,19 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ } } + if( pFile->mmapLimit>0 ){ + int rc; + if( pFile->szChunk<=0 ){ + if( robust_ftruncate(pFile->h, nByte) ){ + pFile->lastErrno = errno; + return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); + } + } + + rc = unixMapfile(pFile, nByte); + return rc; + } + return SQLITE_OK; } @@ -3635,8 +3685,8 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } - case SQLITE_FCNTL_GETFD: { - *(int*)pArg = pFile->h; + case SQLITE_FCNTL_MMAP_SIZE: { + pFile->mmapLimit = *(i64*)pArg; return SQLITE_OK; } #ifdef SQLITE_DEBUG @@ -4451,91 +4501,86 @@ static int unixShmUnmap( */ #define ROUNDUP(x,y) (((x)+y-1)&~(y-1)) -/* -** Map, remap or unmap part of the database file. -*/ -static int unixMremap( - sqlite3_file *fd, /* Main database file */ - int flags, /* Mask of SQLITE_MREMAP_XXX flags */ - sqlite3_int64 iOff, /* Offset to start mapping at */ - sqlite3_int64 nOld, /* Size of old mapping, or zero */ - sqlite3_int64 nNew, /* Size of new mapping, or zero */ - void **ppMap /* IN/OUT: Old/new mappings */ -){ - unixFile *p = (unixFile *)fd; /* The underlying database file */ - int rc = SQLITE_OK; /* Return code */ - void *pNew = 0; /* New mapping */ - i64 nNewRnd; /* nNew rounded up */ - i64 nOldRnd; /* nOld rounded up */ +static void unixUnmapfile(unixFile *pFd){ + assert( pFd->nFetchOut==0 ); + if( pFd->pMapRegion ){ + munmap(pFd->pMapRegion, pFd->mmapOrigsize); + pFd->pMapRegion = 0; + pFd->mmapSize = 0; + pFd->mmapOrigsize = 0; + } +} - assert( iOff==0 ); - /* assert( p->mmapSize==nOld ); */ - assert( p->pMapRegion==0 || p->pMapRegion==(*ppMap) ); +static int unixMapfile(unixFile *pFd, i64 nByte){ + i64 nMap = nByte; + int rc; - /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested - ** mapping (nNew bytes) may be greater than the size of the database file. - ** If this is the case, extend the file on disk using ftruncate(). */ - assert( nNew>0 || (flags & SQLITE_MREMAP_EXTEND)==0 ); - if( flags & SQLITE_MREMAP_EXTEND ){ + assert( nMap>=0 || pFd->nFetchOut==0 ); + if( pFd->nFetchOut>0 ) return SQLITE_OK; + + if( nMap<0 ){ struct stat statbuf; /* Low-level file information */ - rc = osFstat(p->h, &statbuf); - if( rc==SQLITE_OK && nNew>statbuf.st_size ){ - rc = robust_ftruncate(p->h, nNew); + rc = osFstat(pFd->h, &statbuf); + if( rc!=SQLITE_OK ){ + return SQLITE_IOERR_FSTAT; } - if( rc!=SQLITE_OK ) return rc; + nMap = statbuf.st_size; + } + if( nMap>pFd->mmapLimit ){ + nMap = pFd->mmapLimit; } - /* According to some sources, the effect of changing the size of the - ** underlying file on mapped regions that correspond to the added or - ** removed pages is undefined. However, there is reason to believe that - ** on modern platforms like Linux or OSX, things just work. For example, - ** it is possible to create a mapping larger than the file on disk and - ** extend the file on disk later on. - ** - ** Exploit this on Linux and OSX to reduce the number of munmap()/mmap() - ** calls required if the file size is changing. In this case all mappings - ** are rounded up to the nearest 4MB. And if a new mapping is requested - ** that has the same rounded size as an old mapping, the old mapping can - ** be reused as is. */ -#if defined(__APPLE__) || defined(__linux__) - nNewRnd = ROUNDUP(nNew, 4096*1024); - nOldRnd = ROUNDUP(nOld, 4096*1024); -#else - nNewRnd = ROUNDUP(nNew, 4096*1); - nOldRnd = ROUNDUP(nOld, 4096*1); -#endif + if( nMap!=pFd->mmapSize ){ + void *pNew; + unixUnmapfile(pFd); - /* On OSX or Linux, reuse the old mapping if it is the right size. */ -#if defined(__APPLE__) || defined(__linux__) - if( nNewRnd==nOldRnd ){ - VVA_ONLY( p->mmapSize = nNew; ) - return SQLITE_OK; - } -#endif + if( nMap>0 ){ + void *pNew; + int flags = PROT_READ; + if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; + pNew = mmap(0, ROUNDUP(nMap, 4096), flags, MAP_SHARED, pFd->h, 0); + if( pNew==MAP_FAILED ){ + return SQLITE_IOERR_MREMAP; + } - /* If we get this far, unmap any old mapping. */ - if( nOldRnd!=0 ){ - void *pOld = *ppMap; - munmap(pOld, nOldRnd); - VVA_ONLY( p->mmapSize = 0; p->pMapRegion = 0; ); - } - - /* And, if required, use mmap() to create a new mapping. */ - if( nNewRnd>0 ){ - int flags = PROT_READ; - if( (p->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - pNew = mmap(0, nNewRnd, flags, MAP_SHARED, p->h, iOff); - if( pNew==MAP_FAILED ){ - pNew = 0; - VVA_ONLY( p->mmapSize = 0; p->pMapRegion = 0; ) - rc = SQLITE_IOERR_MREMAP; - }else{ - VVA_ONLY( p->mmapSize = nNew; p->pMapRegion = pNew; ) + pFd->pMapRegion = pNew; + pFd->mmapOrigsize = pFd->mmapSize = nMap; } } - *ppMap = pNew; - return rc; + return SQLITE_OK; +} + +static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + *pp = 0; + + if( pFd->mmapLimit>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = unixMapfile(pFd, -1); + if( rc!=SQLITE_OK ) return rc; + } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } + return SQLITE_OK; +} + +static int unixUnfetch(sqlite3_file *fd, void *p){ + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + + assert( (p==0)==(pFd->nFetchOut==0) ); + + if( p ){ + pFd->nFetchOut--; + }else{ + unixUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); + return SQLITE_OK; } /* @@ -4597,7 +4642,8 @@ static const sqlite3_io_methods METHOD = { \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ unixShmUnmap, /* xShmUnmap */ \ - unixMremap, /* xMremap */ \ + unixFetch, /* xFetch */ \ + unixUnfetch, /* xUnfetch */ \ }; \ static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ @@ -4865,7 +4911,6 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; - VVA_ONLY( pNew->mmapSize = 0; ) if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; diff --git a/src/pager.c b/src/pager.c index 09cc4a2375..7e00e1431a 100644 --- a/src/pager.c +++ b/src/pager.c @@ -656,14 +656,10 @@ struct Pager { int nSavepoint; /* Number of elements in aSavepoint[] */ char dbFileVers[16]; /* Changes whenever database file changes */ - void *pMap; /* Memory mapped prefix of database file */ - i64 nMap; /* Size of mapping at pMap in bytes */ - i64 nMapValid; /* Bytes at pMap known to be valid */ - i64 nMapLimit; /* Maximum permitted mapping size */ + u8 bUseFetch; /* True to use xFetch() */ int nMapCfgLimit; /* Configured limit value */ int nMmapOut; /* Number of mmap pages currently outstanding */ PgHdr *pFree; /* List of free mmap page headers (pDirty) */ - int bMapResize; /* Check if the mapping should be resized */ /* ** End of the routinely-changing class members ***************************************************************************/ @@ -2087,24 +2083,6 @@ static void pagerReportSize(Pager *pPager){ # define pagerReportSize(X) /* No-op if we do not support a codec */ #endif -/* -** Write nBuf bytes of data from buffer pBuf to offset iOff of the -** database file. If this part of the database file is memory mapped, -** use memcpy() to do so. Otherwise, call sqlite3OsWrite(). -** -** Return SQLITE_OK if successful, or an SQLite error code if an error -** occurs. -*/ -int sqlite3PagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff){ - int rc = SQLITE_OK; - if( pPager->nMapValid>=(iOff+nBuf) ){ - memcpy(&((u8 *)(pPager->pMap))[iOff], pBuf, nBuf); - }else{ - rc = sqlite3OsWrite(pPager->fd, pBuf, nBuf, iOff); - } - return rc; -} - /* ** Read a single page from either the journal file (if isMainJrnl==1) or ** from the sub-journal (if isMainJrnl==0) and playback that page. @@ -2279,7 +2257,7 @@ static int pager_playback_one_page( i64 ofst = (pgno-1)*(i64)pPager->pageSize; testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); assert( !pagerUseWal(pPager) ); - rc = sqlite3PagerWriteData(pPager, aData, pPager->pageSize, ofst); + rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } @@ -2534,9 +2512,6 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ if( rc==SQLITE_OK && currentSize!=newSize ){ if( currentSize>newSize ){ rc = sqlite3OsTruncate(pPager->fd, newSize); - if( newSizenMapValid ){ - pPager->nMapValid = newSize; - } }else if( (currentSize+szPage)<=newSize ){ char *pTmp = pPager->pTmpSpace; memset(pTmp, 0, szPage); @@ -2884,13 +2859,9 @@ static int readDbPage(PgHdr *pPg, u32 iFrame){ rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData); }else{ i64 iOffset = (pgno-1)*(i64)pPager->pageSize; - if( pPager->pMap && pPager->nMapValid>=iOffset+pPager->pageSize ){ - memcpy(pPg->pData, &((u8 *)(pPager->pMap))[iOffset], pPager->pageSize); - }else{ - rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); - if( rc==SQLITE_IOERR_SHORT_READ ){ - rc = SQLITE_OK; - } + rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; } } @@ -3120,6 +3091,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); if( rc!=SQLITE_OK || changed ){ pager_reset(pPager); + if( pPager->bUseFetch ) sqlite3OsUnfetch(pPager->fd, 0); } return rc; @@ -3382,22 +3354,24 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ } /* -** Set Pager.nMapLimit, the maximum permitted mapping size, based on the -** current values of Pager.nMapCfgLimit and Pager.pageSize. -** -** If this connection should not use mmap at all, set nMapLimit to zero. +** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of nMapCfgLimit. */ static void pagerFixMaplimit(Pager *pPager){ - if( isOpen(pPager->fd)==0 - || pPager->fd->pMethods->iVersion<3 - || pPager->fd->pMethods->xMremap==0 - || pPager->tempFile - ){ - pPager->nMapLimit = 0; - }else if( pPager->nMapCfgLimit<0 ){ - pPager->nMapLimit = (i64)pPager->nMapCfgLimit * -1024; - }else{ - pPager->nMapLimit = (i64)pPager->nMapCfgLimit * pPager->pageSize; + sqlite3_file *fd = pPager->fd; + if( isOpen(fd) ){ + pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->nMapCfgLimit!=0; + if( pPager->bUseFetch ){ + void *p; + i64 nMapLimit; + if( pPager->nMapCfgLimit<0 ){ + nMapLimit = (i64)pPager->nMapCfgLimit * -1024; + }else{ + nMapLimit = (i64)pPager->nMapCfgLimit * pPager->pageSize; + } + + p = (void *)&nMapLimit; + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_SIZE, p); + } } } @@ -3870,67 +3844,22 @@ static int pagerSyncHotJournal(Pager *pPager){ return rc; } -/* -** Unmap any memory mapping of the database file. -*/ -static int pagerUnmap(Pager *pPager){ - assert( pPager->nMmapOut==0 ); - if( pPager->pMap ){ - sqlite3OsMremap(pPager->fd, 0, 0, pPager->nMap, 0, &pPager->pMap); - pPager->nMap = 0; - pPager->nMapValid = 0; - } - return SQLITE_OK; -} - -/* -** Create, or recreate, the memory mapping of the database file. -*/ -static int pagerMap(Pager *pPager, int bExtend){ - int rc = SQLITE_OK; /* Return code */ - Pgno nPg; /* Size of mapping to request in pages */ - i64 sz; /* Size of mapping to request in bytes */ - - assert( isOpen(pPager->fd) && pPager->tempFile==0 ); - assert( pPager->pMap==0 || pPager->nMap>0 ); - /* assert( pPager->eState>=1 ); */ - assert( pPager->nMmapOut==0 ); - assert( pPager->nMapLimit>0 ); - - /* Figure out how large a mapping to request. Set variable sz to this - ** value in bytes. */ - nPg = (pPager->eState==1) ? pPager->dbSize : pPager->dbFileSize; - sz = (i64)nPg * pPager->pageSize; - if( sz>pPager->nMapLimit ) sz = pPager->nMapLimit; - - if( sz!=pPager->nMapValid ){ - int flags = (bExtend ? SQLITE_MREMAP_EXTEND : 0); - rc = sqlite3OsMremap(pPager->fd, flags, 0, pPager->nMap, sz, &pPager->pMap); - if( rc==SQLITE_OK ){ - assert( pPager->pMap!=0 ); - pPager->nMap = sz; - }else{ - assert( pPager->pMap==0 ); - pPager->nMap = 0; - } - pPager->nMapValid = pPager->nMap; - } - pPager->bMapResize = 0; - - return rc; -} - /* ** Obtain a reference to a memory mapped page object for page number pgno. -** The caller must ensure that page pgno lies within the currently mapped -** region. If successful, set *ppPage to point to the new page reference +** The new object will use the pointer pData, obtained from xFetch(). +** If successful, set *ppPage to point to the new page reference ** and return SQLITE_OK. Otherwise, return an SQLite error code and set ** *ppPage to zero. ** ** Page references obtained by calling this function should be released ** by calling pagerReleaseMapPage(). */ -static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ +static int pagerAcquireMapPage( + Pager *pPager, /* Pager object */ + Pgno pgno, /* Page number */ + void *pData, /* xFetch()'d data for this page */ + PgHdr **ppPage /* OUT: Acquired page object */ +){ PgHdr *p; /* Memory mapped page to return */ if( pPager->pFree ){ @@ -3955,8 +3884,8 @@ static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ assert( p->pPager==pPager ); assert( p->nRef==1 ); - p->pData = &((u8 *)pPager->pMap)[(i64)(pgno-1) * pPager->pageSize]; p->pgno = pgno; + p->pData = pData; pPager->nMmapOut++; return SQLITE_OK; @@ -3971,6 +3900,9 @@ static void pagerReleaseMapPage(PgHdr *pPg){ pPager->nMmapOut--; pPg->pDirty = pPager->pFree; pPager->pFree = pPg; + + assert( pPager->fd->pMethods->iVersion>=3 ); + sqlite3OsUnfetch(pPager->fd, pPg->pData); } /* @@ -4006,7 +3938,6 @@ int sqlite3PagerClose(Pager *pPager){ assert( assert_pager_state(pPager) ); disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); - pagerUnmap(pPager); pagerFreeMapHdrs(pPager); /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; @@ -4216,46 +4147,6 @@ static int syncJournal(Pager *pPager, int newHdr){ return SQLITE_OK; } -/* -** This is called by the wal.c module at the start of a checkpoint. If the -** checkpoint runs to completion, it will set the database file size to -** szReq bytes. This function performs two tasks: -** -** * If the file is currently less than szReq bytes in size, an -** xFileControl(SQLITE_FNCTL_SIZE_HINT) is issued to inform the OS -** layer of the expected file size, and -** -** * If mmap is being used, then the mapping is extended to szReq -** bytes in size. -** -** SQLITE_OK is returned if successful, or an error code if an error occurs. -*/ -int sqlite3PagerSetFilesize(Pager *pPager, i64 szReq){ - int rc; - i64 sz; /* Size of file on disk in bytes */ - - assert( pPager->eState==PAGER_OPEN ); - assert( pPager->nMmapOut==0 ); - - rc = sqlite3OsFileSize(pPager->fd, &sz); - if( rc==SQLITE_OK ){ - if( sz>szReq ){ - sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &sz); - } - } - - - if( rc==SQLITE_OK ){ - i64 szMap = (szReq > pPager->nMapLimit) ? pPager->nMapLimit : szReq; - if( pPager->nMapValid!=pPager->nMap || szMap!=pPager->nMap ){ - pPager->dbFileSize = (szReq / pPager->pageSize); - rc = pagerMap(pPager, 1); - } - } - - return rc; -} - /* ** The argument is the first in a linked list of dirty pages connected ** by the PgHdr.pDirty pointer. This function writes each one of the @@ -4315,11 +4206,6 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); pPager->dbHintSize = pPager->dbSize; - - if( pPager->nMmapOut==0 && pPager->nMapLimit>0 ){ - pPager->dbFileSize = pPager->dbSize; - rc = pagerMap(pPager, 1); - } } while( rc==SQLITE_OK && pList ){ @@ -4344,7 +4230,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); /* Write out the page data. */ - rc = sqlite3PagerWriteData(pPager, pData, pPager->pageSize, offset); + rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); /* If page 1 was just written, update Pager.dbFileVers to match ** the value now stored in the database file. If writing this @@ -5164,7 +5050,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( !pPager->tempFile && ( pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0 - || pPager->pMap + || pPager->bUseFetch /* TODO: Currently required for xUnfetch(0) only. */ )){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous @@ -5188,13 +5074,9 @@ int sqlite3PagerSharedLock(Pager *pPager){ rc = pagerPagecount(pPager, &nPage); if( rc ) goto failed; - if( nPage>0 || pPager->pMap ){ + if( nPage>0 ){ IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); - if( pPager->pMap ){ - memcpy(&dbFileVers, &((u8 *)(pPager->pMap))[24], sizeof(dbFileVers)); - }else{ - rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); - } + rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); if( rc!=SQLITE_OK ){ goto failed; } @@ -5211,9 +5093,9 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** In this case there may exist a Pager.pMap mapping that appears ** to be the right size but is not actually valid. Avoid this ** possibility by unmapping the db here. */ - pagerUnmap(pPager); - }else if( pPager->pMap ){ - pPager->bMapResize = 1; + if( pPager->bUseFetch ){ + sqlite3OsUnfetch(pPager->fd, 0); + } } } @@ -5325,7 +5207,7 @@ int sqlite3PagerAcquire( ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY ** flag was specified by the caller. And so long as the db is not a ** temporary or in-memory database. */ - const int bMmapOk = (pPager->nMapLimit>0 && pgno!=1 + const int bMmapOk = (pgno!=1 && pPager->bUseFetch && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY)) ); @@ -5349,15 +5231,20 @@ int sqlite3PagerAcquire( } if( iFrame==0 && bMmapOk ){ - if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){ - rc = pagerMap(pPager, 0); - } - if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){ + void *pData = 0; + + rc = sqlite3OsFetch(pPager->fd, + (i64)(pgno-1) * pPager->pageSize, pPager->pageSize, &pData + ); + + if( rc==SQLITE_OK && pData ){ if( pPager->eState>PAGER_READER ){ (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); } if( pPg==0 ){ - rc = pagerAcquireMapPage(pPager, pgno, &pPg); + rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); + }else{ + sqlite3OsUnfetch(pPager->fd, pData); } if( pPg ){ assert( rc==SQLITE_OK ); @@ -7117,7 +7004,7 @@ static int pagerOpenWal(Pager *pPager){ ** (e.g. due to malloc() failure), return an error code. */ if( rc==SQLITE_OK ){ - rc = sqlite3WalOpen(pPager->pVfs, pPager, + rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, pPager->exclusiveMode, pPager->journalSizeLimit, &pPager->pWal ); @@ -7155,8 +7042,6 @@ int sqlite3PagerOpenWal( assert( pbOpen==0 || *pbOpen==0 ); assert( pbOpen!=0 || (!pPager->tempFile && !pPager->pWal) ); - pagerUnmap(pPager); - if( !pPager->tempFile && !pPager->pWal ){ if( !sqlite3PagerWalSupported(pPager) ) return SQLITE_CANTOPEN; diff --git a/src/pager.h b/src/pager.h index 970b5035bf..066d1f11d7 100644 --- a/src/pager.h +++ b/src/pager.h @@ -176,9 +176,6 @@ int sqlite3SectorSize(sqlite3_file *); void sqlite3PagerTruncateImage(Pager*,Pgno); int sqlite3PagerSetFilesize(Pager *, i64); -/* Write data to the database file */ -int sqlite3PagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff); - #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); #endif diff --git a/src/sqlite.h.in b/src/sqlite.h.in index c0a2803787..666a2e084d 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -707,24 +707,6 @@ struct sqlite3_file { ** fails to zero-fill short reads might seem to work. However, ** failure to zero-fill short reads will eventually lead to ** database corruption. -** -** Assuming parameter nNew is non-zero, the xMremap method should attempt -** to memory map a region nNew bytes in size starting at offset iOffset -** of the file. If successful, it should set *ppMap to point to the -** mapping and return SQLITE_OK. If the file is opened for read-write -** access, then the mapping should also be read-write. -** -** If nOld is non-zero, then the initial value of *ppMap points to a -** mapping returned by a previous call to xMremap. The existing mapping -** is nOld bytes in size and starts at offset iOffset of the file. In -** this case the xMremap method is expected to unmap the existing mapping -** and overwrite *ppMap with the pointer to the new mapping. If nOld is -** zero, then the initial value of *ppMap is undefined. -** -** If nNew is zero, then no new mapping should be created. Any old -** mapping must still be unmapped if nOld is non-zero. If the nOld -** parameter is non-zero, then the existing mapping is always unmapped - -** even if an error occurs. */ typedef struct sqlite3_io_methods sqlite3_io_methods; struct sqlite3_io_methods { @@ -747,14 +729,12 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ - int (*xMremap)(sqlite3_file *fd, int flags, - sqlite3_int64 iOff, sqlite3_int64 nOld, sqlite3_int64 nNew, void **ppMap); + int (*xFetch)(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); + int (*xUnfetch)(sqlite3_file*, void *p); /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; -#define SQLITE_MREMAP_EXTEND 0x0001 /* xMremap call may extend file */ - /* ** CAPI3REF: Standard File Control Opcodes ** @@ -906,6 +886,10 @@ struct sqlite3_io_methods { ** written into memory obtained from [sqlite3_malloc()]. The caller should ** invoke [sqlite3_free()] on the result to avoid a memory leak. ** +**
  • [[SQLITE_FCNTL_MMAP_SIZE]] +** The argument is assumed to point to a value of type sqlite3_int64. An +** advisory maximum amount of this file to memory map in bytes. +** ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -924,7 +908,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_PRAGMA 14 #define SQLITE_FCNTL_BUSYHANDLER 15 #define SQLITE_FCNTL_TEMPFILENAME 16 -#define SQLITE_FCNTL_GETFD 17 +#define SQLITE_FCNTL_MMAP_SIZE 18 /* ** CAPI3REF: Mutex Handle diff --git a/src/wal.c b/src/wal.c index 051b77f515..4602eb2c34 100644 --- a/src/wal.c +++ b/src/wal.c @@ -412,7 +412,6 @@ struct Wal { sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */ sqlite3_file *pDbFd; /* File handle for the database file */ sqlite3_file *pWalFd; /* File handle for WAL file */ - Pager *pPager; /* Pager object */ u32 iCallback; /* Value to pass to log callback (or 0) */ i64 mxWalSize; /* Truncate WAL to this size upon reset */ int nWiData; /* Size of array apWiData */ @@ -1252,7 +1251,6 @@ static void walIndexClose(Wal *pWal, int isDelete){ */ int sqlite3WalOpen( sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */ - Pager *pPager, /* Pager object handle */ sqlite3_file *pDbFd, /* The open database file */ const char *zWalName, /* Name of the WAL file */ int bNoShm, /* True to run in heap-memory mode */ @@ -1293,7 +1291,6 @@ int sqlite3WalOpen( pRet->zWalName = zWalName; pRet->syncHeader = 1; pRet->padToSectorBoundary = 1; - pRet->pPager = pPager; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); /* Open file handle on the write-ahead log file. */ @@ -1725,19 +1722,18 @@ static int walCheckpoint( rc = sqlite3OsSync(pWal->pWalFd, sync_flags); } - /* If the database file is currently smaller than mxPage pages in size, - ** the call below issues an SQLITE_FCNTL_SIZE_HINT to the OS layer to - ** inform it that it is likely to grow to that size. - ** - ** Additionally, if the pager is using mmap(), then the call to - ** SetFilesize() guarantees that the mapping is not larger than mxPage - ** pages. This makes the sqlite3OsTruncate() call below safe - no pages - ** that are part of the mapped region will be truncated away. */ + /* If the database may grow as a result of this checkpoint, hint + ** about the eventual size of the db file to the VFS layer. + */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); - rc = sqlite3PagerSetFilesize(pWal->pPager, nReq); + rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); + if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); + } } + /* Iterate through the contents of the WAL, copying data to the db file. */ while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ i64 iOffset; @@ -1749,7 +1745,7 @@ static int walCheckpoint( if( rc!=SQLITE_OK ) break; iOffset = (iDbpage-1)*(i64)szPage; testcase( IS_BIG_INT(iOffset) ); - rc = sqlite3PagerWriteData(pWal->pPager, zBuf, szPage, iOffset); + rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); if( rc!=SQLITE_OK ) break; } diff --git a/src/wal.h b/src/wal.h index 96d4ff9adf..ff7624af63 100644 --- a/src/wal.h +++ b/src/wal.h @@ -53,8 +53,7 @@ typedef struct Wal Wal; /* Open and close a connection to a write-ahead log. */ -int sqlite3WalOpen( - sqlite3_vfs*, Pager *, sqlite3_file*, const char *, int, i64, Wal**); +int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**); int sqlite3WalClose(Wal *pWal, int sync_flags, int, u8 *); /* Set the limiting size of a WAL file. */ From dba0cb23514248d15da8870951ce5c60e5d478a6 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 24 Mar 2013 22:56:49 +0000 Subject: [PATCH 030/204] Remove the SQLITE_OMIT_MERGE_SORT compile-time option and its related code. The merge sorter is now a required component. FossilOrigin-Name: 8b44d6fb159e85267095e846cded2764c3bcb895 --- manifest | 24 ++++++++++++------------ manifest.uuid | 2 +- src/build.c | 32 -------------------------------- src/ctime.c | 3 --- src/test_config.c | 6 +----- src/vdbe.c | 23 ----------------------- src/vdbeInt.h | 10 ---------- src/vdbesort.c | 3 --- 8 files changed, 14 insertions(+), 89 deletions(-) diff --git a/manifest b/manifest index 4437f787e5..6fe750fd31 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Many\sspelling\sfixes\sin\scomments.\s\sNo\schanges\sto\scode. -D 2013-03-21T21:20:32.533 +C Remove\sthe\sSQLITE_OMIT_MERGE_SORT\scompile-time\soption\sand\sits\srelated\scode.\nThe\smerge\ssorter\sis\snow\sa\srequired\scomponent. +D 2013-03-24T22:56:49.538 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -124,10 +124,10 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 3cebaa69db81a528e115b463a5506133a0043710 F src/btree.h 3ad7964d6c5b1c7bff569aab6adfa075f8bf06cd F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 -F src/build.c b85db6cb50fc63e44776b741452250626bad491a +F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c 72a70dcfda75d3a1f81041ce4573e7afddcd8e4e +F src/ctime.c 2a5f251fcd7393808df77ccfc817e7058df08c4c F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 @@ -197,7 +197,7 @@ F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de F src/test_btree.c 5b89601dcb42a33ba8b820a6b763cc9cb48bac16 -F src/test_config.c 09781397ccc24268cb895be0d4c21b4aad651486 +F src/test_config.c ab6605a41f2020912d5e44b8dd4b9fb21bd75ac0 F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c 1c51e203b2c20235d8c3739f8c1fb13a7502915b @@ -239,14 +239,14 @@ F src/update.c a2a5631d618cbe240fc83725fa9e95c56ae0084c F src/utf.c 8d819e2e5104a430fc2005f018db14347c95a38f F src/util.c 550f2b6a5c0085153a4d00462719fb17ee242792 F src/vacuum.c 2727bdd08847fcb6b2d2da6d14f018910e8645d3 -F src/vdbe.c f9622d5b222f7dd929d7f4a4d42278d47d2b24ab +F src/vdbe.c 1d56617d816098a41bdbe73536a67dcdefad52bd F src/vdbe.h b52887278cb173e66188da84dfab216bea61119d -F src/vdbeInt.h 396bb03eec560f768d1b86092b00f46c25575d3b +F src/vdbeInt.h 3bb8531d6768379f086e1576b31d52331f6eb982 F src/vdbeapi.c 56f9ad6329d093f54b5dd321a2b4a643f89a6cb6 F src/vdbeaux.c ecb43014bcd3019e5aa2b5561e5c3a447f007a08 F src/vdbeblob.c 32f2a4899d67f69634ea4dd93e3f651936d732cb F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab -F src/vdbesort.c c61ca318681c0e7267da8be3abfca8469652a7e9 +F src/vdbesort.c 4fad64071ae120c25f39dcac572d716b9cadeb7f F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5b22053f918d16f593227a432a5d5b4c195bb0b5 -R 0ed9a71acb145119af7cea3063285aea -U mistachkin -Z eedfaec177b15430293d5b03ecbc6bec +P 6f6e2d50941e444ebc83604daddcc034137a05b7 +R c9af7e9245b15a89c65f60e3499063d2 +U drh +Z b277505b37d3e408ea130ed65856e826 diff --git a/manifest.uuid b/manifest.uuid index 3f2050457f..c763374069 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6f6e2d50941e444ebc83604daddcc034137a05b7 \ No newline at end of file +8b44d6fb159e85267095e846cded2764c3bcb895 \ No newline at end of file diff --git a/src/build.c b/src/build.c index c4d14d8943..5920ade059 100644 --- a/src/build.c +++ b/src/build.c @@ -2392,9 +2392,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ int tnum; /* Root page of index */ Vdbe *v; /* Generate code into this virtual machine */ KeyInfo *pKey; /* KeyInfo for index */ -#ifdef SQLITE_OMIT_MERGE_SORT - int regIdxKey; /* Registers containing the index key */ -#endif int regRecord; /* Register holding assemblied index record */ sqlite3 *db = pParse->db; /* The database connection */ int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); @@ -2422,13 +2419,9 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ (char *)pKey, P4_KEYINFO_HANDOFF); sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR|((memRootPage>=0)?OPFLAG_P2ISREG:0)); -#ifndef SQLITE_OMIT_MERGE_SORT /* Open the sorter cursor if we are to use one. */ iSorter = pParse->nTab++; sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, 0, (char*)pKey, P4_KEYINFO); -#else - iSorter = iTab; -#endif /* Open the table. Loop through all rows of the table, inserting index ** records into the sorter. */ @@ -2436,7 +2429,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); regRecord = sqlite3GetTempReg(pParse); -#ifndef SQLITE_OMIT_MERGE_SORT sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); @@ -2456,30 +2448,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord); sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 1); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); -#else - regIdxKey = sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 1); - addr2 = addr1 + 1; - if( pIndex->onError!=OE_None ){ - const int regRowid = regIdxKey + pIndex->nColumn; - const int j2 = sqlite3VdbeCurrentAddr(v) + 2; - void * const pRegKey = SQLITE_INT_TO_PTR(regIdxKey); - - /* The registers accessed by the OP_IsUnique opcode were allocated - ** using sqlite3GetTempRange() inside of the sqlite3GenerateIndexKey() - ** call above. Just before that function was freed they were released - ** (made available to the compiler for reuse) using - ** sqlite3ReleaseTempRange(). So in some ways having the OP_IsUnique - ** opcode use the values stored within seems dangerous. However, since - ** we can be sure that no other temp registers have been allocated - ** since sqlite3ReleaseTempRange() was called, it is safe to do so. - */ - sqlite3VdbeAddOp4(v, OP_IsUnique, iIdx, j2, regRowid, pRegKey, P4_INT32); - sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_UNIQUE, - "indexed columns are not unique", P4_STATIC); - } - sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 0); - sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); -#endif sqlite3ReleaseTempReg(pParse, regRecord); sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); sqlite3VdbeJumpHere(v, addr1); diff --git a/src/ctime.c b/src/ctime.c index 5dee724747..c42454ca71 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -263,9 +263,6 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_OMIT_MEMORYDB "OMIT_MEMORYDB", #endif -#ifdef SQLITE_OMIT_MERGE_SORT - "OMIT_MERGE_SORT", -#endif #ifdef SQLITE_OMIT_OR_OPTIMIZATION "OMIT_OR_OPTIMIZATION", #endif diff --git a/src/test_config.c b/src/test_config.c index f79b455a57..f3c2f61162 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -395,11 +395,7 @@ Tcl_SetVar2(interp, "sqlite_options", "long_double", Tcl_SetVar2(interp, "sqlite_options", "memorymanage", "0", TCL_GLOBAL_ONLY); #endif -#ifdef SQLITE_OMIT_MERGE_SORT - Tcl_SetVar2(interp, "sqlite_options", "mergesort", "0", TCL_GLOBAL_ONLY); -#else - Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); -#endif +Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); #ifdef SQLITE_OMIT_OR_OPTIMIZATION Tcl_SetVar2(interp, "sqlite_options", "or_opt", "0", TCL_GLOBAL_ONLY); diff --git a/src/vdbe.c b/src/vdbe.c index 21c0cf8ec4..688621d951 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -152,11 +152,7 @@ int sqlite3_found_count = 0; && sqlite3VdbeMemMakeWriteable(P) ){ goto no_mem;} /* Return true if the cursor was opened using the OP_OpenSorter opcode. */ -#ifdef SQLITE_OMIT_MERGE_SORT -# define isSorter(x) 0 -#else # define isSorter(x) ((x)->pSorter!=0) -#endif /* ** Argument pMem points at a register that will be passed to a @@ -3321,17 +3317,12 @@ case OP_OpenEphemeral: { case OP_SorterOpen: { VdbeCursor *pCx; -#ifndef SQLITE_OMIT_MERGE_SORT pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( pCx==0 ) goto no_mem; pCx->pKeyInfo = pOp->p4.pKeyInfo; pCx->pKeyInfo->enc = ENC(p->db); pCx->isSorter = 1; rc = sqlite3VdbeSorterInit(db, pCx); -#else - pOp->opcode = OP_OpenEphemeral; - pc--; -#endif break; } @@ -4214,15 +4205,10 @@ case OP_SorterCompare: { case OP_SorterData: { VdbeCursor *pC; -#ifndef SQLITE_OMIT_MERGE_SORT pOut = &aMem[pOp->p2]; pC = p->apCsr[pOp->p1]; assert( pC->isSorter ); rc = sqlite3VdbeSorterRowkey(pC, pOut); -#else - pOp->opcode = OP_RowKey; - pc--; -#endif break; } @@ -4421,9 +4407,6 @@ case OP_Last: { /* jump */ ** correctly optimizing out sorts. */ case OP_SorterSort: /* jump */ -#ifdef SQLITE_OMIT_MERGE_SORT - pOp->opcode = OP_Sort; -#endif case OP_Sort: { /* jump */ #ifdef SQLITE_TEST sqlite3_sort_count++; @@ -4502,9 +4485,6 @@ case OP_Rewind: { /* jump */ ** number P5-1 in the prepared statement is incremented. */ case OP_SorterNext: /* jump */ -#ifdef SQLITE_OMIT_MERGE_SORT - pOp->opcode = OP_Next; -#endif case OP_Prev: /* jump */ case OP_Next: { /* jump */ VdbeCursor *pC; @@ -4555,9 +4535,6 @@ case OP_Next: { /* jump */ ** for tables is OP_Insert. */ case OP_SorterInsert: /* in2 */ -#ifdef SQLITE_OMIT_MERGE_SORT - pOp->opcode = OP_IdxInsert; -#endif case OP_IdxInsert: { /* in2 */ VdbeCursor *pC; BtCursor *pCrsr; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index b2bc3a6342..2372b83849 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -429,15 +429,6 @@ int sqlite3VdbeFrameRestore(VdbeFrame *); void sqlite3VdbeMemStoreType(Mem *pMem); int sqlite3VdbeTransferError(Vdbe *p); -#ifdef SQLITE_OMIT_MERGE_SORT -# define sqlite3VdbeSorterInit(Y,Z) SQLITE_OK -# define sqlite3VdbeSorterWrite(X,Y,Z) SQLITE_OK -# define sqlite3VdbeSorterClose(Y,Z) -# define sqlite3VdbeSorterRowkey(Y,Z) SQLITE_OK -# define sqlite3VdbeSorterRewind(X,Y,Z) SQLITE_OK -# define sqlite3VdbeSorterNext(X,Y,Z) SQLITE_OK -# define sqlite3VdbeSorterCompare(X,Y,Z) SQLITE_OK -#else int sqlite3VdbeSorterInit(sqlite3 *, VdbeCursor *); void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *); int sqlite3VdbeSorterRowkey(const VdbeCursor *, Mem *); @@ -445,7 +436,6 @@ int sqlite3VdbeSorterNext(sqlite3 *, const VdbeCursor *, int *); int sqlite3VdbeSorterRewind(sqlite3 *, const VdbeCursor *, int *); int sqlite3VdbeSorterWrite(sqlite3 *, const VdbeCursor *, Mem *); int sqlite3VdbeSorterCompare(const VdbeCursor *, Mem *, int *); -#endif #if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 void sqlite3VdbeEnter(Vdbe*); diff --git a/src/vdbesort.c b/src/vdbesort.c index d51bbf54a3..fdfc4a79dd 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -18,7 +18,6 @@ #include "sqliteInt.h" #include "vdbeInt.h" -#ifndef SQLITE_OMIT_MERGE_SORT typedef struct VdbeSorterIter VdbeSorterIter; typedef struct SorterRecord SorterRecord; @@ -1037,5 +1036,3 @@ int sqlite3VdbeSorterCompare( vdbeSorterCompare(pCsr, 1, pVal->z, pVal->n, pKey, nKey, pRes); return SQLITE_OK; } - -#endif /* #ifndef SQLITE_OMIT_MERGE_SORT */ From 1cb90c668c1194340e614023120030fab9c4a0be Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 11:38:44 +0000 Subject: [PATCH 031/204] In fts3, when filtering lists for hits in a specific column, edit the list in place in the same way as it is for NEAR filtering. Fix for [38b1ae018f]. FossilOrigin-Name: f85f9103cffa5c8ba6a63a68beb90817147ba080 --- ext/fts3/fts3_write.c | 28 ++++++++++++++++++---------- manifest | 16 ++++++++-------- manifest.uuid | 2 +- test/fts3near.test | 8 ++++++++ 4 files changed, 35 insertions(+), 19 deletions(-) diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index bda7fbbe1a..269d1dd599 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -1482,6 +1482,7 @@ static int fts3SegReaderNextDocid( *pnOffsetList = (int)(p - pReader->pOffsetList - 1); } + /* List may have been edited in place by fts3EvalNearTrim() */ while( papSegment, nMerge, j, xCmp); + if( nList>0 && fts3SegReaderIsPending(apSegment[0]) ){ + rc = fts3MsrBufferData(pMsr, pList, nList+1); + if( rc!=SQLITE_OK ) return rc; + assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); + pList = pMsr->aBuffer; + } + if( pMsr->iColFilter>=0 ){ - fts3ColumnFilter(pMsr->iColFilter, &pList, &nList); + fts3ColumnFilter(pMsr->iColFilter, 1, &pList, &nList); } if( nList>0 ){ - if( fts3SegReaderIsPending(apSegment[0]) ){ - rc = fts3MsrBufferData(pMsr, pList, nList+1); - if( rc!=SQLITE_OK ) return rc; - *paPoslist = pMsr->aBuffer; - assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); - }else{ - *paPoslist = pList; - } + *paPoslist = pList; *piDocid = iDocid; *pnPoslist = nList; break; @@ -2856,7 +2864,7 @@ int sqlite3Fts3SegReaderStep( } if( isColFilter ){ - fts3ColumnFilter(pFilter->iCol, &pList, &nList); + fts3ColumnFilter(pFilter->iCol, 0, &pList, &nList); } if( !isIgnoreEmpty || nList>0 ){ diff --git a/manifest b/manifest index 6fe750fd31..fbf8a6e7a7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sSQLITE_OMIT_MERGE_SORT\scompile-time\soption\sand\sits\srelated\scode.\nThe\smerge\ssorter\sis\snow\sa\srequired\scomponent. -D 2013-03-24T22:56:49.538 +C In\sfts3,\swhen\sfiltering\slists\sfor\shits\sin\sa\sspecific\scolumn,\sedit\sthe\slist\sin\splace\sin\sthe\ssame\sway\sas\sit\sis\sfor\sNEAR\sfiltering.\sFix\sfor\s[38b1ae018f]. +D 2013-03-25T11:38:44.860 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -72,7 +72,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c a863f05f758af36777dffc2facc898bc73fec896 -F ext/fts3/fts3_write.c c2166f7148a4ad8bcdad99a99d647b1091744e6b +F ext/fts3/fts3_write.c d92c6cbe07363791cfe8b62b4dee67e6f8afc9e2 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -491,7 +491,7 @@ F test/fts3fault2.test 3198eef2804deea7cac8403e771d9cbcb752d887 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 F test/fts3malloc.test b86ea33db9e8c58c0c2f8027a9fcadaf6a1568be F test/fts3matchinfo.test ecb08f586d027eb03941bcfcded6cb9d8ccb3a66 -F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 +F test/fts3near.test f78c79a92b1da496cb2a239ba24af35f33445698 F test/fts3prefix.test b36d4f00b128a51e7b386cc013a874246d9d7dc1 F test/fts3prefix2.test 477ca96e67f60745b7ac931cfa6e9b080c562da5 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 6f6e2d50941e444ebc83604daddcc034137a05b7 -R c9af7e9245b15a89c65f60e3499063d2 -U drh -Z b277505b37d3e408ea130ed65856e826 +P 8b44d6fb159e85267095e846cded2764c3bcb895 +R 6c0c94d8e550f4b716beb89e2de0d44b +U dan +Z f4ab0250caa5559575a04c2c550a6a93 diff --git a/manifest.uuid b/manifest.uuid index c763374069..e4241b77b2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8b44d6fb159e85267095e846cded2764c3bcb895 \ No newline at end of file +f85f9103cffa5c8ba6a63a68beb90817147ba080 \ No newline at end of file diff --git a/test/fts3near.test b/test/fts3near.test index 9c4409e1ed..25676cc956 100644 --- a/test/fts3near.test +++ b/test/fts3near.test @@ -580,5 +580,13 @@ do_test fts3near-6.5 { } } {3} +# Ticket 38b1ae018f. +# +do_execsql_test fts3near-7.1 { + CREATE VIRTUAL TABLE x USING fts4(y,z); + INSERT INTO x VALUES('aaa bbb ccc ddd', 'bbb ddd aaa ccc'); + SELECT * FROM x where y MATCH 'bbb NEAR/6 aaa'; +} {{aaa bbb ccc ddd} {bbb ddd aaa ccc}} + finish_test From e115ff8171bb43e56254c9e88e96254d56e3986d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 12:02:45 +0000 Subject: [PATCH 032/204] Add a second test for [38b1ae018f]. FossilOrigin-Name: 5062db672c00c3365d51cd6b39815078f5b6b525 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/fts3near.test | 6 ++++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index fbf8a6e7a7..abc6aa2430 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sfts3,\swhen\sfiltering\slists\sfor\shits\sin\sa\sspecific\scolumn,\sedit\sthe\slist\sin\splace\sin\sthe\ssame\sway\sas\sit\sis\sfor\sNEAR\sfiltering.\sFix\sfor\s[38b1ae018f]. -D 2013-03-25T11:38:44.860 +C Add\sa\ssecond\stest\sfor\s[38b1ae018f]. +D 2013-03-25T12:02:45.865 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -491,7 +491,7 @@ F test/fts3fault2.test 3198eef2804deea7cac8403e771d9cbcb752d887 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 F test/fts3malloc.test b86ea33db9e8c58c0c2f8027a9fcadaf6a1568be F test/fts3matchinfo.test ecb08f586d027eb03941bcfcded6cb9d8ccb3a66 -F test/fts3near.test f78c79a92b1da496cb2a239ba24af35f33445698 +F test/fts3near.test 12895557870b0f9af7cc0be81a0171abb2d12f12 F test/fts3prefix.test b36d4f00b128a51e7b386cc013a874246d9d7dc1 F test/fts3prefix2.test 477ca96e67f60745b7ac931cfa6e9b080c562da5 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 8b44d6fb159e85267095e846cded2764c3bcb895 -R 6c0c94d8e550f4b716beb89e2de0d44b +P f85f9103cffa5c8ba6a63a68beb90817147ba080 +R 8636831e10a59eea94894795b6da6922 U dan -Z f4ab0250caa5559575a04c2c550a6a93 +Z 40d326d8e18513e2868148c4279af040 diff --git a/manifest.uuid b/manifest.uuid index e4241b77b2..14d19b23d1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f85f9103cffa5c8ba6a63a68beb90817147ba080 \ No newline at end of file +5062db672c00c3365d51cd6b39815078f5b6b525 \ No newline at end of file diff --git a/test/fts3near.test b/test/fts3near.test index 25676cc956..9276fa3737 100644 --- a/test/fts3near.test +++ b/test/fts3near.test @@ -588,5 +588,11 @@ do_execsql_test fts3near-7.1 { SELECT * FROM x where y MATCH 'bbb NEAR/6 aaa'; } {{aaa bbb ccc ddd} {bbb ddd aaa ccc}} +do_execsql_test fts3near-7.2 { + CREATE VIRTUAL TABLE t2 USING fts4(a, b); + INSERT INTO t2 VALUES('A B C', 'A D E'); + SELECT * FROM t2 where t2 MATCH 'a:A NEAR E' +} {} + finish_test From a1afc7425a42b774b307416b3256d28c1d4a185f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 13:50:49 +0000 Subject: [PATCH 033/204] Fix a case in the pager where an xFetch() reference was being leaked following an OOM error. FossilOrigin-Name: 5885ba6ce768658ec25b60747430d147b315b55c --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os_unix.c | 2 +- src/pager.c | 1 + src/pager.h | 1 - 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 8d71dc88ab..5e9f5ad099 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Replace\sthe\ssqlite3_io_methods.xMremap\sinterface\swith\ssqlite3_io_methods.xFetch\sand\sxUnfetch. -D 2013-03-23T21:00:41.457 +C Fix\sa\scase\sin\sthe\spager\swhere\san\sxFetch()\sreference\swas\sbeing\sleaked\sfollowing\san\sOOM\serror. +D 2013-03-25T13:50:49.946 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,10 +160,10 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c a7ec2ddc43fb2684bfd8cb30db15ea6cce3298e9 F src/os.h 782980cd0d840b4240a6e29ba9f8ef0ca0d750ca F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c d8cdf331ad08650fc8397eee79b55548070077b0 +F src/os_unix.c 1e61abcb664f3430ce13e5c5f32b2b2e204a6f23 F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac -F src/pager.c 2e63de7185c2a573d26e5087869a9e48326076f9 -F src/pager.h faf4bed79e80cb6527209dfe5cfb08aa5764b941 +F src/pager.c 3da35f3287791354f170824fd847e75ba9f2a0dc +F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 60b9f5e4dd2af54975ba78437239f0bebd472fd2 -R eee4d88d6400cf65980c5b201532d2c9 +P 1431be95579160fb70408d43e17fc23c7b69ab4a +R 2702651dc0367254a02a7905a85901b0 U dan -Z eabd20cdc7aabbc65591f96e4f6dff04 +Z 4827c382c08dac8144315d5de3ed2c37 diff --git a/manifest.uuid b/manifest.uuid index f378aa2e8b..37accb35b1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1431be95579160fb70408d43e17fc23c7b69ab4a \ No newline at end of file +5885ba6ce768658ec25b60747430d147b315b55c \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 334719200a..d865810570 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -1807,6 +1807,7 @@ end_unlock: ** the requested locking level, this routine is a no-op. */ static int unixUnlock(sqlite3_file *id, int eFileLock){ + assert( eFileLock==SHARED_LOCK || ((unixFile *)id)->nFetchOut==0 ); return posixUnlock(id, eFileLock, 0); } @@ -4531,7 +4532,6 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ } if( nMap!=pFd->mmapSize ){ - void *pNew; unixUnmapfile(pFd); if( nMap>0 ){ diff --git a/src/pager.c b/src/pager.c index 7e00e1431a..d6e6b5b440 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3870,6 +3870,7 @@ static int pagerAcquireMapPage( }else{ *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); if( p==0 ){ + sqlite3OsUnfetch(pPager->fd, pData); return SQLITE_NOMEM; } p->pExtra = (void *)&p[1]; diff --git a/src/pager.h b/src/pager.h index 066d1f11d7..81ab30c115 100644 --- a/src/pager.h +++ b/src/pager.h @@ -174,7 +174,6 @@ int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); -int sqlite3PagerSetFilesize(Pager *, i64); #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); From 11ac84635b840afb91178063bd3fa30fff584963 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 14:31:39 +0000 Subject: [PATCH 034/204] Do not return SQLITE_IOERR when the user attempts to open a small file that is not a database with mmap enabled. Instead return SQLITE_NOTADB. FossilOrigin-Name: bbcaab3e80d0ff776d8567094b137d1483b3377b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 5e9f5ad099..01d74c7724 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scase\sin\sthe\spager\swhere\san\sxFetch()\sreference\swas\sbeing\sleaked\sfollowing\san\sOOM\serror. -D 2013-03-25T13:50:49.946 +C Do\snot\sreturn\sSQLITE_IOERR\swhen\sthe\suser\sattempts\sto\sopen\sa\ssmall\sfile\sthat\sis\snot\sa\sdatabase\swith\smmap\senabled.\sInstead\sreturn\sSQLITE_NOTADB. +D 2013-03-25T14:31:39.689 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h 782980cd0d840b4240a6e29ba9f8ef0ca0d750ca F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 1e61abcb664f3430ce13e5c5f32b2b2e204a6f23 F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac -F src/pager.c 3da35f3287791354f170824fd847e75ba9f2a0dc +F src/pager.c 2dd59f366b519d01d08b44f615058e7cfacd0bd7 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 1431be95579160fb70408d43e17fc23c7b69ab4a -R 2702651dc0367254a02a7905a85901b0 +P 5885ba6ce768658ec25b60747430d147b315b55c +R c243df2223b3431445e14c1469df7b20 U dan -Z 4827c382c08dac8144315d5de3ed2c37 +Z 6b851eca8762089c75296a1160d8987e diff --git a/manifest.uuid b/manifest.uuid index 37accb35b1..b4cdb9d207 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5885ba6ce768658ec25b60747430d147b315b55c \ No newline at end of file +bbcaab3e80d0ff776d8567094b137d1483b3377b \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index d6e6b5b440..927126c393 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5051,7 +5051,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( !pPager->tempFile && ( pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0 - || pPager->bUseFetch /* TODO: Currently required for xUnfetch(0) only. */ + || pPager->bUseFetch )){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous @@ -5078,7 +5078,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( nPage>0 ){ IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); - if( rc!=SQLITE_OK ){ + if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ goto failed; } }else{ From aef49d7141d9bbbc68a2c8ce7bd17cdafb06aa6b Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 16:28:54 +0000 Subject: [PATCH 035/204] Remove unnecessary code to round the size of a memory mapping to 4KB from os_unix.c. Rename SQLITE_IOERR_MREMAP to SQLITE_IOERR_MMAP. Fix other small issues in os_unix.c. FossilOrigin-Name: dce35c01a5fe66d2970075b1e3f0376026485e4c --- manifest | 14 ++++++------- manifest.uuid | 2 +- src/os_unix.c | 54 +++++++++++++++++++++++++++++++++++++++---------- src/sqlite.h.in | 2 +- 4 files changed, 52 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 01d74c7724..919b150ea9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sreturn\sSQLITE_IOERR\swhen\sthe\suser\sattempts\sto\sopen\sa\ssmall\sfile\sthat\sis\snot\sa\sdatabase\swith\smmap\senabled.\sInstead\sreturn\sSQLITE_NOTADB. -D 2013-03-25T14:31:39.689 +C Remove\sunnecessary\scode\sto\sround\sthe\ssize\sof\sa\smemory\smapping\sto\s4KB\sfrom\sos_unix.c.\sRename\sSQLITE_IOERR_MREMAP\sto\sSQLITE_IOERR_MMAP.\sFix\sother\ssmall\sissues\sin\sos_unix.c. +D 2013-03-25T16:28:54.437 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c a7ec2ddc43fb2684bfd8cb30db15ea6cce3298e9 F src/os.h 782980cd0d840b4240a6e29ba9f8ef0ca0d750ca F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 1e61abcb664f3430ce13e5c5f32b2b2e204a6f23 +F src/os_unix.c 82efc58c818c69e191f5560935037af365f99ff5 F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac F src/pager.c 2dd59f366b519d01d08b44f615058e7cfacd0bd7 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -176,7 +176,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in f41949e1e778043dd9fc23b33b96e7a6e20de4d6 +F src/sqlite.h.in a0866dafaa412d1e89abbc78d2b44b12cf95d1d1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5885ba6ce768658ec25b60747430d147b315b55c -R c243df2223b3431445e14c1469df7b20 +P bbcaab3e80d0ff776d8567094b137d1483b3377b +R fd25f18074b1cc787a38214a338c9127 U dan -Z 6b851eca8762089c75296a1160d8987e +Z ae05d407cd866087491e3302af269873 diff --git a/manifest.uuid b/manifest.uuid index b4cdb9d207..23ee7e3b08 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bbcaab3e80d0ff776d8567094b137d1483b3377b \ No newline at end of file +dce35c01a5fe66d2970075b1e3f0376026485e4c \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index d865810570..a97aba87f1 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4493,15 +4493,8 @@ static int unixShmUnmap( #endif /* #ifndef SQLITE_OMIT_WAL */ /* -** Arguments x and y are both integers. Argument y must be a power of 2. -** Round x up to the nearest integer multiple of y. For example: -** -** ROUNDUP(0, 8) -> 0 -** ROUNDUP(13, 8) -> 16 -** ROUNDUP(32, 8) -> 32 +** If it is currently memory mapped, unmap file pFd. */ -#define ROUNDUP(x,y) (((x)+y-1)&~(y-1)) - static void unixUnmapfile(unixFile *pFd){ assert( pFd->nFetchOut==0 ); if( pFd->pMapRegion ){ @@ -4512,6 +4505,22 @@ static void unixUnmapfile(unixFile *pFd){ } } +/* +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_SIZE, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. +*/ static int unixMapfile(unixFile *pFd, i64 nByte){ i64 nMap = nByte; int rc; @@ -4538,19 +4547,32 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ void *pNew; int flags = PROT_READ; if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - pNew = mmap(0, ROUNDUP(nMap, 4096), flags, MAP_SHARED, pFd->h, 0); + pNew = mmap(0, nMap, flags, MAP_SHARED, pFd->h, 0); if( pNew==MAP_FAILED ){ - return SQLITE_IOERR_MREMAP; + return SQLITE_IOERR_MMAP; } pFd->pMapRegion = pNew; - pFd->mmapOrigsize = pFd->mmapSize = nMap; + pFd->mmapSize = nMap; + pFd->mmapOrigsize = nMap; } } return SQLITE_OK; } +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling unixUnfetch(). +*/ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ unixFile *pFd = (unixFile *)fd; /* The underlying database file */ *pp = 0; @@ -4568,9 +4590,19 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ return SQLITE_OK; } +/* +** If the second argument is non-NULL, then this function releases a +** reference obtained by an earlier call to unixFetch(). Or, if the second +** argument is NULL, then this function is being called to inform the VFS +** layer that, according to POSIX, any existing mapping may now be invalid +** and should be unmapped. +*/ static int unixUnfetch(sqlite3_file *fd, void *p){ unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ assert( (p==0)==(pFd->nFetchOut==0) ); if( p ){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 666a2e084d..7bbd3fa000 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -470,7 +470,7 @@ int sqlite3_exec( #define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) #define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) #define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) -#define SQLITE_IOERR_MREMAP (SQLITE_IOERR | (24<<8)) +#define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) From df737fe6f51522c478235ff6ddc478243f718bf2 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 17:00:24 +0000 Subject: [PATCH 036/204] Change the signature of the xUnfetch method to "int (*xUnfetch)(sqlite3_file*, sqlite3_int64 iOfst, void *p)". FossilOrigin-Name: 115b830509e8f0aa9d5965c1e9cd4f2ed9d01938 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/os.c | 4 ++-- src/os.h | 2 +- src/os_unix.c | 18 ++++++++++++------ src/pager.c | 10 +++++----- src/sqlite.h.in | 2 +- 7 files changed, 32 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index 919b150ea9..143bb5bf58 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sunnecessary\scode\sto\sround\sthe\ssize\sof\sa\smemory\smapping\sto\s4KB\sfrom\sos_unix.c.\sRename\sSQLITE_IOERR_MREMAP\sto\sSQLITE_IOERR_MMAP.\sFix\sother\ssmall\sissues\sin\sos_unix.c. -D 2013-03-25T16:28:54.437 +C Change\sthe\ssignature\sof\sthe\sxUnfetch\smethod\sto\s"int\s(*xUnfetch)(sqlite3_file*,\ssqlite3_int64\siOfst,\svoid\s*p)". +D 2013-03-25T17:00:24.194 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,12 +157,12 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c a7ec2ddc43fb2684bfd8cb30db15ea6cce3298e9 -F src/os.h 782980cd0d840b4240a6e29ba9f8ef0ca0d750ca +F src/os.c 17c6e6e4a17135ec1c7c468f8d0b97ec6f8a9264 +F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 82efc58c818c69e191f5560935037af365f99ff5 +F src/os_unix.c 664fc178a9478246f6c84e46292e77fb5bee4431 F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac -F src/pager.c 2dd59f366b519d01d08b44f615058e7cfacd0bd7 +F src/pager.c d0cdf47664e487f735b4bea485ee70637ed34a55 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -176,7 +176,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in a0866dafaa412d1e89abbc78d2b44b12cf95d1d1 +F src/sqlite.h.in b04790da19184e883da788d07688f16230d2cb6b F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P bbcaab3e80d0ff776d8567094b137d1483b3377b -R fd25f18074b1cc787a38214a338c9127 +P dce35c01a5fe66d2970075b1e3f0376026485e4c +R 12377e21e3793332cede5cbfb008d2ac U dan -Z ae05d407cd866087491e3302af269873 +Z 1c478466922e4f7b4b376664d80146c7 diff --git a/manifest.uuid b/manifest.uuid index 23ee7e3b08..838ea0b60b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dce35c01a5fe66d2970075b1e3f0376026485e4c \ No newline at end of file +115b830509e8f0aa9d5965c1e9cd4f2ed9d01938 \ No newline at end of file diff --git a/src/os.c b/src/os.c index 08e0d42172..f2d85bba4d 100644 --- a/src/os.c +++ b/src/os.c @@ -144,8 +144,8 @@ int sqlite3OsShmMap( int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ return id->pMethods->xFetch(id, iOff, iAmt, pp); } -int sqlite3OsUnfetch(sqlite3_file *id, void *p){ - return id->pMethods->xUnfetch(id, p); +int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ + return id->pMethods->xUnfetch(id, iOff, p); } /* diff --git a/src/os.h b/src/os.h index a7dacfb6ea..82e07ee1cb 100644 --- a/src/os.h +++ b/src/os.h @@ -260,7 +260,7 @@ int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); -int sqlite3OsUnfetch(sqlite3_file *, void *); +int sqlite3OsUnfetch(sqlite3_file *, i64, void *); /* diff --git a/src/os_unix.c b/src/os_unix.c index a97aba87f1..00c0088f8a 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4591,13 +4591,16 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ } /* -** If the second argument is non-NULL, then this function releases a -** reference obtained by an earlier call to unixFetch(). Or, if the second -** argument is NULL, then this function is being called to inform the VFS -** layer that, according to POSIX, any existing mapping may now be invalid -** and should be unmapped. +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to unixFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the unixFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. */ -static int unixUnfetch(sqlite3_file *fd, void *p){ +static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ unixFile *pFd = (unixFile *)fd; /* The underlying database file */ /* If p==0 (unmap the entire file) then there must be no outstanding @@ -4605,6 +4608,9 @@ static int unixUnfetch(sqlite3_file *fd, void *p){ ** then there must be at least one outstanding. */ assert( (p==0)==(pFd->nFetchOut==0) ); + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + if( p ){ pFd->nFetchOut--; }else{ diff --git a/src/pager.c b/src/pager.c index 927126c393..06a7eefee1 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3091,7 +3091,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); if( rc!=SQLITE_OK || changed ){ pager_reset(pPager); - if( pPager->bUseFetch ) sqlite3OsUnfetch(pPager->fd, 0); + if( pPager->bUseFetch ) sqlite3OsUnfetch(pPager->fd, 0, 0); } return rc; @@ -3870,7 +3870,7 @@ static int pagerAcquireMapPage( }else{ *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); if( p==0 ){ - sqlite3OsUnfetch(pPager->fd, pData); + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1) * pPager->pageSize, pData); return SQLITE_NOMEM; } p->pExtra = (void *)&p[1]; @@ -3903,7 +3903,7 @@ static void pagerReleaseMapPage(PgHdr *pPg){ pPager->pFree = pPg; assert( pPager->fd->pMethods->iVersion>=3 ); - sqlite3OsUnfetch(pPager->fd, pPg->pData); + sqlite3OsUnfetch(pPager->fd, (i64)(pPg->pgno-1)*pPager->pageSize, pPg->pData); } /* @@ -5095,7 +5095,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** to be the right size but is not actually valid. Avoid this ** possibility by unmapping the db here. */ if( pPager->bUseFetch ){ - sqlite3OsUnfetch(pPager->fd, 0); + sqlite3OsUnfetch(pPager->fd, 0, 0); } } } @@ -5245,7 +5245,7 @@ int sqlite3PagerAcquire( if( pPg==0 ){ rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); }else{ - sqlite3OsUnfetch(pPager->fd, pData); + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1)*pPager->pageSize, pData); } if( pPg ){ assert( rc==SQLITE_OK ); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 7bbd3fa000..b68287b0e7 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -730,7 +730,7 @@ struct sqlite3_io_methods { int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ int (*xFetch)(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); - int (*xUnfetch)(sqlite3_file*, void *p); + int (*xUnfetch)(sqlite3_file*, sqlite3_int64 iOfst, void *p); /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; From a64d5a15b7d85ee49c5632ee37f6c0bd3242d258 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 18:25:49 +0000 Subject: [PATCH 037/204] Simulate OOM errors in the sqlite3OsFetch() function. Run malloc.test as part of the "mmap" permutation. FossilOrigin-Name: 77443ef2cd0b29b7822eea544ab8c6c4a93cb67b --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os.c | 1 + test/permutations.test | 2 +- test/speed1p.test | 1 + 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 143bb5bf58..e0bcdeed1d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\ssignature\sof\sthe\sxUnfetch\smethod\sto\s"int\s(*xUnfetch)(sqlite3_file*,\ssqlite3_int64\siOfst,\svoid\s*p)". -D 2013-03-25T17:00:24.194 +C Simulate\sOOM\serrors\sin\sthe\ssqlite3OsFetch()\sfunction.\sRun\smalloc.test\sas\spart\sof\sthe\s"mmap"\spermutation. +D 2013-03-25T18:25:49.003 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c 17c6e6e4a17135ec1c7c468f8d0b97ec6f8a9264 +F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 664fc178a9478246f6c84e46292e77fb5bee4431 @@ -657,7 +657,7 @@ F test/pageropt.test 290cd59782b1890f02bb33795571facfc5ccac43 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test cf5d475f024cfd3fbb0255b8e43bf0c6887c6bb4 +F test/permutations.test 5b5397f33b9d129b734b9c57a5091ad494e025c5 F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -730,7 +730,7 @@ F test/softheap1.test c16709a16ad79fa43b32929b2e623d1d117ccf53 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb -F test/speed1p.test c4a469f29f135f4d76c55b1f2a52f36e209466cc +F test/speed1p.test 9912e8f915f1de2c82fc019a63920bf9dcfc69f7 F test/speed2.test 53177056baf6556dcbdcf032bbdfc41c1aa74ded F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P dce35c01a5fe66d2970075b1e3f0376026485e4c -R 12377e21e3793332cede5cbfb008d2ac +P 115b830509e8f0aa9d5965c1e9cd4f2ed9d01938 +R 81bdcfaea4ca0021abe5572b7c1cd51d U dan -Z 1c478466922e4f7b4b376664d80146c7 +Z 1c0203030ba88df7bcf075001cefd19c diff --git a/manifest.uuid b/manifest.uuid index 838ea0b60b..2ac770c1e3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -115b830509e8f0aa9d5965c1e9cd4f2ed9d01938 \ No newline at end of file +77443ef2cd0b29b7822eea544ab8c6c4a93cb67b \ No newline at end of file diff --git a/src/os.c b/src/os.c index f2d85bba4d..2ba857e2aa 100644 --- a/src/os.c +++ b/src/os.c @@ -142,6 +142,7 @@ int sqlite3OsShmMap( } int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + DO_OS_MALLOC_TEST(id); return id->pMethods->xFetch(id, iOff, iAmt, pp); } int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ diff --git a/test/permutations.test b/test/permutations.test index 6e6b5c90f7..9ff3a801c2 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -143,7 +143,7 @@ test_suite "mmap" -prefix "mm-" -description { } -presql { pragma mmap_size = -65536; } -files [ - test_set $allquicktests -exclude *malloc* *ioerr* *fault* + test_set $allquicktests -exclude *malloc* *ioerr* *fault* -include malloc.test ] test_suite "valgrind" -prefix "" -description { diff --git a/test/speed1p.test b/test/speed1p.test index 915f165354..df5abedde3 100644 --- a/test/speed1p.test +++ b/test/speed1p.test @@ -65,6 +65,7 @@ proc number_name {n} { # do_test speed1p-1.0 { execsql { + PRAGMA mmap_size=1000000; PRAGMA page_size=1024; PRAGMA cache_size=500; PRAGMA locking_mode=EXCLUSIVE; From 893c0ffc290bb59bcd8f505bec45087211579525 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 19:05:07 +0000 Subject: [PATCH 038/204] Add a test that simulates an error in mmap(). FossilOrigin-Name: 6ec7367d8e98425f00eeb8215ca8964313c1d0b7 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/os_unix.c | 7 +++++-- src/test_syscall.c | 20 +++++++++++++++++++- test/malloc_common.tcl | 2 +- test/sysfault.test | 31 +++++++++++++++++++++++++++++++ 6 files changed, 66 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index e0bcdeed1d..215ae805d4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simulate\sOOM\serrors\sin\sthe\ssqlite3OsFetch()\sfunction.\sRun\smalloc.test\sas\spart\sof\sthe\s"mmap"\spermutation. -D 2013-03-25T18:25:49.003 +C Add\sa\stest\sthat\ssimulates\san\serror\sin\smmap(). +D 2013-03-25T19:05:07.846 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 664fc178a9478246f6c84e46292e77fb5bee4431 +F src/os_unix.c 6815e3dd9a1507f2e36ba5a8dcd36b67ac60f3b0 F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac F src/pager.c d0cdf47664e487f735b4bea485ee70637ed34a55 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -226,7 +226,7 @@ F src/test_spellfix.c 56dfa6d583ac34f61af0834d7b58d674e7e18e13 F src/test_sqllog.c 8acb843ddb9928dea8962e31bb09f421a72ffccb F src/test_stat.c d1569c7a4839f13e80187e2c26b2ab4da2d03935 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd -F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae +F src/test_syscall.c fe018f95049d4f24e036d1d649516b60689b4e57 F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c e286f2173563f2a1747c24bcda6b9d030bf4f4e4 F src/test_vfs.c fb16b2d9938cf0c1afc5a423b55b952fcc024275 @@ -611,7 +611,7 @@ F test/mallocH.test 79b65aed612c9b3ed2dcdaa727c85895fd1bfbdb F test/mallocI.test a88c2b9627c8506bf4703d8397420043a786cdb6 F test/mallocJ.test b5d1839da331d96223e5f458856f8ffe1366f62e F test/mallocK.test d79968641d1b70d88f6c01bdb9a7eb4a55582cc9 -F test/malloc_common.tcl 2930895b0962823ec679853e67e58dd6d8198b3c +F test/malloc_common.tcl 9a98856549bfb3fab205edbc1317216edc52e70d F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f F test/memdb.test 708a028d6d373e5b3842e4bdc8ba80998c9a4da6 @@ -747,7 +747,7 @@ F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test bea9bf329bff733c791310244617c2a76974e64a -F test/sysfault.test c79441d88d23696fbec7b147dba98d42a04f523f +F test/sysfault.test 3258d6c8213b44f2669994d798a5eb5327b40146 F test/table.test a59d985ca366e39b17b175f387f9d5db5a18d4e2 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 115b830509e8f0aa9d5965c1e9cd4f2ed9d01938 -R 81bdcfaea4ca0021abe5572b7c1cd51d +P 77443ef2cd0b29b7822eea544ab8c6c4a93cb67b +R 602dbedb509e1b6f815316de036c2541 U dan -Z 1c0203030ba88df7bcf075001cefd19c +Z 3ee6172e1f5d6ed5486bb4fa69d324f2 diff --git a/manifest.uuid b/manifest.uuid index 2ac770c1e3..50a8aea1fc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -77443ef2cd0b29b7822eea544ab8c6c4a93cb67b \ No newline at end of file +6ec7367d8e98425f00eeb8215ca8964313c1d0b7 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 00c0088f8a..8175bd5c9a 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -444,6 +444,9 @@ static struct unix_syscall { { "fchown", (sqlite3_syscall_ptr)posixFchown, 0 }, #define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) + { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, +#define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[21].pCurrent) + }; /* End of the overrideable system calls */ /* @@ -4547,7 +4550,7 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ void *pNew; int flags = PROT_READ; if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - pNew = mmap(0, nMap, flags, MAP_SHARED, pFd->h, 0); + pNew = osMmap(0, nMap, flags, MAP_SHARED, pFd->h, 0); if( pNew==MAP_FAILED ){ return SQLITE_IOERR_MMAP; } @@ -7186,7 +7189,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==21 ); + assert( ArraySize(aSyscall)==22 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ diff --git a/src/test_syscall.c b/src/test_syscall.c index d484f22db4..0b053fc599 100644 --- a/src/test_syscall.c +++ b/src/test_syscall.c @@ -23,7 +23,7 @@ ** ** open close access getcwd stat fstat ** ftruncate fcntl read pread pread64 write -** pwrite pwrite64 fchmod fallocate +** pwrite pwrite64 fchmod fallocate mmap ** ** test_syscall uninstall ** Uninstall all wrapper functions. @@ -81,6 +81,7 @@ /* From test1.c */ extern const char *sqlite3TestErrorName(int); +#include #include #include @@ -106,6 +107,7 @@ static int ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off); static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, off_t off); static int ts_fchmod(int fd, mode_t mode); static int ts_fallocate(int fd, off_t off, off_t len); +static void *ts_mmap(void *, size_t, int, int, int, off_t); struct TestSyscallArray { @@ -131,6 +133,7 @@ struct TestSyscallArray { /* 13 */ { "pwrite64", (sqlite3_syscall_ptr)ts_pwrite64, 0, 0, 0 }, /* 14 */ { "fchmod", (sqlite3_syscall_ptr)ts_fchmod, 0, 0, 0 }, /* 15 */ { "fallocate", (sqlite3_syscall_ptr)ts_fallocate, 0, 0, 0 }, + /* 16 */ { "mmap", (sqlite3_syscall_ptr)ts_mmap, 0, 0, 0 }, { 0, 0, 0, 0, 0 } }; @@ -152,6 +155,7 @@ struct TestSyscallArray { aSyscall[13].xOrig) #define orig_fchmod ((int(*)(int,mode_t))aSyscall[14].xOrig) #define orig_fallocate ((int(*)(int,off_t,off_t))aSyscall[15].xOrig) +#define orig_mmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[16].xOrig) /* ** This function is called exactly once from within each invocation of a @@ -377,6 +381,20 @@ static int ts_fallocate(int fd, off_t off, off_t len){ return orig_fallocate(fd, off, len); } +static void *ts_mmap( + void *pAddr, + size_t nByte, + int prot, + int flags, + int fd, + off_t iOff +){ + if( tsIsFailErrno("mmap") ){ + return MAP_FAILED; + } + return orig_mmap(pAddr, nByte, prot, flags, fd, iOff); +} + static int test_syscall_install( void * clientData, Tcl_Interp *interp, diff --git a/test/malloc_common.tcl b/test/malloc_common.tcl index 5937b958f8..2ac619b1df 100644 --- a/test/malloc_common.tcl +++ b/test/malloc_common.tcl @@ -264,7 +264,7 @@ proc faultsim_test_result_int {args} { set t [list $testrc $testresult] set r $args if { ($testnfail==0 && $t != [lindex $r 0]) || [lsearch $r $t]<0 } { - error "nfail=$testnfail rc=$testrc result=$testresult" + error "nfail=$testnfail rc=$testrc result=$testresult list=$r" } } diff --git a/test/sysfault.test b/test/sysfault.test index 07d525ca9e..bf7df5df5f 100644 --- a/test/sysfault.test +++ b/test/sysfault.test @@ -243,5 +243,36 @@ do_faultsim_test 3 -faults vfsfault-* -prep { faultsim_test_result {0 20000} } +#------------------------------------------------------------------------- +# Test errors in mmap(). +# +proc vfsfault_install {} { + test_syscall reset + test_syscall install {mmap} +} + +faultsim_delete_and_reopen +execsql { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); +} +faultsim_save_and_close + +do_faultsim_test 4 -faults vfsfault-* -prep { + faultsim_restore_and_reopen + file_control_chunksize_test db main 8192 + execsql { + PRAGMA mmap_size = 1000000; + } +} -body { + test_syscall errno mmap EACCES + + execsql { + SELECT * FROM t1; + } +} -test { + faultsim_test_result {0 {1 2}} {1 {disk I/O error}} +} + finish_test From b7e3a326fe84ff10e9b34a9c11c32e441ac0fc5b Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 20:30:13 +0000 Subject: [PATCH 039/204] Use mremap() on Linux. FossilOrigin-Name: 431aecc8600c29c203546e48d256510510238887 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 31 +++++++++++++++++++------------ 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 920c0ece2f..02d611b4ed 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\srecent\strunk\schanges\sinto\sthe\sexperimental-mmap\sbranch. -D 2013-03-25T19:57:26.572 +C Use\smremap()\son\sLinux. +D 2013-03-25T20:30:13.313 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 54b82f341b3c81630aa8c73efd2effe037b69270 +F src/os_unix.c a4d7076a6edd4e6a8594191880e540a31b2c8598 F src/os_win.c e4f17ddf79f2a9373e33ed70565e765d65324589 F src/pager.c 4e1b55e7a50db0466b06f8ec114799b432c18131 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 6ec7367d8e98425f00eeb8215ca8964313c1d0b7 5062db672c00c3365d51cd6b39815078f5b6b525 -R 868df99cd28788ba5be1e70b11892bee -U drh -Z 7385c5f656a5c7af6e1d7e1f0ea19ddb +P a607d63f0b6a3d3785e9385187d3e6b92e14fc70 +R 18e19b2d7d709a7b2b83b278bfc05f98 +U dan +Z f03c297d3465d1cb9dc069838022accc diff --git a/manifest.uuid b/manifest.uuid index 0965ca8390..e4f7651d39 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a607d63f0b6a3d3785e9385187d3e6b92e14fc70 \ No newline at end of file +431aecc8600c29c203546e48d256510510238887 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 9d2e2c130b..33be793423 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4544,21 +4544,28 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ } if( nMap!=pFd->mmapSize ){ - unixUnmapfile(pFd); + void *pNew = 0; - if( nMap>0 ){ - void *pNew; - int flags = PROT_READ; - if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - pNew = osMmap(0, nMap, flags, MAP_SHARED, pFd->h, 0); - if( pNew==MAP_FAILED ){ - return SQLITE_IOERR_MMAP; +#if defined(__linux__) && defined(_GNU_SOURCE) + if( pFd->pMapRegion && nMap>0 ){ + pNew = mremap(pFd->pMapRegion, pFd->mmapOrigsize, nMap, MREMAP_MAYMOVE); + }else +#endif + { + unixUnmapfile(pFd); + if( nMap>0 ){ + int flags = PROT_READ; + if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; + pNew = osMmap(0, nMap, flags, MAP_SHARED, pFd->h, 0); } - - pFd->pMapRegion = pNew; - pFd->mmapSize = nMap; - pFd->mmapOrigsize = nMap; } + + if( pNew==MAP_FAILED ){ + return SQLITE_IOERR_MMAP; + } + pFd->pMapRegion = pNew; + pFd->mmapSize = nMap; + pFd->mmapOrigsize = nMap; } return SQLITE_OK; From d1ab8065c1039db43eb413702cafb3baa500d69a Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Mar 2013 20:50:25 +0000 Subject: [PATCH 040/204] Add munmap and mremap to the set of os interfaces that can be overloaded in os_unix.c. FossilOrigin-Name: 8776047bd776bbf266eb9c3b56683badb84ae73e --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os_unix.c | 20 +++++++++++++++----- test/syscall.test | 2 +- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 02d611b4ed..b0a5aaeb5f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\smremap()\son\sLinux. -D 2013-03-25T20:30:13.313 +C Add\smunmap\sand\smremap\sto\sthe\sset\sof\sos\sinterfaces\sthat\scan\sbe\soverloaded\nin\sos_unix.c. +D 2013-03-25T20:50:25.204 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c a4d7076a6edd4e6a8594191880e540a31b2c8598 +F src/os_unix.c 6b3f972a8b515ce151d1d075080b10d52adbfbd6 F src/os_win.c e4f17ddf79f2a9373e33ed70565e765d65324589 F src/pager.c 4e1b55e7a50db0466b06f8ec114799b432c18131 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -747,7 +747,7 @@ F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 -F test/syscall.test cf59bafccb143f8df21c043b3201d5afa9fe2474 +F test/syscall.test a653783d985108c4912cc64d341ffbbb55ad2806 F test/sysfault.test 3258d6c8213b44f2669994d798a5eb5327b40146 F test/table.test a59d985ca366e39b17b175f387f9d5db5a18d4e2 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a607d63f0b6a3d3785e9385187d3e6b92e14fc70 -R 18e19b2d7d709a7b2b83b278bfc05f98 -U dan -Z f03c297d3465d1cb9dc069838022accc +P 431aecc8600c29c203546e48d256510510238887 +R 420b4ff666854b5074068c14722ea276 +U drh +Z feb03effee9a1e5215de46a6338d0c52 diff --git a/manifest.uuid b/manifest.uuid index e4f7651d39..5b13e95f75 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -431aecc8600c29c203546e48d256510510238887 \ No newline at end of file +8776047bd776bbf266eb9c3b56683badb84ae73e \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 33be793423..625f413406 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -447,6 +447,16 @@ static struct unix_syscall { { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, #define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[21].pCurrent) + { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, +#define osMunmap ((void*(*)(void*,size_t))aSyscall[22].pCurrent) + +#if defined(__linux__) && defined(_GNU_SOURCE) + { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, +#else + { "mremap", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent) + }; /* End of the overrideable system calls */ /* @@ -4005,7 +4015,7 @@ static void unixShmPurge(unixFile *pFd){ sqlite3_mutex_free(p->mutex); for(i=0; inRegion; i++){ if( p->h>=0 ){ - munmap(p->apRegion[i], p->szRegion); + osMunmap(p->apRegion[i], p->szRegion); }else{ sqlite3_free(p->apRegion[i]); } @@ -4278,7 +4288,7 @@ static int unixShmMap( while(pShmNode->nRegion<=iRegion){ void *pMem; if( pShmNode->h>=0 ){ - pMem = mmap(0, szRegion, + pMem = osMmap(0, szRegion, pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, MAP_SHARED, pShmNode->h, szRegion*(i64)pShmNode->nRegion ); @@ -4501,7 +4511,7 @@ static int unixShmUnmap( static void unixUnmapfile(unixFile *pFd){ assert( pFd->nFetchOut==0 ); if( pFd->pMapRegion ){ - munmap(pFd->pMapRegion, pFd->mmapOrigsize); + osMunmap(pFd->pMapRegion, pFd->mmapOrigsize); pFd->pMapRegion = 0; pFd->mmapSize = 0; pFd->mmapOrigsize = 0; @@ -4548,7 +4558,7 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ #if defined(__linux__) && defined(_GNU_SOURCE) if( pFd->pMapRegion && nMap>0 ){ - pNew = mremap(pFd->pMapRegion, pFd->mmapOrigsize, nMap, MREMAP_MAYMOVE); + pNew = osMremap(pFd->pMapRegion, pFd->mmapOrigsize, nMap, MREMAP_MAYMOVE); }else #endif { @@ -7196,7 +7206,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==22 ); + assert( ArraySize(aSyscall)==24 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ diff --git a/test/syscall.test b/test/syscall.test index b9a3b196e3..5bf1225867 100644 --- a/test/syscall.test +++ b/test/syscall.test @@ -60,7 +60,7 @@ foreach s { open close access getcwd stat fstat ftruncate fcntl read pread write pwrite fchmod fallocate pread64 pwrite64 unlink openDirectory mkdir rmdir - statvfs fchown umask mmap + statvfs fchown umask mmap munmap mremap } { if {[test_syscall exists $s]} {lappend syscall_list $s} } From 0d0614bdc6e59c1cb52bc79fdf8dafbbc78f57f9 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Mar 2013 23:09:28 +0000 Subject: [PATCH 041/204] Memory-mapped I/O is now on by default. The "PRAGMA mmap_limit(N)" can be used to issue a hint to the VFS to limit mmap space to N bytes. The VFS is free to ignore that hint if desired. However, if "PRAGMA mmap_limit(0)" is used, xFetch is never called. FossilOrigin-Name: 1b37c4effdd03aa2ea938a71b4f22ed27391689b --- manifest | 42 +++++++++++++++++++++--------------------- manifest.uuid | 2 +- src/btree.c | 4 ++-- src/btree.h | 2 +- src/os_unix.c | 15 ++++++++------- src/pager.c | 22 +++++++--------------- src/pager.h | 2 +- src/pragma.c | 25 +++++++++++-------------- src/sqlite.h.in | 8 ++++---- src/sqliteInt.h | 1 - src/sqliteLimit.h | 7 +++++++ test/dbstatus2.test | 2 +- test/incrblob.test | 8 ++++---- test/mmap1.test | 21 ++++++++++----------- test/permutations.test | 6 +++--- test/speed1p.test | 1 - test/sysfault.test | 3 +-- test/wal5.test | 5 ++--- 18 files changed, 84 insertions(+), 92 deletions(-) diff --git a/manifest b/manifest index b0a5aaeb5f..65993928dd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smunmap\sand\smremap\sto\sthe\sset\sof\sos\sinterfaces\sthat\scan\sbe\soverloaded\nin\sos_unix.c. -D 2013-03-25T20:50:25.204 +C Memory-mapped\sI/O\sis\snow\son\sby\sdefault.\s\sThe\s"PRAGMA\smmap_limit(N)"\scan\sbe\nused\sto\sissue\sa\shint\sto\sthe\sVFS\sto\slimit\smmap\sspace\sto\sN\sbytes.\s\sThe\sVFS\nis\sfree\sto\signore\sthat\shint\sif\sdesired.\s\sHowever,\sif\s"PRAGMA\smmap_limit(0)"\nis\sused,\sxFetch\sis\snever\scalled. +D 2013-03-25T23:09:28.646 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,8 +121,8 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c ef238f2d88c6072cc0d6cff33b72c2a9044a15e4 -F src/btree.h d3259057a38494c4c4358e377032158c762e3d8b +F src/btree.c 771f0391556317de19e02c9533b256e8299dd59b +F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc @@ -160,15 +160,15 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 6b3f972a8b515ce151d1d075080b10d52adbfbd6 +F src/os_unix.c 57306f1d2a2d783dae4365446b9f9a5da8958559 F src/os_win.c e4f17ddf79f2a9373e33ed70565e765d65324589 -F src/pager.c 4e1b55e7a50db0466b06f8ec114799b432c18131 -F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c +F src/pager.c c532d51ccc74de8551a28eb932f3b1a1034017d3 +F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c 86c8088ac6a12d3f3be5f7394542651f03fa9a38 +F src/pragma.c 8875d32d54530300e6aa31a9d5426b2088a69b12 F src/prepare.c 310eaff1ee5f3c700b3545afb095cfe9346efc3a F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -176,11 +176,11 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in 457eb0ba9fbc331281dca53182c90035b087527f +F src/sqlite.h.in 921ba42ec3531fe8cf021492ffd781b161383f1b F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 -F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796 -F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d +F src/sqliteInt.h 0f8f05ee4db4ba9120b38f7a3992b325698f6e8a +F src/sqliteLimit.h 83d0cd0ce3050a80930406d3c7aecb48c1ccdfc5 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 @@ -369,7 +369,7 @@ F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/date.test f3228180c87bbe5d39c9397bf001c0095c3821b9 F test/dbstatus.test 207e5b63fcb7b9c3bb8e1fdf38ebd4654ad0e54b -F test/dbstatus2.test 10418e62b3db5dca070f0c3eef3ea13946f339c2 +F test/dbstatus2.test f329941d5f4a8bc0ba6ec5735897ef0cf34e2f5f F test/default.test 6faf23ccb300114924353007795aa9a8ec0aa9dc F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa @@ -526,7 +526,7 @@ F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test 64f3cc1acde1b9161ccdd8e5bde3daefdb5b2617 F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 -F test/incrblob.test e81846d214f3637622620fbde7cd526781cfe328 +F test/incrblob.test 2100cb8964e4a106e5ca9bf80e2c5c3e8be33f77 F test/incrblob2.test edc3a96e557bd61fb39acc8d2edd43371fbbaa19 F test/incrblob3.test aedbb35ea1b6450c33b98f2b6ed98e5020be8dc7 F test/incrblob4.test 09be37d3dd996a31ea6993bba7837ece549414a8 @@ -630,7 +630,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test e3391ef6c069e5f5733b8339652ce37fd97e3487 +F test/mmap1.test 9fb71daaca72f66fcc7a3b3a37fc3db99988b57a F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 @@ -658,7 +658,7 @@ F test/pageropt.test 290cd59782b1890f02bb33795571facfc5ccac43 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test 5b5397f33b9d129b734b9c57a5091ad494e025c5 +F test/permutations.test d298a74abae42b72b3640dc5505d19d8a3eddc9e F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -731,7 +731,7 @@ F test/softheap1.test c16709a16ad79fa43b32929b2e623d1d117ccf53 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb -F test/speed1p.test 9912e8f915f1de2c82fc019a63920bf9dcfc69f7 +F test/speed1p.test c4a469f29f135f4d76c55b1f2a52f36e209466cc F test/speed2.test 53177056baf6556dcbdcf032bbdfc41c1aa74ded F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 @@ -748,7 +748,7 @@ F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test a653783d985108c4912cc64d341ffbbb55ad2806 -F test/sysfault.test 3258d6c8213b44f2669994d798a5eb5327b40146 +F test/sysfault.test 503f72712b2b21cb80dc9899e53c2e39484d0313 F test/table.test a59d985ca366e39b17b175f387f9d5db5a18d4e2 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 @@ -955,7 +955,7 @@ F test/wal.test 5baa9a0e682c0394012aa9a06a3d06cf92655fbf F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c -F test/wal5.test 091bf7500e97087f1a87b346c8707e93f42d4f80 +F test/wal5.test f4d0aee6a2cf09e326ed2459011d396b4fdf661a F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test b3ee739fe8f7586aaebdc2367f477ebcf3e3b034 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 431aecc8600c29c203546e48d256510510238887 -R 420b4ff666854b5074068c14722ea276 +P 8776047bd776bbf266eb9c3b56683badb84ae73e +R 1fba4ef23570acddc66df7d627b1cb5f U drh -Z feb03effee9a1e5215de46a6338d0c52 +Z c62aae7ee656f659e8aebd3681920a9a diff --git a/manifest.uuid b/manifest.uuid index 5b13e95f75..8e0647f651 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8776047bd776bbf266eb9c3b56683badb84ae73e \ No newline at end of file +1b37c4effdd03aa2ea938a71b4f22ed27391689b \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 59970ded7f..04cc20d5c5 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2134,11 +2134,11 @@ int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ ** Change the limit on the amount of the database file that may be ** memory mapped. */ -int sqlite3BtreeSetMmapSize(Btree *p, int nMap){ +int sqlite3BtreeSetMmapLimit(Btree *p, sqlite3_int64 mxMmap){ BtShared *pBt = p->pBt; assert( sqlite3_mutex_held(p->db->mutex) ); sqlite3BtreeEnter(p); - sqlite3PagerSetMmapsize(pBt->pPager, nMap); + sqlite3PagerSetMmapLimit(pBt->pPager, mxMmap); sqlite3BtreeLeave(p); return SQLITE_OK; } diff --git a/src/btree.h b/src/btree.h index f2cca2fc71..23f6ad7068 100644 --- a/src/btree.h +++ b/src/btree.h @@ -63,7 +63,7 @@ int sqlite3BtreeOpen( int sqlite3BtreeClose(Btree*); int sqlite3BtreeSetCacheSize(Btree*,int); -int sqlite3BtreeSetMmapSize(Btree*, int); +int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); int sqlite3BtreeSetSafetyLevel(Btree*,int,int,int); int sqlite3BtreeSyncDisabled(Btree*); int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); diff --git a/src/os_unix.c b/src/os_unix.c index 625f413406..8a8516bcda 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -225,6 +225,11 @@ struct unixFile { const char *zPath; /* Name of the file */ unixShm *pShm; /* Shared memory segment information */ int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ + int nFetchOut; /* Number of outstanding xFetch refs */ + sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ + sqlite3_int64 mmapOrigsize; /* Actual size of mapping at pMapRegion */ + sqlite3_int64 mmapLimit; /* Configured FCNTL_MMAP_LIMIT value */ + void *pMapRegion; /* Memory mapped region */ #ifdef __QNXNTO__ int sectorSize; /* Device sector size */ int deviceCharacteristics; /* Precomputed device characteristics */ @@ -251,11 +256,6 @@ struct unixFile { unsigned char inNormalWrite; /* True if in a normal write operation */ #endif - sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ - sqlite3_int64 mmapOrigsize; /* Actual size of mapping at pMapRegion */ - sqlite3_int64 mmapLimit; /* Configured FCNTL_MMAP_SIZE value */ - void *pMapRegion; /* Memory mapped region */ - int nFetchOut; /* Number of outstanding xFetch refs */ #ifdef SQLITE_TEST /* In test mode, increase the size of this structure a bit so that @@ -3699,7 +3699,7 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } - case SQLITE_FCNTL_MMAP_SIZE: { + case SQLITE_FCNTL_MMAP_LIMIT: { pFile->mmapLimit = *(i64*)pArg; return SQLITE_OK; } @@ -4528,7 +4528,7 @@ static void unixUnmapfile(unixFile *pFd){ ** the mapping to create. Otherwise, if nByte is less than zero, then the ** requested size is the size of the file on disk. The actual size of the ** created mapping is either the requested size or the value configured -** using SQLITE_FCNTL_MMAP_SIZE, whichever is smaller. +** using SQLITE_FCNTL_MMAP_LIMIT, whichever is smaller. ** ** SQLITE_OK is returned if no error occurs (even if the mapping is not ** recreated as a result of outstanding references) or an SQLite error @@ -4969,6 +4969,7 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; + pNew->mmapLimit = SQLITE_DEFAULT_MMAP_LIMIT; if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; diff --git a/src/pager.c b/src/pager.c index b7534deb09..4cce844237 100644 --- a/src/pager.c +++ b/src/pager.c @@ -657,8 +657,8 @@ struct Pager { char dbFileVers[16]; /* Changes whenever database file changes */ u8 bUseFetch; /* True to use xFetch() */ - int nMapCfgLimit; /* Configured limit value */ int nMmapOut; /* Number of mmap pages currently outstanding */ + sqlite3_int64 mxMmap; /* Desired maximum mmap size */ PgHdr *pFree; /* List of free mmap page headers (pDirty) */ /* ** End of the routinely-changing class members @@ -3354,23 +3354,15 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ } /* -** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of nMapCfgLimit. +** Invoke SQLITE_FCNTL_MMAP_LIMIT based on the current value of mxMmap. */ static void pagerFixMaplimit(Pager *pPager){ sqlite3_file *fd = pPager->fd; if( isOpen(fd) ){ - pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->nMapCfgLimit!=0; + pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->mxMmap>0; if( pPager->bUseFetch ){ - void *p; - i64 nMapLimit; - if( pPager->nMapCfgLimit<0 ){ - nMapLimit = (i64)pPager->nMapCfgLimit * -1024; - }else{ - nMapLimit = (i64)pPager->nMapCfgLimit * pPager->pageSize; - } - - p = (void *)&nMapLimit; - sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_SIZE, p); + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT, + (void*)&pPager->mxMmap); } } } @@ -3378,8 +3370,8 @@ static void pagerFixMaplimit(Pager *pPager){ /* ** Change the maximum size of any memory mapping made of the database file. */ -void sqlite3PagerSetMmapsize(Pager *pPager, int nMap){ - pPager->nMapCfgLimit = nMap; +void sqlite3PagerSetMmapLimit(Pager *pPager, sqlite3_int64 mxMmap){ + pPager->mxMmap = mxMmap; pagerFixMaplimit(pPager); } diff --git a/src/pager.h b/src/pager.h index 81ab30c115..6f659136e2 100644 --- a/src/pager.h +++ b/src/pager.h @@ -108,7 +108,7 @@ void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); int sqlite3PagerSetPagesize(Pager*, u32*, int); int sqlite3PagerMaxPageCount(Pager*, int); void sqlite3PagerSetCachesize(Pager*, int); -void sqlite3PagerSetMmapsize(Pager *, int); +void sqlite3PagerSetMmapLimit(Pager *, sqlite3_int64); void sqlite3PagerShrink(Pager*); void sqlite3PagerSetSafetyLevel(Pager*,int,int,int); int sqlite3PagerLockingMode(Pager *, int); diff --git a/src/pragma.c b/src/pragma.c index 9b31797d3f..cd1cfbe213 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -745,26 +745,23 @@ void sqlite3Pragma( }else /* - ** PRAGMA [database.]mmap_size - ** PRAGMA [database.]mmap_size=N + ** PRAGMA [database.]mmap_limit(N) ** - ** Used to set or query the mapping size limit. The mapping size limit is + ** Used to set mapping size limit. The mapping size limit is ** used to limit the aggregate size of all memory mapped regions of the ** database file. If this parameter is set to zero, then memory mapping - ** is not used at all. If it is set to a positive value, then it is - ** interpreted as a maximum size in pages. If set to less than zero, then - ** the absolute value is interpreted as a size limit in KB. + ** is not used at all. The parameter N is measured in bytes. ** - ** The default value is zero (do not use memory mapped IO). + ** This value is advisory. The underlying VFS is free to memory map + ** as little or as much as it wants. Except, if N is set to 0 then the + ** upper layers will never invoke the xFetch interfaces to the VFS. */ - if( sqlite3StrICmp(zLeft,"mmap_size")==0 ){ + if( sqlite3StrICmp(zLeft,"mmap_limit")==0 ){ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - if( !zRight ){ - returnSingleInt(pParse, "mmap_size", pDb->pSchema->mmap_size); - }else{ - int size = sqlite3Atoi(zRight); - pDb->pSchema->mmap_size = size; - sqlite3BtreeSetMmapSize(pDb->pBt, pDb->pSchema->mmap_size); + if( zRight ){ + sqlite3_int64 size; + sqlite3Atoi64(zRight, &size, 1000, SQLITE_UTF8); + sqlite3BtreeSetMmapLimit(pDb->pBt, size); } }else diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 72f8d32939..acb49bf5f9 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -886,9 +886,9 @@ struct sqlite3_io_methods { ** written into memory obtained from [sqlite3_malloc()]. The caller should ** invoke [sqlite3_free()] on the result to avoid a memory leak. ** -**
  • [[SQLITE_FCNTL_MMAP_SIZE]] -** The argument is assumed to point to a value of type sqlite3_int64. An -** advisory maximum amount of this file to memory map in bytes. +**
  • [[SQLITE_FCNTL_MMAP_LIMIT]] +** The argument is assumed to pointer to a value of type sqlite3_int64 that +** is an advisory maximum number of bytes in the file to memory map. ** ** */ @@ -908,7 +908,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_PRAGMA 14 #define SQLITE_FCNTL_BUSYHANDLER 15 #define SQLITE_FCNTL_TEMPFILENAME 16 -#define SQLITE_FCNTL_MMAP_SIZE 18 +#define SQLITE_FCNTL_MMAP_LIMIT 18 /* ** CAPI3REF: Mutex Handle diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 59483bc87b..129c4c5ea5 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -746,7 +746,6 @@ struct Schema { u8 enc; /* Text encoding used by this database */ u16 flags; /* Flags associated with this schema */ int cache_size; /* Number of pages to use in the cache */ - int mmap_size; /* Number of pages to memory map */ }; /* diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index c7aee53ceb..2013139456 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -206,3 +206,10 @@ #ifndef SQLITE_MAX_TRIGGER_DEPTH # define SQLITE_MAX_TRIGGER_DEPTH 1000 #endif + +/* +** Default maximum size of memory used by xFetch in the VFS. +*/ +#ifndef SQLITE_DEFAULT_MMAP_LIMIT +# define SQLITE_DEFAULT_MMAP_LIMIT (256*1024*1024) +#endif diff --git a/test/dbstatus2.test b/test/dbstatus2.test index 2541a1a823..c2d9164fa2 100644 --- a/test/dbstatus2.test +++ b/test/dbstatus2.test @@ -40,7 +40,7 @@ proc db_write {db {reset 0}} { do_test 1.1 { db close sqlite3 db test.db - execsql { PRAGMA mmap_size = 0 } + execsql { PRAGMA mmap_limit = 0 } expr {[file size test.db] / 1024} } 6 diff --git a/test/incrblob.test b/test/incrblob.test index 4277e5c4c1..fa564de0cf 100644 --- a/test/incrblob.test +++ b/test/incrblob.test @@ -123,7 +123,7 @@ foreach AutoVacuumMode [list 0 1] { forcedelete test.db test.db-journal sqlite3 db test.db - execsql "PRAGMA mmap_size = 0" + execsql "PRAGMA mmap_limit = 0" execsql "PRAGMA auto_vacuum = $AutoVacuumMode" do_test incrblob-2.$AutoVacuumMode.1 { @@ -150,7 +150,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db - execsql "PRAGMA mmap_size = 0" + execsql "PRAGMA mmap_limit = 0" # Read the last 20 bytes of the blob via a blob handle. set ::blob [db incrblob blobs v 1] @@ -173,7 +173,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db - execsql "PRAGMA mmap_size = 0" + execsql "PRAGMA mmap_limit = 0" # Write the second-to-last 20 bytes of the blob via a blob handle. # @@ -203,7 +203,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db - execsql { PRAGMA mmap_size = 0 } + execsql { PRAGMA mmap_limit = 0 } execsql { SELECT i FROM blobs } } {45} diff --git a/test/mmap1.test b/test/mmap1.test index 41714e29d8..d97ada7e57 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -23,16 +23,16 @@ proc nRead {db} { return $stats(read) } -foreach {t mmap_size nRead c2init} { - 1.1 { PRAGMA mmap_size = -65536 } 4 {} - 1.2 { PRAGMA mmap_size = -50 } 156 {} - 1.3 { PRAGMA mmap_size = 0 } 344 {} - 1.4 { PRAGMA mmap_size = -65536 } 4 {PRAGMA mmap_size = -65536} - 1.5 { PRAGMA mmap_size = -50 } 156 {PRAGMA mmap_size = -65536} - 1.6 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = -65536} +foreach {t mmap_limit nRead c2init} { + 1.1 { PRAGMA mmap_limit = 70000000 } 4 {} + 1.2 { PRAGMA mmap_limit = 51200 } 156 {} + 1.3 { PRAGMA mmap_limit = 0 } 344 {} + 1.4 { PRAGMA mmap_limit = 70000000 } 4 {PRAGMA mmap_limit = 70000000 } + 1.5 { PRAGMA mmap_limit = 51200 } 156 {PRAGMA mmap_limit = 70000000 } + 1.6 { PRAGMA mmap_limit = 0 } 344 {PRAGMA mmap_limit = 70000000 } } { do_multiclient_test tn { - sql1 $mmap_size + sql1 $mmap_limit sql2 $c2init code2 { @@ -78,7 +78,7 @@ foreach {t mmap_size nRead c2init} { } {64 ok 149} # Check that the number of pages read by connection 1 indicates that the - # "PRAGMA mmap_size" command worked. + # "PRAGMA mmap_limit" command worked. do_test $t.$tn.5 { nRead db } $nRead } } @@ -95,7 +95,7 @@ db func rblob rblob do_execsql_test 2.1 { PRAGMA auto_vacuum = 1; - PRAGMA mmap_size = -65536; + PRAGMA mmap_limit = 70000000; PRAGMA journal_mode = wal; CREATE TABLE t1(a, b, UNIQUE(a, b)); INSERT INTO t1 VALUES(rblob(500), rblob(500)); @@ -130,4 +130,3 @@ do_execsql_test 2.4 { } {0 24 24} finish_test - diff --git a/test/permutations.test b/test/permutations.test index 9ff3a801c2..2e529fa880 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -138,10 +138,10 @@ test_suite "veryquick" -prefix "" -description { test_set $allquicktests -exclude *malloc* *ioerr* *fault* ] -test_suite "mmap" -prefix "mm-" -description { - Similar to veryquick. Except with memory mapping enabled. +test_suite "no-mmap" -prefix "nomm-" -description { + Similar to veryquick. Except with memory mapping disabled. } -presql { - pragma mmap_size = -65536; + pragma mmap_size = 0; } -files [ test_set $allquicktests -exclude *malloc* *ioerr* *fault* -include malloc.test ] diff --git a/test/speed1p.test b/test/speed1p.test index df5abedde3..915f165354 100644 --- a/test/speed1p.test +++ b/test/speed1p.test @@ -65,7 +65,6 @@ proc number_name {n} { # do_test speed1p-1.0 { execsql { - PRAGMA mmap_size=1000000; PRAGMA page_size=1024; PRAGMA cache_size=500; PRAGMA locking_mode=EXCLUSIVE; diff --git a/test/sysfault.test b/test/sysfault.test index bf7df5df5f..4ecaa1fabc 100644 --- a/test/sysfault.test +++ b/test/sysfault.test @@ -262,7 +262,7 @@ do_faultsim_test 4 -faults vfsfault-* -prep { faultsim_restore_and_reopen file_control_chunksize_test db main 8192 execsql { - PRAGMA mmap_size = 1000000; + PRAGMA mmap_limit = 1000000; } } -body { test_syscall errno mmap EACCES @@ -275,4 +275,3 @@ do_faultsim_test 4 -faults vfsfault-* -prep { } finish_test - diff --git a/test/wal5.test b/test/wal5.test index d8bf541a4b..ebd1dbfc31 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -241,8 +241,8 @@ foreach {testprefix do_wal_checkpoint} { # [sql2] commmand above. So normally, the db is 1 page in size here. # However, in mmap() mode, the db is pre-allocated to 2 pages at the # start of the checkpoint, even though page 2 cannot be written. - set nDb 1 - if {[permutation]=="mmap"} {set nDb 2} + set nDb 2 + if {[permutation]=="no-mmap"} {set nDb 1} do_test 2.3.$tn.8 { file_page_counts } [list $nDb 4 2 4] } @@ -351,4 +351,3 @@ foreach {testprefix do_wal_checkpoint} { finish_test - From 41f89cc6763b71bcad37964745c1ffdf2a391452 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 26 Mar 2013 01:07:50 +0000 Subject: [PATCH 042/204] Previous check-in accidently left mmap turned off by default. This checkin fixes that. Unfortunately, shared.test is now segfaulting. All other veryquick tests appear to work, however. FossilOrigin-Name: a850c7319c20b5757983443df05cf2aa4250053b --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/pager.c | 1 + test/exclusive2.test | 2 +- test/func.test | 2 +- test/pageropt.test | 2 +- test/permutations.test | 4 ++-- test/wal.test | 4 ++-- 8 files changed, 20 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 65993928dd..f9fdab7968 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Memory-mapped\sI/O\sis\snow\son\sby\sdefault.\s\sThe\s"PRAGMA\smmap_limit(N)"\scan\sbe\nused\sto\sissue\sa\shint\sto\sthe\sVFS\sto\slimit\smmap\sspace\sto\sN\sbytes.\s\sThe\sVFS\nis\sfree\sto\signore\sthat\shint\sif\sdesired.\s\sHowever,\sif\s"PRAGMA\smmap_limit(0)"\nis\sused,\sxFetch\sis\snever\scalled. -D 2013-03-25T23:09:28.646 +C Previous\scheck-in\saccidently\sleft\smmap\sturned\soff\sby\sdefault.\sThis\scheckin\nfixes\sthat.\s\sUnfortunately,\sshared.test\sis\snow\ssegfaulting.\s\sAll\sother\nveryquick\stests\sappear\sto\swork,\showever. +D 2013-03-26T01:07:50.891 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 57306f1d2a2d783dae4365446b9f9a5da8958559 F src/os_win.c e4f17ddf79f2a9373e33ed70565e765d65324589 -F src/pager.c c532d51ccc74de8551a28eb932f3b1a1034017d3 +F src/pager.c 8ac98fd95106e759870e92ff316b188e78f469cf F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -403,7 +403,7 @@ F test/eqp.test 46aa946dd55c90635327898275d3e533d23a9845 F test/errmsg.test 050717f1c6a5685de9c79f5f9f6b83d7c592f73a F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test a1b324cb21834a490cd052d409d34789cfef57cb -F test/exclusive2.test 881193eccec225cfed9d7f744b65e57f26adee25 +F test/exclusive2.test 354bdabe299a2546c898dff42f79079ff1590d88 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30 F test/expr.test 67c9fd6f8f829e239dc8b0f4a08a73c08b09196d @@ -508,7 +508,7 @@ F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7 F test/fts4unicode.test 25ccad45896f8e50f6a694cff738a35f798cdb40 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d -F test/func.test b0fc34fdc36897769651975a2b0a606312753643 +F test/func.test a4f24707e6af0bcd4cb5b856af4462db40b047b1 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a F test/fuzz-oss1.test 4912e528ec9cf2f42134456933659d371c9e0d74 @@ -654,11 +654,11 @@ F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f F test/pagerfault.test 452f2cc23e3bfcfa935f4442aec1da4fe1dc0442 F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 -F test/pageropt.test 290cd59782b1890f02bb33795571facfc5ccac43 +F test/pageropt.test 32cb5a6ed7ccddf8e8c842cb44240bd9340223ce F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test d298a74abae42b72b3640dc5505d19d8a3eddc9e +F test/permutations.test b6cb45ce4d3193b831231025c9d30c77317dd240 F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -951,7 +951,7 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 -F test/wal.test 5baa9a0e682c0394012aa9a06a3d06cf92655fbf +F test/wal.test bbd98ac09d8fbcaa80ec233757056eb1bbb7cd95 F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 8776047bd776bbf266eb9c3b56683badb84ae73e -R 1fba4ef23570acddc66df7d627b1cb5f +P 1b37c4effdd03aa2ea938a71b4f22ed27391689b +R 66cf00b9dd2b30a0d14b8b5c617a6bbb U drh -Z c62aae7ee656f659e8aebd3681920a9a +Z c802a777b217fa0e42ce25209bcf7fef diff --git a/manifest.uuid b/manifest.uuid index 8e0647f651..dc7e369204 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1b37c4effdd03aa2ea938a71b4f22ed27391689b \ No newline at end of file +a850c7319c20b5757983443df05cf2aa4250053b \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 4cce844237..65969bcc3e 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4749,6 +4749,7 @@ int sqlite3PagerOpen( /* pPager->pBusyHandlerArg = 0; */ pPager->xReiniter = xReinit; /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ + pPager->mxMmap = SQLITE_DEFAULT_MMAP_LIMIT; *ppPager = pPager; return SQLITE_OK; diff --git a/test/exclusive2.test b/test/exclusive2.test index 54203e3d8f..d09097671d 100644 --- a/test/exclusive2.test +++ b/test/exclusive2.test @@ -28,7 +28,7 @@ ifcapable {!pager_pragmas} { # Tests in this file verify that locking_mode=exclusive causes SQLite to # use cached pages even if the database is changed on disk. This doesn't # work with mmap. -if {[permutation]=="mmap"} { +if {[permutation]!="nommap"} { finish_test return } diff --git a/test/func.test b/test/func.test index 4ab7688461..032becc94f 100644 --- a/test/func.test +++ b/test/func.test @@ -1273,7 +1273,7 @@ do_test func-29.3 { sqlite3_db_status db CACHE_MISS 1 db eval {SELECT typeof(+x) FROM t29 ORDER BY id} } {integer null real blob text} -if {[permutation] != "mmap"} { +if {[permutation] == "nommap"} { do_test func-29.4 { set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] if {$x>100} {set x many} diff --git a/test/pageropt.test b/test/pageropt.test index de39bbaf05..10d3d4917e 100644 --- a/test/pageropt.test +++ b/test/pageropt.test @@ -92,7 +92,7 @@ do_test pageropt-1.5 { pagercount_sql { SELECT hex(x) FROM t1 } -} [list [expr {[permutation]=="mmap" ? 1 : 6}] 0 0 $blobcontent] +} [list [expr {[permutation]=="nommap" ? 6 : 1}] 0 0 $blobcontent] do_test pageropt-1.6 { pagercount_sql { SELECT hex(x) FROM t1 diff --git a/test/permutations.test b/test/permutations.test index 2e529fa880..3aba9d9b38 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -138,10 +138,10 @@ test_suite "veryquick" -prefix "" -description { test_set $allquicktests -exclude *malloc* *ioerr* *fault* ] -test_suite "no-mmap" -prefix "nomm-" -description { +test_suite "nommap" -prefix "nomm-" -description { Similar to veryquick. Except with memory mapping disabled. } -presql { - pragma mmap_size = 0; + pragma mmap_limit = 0; } -files [ test_set $allquicktests -exclude *malloc* *ioerr* *fault* -include malloc.test ] diff --git a/test/wal.test b/test/wal.test index ca7edef952..0539192d25 100644 --- a/test/wal.test +++ b/test/wal.test @@ -727,8 +727,8 @@ do_test wal-11.9 { list [expr [file size test.db]/1024] [log_deleted test.db-wal] } {37 1} sqlite3_wal db test.db -set nWal 37 -if {[permutation]=="mmap"} {set nWal 39} +set nWal 39 +if {[permutation]=="nommap"} {set nWal 37} do_test wal-11.10 { execsql { PRAGMA cache_size = 10; From 0aed84d19f8e1fcf90340e17cff6b774b0379255 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 26 Mar 2013 14:16:20 +0000 Subject: [PATCH 043/204] In btree.c, save the positions of any open cursors before moving any pages around to auto-vacuum the database on commit. FossilOrigin-Name: 30c0a69363931a72d1c34e5be71646932398d172 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 6 ++++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index f9fdab7968..7b9df5476e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Previous\scheck-in\saccidently\sleft\smmap\sturned\soff\sby\sdefault.\sThis\scheckin\nfixes\sthat.\s\sUnfortunately,\sshared.test\sis\snow\ssegfaulting.\s\sAll\sother\nveryquick\stests\sappear\sto\swork,\showever. -D 2013-03-26T01:07:50.891 +C In\sbtree.c,\ssave\sthe\spositions\sof\sany\sopen\scursors\sbefore\smoving\sany\spages\saround\sto\sauto-vacuum\sthe\sdatabase\son\scommit. +D 2013-03-26T14:16:20.450 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 771f0391556317de19e02c9533b256e8299dd59b +F src/btree.c 19b587b32569d4b92651b5eaa9de2e59c6f8f704 F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 1b37c4effdd03aa2ea938a71b4f22ed27391689b -R 66cf00b9dd2b30a0d14b8b5c617a6bbb -U drh -Z c802a777b217fa0e42ce25209bcf7fef +P a850c7319c20b5757983443df05cf2aa4250053b +R 211ffabed726a7468ad9f8d3eeabe8a8 +U dan +Z 3ec7da7f1db0278991c8541b22037fbf diff --git a/manifest.uuid b/manifest.uuid index dc7e369204..12eaba3359 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a850c7319c20b5757983443df05cf2aa4250053b \ No newline at end of file +30c0a69363931a72d1c34e5be71646932398d172 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 04cc20d5c5..7d2f2d3840 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3185,7 +3185,9 @@ static int autoVacuumCommit(BtShared *pBt){ nFree = get4byte(&pBt->pPage1->aData[36]); nFin = finalDbSize(pBt, nOrig, nFree); if( nFin>nOrig ) return SQLITE_CORRUPT_BKPT; - + if( nFinnFin && rc==SQLITE_OK; iFree--){ rc = incrVacuumStep(pBt, nFin, iFree, 1); } @@ -3202,7 +3204,7 @@ static int autoVacuumCommit(BtShared *pBt){ } } - assert( nRef==sqlite3PagerRefcount(pPager) ); + assert( nRef>=sqlite3PagerRefcount(pPager) ); return rc; } From c86e51358b3e6cd21d2b324116933b9fb7943956 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 26 Mar 2013 14:36:11 +0000 Subject: [PATCH 044/204] Change the name of the Pager.pFree field to Pager.pMmapFreelist. FossilOrigin-Name: 611bd824c24a60d298f28705de323fa2e813a308 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 16 ++++++++-------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 7b9df5476e..a2e312e3a6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sbtree.c,\ssave\sthe\spositions\sof\sany\sopen\scursors\sbefore\smoving\sany\spages\saround\sto\sauto-vacuum\sthe\sdatabase\son\scommit. -D 2013-03-26T14:16:20.450 +C Change\sthe\sname\sof\sthe\sPager.pFree\sfield\sto\sPager.pMmapFreelist. +D 2013-03-26T14:36:11.815 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 57306f1d2a2d783dae4365446b9f9a5da8958559 F src/os_win.c e4f17ddf79f2a9373e33ed70565e765d65324589 -F src/pager.c 8ac98fd95106e759870e92ff316b188e78f469cf +F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a850c7319c20b5757983443df05cf2aa4250053b -R 211ffabed726a7468ad9f8d3eeabe8a8 -U dan -Z 3ec7da7f1db0278991c8541b22037fbf +P 30c0a69363931a72d1c34e5be71646932398d172 +R 11ebc3388f5e3939639a27582f5015a5 +U drh +Z 2c5cf3d5be983a0c21f765acf3ca05b3 diff --git a/manifest.uuid b/manifest.uuid index 12eaba3359..cd71ef1d3b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -30c0a69363931a72d1c34e5be71646932398d172 \ No newline at end of file +611bd824c24a60d298f28705de323fa2e813a308 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 65969bcc3e..550c6c7564 100644 --- a/src/pager.c +++ b/src/pager.c @@ -659,7 +659,7 @@ struct Pager { u8 bUseFetch; /* True to use xFetch() */ int nMmapOut; /* Number of mmap pages currently outstanding */ sqlite3_int64 mxMmap; /* Desired maximum mmap size */ - PgHdr *pFree; /* List of free mmap page headers (pDirty) */ + PgHdr *pMmapFreelist; /* List of free mmap page headers (pDirty) */ /* ** End of the routinely-changing class members ***************************************************************************/ @@ -3854,9 +3854,9 @@ static int pagerAcquireMapPage( ){ PgHdr *p; /* Memory mapped page to return */ - if( pPager->pFree ){ - *ppPage = p = pPager->pFree; - pPager->pFree = p->pDirty; + if( pPager->pMmapFreelist ){ + *ppPage = p = pPager->pMmapFreelist; + pPager->pMmapFreelist = p->pDirty; p->pDirty = 0; memset(p->pExtra, 0, pPager->nExtra); }else{ @@ -3891,20 +3891,20 @@ static int pagerAcquireMapPage( static void pagerReleaseMapPage(PgHdr *pPg){ Pager *pPager = pPg->pPager; pPager->nMmapOut--; - pPg->pDirty = pPager->pFree; - pPager->pFree = pPg; + pPg->pDirty = pPager->pMmapFreelist; + pPager->pMmapFreelist = pPg; assert( pPager->fd->pMethods->iVersion>=3 ); sqlite3OsUnfetch(pPager->fd, (i64)(pPg->pgno-1)*pPager->pageSize, pPg->pData); } /* -** Free all PgHdr objects stored in the Pager.pFree list. +** Free all PgHdr objects stored in the Pager.pMmapFreelist list. */ static void pagerFreeMapHdrs(Pager *pPager){ PgHdr *p; PgHdr *pNext; - for(p=pPager->pFree; p; p=pNext){ + for(p=pPager->pMmapFreelist; p; p=pNext){ pNext = p->pDirty; sqlite3_free(p); } From 6c5696381e5720eaa9bd6c780bd75f5669f04c96 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 26 Mar 2013 18:48:11 +0000 Subject: [PATCH 045/204] Fix a comment in os_unix.c. No code changes. FossilOrigin-Name: 72813b8ec924b91583c679668f7c4561dff82a02 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index a2e312e3a6..715e199440 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sname\sof\sthe\sPager.pFree\sfield\sto\sPager.pMmapFreelist. -D 2013-03-26T14:36:11.815 +C Fix\sa\scomment\sin\sos_unix.c.\s\sNo\scode\schanges. +D 2013-03-26T18:48:11.374 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 57306f1d2a2d783dae4365446b9f9a5da8958559 +F src/os_unix.c d6981218583748080374ed98a03f6a87e2bdc9e0 F src/os_win.c e4f17ddf79f2a9373e33ed70565e765d65324589 F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 30c0a69363931a72d1c34e5be71646932398d172 -R 11ebc3388f5e3939639a27582f5015a5 +P 611bd824c24a60d298f28705de323fa2e813a308 +R 8f591838a5bc479ac7493be19f46439f U drh -Z 2c5cf3d5be983a0c21f765acf3ca05b3 +Z be06b34348d93c4a77e7034db0a8802f diff --git a/manifest.uuid b/manifest.uuid index cd71ef1d3b..855425c986 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -611bd824c24a60d298f28705de323fa2e813a308 \ No newline at end of file +72813b8ec924b91583c679668f7c4561dff82a02 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 8a8516bcda..f91f4c4e7a 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3107,8 +3107,8 @@ static int unixRead( ); #endif - /* Deal with as much of this write request as possible by transfering - ** data to the memory mapping using memcpy(). */ + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); From 138eeeb1b0e9d59c0c2ee5caa1463edd1a62ffcc Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 27 Mar 2013 03:15:23 +0000 Subject: [PATCH 046/204] Candidate fix for ticket [6bfb98dfc0c]: Make sure invalid cursors drop all references to database pages prior to doing any insert or update. FossilOrigin-Name: 322a5f086d9ee46017f750df81527799a54ae258 --- manifest | 15 +++++----- manifest.uuid | 2 +- src/btree.c | 34 ++++++++++++++-------- test/tkt-6bfb98dfc0.test | 61 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 19 deletions(-) create mode 100644 test/tkt-6bfb98dfc0.test diff --git a/manifest b/manifest index abc6aa2430..007356f176 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\ssecond\stest\sfor\s[38b1ae018f]. -D 2013-03-25T12:02:45.865 +C Candidate\sfix\sfor\sticket\s[6bfb98dfc0c]:\sMake\ssure\sinvalid\scursors\sdrop\sall\nreferences\sto\sdatabase\spages\sprior\sto\sdoing\sany\sinsert\sor\supdate. +D 2013-03-27T03:15:23.329 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 3cebaa69db81a528e115b463a5506133a0043710 +F src/btree.c 62ba5954765efc711c873a20a53f60d9fc2843ba F src/btree.h 3ad7964d6c5b1c7bff569aab6adfa075f8bf06cd F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -784,6 +784,7 @@ F test/tkt-54844eea3f.test a12b851128f46a695e4e378cca67409b9b8f5894 F test/tkt-5d863f876e.test c9f36ca503fa154a3655f92a69d2c30da1747bfa F test/tkt-5e10420e8d.test 904d1687b3c06d43e5b3555bbcf6802e7c0ffd84 F test/tkt-5ee23731f.test 9db6e1d7209dc0794948b260d6f82b2b1de83a9f +F test/tkt-6bfb98dfc0.test 24780633627b5cfc0635a5500c2389ebfb563336 F test/tkt-752e1646fc.test ea78d88d14fe9866bdd991c634483334639e13bf F test/tkt-78e04e52ea.test 703e0bfb23d543edf0426a97e3bbd0ca346508ec F test/tkt-7a31705a7e6.test 5a7889fdb095ffbe1622413e0145de1637d421bd @@ -1039,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f85f9103cffa5c8ba6a63a68beb90817147ba080 -R 8636831e10a59eea94894795b6da6922 -U dan -Z 40d326d8e18513e2868148c4279af040 +P 5062db672c00c3365d51cd6b39815078f5b6b525 +R 444ec66222f6e3876b5a1bc78bf6602c +U drh +Z 3d135e15ecdab16eb6decd45c9d13940 diff --git a/manifest.uuid b/manifest.uuid index 14d19b23d1..ed40effb49 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5062db672c00c3365d51cd6b39815078f5b6b525 \ No newline at end of file +322a5f086d9ee46017f750df81527799a54ae258 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index b3549fa697..96140d68c6 100644 --- a/src/btree.c +++ b/src/btree.c @@ -575,6 +575,19 @@ static void btreeClearHasContent(BtShared *pBt){ pBt->pHasContent = 0; } +/* +** Release all of the apPage[] pages for a cursor. +*/ +static void btreeReleaseAllCursorPages(BtCursor *pCur){ + int i; + for(i=0; i<=pCur->iPage; i++){ + releasePage(pCur->apPage[i]); + pCur->apPage[i] = 0; + } + pCur->iPage = -1; +} + + /* ** Save the current cursor position in the variables BtCursor.nKey ** and BtCursor.pKey. The cursor's state is set to CURSOR_REQUIRESEEK. @@ -614,12 +627,7 @@ static int saveCursorPosition(BtCursor *pCur){ assert( !pCur->apPage[0]->intKey || !pCur->pKey ); if( rc==SQLITE_OK ){ - int i; - for(i=0; i<=pCur->iPage; i++){ - releasePage(pCur->apPage[i]); - pCur->apPage[i] = 0; - } - pCur->iPage = -1; + btreeReleaseAllCursorPages(pCur); pCur->eState = CURSOR_REQUIRESEEK; } @@ -637,11 +645,15 @@ static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ assert( sqlite3_mutex_held(pBt->mutex) ); assert( pExcept==0 || pExcept->pBt==pBt ); for(p=pBt->pCursor; p; p=p->pNext){ - if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) && - p->eState==CURSOR_VALID ){ - int rc = saveCursorPosition(p); - if( SQLITE_OK!=rc ){ - return rc; + if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ){ + if( p->eState==CURSOR_VALID ){ + int rc = saveCursorPosition(p); + if( SQLITE_OK!=rc ){ + return rc; + } + }else{ + testcase( p->iPage>0 ); + btreeReleaseAllCursorPages(p); } } } diff --git a/test/tkt-6bfb98dfc0.test b/test/tkt-6bfb98dfc0.test new file mode 100644 index 0000000000..675a3fc2e9 --- /dev/null +++ b/test/tkt-6bfb98dfc0.test @@ -0,0 +1,61 @@ +# 2013 March 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. Specifically, +# it tests that ticket [6bfb98dfc0] +# +# The final INSERT in the script below reports that the database is +# corrupt (SQLITE_CORRUPT) and aborts even though the database is not +# corrupt. +# +# PRAGMA page_size=512; +# CREATE TABLE t1(x INTEGER PRIMARY KEY, y); +# INSERT INTO t1 VALUES(1,randomblob(400)); +# INSERT INTO t1 VALUES(2,randomblob(400)); +# INSERT INTO t1 SELECT x+2, randomblob(400) FROM t1; +# INSERT INTO t1 SELECT x+4, randomblob(400) FROM t1; +# INSERT INTO t1 SELECT x+8, randomblob(400) FROM t1; +# INSERT INTO t1 SELECT x+16, randomblob(400) FROM t1; +# INSERT INTO t1 SELECT x+32, randomblob(400) FROM t1; +# INSERT INTO t1 SELECT x+64, randomblob(400) FROM t1 WHERE x<10; +# CREATE TRIGGER r1 AFTER INSERT ON t1 WHEN new.x=74 BEGIN +# DELETE FROM t1; +# INSERT INTO t1 VALUES(75, randomblob(400)); +# INSERT INTO t1 VALUES(76, randomblob(400)); +# END; +# INSERT INTO t1 VALUES(74, randomblob(400)); +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_test tkt-6bfb98dfc0.100 { + db eval { + PRAGMA page_size=512; + CREATE TABLE t1(x INTEGER PRIMARY KEY, y); + INSERT INTO t1 VALUES(1,randomblob(400)); + INSERT INTO t1 VALUES(2,randomblob(400)); + INSERT INTO t1 SELECT x+2, randomblob(400) FROM t1; + INSERT INTO t1 SELECT x+4, randomblob(400) FROM t1; + INSERT INTO t1 SELECT x+8, randomblob(400) FROM t1; + INSERT INTO t1 SELECT x+16, randomblob(400) FROM t1; + INSERT INTO t1 SELECT x+32, randomblob(400) FROM t1; + INSERT INTO t1 SELECT x+64, randomblob(400) FROM t1 WHERE x<10; + CREATE TRIGGER r1 AFTER INSERT ON t1 WHEN new.x=74 BEGIN + DELETE FROM t1; + INSERT INTO t1 VALUES(75, randomblob(400)); + INSERT INTO t1 VALUES(76, randomblob(400)); + END; + INSERT INTO t1 VALUES(74, randomblob(400)); + SELECT x, length(y) FROM t1 ORDER BY x; + } +} {75 400 76 400} + +finish_test From 8a1b87cb971a52b9d782def9b70827d9fc302a3f Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 27 Mar 2013 15:04:28 +0000 Subject: [PATCH 047/204] A fix an test-case for the ORDER BY problem identified by ticket [a179fe7465]. This change causes sorting to occur in some cases where it is not strictly necessary. Further work is needed to avoid those extra sorts. FossilOrigin-Name: 488089e615e375c01b31544f06e801af950ae3db --- manifest | 16 ++++++++----- manifest.uuid | 2 +- src/where.c | 30 ++++++++++++++++++++----- test/orderby4.test | 56 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 91 insertions(+), 13 deletions(-) create mode 100644 test/orderby4.test diff --git a/manifest b/manifest index 007356f176..391a2dd073 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Candidate\sfix\sfor\sticket\s[6bfb98dfc0c]:\sMake\ssure\sinvalid\scursors\sdrop\sall\nreferences\sto\sdatabase\spages\sprior\sto\sdoing\sany\sinsert\sor\supdate. -D 2013-03-27T03:15:23.329 +C A\sfix\san\stest-case\sfor\sthe\sORDER\sBY\sproblem\sidentified\sby\sticket\s[a179fe7465].\nThis\schange\scauses\ssorting\sto\soccur\sin\ssome\scases\swhere\sit\sis\snot\sstrictly\nnecessary.\s\sFurther\swork\sis\sneeded\sto\savoid\sthose\sextra\ssorts. +D 2013-03-27T15:04:28.049 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -252,7 +252,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 9a16c0b84bbeb054d11fda96e9e037ae310bd54e +F src/where.c 979868386da8fa8e50d74bde97216849add43db2 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -646,6 +646,7 @@ F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 F test/orderby1.test f33968647da5c546528fe4d2bf86c6a6a2e5a7ae F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 +F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 31c04bec797dda1bde337810b52efa08d1f1f08e F test/pager2.test 745b911dde3d1f24ae0870bd433dfa83d7c658c1 @@ -1040,7 +1041,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5062db672c00c3365d51cd6b39815078f5b6b525 -R 444ec66222f6e3876b5a1bc78bf6602c +P 322a5f086d9ee46017f750df81527799a54ae258 +R b585a02d8fa68b00b6cca0df4ff7837d +T *branch * orderby-fix +T *sym-orderby-fix * +T -sym-trunk * U drh -Z 3d135e15ecdab16eb6decd45c9d13940 +Z 030131c01bf39bf419a41cdbf4bff578 diff --git a/manifest.uuid b/manifest.uuid index ed40effb49..22f49c1cad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -322a5f086d9ee46017f750df81527799a54ae258 \ No newline at end of file +488089e615e375c01b31544f06e801af950ae3db \ No newline at end of file diff --git a/src/where.c b/src/where.c index 9b64fd72fe..323b568c83 100644 --- a/src/where.c +++ b/src/where.c @@ -262,6 +262,8 @@ struct WhereCost { #define WHERE_REVERSE 0x01000000 /* Scan in reverse order */ #define WHERE_UNIQUE 0x02000000 /* Selects no more than one row */ #define WHERE_ALL_UNIQUE 0x04000000 /* This and all prior have one row */ +#define WHERE_OB_UNIQUE 0x00004000 /* Values in ORDER BY columns are + ** different for every output row */ #define WHERE_VIRTUALTABLE 0x08000000 /* Use virtual-table processing */ #define WHERE_MULTI_OR 0x10000000 /* OR using multiple indices */ #define WHERE_TEMP_INDEX 0x20000000 /* Uses an ephemeral index */ @@ -2903,7 +2905,8 @@ static int isSortingIndex( WhereBestIdx *p, /* Best index search context */ Index *pIdx, /* The index we are testing */ int base, /* Cursor number for the table to be sorted */ - int *pbRev /* Set to 1 for reverse-order scan of pIdx */ + int *pbRev, /* Set to 1 for reverse-order scan of pIdx */ + int *pbObUnique /* ORDER BY column values will different in every row */ ){ int i; /* Number of pIdx terms used */ int j; /* Number of ORDER BY terms satisfied */ @@ -2917,9 +2920,12 @@ static int isSortingIndex( int nPriorSat; /* ORDER BY terms satisfied by outer loops */ int seenRowid = 0; /* True if an ORDER BY rowid term is seen */ int uniqueNotNull; /* pIdx is UNIQUE with all terms are NOT NULL */ + int outerObUnique; /* Outer loops generate different values in + ** every row for the ORDER BY columns */ if( p->i==0 ){ nPriorSat = 0; + outerObUnique = 1; }else{ nPriorSat = p->aLevel[p->i-1].plan.nOBSat; if( (p->aLevel[p->i-1].plan.wsFlags & WHERE_ORDERED)==0 ){ @@ -2932,6 +2938,7 @@ static int isSortingIndex( ** optimization is disabled */ return nPriorSat; } + outerObUnique = (p->aLevel[p->i-1].plan.wsFlags & WHERE_OB_UNIQUE)!=0; } pOrderBy = p->pOrderBy; assert( pOrderBy!=0 ); @@ -3073,11 +3080,20 @@ static int isSortingIndex( uniqueNotNull = 0; } } + if( seenRowid ){ + uniqueNotNull = 1; + }else if( uniqueNotNull==0 || inColumn ){ + uniqueNotNull = 0; + } /* If we have not found at least one ORDER BY term that matches the ** index, then show no progress. */ if( pOBItem==&pOrderBy->a[nPriorSat] ) return nPriorSat; + /* */ + if( outerObUnique==0 && uniqueNotNull==0 ) return nPriorSat; + *pbObUnique = uniqueNotNull; + /* Return the necessary scan order back to the caller */ *pbRev = sortOrder & 1; @@ -3085,7 +3101,7 @@ static int isSortingIndex( ** possible for a single row from this table to match, then skip over ** any additional ORDER BY terms dealing with this table. */ - if( seenRowid || (uniqueNotNull && i>=pIdx->nColumn) ){ + if( uniqueNotNull ){ /* Advance j over additional ORDER BY terms associated with base */ WhereMaskSet *pMS = p->pWC->pMaskSet; Bitmask m = ~getMask(pMS, base); @@ -3369,12 +3385,14 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** variable. */ if( bSort && (pSrc->jointype & JT_LEFT)==0 ){ int bRev = 2; - WHERETRACE((" --> before isSortingIndex: nPriorSat=%d\n",nPriorSat)); - pc.plan.nOBSat = isSortingIndex(p, pProbe, iCur, &bRev); - WHERETRACE((" --> after isSortingIndex: bRev=%d nOBSat=%d\n", - bRev, pc.plan.nOBSat)); + int bObUnique = 0; + WHERETRACE((" --> before isSortIndex: nPriorSat=%d\n",nPriorSat)); + pc.plan.nOBSat = isSortingIndex(p, pProbe, iCur, &bRev, &bObUnique); + WHERETRACE((" --> after isSortIndex: bRev=%d bObU=%d nOBSat=%d\n", + bRev, bObUnique, pc.plan.nOBSat)); if( nPriorSat Date: Wed, 27 Mar 2013 16:05:30 +0000 Subject: [PATCH 048/204] Improved optimization of ORDER BY. FossilOrigin-Name: 97e5c70f2f19a1bca70dd2221b757b9e5418fce0 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/where.c | 3 ++- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 391a2dd073..d7b9619908 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C A\sfix\san\stest-case\sfor\sthe\sORDER\sBY\sproblem\sidentified\sby\sticket\s[a179fe7465].\nThis\schange\scauses\ssorting\sto\soccur\sin\ssome\scases\swhere\sit\sis\snot\sstrictly\nnecessary.\s\sFurther\swork\sis\sneeded\sto\savoid\sthose\sextra\ssorts. -D 2013-03-27T15:04:28.049 +C Improved\soptimization\sof\sORDER\sBY. +D 2013-03-27T16:05:30.603 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -252,7 +252,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 979868386da8fa8e50d74bde97216849add43db2 +F src/where.c fc912c1d321f765840b2cf63a903cf795c871386 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1041,10 +1041,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 322a5f086d9ee46017f750df81527799a54ae258 -R b585a02d8fa68b00b6cca0df4ff7837d -T *branch * orderby-fix -T *sym-orderby-fix * -T -sym-trunk * +P 488089e615e375c01b31544f06e801af950ae3db +R 7011d42dba6b11445611fd2df04727aa U drh -Z 030131c01bf39bf419a41cdbf4bff578 +Z 55f146dee0dda61d0272032f415b56a9 diff --git a/manifest.uuid b/manifest.uuid index 22f49c1cad..41967bd9a4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -488089e615e375c01b31544f06e801af950ae3db \ No newline at end of file +97e5c70f2f19a1bca70dd2221b757b9e5418fce0 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 323b568c83..409d73a059 100644 --- a/src/where.c +++ b/src/where.c @@ -3486,7 +3486,8 @@ static void bestBtreeIndex(WhereBestIdx *p){ ** So this computation assumes table records are about twice as big ** as index records */ - if( (pc.plan.wsFlags&~(WHERE_REVERSE|WHERE_ORDERED))==WHERE_IDX_ONLY + if( (pc.plan.wsFlags&~(WHERE_REVERSE|WHERE_ORDERED|WHERE_OB_UNIQUE)) + ==WHERE_IDX_ONLY && (pWC->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 && sqlite3GlobalConfig.bUseCis && OptimizationEnabled(pParse->db, SQLITE_CoverIdxScan) From e6a8bbbb623169b08390c0db0d52e8ad07653ae6 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 27 Mar 2013 16:42:21 +0000 Subject: [PATCH 049/204] Restore additional ORDER BY optimizations that where broken by the recent ORDER BY fix. FossilOrigin-Name: c77ee6e20d3a8c91a8bf4c9063c36a95c70e17cf --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 7 +++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index d7b9619908..5f6aa42e1a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\soptimization\sof\sORDER\sBY. -D 2013-03-27T16:05:30.603 +C Restore\sadditional\sORDER\sBY\soptimizations\sthat\swhere\sbroken\sby\sthe\nrecent\sORDER\sBY\sfix. +D 2013-03-27T16:42:21.685 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -252,7 +252,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c fc912c1d321f765840b2cf63a903cf795c871386 +F src/where.c 8179f4c891d2742ff7c00237b2f7b534c826503c F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1041,7 +1041,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 488089e615e375c01b31544f06e801af950ae3db -R 7011d42dba6b11445611fd2df04727aa +P 97e5c70f2f19a1bca70dd2221b757b9e5418fce0 +R 4ded79a01068c5e1c5e1e2eba18cd5f5 U drh -Z 55f146dee0dda61d0272032f415b56a9 +Z 80c245e483b26492a6a00ec636bab1bb diff --git a/manifest.uuid b/manifest.uuid index 41967bd9a4..ac70cc5249 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -97e5c70f2f19a1bca70dd2221b757b9e5418fce0 \ No newline at end of file +c77ee6e20d3a8c91a8bf4c9063c36a95c70e17cf \ No newline at end of file diff --git a/src/where.c b/src/where.c index 409d73a059..5c0f664aec 100644 --- a/src/where.c +++ b/src/where.c @@ -2927,8 +2927,9 @@ static int isSortingIndex( nPriorSat = 0; outerObUnique = 1; }else{ + u32 wsFlags = p->aLevel[p->i-1].plan.wsFlags; nPriorSat = p->aLevel[p->i-1].plan.nOBSat; - if( (p->aLevel[p->i-1].plan.wsFlags & WHERE_ORDERED)==0 ){ + if( (wsFlags & WHERE_ORDERED)==0 ){ /* This loop cannot be ordered unless the next outer loop is ** also ordered */ return nPriorSat; @@ -2938,7 +2939,9 @@ static int isSortingIndex( ** optimization is disabled */ return nPriorSat; } - outerObUnique = (p->aLevel[p->i-1].plan.wsFlags & WHERE_OB_UNIQUE)!=0; + testcase( wsFlags & WHERE_OB_UNIQUE ); + testcase( wsFlags & WHERE_ALL_UNIQUE ); + outerObUnique = (wsFlags & (WHERE_OB_UNIQUE|WHERE_ALL_UNIQUE))!=0; } pOrderBy = p->pOrderBy; assert( pOrderBy!=0 ); From ab5190a46cbc92b06de6a37becb8d1561d8a3f28 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 27 Mar 2013 19:46:33 +0000 Subject: [PATCH 050/204] Increment the version number to 3.7.16.1. FossilOrigin-Name: 7e32eb7b66290ba7323520f7ea92ffa584ee7193 --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/VERSION b/VERSION index 36f601f10e..9efca4c4be 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.7.16 +3.7.16.1 diff --git a/configure b/configure index f8a60e1625..ddd6364831 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.7.16. +# Generated by GNU Autoconf 2.62 for sqlite 3.7.16.1. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.7.16' -PACKAGE_STRING='sqlite 3.7.16' +PACKAGE_VERSION='3.7.16.1' +PACKAGE_STRING='sqlite 3.7.16.1' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1484,7 +1484,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.7.16 to adapt to many kinds of systems. +\`configure' configures sqlite 3.7.16.1 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1549,7 +1549,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.7.16:";; + short | recursive ) echo "Configuration of sqlite 3.7.16.1:";; esac cat <<\_ACEOF @@ -1665,7 +1665,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.7.16 +sqlite configure 3.7.16.1 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1679,7 +1679,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.7.16, which was +It was created by sqlite $as_me 3.7.16.1, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -14032,7 +14032,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.7.16, which was +This file was extended by sqlite $as_me 3.7.16.1, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14085,7 +14085,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.7.16 +sqlite config.status 3.7.16.1 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/manifest b/manifest index d1bf95f820..7f1c24774e 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C In\sorder\sto\soptimize\sout\sthe\sORDER\sBY\sclause,\s\nouter\sloops\smust\sgenerate\svalues\sfor\sORDER\sBY\sterms\sthat\sare\sunique\sor\selse\nthe\sinner\sloops\smust\sgenerate\sno\smore\sthan\sa\ssingle\srow.\nFix\sfor\sticket\s[a179fe7465]. -D 2013-03-27T17:20:10.024 +C Increment\sthe\sversion\snumber\sto\s3.7.16.1. +D 2013-03-27T19:46:33.857 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc 0c1abc21c8deefc88e8a32ad6a07e5f96e158761 F Makefile.vxworks b18ad88e9a8c6a001f5cf4a389116a4f1a7ab45f F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 -F VERSION 6d4f66eaebabc42ef8c2a4d2d0caf4ce7ee81137 +F VERSION 05604ccde96fe1b37f922eddbdcb5654308261db F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 17dc593f791f874d2c23a0f9360850ded0286531 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -15,7 +15,7 @@ F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 0921066a13130082764ab4ab6456f7b5bebe56de F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure 00d3a5fda47ee7dd3e68dc9dad7eeb439cd1ffea x +F configure 2f9994fd2e2005753809a2246c7a0816c80b2d2f x F configure.ac 81c43d151d0b0e406be056394cc9ff4cb3fd0444 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 @@ -1041,7 +1041,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 322a5f086d9ee46017f750df81527799a54ae258 c77ee6e20d3a8c91a8bf4c9063c36a95c70e17cf -R 7a3bcce0c2bcd988dcd25474f2b83a5d +P 2936f7466e162dfb003bda26d35358d1b3063112 +R 26cbf43287127f8a1cfd09b97c6bcc02 U drh -Z 25cf293fe59d9bbc42bbcd9448a91b58 +Z 184f78ab5133d283510134f5a6061c7a diff --git a/manifest.uuid b/manifest.uuid index 8b756b3952..6b2a81dd59 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2936f7466e162dfb003bda26d35358d1b3063112 \ No newline at end of file +7e32eb7b66290ba7323520f7ea92ffa584ee7193 \ No newline at end of file From 4b11bad35824c3f4f56e3d1560908a43d074443a Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 27 Mar 2013 20:41:15 +0000 Subject: [PATCH 051/204] Fix harmless compiler warnings coming from MSVC64 in test code. No changes to the core. FossilOrigin-Name: 274d2a22660c7b34b8bbd85f3c29cbafbcb1b4e7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/test_fs.c | 2 +- src/test_regexp.c | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 7f1c24774e..dc96b26eee 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increment\sthe\sversion\snumber\sto\s3.7.16.1. -D 2013-03-27T19:46:33.857 +C Fix\sharmless\scompiler\swarnings\scoming\sfrom\sMSVC64\sin\stest\scode.\s\sNo\nchanges\sto\sthe\score. +D 2013-03-27T20:41:15.391 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -200,7 +200,7 @@ F src/test_btree.c 5b89601dcb42a33ba8b820a6b763cc9cb48bac16 F src/test_config.c ab6605a41f2020912d5e44b8dd4b9fb21bd75ac0 F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc -F src/test_fs.c 1c51e203b2c20235d8c3739f8c1fb13a7502915b +F src/test_fs.c 8f786bfd0ad48030cf2a06fb1f050e9c60a150d7 F src/test_func.c 3a8dd37c08ab43b76d38eea2836e34a3897bf170 F src/test_fuzzer.c 1d26aa965120420bc14807da29d4d4541bfa6148 F src/test_hexio.c abfdecb6fa58c354623978efceb088ca18e379cd @@ -218,7 +218,7 @@ F src/test_osinst.c 90a845c8183013d80eccb1f29e8805608516edba F src/test_pcache.c a5cd24730cb43c5b18629043314548c9169abb00 F src/test_quota.c 1ec82e02fd3643899e9a5de9684515e84641c91f F src/test_quota.h 8761e463b25e75ebc078bd67d70e39b9c817a0cb -F src/test_regexp.c 5ff0eafd66c9a209417142fa319006f7028fbda8 +F src/test_regexp.c 06ae8138d41a793330f62351283dd6f6f21104f4 F src/test_rtree.c aba603c949766c4193f1068b91c787f57274e0d9 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f @@ -1041,7 +1041,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 2936f7466e162dfb003bda26d35358d1b3063112 -R 26cbf43287127f8a1cfd09b97c6bcc02 +P 7e32eb7b66290ba7323520f7ea92ffa584ee7193 +R 146cc38cc60deb4264a54dc6e7f5cc2c U drh -Z 184f78ab5133d283510134f5a6061c7a +Z 01cce950cabd5af1f79f5298131f812c diff --git a/manifest.uuid b/manifest.uuid index 6b2a81dd59..005d2f5c58 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7e32eb7b66290ba7323520f7ea92ffa584ee7193 \ No newline at end of file +274d2a22660c7b34b8bbd85f3c29cbafbcb1b4e7 \ No newline at end of file diff --git a/src/test_fs.c b/src/test_fs.c index 532f92528b..478cad80b1 100644 --- a/src/test_fs.c +++ b/src/test_fs.c @@ -99,7 +99,7 @@ static int fsConnect( } zTbl = argv[3]; - nByte = sizeof(fs_vtab) + strlen(zTbl) + 1 + strlen(zDb) + 1; + nByte = sizeof(fs_vtab) + (int)strlen(zTbl) + 1 + (int)strlen(zDb) + 1; pVtab = (fs_vtab *)sqlite3MallocZero( nByte ); if( !pVtab ) return SQLITE_NOMEM; diff --git a/src/test_regexp.c b/src/test_regexp.c index a1969ada40..829d22ad25 100644 --- a/src/test_regexp.c +++ b/src/test_regexp.c @@ -188,7 +188,7 @@ int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){ in.z = zIn; in.i = 0; - in.mx = nIn>=0 ? nIn : strlen((char const*)zIn); + in.mx = nIn>=0 ? nIn : (int)strlen((char const*)zIn); /* Look for the initial prefix match, if there is one. */ if( pRe->nInit ){ @@ -628,7 +628,7 @@ const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){ } pRe->sIn.z = (unsigned char*)zIn; pRe->sIn.i = 0; - pRe->sIn.mx = strlen(zIn); + pRe->sIn.mx = (int)strlen(zIn); zErr = re_subcompile_re(pRe); if( zErr ){ re_free(pRe); From 12886632c694ddf99471d51b20aefbc524f69426 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 28 Mar 2013 11:40:14 +0000 Subject: [PATCH 052/204] Adjustment to an extreme corner case in the sqlite3Atoi64() routine. FossilOrigin-Name: 39534b4baa53b9360696a9d5ee8cc2108af34592 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/util.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index dc96b26eee..9ccb151d94 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\scoming\sfrom\sMSVC64\sin\stest\scode.\s\sNo\nchanges\sto\sthe\score. -D 2013-03-27T20:41:15.391 +C Adjustment\sto\san\sextreme\scorner\scase\sin\sthe\ssqlite3Atoi64()\sroutine. +D 2013-03-28T11:40:14.945 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -237,7 +237,7 @@ F src/tokenize.c 1e86210d3976717a19238ea7b047fac481fe8c12 F src/trigger.c cd95ac64efa60e39faf9b5597443192ff27a22fa F src/update.c a2a5631d618cbe240fc83725fa9e95c56ae0084c F src/utf.c 8d819e2e5104a430fc2005f018db14347c95a38f -F src/util.c 550f2b6a5c0085153a4d00462719fb17ee242792 +F src/util.c f566b5138099a2df8533b190d0dcc74b7dfbe0c9 F src/vacuum.c 2727bdd08847fcb6b2d2da6d14f018910e8645d3 F src/vdbe.c 1d56617d816098a41bdbe73536a67dcdefad52bd F src/vdbe.h b52887278cb173e66188da84dfab216bea61119d @@ -1041,7 +1041,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 7e32eb7b66290ba7323520f7ea92ffa584ee7193 -R 146cc38cc60deb4264a54dc6e7f5cc2c +P 274d2a22660c7b34b8bbd85f3c29cbafbcb1b4e7 +R e8e4e98970281a95c0de62c6e5f4d562 U drh -Z 01cce950cabd5af1f79f5298131f812c +Z e440add500ad1771a8d916396dcab1f1 diff --git a/manifest.uuid b/manifest.uuid index 005d2f5c58..d68efa3fcb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -274d2a22660c7b34b8bbd85f3c29cbafbcb1b4e7 \ No newline at end of file +39534b4baa53b9360696a9d5ee8cc2108af34592 \ No newline at end of file diff --git a/src/util.c b/src/util.c index 937067a06f..d83a63015f 100644 --- a/src/util.c +++ b/src/util.c @@ -520,7 +520,7 @@ int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc){ testcase( i==18 ); testcase( i==19 ); testcase( i==20 ); - if( (c+nonNum!=0 && &zNum[i]19*incr ){ + if( (c!=0 && &zNum[i]19*incr || nonNum ){ /* zNum is empty or contains non-numeric text or is longer ** than 19 digits (thus guaranteeing that it is too large) */ return 1; From 50126644a52ccbdf8af18da9552d8b1d594b642e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 29 Mar 2013 13:44:34 +0000 Subject: [PATCH 053/204] Version 3.7.16.1 FossilOrigin-Name: 527231bc67285f01fb18d4451b28f61da3c4e39d --- manifest | 11 +++++++---- manifest.uuid | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/manifest b/manifest index 9ccb151d94..5cf11cc2fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjustment\sto\san\sextreme\scorner\scase\sin\sthe\ssqlite3Atoi64()\sroutine. -D 2013-03-28T11:40:14.945 +C Version\s3.7.16.1 +D 2013-03-29T13:44:34.265 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1041,7 +1041,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 274d2a22660c7b34b8bbd85f3c29cbafbcb1b4e7 +P 39534b4baa53b9360696a9d5ee8cc2108af34592 R e8e4e98970281a95c0de62c6e5f4d562 +T +bgcolor * #d0c0ff +T +sym-release * +T +sym-version-3.7.16.1 * U drh -Z e440add500ad1771a8d916396dcab1f1 +Z 17d1076ce5eb9d83e75734278cd22a73 diff --git a/manifest.uuid b/manifest.uuid index d68efa3fcb..68abab9466 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -39534b4baa53b9360696a9d5ee8cc2108af34592 \ No newline at end of file +527231bc67285f01fb18d4451b28f61da3c4e39d \ No newline at end of file From fde3b1c71596165d04dc3e4a46ae70f173a1adb3 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 29 Mar 2013 19:52:04 +0000 Subject: [PATCH 054/204] For the clean target, tidy up a couple more generated (or copied) source files. FossilOrigin-Name: 5687e5ee7bafa00d2b353c3eda1e5dfb219cb185 --- Makefile.in | 2 ++ Makefile.msc | 1 + Makefile.vxworks | 2 ++ main.mk | 2 ++ manifest | 23 ++++++++++------------- manifest.uuid | 2 +- 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/Makefile.in b/Makefile.in index 1ecaf2bbd5..edd29ae011 100644 --- a/Makefile.in +++ b/Makefile.in @@ -946,6 +946,8 @@ clean: rm -f testfixture$(TEXE) test.db rm -f sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def rm -f sqlite3.c + rm -f sqlite3rc.h + rm -f shell.c sqlite3ext.h rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c rm -f sqlite-*-output.vsix diff --git a/Makefile.msc b/Makefile.msc index df6ed4d46d..70be76afa8 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1250,6 +1250,7 @@ clean: del /Q sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def del /Q sqlite3.c del /Q sqlite3rc.h + del /Q shell.c sqlite3ext.h del /Q sqlite3_analyzer.exe sqlite3_analyzer.exp sqlite3_analyzer.c del /Q sqlite-*-output.vsix diff --git a/Makefile.vxworks b/Makefile.vxworks index d04028833c..c9058da3e5 100644 --- a/Makefile.vxworks +++ b/Makefile.vxworks @@ -662,4 +662,6 @@ clean: rm -rf tsrc target_source rm -f testloadext.dll libtestloadext.so rm -f sqlite3.c fts?amal.c tclsqlite3.c + rm -f sqlite3rc.h + rm -f shell.c sqlite3ext.h rm -f $(SHPREFIX)sqlite3.$(SO) diff --git a/main.mk b/main.mk index d644079f59..1294121c0a 100644 --- a/main.mk +++ b/main.mk @@ -622,5 +622,7 @@ clean: rm -f testfixture testfixture.exe rm -f threadtest3 threadtest3.exe rm -f sqlite3.c fts?amal.c tclsqlite3.c + rm -f sqlite3rc.h + rm -f shell.c sqlite3ext.h rm -f sqlite3_analyzer sqlite3_analyzer.exe sqlite3_analyzer.c rm -f sqlite-*-output.vsix diff --git a/manifest b/manifest index 5cf11cc2fb..453b33c22e 100644 --- a/manifest +++ b/manifest @@ -1,10 +1,10 @@ -C Version\s3.7.16.1 -D 2013-03-29T13:44:34.265 +C For\sthe\sclean\starget,\stidy\sup\sa\scouple\smore\sgenerated\s(or\scopied)\ssource\sfiles. +D 2013-03-29T19:52:04.965 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 +F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 0c1abc21c8deefc88e8a32ad6a07e5f96e158761 -F Makefile.vxworks b18ad88e9a8c6a001f5cf4a389116a4f1a7ab45f +F Makefile.msc 9ab787d472c9cb8bad29c2c4aeba15b06fba7bc4 +F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 05604ccde96fe1b37f922eddbdcb5654308261db F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -103,7 +103,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk c5437f64aad34844d18efe7b2d2adaa36efca49c +F main.mk b25a7a0b657409b2dcf9b09d39562703d5df702d F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -1041,10 +1041,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 39534b4baa53b9360696a9d5ee8cc2108af34592 -R e8e4e98970281a95c0de62c6e5f4d562 -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.7.16.1 * -U drh -Z 17d1076ce5eb9d83e75734278cd22a73 +P 527231bc67285f01fb18d4451b28f61da3c4e39d +R c463228a20804b5f57c5669f0d0e45b8 +U mistachkin +Z bfb2a9ca80a29b00e5f7b4db495d38d4 diff --git a/manifest.uuid b/manifest.uuid index 68abab9466..6960809f06 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -527231bc67285f01fb18d4451b28f61da3c4e39d \ No newline at end of file +5687e5ee7bafa00d2b353c3eda1e5dfb219cb185 \ No newline at end of file From 71be5c5c876747be8a8352170519271a20b7ca9f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 1 Apr 2013 14:29:33 +0000 Subject: [PATCH 055/204] Fix a problem in btree.c that could cause a crash following an OOM. FossilOrigin-Name: b724aa5889f2e09cf0cbef133a672e6486755104 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 11 ++++++----- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 715e199440..8a6c9f5c8d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scomment\sin\sos_unix.c.\s\sNo\scode\schanges. -D 2013-03-26T18:48:11.374 +C Fix\sa\sproblem\sin\sbtree.c\sthat\scould\scause\sa\scrash\sfollowing\san\sOOM. +D 2013-04-01T14:29:33.960 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 19b587b32569d4b92651b5eaa9de2e59c6f8f704 +F src/btree.c 84c9a4a20b827a6f7dbcbcda59607dbee57af446 F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 611bd824c24a60d298f28705de323fa2e813a308 -R 8f591838a5bc479ac7493be19f46439f -U drh -Z be06b34348d93c4a77e7034db0a8802f +P 72813b8ec924b91583c679668f7c4561dff82a02 +R ff1f51b04ee42accb0ddfde388849db6 +U dan +Z fbb073bc333f3d3b9c496b3226200b28 diff --git a/manifest.uuid b/manifest.uuid index 855425c986..069c9bb2ee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -72813b8ec924b91583c679668f7c4561dff82a02 \ No newline at end of file +b724aa5889f2e09cf0cbef133a672e6486755104 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 7d2f2d3840..03764a9267 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2596,12 +2596,13 @@ static int btreeSwapOutMmap(BtShared *pBt){ if( pPg && pPg->pDbPage->flags & PGHDR_MMAP ){ MemPage *pNew = 0; rc = getAndInitPage(pBt, pPg->pgno, &pNew, 0); - if( rc==SQLITE_OK && pCsr->iPage==0 ){ - pCsr->info.pCell = pNew->aData + (pCsr->info.pCell - pPg->aData); + if( rc==SQLITE_OK ){ + if( pCsr->iPage==0 ){ + pCsr->info.pCell = pNew->aData + (pCsr->info.pCell - pPg->aData); + } + pCsr->apPage[0] = pNew; + releasePage(pPg); } - pCsr->apPage[0] = pNew; - releasePage(pPg); - if( rc!=SQLITE_OK ) return rc; } } } From 0943f0bdc870249bf1d2e8692de9fd2f296190a3 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 1 Apr 2013 14:35:01 +0000 Subject: [PATCH 056/204] Fix a typo in a log message in wal.c. FossilOrigin-Name: 83fc48d16572443373e5de62adbd4cd53c2f7786 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 8a6c9f5c8d..2da91083ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sbtree.c\sthat\scould\scause\sa\scrash\sfollowing\san\sOOM. -D 2013-04-01T14:29:33.960 +C Fix\sa\stypo\sin\sa\slog\smessage\sin\swal.c. +D 2013-04-01T14:35:01.078 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c 4fad64071ae120c25f39dcac572d716b9cadeb7f F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 -F src/wal.c e84eff498c57ec2d79ca4496a3f4a638af378fb3 +F src/wal.c 04029f2691e937d7f5dd638580679a560fec93bf F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b F src/where.c 9a16c0b84bbeb054d11fda96e9e037ae310bd54e @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 72813b8ec924b91583c679668f7c4561dff82a02 -R ff1f51b04ee42accb0ddfde388849db6 +P b724aa5889f2e09cf0cbef133a672e6486755104 +R 96d4d8288650bd963603cf34d02b8e9c U dan -Z fbb073bc333f3d3b9c496b3226200b28 +Z 48fdc69d5f63c210da250932a430f955 diff --git a/manifest.uuid b/manifest.uuid index 069c9bb2ee..9b3ffd9c61 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b724aa5889f2e09cf0cbef133a672e6486755104 \ No newline at end of file +83fc48d16572443373e5de62adbd4cd53c2f7786 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 4602eb2c34..abbd7032f5 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1208,7 +1208,7 @@ finished: */ if( pWal->hdr.nPage ){ sqlite3_log(SQLITE_OK, "Recovered %d frames from WAL file %s", - pWal->hdr.nPage, pWal->zWalName + pWal->hdr.mxFrame, pWal->zWalName ); } } From f55a4cf80c73937610e8b39d553f2c8aea02af8f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 1 Apr 2013 16:56:41 +0000 Subject: [PATCH 057/204] Ensure that a checkpoint does not use an out-of-date mapping. FossilOrigin-Name: a1040f0397d57855500926494c978623286ddc77 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 2da91083ab..334e902f99 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sa\slog\smessage\sin\swal.c. -D 2013-04-01T14:35:01.078 +C Ensure\sthat\sa\scheckpoint\sdoes\snot\suse\san\sout-of-date\smapping. +D 2013-04-01T16:56:41.264 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c 4fad64071ae120c25f39dcac572d716b9cadeb7f F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 -F src/wal.c 04029f2691e937d7f5dd638580679a560fec93bf +F src/wal.c 94b5fed2df988fb12f5bf17256e2840e56957a85 F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b F src/where.c 9a16c0b84bbeb054d11fda96e9e037ae310bd54e @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b724aa5889f2e09cf0cbef133a672e6486755104 -R 96d4d8288650bd963603cf34d02b8e9c +P 83fc48d16572443373e5de62adbd4cd53c2f7786 +R 8c156c43b98e65e90e42898654b4787f U dan -Z 48fdc69d5f63c210da250932a430f955 +Z 8dac7ace59503631e98c649abc9ec1b4 diff --git a/manifest.uuid b/manifest.uuid index 9b3ffd9c61..b6b3710dd3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83fc48d16572443373e5de62adbd4cd53c2f7786 \ No newline at end of file +a1040f0397d57855500926494c978623286ddc77 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index abbd7032f5..881f0a4c78 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2956,6 +2956,9 @@ int sqlite3WalCheckpoint( /* Read the wal-index header. */ if( rc==SQLITE_OK ){ rc = walIndexReadHdr(pWal, &isChanged); + if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ + sqlite3OsUnfetch(pWal->pDbFd, 0, 0); + } } /* Copy data from the log to the database file. */ From 5175b324b84ecab7132382bbb8de698988fcd4b9 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Apr 2013 17:22:51 +0000 Subject: [PATCH 058/204] Add xFetch and xUnfetch methods to the os_win.c VFS. FossilOrigin-Name: a1653a257d6af6e8b10c819e68b12f6c2f485811 --- manifest | 14 +-- manifest.uuid | 2 +- src/os_win.c | 288 ++++++++++++++++++++++++++++++-------------------- 3 files changed, 182 insertions(+), 122 deletions(-) diff --git a/manifest b/manifest index 334e902f99..e6eca96ae6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sa\scheckpoint\sdoes\snot\suse\san\sout-of-date\smapping. -D 2013-04-01T16:56:41.264 +C Add\sxFetch\sand\sxUnfetch\smethods\sto\sthe\sos_win.c\sVFS. +D 2013-04-01T17:22:51.830 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d6981218583748080374ed98a03f6a87e2bdc9e0 -F src/os_win.c e4f17ddf79f2a9373e33ed70565e765d65324589 +F src/os_win.c f705e7ce230f86104dedcc2987a21d564b236659 F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 83fc48d16572443373e5de62adbd4cd53c2f7786 -R 8c156c43b98e65e90e42898654b4787f -U dan -Z 8dac7ace59503631e98c649abc9ec1b4 +P a1040f0397d57855500926494c978623286ddc77 +R 52b13f77704d4760e77e0cfd9ab57bff +U drh +Z 878a27609130450cef13cc90a8feab48 diff --git a/manifest.uuid b/manifest.uuid index b6b3710dd3..c685b6feac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a1040f0397d57855500926494c978623286ddc77 \ No newline at end of file +a1653a257d6af6e8b10c819e68b12f6c2f485811 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index a836664565..b1f5748638 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -150,9 +150,12 @@ struct winFile { winceLock local; /* Locks obtained by this instance of winFile */ winceLock *shared; /* Global shared lock memory for the file */ #endif - HANDLE hMap; /* Handle for accessing memory mapping */ - void *pMapRegion; /* Area memory mapped */ - sqlite3_int64 mmapSize; /* Size of xMremap() */ + int nFetchOut; /* Number of outstanding xFetch references */ + HANDLE hMap; /* Handle for accessing memory mapping */ + void *pMapRegion; /* Area memory mapped */ + sqlite3_int64 mmapSize; /* Usable size of mapped region */ + sqlite3_int64 mmapOrigsize; /* Actual size of mapped region */ + sqlite3_int64 mmapLimit; /* Configured FCNTL_MMAP_LIMIT value */ }; /* @@ -2066,7 +2069,7 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ } /* Forward references to VFS methods */ -static int winUnmap(sqlite3_file *); +static int winUnmapfile(winFile*); /* ** Close a file. @@ -2090,7 +2093,7 @@ static int winClose(sqlite3_file *id){ OSTRACE(("CLOSE %d\n", pFile->h)); assert( pFile->h!=NULL && pFile->h!=INVALID_HANDLE_VALUE ); - rc = winUnmap(id); + rc = winUnmapfile(pFile); if( rc!=SQLITE_OK ) return rc; do{ @@ -2142,10 +2145,24 @@ static int winRead( assert( id!=0 ); assert( amt>0 ); - assert( offset>=pFile->mmapSize ); /* Never read from the mmapped region */ SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } + #if SQLITE_OS_WINCE if( seekWinFile(pFile, offset) ){ return SQLITE_FULL; @@ -2189,13 +2206,27 @@ static int winWrite( int nRetry = 0; /* Number of retries */ assert( amt>0 ); - assert( offset>=pFile->mmapSize ); /* Never write into the mmapped region */ assert( pFile ); SimulateIOError(return SQLITE_IOERR_WRITE); SimulateDiskfullError(return SQLITE_FULL); OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } + #if SQLITE_OS_WINCE rc = seekWinFile(pFile, offset); if( rc==0 ){ @@ -2288,14 +2319,14 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ pFile->lastErrno = lastErrno; rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, "winTruncate2", pFile->zPath); - }else{ - /* If the file was just truncated to a size smaller than the currently - ** mapped region, reduce the effective mapping size as well. SQLite will - ** use read() and write() to access data beyond this point from now on. - */ - if( pFile->pMapRegion && nBytemmapSize ){ - pFile->mmapSize = nByte; - } + } + + /* If the file was truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( pFile->pMapRegion && nBytemmapSize ){ + pFile->mmapSize = nByte; } OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); @@ -2805,6 +2836,10 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } + case SQLITE_FCNTL_MMAP_LIMIT: { + pFile->mmapLimit = *(i64*)pArg; + return SQLITE_OK; + } } return SQLITE_NOTFOUND; } @@ -3475,37 +3510,25 @@ shmpage_out: # define winShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ -/* -** Arguments x and y are both integers. Argument y must be a power of 2. -** Round x up to the nearest integer multiple of y. For example: -** -** ROUNDUP(0, 8) -> 0 -** ROUNDUP(13, 8) -> 16 -** ROUNDUP(32, 8) -> 32 -*/ -#define ROUNDUP(x,y) (((x)+y-1)&~(y-1)) - /* ** Cleans up the mapped region of the specified file, if any. */ -static int winUnmap( - sqlite3_file *id -){ - winFile *pFile = (winFile*)id; +static int winUnmapfile(winFile *pFile){ assert( pFile!=0 ); if( pFile->pMapRegion ){ if( !osUnmapViewOfFile(pFile->pMapRegion) ){ pFile->lastErrno = osGetLastError(); - return winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, "winUnmap1", pFile->zPath); } pFile->pMapRegion = 0; pFile->mmapSize = 0; + pFile->mmapOrigsize = 0; } if( pFile->hMap!=NULL ){ if( !osCloseHandle(pFile->hMap) ){ pFile->lastErrno = osGetLastError(); - return winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, "winUnmap2", pFile->zPath); } pFile->hMap = NULL; @@ -3514,116 +3537,150 @@ static int winUnmap( } /* -** Map, remap or unmap part of the database file. +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_LIMIT, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. */ -static int winMremap( - sqlite3_file *id, /* Main database file */ - int flags, /* Mask of SQLITE_MREMAP_XXX flags */ - sqlite3_int64 iOff, /* Offset to start mapping at */ - sqlite3_int64 nOld, /* Size of old mapping, or zero */ - sqlite3_int64 nNew, /* Size of new mapping, or zero */ - void **ppMap /* IN/OUT: Old/new mappings */ -){ - winFile *pFile = (winFile*)id; /* The underlying database file */ - int rc = SQLITE_OK; /* Return code */ +static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ + sqlite3_int64 nMap = nByte; + int rc; HANDLE hMap = NULL; /* New mapping handle */ - void *pNew = 0; /* New mapping */ - i64 nNewRnd; /* nNew rounded up */ - i64 nOldRnd; /* nOld rounded up */ - assert( pFile!=0 ); - assert( iOff==0 ); - assert( nOld>=0 ); - assert( nOld==0 || pFile->pMapRegion==(*ppMap) ); - assert( nNew>=0 ); - assert( ppMap ); - assert( pFile->hMap==NULL || pFile->pMapRegion==(*ppMap) ); - assert( pFile->pMapRegion==0 || pFile->pMapRegion==(*ppMap) ); - /* assert( pFile->mmapSize==nOld ); */ + assert( nMap>=0 || pFd->nFetchOut==0 ); + if( pFd->nFetchOut>0 ) return SQLITE_OK; - assert( winSysInfo.dwPageSize>0 ); - nNewRnd = ROUNDUP(nNew, winSysInfo.dwPageSize*1); - assert( nNewRnd>=0 ); - nOldRnd = ROUNDUP(nOld, winSysInfo.dwPageSize*1); - assert( nOldRnd>=0 ); - - if( nNewRnd==nOldRnd ){ - pFile->mmapSize = nNew; - return SQLITE_OK; - } - - /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested - ** mapping (nNew bytes) may be greater than the size of the database file. - ** If this is the case, extend the file on disk using ftruncate(). */ - assert( nNewRnd>0 || (flags & SQLITE_MREMAP_EXTEND)==0 ); - if( flags & SQLITE_MREMAP_EXTEND ){ - sqlite3_int64 oldSz; - rc = winFileSize(id, &oldSz); - if( rc==SQLITE_OK && nNewRnd>oldSz ){ - rc = winUnmap(id); /* Cannot truncate the file with an open mapping. */ - if( rc==SQLITE_OK ){ - rc = winTruncate(id, nNewRnd); - } + if( nMap<0 ){ + rc = winFileSize((sqlite3_file*)pFd, &nMap); + if( rc ){ + return SQLITE_IOERR_FSTAT; } - if( rc!=SQLITE_OK ) return rc; } - - /* If we get this far, unmap any old mapping. */ - rc = winUnmap(id); - if( rc!=SQLITE_OK ) return rc; - - /* And, if required, create a new mapping. */ - if( nNewRnd>0 ){ - i64 offset = ((iOff / winSysInfo.dwAllocationGranularity) * - (winSysInfo.dwAllocationGranularity)); + if( nMap>pFd->mmapLimit ){ + nMap = pFd->mmapLimit; + } + + if( nMap==0 && pFd->mmapSize>0 ){ + winUnmapfile(pFd); + } + if( nMap!=pFd->mmapSize ){ + void *pNew = 0; DWORD protect = PAGE_READONLY; DWORD flags = FILE_MAP_READ; - if( (pFile->ctrlFlags & WINFILE_RDONLY)==0 ){ + + winUnmapfile(pFd); + if( (pFd->ctrlFlags & WINFILE_RDONLY)==0 ){ protect = PAGE_READWRITE; flags |= FILE_MAP_WRITE; } #if SQLITE_OS_WINRT - hMap = osCreateFileMappingFromApp(pFile->h, NULL, protect, nNewRnd, NULL); + hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); #elif defined(SQLITE_WIN32_HAS_WIDE) - hMap = osCreateFileMappingW(pFile->h, NULL, protect, - (DWORD)((nNewRnd>>32) & 0xffffffff), - (DWORD)(nNewRnd & 0xffffffff), NULL); + hMap = osCreateFileMappingW(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); #elif defined(SQLITE_WIN32_HAS_ANSI) - hMap = osCreateFileMappingA(pFile->h, NULL, protect, - (DWORD)((nNewRnd>>32) & 0xffffffff), - (DWORD)(nNewRnd & 0xffffffff), NULL); + hMap = osCreateFileMappingA(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); #endif if( hMap==NULL ){ - pFile->lastErrno = osGetLastError(); - rc = winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, - "winMremap1", pFile->zPath); - return rc; + pFd->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile", pFd->zPath); + /* Log the error, but continue normal operation using xRead/xWrite */ + return SQLITE_OK; } assert( (nNewRnd % winSysInfo.dwPageSize)==0 ); #if SQLITE_OS_WINRT - pNew = osMapViewOfFileFromApp(hMap, flags, offset, nNewRnd); + pNew = osMapViewOfFileFromApp(hMap, flags, 0, nMap); #else - assert( sizeof(SIZE_T)==sizeof(sqlite3_int64) || nNewRnd<=0xffffffff ); - pNew = osMapViewOfFile(hMap, flags, - (DWORD)((offset>>32) & 0xffffffff), - (DWORD)(offset & 0xffffffff), - (SIZE_T)nNewRnd); + pNew = osMapViewOfFile(hMap, flags, 0, 0, (SIZE_T)nMap); #endif if( pNew==NULL ){ osCloseHandle(hMap); hMap = NULL; - pFile->lastErrno = osGetLastError(); - rc = winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, - "winMremap2", pFile->zPath); + pFd->lastErrno = osGetLastError(); + winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile", pFd->zPath); + return SQLITE_OK; } + pFd->pMapRegion = pNew; + pFd->mmapSize = nMap; + pFd->mmapOrigsize = nMap; } - pFile->hMap = hMap; - pFile->pMapRegion = pNew; - pFile->mmapSize = nNew; + return SQLITE_OK; +} - *ppMap = pNew; - return rc; +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling unixUnfetch(). +*/ +static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ + winFile *pFd = (winFile*)fd; /* The underlying database file */ + *pp = 0; + + if( pFd->mmapLimit>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = winMapfile(pFd, -1); + if( rc!=SQLITE_OK ) return rc; + } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } + return SQLITE_OK; +} + +/* +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to unixFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the unixFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. +*/ +static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ + winFile *pFd = (winFile*)fd; /* The underlying database file */ + + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ + assert( (p==0)==(pFd->nFetchOut==0) ); + + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + + if( p ){ + pFd->nFetchOut--; + }else{ + winUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); + return SQLITE_OK; } /* @@ -3654,7 +3711,8 @@ static const sqlite3_io_methods winIoMethod = { winShmLock, /* xShmLock */ winShmBarrier, /* xShmBarrier */ winShmUnmap, /* xShmUnmap */ - winMremap, /* xMremap */ + winFetch, /* xFetch */ + winUnfetch /* xUnfetch */ }; /**************************************************************************** @@ -4052,6 +4110,8 @@ static int winOpen( pFile->hMap = NULL; pFile->pMapRegion = 0; pFile->mmapSize = 0; + pFile->mmapOrigsize = 0; + pFile->mmapLimit = SQLITE_DEFAULT_MMAP_LIMIT; OpenCounter(+1); return rc; From f9d18e472bd853c2849de8aee6b8498f20e254f4 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Apr 2013 17:56:28 +0000 Subject: [PATCH 059/204] Bug fix in the winMapfile() subroutine: Be sure to record the map object handle in the sqlite3_file object. FossilOrigin-Name: ee4d188e207efa24a26776fa4f025c6ac39cce73 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 17 ++++++++--------- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index e6eca96ae6..e46bdd10e1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sxFetch\sand\sxUnfetch\smethods\sto\sthe\sos_win.c\sVFS. -D 2013-04-01T17:22:51.830 +C Bug\sfix\sin\sthe\swinMapfile()\ssubroutine:\s\sBe\ssure\sto\srecord\sthe\smap\sobject\nhandle\sin\sthe\ssqlite3_file\sobject. +D 2013-04-01T17:56:28.844 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d6981218583748080374ed98a03f6a87e2bdc9e0 -F src/os_win.c f705e7ce230f86104dedcc2987a21d564b236659 +F src/os_win.c 390653f5c9b216bd445366f3c2b4863e52d51c24 F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a1040f0397d57855500926494c978623286ddc77 -R 52b13f77704d4760e77e0cfd9ab57bff +P a1653a257d6af6e8b10c819e68b12f6c2f485811 +R 88910ffb14df2e25c8fd9ccc5fce98d7 U drh -Z 878a27609130450cef13cc90a8feab48 +Z d2117afc31dea61942683c2e81660a00 diff --git a/manifest.uuid b/manifest.uuid index c685b6feac..2f1539dbd2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a1653a257d6af6e8b10c819e68b12f6c2f485811 \ No newline at end of file +ee4d188e207efa24a26776fa4f025c6ac39cce73 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index b1f5748638..5ce5a4762a 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3555,7 +3555,6 @@ static int winUnmapfile(winFile *pFile){ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ sqlite3_int64 nMap = nByte; int rc; - HANDLE hMap = NULL; /* New mapping handle */ assert( nMap>=0 || pFd->nFetchOut==0 ); if( pFd->nFetchOut>0 ) return SQLITE_OK; @@ -3584,17 +3583,17 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ flags |= FILE_MAP_WRITE; } #if SQLITE_OS_WINRT - hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); + pFd->hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); #elif defined(SQLITE_WIN32_HAS_WIDE) - hMap = osCreateFileMappingW(pFd->h, NULL, protect, + pFd->hMap = osCreateFileMappingW(pFd->h, NULL, protect, (DWORD)((nMap>>32) & 0xffffffff), (DWORD)(nMap & 0xffffffff), NULL); #elif defined(SQLITE_WIN32_HAS_ANSI) - hMap = osCreateFileMappingA(pFd->h, NULL, protect, + pFd->hMap = osCreateFileMappingA(pFd->h, NULL, protect, (DWORD)((nMap>>32) & 0xffffffff), (DWORD)(nMap & 0xffffffff), NULL); #endif - if( hMap==NULL ){ + if( pFd->hMap==NULL ){ pFd->lastErrno = osGetLastError(); rc = winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, "winMapfile", pFd->zPath); @@ -3603,13 +3602,13 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ } assert( (nNewRnd % winSysInfo.dwPageSize)==0 ); #if SQLITE_OS_WINRT - pNew = osMapViewOfFileFromApp(hMap, flags, 0, nMap); + pNew = osMapViewOfFileFromApp(pFd->hMap, flags, 0, nMap); #else - pNew = osMapViewOfFile(hMap, flags, 0, 0, (SIZE_T)nMap); + pNew = osMapViewOfFile(pFd->hMap, flags, 0, 0, (SIZE_T)nMap); #endif if( pNew==NULL ){ - osCloseHandle(hMap); - hMap = NULL; + osCloseHandle(pFd->hMap); + pFd->hMap = NULL; pFd->lastErrno = osGetLastError(); winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, "winMapfile", pFd->zPath); From e6ecd6630dd7fadec974ffd94ea678212e959717 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 1 Apr 2013 17:56:59 +0000 Subject: [PATCH 060/204] Attempt to emulate mremap() on non-Linux systems by allocating a second mapping immediately following the first in virtual memory. FossilOrigin-Name: 4d67433db8fb4754ae6b192945e479f3d7bad579 --- manifest | 14 +++--- manifest.uuid | 2 +- src/os_unix.c | 137 +++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 122 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index e6eca96ae6..52c1269c2e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sxFetch\sand\sxUnfetch\smethods\sto\sthe\sos_win.c\sVFS. -D 2013-04-01T17:22:51.830 +C Attempt\sto\semulate\smremap()\son\snon-Linux\ssystems\sby\sallocating\sa\ssecond\smapping\simmediately\sfollowing\sthe\sfirst\sin\svirtual\smemory. +D 2013-04-01T17:56:59.408 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c d6981218583748080374ed98a03f6a87e2bdc9e0 +F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c F src/os_win.c f705e7ce230f86104dedcc2987a21d564b236659 F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a1040f0397d57855500926494c978623286ddc77 -R 52b13f77704d4760e77e0cfd9ab57bff -U drh -Z 878a27609130450cef13cc90a8feab48 +P a1653a257d6af6e8b10c819e68b12f6c2f485811 +R 144bfd1ec5d6fde06365c0d3ea4b879a +U dan +Z 4e8a79b712821b5d3d9730a626894d79 diff --git a/manifest.uuid b/manifest.uuid index c685b6feac..694ae46ba8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a1653a257d6af6e8b10c819e68b12f6c2f485811 \ No newline at end of file +4d67433db8fb4754ae6b192945e479f3d7bad579 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index f91f4c4e7a..e92c7cc4a4 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -313,6 +313,17 @@ struct unixFile { #define threadid 0 #endif +/* +** HAVE_MREMAP defaults to true on Linux and false everywhere else. +*/ +#if !defined(HAVE_MREMAP) +# if defined(__linux__) && defined(_GNU_SOURCE) +# define HAVE_MREMAP 1 +# else +# define HAVE_MREMAP 0 +# endif +#endif + /* ** Different Unix systems declare open() in different ways. Same use ** open(const char*,int,mode_t). Others use open(const char*,int,...). @@ -450,7 +461,7 @@ static struct unix_syscall { { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, #define osMunmap ((void*(*)(void*,size_t))aSyscall[22].pCurrent) -#if defined(__linux__) && defined(_GNU_SOURCE) +#if HAVE_MREMAP { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, #else { "mremap", (sqlite3_syscall_ptr)0, 0 }, @@ -1124,7 +1135,6 @@ static int unixLogErrorAtLine( zErr = strerror(iErrno); #endif - assert( errcode!=SQLITE_OK ); if( zPath==0 ) zPath = ""; sqlite3_log(errcode, "os_unix.c:%d: (%d) %s(%s) - %s", @@ -3619,7 +3629,7 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ } } - if( pFile->mmapLimit>0 ){ + if( pFile->mmapLimit>0 && nByte>pFile->mmapSize ){ int rc; if( pFile->szChunk<=0 ){ if( robust_ftruncate(pFile->h, nByte) ){ @@ -4518,6 +4528,104 @@ static void unixUnmapfile(unixFile *pFd){ } } +/* +** Return the system page size. +*/ +static int unixGetPagesize(void){ +#if HAVE_MREMAP + return 512; +#elif _BSD_SOURCE + return getpagesize(); +#else + return (int)sysconf(_SC_PAGESIZE); +#endif +} + +/* +** Attempt to set the size of the memory mapping maintained by file +** descriptor pFd to nNew bytes. Any existing mapping is discarded. +** +** If successful, this function sets the following variables: +** +** unixFile.pMapRegion +** unixFile.mmapSize +** unixFile.mmapOrigsize +** +** If unsuccessful, an error message is logged via sqlite3_log() and +** the three variables above are zeroed. In this case SQLite should +** continue accessing the database using the xRead() and xWrite() +** methods. +*/ +static void unixRemapfile( + unixFile *pFd, /* File descriptor object */ + i64 nNew /* Required mapping size */ +){ + int h = pFd->h; /* File descriptor open on db file */ + u8 *pOrig = (u8 *)pFd->pMapRegion; /* Pointer to current file mapping */ + i64 nOrig = pFd->mmapOrigsize; /* Size of pOrig region in bytes */ + u8 *pNew = 0; /* Location of new mapping */ + int flags = PROT_READ; /* Flags to pass to mmap() */ + + assert( pFd->nFetchOut==0 ); + assert( nNew>pFd->mmapSize ); + assert( nNew<=pFd->mmapLimit ); + assert( nNew>0 ); + assert( pFd->mmapOrigsize>=pFd->mmapSize ); + + if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; + + if( pOrig ){ + const int szSyspage = unixGetPagesize(); + i64 nReuse = (pFd->mmapSize & ~(szSyspage-1)); + u8 *pReq = &pOrig[nReuse]; + + /* Unmap any pages of the existing mapping that cannot be reused. */ + if( nReuse!=nOrig ){ + osMunmap(pReq, nOrig-nReuse); + } + +#if HAVE_MREMAP + pNew = osMremap(pOrig, nReuse, nNew, MREMAP_MAYMOVE); +#else + pNew = osMmap(pReq, nNew-nReuse, flags, MAP_SHARED, h, nReuse); + if( pNew!=MAP_FAILED ){ + if( pNew!=pReq ){ + osMunmap(pNew, nNew - nReuse); + pNew = MAP_FAILED; + }else{ + pNew = pOrig; + } + } +#endif + + /* The attempt to extend the existing mapping failed. Free the existing + ** mapping and set pNew to NULL so that the code below will create a + ** new mapping from scratch. */ + if( pNew==MAP_FAILED ){ + pNew = 0; + osMunmap(pOrig, nReuse); + } + } + + /* If pNew is still NULL, try to create an entirely new mapping. */ + if( pNew==0 ){ + pNew = osMmap(0, nNew, flags, MAP_SHARED, h, 0); + if( pNew==MAP_FAILED ){ + pNew = 0; + nNew = 0; + unixLogError(SQLITE_OK, "mmap", pFd->zPath); + + /* If the mmap() above failed, assume that all subsequent mmap() calls + ** will probably fail too. Fall back to using xRead/xWrite exclusively + ** in this case. */ + pFd->mmapLimit = 0; + } + } + + pFd->pMapRegion = (void *)pNew; + pFd->mmapSize = pFd->mmapOrigsize = nNew; +} + /* ** Memory map or remap the file opened by file-descriptor pFd (if the file ** is already mapped, the existing mapping is replaced by the new). Or, if @@ -4554,28 +4662,11 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ } if( nMap!=pFd->mmapSize ){ - void *pNew = 0; - -#if defined(__linux__) && defined(_GNU_SOURCE) - if( pFd->pMapRegion && nMap>0 ){ - pNew = osMremap(pFd->pMapRegion, pFd->mmapOrigsize, nMap, MREMAP_MAYMOVE); - }else -#endif - { + if( nMap>0 ){ + unixRemapfile(pFd, nMap); + }else{ unixUnmapfile(pFd); - if( nMap>0 ){ - int flags = PROT_READ; - if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - pNew = osMmap(0, nMap, flags, MAP_SHARED, pFd->h, 0); - } } - - if( pNew==MAP_FAILED ){ - return SQLITE_IOERR_MMAP; - } - pFd->pMapRegion = pNew; - pFd->mmapSize = nMap; - pFd->mmapOrigsize = nMap; } return SQLITE_OK; From db56bcbe42c7d847eda67533717316e4ff6212ae Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Apr 2013 18:15:50 +0000 Subject: [PATCH 061/204] In windows, round down the size of the memory mapped region to the next smaller multiple of the page size. FossilOrigin-Name: 0e44ed1badde6d766f18d1373d968e5b286b15bd --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ed4a17e30f..d43ccc6be3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\saccidental\sfork. -D 2013-04-01T17:58:10.824 +C In\swindows,\sround\sdown\sthe\ssize\sof\sthe\smemory\smapped\sregion\sto\sthe\snext\nsmaller\smultiple\sof\sthe\spage\ssize. +D 2013-04-01T18:15:50.853 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c -F src/os_win.c 390653f5c9b216bd445366f3c2b4863e52d51c24 +F src/os_win.c 4cb9b2621f4869e17c8caa627903dc114b1dfb37 F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 4d67433db8fb4754ae6b192945e479f3d7bad579 ee4d188e207efa24a26776fa4f025c6ac39cce73 -R 5abed6f6d2f091dbf577966219204ad2 -U dan -Z 994fbbedca554dbff615409d58ee1a84 +P 5f4437c0e35c8aa740515aac3e81e0146716d2bb +R bde1c152af7414f4a8d87d63f33cdc11 +U drh +Z 9887751c05c44b393c15dab1850774d3 diff --git a/manifest.uuid b/manifest.uuid index 5c2360cd0a..91d9770506 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5f4437c0e35c8aa740515aac3e81e0146716d2bb \ No newline at end of file +0e44ed1badde6d766f18d1373d968e5b286b15bd \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 5ce5a4762a..6965ee90bb 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3568,6 +3568,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ if( nMap>pFd->mmapLimit ){ nMap = pFd->mmapLimit; } + nMap &= ~(sqlite3_int64)(winSysInfo.dwPageSize - 1); if( nMap==0 && pFd->mmapSize>0 ){ winUnmapfile(pFd); From a539c8a4642c91fed157ee48ddf60f92985c5ac9 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Apr 2013 18:25:48 +0000 Subject: [PATCH 062/204] Add a comment to xUnfetch on windows to say that the call to winUnmapfile() might be unnecessary. FossilOrigin-Name: e3c63c31a21b27806592d066a013a111b280c61f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index d43ccc6be3..9d43650636 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\swindows,\sround\sdown\sthe\ssize\sof\sthe\smemory\smapped\sregion\sto\sthe\snext\nsmaller\smultiple\sof\sthe\spage\ssize. -D 2013-04-01T18:15:50.853 +C Add\sa\scomment\sto\sxUnfetch\son\swindows\sto\ssay\sthat\sthe\scall\sto\swinUnmapfile()\nmight\sbe\sunnecessary. +D 2013-04-01T18:25:48.030 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c -F src/os_win.c 4cb9b2621f4869e17c8caa627903dc114b1dfb37 +F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5f4437c0e35c8aa740515aac3e81e0146716d2bb -R bde1c152af7414f4a8d87d63f33cdc11 +P 0e44ed1badde6d766f18d1373d968e5b286b15bd +R 8c3ade0fe410bf8f08819b27491eb958 U drh -Z 9887751c05c44b393c15dab1850774d3 +Z 1d97c5c3c6a55788c16fe769f3e976ae diff --git a/manifest.uuid b/manifest.uuid index 91d9770506..224e9238f1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0e44ed1badde6d766f18d1373d968e5b286b15bd \ No newline at end of file +e3c63c31a21b27806592d066a013a111b280c61f \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 6965ee90bb..ee7a71018e 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3676,6 +3676,10 @@ static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ if( p ){ pFd->nFetchOut--; }else{ + /* FIXME: If Windows truly always prevents truncating or deleting a + ** file while a mapping is held, then the following winUnmapfile() call + ** is unnecessary can can be omitted - potentially improving + ** performance. */ winUnmapfile(pFd); } From a1f42c7c32b7326568305f64cf5c592910928a3e Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Apr 2013 22:38:06 +0000 Subject: [PATCH 063/204] Add the SQLITE_CONFIG_MMAP_LIMIT configuration option for overriding the SQLITE_DEFAULT_MMAP_LIMIT compile-time setting. Enhance "PRAGMA mmap_limit" so that without a specific database name, it sets the limit on all database files and changes the default for any future databases that might be added using ATTACH. FossilOrigin-Name: 78141d0a16dd1d56b575fccd149de7fa789cb06c --- manifest | 26 +++++++++++++------------- manifest.uuid | 2 +- src/btree.c | 1 + src/global.c | 1 + src/main.c | 8 ++++++++ src/os_unix.c | 1 + src/pager.c | 2 +- src/pragma.c | 13 +++++++++++-- src/sqlite.h.in | 14 ++++++++++++++ src/sqliteInt.h | 2 ++ 10 files changed, 53 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 9d43650636..995cb45fa0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\scomment\sto\sxUnfetch\son\swindows\sto\ssay\sthat\sthe\scall\sto\swinUnmapfile()\nmight\sbe\sunnecessary. -D 2013-04-01T18:25:48.030 +C Add\sthe\sSQLITE_CONFIG_MMAP_LIMIT\sconfiguration\soption\sfor\soverriding\sthe\nSQLITE_DEFAULT_MMAP_LIMIT\scompile-time\ssetting.\s\sEnhance\s"PRAGMA\smmap_limit"\nso\sthat\swithout\sa\sspecific\sdatabase\sname,\sit\ssets\sthe\slimit\son\sall\sdatabase\nfiles\sand\schanges\sthe\sdefault\sfor\sany\sfuture\sdatabases\sthat\smight\sbe\sadded\nusing\sATTACH. +D 2013-04-01T22:38:06.929 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 84c9a4a20b827a6f7dbcbcda59607dbee57af446 +F src/btree.c bb679fddb7269dd7cfb950f845b1b18f19df4f4a F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -134,7 +134,7 @@ F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c e16942bd5c8a868ac53287886464a5ed0e72b179 F src/func.c 48987c025d69399f59a1c2a553cea5da41bf105d -F src/global.c e59ecd2c553ad0d4bfbc84ca71231336f8993a7a +F src/global.c d2494a1cea8f66a2cab8258449df07f8f0ae6330 F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4 F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -143,7 +143,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 1422eba4aa2b1fb5f7b3aef574752272477d21e2 -F src/main.c 379160ec3680e3009aa4978eac47027c3ef27ac5 +F src/main.c 54a841854734b6731c4d026834788cac6a19f3d1 F src/malloc.c fe085aa851b666b7c375c1ff957643dc20a04bf6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa @@ -160,15 +160,15 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c +F src/os_unix.c 9912ab0cfcd7fa2b058544d861a9fead3adac8da F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 -F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec +F src/pager.c 072512d05dacd61b1f147736825bd32fdc890ba4 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c 8875d32d54530300e6aa31a9d5426b2088a69b12 +F src/pragma.c d0231f412dea33c35c1309b45b535cf0c10c63c8 F src/prepare.c 310eaff1ee5f3c700b3545afb095cfe9346efc3a F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -176,10 +176,10 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in 921ba42ec3531fe8cf021492ffd781b161383f1b +F src/sqlite.h.in 42c69ea00dfcce84394d6ce0d8dac7bb3d2ecff2 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 -F src/sqliteInt.h 0f8f05ee4db4ba9120b38f7a3992b325698f6e8a +F src/sqliteInt.h 1664dc5ad6f8d4dab871416628aa3271044d66c0 F src/sqliteLimit.h 83d0cd0ce3050a80930406d3c7aecb48c1ccdfc5 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 0e44ed1badde6d766f18d1373d968e5b286b15bd -R 8c3ade0fe410bf8f08819b27491eb958 +P e3c63c31a21b27806592d066a013a111b280c61f +R 6d1c950d2134f5ef56d487146725ba7a U drh -Z 1d97c5c3c6a55788c16fe769f3e976ae +Z 72c1547cadf3e5127066e34139489015 diff --git a/manifest.uuid b/manifest.uuid index 224e9238f1..838dd7eeb0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e3c63c31a21b27806592d066a013a111b280c61f \ No newline at end of file +78141d0a16dd1d56b575fccd149de7fa789cb06c \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 03764a9267..9672e0e000 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1864,6 +1864,7 @@ int sqlite3BtreeOpen( rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, EXTRA_SIZE, flags, vfsFlags, pageReinit); if( rc==SQLITE_OK ){ + sqlite3PagerSetMmapLimit(pBt->pPager, db->mxMmap); rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); } if( rc!=SQLITE_OK ){ diff --git a/src/global.c b/src/global.c index f5da7e7f1f..aec3958c1f 100644 --- a/src/global.c +++ b/src/global.c @@ -156,6 +156,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { (void*)0, /* pHeap */ 0, /* nHeap */ 0, 0, /* mnHeap, mxHeap */ + SQLITE_DEFAULT_MMAP_LIMIT, /* mxMmap */ (void*)0, /* pScratch */ 0, /* szScratch */ 0, /* nScratch */ diff --git a/src/main.c b/src/main.c index 2fcb236e83..288d4535f3 100644 --- a/src/main.c +++ b/src/main.c @@ -496,6 +496,13 @@ int sqlite3_config(int op, ...){ } #endif + case SQLITE_CONFIG_MMAP_LIMIT: { + sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); + if( mxMmap<0 ) mxMmap = SQLITE_DEFAULT_MMAP_LIMIT; + sqlite3GlobalConfig.mxMmap = mxMmap; + break; + } + default: { rc = SQLITE_ERROR; break; @@ -2316,6 +2323,7 @@ static int openDatabase( memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); db->autoCommit = 1; db->nextAutovac = -1; + db->mxMmap = sqlite3GlobalConfig.mxMmap; db->nextPagesize = 0; db->flags |= SQLITE_ShortColNames | SQLITE_AutoIndex | SQLITE_EnableTrigger #if SQLITE_DEFAULT_FILE_FORMAT<4 diff --git a/src/os_unix.c b/src/os_unix.c index e92c7cc4a4..c10841abf9 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3711,6 +3711,7 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } case SQLITE_FCNTL_MMAP_LIMIT: { pFile->mmapLimit = *(i64*)pArg; +printf("MMAP-LIMIT(%s) -> %lld\n", pFile->zPath, pFile->mmapLimit); return SQLITE_OK; } #ifdef SQLITE_DEBUG diff --git a/src/pager.c b/src/pager.c index 550c6c7564..68b2e2d667 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4749,7 +4749,7 @@ int sqlite3PagerOpen( /* pPager->pBusyHandlerArg = 0; */ pPager->xReiniter = xReinit; /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ - pPager->mxMmap = SQLITE_DEFAULT_MMAP_LIMIT; + /* pPager->mxMmap = SQLITE_DEFAULT_MMAP_LIMIT // will be set by btree.c */ *ppPager = pPager; return SQLITE_OK; diff --git a/src/pragma.c b/src/pragma.c index cd1cfbe213..7a515cba2e 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -750,7 +750,9 @@ void sqlite3Pragma( ** Used to set mapping size limit. The mapping size limit is ** used to limit the aggregate size of all memory mapped regions of the ** database file. If this parameter is set to zero, then memory mapping - ** is not used at all. The parameter N is measured in bytes. + ** is not used at all. If N is negative, then the default memory map + ** limit determined by sqlite3_config(SQLITE_CONFIG_MMAP_LIMIT) is set. + ** The parameter N is measured in bytes. ** ** This value is advisory. The underlying VFS is free to memory map ** as little or as much as it wants. Except, if N is set to 0 then the @@ -760,8 +762,15 @@ void sqlite3Pragma( assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( zRight ){ sqlite3_int64 size; + int ii; sqlite3Atoi64(zRight, &size, 1000, SQLITE_UTF8); - sqlite3BtreeSetMmapLimit(pDb->pBt, size); + if( size<0 ) size = sqlite3GlobalConfig.mxMmap; + if( pId2->n==0 ) db->mxMmap = size; + for(ii=db->nDb-1; ii>=0; ii--){ + if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ + sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, size); + } + } } }else diff --git a/src/sqlite.h.in b/src/sqlite.h.in index acb49bf5f9..de1316606b 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1637,6 +1637,19 @@ struct sqlite3_mem_methods { ** points to has just been executed. Or, if the fourth parameter is 2, then ** the connection being passed as the second parameter is being closed. The ** third parameter is passed NULL In this case. +** +** [[SQLITE_CONFIG_MMAP_LIMIT]] +**
    SQLITE_CONFIG_MMAP_LIMIT +**
    The sole argument should be a 64-bit integer (an sqlite3_int64) that +** is the default maximum number of bytes of process address space that +** should be used for accessing each database file using memory mapping. +** The default setting can be overridden by each database connection using +** either the [PRAGMA mmap_limit] command or the "mmaplimit" query parameter +** on the [URI filename] when opening the databaes file or by using the +** [SQLITE_FCNTL_MMAP_LIMIT] file control. The value set here overrides the +** compile-time default that is set using the [SQLITE_DEFAULT_MMAP_LIMIT] +** compile-time option. If the argument to this option is negative, then +** the memory map limit is set to the compile-time default. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -1660,6 +1673,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ #define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_LIMIT 22 /* sqlite3_int64 */ /* ** CAPI3REF: Database Connection Configuration Options diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 129c4c5ea5..d894051790 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -834,6 +834,7 @@ struct sqlite3 { int nDb; /* Number of backends currently in use */ int flags; /* Miscellaneous flags. See below */ i64 lastRowid; /* ROWID of most recent insert (see above) */ + i64 mxMmap; /* Default mmap_limit setting */ unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ @@ -2505,6 +2506,7 @@ struct Sqlite3Config { void *pHeap; /* Heap storage space */ int nHeap; /* Size of pHeap[] */ int mnReq, mxReq; /* Min and max heap requests sizes */ + sqlite3_int64 mxMmap; /* Maximum mmap() space per open file */ void *pScratch; /* Scratch memory */ int szScratch; /* Size of each scratch buffer */ int nScratch; /* Number of scratch buffers */ From 3861f546e3f3752cdce48fb75e37da7394aba882 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Apr 2013 22:42:48 +0000 Subject: [PATCH 064/204] Remove a debugging printf() accidently left in the previous check-in. FossilOrigin-Name: 8198cdd8ac5dcc1c677fffa869ac965186b96abf --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 995cb45fa0..b2e4dbad76 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_CONFIG_MMAP_LIMIT\sconfiguration\soption\sfor\soverriding\sthe\nSQLITE_DEFAULT_MMAP_LIMIT\scompile-time\ssetting.\s\sEnhance\s"PRAGMA\smmap_limit"\nso\sthat\swithout\sa\sspecific\sdatabase\sname,\sit\ssets\sthe\slimit\son\sall\sdatabase\nfiles\sand\schanges\sthe\sdefault\sfor\sany\sfuture\sdatabases\sthat\smight\sbe\sadded\nusing\sATTACH. -D 2013-04-01T22:38:06.929 +C Remove\sa\sdebugging\sprintf()\saccidently\sleft\sin\sthe\sprevious\scheck-in. +D 2013-04-01T22:42:48.981 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 9912ab0cfcd7fa2b058544d861a9fead3adac8da +F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 F src/pager.c 072512d05dacd61b1f147736825bd32fdc890ba4 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e3c63c31a21b27806592d066a013a111b280c61f -R 6d1c950d2134f5ef56d487146725ba7a +P 78141d0a16dd1d56b575fccd149de7fa789cb06c +R fed26fbe7b97d6a5db36e7e764991fa9 U drh -Z 72c1547cadf3e5127066e34139489015 +Z c0440f45f7713ec5c0ea751d4cf10d7e diff --git a/manifest.uuid b/manifest.uuid index 838dd7eeb0..ee803ab26e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -78141d0a16dd1d56b575fccd149de7fa789cb06c \ No newline at end of file +8198cdd8ac5dcc1c677fffa869ac965186b96abf \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index c10841abf9..e92c7cc4a4 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3711,7 +3711,6 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } case SQLITE_FCNTL_MMAP_LIMIT: { pFile->mmapLimit = *(i64*)pArg; -printf("MMAP-LIMIT(%s) -> %lld\n", pFile->zPath, pFile->mmapLimit); return SQLITE_OK; } #ifdef SQLITE_DEBUG From 98d2038fc2325c968620afe515e3a99a8a08f098 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 00:15:23 +0000 Subject: [PATCH 065/204] Always send the SQLITE_FCNTL_MMAP_LIMIT pragma to the VFS, even if the limit is zero and even if the VFS does not support xFetch(). FossilOrigin-Name: 01ffdabbad30f1c157f2b33b1e85ee4d6c4632dd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 6 ++---- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index b2e4dbad76..3d3492ec7c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\sdebugging\sprintf()\saccidently\sleft\sin\sthe\sprevious\scheck-in. -D 2013-04-01T22:42:48.981 +C Always\ssend\sthe\sSQLITE_FCNTL_MMAP_LIMIT\spragma\sto\sthe\sVFS,\seven\sif\sthe\slimit\nis\szero\sand\seven\sif\sthe\sVFS\sdoes\snot\ssupport\sxFetch(). +D 2013-04-02T00:15:23.044 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 -F src/pager.c 072512d05dacd61b1f147736825bd32fdc890ba4 +F src/pager.c 95e7a64bfdd244603d595b455f4e9ef6cf124b54 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 78141d0a16dd1d56b575fccd149de7fa789cb06c -R fed26fbe7b97d6a5db36e7e764991fa9 +P 8198cdd8ac5dcc1c677fffa869ac965186b96abf +R a603a97de401ee9321fd272228a03ad0 U drh -Z c0440f45f7713ec5c0ea751d4cf10d7e +Z 4afc43008f65fbdcf2b0b7902c3ce20b diff --git a/manifest.uuid b/manifest.uuid index ee803ab26e..4e3c99ee29 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8198cdd8ac5dcc1c677fffa869ac965186b96abf \ No newline at end of file +01ffdabbad30f1c157f2b33b1e85ee4d6c4632dd \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 68b2e2d667..e391652f48 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3360,10 +3360,8 @@ static void pagerFixMaplimit(Pager *pPager){ sqlite3_file *fd = pPager->fd; if( isOpen(fd) ){ pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->mxMmap>0; - if( pPager->bUseFetch ){ - sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT, - (void*)&pPager->mxMmap); - } + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT, + (void*)&pPager->mxMmap); } } From 092d993cb79d11675246e04b1e15d2c48a825f2a Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 00:41:18 +0000 Subject: [PATCH 066/204] Remove an unreachable condition from pager.c. FossilOrigin-Name: 3628e86bf131cb205f08a4b299d84007cd6b25c3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 5 ++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 3d3492ec7c..767726d357 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Always\ssend\sthe\sSQLITE_FCNTL_MMAP_LIMIT\spragma\sto\sthe\sVFS,\seven\sif\sthe\slimit\nis\szero\sand\seven\sif\sthe\sVFS\sdoes\snot\ssupport\sxFetch(). -D 2013-04-02T00:15:23.044 +C Remove\san\sunreachable\scondition\sfrom\spager.c. +D 2013-04-02T00:41:18.590 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 -F src/pager.c 95e7a64bfdd244603d595b455f4e9ef6cf124b54 +F src/pager.c 37ac3175cb46bf4eba5c23005c49700b33ed2dbe F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 8198cdd8ac5dcc1c677fffa869ac965186b96abf -R a603a97de401ee9321fd272228a03ad0 +P 01ffdabbad30f1c157f2b33b1e85ee4d6c4632dd +R 683cfbd9fb419480e60db9265a2009fc U drh -Z 4afc43008f65fbdcf2b0b7902c3ce20b +Z c53b9215e567f2251d489b56e620483a diff --git a/manifest.uuid b/manifest.uuid index 4e3c99ee29..cd77251101 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -01ffdabbad30f1c157f2b33b1e85ee4d6c4632dd \ No newline at end of file +3628e86bf131cb205f08a4b299d84007cd6b25c3 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index e391652f48..75d12cdc1e 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2936,15 +2936,14 @@ static int pagerUndoCallback(void *pCtx, Pgno iPg){ Pager *pPager = (Pager *)pCtx; PgHdr *pPg; + assert( pagerUseWal(pPager) ); pPg = sqlite3PagerLookup(pPager, iPg); if( pPg ){ if( sqlite3PcachePageRefcount(pPg)==1 ){ sqlite3PcacheDrop(pPg); }else{ u32 iFrame = 0; - if( pagerUseWal(pPager) ){ - rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); - } + rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); if( rc==SQLITE_OK ){ rc = readDbPage(pPg, iFrame); } From 6d81449fc6cabb4ee186cf72547eb1e949e011ac Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 01:45:10 +0000 Subject: [PATCH 067/204] Reorder two conditions to make coverage testing easier. FossilOrigin-Name: 793ba4e996426522eeaa86589a9e974fa1fc1522 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 767726d357..37da7c88aa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunreachable\scondition\sfrom\spager.c. -D 2013-04-02T00:41:18.590 +C Reorder\stwo\sconditions\sto\smake\scoverage\stesting\seasier. +D 2013-04-02T01:45:10.945 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 -F src/pager.c 37ac3175cb46bf4eba5c23005c49700b33ed2dbe +F src/pager.c 46fa7d8b29e7fb8baa6111690d900c9111a5a397 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 01ffdabbad30f1c157f2b33b1e85ee4d6c4632dd -R 683cfbd9fb419480e60db9265a2009fc +P 3628e86bf131cb205f08a4b299d84007cd6b25c3 +R 5022c5197e31d50bddc957f6984f11f1 U drh -Z c53b9215e567f2251d489b56e620483a +Z 379e6b5efd0a997c87c1f9a5494b7aa5 diff --git a/manifest.uuid b/manifest.uuid index cd77251101..8133886ec9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3628e86bf131cb205f08a4b299d84007cd6b25c3 \ No newline at end of file +793ba4e996426522eeaa86589a9e974fa1fc1522 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 75d12cdc1e..cb2b39a992 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5128,7 +5128,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** nothing to rollback, so this routine is a no-op. */ static void pagerUnlockIfUnused(Pager *pPager){ - if( (sqlite3PcacheRefCount(pPager->pPCache)==0) && pPager->nMmapOut==0 ){ + if( pPager->nMmapOut==0 && (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ pagerUnlockAndRollback(pPager); } } From a3a2b3bdc0cfad4e96a547ff953f240ca9845061 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 10:29:13 +0000 Subject: [PATCH 068/204] Proposed template preprocessor magic for activating mmap only on platforms where we know it works. FossilOrigin-Name: d96272f031102514b02bd839aac9e37618872200 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqliteLimit.h | 21 +++++++++++++++++++-- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 37da7c88aa..edce1feec8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reorder\stwo\sconditions\sto\smake\scoverage\stesting\seasier. -D 2013-04-02T01:45:10.945 +C Proposed\stemplate\spreprocessor\smagic\sfor\sactivating\smmap\sonly\son\splatforms\nwhere\swe\sknow\sit\sworks. +D 2013-04-02T10:29:13.935 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -180,7 +180,7 @@ F src/sqlite.h.in 42c69ea00dfcce84394d6ce0d8dac7bb3d2ecff2 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 1664dc5ad6f8d4dab871416628aa3271044d66c0 -F src/sqliteLimit.h 83d0cd0ce3050a80930406d3c7aecb48c1ccdfc5 +F src/sqliteLimit.h 4d8d04c43bf2886d94f1ab7992fee5081e09a7f5 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3628e86bf131cb205f08a4b299d84007cd6b25c3 -R 5022c5197e31d50bddc957f6984f11f1 +P 793ba4e996426522eeaa86589a9e974fa1fc1522 +R 2b5ee6ceec98b53d533d4ea410a1bfc7 U drh -Z 379e6b5efd0a997c87c1f9a5494b7aa5 +Z ead310385cbe70188969099b54288940 diff --git a/manifest.uuid b/manifest.uuid index 8133886ec9..6c51e2ad5a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -793ba4e996426522eeaa86589a9e974fa1fc1522 \ No newline at end of file +d96272f031102514b02bd839aac9e37618872200 \ No newline at end of file diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index 2013139456..6adf3e8726 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -210,6 +210,23 @@ /* ** Default maximum size of memory used by xFetch in the VFS. */ -#ifndef SQLITE_DEFAULT_MMAP_LIMIT -# define SQLITE_DEFAULT_MMAP_LIMIT (256*1024*1024) +#ifdef __APPLE__ +# include +# if defined(TARGET_OS_IPHONE) +# define SQLITE_DEFAULT_MMAP_LIMIT 0 +# endif +#endif +#ifndef SQLITE_DEFAULT_MMAP_LIMIT +# if defined(__linux__) \ + || defined(_WIN32) \ + || (defined(__APPLE__) && defined(__MACH__) && !defined(TARGET_OS_IPHONE)) \ + || defined(__sun) \ + || defined(__DragonFly__) \ + || defined(__FreeBSD__) \ + || defined(__NetBSD__) \ + || defined(__OpenBSD__) +# define SQLITE_DEFAULT_MMAP_LIMIT (256*1024*1024) +# else +# define SQLITE_DEFAULT_MMAP_LIMIT 0 +# endif #endif From 4ff7bc45c6c6a7156e433ff50395dfe5e2885c38 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Apr 2013 12:04:09 +0000 Subject: [PATCH 069/204] Add test cases for errors in mmap() or mremap() is os_unix.c. FossilOrigin-Name: 3098a3c1e7305033904a496ef534cb312a876fab --- manifest | 19 +++++------ manifest.uuid | 2 +- src/os_unix.c | 26 ++++++++------- src/test_syscall.c | 15 ++++++++- test/mmap2.test | 79 ++++++++++++++++++++++++++++++++++++++++++++++ test/speed1p.test | 3 +- 6 files changed, 120 insertions(+), 24 deletions(-) create mode 100644 test/mmap2.test diff --git a/manifest b/manifest index edce1feec8..645d467c46 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Proposed\stemplate\spreprocessor\smagic\sfor\sactivating\smmap\sonly\son\splatforms\nwhere\swe\sknow\sit\sworks. -D 2013-04-02T10:29:13.935 +C Add\stest\scases\sfor\serrors\sin\smmap()\sor\smremap()\sis\sos_unix.c. +D 2013-04-02T12:04:09.729 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c +F src/os_unix.c 42684a59b06b912b3ab80a8ad9f4295c0572a1f4 F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 F src/pager.c 46fa7d8b29e7fb8baa6111690d900c9111a5a397 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -226,7 +226,7 @@ F src/test_spellfix.c 56dfa6d583ac34f61af0834d7b58d674e7e18e13 F src/test_sqllog.c bc50e5afeb7fb50e77b4594e42302df9d05446aa F src/test_stat.c d1569c7a4839f13e80187e2c26b2ab4da2d03935 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd -F src/test_syscall.c fe018f95049d4f24e036d1d649516b60689b4e57 +F src/test_syscall.c 7e8293e4e6971b0f44c7f7f37b1315a8cc9f6018 F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c e286f2173563f2a1747c24bcda6b9d030bf4f4e4 F src/test_vfs.c fb16b2d9938cf0c1afc5a423b55b952fcc024275 @@ -631,6 +631,7 @@ F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 F test/mmap1.test 9fb71daaca72f66fcc7a3b3a37fc3db99988b57a +F test/mmap2.test bb0c97cdac0db06a5acdc6836eef9866815481b1 F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 @@ -731,7 +732,7 @@ F test/softheap1.test c16709a16ad79fa43b32929b2e623d1d117ccf53 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb -F test/speed1p.test c4a469f29f135f4d76c55b1f2a52f36e209466cc +F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 F test/speed2.test 53177056baf6556dcbdcf032bbdfc41c1aa74ded F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 @@ -1040,7 +1041,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 793ba4e996426522eeaa86589a9e974fa1fc1522 -R 2b5ee6ceec98b53d533d4ea410a1bfc7 -U drh -Z ead310385cbe70188969099b54288940 +P d96272f031102514b02bd839aac9e37618872200 +R 323dfbce5d7140c5ce46f22a5112dcc8 +U dan +Z 3d1807f4fa3a8cf06a6141b170b215ce diff --git a/manifest.uuid b/manifest.uuid index 6c51e2ad5a..dfa80de2b5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d96272f031102514b02bd839aac9e37618872200 \ No newline at end of file +3098a3c1e7305033904a496ef534cb312a876fab \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index e92c7cc4a4..07c919c983 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4560,6 +4560,7 @@ static void unixRemapfile( unixFile *pFd, /* File descriptor object */ i64 nNew /* Required mapping size */ ){ + const char *zErr = "mmap"; int h = pFd->h; /* File descriptor open on db file */ u8 *pOrig = (u8 *)pFd->pMapRegion; /* Pointer to current file mapping */ i64 nOrig = pFd->mmapOrigsize; /* Size of pOrig region in bytes */ @@ -4571,6 +4572,7 @@ static void unixRemapfile( assert( nNew<=pFd->mmapLimit ); assert( nNew>0 ); assert( pFd->mmapOrigsize>=pFd->mmapSize ); + assert( MAP_FAILED!=0 ); if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; @@ -4586,12 +4588,13 @@ static void unixRemapfile( #if HAVE_MREMAP pNew = osMremap(pOrig, nReuse, nNew, MREMAP_MAYMOVE); + zErr = "mremap"; #else pNew = osMmap(pReq, nNew-nReuse, flags, MAP_SHARED, h, nReuse); if( pNew!=MAP_FAILED ){ if( pNew!=pReq ){ osMunmap(pNew, nNew - nReuse); - pNew = MAP_FAILED; + pNew = 0; }else{ pNew = pOrig; } @@ -4602,7 +4605,6 @@ static void unixRemapfile( ** mapping and set pNew to NULL so that the code below will create a ** new mapping from scratch. */ if( pNew==MAP_FAILED ){ - pNew = 0; osMunmap(pOrig, nReuse); } } @@ -4610,18 +4612,18 @@ static void unixRemapfile( /* If pNew is still NULL, try to create an entirely new mapping. */ if( pNew==0 ){ pNew = osMmap(0, nNew, flags, MAP_SHARED, h, 0); - if( pNew==MAP_FAILED ){ - pNew = 0; - nNew = 0; - unixLogError(SQLITE_OK, "mmap", pFd->zPath); - - /* If the mmap() above failed, assume that all subsequent mmap() calls - ** will probably fail too. Fall back to using xRead/xWrite exclusively - ** in this case. */ - pFd->mmapLimit = 0; - } } + if( pNew==MAP_FAILED ){ + pNew = 0; + nNew = 0; + unixLogError(SQLITE_OK, zErr, pFd->zPath); + + /* If the mmap() above failed, assume that all subsequent mmap() calls + ** will probably fail too. Fall back to using xRead/xWrite exclusively + ** in this case. */ + pFd->mmapLimit = 0; + } pFd->pMapRegion = (void *)pNew; pFd->mmapSize = pFd->mmapOrigsize = nNew; } diff --git a/src/test_syscall.c b/src/test_syscall.c index 0b053fc599..3d117fb2ab 100644 --- a/src/test_syscall.c +++ b/src/test_syscall.c @@ -108,7 +108,7 @@ static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, off_t off); static int ts_fchmod(int fd, mode_t mode); static int ts_fallocate(int fd, off_t off, off_t len); static void *ts_mmap(void *, size_t, int, int, int, off_t); - +static void *ts_mremap(void*, size_t, size_t, int, ...); struct TestSyscallArray { const char *zName; @@ -134,6 +134,7 @@ struct TestSyscallArray { /* 14 */ { "fchmod", (sqlite3_syscall_ptr)ts_fchmod, 0, 0, 0 }, /* 15 */ { "fallocate", (sqlite3_syscall_ptr)ts_fallocate, 0, 0, 0 }, /* 16 */ { "mmap", (sqlite3_syscall_ptr)ts_mmap, 0, 0, 0 }, + /* 17 */ { "mremap", (sqlite3_syscall_ptr)ts_mremap, 0, 0, 0 }, { 0, 0, 0, 0, 0 } }; @@ -156,6 +157,7 @@ struct TestSyscallArray { #define orig_fchmod ((int(*)(int,mode_t))aSyscall[14].xOrig) #define orig_fallocate ((int(*)(int,off_t,off_t))aSyscall[15].xOrig) #define orig_mmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[16].xOrig) +#define orig_mremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[17].xOrig) /* ** This function is called exactly once from within each invocation of a @@ -395,6 +397,17 @@ static void *ts_mmap( return orig_mmap(pAddr, nByte, prot, flags, fd, iOff); } +static void *ts_mremap(void *a, size_t b, size_t c, int d, ...){ + va_list ap; + void *pArg; + if( tsIsFailErrno("mremap") ){ + return MAP_FAILED; + } + va_start(ap, d); + pArg = va_arg(ap, void *); + return orig_mremap(a, b, c, d, pArg); +} + static int test_syscall_install( void * clientData, Tcl_Interp *interp, diff --git a/test/mmap2.test b/test/mmap2.test new file mode 100644 index 0000000000..d43ea97adf --- /dev/null +++ b/test/mmap2.test @@ -0,0 +1,79 @@ +# 2013 March 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file tests the effect of the mmap() or mremap() system calls +# returning an error on the library. +# +# If either mmap() or mremap() fails, SQLite should log an error +# message, then continue accessing the database using read() and +# write() exclusively. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix mmap2 + +if {[test_syscall defaultvfs] != "unix"} { + finish_test + return +} + +db close +sqlite3_shutdown +test_sqlite3_log xLog +proc xLog {error_code msg} { + if {[string match os_unix.c* $msg]} { + lappend ::log $msg + } +} + +foreach syscall {mmap mremap} { + test_syscall uninstall + if {[catch {test_syscall install $syscall}]} continue + + for {set i 1} {$i < 20} {incr i} { + reset_db + + test_syscall fault $i 1 + test_syscall errno $syscall ENOMEM + set ::log "" + + do_execsql_test 1.$syscall.$i.1 { + CREATE TABLE t1(a, b, UNIQUE(a, b)); + INSERT INTO t1 VALUES(randomblob(1000), randomblob(1000)); + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + } + + set nFail [test_syscall fault 0 0] + + do_execsql_test 1.$syscall.$i.2 { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } {64 ok} + + do_test 1.$syscall.$i.3 { + expr {$nFail==0 || $nFail==1} + } {1} + + do_test 1.$syscall.$i.4.nFail=$nFail { + regexp ".*${syscall}.*" $::log + } [expr $nFail>0] + } +} + +test_syscall uninstall +finish_test + diff --git a/test/speed1p.test b/test/speed1p.test index 915f165354..6bf7b10e1f 100644 --- a/test/speed1p.test +++ b/test/speed1p.test @@ -24,6 +24,8 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl speed_trial_init speed1 +sqlite3_memdebug_vfs_oom_test 0 + # Set a uniform random seed expr srand(0) @@ -78,7 +80,6 @@ do_test speed1p-1.0 { } } {i2a i2b t1 t2} - # 50000 INSERTs on an unindexed table # set list {} From 936c6d7e9e640e1269545ed9e878186a9b2b83f2 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 13:56:53 +0000 Subject: [PATCH 070/204] If the SQLITE_TRACE_SIZE_LIMIT compile-time parameter is set to a positive integer then limit the expansion of strings and blobs in trace output to approximately that many bytes. FossilOrigin-Name: e5b710849dd66673ba0e0d935b103cb29abfcc4b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbetrace.c | 40 ++++++++++++++++++++++++++++++++-------- 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 453b33c22e..dce40a05fc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\sthe\sclean\starget,\stidy\sup\sa\scouple\smore\sgenerated\s(or\scopied)\ssource\sfiles. -D 2013-03-29T19:52:04.965 +C If\sthe\sSQLITE_TRACE_SIZE_LIMIT\scompile-time\sparameter\sis\sset\sto\sa\spositive\ninteger\sthen\slimit\sthe\sexpansion\sof\sstrings\sand\sblobs\sin\strace\soutput\sto\napproximately\sthat\smany\sbytes. +D 2013-04-02T13:56:53.795 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -247,7 +247,7 @@ F src/vdbeaux.c ecb43014bcd3019e5aa2b5561e5c3a447f007a08 F src/vdbeblob.c 32f2a4899d67f69634ea4dd93e3f651936d732cb F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c 4fad64071ae120c25f39dcac572d716b9cadeb7f -F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 +F src/vdbetrace.c a22263ab47f6ba4fcd176515cec1e732866b25f0 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 @@ -1041,7 +1041,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 527231bc67285f01fb18d4451b28f61da3c4e39d -R c463228a20804b5f57c5669f0d0e45b8 -U mistachkin -Z bfb2a9ca80a29b00e5f7b4db495d38d4 +P 5687e5ee7bafa00d2b353c3eda1e5dfb219cb185 +R b688b887865b3d8981922c40426941a0 +U drh +Z 66f86621cb15eccea274cb5b5e71ec26 diff --git a/manifest.uuid b/manifest.uuid index 6960809f06..ec266fb546 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5687e5ee7bafa00d2b353c3eda1e5dfb219cb185 \ No newline at end of file +e5b710849dd66673ba0e0d935b103cb29abfcc4b \ No newline at end of file diff --git a/src/vdbetrace.c b/src/vdbetrace.c index 35825c8736..91554d6717 100644 --- a/src/vdbetrace.c +++ b/src/vdbetrace.c @@ -53,6 +53,11 @@ static int findNextHostParameter(const char *zSql, int *pnToken){ ** then the returned string holds a copy of zRawSql with "-- " prepended ** to each line of text. ** +** If the SQLITE_TRACE_SIZE_LIMIT macro is defined to an integer, then +** then long strings and blobs are truncated to that many bytes. This +** can be used to prevent unreasonably large trace strings when dealing +** with large (multi-megabyte) strings and blobs. +** ** The calling function is responsible for making sure the memory returned ** is eventually freed. ** @@ -123,30 +128,49 @@ char *sqlite3VdbeExpandSql( }else if( pVar->flags & MEM_Real ){ sqlite3XPrintf(&out, "%!.15g", pVar->r); }else if( pVar->flags & MEM_Str ){ + int n; /* Number of bytes of the string text to include in output */ #ifndef SQLITE_OMIT_UTF16 u8 enc = ENC(db); + Mem utf8; if( enc!=SQLITE_UTF8 ){ - Mem utf8; memset(&utf8, 0, sizeof(utf8)); utf8.db = db; sqlite3VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE_STATIC); sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8); - sqlite3XPrintf(&out, "'%.*q'", utf8.n, utf8.z); - sqlite3VdbeMemRelease(&utf8); - }else -#endif - { - sqlite3XPrintf(&out, "'%.*q'", pVar->n, pVar->z); + pVar = &utf8; } +#endif + n = pVar->n; +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( n>SQLITE_TRACE_SIZE_LIMIT ){ + n = SQLITE_TRACE_SIZE_LIMIT; + while( nn && (pVar->z[n]&0xc0)==0x80 ){ n++; } + } +#endif + sqlite3XPrintf(&out, "'%.*q'", n, pVar->z); +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nn ) sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-n); +#endif +#ifndef SQLITE_OMIT_UTF16 + if( enc!=SQLITE_UTF8 ) sqlite3VdbeMemRelease(&utf8); +#endif }else if( pVar->flags & MEM_Zero ){ sqlite3XPrintf(&out, "zeroblob(%d)", pVar->u.nZero); }else{ + int n; /* Number of bytes of the blob to include in output */ assert( pVar->flags & MEM_Blob ); sqlite3StrAccumAppend(&out, "x'", 2); - for(i=0; in; i++){ + n = pVar->n; +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( n>SQLITE_TRACE_SIZE_LIMIT ) n = SQLITE_TRACE_SIZE_LIMIT; +#endif + for(i=0; iz[i]&0xff); } sqlite3StrAccumAppend(&out, "'", 1); +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nn ) sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-n); +#endif } } } From 07fa864d15ae88e9fb999b37128b7698cbea037a Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 14:37:40 +0000 Subject: [PATCH 071/204] Fix a faulty assert() in the os_win.c VFS. FossilOrigin-Name: fd6ee54969322203c02ce9bd8744e095faf6a69c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 645d467c46..7730e2979b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\scases\sfor\serrors\sin\smmap()\sor\smremap()\sis\sos_unix.c. -D 2013-04-02T12:04:09.729 +C Fix\sa\sfaulty\sassert()\sin\sthe\sos_win.c\sVFS. +D 2013-04-02T14:37:40.848 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 42684a59b06b912b3ab80a8ad9f4295c0572a1f4 -F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 +F src/os_win.c 6822b2cda33f025ce7fe240ec050b9bcba213903 F src/pager.c 46fa7d8b29e7fb8baa6111690d900c9111a5a397 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1041,7 +1041,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d96272f031102514b02bd839aac9e37618872200 -R 323dfbce5d7140c5ce46f22a5112dcc8 -U dan -Z 3d1807f4fa3a8cf06a6141b170b215ce +P 3098a3c1e7305033904a496ef534cb312a876fab +R 3eb86ed3ddaa0095d1a3da7f2b0ccdb4 +U drh +Z 7afe4013f9081817d8523bb6f04fa909 diff --git a/manifest.uuid b/manifest.uuid index dfa80de2b5..c2883df6c5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3098a3c1e7305033904a496ef534cb312a876fab \ No newline at end of file +fd6ee54969322203c02ce9bd8744e095faf6a69c \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index ee7a71018e..d66faba4cb 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3601,7 +3601,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ /* Log the error, but continue normal operation using xRead/xWrite */ return SQLITE_OK; } - assert( (nNewRnd % winSysInfo.dwPageSize)==0 ); + assert( (nMap % winSysInfo.dwPageSize)==0 ); #if SQLITE_OS_WINRT pNew = osMapViewOfFileFromApp(pFd->hMap, flags, 0, nMap); #else From f6653871b218c18667a5eadd6b449c7d5bdb768d Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Apr 2013 15:37:30 +0000 Subject: [PATCH 072/204] Add new test file btreefault.test. FossilOrigin-Name: 7fe908afeba16f64edc16824c67b396138581a8f --- manifest | 13 ++++++------ manifest.uuid | 2 +- test/btreefault.test | 50 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 7 deletions(-) create mode 100644 test/btreefault.test diff --git a/manifest b/manifest index 7730e2979b..4e4e7e16f5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sfaulty\sassert()\sin\sthe\sos_win.c\sVFS. -D 2013-04-02T14:37:40.848 +C Add\snew\stest\sfile\sbtreefault.test. +D 2013-04-02T15:37:30.257 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -314,6 +314,7 @@ F test/boundary3.tcl 8901d6a503d0bf64251dd81cc74e5ad3add4b119 F test/boundary3.test 56ef82096b4329aca2be74fa1e2b0f762ea0eb45 F test/boundary4.tcl 0bb4b1a94f4fc5ae59b79b9a2b7a140c405e2983 F test/boundary4.test 89e02fa66397b8a325d5eb102b5806f961f8ec4b +F test/btreefault.test 06899a377f31a8c1a3048ec69831522d4e5c6045 F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de F test/capi2.test e8b18cc61090b6e5e388f54d6b125d711d1b265a @@ -1041,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3098a3c1e7305033904a496ef534cb312a876fab -R 3eb86ed3ddaa0095d1a3da7f2b0ccdb4 -U drh -Z 7afe4013f9081817d8523bb6f04fa909 +P fd6ee54969322203c02ce9bd8744e095faf6a69c +R afe8ecd9cb367696b6be602590bc8be6 +U dan +Z 42e6019db12bcc782c7a96e67c5fd3c7 diff --git a/manifest.uuid b/manifest.uuid index c2883df6c5..005403f1ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fd6ee54969322203c02ce9bd8744e095faf6a69c \ No newline at end of file +7fe908afeba16f64edc16824c67b396138581a8f \ No newline at end of file diff --git a/test/btreefault.test b/test/btreefault.test new file mode 100644 index 0000000000..45d326da7f --- /dev/null +++ b/test/btreefault.test @@ -0,0 +1,50 @@ +# 2013 April 02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains fault injection tests designed to test the btree.c +# module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set testprefix btreefault + +do_test 1-pre1 { + execsql { + PRAGMA auto_vacuum = incremental; + PRAGMA journal_mode = DELETE; + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES(randomblob(1000), randomblob(100)); + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + DELETE FROM t1 WHERE rowid%2; + } + faultsim_save_and_close +} {} + +do_faultsim_test 1 -prep { + faultsim_restore_and_reopen + set ::STMT [sqlite3_prepare db "SELECT * FROM t1 ORDER BY a" -1 DUMMY] + sqlite3_step $::STMT + sqlite3_step $::STMT +} -body { + execsql { PRAGMA incremental_vacuum = 10 } +} -test { + sqlite3_finalize $::STMT + faultsim_test_result {0 {}} + faultsim_integrity_check +} + +finish_test + From a2c16c084a8a6d6654ac9a8be7bfe5d181638900 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 18:25:04 +0000 Subject: [PATCH 073/204] Fix the build for Mac. FossilOrigin-Name: d36ce0bbad58445c983597ee93438fe07b659cdc --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqliteLimit.h | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 4e4e7e16f5..0d221db3d2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\stest\sfile\sbtreefault.test. -D 2013-04-02T15:37:30.257 +C Fix\sthe\sbuild\sfor\sMac. +D 2013-04-02T18:25:04.196 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -180,7 +180,7 @@ F src/sqlite.h.in 42c69ea00dfcce84394d6ce0d8dac7bb3d2ecff2 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 1664dc5ad6f8d4dab871416628aa3271044d66c0 -F src/sqliteLimit.h 4d8d04c43bf2886d94f1ab7992fee5081e09a7f5 +F src/sqliteLimit.h edc1e61e6b745dc26ea16542838c100cef02df0e F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P fd6ee54969322203c02ce9bd8744e095faf6a69c -R afe8ecd9cb367696b6be602590bc8be6 -U dan -Z 42e6019db12bcc782c7a96e67c5fd3c7 +P 7fe908afeba16f64edc16824c67b396138581a8f +R 1e903cf5fbaea8405561162b82cab3c1 +U drh +Z 5ee32a75ebc407142eddeaec11f8a36d diff --git a/manifest.uuid b/manifest.uuid index 005403f1ad..d710366705 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7fe908afeba16f64edc16824c67b396138581a8f \ No newline at end of file +d36ce0bbad58445c983597ee93438fe07b659cdc \ No newline at end of file diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index 6adf3e8726..826d19dd9c 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -211,7 +211,7 @@ ** Default maximum size of memory used by xFetch in the VFS. */ #ifdef __APPLE__ -# include +# include # if defined(TARGET_OS_IPHONE) # define SQLITE_DEFAULT_MMAP_LIMIT 0 # endif From d409792838233ae25045e223278a2d195d47d347 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Apr 2013 18:31:29 +0000 Subject: [PATCH 074/204] Change an assert in pager.c to acknowledge that it is possible for sqlite3PagerRollback() to return SQLITE_CORRUPT. FossilOrigin-Name: d641d3d20d80fdb86d91de31fcf2eb5ca3c5c715 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pager.c | 2 +- test/wal.test | 20 ++++++++++++++++++++ 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 0d221db3d2..a0457959ef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sbuild\sfor\sMac. -D 2013-04-02T18:25:04.196 +C Change\san\sassert\sin\spager.c\sto\sacknowledge\sthat\sit\sis\spossible\sfor\ssqlite3PagerRollback()\sto\sreturn\sSQLITE_CORRUPT. +D 2013-04-02T18:31:29.703 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 42684a59b06b912b3ab80a8ad9f4295c0572a1f4 F src/os_win.c 6822b2cda33f025ce7fe240ec050b9bcba213903 -F src/pager.c 46fa7d8b29e7fb8baa6111690d900c9111a5a397 +F src/pager.c e26184d451207542d56bd09b84339252a194791e F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -953,7 +953,7 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 -F test/wal.test bbd98ac09d8fbcaa80ec233757056eb1bbb7cd95 +F test/wal.test 62454b2cf00b311e9e65f898aad0fef536ed561a F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 7fe908afeba16f64edc16824c67b396138581a8f -R 1e903cf5fbaea8405561162b82cab3c1 -U drh -Z 5ee32a75ebc407142eddeaec11f8a36d +P d36ce0bbad58445c983597ee93438fe07b659cdc +R 338634a1c64bdacb6f3ed638ff14c520 +U dan +Z 2347325502a78b990959c6fe11156215 diff --git a/manifest.uuid b/manifest.uuid index d710366705..b9069347bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d36ce0bbad58445c983597ee93438fe07b659cdc \ No newline at end of file +d641d3d20d80fdb86d91de31fcf2eb5ca3c5c715 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index cb2b39a992..f626a21d89 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6261,7 +6261,7 @@ int sqlite3PagerRollback(Pager *pPager){ } assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); - assert( rc==SQLITE_OK || rc==SQLITE_FULL + assert( rc==SQLITE_OK || rc==SQLITE_FULL || rc==SQLITE_CORRUPT || rc==SQLITE_NOMEM || (rc&0xFF)==SQLITE_IOERR ); /* If an error occurs during a ROLLBACK, we can no longer trust the pager diff --git a/test/wal.test b/test/wal.test index 0539192d25..f833b6ab1a 100644 --- a/test/wal.test +++ b/test/wal.test @@ -1569,6 +1569,26 @@ ifcapable autovacuum { } [wal_file_size 1 1024] } +reset_db +do_test 25 { + sqlite3 db test.db + + execsql { + CREATE TABLE t1(x, y); + PRAGMA journal_mode = WAL; + INSERT INTO t1 VALUES(1, 2); + } + + execsql { + BEGIN; + CREATE TABLE t2(a, b); + } + + hexio_write test.db-shm [expr 16*1024] [string repeat 0055 8192] + catchsql ROLLBACK +} {0 {}} + + db close sqlite3_shutdown test_sqlite3_log From db082408c6d68e5c8b6958806c5364bda3849a16 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Apr 2013 18:33:55 +0000 Subject: [PATCH 075/204] Add test cases to mmap1.test and pagerfault.test. FossilOrigin-Name: 3050136be962427cc4dcced6077ef29b2a941405 --- manifest | 14 +++++------ manifest.uuid | 2 +- test/mmap1.test | 25 +++++++++++++++++++ test/pagerfault.test | 57 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a0457959ef..856014e0b4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\san\sassert\sin\spager.c\sto\sacknowledge\sthat\sit\sis\spossible\sfor\ssqlite3PagerRollback()\sto\sreturn\sSQLITE_CORRUPT. -D 2013-04-02T18:31:29.703 +C Add\stest\scases\sto\smmap1.test\sand\spagerfault.test. +D 2013-04-02T18:33:55.243 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -631,7 +631,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 9fb71daaca72f66fcc7a3b3a37fc3db99988b57a +F test/mmap1.test 245eb4a6280d4b31523eb79486863afadac4f3c6 F test/mmap2.test bb0c97cdac0db06a5acdc6836eef9866815481b1 F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -653,7 +653,7 @@ F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 31c04bec797dda1bde337810b52efa08d1f1f08e F test/pager2.test 745b911dde3d1f24ae0870bd433dfa83d7c658c1 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f -F test/pagerfault.test 452f2cc23e3bfcfa935f4442aec1da4fe1dc0442 +F test/pagerfault.test fc2e37b2da626826dd54bb13720b4d721719b660 F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 F test/pageropt.test 32cb5a6ed7ccddf8e8c842cb44240bd9340223ce @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d36ce0bbad58445c983597ee93438fe07b659cdc -R 338634a1c64bdacb6f3ed638ff14c520 +P d641d3d20d80fdb86d91de31fcf2eb5ca3c5c715 +R 93a80e505d81032f015b51889ab16396 U dan -Z 2347325502a78b990959c6fe11156215 +Z 374a5f4f4b29eb3200a93246d589254e diff --git a/manifest.uuid b/manifest.uuid index b9069347bf..66226dcc40 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d641d3d20d80fdb86d91de31fcf2eb5ca3c5c715 \ No newline at end of file +3050136be962427cc4dcced6077ef29b2a941405 \ No newline at end of file diff --git a/test/mmap1.test b/test/mmap1.test index d97ada7e57..e63e09481c 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -129,4 +129,29 @@ do_execsql_test 2.4 { PRAGMA wal_checkpoint; } {0 24 24} + +reset_db +db func rblob rblob +do_execsql_test 3.1 { + PRAGMA auto_vacuum = 1; + + CREATE TABLE t1(a, b, UNIQUE(a, b)); + INSERT INTO t1 VALUES(rblob(500), rblob(500)); + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 2 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 4 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 8 + + CREATE TABLE t2(a, b, UNIQUE(a, b)); + INSERT INTO t2 SELECT * FROM t1; +} {} + +do_test 3.2 { + set nRow 0 + db eval {SELECT * FROM t2 ORDER BY a, b} { + if {$nRow==4} { db eval { DELETE FROM t1 } } + incr nRow + } + set nRow +} {8} + finish_test diff --git a/test/pagerfault.test b/test/pagerfault.test index e04e97e4fc..9ea1aa6328 100644 --- a/test/pagerfault.test +++ b/test/pagerfault.test @@ -1246,5 +1246,62 @@ do_faultsim_test pagerfault-27 -faults ioerr-persistent -prep { faultsim_integrity_check } +#------------------------------------------------------------------------- +# +do_test pagerfault-28-pre { + faultsim_delete_and_reopen + db func a_string a_string + execsql { + PRAGMA page_size = 512; + + PRAGMA journal_mode = wal; + PRAGMA wal_autocheckpoint = 0; + PRAGMA cache_size = 100000; + + BEGIN; + CREATE TABLE t2(a UNIQUE, b UNIQUE); + INSERT INTO t2 VALUES( a_string(800), a_string(800) ); + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + COMMIT; + CREATE TABLE t1(a PRIMARY KEY, b); + } + expr {[file size test.db-shm] >= 96*1024} +} {1} +faultsim_save_and_close + +do_faultsim_test pagerfault-28 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { PRAGMA mmap_limit=0 } + + sqlite3 db2 test.db + db2 eval { SELECT count(*) FROM t2 } + + db func a_string a_string + execsql { + BEGIN; + INSERT INTO t1 VALUES(a_string(2000), a_string(2000)); + INSERT INTO t1 VALUES(a_string(2000), a_string(2000)); + } + set ::STMT [sqlite3_prepare db "SELECT * FROM t1 ORDER BY a" -1 DUMMY] + sqlite3_step $::STMT +} -body { + execsql { ROLLBACK } +} -test { + db2 close + sqlite3_finalize $::STMT + catchsql { ROLLBACK } + faultsim_integrity_check +} + finish_test From a9d79ae95d54d4138356ac5801ff98b555bea939 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 2 Apr 2013 20:13:04 +0000 Subject: [PATCH 076/204] Add assert to the Windows VFS pertaining to the size of the SIZE_T type. FossilOrigin-Name: 4eeea58354e21797ca1f4dc4ea785d0330381f0b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 856014e0b4..1ca22b55cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\scases\sto\smmap1.test\sand\spagerfault.test. -D 2013-04-02T18:33:55.243 +C Add\sassert\sto\sthe\sWindows\sVFS\spertaining\sto\sthe\ssize\sof\sthe\sSIZE_T\stype. +D 2013-04-02T20:13:04.021 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 42684a59b06b912b3ab80a8ad9f4295c0572a1f4 -F src/os_win.c 6822b2cda33f025ce7fe240ec050b9bcba213903 +F src/os_win.c 6a7a1017fb6a0783a3aa7c3f5eadd9b1600a9ddf F src/pager.c e26184d451207542d56bd09b84339252a194791e F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d641d3d20d80fdb86d91de31fcf2eb5ca3c5c715 -R 93a80e505d81032f015b51889ab16396 -U dan -Z 374a5f4f4b29eb3200a93246d589254e +P 3050136be962427cc4dcced6077ef29b2a941405 +R 69136e8e9adf1c4824020b7a88c6a9a7 +U mistachkin +Z 0311bae3bb3a8c1122f90eda3126446b diff --git a/manifest.uuid b/manifest.uuid index 66226dcc40..341d71bfa9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3050136be962427cc4dcced6077ef29b2a941405 \ No newline at end of file +4eeea58354e21797ca1f4dc4ea785d0330381f0b \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index d66faba4cb..a978613e2e 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3605,6 +3605,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ #if SQLITE_OS_WINRT pNew = osMapViewOfFileFromApp(pFd->hMap, flags, 0, nMap); #else + assert( sizeof(SIZE_T)==sizeof(sqlite3_int64) || nMap<=0xffffffff ); pNew = osMapViewOfFile(pFd->hMap, flags, 0, 0, (SIZE_T)nMap); #endif if( pNew==NULL ){ From 81d1765ad837a99c2ddd9cb2001aba3213779395 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Apr 2013 20:19:22 +0000 Subject: [PATCH 077/204] Fix a bug in mmap2.test that was leaving an invalid sqlite3_log() callback installed, causing a crash. FossilOrigin-Name: c0cdaa07355f51bd217881b5b91ec2a609fa56f4 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/mmap2.test | 4 ++++ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 1ca22b55cf..aa9f6ee2cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sassert\sto\sthe\sWindows\sVFS\spertaining\sto\sthe\ssize\sof\sthe\sSIZE_T\stype. -D 2013-04-02T20:13:04.021 +C Fix\sa\sbug\sin\smmap2.test\sthat\swas\sleaving\san\sinvalid\ssqlite3_log()\scallback\sinstalled,\scausing\sa\scrash. +D 2013-04-02T20:19:22.780 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -632,7 +632,7 @@ F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 F test/mmap1.test 245eb4a6280d4b31523eb79486863afadac4f3c6 -F test/mmap2.test bb0c97cdac0db06a5acdc6836eef9866815481b1 +F test/mmap2.test c0cbb978eda8d06d755ba8d9e59ec06ebf60c5cb F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3050136be962427cc4dcced6077ef29b2a941405 -R 69136e8e9adf1c4824020b7a88c6a9a7 -U mistachkin -Z 0311bae3bb3a8c1122f90eda3126446b +P 4eeea58354e21797ca1f4dc4ea785d0330381f0b +R 6d15151e9d49c450ba09b87b5b0a03e1 +U dan +Z 90fc4c0a95395512a6e1551c81a3562f diff --git a/manifest.uuid b/manifest.uuid index 341d71bfa9..6ec6c81697 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4eeea58354e21797ca1f4dc4ea785d0330381f0b \ No newline at end of file +c0cdaa07355f51bd217881b5b91ec2a609fa56f4 \ No newline at end of file diff --git a/test/mmap2.test b/test/mmap2.test index d43ea97adf..fb36507efd 100644 --- a/test/mmap2.test +++ b/test/mmap2.test @@ -74,6 +74,10 @@ foreach syscall {mmap mremap} { } } +db close test_syscall uninstall +sqlite3_shutdown +test_sqlite3_log +sqlite3_initialize finish_test From c3d53189fca4ec61a2f697ecc23bfb16328360b0 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 20:29:33 +0000 Subject: [PATCH 078/204] Adjustments to mmap1.test so that it works better on windows. It still gets some answers wrong, but it no longer crashes. FossilOrigin-Name: ee8d3ceeec40d84adb8798f084b6f1215ab56d92 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/mmap1.test | 19 +++++++++++-------- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index aa9f6ee2cd..abae49480a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sin\smmap2.test\sthat\swas\sleaving\san\sinvalid\ssqlite3_log()\scallback\sinstalled,\scausing\sa\scrash. -D 2013-04-02T20:19:22.780 +C Adjustments\sto\smmap1.test\sso\sthat\sit\sworks\sbetter\son\swindows.\s\sIt\sstill\sgets\nsome\sanswers\swrong,\sbut\sit\sno\slonger\scrashes. +D 2013-04-02T20:29:33.219 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -631,7 +631,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 245eb4a6280d4b31523eb79486863afadac4f3c6 +F test/mmap1.test 95f8f90b7f4e1ef3499ea5cc69a9c3e7b1d36227 F test/mmap2.test c0cbb978eda8d06d755ba8d9e59ec06ebf60c5cb F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 4eeea58354e21797ca1f4dc4ea785d0330381f0b -R 6d15151e9d49c450ba09b87b5b0a03e1 -U dan -Z 90fc4c0a95395512a6e1551c81a3562f +P c0cdaa07355f51bd217881b5b91ec2a609fa56f4 +R 2d6cafbf0f715d3130e0ad2821f95af0 +U drh +Z eb8137777309edfff1693e4a3a7e15de diff --git a/manifest.uuid b/manifest.uuid index 6ec6c81697..6a1646888a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c0cdaa07355f51bd217881b5b91ec2a609fa56f4 \ No newline at end of file +ee8d3ceeec40d84adb8798f084b6f1215ab56d92 \ No newline at end of file diff --git a/test/mmap1.test b/test/mmap1.test index e63e09481c..f3e7268105 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -20,18 +20,20 @@ proc nRead {db} { db_enter $db array set stats [btree_pager_stats $bt] db_leave $db + # puts [array get stats] return $stats(read) } foreach {t mmap_limit nRead c2init} { - 1.1 { PRAGMA mmap_limit = 70000000 } 4 {} - 1.2 { PRAGMA mmap_limit = 51200 } 156 {} + 1.1 { PRAGMA mmap_limit = 67108864 } 4 {} + 1.2 { PRAGMA mmap_limit = 53248 } 150 {} 1.3 { PRAGMA mmap_limit = 0 } 344 {} - 1.4 { PRAGMA mmap_limit = 70000000 } 4 {PRAGMA mmap_limit = 70000000 } - 1.5 { PRAGMA mmap_limit = 51200 } 156 {PRAGMA mmap_limit = 70000000 } - 1.6 { PRAGMA mmap_limit = 0 } 344 {PRAGMA mmap_limit = 70000000 } + 1.4 { PRAGMA mmap_limit = 67108864 } 4 {PRAGMA mmap_limit = 67108864 } + 1.5 { PRAGMA mmap_limit = 53248 } 150 {PRAGMA mmap_limit = 67108864 } + 1.6 { PRAGMA mmap_limit = 0 } 344 {PRAGMA mmap_limit = 67108864 } } { do_multiclient_test tn { + sql1 {PRAGMA page_size=1024} sql1 $mmap_limit sql2 $c2init @@ -45,7 +47,8 @@ foreach {t mmap_limit nRead c2init} { db2 func rblob rblob } - sql2 { + sql2 { + PRAGMA page_size=1024; PRAGMA auto_vacuum = 1; CREATE TABLE t1(a, b, UNIQUE(a, b)); INSERT INTO t1 VALUES(rblob(500), rblob(500)); @@ -95,7 +98,7 @@ db func rblob rblob do_execsql_test 2.1 { PRAGMA auto_vacuum = 1; - PRAGMA mmap_limit = 70000000; + PRAGMA mmap_limit = 67108864; PRAGMA journal_mode = wal; CREATE TABLE t1(a, b, UNIQUE(a, b)); INSERT INTO t1 VALUES(rblob(500), rblob(500)); @@ -129,7 +132,7 @@ do_execsql_test 2.4 { PRAGMA wal_checkpoint; } {0 24 24} - +db2 close reset_db db func rblob rblob do_execsql_test 3.1 { From 48ccef80595731d2c2b6d759455837db3113f647 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Apr 2013 20:55:01 +0000 Subject: [PATCH 079/204] Fix a resource leak in os_unix.c. FossilOrigin-Name: b29cda03fe4e8d8f5b5acbbea2d69f284a2bdf23 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 6 ++---- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index abae49480a..041087f917 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjustments\sto\smmap1.test\sso\sthat\sit\sworks\sbetter\son\swindows.\s\sIt\sstill\sgets\nsome\sanswers\swrong,\sbut\sit\sno\slonger\scrashes. -D 2013-04-02T20:29:33.219 +C Fix\sa\sresource\sleak\sin\sos_unix.c. +D 2013-04-02T20:55:01.697 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 42684a59b06b912b3ab80a8ad9f4295c0572a1f4 +F src/os_unix.c e8e76c95425e60a4d0c749c1ed5d6e00109782f5 F src/os_win.c 6a7a1017fb6a0783a3aa7c3f5eadd9b1600a9ddf F src/pager.c e26184d451207542d56bd09b84339252a194791e F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c0cdaa07355f51bd217881b5b91ec2a609fa56f4 -R 2d6cafbf0f715d3130e0ad2821f95af0 -U drh -Z eb8137777309edfff1693e4a3a7e15de +P ee8d3ceeec40d84adb8798f084b6f1215ab56d92 +R 40861d78d26ec4eddf163e4d116b8b4f +U dan +Z 36ae50dfb6860cc1a436ea46d4e8b8f3 diff --git a/manifest.uuid b/manifest.uuid index 6a1646888a..cb24cad099 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ee8d3ceeec40d84adb8798f084b6f1215ab56d92 \ No newline at end of file +b29cda03fe4e8d8f5b5acbbea2d69f284a2bdf23 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 07c919c983..581afdbc91 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4601,10 +4601,8 @@ static void unixRemapfile( } #endif - /* The attempt to extend the existing mapping failed. Free the existing - ** mapping and set pNew to NULL so that the code below will create a - ** new mapping from scratch. */ - if( pNew==MAP_FAILED ){ + /* The attempt to extend the existing mapping failed. Free it. */ + if( pNew==MAP_FAILED || pNew==0 ){ osMunmap(pOrig, nReuse); } } From 85830a70c2d07e345564211e13e7f5e6182f4135 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 00:42:01 +0000 Subject: [PATCH 080/204] Fix the unix driver to check defined(_BSD_SOURCE) rather than just the plain _BSD_SOURCE macro. This fixes the build for OpenBSD. FossilOrigin-Name: 1dd42ef4144ee08fb4ee1676d934a56a0e34bac2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 041087f917..f907c58839 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sresource\sleak\sin\sos_unix.c. -D 2013-04-02T20:55:01.697 +C Fix\sthe\sunix\sdriver\sto\scheck\sdefined(_BSD_SOURCE)\srather\sthan\sjust\sthe\splain\n_BSD_SOURCE\smacro.\s\sThis\sfixes\sthe\sbuild\sfor\sOpenBSD. +D 2013-04-03T00:42:01.544 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c e8e76c95425e60a4d0c749c1ed5d6e00109782f5 +F src/os_unix.c f7d8c6ef5f45111f62a77984053c17e767f9679e F src/os_win.c 6a7a1017fb6a0783a3aa7c3f5eadd9b1600a9ddf F src/pager.c e26184d451207542d56bd09b84339252a194791e F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ee8d3ceeec40d84adb8798f084b6f1215ab56d92 -R 40861d78d26ec4eddf163e4d116b8b4f -U dan -Z 36ae50dfb6860cc1a436ea46d4e8b8f3 +P b29cda03fe4e8d8f5b5acbbea2d69f284a2bdf23 +R 12d119a8bfdd7260b7f76958004bb415 +U drh +Z 8724856398770e48d9cea9c9c7fa6257 diff --git a/manifest.uuid b/manifest.uuid index cb24cad099..4593a953a2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b29cda03fe4e8d8f5b5acbbea2d69f284a2bdf23 \ No newline at end of file +1dd42ef4144ee08fb4ee1676d934a56a0e34bac2 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 581afdbc91..6ca99fa532 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4534,7 +4534,7 @@ static void unixUnmapfile(unixFile *pFd){ static int unixGetPagesize(void){ #if HAVE_MREMAP return 512; -#elif _BSD_SOURCE +#elif defined(_BSD_SOURCE) return getpagesize(); #else return (int)sysconf(_SC_PAGESIZE); From 7d9f3943ba060edd2389b4b2c27db0d02524f11d Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 01:26:54 +0000 Subject: [PATCH 081/204] Add the -mmap option to the command-line shell, for setting the default mmap_limit. FossilOrigin-Name: fc30d06c94c8212abb0477fb4cec4520d05bea34 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 53 +++++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 52 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index f907c58839..0b74d42186 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sunix\sdriver\sto\scheck\sdefined(_BSD_SOURCE)\srather\sthan\sjust\sthe\splain\n_BSD_SOURCE\smacro.\s\sThis\sfixes\sthe\sbuild\sfor\sOpenBSD. -D 2013-04-03T00:42:01.544 +C Add\sthe\s-mmap\soption\sto\sthe\scommand-line\sshell,\sfor\ssetting\sthe\sdefault\nmmap_limit. +D 2013-04-03T01:26:54.789 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -175,7 +175,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 -F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca +F src/shell.c 319b7791cee6c763b60fde1b590bfaf62613cf37 F src/sqlite.h.in 42c69ea00dfcce84394d6ce0d8dac7bb3d2ecff2 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b29cda03fe4e8d8f5b5acbbea2d69f284a2bdf23 -R 12d119a8bfdd7260b7f76958004bb415 +P 1dd42ef4144ee08fb4ee1676d934a56a0e34bac2 +R 414201a51e65d2b5d08b22317803af41 U drh -Z 8724856398770e48d9cea9c9c7fa6257 +Z ee0a75b1373c6c61317486123f3b4064 diff --git a/manifest.uuid b/manifest.uuid index 4593a953a2..ac22576f6d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1dd42ef4144ee08fb4ee1676d934a56a0e34bac2 \ No newline at end of file +fc30d06c94c8212abb0477fb4cec4520d05bea34 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index c6d7fa3d53..6a0e047475 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1552,6 +1552,43 @@ static int booleanValue(char *zArg){ return 0; } +/* +** Interpret zArg as an integer value, possibly with suffixes. +*/ +static sqlite3_int64 integerValue(const char *zArg){ + sqlite3_int64 v = 0; + static const struct { char *zSuffix; int iMult; } aMult[] = { + { "KiB", 1024 }, + { "MiB", 1024*1024 }, + { "GiB", 1024*1024*1024 }, + { "KB", 1000 }, + { "MB", 1000000 }, + { "GB", 1000000000 }, + { "K", 1000 }, + { "M", 1000000 }, + { "G", 1000000000 }, + }; + int i; + int isNeg = 0; + if( zArg[0]=='-' ){ + isNeg = 1; + zArg++; + }else if( zArg[0]=='+' ){ + zArg++; + } + while( isdigit(zArg[0]) ){ + v = v*10 + zArg[0] - '0'; + zArg++; + } + for(i=0; i0x7fff0000 ) szHeap = 0x7fff0000; sqlite3_config(SQLITE_CONFIG_HEAP, malloc((int)szHeap), (int)szHeap, 64); #endif @@ -3026,6 +3059,8 @@ int main(int argc, char **argv){ extern int sqlite3_multiple_initialize(const char*,int); sqlite3_multiplex_initialize(0, 1); #endif + }else if( strcmp(z,"-mmap")==0 ){ + sqlite3_config(SQLITE_CONFIG_MMAP_LIMIT, integerValue(cmdline_option_value(argc,argv,++i))); }else if( strcmp(z,"-vfs")==0 ){ sqlite3_vfs *pVfs = sqlite3_vfs_find(cmdline_option_value(argc,argv,++i)); if( pVfs ){ @@ -3111,6 +3146,8 @@ int main(int argc, char **argv){ stdin_is_interactive = 0; }else if( strcmp(z,"-heap")==0 ){ i++; + }else if( strcmp(z,"-mmap")==0 ){ + i++; }else if( strcmp(z,"-vfs")==0 ){ i++; #ifdef SQLITE_ENABLE_VFSTRACE From 7e36962e6c152768483b9282149ff975359d7ed4 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 03:53:15 +0000 Subject: [PATCH 082/204] Make use of xFetch, if it is available when doing a VACUUM or a backup. FossilOrigin-Name: 83bc37af07857960c11275891f853a358dcbbf05 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/backup.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 0b74d42186..cbde44af45 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s-mmap\soption\sto\sthe\scommand-line\sshell,\sfor\ssetting\sthe\sdefault\nmmap_limit. -D 2013-04-03T01:26:54.789 +C Make\suse\sof\sxFetch,\sif\sit\sis\savailable\swhen\sdoing\sa\sVACUUM\sor\sa\sbackup. +D 2013-04-03T03:53:15.647 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -118,7 +118,7 @@ F src/alter.c f8db986c03eb0bfb221523fc9bbb9d0b70de3168 F src/analyze.c d5f895810e8ff9737c9ec7b76abc3dcff5860335 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 -F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 +F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c bb679fddb7269dd7cfb950f845b1b18f19df4f4a @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 1dd42ef4144ee08fb4ee1676d934a56a0e34bac2 -R 414201a51e65d2b5d08b22317803af41 +P fc30d06c94c8212abb0477fb4cec4520d05bea34 +R 8f21f6233365f9239c040421dd942d16 U drh -Z ee0a75b1373c6c61317486123f3b4064 +Z fab31801d3b721c34a1407901bd055e3 diff --git a/manifest.uuid b/manifest.uuid index ac22576f6d..847cb431dc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fc30d06c94c8212abb0477fb4cec4520d05bea34 \ No newline at end of file +83bc37af07857960c11275891f853a358dcbbf05 \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 71a8a1a3e7..252f61cfca 100644 --- a/src/backup.c +++ b/src/backup.c @@ -397,7 +397,8 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ const Pgno iSrcPg = p->iNext; /* Source page number */ if( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ){ DbPage *pSrcPg; /* Source page object */ - rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); + rc = sqlite3PagerAcquire(pSrcPager, iSrcPg, &pSrcPg, + PAGER_ACQUIRE_READONLY); if( rc==SQLITE_OK ){ rc = backupOnePage(p, iSrcPg, sqlite3PagerGetData(pSrcPg), 0); sqlite3PagerUnref(pSrcPg); From 2b8246e3e23cb98b6d7339de019d446aa3bc3bb8 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 10:50:02 +0000 Subject: [PATCH 083/204] Initialize the mmap_limit of temporary files to the configured mmap_limit. FossilOrigin-Name: 24bab7596bb7385981a5d331df5eeb05353547f7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 2 +- src/os_win.c | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index cbde44af45..78b0bffaa7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\suse\sof\sxFetch,\sif\sit\sis\savailable\swhen\sdoing\sa\sVACUUM\sor\sa\sbackup. -D 2013-04-03T03:53:15.647 +C Initialize\sthe\smmap_limit\sof\stemporary\sfiles\sto\sthe\sconfigured\smmap_limit. +D 2013-04-03T10:50:02.049 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,8 +160,8 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c f7d8c6ef5f45111f62a77984053c17e767f9679e -F src/os_win.c 6a7a1017fb6a0783a3aa7c3f5eadd9b1600a9ddf +F src/os_unix.c 626ddccfd5b1be0c91ca5a9d147c8c3e824d7013 +F src/os_win.c 6718b053fe8295ede770b7a1e48ba4ef73310e9c F src/pager.c e26184d451207542d56bd09b84339252a194791e F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P fc30d06c94c8212abb0477fb4cec4520d05bea34 -R 8f21f6233365f9239c040421dd942d16 +P 83bc37af07857960c11275891f853a358dcbbf05 +R 32aa00c0c8e64a488f9b37d8aa36924f U drh -Z fab31801d3b721c34a1407901bd055e3 +Z db8bee74489339af40a69aa3740a7e67 diff --git a/manifest.uuid b/manifest.uuid index 847cb431dc..56b76309a3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83bc37af07857960c11275891f853a358dcbbf05 \ No newline at end of file +24bab7596bb7385981a5d331df5eeb05353547f7 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 6ca99fa532..39a31cb188 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5060,7 +5060,7 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; - pNew->mmapLimit = SQLITE_DEFAULT_MMAP_LIMIT; + pNew->mmapLimit = sqlite3GlobalConfig.mxMmap; if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; diff --git a/src/os_win.c b/src/os_win.c index a978613e2e..74aede5704 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -4116,7 +4116,7 @@ static int winOpen( pFile->pMapRegion = 0; pFile->mmapSize = 0; pFile->mmapOrigsize = 0; - pFile->mmapLimit = SQLITE_DEFAULT_MMAP_LIMIT; + pFile->mmapLimit = sqlite3GlobalConfig.mxMmap; OpenCounter(+1); return rc; From 227a1c482cb2e725712f5a5f75fa0fd977a586ef Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 3 Apr 2013 11:17:39 +0000 Subject: [PATCH 084/204] In btree.c, save the positions of other cursors open on the same table when writing via an incremental blob handle. Otherwise, they may be left holding an out-of-date xFetch page reference. FossilOrigin-Name: 3f09fba18f7e61e21381ffea13c31b968efd7d77 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 9 +++++++++ test/mmap1.test | 51 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 78b0bffaa7..a260417387 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initialize\sthe\smmap_limit\sof\stemporary\sfiles\sto\sthe\sconfigured\smmap_limit. -D 2013-04-03T10:50:02.049 +C In\sbtree.c,\ssave\sthe\spositions\sof\sother\scursors\sopen\son\sthe\ssame\stable\swhen\swriting\svia\san\sincremental\sblob\shandle.\sOtherwise,\sthey\smay\sbe\sleft\sholding\san\sout-of-date\sxFetch\spage\sreference. +D 2013-04-03T11:17:39.067 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c bb679fddb7269dd7cfb950f845b1b18f19df4f4a +F src/btree.c b38cb3a1758d46e76bec080994fed7313622687d F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -631,7 +631,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 95f8f90b7f4e1ef3499ea5cc69a9c3e7b1d36227 +F test/mmap1.test 9f4cee0877549c80b13acd688c775254b2d61ec7 F test/mmap2.test c0cbb978eda8d06d755ba8d9e59ec06ebf60c5cb F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 83bc37af07857960c11275891f853a358dcbbf05 -R 32aa00c0c8e64a488f9b37d8aa36924f -U drh -Z db8bee74489339af40a69aa3740a7e67 +P 24bab7596bb7385981a5d331df5eeb05353547f7 +R 5d2a80f93297e2bae93eb7d43d59c6c0 +U dan +Z a5b2fa9ebb6e4e86d546b82b43bf4939 diff --git a/manifest.uuid b/manifest.uuid index 56b76309a3..d1325fe2f5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -24bab7596bb7385981a5d331df5eeb05353547f7 \ No newline at end of file +3f09fba18f7e61e21381ffea13c31b968efd7d77 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9672e0e000..a1877ebfe8 100644 --- a/src/btree.c +++ b/src/btree.c @@ -8400,6 +8400,15 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ return SQLITE_ABORT; } + /* Save the positions of all other cursors open on this table. This is + ** required in case any of them are holding references to an xFetch + ** version of the b-tree page modified by the accessPayload call below. + */ + rc = saveAllCursors(pCsr->pBt, pCsr->pgnoRoot, pCsr); + if( rc!=SQLITE_OK ){ + return SQLITE_OK; + } + /* Check some assumptions: ** (a) the cursor is open for writing, ** (b) there is a read/write transaction open, diff --git a/test/mmap1.test b/test/mmap1.test index f3e7268105..802b921048 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -157,4 +157,55 @@ do_test 3.2 { set nRow } {8} +#------------------------------------------------------------------------- +# Ensure that existing cursors using xFetch() pages see changes made +# to rows using the incrblob API. +# +reset_db +set aaa [string repeat a 400] +set bbb [string repeat b 400] +set ccc [string repeat c 400] +set ddd [string repeat d 400] +set eee [string repeat e 400] + +do_execsql_test 4.1 { + PRAGMA page_size = 1024; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES($aaa); + INSERT INTO t1 VALUES($bbb); + INSERT INTO t1 VALUES($ccc); + INSERT INTO t1 VALUES($ddd); + SELECT * FROM t1; + BEGIN; +} [list $aaa $bbb $ccc $ddd] + +do_test 4.2 { + set ::STMT [sqlite3_prepare db "SELECT * FROM t1 ORDER BY rowid" -1 dummy] + sqlite3_step $::STMT + sqlite3_column_text $::STMT 0 +} $aaa + +do_test 4.3 { + foreach r {2 3 4} { + set fd [db incrblob t1 x $r] + puts -nonewline $fd $eee + close $fd + } + + set res [list] + while {"SQLITE_ROW" == [sqlite3_step $::STMT]} { + lappend res [sqlite3_column_text $::STMT 0] + } + set res +} [list $eee $eee $eee] + +do_test 4.4 { + sqlite3_finalize $::STMT +} SQLITE_OK + +do_execsql_test 4.5 { COMMIT } + finish_test + + + From f7679ad19eaaf249bd803010eab2da14f21c61e9 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 3 Apr 2013 11:38:36 +0000 Subject: [PATCH 085/204] When moving a page to make way for the root page of a new table or index in an auto-vacuum database, save the positions of any cursors that may be holding xFetch references to the page being moved. FossilOrigin-Name: 9d9b1da54a555e8fb6037d63d1952458c12956d2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 8 ++++++++ test/mmap1.test | 41 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index a260417387..cf7c875169 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sbtree.c,\ssave\sthe\spositions\sof\sother\scursors\sopen\son\sthe\ssame\stable\swhen\swriting\svia\san\sincremental\sblob\shandle.\sOtherwise,\sthey\smay\sbe\sleft\sholding\san\sout-of-date\sxFetch\spage\sreference. -D 2013-04-03T11:17:39.067 +C When\smoving\sa\spage\sto\smake\sway\sfor\sthe\sroot\spage\sof\sa\snew\stable\sor\sindex\sin\san\sauto-vacuum\sdatabase,\ssave\sthe\spositions\sof\sany\scursors\sthat\smay\sbe\sholding\sxFetch\sreferences\sto\sthe\spage\sbeing\smoved. +D 2013-04-03T11:38:36.388 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c b38cb3a1758d46e76bec080994fed7313622687d +F src/btree.c 4f4b23941b38f68db425f68960f26031204d9807 F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -631,7 +631,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 9f4cee0877549c80b13acd688c775254b2d61ec7 +F test/mmap1.test 5a662d506c367b082b24cb9186acb11efa6cbe3d F test/mmap2.test c0cbb978eda8d06d755ba8d9e59ec06ebf60c5cb F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 24bab7596bb7385981a5d331df5eeb05353547f7 -R 5d2a80f93297e2bae93eb7d43d59c6c0 +P 3f09fba18f7e61e21381ffea13c31b968efd7d77 +R 7ec6dcb52e20c580b46b605bb884042f U dan -Z a5b2fa9ebb6e4e86d546b82b43bf4939 +Z c27841a9d742e32f5c92b396fb592137 diff --git a/manifest.uuid b/manifest.uuid index d1325fe2f5..ab30a2773c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3f09fba18f7e61e21381ffea13c31b968efd7d77 \ No newline at end of file +9d9b1da54a555e8fb6037d63d1952458c12956d2 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index a1877ebfe8..dd37112c39 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7269,6 +7269,14 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ u8 eType = 0; Pgno iPtrPage = 0; + /* Save the positions of any open cursors. This is required in + ** case they are holding a reference to an xFetch reference + ** corresponding to page pgnoRoot. */ + rc = saveAllCursors(pBt, 0, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + releasePage(pPageMove); /* Move the page currently at pgnoRoot to pgnoMove. */ diff --git a/test/mmap1.test b/test/mmap1.test index 802b921048..d9babe3d86 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -205,7 +205,46 @@ do_test 4.4 { do_execsql_test 4.5 { COMMIT } +#------------------------------------------------------------------------- +# Ensure that existing cursors holding xFetch() references are not +# confused if those pages are moved to make way for the root page of a +# new table or index. +# +reset_db +do_execsql_test 5.1 { + PRAGMA auto_vacuum = 2; + PRAGMA page_size = 1024; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES($aaa); + INSERT INTO t1 VALUES($bbb); + INSERT INTO t1 VALUES($ccc); + INSERT INTO t1 VALUES($ddd); + + PRAGMA auto_vacuum; + SELECT * FROM t1; +} [list 2 $aaa $bbb $ccc $ddd] + +do_test 5.2 { + set ::STMT [sqlite3_prepare db "SELECT * FROM t1 ORDER BY rowid" -1 dummy] + sqlite3_step $::STMT + sqlite3_column_text $::STMT 0 +} $aaa + +do_execsql_test 5.3 { + CREATE TABLE t2(x); + INSERT INTO t2 VALUES('tricked you!'); + INSERT INTO t2 VALUES('tricked you!'); +} + +do_test 5.4 { + sqlite3_step $::STMT + sqlite3_column_text $::STMT 0 +} $bbb + +do_test 5.5 { + sqlite3_finalize $::STMT +} SQLITE_OK + finish_test - From 6c96946475fd5b61168ba6b7e43b33e46fee2a9f Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 3 Apr 2013 11:52:16 +0000 Subject: [PATCH 086/204] Remove the restriction on using xFetch to load the root pages of tables and indexes. It appears to have been based on a misconception. FossilOrigin-Name: 5b082efead488a2fccc18171e640e0aa5252d1d0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 33 +-------------------------------- 3 files changed, 8 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index cf7c875169..5aaf2a1f4f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\smoving\sa\spage\sto\smake\sway\sfor\sthe\sroot\spage\sof\sa\snew\stable\sor\sindex\sin\san\sauto-vacuum\sdatabase,\ssave\sthe\spositions\sof\sany\scursors\sthat\smay\sbe\sholding\sxFetch\sreferences\sto\sthe\spage\sbeing\smoved. -D 2013-04-03T11:38:36.388 +C Remove\sthe\srestriction\son\susing\sxFetch\sto\sload\sthe\sroot\spages\sof\stables\sand\sindexes.\sIt\sappears\sto\shave\sbeen\sbased\son\sa\smisconception. +D 2013-04-03T11:52:16.489 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 4f4b23941b38f68db425f68960f26031204d9807 +F src/btree.c fd492d65eee0eb35f4000ddf478215c99dca6838 F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3f09fba18f7e61e21381ffea13c31b968efd7d77 -R 7ec6dcb52e20c580b46b605bb884042f +P 9d9b1da54a555e8fb6037d63d1952458c12956d2 +R a9b149f10f100efb054372d5cd9076ff U dan -Z c27841a9d742e32f5c92b396fb592137 +Z cc4101c000adda873dbba809b24aeeeb diff --git a/manifest.uuid b/manifest.uuid index ab30a2773c..a21ce49234 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9d9b1da54a555e8fb6037d63d1952458c12956d2 \ No newline at end of file +5b082efead488a2fccc18171e640e0aa5252d1d0 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index dd37112c39..42c4ee3cc0 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2583,34 +2583,6 @@ int sqlite3BtreeNewDb(Btree *p){ return rc; } -/* -** Ensure that any root page references held by open cursors are not -** mmap pages. -*/ -static int btreeSwapOutMmap(BtShared *pBt){ - int rc = SQLITE_OK; /* Return code */ - BtCursor *pCsr; /* Used to iterate through all open cursors */ - - for(pCsr=pBt->pCursor; pCsr && rc==SQLITE_OK; pCsr=pCsr->pNext){ - if( pCsr->iPage>=0 ){ - MemPage *pPg = pCsr->apPage[0]; - if( pPg && pPg->pDbPage->flags & PGHDR_MMAP ){ - MemPage *pNew = 0; - rc = getAndInitPage(pBt, pPg->pgno, &pNew, 0); - if( rc==SQLITE_OK ){ - if( pCsr->iPage==0 ){ - pCsr->info.pCell = pNew->aData + (pCsr->info.pCell - pPg->aData); - } - pCsr->apPage[0] = pNew; - releasePage(pPg); - } - } - } - } - - return rc; -} - /* ** Attempt to start a new transaction. A write-transaction ** is started if the second argument is nonzero, otherwise a read- @@ -2717,9 +2689,6 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ rc = SQLITE_READONLY; }else{ rc = sqlite3PagerBegin(pBt->pPager,wrflag>1,sqlite3TempInMemory(p->db)); - if( rc==SQLITE_OK ){ - rc = btreeSwapOutMmap(pBt); - } if( rc==SQLITE_OK ){ rc = newDatabase(pBt); } @@ -4419,7 +4388,7 @@ static int moveToRoot(BtCursor *pCur){ pCur->eState = CURSOR_INVALID; return SQLITE_OK; }else{ - rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0], 0); + rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0], pCur->wrFlag==0); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; From 34f7490311f77d0da8c90260d083f19550fd6c61 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 13:09:18 +0000 Subject: [PATCH 087/204] Change the mmap_limit pragma to report the new limit, or to report the existing limit if called with no arguments. Report the default mmap_limit as part of PRAGMA compile_options. Set the default mmmap_limit to 0 for all systems other than linux, mac, windows, and solaris. FossilOrigin-Name: 2d9f1327fe79e40435ce1e2594d7cd9a5aea0ef2 --- manifest | 28 ++++++++++++++-------------- manifest.uuid | 2 +- src/ctime.c | 3 +++ src/os_unix.c | 4 +++- src/os_win.c | 4 +++- src/pager.c | 5 +++-- src/pragma.c | 14 +++++++++----- src/sqlite.h.in | 4 +++- src/sqliteLimit.h | 10 +++------- test/mmap1.test | 2 +- 10 files changed, 43 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 5aaf2a1f4f..a97a1a1a46 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\srestriction\son\susing\sxFetch\sto\sload\sthe\sroot\spages\sof\stables\sand\sindexes.\sIt\sappears\sto\shave\sbeen\sbased\son\sa\smisconception. -D 2013-04-03T11:52:16.489 +C Change\sthe\smmap_limit\spragma\sto\sreport\sthe\snew\slimit,\sor\sto\sreport\sthe\nexisting\slimit\sif\scalled\swith\sno\sarguments.\s\sReport\sthe\sdefault\smmap_limit\nas\spart\sof\sPRAGMA\scompile_options.\s\sSet\sthe\sdefault\smmmap_limit\sto\s0\sfor\nall\ssystems\sother\sthan\slinux,\smac,\swindows,\sand\ssolaris. +D 2013-04-03T13:09:18.208 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -127,7 +127,7 @@ F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c 2a5f251fcd7393808df77ccfc817e7058df08c4c +F src/ctime.c 16658a257bc6a3ca8d8961f574cf61a57e4d6faf F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 @@ -160,15 +160,15 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 626ddccfd5b1be0c91ca5a9d147c8c3e824d7013 -F src/os_win.c 6718b053fe8295ede770b7a1e48ba4ef73310e9c -F src/pager.c e26184d451207542d56bd09b84339252a194791e +F src/os_unix.c f0ecce40d92469d5cc737ae883e776eb3e5c0af5 +F src/os_win.c 1d0ccc1880b626ac2b6868284f6338c90687e8d8 +F src/pager.c 2e68df46d4086027cb6b527d47a6dedbf1a6b7ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c d0231f412dea33c35c1309b45b535cf0c10c63c8 +F src/pragma.c 682e97f3e3b77fd6c9b569eabfbf4a14c987aca3 F src/prepare.c 310eaff1ee5f3c700b3545afb095cfe9346efc3a F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -176,11 +176,11 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c 319b7791cee6c763b60fde1b590bfaf62613cf37 -F src/sqlite.h.in 42c69ea00dfcce84394d6ce0d8dac7bb3d2ecff2 +F src/sqlite.h.in faeb6b3470193e599d79289f77f984b8a78136e1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 1664dc5ad6f8d4dab871416628aa3271044d66c0 -F src/sqliteLimit.h edc1e61e6b745dc26ea16542838c100cef02df0e +F src/sqliteLimit.h 299a15148feccc4b85db3b9f3c652ee5b60267b4 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 @@ -631,7 +631,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 5a662d506c367b082b24cb9186acb11efa6cbe3d +F test/mmap1.test df5105f08e6000e57b4de7e748f8c2ae3fed75da F test/mmap2.test c0cbb978eda8d06d755ba8d9e59ec06ebf60c5cb F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9d9b1da54a555e8fb6037d63d1952458c12956d2 -R a9b149f10f100efb054372d5cd9076ff -U dan -Z cc4101c000adda873dbba809b24aeeeb +P 5b082efead488a2fccc18171e640e0aa5252d1d0 +R 779b4d1891b69fa3c237308161c44940 +U drh +Z c380833e5cb3768ffb9f1e28bd39c0ad diff --git a/manifest.uuid b/manifest.uuid index a21ce49234..a842cd26d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5b082efead488a2fccc18171e640e0aa5252d1d0 \ No newline at end of file +2d9f1327fe79e40435ce1e2594d7cd9a5aea0ef2 \ No newline at end of file diff --git a/src/ctime.c b/src/ctime.c index c42454ca71..ee6d4cbe2e 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -57,6 +57,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_DEFAULT_LOCKING_MODE "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), #endif +#ifdef SQLITE_DEFAULT_MMAP_LIMIT + "DEFAULT_MMAP_LIMIT=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_LIMIT), +#endif #ifdef SQLITE_DISABLE_DIRSYNC "DISABLE_DIRSYNC", #endif diff --git a/src/os_unix.c b/src/os_unix.c index 39a31cb188..75f9236013 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3710,7 +3710,9 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ return SQLITE_OK; } case SQLITE_FCNTL_MMAP_LIMIT: { - pFile->mmapLimit = *(i64*)pArg; + i64 newLimit = *(i64*)pArg; + *(i64*)pArg = pFile->mmapLimit; + if( newLimit>=0 ) pFile->mmapLimit = newLimit; return SQLITE_OK; } #ifdef SQLITE_DEBUG diff --git a/src/os_win.c b/src/os_win.c index 74aede5704..dc69b47ae2 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2837,7 +2837,9 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ return SQLITE_OK; } case SQLITE_FCNTL_MMAP_LIMIT: { - pFile->mmapLimit = *(i64*)pArg; + i64 newLimit = *(i64*)pArg; + *(i64*) = pFile->mmapLimit; + if( newLimit>=0 ) pFile->mmapLimit = newLimit; return SQLITE_OK; } } diff --git a/src/pager.c b/src/pager.c index f626a21d89..ae2f10069f 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3358,9 +3358,10 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ static void pagerFixMaplimit(Pager *pPager){ sqlite3_file *fd = pPager->fd; if( isOpen(fd) ){ + sqlite3_int64 mx; pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->mxMmap>0; - sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT, - (void*)&pPager->mxMmap); + mx = pPager->mxMmap; + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT, &mx); } } diff --git a/src/pragma.c b/src/pragma.c index 7a515cba2e..3527c0974c 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -759,19 +759,23 @@ void sqlite3Pragma( ** upper layers will never invoke the xFetch interfaces to the VFS. */ if( sqlite3StrICmp(zLeft,"mmap_limit")==0 ){ + sqlite3_int64 mx; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( zRight ){ - sqlite3_int64 size; int ii; - sqlite3Atoi64(zRight, &size, 1000, SQLITE_UTF8); - if( size<0 ) size = sqlite3GlobalConfig.mxMmap; - if( pId2->n==0 ) db->mxMmap = size; + sqlite3Atoi64(zRight, &mx, 1000, SQLITE_UTF8); + if( mx<0 ) mx = sqlite3GlobalConfig.mxMmap; + if( pId2->n==0 ) db->mxMmap = mx; for(ii=db->nDb-1; ii>=0; ii--){ if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ - sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, size); + sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, mx); } } } + mx = -1; + if( sqlite3_file_control(db,zDb,SQLITE_FCNTL_MMAP_LIMIT,&mx)==SQLITE_OK ){ + returnSingleInt(pParse, "mmap_limit", mx); + } }else /* diff --git a/src/sqlite.h.in b/src/sqlite.h.in index de1316606b..a4fb1a8a9a 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -888,7 +888,9 @@ struct sqlite3_io_methods { ** **
  • [[SQLITE_FCNTL_MMAP_LIMIT]] ** The argument is assumed to pointer to a value of type sqlite3_int64 that -** is an advisory maximum number of bytes in the file to memory map. +** is an advisory maximum number of bytes in the file to memory map. The +** pointer is overwritten with the old value. The limit is not changed if +** the original value pointed to is negative. ** ** */ diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index 826d19dd9c..2b02cc5662 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -219,13 +219,9 @@ #ifndef SQLITE_DEFAULT_MMAP_LIMIT # if defined(__linux__) \ || defined(_WIN32) \ - || (defined(__APPLE__) && defined(__MACH__) && !defined(TARGET_OS_IPHONE)) \ - || defined(__sun) \ - || defined(__DragonFly__) \ - || defined(__FreeBSD__) \ - || defined(__NetBSD__) \ - || defined(__OpenBSD__) -# define SQLITE_DEFAULT_MMAP_LIMIT (256*1024*1024) + || (defined(__APPLE__) && defined(__MACH__)) \ + || defined(__sun) +# define SQLITE_DEFAULT_MMAP_LIMIT 268435456 /* = 256*1024*1024 */ # else # define SQLITE_DEFAULT_MMAP_LIMIT 0 # endif diff --git a/test/mmap1.test b/test/mmap1.test index d9babe3d86..805fb76525 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -108,7 +108,7 @@ do_execsql_test 2.1 { INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 32 PRAGMA wal_checkpoint; -} {wal 0 103 103} +} {67108864 wal 0 103 103} do_execsql_test 2.2 { PRAGMA auto_vacuum; From 716d6d2da760f51b7c22e3091bfa2a35fccdd005 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 13:38:46 +0000 Subject: [PATCH 088/204] Fix a bug in the detection of iOS when setting the default mmap_limit. FossilOrigin-Name: ac1432b3a8e968be4cbc138b4a35f34187c0c36f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqliteLimit.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a97a1a1a46..d40b31d70b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\smmap_limit\spragma\sto\sreport\sthe\snew\slimit,\sor\sto\sreport\sthe\nexisting\slimit\sif\scalled\swith\sno\sarguments.\s\sReport\sthe\sdefault\smmap_limit\nas\spart\sof\sPRAGMA\scompile_options.\s\sSet\sthe\sdefault\smmmap_limit\sto\s0\sfor\nall\ssystems\sother\sthan\slinux,\smac,\swindows,\sand\ssolaris. -D 2013-04-03T13:09:18.208 +C Fix\sa\sbug\sin\sthe\sdetection\sof\siOS\swhen\ssetting\sthe\sdefault\smmap_limit. +D 2013-04-03T13:38:46.133 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -180,7 +180,7 @@ F src/sqlite.h.in faeb6b3470193e599d79289f77f984b8a78136e1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 1664dc5ad6f8d4dab871416628aa3271044d66c0 -F src/sqliteLimit.h 299a15148feccc4b85db3b9f3c652ee5b60267b4 +F src/sqliteLimit.h 1097d2c541147d303e66515040067c6bfcf8cf21 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5b082efead488a2fccc18171e640e0aa5252d1d0 -R 779b4d1891b69fa3c237308161c44940 +P 2d9f1327fe79e40435ce1e2594d7cd9a5aea0ef2 +R e2d5c255f4ba7e0fb00c083dac088003 U drh -Z c380833e5cb3768ffb9f1e28bd39c0ad +Z 519f2edad4b962613f9e09a344e900e8 diff --git a/manifest.uuid b/manifest.uuid index a842cd26d5..7c7984449a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2d9f1327fe79e40435ce1e2594d7cd9a5aea0ef2 \ No newline at end of file +ac1432b3a8e968be4cbc138b4a35f34187c0c36f \ No newline at end of file diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index 2b02cc5662..d3a3b4ac95 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -212,7 +212,7 @@ */ #ifdef __APPLE__ # include -# if defined(TARGET_OS_IPHONE) +# if TARGET_OS_IPHONE # define SQLITE_DEFAULT_MMAP_LIMIT 0 # endif #endif From 370c9f498dbb223c05197e532668a6ba5595f78e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 20:04:04 +0000 Subject: [PATCH 089/204] Remove an unreachable branch in the direct blob I/O logic of btree.c. FossilOrigin-Name: f97d7274f48e3bb98ed17eae11abb55064134874 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 10 ++++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index d40b31d70b..e986a3818e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sin\sthe\sdetection\sof\siOS\swhen\ssetting\sthe\sdefault\smmap_limit. -D 2013-04-03T13:38:46.133 +C Remove\san\sunreachable\sbranch\sin\sthe\sdirect\sblob\sI/O\slogic\sof\sbtree.c. +D 2013-04-03T20:04:04.697 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c fd492d65eee0eb35f4000ddf478215c99dca6838 +F src/btree.c 4e3d07b3781a41e6ba9ea13ca97d7baab0a252a3 F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 2d9f1327fe79e40435ce1e2594d7cd9a5aea0ef2 -R e2d5c255f4ba7e0fb00c083dac088003 +P ac1432b3a8e968be4cbc138b4a35f34187c0c36f +R d830cc1d5cf92f6d3df5e32e65cb6d76 U drh -Z 519f2edad4b962613f9e09a344e900e8 +Z d56f9eb968ec676dc0a522c6c207a8e6 diff --git a/manifest.uuid b/manifest.uuid index 7c7984449a..d557c3a92c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac1432b3a8e968be4cbc138b4a35f34187c0c36f \ No newline at end of file +f97d7274f48e3bb98ed17eae11abb55064134874 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 42c4ee3cc0..c24481bdf7 100644 --- a/src/btree.c +++ b/src/btree.c @@ -8380,11 +8380,13 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ /* Save the positions of all other cursors open on this table. This is ** required in case any of them are holding references to an xFetch ** version of the b-tree page modified by the accessPayload call below. + ** + ** Note that pCsr must be open on a BTREE_INTKEY table and saveCursorPosition() + ** and hence saveAllCursors() cannot fail on a BTREE_INTKEY table, hence + ** saveAllCursors can only return SQLITE_OK. */ - rc = saveAllCursors(pCsr->pBt, pCsr->pgnoRoot, pCsr); - if( rc!=SQLITE_OK ){ - return SQLITE_OK; - } + VVA_ONLY(rc =) saveAllCursors(pCsr->pBt, pCsr->pgnoRoot, pCsr); + assert( rc==SQLITE_OK ); /* Check some assumptions: ** (a) the cursor is open for writing, From b0a8ea046a5083b6f0e862e5e5ddcb50c6957334 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 21:23:28 +0000 Subject: [PATCH 090/204] Release resources prior to bailing out of the btreeCreateTable() routine following an OOM error. FossilOrigin-Name: b2a72be9bab77f050bef75477a278a6294d3e854 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e986a3818e..1dc8e386cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunreachable\sbranch\sin\sthe\sdirect\sblob\sI/O\slogic\sof\sbtree.c. -D 2013-04-03T20:04:04.697 +C Release\sresources\sprior\sto\sbailing\sout\sof\sthe\sbtreeCreateTable()\sroutine\nfollowing\san\sOOM\serror. +D 2013-04-03T21:23:28.794 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 4e3d07b3781a41e6ba9ea13ca97d7baab0a252a3 +F src/btree.c 58c4d68415ca7b13a501b682a9c0ad1e287bcacb F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ac1432b3a8e968be4cbc138b4a35f34187c0c36f -R d830cc1d5cf92f6d3df5e32e65cb6d76 +P f97d7274f48e3bb98ed17eae11abb55064134874 +R 133fe42d87ba7d3deff6d35af20e1f64 U drh -Z d56f9eb968ec676dc0a522c6c207a8e6 +Z 8d00d860d2c818ad1f98631f4c1a926a diff --git a/manifest.uuid b/manifest.uuid index d557c3a92c..a6f66ce7d4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f97d7274f48e3bb98ed17eae11abb55064134874 \ No newline at end of file +b2a72be9bab77f050bef75477a278a6294d3e854 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index c24481bdf7..2fb5becb57 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7242,12 +7242,11 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ ** case they are holding a reference to an xFetch reference ** corresponding to page pgnoRoot. */ rc = saveAllCursors(pBt, 0, 0); + releasePage(pPageMove); if( rc!=SQLITE_OK ){ return rc; } - releasePage(pPageMove); - /* Move the page currently at pgnoRoot to pgnoMove. */ rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0, 0); if( rc!=SQLITE_OK ){ From 6109e50f87cbf45c2f0dd9d5939e0471ca377e4d Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 4 Apr 2013 00:51:14 +0000 Subject: [PATCH 091/204] Increase the version number to 3.7.17. FossilOrigin-Name: 5c8c5a2a12f3f5e29c91c4d16e8b4afd4307354e --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/VERSION b/VERSION index 9efca4c4be..9a4c4371ee 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.7.16.1 +3.7.17 diff --git a/configure b/configure index ddd6364831..90452c95a2 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.7.16.1. +# Generated by GNU Autoconf 2.62 for sqlite 3.7.17. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.7.16.1' -PACKAGE_STRING='sqlite 3.7.16.1' +PACKAGE_VERSION='3.7.17' +PACKAGE_STRING='sqlite 3.7.17' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1484,7 +1484,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.7.16.1 to adapt to many kinds of systems. +\`configure' configures sqlite 3.7.17 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1549,7 +1549,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.7.16.1:";; + short | recursive ) echo "Configuration of sqlite 3.7.17:";; esac cat <<\_ACEOF @@ -1665,7 +1665,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.7.16.1 +sqlite configure 3.7.17 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1679,7 +1679,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.7.16.1, which was +It was created by sqlite $as_me 3.7.17, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -14032,7 +14032,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.7.16.1, which was +This file was extended by sqlite $as_me 3.7.17, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14085,7 +14085,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.7.16.1 +sqlite config.status 3.7.17 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/manifest b/manifest index ebd18074c8..b53e120abe 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Try\sto\suse\smmap()\sto\sspeed\saccess\sto\sthe\sdatabase\sfile\son\swindows,\slinux,\nand\smac. -D 2013-04-04T00:40:17.148 +C Increase\sthe\sversion\snumber\sto\s3.7.17. +D 2013-04-04T00:51:14.877 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc 9ab787d472c9cb8bad29c2c4aeba15b06fba7bc4 F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 -F VERSION 05604ccde96fe1b37f922eddbdcb5654308261db +F VERSION 05c7bd63b96f31cfdef5c766ed91307ac121f5aa F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 17dc593f791f874d2c23a0f9360850ded0286531 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -15,7 +15,7 @@ F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 0921066a13130082764ab4ab6456f7b5bebe56de F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure 2f9994fd2e2005753809a2246c7a0816c80b2d2f x +F configure 8bb8bd13d3c918c4c1c73480930e81f955ac298a x F configure.ac 81c43d151d0b0e406be056394cc9ff4cb3fd0444 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 @@ -1044,7 +1044,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e5b710849dd66673ba0e0d935b103cb29abfcc4b b2a72be9bab77f050bef75477a278a6294d3e854 -R 60af9b4d9e73e7757090b3abb77cbb41 +P fff2be60779571c8fb89158db976ec3755e9a223 +R 31e6af012b822f077ea81ef26342c1b3 U drh -Z d05e425b384f57133414b300483bca27 +Z 4751dcf9197532f06483d5731e95cbb0 diff --git a/manifest.uuid b/manifest.uuid index 4f646cb565..c01513bb51 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fff2be60779571c8fb89158db976ec3755e9a223 \ No newline at end of file +5c8c5a2a12f3f5e29c91c4d16e8b4afd4307354e \ No newline at end of file From 67f7c780f305ea6717448f68d2a4b64c03567337 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 4 Apr 2013 01:54:10 +0000 Subject: [PATCH 092/204] Updates to the file-control documentation. No changes to code. FossilOrigin-Name: 51fab480d0487107709e6c9e00bc0512b82ef1aa --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 21 +++++++++++++-------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index b53e120abe..30005c082d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sversion\snumber\sto\s3.7.17. -D 2013-04-04T00:51:14.877 +C Updates\sto\sthe\sfile-control\sdocumentation.\s\sNo\schanges\sto\scode. +D 2013-04-04T01:54:10.804 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,7 +176,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c 319b7791cee6c763b60fde1b590bfaf62613cf37 -F src/sqlite.h.in faeb6b3470193e599d79289f77f984b8a78136e1 +F src/sqlite.h.in 5fdc866a06ac82fd3c6ccb9f59a100151a36568c F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 1664dc5ad6f8d4dab871416628aa3271044d66c0 @@ -1044,7 +1044,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P fff2be60779571c8fb89158db976ec3755e9a223 -R 31e6af012b822f077ea81ef26342c1b3 +P 5c8c5a2a12f3f5e29c91c4d16e8b4afd4307354e +R 88aa21f5b7ea257c6d372937f19d04ff U drh -Z 4751dcf9197532f06483d5731e95cbb0 +Z 1441bae4ec4fe09d305292864aa8123c diff --git a/manifest.uuid b/manifest.uuid index c01513bb51..556e99572e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5c8c5a2a12f3f5e29c91c4d16e8b4afd4307354e \ No newline at end of file +51fab480d0487107709e6c9e00bc0512b82ef1aa \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index a4fb1a8a9a..74cecbcf22 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -868,7 +868,8 @@ struct sqlite3_io_methods { ** it is able to override built-in [PRAGMA] statements. ** **
  • [[SQLITE_FCNTL_BUSYHANDLER]] -** ^This file-control may be invoked by SQLite on the database file handle +** ^The [SQLITE_FCNTL_BUSYHANDLER] +** file-control may be invoked by SQLite on the database file handle ** shortly after it is opened in order to provide a custom VFS with access ** to the connections busy-handler callback. The argument is of type (void **) ** - an array of two (void *) values. The first (void *) actually points @@ -879,7 +880,8 @@ struct sqlite3_io_methods { ** current operation. ** **
  • [[SQLITE_FCNTL_TEMPFILENAME]] -** ^Application can invoke this file-control to have SQLite generate a +** ^Application can invoke the [SQLITE_FCNTL_TEMPFILENAME] file-control +** to have SQLite generate a ** temporary filename using the same algorithm that is followed to generate ** temporary filenames for TEMP tables and other internal uses. The ** argument should be a char** which will be filled with the filename @@ -887,10 +889,14 @@ struct sqlite3_io_methods { ** invoke [sqlite3_free()] on the result to avoid a memory leak. ** **
  • [[SQLITE_FCNTL_MMAP_LIMIT]] -** The argument is assumed to pointer to a value of type sqlite3_int64 that +** The [SQLITE_FCNTL_MMAP_LIMIT] file control is used to query or set the +** maximum number of bytes that will be used for memory-mapped I/O. +** The argument is a pointer to a value of type sqlite3_int64 that ** is an advisory maximum number of bytes in the file to memory map. The ** pointer is overwritten with the old value. The limit is not changed if -** the original value pointed to is negative. +** the originally pointed to is negative, and so the current limit can be +** queried by passing in a pointer to a negative number. This file-control +** is used internally to implement [PRAGMA mmap_limit]. ** ** */ @@ -1646,11 +1652,10 @@ struct sqlite3_mem_methods { ** is the default maximum number of bytes of process address space that ** should be used for accessing each database file using memory mapping. ** The default setting can be overridden by each database connection using -** either the [PRAGMA mmap_limit] command or the "mmaplimit" query parameter -** on the [URI filename] when opening the databaes file or by using the +** either the [PRAGMA mmap_limit] command, or by using the ** [SQLITE_FCNTL_MMAP_LIMIT] file control. The value set here overrides the -** compile-time default that is set using the [SQLITE_DEFAULT_MMAP_LIMIT] -** compile-time option. If the argument to this option is negative, then +** compile-time default that is set using [SQLITE_DEFAULT_MMAP_LIMIT]. +** If the argument to this option is negative, then ** the memory map limit is set to the compile-time default. ** */ From 75f2d540acf87280e506c6ca8bf21430d2b124fe Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 4 Apr 2013 17:28:11 +0000 Subject: [PATCH 093/204] Disable memory-mapped I/O for the win32lock.test module FossilOrigin-Name: 2b3d9805a28e51f646113e22c2dff2272342306e --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/win32lock.test | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 30005c082d..e8f2288842 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\sthe\sfile-control\sdocumentation.\s\sNo\schanges\sto\scode. -D 2013-04-04T01:54:10.804 +C Disable\smemory-mapped\sI/O\sfor\sthe\swin32lock.test\smodule +D 2013-04-04T17:28:11.286 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -997,7 +997,7 @@ F test/whereD.test 3f3ee93825c94804f1fc91eef2de0d365981759a F test/whereE.test 7bd34945797efef15819368479bacc34215e4e1d F test/whereF.test a0e296643cabe5278379bc1a0aa158cf3c54a1c9 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 -F test/win32lock.test b2a539e85ae6b2d78475e016a9636b4451dc7fb9 +F test/win32lock.test 21810396eee150680cc398970839e6abdcb807b9 F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F test/zerodamage.test e7f77fded01dfcdf92ac2c5400f1e35d7a21463c F tool/build-all-msvc.bat 74fb6e5cca66ebdb6c9bbafb2f8b802f08146d38 x @@ -1044,7 +1044,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5c8c5a2a12f3f5e29c91c4d16e8b4afd4307354e -R 88aa21f5b7ea257c6d372937f19d04ff +P 51fab480d0487107709e6c9e00bc0512b82ef1aa +R c0f8ddd1641a176c1bcbffa647af40c5 U drh -Z 1441bae4ec4fe09d305292864aa8123c +Z f901f35bfd9928cf6d746ecf21982f94 diff --git a/manifest.uuid b/manifest.uuid index 556e99572e..d4cf081120 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -51fab480d0487107709e6c9e00bc0512b82ef1aa \ No newline at end of file +2b3d9805a28e51f646113e22c2dff2272342306e \ No newline at end of file diff --git a/test/win32lock.test b/test/win32lock.test index d014be4d2c..50c9f1d5f4 100644 --- a/test/win32lock.test +++ b/test/win32lock.test @@ -27,6 +27,7 @@ proc xLog {error_code msg} { lappend ::log $msg } sqlite3 db test.db +db eval {PRAGMA mmap_limit=0} do_test win32lock-1.1 { db eval { From ced9813b12a9320e0a16ddf7926a323d672ce36b Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 4 Apr 2013 19:32:53 +0000 Subject: [PATCH 094/204] Add further multi-client tests to mmap1.test. FossilOrigin-Name: a107f75d93b8acd3403d8473a75137378041a833 --- manifest | 14 ++++---- manifest.uuid | 2 +- test/mmap1.test | 89 ++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 85 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index e8f2288842..16f4057920 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\smemory-mapped\sI/O\sfor\sthe\swin32lock.test\smodule -D 2013-04-04T17:28:11.286 +C Add\sfurther\smulti-client\stests\sto\smmap1.test. +D 2013-04-04T19:32:53.279 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -631,7 +631,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test df5105f08e6000e57b4de7e748f8c2ae3fed75da +F test/mmap1.test 146a755bb382677b6f9993c61ddc7732b157d730 F test/mmap2.test 62dbb5d718e66d654d232116c5a2d96e26a071a5 F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -1044,7 +1044,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 51fab480d0487107709e6c9e00bc0512b82ef1aa -R c0f8ddd1641a176c1bcbffa647af40c5 -U drh -Z f901f35bfd9928cf6d746ecf21982f94 +P 2b3d9805a28e51f646113e22c2dff2272342306e +R 4c631120a8e96e37be49975ee7d63ac7 +U dan +Z d6d074a538192756f7d91b52bba8b7ce diff --git a/manifest.uuid b/manifest.uuid index d4cf081120..d68eca1adf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2b3d9805a28e51f646113e22c2dff2272342306e \ No newline at end of file +a107f75d93b8acd3403d8473a75137378041a833 \ No newline at end of file diff --git a/test/mmap1.test b/test/mmap1.test index 805fb76525..417a0756b1 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -24,10 +24,22 @@ proc nRead {db} { return $stats(read) } +proc register_rblob_code {dbname seed} { + return [subst -nocommands { + set ::rcnt $seed + proc rblob {n} { + set ::rcnt [expr (([set ::rcnt] << 3) + [set ::rcnt] + 456) & 0xFFFFFFFF] + set str [format %.8x [expr [set ::rcnt] ^ 0xbdf20da3]] + string range [string repeat [set str] [expr [set n]/4]] 1 [set n] + } + $dbname func rblob rblob + }] +} + foreach {t mmap_limit nRead c2init} { - 1.1 { PRAGMA mmap_limit = 67108864 } 4 {} - 1.2 { PRAGMA mmap_limit = 53248 } 150 {} - 1.3 { PRAGMA mmap_limit = 0 } 344 {} + 1.1 { PRAGMA mmap_limit = 67108864 } 4 {PRAGMA mmap_limit = 0} + 1.2 { PRAGMA mmap_limit = 53248 } 150 {PRAGMA mmap_limit = 0} + 1.3 { PRAGMA mmap_limit = 0 } 344 {PRAGMA mmap_limit = 0} 1.4 { PRAGMA mmap_limit = 67108864 } 4 {PRAGMA mmap_limit = 67108864 } 1.5 { PRAGMA mmap_limit = 53248 } 150 {PRAGMA mmap_limit = 67108864 } 1.6 { PRAGMA mmap_limit = 0 } 344 {PRAGMA mmap_limit = 67108864 } @@ -37,15 +49,7 @@ foreach {t mmap_limit nRead c2init} { sql1 $mmap_limit sql2 $c2init - code2 { - set ::rcnt 0 - proc rblob {n} { - set ::rcnt [expr (($::rcnt << 3) + $::rcnt + 456) & 0xFFFFFFFF] - set str [format %.8x [expr $::rcnt ^ 0xbdf20da3]] - string range [string repeat $str [expr $n/4]] 1 $n - } - db2 func rblob rblob - } + code2 [register_rblob_code db2 0] sql2 { PRAGMA page_size=1024; @@ -245,6 +249,67 @@ do_test 5.5 { sqlite3_finalize $::STMT } SQLITE_OK +#------------------------------------------------------------------------- +# Test various mmap_limit settings. +# +foreach {tn1 mmap1 mmap2} { + 1 6144 167773 + 2 18432 140399 + 3 43008 401302 + 4 92160 253899 + 5 190464 2 + 6 387072 752431 + 7 780288 291143 + 8 1566720 594306 + 9 3139584 829137 + 10 6285312 793963 + 11 12576768 1015590 +} { + do_multiclient_test tn { + sql1 { + CREATE TABLE t1(a PRIMARY KEY); + CREATE TABLE t2(x); + INSERT INTO t2 VALUES(''); + } + + code1 [register_rblob_code db 0] + code2 [register_rblob_code db2 444] + + sql1 "PRAGMA mmap_limit = $mmap1" + sql2 "PRAGMA mmap_limit = $mmap2" + + do_test $tn1.$tn { + for {set i 1} {$i <= 100} {incr i} { + if {$i % 2} { + set c1 sql1 + set c2 sql2 + } else { + set c1 sql2 + set c2 sql1 + } + + $c1 { + INSERT INTO t1 VALUES( rblob(5000) ); + UPDATE t2 SET x = (SELECT md5sum(a) FROM t1); + } + + set res [$c2 { + SELECT count(*) FROM t1; + SELECT x == (SELECT md5sum(a) FROM t1) FROM t2; + PRAGMA integrity_check; + }] + if {$res != [list $i 1 ok]} { + do_test $tn1.$tn.$i { + set ::res + } [list $i 1 ok] + } + } + set res 1 + } {1} + } +} + + finish_test From d7a558a9fbf6feb84ed990c67e41fe45dafd4a18 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 5 Apr 2013 20:40:43 +0000 Subject: [PATCH 095/204] Improve the code coverage of "permutations.test coverage-pager". FossilOrigin-Name: b095e2cdb61ca8487255687f58fb1024d40f3986 --- manifest | 20 ++-- manifest.uuid | 2 +- src/test_vfs.c | 12 ++- test/pager1.test | 211 ++++++++++++++++++++++++++++++++++++++++++- test/pagerfault.test | 183 ++++++++++++++++++++++++++++++++++++- test/wal.test | 20 ---- test/walfault.test | 40 +++++++- 7 files changed, 451 insertions(+), 37 deletions(-) diff --git a/manifest b/manifest index 16f4057920..2b89f591ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfurther\smulti-client\stests\sto\smmap1.test. -D 2013-04-04T19:32:53.279 +C Improve\sthe\scode\scoverage\sof\s"permutations.test\scoverage-pager". +D 2013-04-05T20:40:43.227 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd F src/test_syscall.c 7e8293e4e6971b0f44c7f7f37b1315a8cc9f6018 F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c e286f2173563f2a1747c24bcda6b9d030bf4f4e4 -F src/test_vfs.c fb16b2d9938cf0c1afc5a423b55b952fcc024275 +F src/test_vfs.c 8e6087a8b3dcc260282074b0efba14b76311120c F src/test_vfstrace.c 34b544e80ba7fb77be15395a609c669df2e660a2 F src/test_wholenumber.c 3d2b9ed1505c40ad5c5ca2ad16ae7a289d6cc251 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 @@ -651,10 +651,10 @@ F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 -F test/pager1.test 31c04bec797dda1bde337810b52efa08d1f1f08e +F test/pager1.test 622bfaea80604a96010f35256a4fc22fce584e95 F test/pager2.test 745b911dde3d1f24ae0870bd433dfa83d7c658c1 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f -F test/pagerfault.test fc2e37b2da626826dd54bb13720b4d721719b660 +F test/pagerfault.test 636c01bdbdb2bb2baf68532d6c4ef4c633022743 F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 F test/pageropt.test 32cb5a6ed7ccddf8e8c842cb44240bd9340223ce @@ -955,7 +955,7 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 -F test/wal.test 62454b2cf00b311e9e65f898aad0fef536ed561a +F test/wal.test bbd98ac09d8fbcaa80ec233757056eb1bbb7cd95 F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c @@ -971,7 +971,7 @@ F test/walcksum.test f5447800a157c9e2234fbb8e80243f0813941bde F test/walcrash.test 4457436593be8c136f9148487c7dccd5e9013af2 F test/walcrash2.test 019d60b89d96c1937adb2b30b850ac7e86e5a142 F test/walcrash3.test 595e44c6197f0d0aa509fc135be2fd0209d11a2c -F test/walfault.test 97394d8de82a99f7abf1c12ed229640607fd0ad2 +F test/walfault.test 54ad6e849c727f4da463964b9eb8c8e8e155cf82 F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483 F test/walmode.test 4022fe03ae6e830583672caa101f046438a0473c F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496 @@ -1044,7 +1044,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 2b3d9805a28e51f646113e22c2dff2272342306e -R 4c631120a8e96e37be49975ee7d63ac7 +P a107f75d93b8acd3403d8473a75137378041a833 +R 6d11c98ab18b87c4b8798ffe987d9d38 U dan -Z d6d074a538192756f7d91b52bba8b7ce +Z 50aa2590d101ae076870fa94f81e976a diff --git a/manifest.uuid b/manifest.uuid index d68eca1adf..8dbe787aae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a107f75d93b8acd3403d8473a75137378041a833 \ No newline at end of file +b095e2cdb61ca8487255687f58fb1024d40f3986 \ No newline at end of file diff --git a/src/test_vfs.c b/src/test_vfs.c index 933de99515..fcd577439c 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -125,8 +125,9 @@ struct Testvfs { #define TESTVFS_ACCESS_MASK 0x00004000 #define TESTVFS_FULLPATHNAME_MASK 0x00008000 #define TESTVFS_READ_MASK 0x00010000 +#define TESTVFS_UNLOCK_MASK 0x00020000 -#define TESTVFS_ALL_MASK 0x0001FFFF +#define TESTVFS_ALL_MASK 0x0003FFFF #define TESTVFS_MAX_PAGES 1024 @@ -467,8 +468,12 @@ static int tvfsLock(sqlite3_file *pFile, int eLock){ ** Unlock an tvfs-file. */ static int tvfsUnlock(sqlite3_file *pFile, int eLock){ - TestvfsFd *p = tvfsGetFd(pFile); - return sqlite3OsUnlock(p->pReal, eLock); + TestvfsFd *pFd = tvfsGetFd(pFile); + Testvfs *p = (Testvfs *)pFd->pVfs->pAppData; + if( p->mask&TESTVFS_WRITE_MASK && tvfsInjectIoerr(p) ){ + return SQLITE_IOERR_UNLOCK; + } + return sqlite3OsUnlock(pFd->pReal, eLock); } /* @@ -1101,6 +1106,7 @@ static int testvfs_obj_cmd( { "xClose", TESTVFS_CLOSE_MASK }, { "xAccess", TESTVFS_ACCESS_MASK }, { "xFullPathname", TESTVFS_FULLPATHNAME_MASK }, + { "xUnlock", TESTVFS_UNLOCK_MASK }, }; Tcl_Obj **apElem = 0; int nElem = 0; diff --git a/test/pager1.test b/test/pager1.test index ae47b3d932..22c84d55d0 100644 --- a/test/pager1.test +++ b/test/pager1.test @@ -15,6 +15,7 @@ source $testdir/tester.tcl source $testdir/lock_common.tcl source $testdir/malloc_common.tcl source $testdir/wal_common.tcl +set testprefix pager1 # Do not use a codec for tests in this file, as the database file is # manipulated directly using tcl scripts (using the [hexio_write] command). @@ -1382,6 +1383,7 @@ db2 close # testvfs tv -default 1 foreach sectorsize { + 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 131072 262144 } { @@ -1404,7 +1406,7 @@ foreach sectorsize { COMMIT; } file size test.db-journal - } [expr $sectorsize > 65536 ? 65536 : $sectorsize] + } [expr $sectorsize > 65536 ? 65536 : ($sectorsize<32 ? 512 : $sectorsize)] do_test pager1-10.$sectorsize.2 { execsql { @@ -2523,4 +2525,211 @@ if {$::tcl_platform(platform)=="unix"} { } {one two} } +#------------------------------------------------------------------------- +# Test that appending pages to the database file then moving those pages +# to the free-list before the transaction is committed does not cause +# an error. +# +foreach {tn pragma strsize} { + 1 { PRAGMA mmap_limit = 0 } 2400 + 2 { } 2400 + 3 { PRAGMA mmap_limit = 0 } 4400 + 4 { } 4400 +} { + reset_db + db func a_string a_string + db eval $pragma + do_execsql_test 34.$tn.1 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + } + breakpoint + do_execsql_test 34.$tn.2 { + BEGIN; + INSERT INTO t1 VALUES(2, a_string($strsize)); + DELETE FROM t1 WHERE oid=2; + COMMIT; + PRAGMA integrity_check; + } {ok} +} + +#------------------------------------------------------------------------- +# +reset_db +do_test 35 { + sqlite3 db test.db + + execsql { + CREATE TABLE t1(x, y); + PRAGMA journal_mode = WAL; + INSERT INTO t1 VALUES(1, 2); + } + + execsql { + BEGIN; + CREATE TABLE t2(a, b); + } + + hexio_write test.db-shm [expr 16*1024] [string repeat 0055 8192] + catchsql ROLLBACK +} {0 {}} + +do_multiclient_test tn { + sql1 { + PRAGMA auto_vacuum = 0; + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(1, 2); + } + + do_test 36.$tn.1 { + sql2 { PRAGMA max_page_count = 2 } + list [catch { sql2 { CREATE TABLE t2(x) } } msg] $msg + } {1 {database or disk is full}} + + sql1 { PRAGMA checkpoint_fullfsync = 1 } + sql1 { CREATE TABLE t2(x) } + + do_test 36.$tn.2 { + sql2 { INSERT INTO t2 VALUES('xyz') } + list [catch { sql2 { CREATE TABLE t3(x) } } msg] $msg + } {1 {database or disk is full}} +} + +forcedelete test1 test2 +foreach {tn uri} { + 1 {file:?mode=memory&cache=shared} + 2 {file:one?mode=memory&cache=shared} + 3 {file:test1?cache=shared} + 4 {file:test2?another=parameter&yet=anotherone} +} { + do_test 37.$tn { + catch { db close } + sqlite3_shutdown + sqlite3_config_uri 1 + sqlite3 db $uri + db eval { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(1); + SELECT * FROM t1; + } + } {1} + db close + sqlite3_shutdown + sqlite3_config_uri 0 +} + +do_test 38.1 { + catch { db close } + forcedelete test.db + set fd [open test.db w] + puts $fd "hello world" + close $fd + sqlite3 db test.db + catchsql { CREATE TABLE t1(x) } +} {1 {file is encrypted or is not a database}} +do_test 38.2 { + catch { db close } + forcedelete test.db +} {} + +do_test 39.1 { + sqlite3 db test.db + execsql { + PRAGMA auto_vacuum = 1; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES('xxx'); + INSERT INTO t1 VALUES('two'); + INSERT INTO t1 VALUES(randomblob(400)); + INSERT INTO t1 VALUES(randomblob(400)); + INSERT INTO t1 VALUES(randomblob(400)); + INSERT INTO t1 VALUES(randomblob(400)); + BEGIN; + UPDATE t1 SET x = 'one' WHERE rowid=1; + } + set ::stmt [sqlite3_prepare db "SELECT * FROM t1 ORDER BY rowid" -1 dummy] + sqlite3_step $::stmt + sqlite3_column_text $::stmt 0 +} {one} +do_test 39.2 { + execsql { CREATE TABLE t2(x) } + sqlite3_step $::stmt + sqlite3_column_text $::stmt 0 +} {two} +do_test 39.3 { + sqlite3_finalize $::stmt + execsql COMMIT +} {} + +do_execsql_test 39.4 { + PRAGMA auto_vacuum = 2; + CREATE TABLE t3(x); + CREATE TABLE t4(x); + + DROP TABLE t2; + DROP TABLE t3; + DROP TABLE t4; +} +do_test 39.5 { + db close + sqlite3 db test.db + execsql { + PRAGMA cache_size = 1; + PRAGMA incremental_vacuum; + PRAGMA integrity_check; + } +} {ok} + +do_test 40.1 { + reset_db + execsql { + PRAGMA auto_vacuum = 1; + CREATE TABLE t1(x PRIMARY KEY); + INSERT INTO t1 VALUES(randomblob(1200)); + PRAGMA page_count; + } +} {6} +do_test 40.2 { + execsql { + INSERT INTO t1 VALUES(randomblob(1200)); + INSERT INTO t1 VALUES(randomblob(1200)); + INSERT INTO t1 VALUES(randomblob(1200)); + } +} {} +do_test 40.3 { + db close + sqlite3 db test.db + execsql { + PRAGMA cache_size = 1; + CREATE TABLE t2(x); + PRAGMA integrity_check; + } +} {ok} + +do_test 41.1 { + reset_db + execsql { + CREATE TABLE t1(x PRIMARY KEY); + INSERT INTO t1 VALUES(randomblob(200)); + INSERT INTO t1 SELECT randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200) FROM t1; + } +} {} +do_test 41.2 { + testvfs tv -default 1 + tv sectorsize 16384; + tv devchar [list] + db close + sqlite3 db test.db + execsql { + PRAGMA cache_size = 1; + DELETE FROM t1 WHERE rowid%4; + PRAGMA integrity_check; + } +} {ok} + + finish_test diff --git a/test/pagerfault.test b/test/pagerfault.test index 9ea1aa6328..440909d1a1 100644 --- a/test/pagerfault.test +++ b/test/pagerfault.test @@ -1246,6 +1246,7 @@ do_faultsim_test pagerfault-27 -faults ioerr-persistent -prep { faultsim_integrity_check } + #------------------------------------------------------------------------- # do_test pagerfault-28-pre { @@ -1279,7 +1280,7 @@ do_test pagerfault-28-pre { } {1} faultsim_save_and_close -do_faultsim_test pagerfault-28 -faults oom* -prep { +do_faultsim_test pagerfault-28a -faults oom* -prep { faultsim_restore_and_reopen execsql { PRAGMA mmap_limit=0 } @@ -1303,5 +1304,185 @@ do_faultsim_test pagerfault-28 -faults oom* -prep { faultsim_integrity_check } +faultsim_restore_and_reopen +sqlite3 db2 test.db +db2 eval {SELECT count(*) FROM t2} +db close + +do_faultsim_test pagerfault-28b -faults oom* -prep { + sqlite3 db test.db +} -body { + execsql { SELECT count(*) FROM t2 } +} -test { + faultsim_test_result {0 2048} + db close +} + +db2 close + +#------------------------------------------------------------------------- +# Try this: +# +# 1) Put the pager in ERROR state (error during rollback) +# +# 2) Next time the connection is used inject errors into all xWrite() and +# xUnlock() calls. This causes the hot-journal rollback to fail and +# the pager to declare its locking state UNKNOWN. +# +# 3) Same again. +# +# 4a) Stop injecting errors. Allow the rollback to succeed. Check that +# the database is Ok. Or, +# +# 4b) Close and reopen the db. Check that the db is Ok. +# +proc custom_injectinstall {} { + testvfs custom -default true + custom filter {xWrite xUnlock} +} +proc custom_injectuninstall {} { + catch {db close} + catch {db2 close} + custom delete +} +proc custom_injectstart {iFail} { + custom ioerr $iFail 1 +} +proc custom_injectstop {} { + custom ioerr +} +set ::FAULTSIM(custom) [list \ + -injectinstall custom_injectinstall \ + -injectstart custom_injectstart \ + -injectstop custom_injectstop \ + -injecterrlist {{1 {disk I/O error}}} \ + -injectuninstall custom_injectuninstall \ +] + +do_test pagerfault-29-pre { + faultsim_delete_and_reopen + db func a_string a_string + execsql { + PRAGMA page_size = 1024; + PRAGMA cache_size = 5; + + BEGIN; + CREATE TABLE t2(a UNIQUE, b UNIQUE); + INSERT INTO t2 VALUES( a_string(800), a_string(800) ); + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + COMMIT; + } + expr {[file size test.db] >= 50*1024} +} {1} +faultsim_save_and_close +foreach {tn tt} { + 29 { catchsql ROLLBACK } + 30 { db close ; sqlite3 db test.db } +} { + do_faultsim_test pagerfault-$tn -faults custom -prep { + faultsim_restore_and_reopen + db func a_string a_string + execsql { + PRAGMA cache_size = 5; + BEGIN; + UPDATE t2 SET a = a_string(799); + } + } -body { + catchsql ROLLBACK + catchsql ROLLBACK + catchsql ROLLBACK + } -test { + eval $::tt + if {"ok" != [db one {PRAGMA integrity_check}]} { + error "integrity check failed" + } + } +} + +do_test pagerfault-31-pre { + sqlite3_shutdown + sqlite3_config_uri 1 +} {SQLITE_OK} +do_faultsim_test pagerfault-31 -faults oom* -body { + sqlite3 db {file:one?mode=memory&cache=shared} + db eval { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(1); + SELECT * FROM t1; + } +} -test { + faultsim_test_result {0 1} {1 {}} + catch { db close } +} +sqlite3_shutdown +sqlite3_config_uri 0 + +do_test pagerfault-32-pre { + reset_db + execsql { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES('one'); + } +} {} +faultsim_save_and_close + +do_faultsim_test pagerfault-32 -prep { + faultsim_restore_and_reopen + db eval { SELECT * FROM t1; } +} -body { + execsql { SELECT * FROM t1; } +} -test { + faultsim_test_result {0 one} +} +sqlite3_shutdown +sqlite3_config_uri 0 + +do_faultsim_test pagerfault-33a -prep { + sqlite3 db :memory: + execsql { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + } +} -body { + execsql { VACUUM } +} -test { + faultsim_test_result {0 {}} +} +do_faultsim_test pagerfault-33b -prep { + sqlite3 db "" + execsql { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + } +} -body { + execsql { VACUUM } +} -test { + faultsim_test_result {0 {}} +} + +do_test pagerfault-34-pre { + reset_db + execsql { + CREATE TABLE t1(x PRIMARY KEY); + } +} {} +faultsim_save_and_close +do_faultsim_test pagerfault-34 -prep { + faultsim_restore_and_reopen + execsql { + BEGIN; + INSERT INTO t1 VALUES( randomblob(4000) ); + DELETE FROM t1; + } +} -body { + execsql COMMIT +} -test { + faultsim_test_result {0 {}} +} + + finish_test diff --git a/test/wal.test b/test/wal.test index f833b6ab1a..0539192d25 100644 --- a/test/wal.test +++ b/test/wal.test @@ -1569,26 +1569,6 @@ ifcapable autovacuum { } [wal_file_size 1 1024] } -reset_db -do_test 25 { - sqlite3 db test.db - - execsql { - CREATE TABLE t1(x, y); - PRAGMA journal_mode = WAL; - INSERT INTO t1 VALUES(1, 2); - } - - execsql { - BEGIN; - CREATE TABLE t2(a, b); - } - - hexio_write test.db-shm [expr 16*1024] [string repeat 0055 8192] - catchsql ROLLBACK -} {0 {}} - - db close sqlite3_shutdown test_sqlite3_log diff --git a/test/walfault.test b/test/walfault.test index 6f9aeddea2..4a9d98afda 100644 --- a/test/walfault.test +++ b/test/walfault.test @@ -548,6 +548,44 @@ do_faultsim_test walfault-14 -prep { set nRow [db eval {SELECT count(*) FROM abc}] if {!(($nRow==2 && $testrc) || $nRow==3)} { error "Bad db content" } } -finish_test + +#------------------------------------------------------------------------- +# Test fault-handling when switching out of exclusive-locking mode. +# +do_test walfault-14-pre { + faultsim_delete_and_reopen + execsql { + PRAGMA auto_vacuum = 0; + PRAGMA journal_mode = WAL; + BEGIN; + CREATE TABLE abc(a PRIMARY KEY); + INSERT INTO abc VALUES(randomblob(1500)); + INSERT INTO abc VALUES(randomblob(1500)); + COMMIT; + } + faultsim_save_and_close +} {} +do_faultsim_test walfault-14 -prep { + faultsim_restore_and_reopen + breakpoint + execsql { + SELECT count(*) FROM abc; + PRAGMA locking_mode = exclusive; + BEGIN; + INSERT INTO abc VALUES(randomblob(1500)); + COMMIT; + } +} -body { + db eval { + PRAGMA locking_mode = normal; + BEGIN; + INSERT INTO abc VALUES(randomblob(1500)); + COMMIT; + } +} -test { + faultsim_integrity_check + set nRow [db eval {SELECT count(*) FROM abc}] + if {$nRow!=3 && $nRow!=4} { error "Bad db content" } +} finish_test From 27338e6ed7eb0436896329dc6ca29959038e27a2 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 6 Apr 2013 00:19:37 +0000 Subject: [PATCH 096/204] Add a prototype TH3-script-style test harness that starts multiple processes operating on the same database file at the same time. FossilOrigin-Name: c318fafe686120d7fb8e487eb3bb4942d497665c --- manifest | 16 +- manifest.uuid | 2 +- mptest/mptest.c | 1031 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1042 insertions(+), 7 deletions(-) create mode 100644 mptest/mptest.c diff --git a/manifest b/manifest index 2b89f591ab..0c4454579c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\scode\scoverage\sof\s"permutations.test\scoverage-pager". -D 2013-04-05T20:40:43.227 +C Add\sa\sprototype\sTH3-script-style\stest\sharness\sthat\sstarts\smultiple\sprocesses\noperating\son\sthe\ssame\sdatabase\sfile\sat\sthe\ssame\stime. +D 2013-04-06T00:19:37.887 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,6 +110,7 @@ F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 +F mptest/mptest.c 04110c35f415218e54e030b2e37215ad4adf01f3 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc @@ -1044,7 +1045,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a107f75d93b8acd3403d8473a75137378041a833 -R 6d11c98ab18b87c4b8798ffe987d9d38 -U dan -Z 50aa2590d101ae076870fa94f81e976a +P b095e2cdb61ca8487255687f58fb1024d40f3986 +R c143d7671ed86699bfdf0a0519618d31 +T *branch * mptest +T *sym-mptest * +T -sym-trunk * +U drh +Z f0aabffa9579562b9fff6a9c66b5a272 diff --git a/manifest.uuid b/manifest.uuid index 8dbe787aae..fcf249224b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b095e2cdb61ca8487255687f58fb1024d40f3986 \ No newline at end of file +c318fafe686120d7fb8e487eb3bb4942d497665c \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c new file mode 100644 index 0000000000..01660af690 --- /dev/null +++ b/mptest/mptest.c @@ -0,0 +1,1031 @@ +/* +** 2013-04-05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a program used for testing SQLite, and specifically for testing +** the ability of independent processes to access the same SQLite database +** concurrently. +** +** Compile this program as follows: +** +** gcc -g -c -Wall sqlite3.c $(OPTS) +** gcc -g -o mptest mptest.c sqlite3.o $(LIBS) +** +** Recommended options: +** +** -DHAVE_USLEEP +** -DSQLITE_NO_SYNC +** -DSQLITE_THREADSAFE=0 +** -DSQLITE_OMIT_LOAD_EXTENSION +** +** Run like this: +** +** ./mptest $database $script +** +** where $database is the database to use for testing and $script is a +** test script. +*/ +#include "sqlite3.h" +#include +#include +#include +#include +#include +#include + +/* Global data +*/ +static struct Global { + char *argv0; /* Name of the executable */ + const char *zVfs; /* Name of VFS to use. Often NULL meaning "default" */ + char *zDbFile; /* Name of the database */ + sqlite3 *db; /* Open connection to database */ + char *zErrLog; /* Filename for error log */ + FILE *pErrLog; /* Where to write errors */ + char *zLog; /* Name of output log file */ + FILE *pLog; /* Where to write log messages */ + char zName[12]; /* Symbolic name of this process */ + int taskId; /* Task ID. 0 means supervisor. */ + int iTrace; /* Tracing level */ + int bSqlTrace; /* True to trace SQL commands */ + int nError; /* Number of errors */ +} g; + +/* +** Print a message adding zPrefix[] to the beginning of every line. +*/ +static void printWithPrefix(FILE *pOut, const char *zPrefix, const char *zMsg){ + while( zMsg && zMsg[0] ){ + int i; + for(i=0; zMsg[i] && zMsg[i]!='\n' && zMsg[i]!='\r'; i++){} + fprintf(pOut, "%s%.*s\n", zPrefix, i, zMsg); + zMsg += i; + while( zMsg[0]=='\n' || zMsg[0]=='\r' ) zMsg++; + } +} + +/* +** Compare two pointers to strings, where the pointers might be NULL. +*/ +static int safe_strcmp(const char *a, const char *b){ + if( a==b ) return 0; + if( a==0 ) return -1; + if( b==0 ) return 1; + return strcmp(a,b); +} + +/* +** Return TRUE if string z[] matches glob pattern zGlob[]. +** Return FALSE if the pattern does not match. +** +** Globbing rules: +** +** '*' Matches any sequence of zero or more characters. +** +** '?' Matches exactly one character. +** +** [...] Matches one character from the enclosed list of +** characters. +** +** [^...] Matches one character not in the enclosed list. +** +** '#' Matches any sequence of one or more digits with an +** optional + or - sign in front +*/ +int strglob(const char *zGlob, const char *z){ + int c, c2; + int invert; + int seen; + + while( (c = (*(zGlob++)))!=0 ){ + if( c=='*' ){ + while( (c=(*(zGlob++))) == '*' || c=='?' ){ + if( c=='?' && (*(z++))==0 ) return 0; + } + if( c==0 ){ + return 1; + }else if( c=='[' ){ + while( *z && strglob(zGlob-1,z) ){ + z++; + } + return (*z)!=0; + } + while( (c2 = (*(z++)))!=0 ){ + while( c2!=c ){ + c2 = *(z++); + if( c2==0 ) return 0; + } + if( strglob(zGlob,z) ) return 1; + } + return 0; + }else if( c=='?' ){ + if( (*(z++))==0 ) return 0; + }else if( c=='[' ){ + int prior_c = 0; + seen = 0; + invert = 0; + c = *(z++); + if( c==0 ) return 0; + c2 = *(zGlob++); + if( c2=='^' ){ + invert = 1; + c2 = *(zGlob++); + } + if( c2==']' ){ + if( c==']' ) seen = 1; + c2 = *(zGlob++); + } + while( c2 && c2!=']' ){ + if( c2=='-' && zGlob[0]!=']' && zGlob[0]!=0 && prior_c>0 ){ + c2 = *(zGlob++); + if( c>=prior_c && c<=c2 ) seen = 1; + prior_c = 0; + }else{ + if( c==c2 ){ + seen = 1; + } + prior_c = c2; + } + c2 = *(zGlob++); + } + if( c2==0 || (seen ^ invert)==0 ) return 0; + }else if( c=='#' ){ + if( (z[0]=='-' || z[0]=='+') && isdigit(z[1]) ) z++; + if( !isdigit(z[0]) ) return 0; + z++; + while( isdigit(z[0]) ){ z++; } + }else{ + if( c!=(*(z++)) ) return 0; + } + } + return *z==0; +} + +/* +** Close output stream pOut if it is not stdout or stderr +*/ +static void maybeClose(FILE *pOut){ + if( pOut!=stdout && pOut!=stderr ) fclose(pOut); +} + +/* +** Print an error message +*/ +static void errorMessage(const char *zFormat, ...){ + va_list ap; + char *zMsg; + char zPrefix[30]; + va_start(ap, zFormat); + zMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + sqlite3_snprintf(sizeof(zPrefix), zPrefix, "%s:ERROR: ", g.zName); + if( g.pLog ){ + printWithPrefix(g.pLog, zPrefix, zMsg); + fflush(g.pLog); + } + if( g.pErrLog && safe_strcmp(g.zErrLog,g.zLog) ){ + printWithPrefix(g.pErrLog, zPrefix, zMsg); + fflush(g.pErrLog); + } + sqlite3_free(zMsg); + g.nError++; +} + +/* Forward declaration */ +static int trySql(const char*, ...); + +/* +** Print an error message and then quit. +*/ +static void fatalError(const char *zFormat, ...){ + va_list ap; + char *zMsg; + char zPrefix[30]; + va_start(ap, zFormat); + zMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + sqlite3_snprintf(sizeof(zPrefix), zPrefix, "%s:FATAL: ", g.zName); + if( g.pLog ){ + printWithPrefix(g.pLog, zPrefix, zMsg); + fflush(g.pLog); + maybeClose(g.pLog); + } + if( g.pErrLog && safe_strcmp(g.zErrLog,g.zLog) ){ + printWithPrefix(g.pErrLog, zPrefix, zMsg); + fflush(g.pErrLog); + maybeClose(g.pErrLog); + } + sqlite3_free(zMsg); + if( g.db ){ + int nTry = 0; + while( trySql("CREATE TABLE halt(x);")==SQLITE_BUSY && (nTry++)<100 ){ + sqlite3_sleep(10); + } + } + sqlite3_close(g.db); + exit(1); +} + + +/* +** Print a log message +*/ +static void logMessage(const char *zFormat, ...){ + va_list ap; + char *zMsg; + char zPrefix[30]; + va_start(ap, zFormat); + zMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + sqlite3_snprintf(sizeof(zPrefix), zPrefix, "%s: ", g.zName); + if( g.pLog ){ + printWithPrefix(g.pLog, zPrefix, zMsg); + fflush(g.pLog); + } + sqlite3_free(zMsg); +} + +/* +** Return the length of a string omitting trailing whitespace +*/ +static int clipLength(const char *z){ + int n = (int)strlen(z); + while( n>0 && isspace(z[n-1]) ){ n--; } + return n; +} + +/* +** SQL Trace callback +*/ +static void sqlTraceCallback(void *NotUsed1, const char *zSql){ + logMessage("[%.*s]", clipLength(zSql), zSql); +} + +/* +** Prepare an SQL statement. Issue a fatal error if unable. +*/ +static sqlite3_stmt *prepareSql(const char *zFormat, ...){ + va_list ap; + char *zSql; + int rc; + sqlite3_stmt *pStmt = 0; + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + rc = sqlite3_prepare_v2(g.db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + sqlite3_finalize(pStmt); + fatalError("%s\n%s\n", sqlite3_errmsg(g.db), zSql); + } + sqlite3_free(zSql); + return pStmt; +} + +/* +** Run arbitrary SQL. Issue a fatal error on failure. +*/ +static void runSql(const char *zFormat, ...){ + va_list ap; + char *zSql; + int rc; + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + rc = sqlite3_exec(g.db, zSql, 0, 0, 0); + if( rc!=SQLITE_OK ){ + fatalError("%s\n%s\n", sqlite3_errmsg(g.db), zSql); + } + sqlite3_free(zSql); +} + +/* +** Try to run arbitrary SQL. Return success code. +*/ +static int trySql(const char *zFormat, ...){ + va_list ap; + char *zSql; + int rc; + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + rc = sqlite3_exec(g.db, zSql, 0, 0, 0); + sqlite3_free(zSql); + return rc; +} + +/* Structure for holding an arbitrary length string +*/ +typedef struct String String; +struct String { + char *z; /* the string */ + int n; /* Slots of z[] used */ + int nAlloc; /* Slots of z[] allocated */ +}; + +/* Free a string */ +static void stringFree(String *p){ + if( p->z ) sqlite3_free(p->z); + memset(p, 0, sizeof(*p)); +} + +/* Append n bytes of text to a string. If n<0 append the entire string. */ +static void stringAppend(String *p, const char *z, int n){ + if( n<0 ) n = (int)strlen(z); + if( p->n+n>=p->nAlloc ){ + int nAlloc = p->nAlloc*2 + n + 100; + char *z = sqlite3_realloc(p->z, nAlloc); + if( z==0 ) fatalError("out of memory"); + p->z = z; + p->nAlloc = nAlloc; + } + memcpy(p->z+p->n, z, n); + p->n += n; + p->z[p->n] = 0; +} + +/* Reset a string to an empty string */ +static void stringReset(String *p){ + if( p->z==0 ) stringAppend(p, " ", 1); + p->n = 0; + p->z[0] = 0; +} + +/* Append a new token onto the end of the string */ +static void stringAppendTerm(String *p, const char *z){ + int i; + if( p->n ) stringAppend(p, " ", 1); + if( z==0 ){ + stringAppend(p, "nil", 3); + return; + } + for(i=0; z[i] && !isspace(z[i]); i++){} + if( i>0 && z[i]==0 ){ + stringAppend(p, z, i); + return; + } + stringAppend(p, "'", 1); + while( z[0] ){ + for(i=0; z[i] && z[i]!='\''; i++){} + if( z[i] ){ + stringAppend(p, z, i+1); + stringAppend(p, "'", 1); + z += i+1; + }else{ + stringAppend(p, z, i); + break; + } + } + stringAppend(p, "'", 1); +} + +/* +** Callback function for evalSql() +*/ +static int evalCallback(void *pCData, int argc, char **argv, char **azCol){ + String *p = (String*)pCData; + int i; + for(i=0; i0 ){ + pStmt = prepareSql( + "SELECT 1 FROM task WHERE client=%d AND endtime IS NULL", + iClient); + }else{ + pStmt = prepareSql( + "SELECT 1 FROM task WHERE client=%d AND endtime IS NULL", + iClient); + } + while( ((rc = sqlite3_step(pStmt))==SQLITE_BUSY || rc==SQLITE_ROW) + && iTimeout>0 + ){ + sqlite3_reset(pStmt); + sqlite3_sleep(50); + iTimeout -= 50; + } + sqlite3_finalize(pStmt); + if( rc!=SQLITE_DONE ){ + if( zErrPrefix==0 ) zErrPrefix = ""; + if( iClient>0 ){ + errorMessage("%stimeout waiting for client %d", zErrPrefix, iClient); + }else{ + errorMessage("%stimeout waiting for all clients", zErrPrefix); + } + } +} + +/* Maximum number of arguments to a --command */ +#define MX_ARG 5 + +/* +** Run a script. +*/ +static void runScript( + int iClient, /* The client number, or 0 for the master */ + int taskId, /* The task ID for clients. 0 for master */ + char *zScript, /* Text of the script */ + char *zFilename /* File from which script was read. */ +){ + int lineno = 1; + int prevLine = 1; + int ii = 0; + int iBegin = 0; + int n, c, j; + int rc; + int len; + int nArg; + String sResult; + char zCmd[30]; + char zError[1000]; + char azArg[MX_ARG][100]; + unsigned char isRunning[100]; + + memset(isRunning, 0, sizeof(isRunning)); + memset(&sResult, 0, sizeof(sResult)); + stringReset(&sResult); + while( (c = zScript[ii])!=0 ){ + prevLine = lineno; + len = tokenLength(zScript+ii, &lineno); + if( isspace(c) || (c=='/' && zScript[ii+1]=='*') ){ + ii += len; + continue; + } + if( c!='-' || zScript[ii+1]!='-' || !isalpha(zScript[ii+2]) ){ + ii += len; + continue; + } + + /* Run any prior SQL before processing the new --command */ + if( ii>iBegin ){ + char *zSql = sqlite3_mprintf("%.*s", ii-iBegin, zScript+iBegin); + evalSql(&sResult, zSql); + sqlite3_free(zSql); + iBegin = ii + len; + } + + /* Parse the --command */ + if( g.iTrace>=2 ) logMessage("%.*s", len, zScript+ii); + n = extractToken(zScript+ii+2, len-2, zCmd, sizeof(zCmd)); + for(nArg=0; n=len-2 ) break; + n += extractToken(zScript+ii+2+n, len-2-n, + azArg[nArg], sizeof(azArg[nArg])); + } + for(j=nArg; j0 then exit without shutting down + ** SQLite. (In other words, simulate a crash.) + */ + if( strcmp(zCmd, "exit")==0 ){ + int rc = atoi(azArg[0]); + finishScript(taskId); + if( rc==0 ) sqlite3_close(g.db); + exit(rc); + }else + + /* + ** --result + ** + ** Reset accumulated results back to an empty string + */ + if( strcmp(zCmd, "reset")==0 ){ + stringReset(&sResult); + }else + + /* + ** --match ANSWER... + ** + ** Check to see if output matches ANSWER. Report an error if not. + */ + if( strcmp(zCmd, "match")==0 ){ + int jj; + char *zAns = zScript+ii; + for(jj=7; jj=sizeof(isRunning) ){ + errorMessage("line %d of %s: bad client number: %d", + prevLine, zFilename, iNewClient); + goto start_error; + } + if( isRunning[iNewClient] ){ + errorMessage("line %d of %s: client already running: %d", + prevLine, zFilename, iNewClient); + goto start_error; + } + zSys = sqlite3_mprintf( + "%s \"%s\" --client %d --trace %d %s&", + g.argv0, g.zDbFile, iNewClient, g.iTrace, + g.bSqlTrace ? "--sqltrace " : ""); + + system(zSys); + sqlite3_free(zSys); + isRunning[iNewClient] = 1; + start_error: {/* no-op */} + }else + + /* + ** --wait CLIENT TIMEOUT + ** + ** Wait until all tasks complete for the given client. If CLIENT is + ** "all" then wait for all clients to complete. Wait no longer than + ** TIMEOUT milliseconds (default 10,000) + */ + if( strcmp(zCmd, "wait")==0 ){ + int iTimeout = nArg>=2 ? atoi(azArg[1]) : 10000; + sqlite3_snprintf(sizeof(zError),zError,"line %d of %s\n", + prevLine, zFilename); + waitForClient(atoi(azArg[0]), iTimeout, zError); + }else + + /* + ** --task CLIENT + ** + ** --end + ** + ** Assign work to a client. + */ + if( strcmp(zCmd, "task")==0 ){ + int iTarget = atoi(azArg[0]); + int iEnd; + char *zTask; + sqlite3_stmt *pStmt; + iEnd = findEnd(zScript+ii+len, &lineno); + if( iTarget<0 || iTarget>=sizeof(isRunning) + || !isRunning[iTarget] ){ + errorMessage("line %d of %s: client %d is not running", + prevLine, zFilename, iTarget); + goto task_error; + } + zTask = sqlite3_mprintf("%.*s", iEnd, zScript+ii+len); + pStmt = prepareSql("INSERT INTO task(client,script)" + " VALUES(%d,'%q')", iTarget, zTask); + sqlite3_free(zTask); + while( (rc = sqlite3_step(pStmt))==SQLITE_BUSY ){ + sqlite3_reset(pStmt); + sqlite3_sleep(10); + } + sqlite3_finalize(pStmt); + task_error: + iEnd += tokenLength(zScript+ii+len+iEnd, &lineno); + len += iEnd; + iBegin = ii+len; + }else + + /* error */{ + errorMessage("line %d of %s: unknown command --%s", + prevLine, zFilename, zCmd); + } + ii += len; + } + if( iBegin= nArg ) break; + z = azArg[i]; + if( z[0]!='-' ) continue; + z++; + if( z[0]=='-' ){ + if( z[1]==0 ) break; + z++; + } + if( strcmp(z,zOption)==0 ){ + if( hasArg && i==nArg-1 ){ + fatalError("command-line option \"--%s\" requires an argument", z); + } + if( hasArg ){ + zReturn = azArg[i+1]; + }else{ + zReturn = azArg[i]; + } + j = i+1+(hasArg!=0); + while( j0 ){ + if( n>0 ) unrecognizedArguments(argv[0], n, argv+2); + if( g.iTrace ) logMessage("start-client"); + while(1){ + char zTaskName[50]; + rc = startScript(iClient, &zScript, &taskId); + if( rc==SQLITE_DONE ) break; + if( g.iTrace ) logMessage("begin task %d", taskId); + sqlite3_snprintf(sizeof(zTaskName), zTaskName, "client%02d-task-%d", + iClient, taskId); + runScript(iClient, taskId, zScript, zTaskName); + if( g.iTrace ) logMessage("end task %d", taskId); + finishScript(taskId); + sqlite3_sleep(10); + } + if( g.iTrace ) logMessage("end-client"); + }else{ + sqlite3_stmt *pStmt; + if( n==0 ){ + fatalError("missing script filename"); + } + if( n>1 ) unrecognizedArguments(argv[0], n, argv+2); + runSql( + "CREATE TABLE task(\n" + " id INTEGER PRIMARY KEY,\n" + " client INTEGER,\n" + " starttime DATE,\n" + " endtime DATE,\n" + " script TEXT\n" + ");" + "CREATE TABLE clienterror(cnt);\n" + "INSERT INTO clienterror VALUES(0);\n" + ); + zScript = readFile(argv[2]); + if( g.iTrace ) logMessage("begin script [%s]\n", argv[2]); + runScript(0, 0, zScript, argv[2]); + sqlite3_free(zScript); + if( g.iTrace ) logMessage("end script [%s]\n", argv[2]); + waitForClient(0, 2000, "during shutdown...\n"); + while( trySql("CREATE TABLE halt(x);")==SQLITE_BUSY ){ + sqlite3_sleep(10); + } + sqlite3_sleep(100); + pStmt = prepareSql("SELECT cnt FROM clienterror"); + while( (rc = sqlite3_step(pStmt))==SQLITE_BUSY ){ + sqlite3_sleep(10); + } + if( rc==SQLITE_ROW ){ + g.nError += sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + } + sqlite3_close(g.db); + maybeClose(g.pLog); + maybeClose(g.pErrLog); + if( iClient==0 ){ + if( g.nError ){ + printf("ERRORS: %d\n", g.nError); + }else if( g.iTrace ){ + printf("All OK\n"); + } + } + return g.nError>0; +} From abd6d84a11781229fb3f09a03c45516f1081a4dc Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 6 Apr 2013 11:03:09 +0000 Subject: [PATCH 097/204] Add tests to "permutations.test coverage-pager" to cover uncovered branches. FossilOrigin-Name: 07a0e4e9820a85bf1690214fef3577866c763f92 --- manifest | 16 ++++---- manifest.uuid | 2 +- test/pager1.test | 91 ++++++++++++++++++++++++++++++++++++++++++-- test/pager2.test | 19 ++++++++- test/pagerfault.test | 61 +++++++++++++++++++++++++++++ 5 files changed, 175 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 2b89f591ab..7e72d182cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\scode\scoverage\sof\s"permutations.test\scoverage-pager". -D 2013-04-05T20:40:43.227 +C Add\stests\sto\s"permutations.test\scoverage-pager"\sto\scover\suncovered\sbranches. +D 2013-04-06T11:03:09.144 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -651,10 +651,10 @@ F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 -F test/pager1.test 622bfaea80604a96010f35256a4fc22fce584e95 -F test/pager2.test 745b911dde3d1f24ae0870bd433dfa83d7c658c1 +F test/pager1.test 134144862f6a98f862dc2edda1b10ff499141bfa +F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f -F test/pagerfault.test 636c01bdbdb2bb2baf68532d6c4ef4c633022743 +F test/pagerfault.test bba948be0564254b823ec6ca1fd95fbbfb0839a3 F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 F test/pageropt.test 32cb5a6ed7ccddf8e8c842cb44240bd9340223ce @@ -1044,7 +1044,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a107f75d93b8acd3403d8473a75137378041a833 -R 6d11c98ab18b87c4b8798ffe987d9d38 +P b095e2cdb61ca8487255687f58fb1024d40f3986 +R bbe90127c799120f0deb57d78c345328 U dan -Z 50aa2590d101ae076870fa94f81e976a +Z 0a43fd582a2aa921b5f65df032585318 diff --git a/manifest.uuid b/manifest.uuid index 8dbe787aae..10faadaead 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b095e2cdb61ca8487255687f58fb1024d40f3986 \ No newline at end of file +07a0e4e9820a85bf1690214fef3577866c763f92 \ No newline at end of file diff --git a/test/pager1.test b/test/pager1.test index 22c84d55d0..ab0ec67526 100644 --- a/test/pager1.test +++ b/test/pager1.test @@ -2245,7 +2245,6 @@ do_test pager1-25-1 { } db close } {} -breakpoint do_test pager1-25-2 { faultsim_delete_and_reopen execsql { @@ -2543,7 +2542,6 @@ foreach {tn pragma strsize} { CREATE TABLE t1(a, b); INSERT INTO t1 VALUES(1, 2); } - breakpoint do_execsql_test 34.$tn.2 { BEGIN; INSERT INTO t1 VALUES(2, a_string($strsize)); @@ -2605,14 +2603,21 @@ foreach {tn uri} { do_test 37.$tn { catch { db close } sqlite3_shutdown - sqlite3_config_uri 1 - sqlite3 db $uri + sqlite3_config_uri 1 + sqlite3 db $uri + db eval { CREATE TABLE t1(x); INSERT INTO t1 VALUES(1); SELECT * FROM t1; } } {1} + + do_execsql_test 37.$tn.2 { + VACUUM; + SELECT * FROM t1; + } {1} + db close sqlite3_shutdown sqlite3_config_uri 0 @@ -2730,6 +2735,84 @@ do_test 41.2 { PRAGMA integrity_check; } } {ok} +db close +tv delete +set pending_prev [sqlite3_test_control_pending_byte 0x1000000] +do_test 42.1 { + reset_db + execsql { + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(randomblob(200), randomblob(200)); + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + } + db close + sqlite3_test_control_pending_byte 0x0010000 + sqlite3 db test.db + db eval { PRAGMA mmap_limit = 0 } + catchsql { SELECT sum(length(y)) FROM t1 } +} {1 {database disk image is malformed}} +do_test 42.2 { + reset_db + execsql { + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(randomblob(200), randomblob(200)); + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + } + db close + + testvfs tv -default 1 + tv sectorsize 16384; + tv devchar [list] + sqlite3 db test.db -vfs tv + execsql { UPDATE t1 SET x = randomblob(200) } +} {} +db close +tv delete +sqlite3_test_control_pending_byte $pending_prev + +do_test 43.1 { + reset_db + execsql { + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(1, 2); + CREATE TABLE t2(x, y); + INSERT INTO t2 VALUES(1, 2); + CREATE TABLE t3(x, y); + INSERT INTO t3 VALUES(1, 2); + } + db close + sqlite3 db test.db + + db eval { PRAGMA mmap_limit = 0 } + db eval { SELECT * FROM t1 } + sqlite3_db_status db CACHE_MISS 0 +} {0 2 0} + +do_test 43.2 { + db eval { SELECT * FROM t2 } + sqlite3_db_status db CACHE_MISS 1 +} {0 3 0} + +do_test 43.3 { + db eval { SELECT * FROM t3 } + sqlite3_db_status db CACHE_MISS 0 +} {0 1 0} finish_test + diff --git a/test/pager2.test b/test/pager2.test index fa5f7b9611..0e2b33b833 100644 --- a/test/pager2.test +++ b/test/pager2.test @@ -118,7 +118,6 @@ tv delete #------------------------------------------------------------------------- -# # pager2-2.1: Test a ROLLBACK with journal_mode=off. # pager2-2.2: Test shrinking the database (auto-vacuum) with # journal_mode=off @@ -148,4 +147,22 @@ do_test pager2-2.2 { file size test.db } {3072} +#------------------------------------------------------------------------- +# Test that shared in-memory databases seem to work. +# +db close +do_test pager2-3.1 { + forcedelete test.db + sqlite3_shutdown + sqlite3_config_uri 1 + + sqlite3 db1 {file:test.db?mode=memory&cache=shared} + sqlite3 db2 {file:test.db?mode=memory&cache=shared} + sqlite3 db3 test.db + + db1 eval { CREATE TABLE t1(a, b) } + db2 eval { INSERT INTO t1 VALUES(1, 2) } + list [catch { db3 eval { INSERT INTO t1 VALUES(3, 4) } } msg] $msg +} {1 {no such table: t1}} + finish_test diff --git a/test/pagerfault.test b/test/pagerfault.test index 440909d1a1..9b005c7a03 100644 --- a/test/pagerfault.test +++ b/test/pagerfault.test @@ -1483,6 +1483,67 @@ do_faultsim_test pagerfault-34 -prep { faultsim_test_result {0 {}} } +do_test pagerfault-35-pre { + faultsim_delete_and_reopen + execsql { + CREATE TABLE t1(x PRIMARY KEY, y); + INSERT INTO t1 VALUES(randomblob(200), randomblob(200)); + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + } + faultsim_save_and_close +} {} +testvfs tv -default 1 +tv sectorsize 8192; +tv devchar [list] +do_faultsim_test pagerfault-35 -prep { + faultsim_restore_and_reopen +} -body { + execsql { UPDATE t1 SET x=randomblob(200) } +} -test { + faultsim_test_result {0 {}} +} +catch {db close} +tv delete + +sqlite3_shutdown +sqlite3_config_uri 1 +do_test pagerfault-36-pre { + faultsim_delete_and_reopen + execsql { + CREATE TABLE t1(x PRIMARY KEY, y); + INSERT INTO t1 VALUES(randomblob(200), randomblob(200)); + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + } + faultsim_save_and_close +} {} +do_faultsim_test pagerfault-36 -prep { + faultsim_restore + sqlite3 db file:test.db?cache=shared + sqlite3 db2 file:test.db?cache=shared + db2 eval { + BEGIN; + SELECT count(*) FROM sqlite_master; + } + db eval { + PRAGMA cache_size = 1; + BEGIN; + UPDATE t1 SET x = randomblob(200); + } +} -body { + execsql ROLLBACK db +} -test { + catch { db eval {UPDATE t1 SET x = randomblob(200)} } + faultsim_test_result {0 {}} + catch { db close } + catch { db2 close } +} + +sqlite3_shutdown +sqlite3_config_uri 0 finish_test From 3f5bc38037eda131bd0505cd8961e4955d5ce0cf Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 6 Apr 2013 13:09:11 +0000 Subject: [PATCH 098/204] Many improvements to the mptest program. Added a simple test script. FossilOrigin-Name: 07b0401a9b61b1664fc6dcddac3b5969fc0f481a --- manifest | 17 ++-- manifest.uuid | 2 +- mptest/config01.test | 19 ++++ mptest/mptest.c | 194 ++++++++++++++++++++++++--------------- mptest/multiwrite01.test | 141 ++++++++++++++++++++++++++++ 5 files changed, 289 insertions(+), 84 deletions(-) create mode 100644 mptest/config01.test create mode 100644 mptest/multiwrite01.test diff --git a/manifest b/manifest index 0c4454579c..53c2472e9b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sprototype\sTH3-script-style\stest\sharness\sthat\sstarts\smultiple\sprocesses\noperating\son\sthe\ssame\sdatabase\sfile\sat\sthe\ssame\stime. -D 2013-04-06T00:19:37.887 +C Many\simprovements\sto\sthe\smptest\sprogram.\s\sAdded\sa\ssimple\stest\sscript. +D 2013-04-06T13:09:11.141 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,7 +110,9 @@ F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 -F mptest/mptest.c 04110c35f415218e54e030b2e37215ad4adf01f3 +F mptest/config01.test 058a9bc2b0db710d36003ab06dc1618566f27b52 +F mptest/mptest.c d73b294c354719f33086d8d3ebc273553b26073c +F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc @@ -1045,10 +1047,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b095e2cdb61ca8487255687f58fb1024d40f3986 -R c143d7671ed86699bfdf0a0519618d31 -T *branch * mptest -T *sym-mptest * -T -sym-trunk * +P c318fafe686120d7fb8e487eb3bb4942d497665c +R e5c361902695f1e216fb1ed024ee2204 U drh -Z f0aabffa9579562b9fff6a9c66b5a272 +Z 1a794dfe7c097a8f7d97d3bcdba2a731 diff --git a/manifest.uuid b/manifest.uuid index fcf249224b..a5a84885b3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c318fafe686120d7fb8e487eb3bb4942d497665c \ No newline at end of file +07b0401a9b61b1664fc6dcddac3b5969fc0f481a \ No newline at end of file diff --git a/mptest/config01.test b/mptest/config01.test new file mode 100644 index 0000000000..be6b570678 --- /dev/null +++ b/mptest/config01.test @@ -0,0 +1,19 @@ +/* +** Configure five tasks in different ways, then run tests. +*/ +PRAGMA page_size=8192; +--task 1 + PRAGMA journal_mode=PERSIST; + PRAGMA mmap_limit=0; +--end +--task 2 + PRAGMA journal_mode=TRUNCATE; + PRAGMA mmap_limit=28672; +--end +--task 3 + PRAGMA journal_mode=MEMORY; +--end +--task 4 + PRAGMA journal_mode=OFF; +--end +--source multiwrite01.test diff --git a/mptest/mptest.c b/mptest/mptest.c index 01660af690..e07e974de3 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -57,8 +57,13 @@ static struct Global { int iTrace; /* Tracing level */ int bSqlTrace; /* True to trace SQL commands */ int nError; /* Number of errors */ + int nTest; /* Number of --match operators */ + int iTimeout; /* Milliseconds until a busy timeout */ } g; +/* Default timeout */ +#define DEFAULT_TIMEOUT 10000 + /* ** Print a message adding zPrefix[] to the beginning of every line. */ @@ -226,7 +231,9 @@ static void fatalError(const char *zFormat, ...){ sqlite3_free(zMsg); if( g.db ){ int nTry = 0; - while( trySql("CREATE TABLE halt(x);")==SQLITE_BUSY && (nTry++)<100 ){ + g.iTimeout = 0; + while( trySql("UPDATE client SET wantHalt=1;")==SQLITE_BUSY + && (nTry++)<100 ){ sqlite3_sleep(10); } } @@ -262,6 +269,18 @@ static int clipLength(const char *z){ return n; } +/* +** Busy handler with a g.iTimeout-millisecond timeout +*/ +static int busyHandler(void *pCD, int count){ + if( count*10>g.iTimeout ){ + if( g.iTimeout>0 ) errorMessage("timeout after %dms", g.iTimeout); + return 0; + } + sqlite3_sleep(10); + return 1; +} + /* ** SQL Trace callback */ @@ -408,6 +427,7 @@ static int evalSql(String *p, const char *zFormat, ...){ va_start(ap, zFormat); zSql = sqlite3_vmprintf(zFormat, ap); va_end(ap); + assert( g.iTimeout>0 ); rc = sqlite3_exec(g.db, zSql, evalCallback, p, &zErrMsg); sqlite3_free(zSql); if( rc ){ @@ -435,21 +455,34 @@ static int startScript( sqlite3_stmt *pStmt = 0; int taskId; int rc; + int totalTime = 0; *pzScript = 0; + g.iTimeout = 0; while(1){ - if( trySql("SELECT * FROM halt")==SQLITE_OK ) return SQLITE_DONE; rc = trySql("BEGIN IMMEDIATE"); if( rc==SQLITE_BUSY ){ sqlite3_sleep(10); + totalTime += 10; continue; } if( rc!=SQLITE_OK ){ fatalError("%s\nBEGIN IMMEDIATE", sqlite3_errmsg(g.db)); } - if( g.nError ){ - runSql("UPDATE clienterror SET cnt=cnt+%d", g.nError); + if( g.nError || g.nTest ){ + runSql("UPDATE counters SET nError=nError+%d, nTest=nTest+%d", + g.nError, g.nTest); g.nError = 0; + g.nTest = 0; + } + pStmt = prepareSql("SELECT 1 FROM client WHERE id=%d AND wantHalt",iClient); + rc = sqlite3_step(pStmt); + sqlite3_finalize(pStmt); + if( rc==SQLITE_ROW ){ + runSql("DELETE FROM client WHERE id=%d", iClient); + runSql("COMMIT"); + g.iTimeout = DEFAULT_TIMEOUT; + return SQLITE_DONE; } pStmt = prepareSql( "SELECT script, id FROM task" @@ -466,37 +499,58 @@ static int startScript( " SET starttime=strftime('%%Y-%%m-%%d %%H:%%M:%%f','now')" " WHERE id=%d;", taskId); runSql("COMMIT;"); + g.iTimeout = DEFAULT_TIMEOUT; return SQLITE_OK; } sqlite3_finalize(pStmt); if( rc==SQLITE_DONE ){ + if( totalTime>30000 ){ + errorMessage("Waited over 30 seconds with no work. Giving up."); + runSql("DELETE FROM client WHERE id=%d; COMMIT;", iClient); + sqlite3_close(g.db); + exit(1); + } runSql("COMMIT;"); sqlite3_sleep(100); + totalTime += 100; continue; } fatalError("%s", sqlite3_errmsg(g.db)); } + g.iTimeout = DEFAULT_TIMEOUT; } /* -** Mark a script as having finished. +** Mark a script as having finished. Remove the CLIENT table entry +** if bShutdown is true. */ -static int finishScript(int taskId){ - sqlite3_stmt *pStmt; - int rc; - pStmt = prepareSql( - "UPDATE task" - " SET endtime=strftime('%%Y-%%m-%%d %%H:%%M:%%f','now')" - " WHERE id=%d;", taskId); - do{ - rc = sqlite3_step(pStmt); - sqlite3_reset(pStmt); - }while( rc==SQLITE_BUSY ); - sqlite3_finalize(pStmt); - if( rc!=SQLITE_DONE ){ - fatalError("%s\n", sqlite3_errmsg(g.db)); +static int finishScript(int iClient, int taskId, int bShutdown){ + runSql("UPDATE task" + " SET endtime=strftime('%%Y-%%m-%%d %%H:%%M:%%f','now')" + " WHERE id=%d;", taskId); + if( bShutdown ){ + runSql("DELETE FROM client WHERE id=%d", iClient); + } + return SQLITE_OK; +} + +/* +** Start up a client process for iClient, if it is not already +** running. If the client is already running, then this routine +** is a no-op. +*/ +static void startClient(int iClient){ + runSql("INSERT OR IGNORE INTO client VALUES(%d,0)", iClient); + if( sqlite3_changes(g.db) ){ + char *zSys; + zSys = sqlite3_mprintf( + "%s \"%s\" --client %d --trace %d %s&", + g.argv0, g.zDbFile, iClient, g.iTrace, + g.bSqlTrace ? "--sqltrace " : ""); + + system(zSys); + sqlite3_free(zSys); } - return rc; } /* @@ -597,13 +651,18 @@ static void waitForClient(int iClient, int iTimeout, char *zErrPrefix){ int rc; if( iClient>0 ){ pStmt = prepareSql( - "SELECT 1 FROM task WHERE client=%d AND endtime IS NULL", + "SELECT 1 FROM task" + " WHERE client=%d" + " AND client IN (SELECT id FROM client)" + " AND endtime IS NULL", iClient); }else{ pStmt = prepareSql( - "SELECT 1 FROM task WHERE client=%d AND endtime IS NULL", - iClient); + "SELECT 1 FROM task" + " WHERE client IN (SELECT id FROM client)" + " AND endtime IS NULL"); } + g.iTimeout = 0; while( ((rc = sqlite3_step(pStmt))==SQLITE_BUSY || rc==SQLITE_ROW) && iTimeout>0 ){ @@ -612,6 +671,7 @@ static void waitForClient(int iClient, int iTimeout, char *zErrPrefix){ iTimeout -= 50; } sqlite3_finalize(pStmt); + g.iTimeout = DEFAULT_TIMEOUT; if( rc!=SQLITE_DONE ){ if( zErrPrefix==0 ) zErrPrefix = ""; if( iClient>0 ){ @@ -639,16 +699,13 @@ static void runScript( int ii = 0; int iBegin = 0; int n, c, j; - int rc; int len; int nArg; String sResult; char zCmd[30]; char zError[1000]; char azArg[MX_ARG][100]; - unsigned char isRunning[100]; - memset(isRunning, 0, sizeof(isRunning)); memset(&sResult, 0, sizeof(sResult)); stringReset(&sResult); while( (c = zScript[ii])!=0 ){ @@ -699,7 +756,7 @@ static void runScript( */ if( strcmp(zCmd, "exit")==0 ){ int rc = atoi(azArg[0]); - finishScript(taskId); + finishScript(iClient, taskId, 1); if( rc==0 ) sqlite3_close(g.db); exit(rc); }else @@ -727,6 +784,7 @@ static void runScript( errorMessage("line %d of %s:\nExpected [%.*s]\n Got [%s]", prevLine, zFilename, len-jj-1, zAns, sResult.z); } + g.nTest++; stringReset(&sResult); }else @@ -736,7 +794,7 @@ static void runScript( ** Run a subscript from a separate file. */ if( strcmp(zCmd, "source")==0 ){ - char *zNewFile = azArg[1]; + char *zNewFile = azArg[0]; char *zNewScript = readFile(zNewFile); if( g.iTrace ) logMessage("begin script [%s]\n", zNewFile); runScript(0, 0, zNewScript, zNewFile); @@ -762,26 +820,9 @@ static void runScript( */ if( strcmp(zCmd, "start")==0 ){ int iNewClient = atoi(azArg[0]); - char *zSys; - if( iNewClient<1 || iNewClient>=sizeof(isRunning) ){ - errorMessage("line %d of %s: bad client number: %d", - prevLine, zFilename, iNewClient); - goto start_error; + if( iNewClient>0 ){ + startClient(iNewClient); } - if( isRunning[iNewClient] ){ - errorMessage("line %d of %s: client already running: %d", - prevLine, zFilename, iNewClient); - goto start_error; - } - zSys = sqlite3_mprintf( - "%s \"%s\" --client %d --trace %d %s&", - g.argv0, g.zDbFile, iNewClient, g.iTrace, - g.bSqlTrace ? "--sqltrace " : ""); - - system(zSys); - sqlite3_free(zSys); - isRunning[iNewClient] = 1; - start_error: {/* no-op */} }else /* @@ -803,30 +844,24 @@ static void runScript( ** ** --end ** - ** Assign work to a client. + ** Assign work to a client. Start the client if it is not running + ** already. */ if( strcmp(zCmd, "task")==0 ){ int iTarget = atoi(azArg[0]); int iEnd; char *zTask; - sqlite3_stmt *pStmt; iEnd = findEnd(zScript+ii+len, &lineno); - if( iTarget<0 || iTarget>=sizeof(isRunning) - || !isRunning[iTarget] ){ - errorMessage("line %d of %s: client %d is not running", + if( iTarget<0 ){ + errorMessage("line %d of %s: bad client number: %d", prevLine, zFilename, iTarget); - goto task_error; + }else{ + zTask = sqlite3_mprintf("%.*s", iEnd, zScript+ii+len); + startClient(iTarget); + runSql("INSERT INTO task(client,script)" + " VALUES(%d,'%q')", iTarget, zTask); + sqlite3_free(zTask); } - zTask = sqlite3_mprintf("%.*s", iEnd, zScript+ii+len); - pStmt = prepareSql("INSERT INTO task(client,script)" - " VALUES(%d,'%q')", iTarget, zTask); - sqlite3_free(zTask); - while( (rc = sqlite3_step(pStmt))==SQLITE_BUSY ){ - sqlite3_reset(pStmt); - sqlite3_sleep(10); - } - sqlite3_finalize(pStmt); - task_error: iEnd += tokenLength(zScript+ii+len+iEnd, &lineno); len += iEnd; iBegin = ii+len; @@ -964,6 +999,8 @@ int main(int argc, char **argv){ } rc = sqlite3_open_v2(g.zDbFile, &g.db, openFlags, g.zVfs); if( rc ) fatalError("cannot open [%s]", g.zDbFile); + sqlite3_busy_handler(g.db, busyHandler, 0); + g.iTimeout = DEFAULT_TIMEOUT; if( g.bSqlTrace ) sqlite3_trace(g.db, sqlTraceCallback, 0); if( iClient>0 ){ if( n>0 ) unrecognizedArguments(argv[0], n, argv+2); @@ -977,12 +1014,13 @@ int main(int argc, char **argv){ iClient, taskId); runScript(iClient, taskId, zScript, zTaskName); if( g.iTrace ) logMessage("end task %d", taskId); - finishScript(taskId); + finishScript(iClient, taskId, 0); sqlite3_sleep(10); } if( g.iTrace ) logMessage("end-client"); }else{ sqlite3_stmt *pStmt; + int iTimeout; if( n==0 ){ fatalError("missing script filename"); } @@ -995,8 +1033,11 @@ int main(int argc, char **argv){ " endtime DATE,\n" " script TEXT\n" ");" - "CREATE TABLE clienterror(cnt);\n" - "INSERT INTO clienterror VALUES(0);\n" + "CREATE INDEX task_i1 ON task(client, starttime);\n" + "CREATE INDEX task_i2 ON task(client, endtime);\n" + "CREATE TABLE counters(nError,nTest);\n" + "INSERT INTO counters VALUES(0,0);\n" + "CREATE TABLE client(id INTEGER PRIMARY KEY, wantHalt);\n" ); zScript = readFile(argv[2]); if( g.iTrace ) logMessage("begin script [%s]\n", argv[2]); @@ -1004,16 +1045,25 @@ int main(int argc, char **argv){ sqlite3_free(zScript); if( g.iTrace ) logMessage("end script [%s]\n", argv[2]); waitForClient(0, 2000, "during shutdown...\n"); - while( trySql("CREATE TABLE halt(x);")==SQLITE_BUSY ){ + trySql("UPDATE client SET wantHalt=1"); + sqlite3_sleep(10); + g.iTimeout = 0; + iTimeout = 1000; + while( ((rc = trySql("SELECT 1 FROM client"))==SQLITE_BUSY + || rc==SQLITE_ROW) && iTimeout>0 ){ sqlite3_sleep(10); + iTimeout -= 10; } sqlite3_sleep(100); - pStmt = prepareSql("SELECT cnt FROM clienterror"); - while( (rc = sqlite3_step(pStmt))==SQLITE_BUSY ){ + pStmt = prepareSql("SELECT nError, nTest FROM counters"); + iTimeout = 1000; + while( (rc = sqlite3_step(pStmt))==SQLITE_BUSY && iTimeout>0 ){ sqlite3_sleep(10); + iTimeout -= 10; } if( rc==SQLITE_ROW ){ g.nError += sqlite3_column_int(pStmt, 0); + g.nTest += sqlite3_column_int(pStmt, 1); } sqlite3_finalize(pStmt); } @@ -1021,11 +1071,7 @@ int main(int argc, char **argv){ maybeClose(g.pLog); maybeClose(g.pErrLog); if( iClient==0 ){ - if( g.nError ){ - printf("ERRORS: %d\n", g.nError); - }else if( g.iTrace ){ - printf("All OK\n"); - } + printf("Summary: %d errors in %d tests\n", g.nError, g.nTest); } return g.nError>0; } diff --git a/mptest/multiwrite01.test b/mptest/multiwrite01.test new file mode 100644 index 0000000000..dc71884047 --- /dev/null +++ b/mptest/multiwrite01.test @@ -0,0 +1,141 @@ +/* +** This script sets up five different tasks all writing and updating +** the database at the same time, but each in its own table. +*/ +--task 1 + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + --sleep 1 + INSERT INTO t1 VALUES(1, randomblob(2000)); + INSERT INTO t1 VALUES(2, randomblob(1000)); + --sleep 1 + INSERT INTO t1 SELECT a+2, randomblob(1500) FROM t1; + INSERT INTO t1 SELECT a+4, randomblob(1500) FROM t1; + INSERT INTO t1 SELECT a+8, randomblob(1500) FROM t1; + --sleep 1 + INSERT INTO t1 SELECT a+16, randomblob(1500) FROM t1; + --sleep 1 + INSERT INTO t1 SELECT a+32, randomblob(1500) FROM t1; + SELECT count(*) FROM t1; + --match 64 + SELECT avg(length(b)) FROM t1; + --match 1500.0 + --sleep 2 + UPDATE t1 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t1; + --match 247 +--end + + +--task 2 + DROP TABLE IF EXISTS t2; + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + --sleep 1 + INSERT INTO t2 VALUES(1, randomblob(2000)); + INSERT INTO t2 VALUES(2, randomblob(1000)); + --sleep 1 + INSERT INTO t2 SELECT a+2, randomblob(1500) FROM t2; + INSERT INTO t2 SELECT a+4, randomblob(1500) FROM t2; + INSERT INTO t2 SELECT a+8, randomblob(1500) FROM t2; + --sleep 1 + INSERT INTO t2 SELECT a+16, randomblob(1500) FROM t2; + --sleep 1 + INSERT INTO t2 SELECT a+32, randomblob(1500) FROM t2; + SELECT count(*) FROM t2; + --match 64 + SELECT avg(length(b)) FROM t2; + --match 1500.0 + --sleep 2 + UPDATE t2 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t2; + --match 247 +--end + +--task 3 + DROP TABLE IF EXISTS t3; + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + --sleep 1 + INSERT INTO t3 VALUES(1, randomblob(2000)); + INSERT INTO t3 VALUES(2, randomblob(1000)); + --sleep 1 + INSERT INTO t3 SELECT a+2, randomblob(1500) FROM t3; + INSERT INTO t3 SELECT a+4, randomblob(1500) FROM t3; + INSERT INTO t3 SELECT a+8, randomblob(1500) FROM t3; + --sleep 1 + INSERT INTO t3 SELECT a+16, randomblob(1500) FROM t3; + --sleep 1 + INSERT INTO t3 SELECT a+32, randomblob(1500) FROM t3; + SELECT count(*) FROM t3; + --match 64 + SELECT avg(length(b)) FROM t3; + --match 1500.0 + --sleep 2 + UPDATE t3 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t3; + --match 247 +--end + +--task 4 + DROP TABLE IF EXISTS t4; + CREATE TABLE t4(a INTEGER PRIMARY KEY, b); + --sleep 1 + INSERT INTO t4 VALUES(1, randomblob(2000)); + INSERT INTO t4 VALUES(2, randomblob(1000)); + --sleep 1 + INSERT INTO t4 SELECT a+2, randomblob(1500) FROM t4; + INSERT INTO t4 SELECT a+4, randomblob(1500) FROM t4; + INSERT INTO t4 SELECT a+8, randomblob(1500) FROM t4; + --sleep 1 + INSERT INTO t4 SELECT a+16, randomblob(1500) FROM t4; + --sleep 1 + INSERT INTO t4 SELECT a+32, randomblob(1500) FROM t4; + SELECT count(*) FROM t4; + --match 64 + SELECT avg(length(b)) FROM t4; + --match 1500.0 + --sleep 2 + UPDATE t4 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t4; + --match 247 +--end + +--task 5 + DROP TABLE IF EXISTS t5; + CREATE TABLE t5(a INTEGER PRIMARY KEY, b); + --sleep 1 + INSERT INTO t5 VALUES(1, randomblob(2000)); + INSERT INTO t5 VALUES(2, randomblob(1000)); + --sleep 1 + INSERT INTO t5 SELECT a+2, randomblob(1500) FROM t5; + INSERT INTO t5 SELECT a+4, randomblob(1500) FROM t5; + INSERT INTO t5 SELECT a+8, randomblob(1500) FROM t5; + --sleep 1 + INSERT INTO t5 SELECT a+16, randomblob(1500) FROM t5; + --sleep 1 + INSERT INTO t5 SELECT a+32, randomblob(1500) FROM t5; + SELECT count(*) FROM t5; + --match 64 + SELECT avg(length(b)) FROM t5; + --match 1500.0 + --sleep 2 + UPDATE t5 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t5; + --match 247 +--end + +--wait all +SELECT count(*), total(length(b)) FROM t1; +--match 64 247 +SELECT count(*), total(length(b)) FROM t2; +--match 64 247 +SELECT count(*), total(length(b)) FROM t3; +--match 64 247 +SELECT count(*), total(length(b)) FROM t4; +--match 64 247 +SELECT count(*), total(length(b)) FROM t5; +--match 64 247 +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +DROP TABLE t4; +DROP TABLE t5; From f012ae07593dfe0006e579200f27e4f53f420d5e Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 6 Apr 2013 14:04:22 +0000 Subject: [PATCH 099/204] Get the mptest program running on windows. FossilOrigin-Name: 3966b33284355325afda627ed6f4e26f967438f8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 24 +++++++++++++++++++++++- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 53c2472e9b..df3387a104 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Many\simprovements\sto\sthe\smptest\sprogram.\s\sAdded\sa\ssimple\stest\sscript. -D 2013-04-06T13:09:11.141 +C Get\sthe\smptest\sprogram\srunning\son\swindows. +D 2013-04-06T14:04:22.154 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -111,7 +111,7 @@ F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 058a9bc2b0db710d36003ab06dc1618566f27b52 -F mptest/mptest.c d73b294c354719f33086d8d3ebc273553b26073c +F mptest/mptest.c 7dcc7afacc87be958c181f36a8494ef145ebd131 F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1047,7 +1047,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c318fafe686120d7fb8e487eb3bb4942d497665c -R e5c361902695f1e216fb1ed024ee2204 +P 07b0401a9b61b1664fc6dcddac3b5969fc0f481a +R 61768559c8a66440452ca26012ff3963 U drh -Z 1a794dfe7c097a8f7d97d3bcdba2a731 +Z 2d9f9bfb3324d80756ef1485b1717ef9 diff --git a/manifest.uuid b/manifest.uuid index a5a84885b3..6a4ccb2841 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07b0401a9b61b1664fc6dcddac3b5969fc0f481a \ No newline at end of file +3966b33284355325afda627ed6f4e26f967438f8 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index e07e974de3..ae77770380 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -22,6 +22,7 @@ ** Recommended options: ** ** -DHAVE_USLEEP +** -DSQLITE_MAX_SCHEMA_RETRY=100 ** -DSQLITE_NO_SYNC ** -DSQLITE_THREADSAFE=0 ** -DSQLITE_OMIT_LOAD_EXTENSION @@ -542,14 +543,35 @@ static int finishScript(int iClient, int taskId, int bShutdown){ static void startClient(int iClient){ runSql("INSERT OR IGNORE INTO client VALUES(%d,0)", iClient); if( sqlite3_changes(g.db) ){ +#if defined(__unix__) char *zSys; zSys = sqlite3_mprintf( "%s \"%s\" --client %d --trace %d %s&", g.argv0, g.zDbFile, iClient, g.iTrace, g.bSqlTrace ? "--sqltrace " : ""); - system(zSys); sqlite3_free(zSys); +#endif +#if defined(_WIN32) + char *argv[10]; + char zClient[20]; + char zTrace[20]; + argv[0] = g.argv0; + argv[1] = g.zDbFile; + argv[2] = "--client"; + sqlite3_snprintf(sizeof(zClient),zClient,"%d",iClient); + argv[3] = zClient; + argv[4] = "--trace"; + sqlite3_snprintf(sizeof(zTrace),zTrace,"%d",g.iTrace); + argv[5] = zTrace; + if( g.bSqlTrace ){ + argv[6] = "--sqltrace"; + argv[7] = 0; + }else{ + argv[6] = 0; + } + _spawnv(_P_NOWAIT, g.argv0, argv); +#endif } } From 896be7d93ac2f92443e40e9015235f224616a4f5 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 6 Apr 2013 14:16:41 +0000 Subject: [PATCH 100/204] Get the build of mptest working on mac. FossilOrigin-Name: a0bae686c9d7f08afac00f66067f5147be6231e0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index df3387a104..5709059ad5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sthe\smptest\sprogram\srunning\son\swindows. -D 2013-04-06T14:04:22.154 +C Get\sthe\sbuild\sof\smptest\sworking\son\smac. +D 2013-04-06T14:16:41.777 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -111,7 +111,7 @@ F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 058a9bc2b0db710d36003ab06dc1618566f27b52 -F mptest/mptest.c 7dcc7afacc87be958c181f36a8494ef145ebd131 +F mptest/mptest.c 5d48c89704556ccf01602627fb26b9ec920ae0a6 F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1047,7 +1047,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 07b0401a9b61b1664fc6dcddac3b5969fc0f481a -R 61768559c8a66440452ca26012ff3963 +P 3966b33284355325afda627ed6f4e26f967438f8 +R 8058bb54d4c385606834ce76ac0c79c1 U drh -Z 2d9f9bfb3324d80756ef1485b1717ef9 +Z 71f79ad69c84c0cc8a1fd074e63ccdd7 diff --git a/manifest.uuid b/manifest.uuid index 6a4ccb2841..b1f322192d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3966b33284355325afda627ed6f4e26f967438f8 \ No newline at end of file +a0bae686c9d7f08afac00f66067f5147be6231e0 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index ae77770380..517bb2e302 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -543,7 +543,7 @@ static int finishScript(int iClient, int taskId, int bShutdown){ static void startClient(int iClient){ runSql("INSERT OR IGNORE INTO client VALUES(%d,0)", iClient); if( sqlite3_changes(g.db) ){ -#if defined(__unix__) +#if !defined(_WIN32) char *zSys; zSys = sqlite3_mprintf( "%s \"%s\" --client %d --trace %d %s&", From 1790bb36c57b7ba86477a6b2df097b80df279202 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 6 Apr 2013 14:30:29 +0000 Subject: [PATCH 101/204] Add SQLITE_CONFIG_LOG error logging to the mptest program. FossilOrigin-Name: 716c25bd125c8934499c1fb59a60d7e844c0691c --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 9 +++++++++ 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 5709059ad5..e1cc87dabd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sthe\sbuild\sof\smptest\sworking\son\smac. -D 2013-04-06T14:16:41.777 +C Add\sSQLITE_CONFIG_LOG\serror\slogging\sto\sthe\smptest\sprogram. +D 2013-04-06T14:30:29.439 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -111,7 +111,7 @@ F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 058a9bc2b0db710d36003ab06dc1618566f27b52 -F mptest/mptest.c 5d48c89704556ccf01602627fb26b9ec920ae0a6 +F mptest/mptest.c 4f1a7ad2b9d089fb1bfbbac8cd9f6fe840575f10 F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1047,7 +1047,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3966b33284355325afda627ed6f4e26f967438f8 -R 8058bb54d4c385606834ce76ac0c79c1 +P a0bae686c9d7f08afac00f66067f5147be6231e0 +R 861169f81b752fa53a7328f523d2af55 U drh -Z 71f79ad69c84c0cc8a1fd074e63ccdd7 +Z 4d9e999545d1ee7e59c6e7a0f5ed5b2c diff --git a/manifest.uuid b/manifest.uuid index b1f322192d..f6ccd0c84f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a0bae686c9d7f08afac00f66067f5147be6231e0 \ No newline at end of file +716c25bd125c8934499c1fb59a60d7e844c0691c \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 517bb2e302..768a73ed72 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -289,6 +289,14 @@ static void sqlTraceCallback(void *NotUsed1, const char *zSql){ logMessage("[%.*s]", clipLength(zSql), zSql); } +/* +** SQL error log callback +*/ +static void sqlErrorCallback(void *pArg, int iErrCode, const char *zMsg){ + if( (iErrCode&0xff)==SQLITE_SCHEMA && g.iTrace<3 ) return; + errorMessage("(errcode=%d) %s", iErrCode, zMsg); +} + /* ** Prepare an SQL statement. Issue a fatal error if unable. */ @@ -1010,6 +1018,7 @@ int main(int argc, char **argv){ }else{ g.pLog = stdout; } + sqlite3_config(SQLITE_CONFIG_LOG, sqlErrorCallback, 0); if( zClient ){ iClient = atoi(zClient); if( iClient<1 ) fatalError("illegal client number: %d\n", iClient); From 60625313b04bd9dc09c8f770095b9b02b24a407c Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 6 Apr 2013 18:06:51 +0000 Subject: [PATCH 102/204] Increase the default SQLITE_MAX_SCHEMA_RETRY to 50. Make sure that macro covers every case where a prepared statement might need to be reprepared due to a schema change. The sqlite3_exec() interface now uses sqlite3_prepare_v2(). FossilOrigin-Name: c1d7304c80c4a6244c8a9f6fad1eebd0f339c724 --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/legacy.c | 12 ++++-------- src/prepare.c | 1 - src/vdbeInt.h | 8 ++++++++ src/vdbeapi.c | 8 -------- src/vdbeblob.c | 2 +- 7 files changed, 25 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index 7e72d182cd..5497db1fb1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sto\s"permutations.test\scoverage-pager"\sto\scover\suncovered\sbranches. -D 2013-04-06T11:03:09.144 +C Increase\sthe\sdefault\sSQLITE_MAX_SCHEMA_RETRY\sto\s50.\s\sMake\ssure\sthat\smacro\ncovers\severy\scase\swhere\sa\sprepared\sstatement\smight\sneed\sto\sbe\sreprepared\sdue\nto\sa\sschema\schange.\s\sThe\ssqlite3_exec()\sinterface\snow\suses\s\nsqlite3_prepare_v2(). +D 2013-04-06T18:06:51.485 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -140,7 +140,7 @@ F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 F src/insert.c f7cb141e8ce257cb6b15c497f09e4e23d6055599 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d -F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f +F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 1422eba4aa2b1fb5f7b3aef574752272477d21e2 F src/main.c 54a841854734b6731c4d026834788cac6a19f3d1 @@ -169,7 +169,7 @@ F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 F src/pragma.c 682e97f3e3b77fd6c9b569eabfbf4a14c987aca3 -F src/prepare.c 310eaff1ee5f3c700b3545afb095cfe9346efc3a +F src/prepare.c 743e484233c51109666d402f470523553b41797c F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 @@ -241,10 +241,10 @@ F src/util.c f566b5138099a2df8533b190d0dcc74b7dfbe0c9 F src/vacuum.c 2727bdd08847fcb6b2d2da6d14f018910e8645d3 F src/vdbe.c 1d56617d816098a41bdbe73536a67dcdefad52bd F src/vdbe.h b52887278cb173e66188da84dfab216bea61119d -F src/vdbeInt.h 3bb8531d6768379f086e1576b31d52331f6eb982 -F src/vdbeapi.c 56f9ad6329d093f54b5dd321a2b4a643f89a6cb6 +F src/vdbeInt.h c1e830268b75f04a2901dd895b51a637a26c7045 +F src/vdbeapi.c 085cf9bf169b859a6c8fa43791702bac805cb7aa F src/vdbeaux.c ecb43014bcd3019e5aa2b5561e5c3a447f007a08 -F src/vdbeblob.c 32f2a4899d67f69634ea4dd93e3f651936d732cb +F src/vdbeblob.c 5dc79627775bd9a9b494dd956e26297946417d69 F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c 4fad64071ae120c25f39dcac572d716b9cadeb7f F src/vdbetrace.c a22263ab47f6ba4fcd176515cec1e732866b25f0 @@ -1044,7 +1044,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b095e2cdb61ca8487255687f58fb1024d40f3986 -R bbe90127c799120f0deb57d78c345328 -U dan -Z 0a43fd582a2aa921b5f65df032585318 +P 07a0e4e9820a85bf1690214fef3577866c763f92 +R 2fd4d54ffecba80bddc0b0bfdd8ef6ea +U drh +Z d5343d886a406d77dc84001af2f5f17d diff --git a/manifest.uuid b/manifest.uuid index 10faadaead..85d44ed73b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07a0e4e9820a85bf1690214fef3577866c763f92 \ No newline at end of file +c1d7304c80c4a6244c8a9f6fad1eebd0f339c724 \ No newline at end of file diff --git a/src/legacy.c b/src/legacy.c index ebab2de37d..94649ae705 100644 --- a/src/legacy.c +++ b/src/legacy.c @@ -38,7 +38,6 @@ int sqlite3_exec( const char *zLeftover; /* Tail of unprocessed SQL */ sqlite3_stmt *pStmt = 0; /* The current SQL statement */ char **azCols = 0; /* Names of result columns */ - int nRetry = 0; /* Number of retry attempts */ int callbackIsInit; /* True if callback data is initialized */ if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; @@ -46,12 +45,12 @@ int sqlite3_exec( sqlite3_mutex_enter(db->mutex); sqlite3Error(db, SQLITE_OK, 0); - while( (rc==SQLITE_OK || (rc==SQLITE_SCHEMA && (++nRetry)<2)) && zSql[0] ){ + while( rc==SQLITE_OK && zSql[0] ){ int nCol; char **azVals = 0; pStmt = 0; - rc = sqlite3_prepare(db, zSql, -1, &pStmt, &zLeftover); + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover); assert( rc==SQLITE_OK || pStmt==0 ); if( rc!=SQLITE_OK ){ continue; @@ -108,11 +107,8 @@ int sqlite3_exec( if( rc!=SQLITE_ROW ){ rc = sqlite3VdbeFinalize((Vdbe *)pStmt); pStmt = 0; - if( rc!=SQLITE_SCHEMA ){ - nRetry = 0; - zSql = zLeftover; - while( sqlite3Isspace(zSql[0]) ) zSql++; - } + zSql = zLeftover; + while( sqlite3Isspace(zSql[0]) ) zSql++; break; } } diff --git a/src/prepare.c b/src/prepare.c index 26d6c2614f..d78d83cbd8 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -654,7 +654,6 @@ static int sqlite3Prepare( } #endif - assert( db->init.busy==0 || saveSqlFlag==0 ); if( db->init.busy==0 ){ Vdbe *pVdbe = pParse->pVdbe; sqlite3VdbeSetSql(pVdbe, zSql, (int)(pParse->zTail-zSql), saveSqlFlag); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 2372b83849..3a5b4028bb 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -18,6 +18,14 @@ #ifndef _VDBEINT_H_ #define _VDBEINT_H_ +/* +** The maximum number of times that a statement will try to reparse +** itself before giving up and returning SQLITE_SCHEMA. +*/ +#ifndef SQLITE_MAX_SCHEMA_RETRY +# define SQLITE_MAX_SCHEMA_RETRY 50 +#endif + /* ** SQL is translated into a sequence of instructions to be ** executed by a virtual machine. Each instruction is an instance diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 2387438356..7c861e2d47 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -453,14 +453,6 @@ end_of_step: return (rc&db->errMask); } -/* -** The maximum number of times that a statement will try to reparse -** itself before giving up and returning SQLITE_SCHEMA. -*/ -#ifndef SQLITE_MAX_SCHEMA_RETRY -# define SQLITE_MAX_SCHEMA_RETRY 5 -#endif - /* ** This is the top-level implementation of sqlite3_step(). Call ** sqlite3Step() to do most of the work. If a schema error occurs, diff --git a/src/vdbeblob.c b/src/vdbeblob.c index ae77a47ba3..2e8fd8ee74 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -313,7 +313,7 @@ int sqlite3_blob_open( } sqlite3_bind_int64(pBlob->pStmt, 1, iRow); rc = blobSeekToRow(pBlob, iRow, &zErr); - } while( (++nAttempt)<5 && rc==SQLITE_SCHEMA ); + } while( (++nAttempt)mallocFailed==0 ){ From e348fc7fe8433fe5f8dcb36317cf0e3661bb7f19 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 6 Apr 2013 18:35:07 +0000 Subject: [PATCH 103/204] Fix the --source command so that its argument is relative to the script. Verify that the SQLite header and library match. Print SQLite version and compile-time configuration information on startup. FossilOrigin-Name: 15cb0db7583c3a24cbea0c72576047a93fba0801 --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 33 +++++++++++++++++++++++++++++---- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index bfff919823..36209600bd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sthe\smptest\sbranch.\s\sThe\smptest\sneeds\sthe\nSQLITE_MAX_SCHEMA_RETRY\sfix\sin\sparticular. -D 2013-04-06T18:10:27.166 +C Fix\sthe\s--source\scommand\sso\sthat\sits\sargument\sis\srelative\sto\sthe\sscript.\nVerify\sthat\sthe\sSQLite\sheader\sand\slibrary\smatch.\s\sPrint\sSQLite\sversion\nand\scompile-time\sconfiguration\sinformation\son\sstartup. +D 2013-04-06T18:35:07.098 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -111,7 +111,7 @@ F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 058a9bc2b0db710d36003ab06dc1618566f27b52 -F mptest/mptest.c 4f1a7ad2b9d089fb1bfbbac8cd9f6fe840575f10 +F mptest/mptest.c 6e23dade7b4cde4801b7a6df29f52a492d82d450 F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1047,7 +1047,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 716c25bd125c8934499c1fb59a60d7e844c0691c c1d7304c80c4a6244c8a9f6fad1eebd0f339c724 -R 865d94cb53d70dd89436e58ffa63ebac +P fb7a7773965697855c12193408cb75300decc23c +R 22a5a5addda1bdaa7f5e87ce6cbfc935 U drh -Z 83b40ff56b301adcba03b758b3b36603 +Z 9527a4645b09957de5c779d28c3eb994 diff --git a/manifest.uuid b/manifest.uuid index 284652c9b5..aa637d58c4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb7a7773965697855c12193408cb75300decc23c \ No newline at end of file +15cb0db7583c3a24cbea0c72576047a93fba0801 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 768a73ed72..d2fdb64396 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -22,7 +22,6 @@ ** Recommended options: ** ** -DHAVE_USLEEP -** -DSQLITE_MAX_SCHEMA_RETRY=100 ** -DSQLITE_NO_SYNC ** -DSQLITE_THREADSAFE=0 ** -DSQLITE_OMIT_LOAD_EXTENSION @@ -824,11 +823,21 @@ static void runScript( ** Run a subscript from a separate file. */ if( strcmp(zCmd, "source")==0 ){ - char *zNewFile = azArg[0]; - char *zNewScript = readFile(zNewFile); + char *zNewFile, *zNewScript; + char *zToDel = 0; + zNewFile = azArg[0]; + if( zNewFile[0]!='/' ){ + int k; + for(k=(int)strlen(zFilename)-1; k>=0 && zFilename[k]!='/'; k--){} + if( k>0 ){ + zNewFile = zToDel = sqlite3_mprintf("%.*s/%s", k,zFilename,zNewFile); + } + } + zNewScript = readFile(zNewFile); if( g.iTrace ) logMessage("begin script [%s]\n", zNewFile); runScript(0, 0, zNewScript, zNewFile); sqlite3_free(zNewScript); + sqlite3_free(zToDel); if( g.iTrace ) logMessage("end script [%s]\n", zNewFile); }else @@ -987,17 +996,26 @@ static void unrecognizedArguments( int main(int argc, char **argv){ const char *zClient; int iClient; - int n; + int n, i; int openFlags = SQLITE_OPEN_READWRITE; int rc; char *zScript; int taskId; const char *zTrace; + const char *zCOption; g.argv0 = argv[0]; g.iTrace = 1; if( argc<2 ) usage(argv[0]); g.zDbFile = argv[1]; + if( strglob("*.test", g.zDbFile) ) usage(argv[0]); + if( strcmp(sqlite3_sourceid(), SQLITE_SOURCE_ID)!=0 ){ + fprintf(stderr, "SQLite library and header mismatch\n" + "Library: %s\n" + "Header: %s\n", + sqlite3_sourceid(), SQLITE_SOURCE_ID); + exit(1); + } n = argc-2; sqlite3_snprintf(sizeof(g.zName), g.zName, "mptest"); g.zVfs = findOption(argv+2, &n, "vfs", 1); @@ -1024,6 +1042,13 @@ int main(int argc, char **argv){ if( iClient<1 ) fatalError("illegal client number: %d\n", iClient); sqlite3_snprintf(sizeof(g.zName), g.zName, "client%02d", iClient); }else{ + if( g.iTrace>0 ){ + printf("With SQLite " SQLITE_VERSION " " SQLITE_SOURCE_ID "\n" ); + for(i=0; (zCOption = sqlite3_compileoption_get(i))!=0; i++){ + printf("-DSQLITE_%s\n", zCOption); + } + fflush(stdout); + } iClient = 0; unlink(g.zDbFile); openFlags |= SQLITE_OPEN_CREATE; From fbfe388d314f44de6891ede3eaaaa926b0c7769e Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 8 Apr 2013 10:38:57 +0000 Subject: [PATCH 104/204] Fix an incorrect type cast in an assignment in vdbe.c. FossilOrigin-Name: 320b5295da57023399c7b9eed82eae40cf619803 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 5497db1fb1..20d73d0f91 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sdefault\sSQLITE_MAX_SCHEMA_RETRY\sto\s50.\s\sMake\ssure\sthat\smacro\ncovers\severy\scase\swhere\sa\sprepared\sstatement\smight\sneed\sto\sbe\sreprepared\sdue\nto\sa\sschema\schange.\s\sThe\ssqlite3_exec()\sinterface\snow\suses\s\nsqlite3_prepare_v2(). -D 2013-04-06T18:06:51.485 +C Fix\san\sincorrect\stype\scast\sin\san\sassignment\sin\svdbe.c. +D 2013-04-08T10:38:57.721 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -239,7 +239,7 @@ F src/update.c a2a5631d618cbe240fc83725fa9e95c56ae0084c F src/utf.c 8d819e2e5104a430fc2005f018db14347c95a38f F src/util.c f566b5138099a2df8533b190d0dcc74b7dfbe0c9 F src/vacuum.c 2727bdd08847fcb6b2d2da6d14f018910e8645d3 -F src/vdbe.c 1d56617d816098a41bdbe73536a67dcdefad52bd +F src/vdbe.c 5f0047130f80c7fd0bc41bc51a653b5542c4fbd5 F src/vdbe.h b52887278cb173e66188da84dfab216bea61119d F src/vdbeInt.h c1e830268b75f04a2901dd895b51a637a26c7045 F src/vdbeapi.c 085cf9bf169b859a6c8fa43791702bac805cb7aa @@ -1044,7 +1044,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 07a0e4e9820a85bf1690214fef3577866c763f92 -R 2fd4d54ffecba80bddc0b0bfdd8ef6ea -U drh -Z d5343d886a406d77dc84001af2f5f17d +P c1d7304c80c4a6244c8a9f6fad1eebd0f339c724 +R 9455b151c7a6a645888f7422c4787f65 +U dan +Z 3be6f6ee1162b72435a62041d8441b1f diff --git a/manifest.uuid b/manifest.uuid index 85d44ed73b..8700cad9be 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c1d7304c80c4a6244c8a9f6fad1eebd0f339c724 \ No newline at end of file +320b5295da57023399c7b9eed82eae40cf619803 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 688621d951..f343e13d4e 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3515,7 +3515,7 @@ case OP_SeekGt: { /* jump, in3 */ ** r.flags = 0; ** } */ - r.flags = (u16)(UNPACKED_INCRKEY * (1 & (oc - OP_SeekLt))); + r.flags = (u8)(UNPACKED_INCRKEY * (1 & (oc - OP_SeekLt))); assert( oc!=OP_SeekGt || r.flags==UNPACKED_INCRKEY ); assert( oc!=OP_SeekLe || r.flags==UNPACKED_INCRKEY ); assert( oc!=OP_SeekGe || r.flags==0 ); From 7dfe8e2bde9b7dc655fa8b112efaecbc3acf0dfd Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Apr 2013 13:13:43 +0000 Subject: [PATCH 105/204] Add --if, --else, --endif processing to mptest.c. FossilOrigin-Name: 51265acae3088a51ac0ce6ab8731e6e15a48d4ae --- manifest | 12 ++++---- manifest.uuid | 2 +- mptest/mptest.c | 73 +++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 74 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 36209600bd..3e8ee81047 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s--source\scommand\sso\sthat\sits\sargument\sis\srelative\sto\sthe\sscript.\nVerify\sthat\sthe\sSQLite\sheader\sand\slibrary\smatch.\s\sPrint\sSQLite\sversion\nand\scompile-time\sconfiguration\sinformation\son\sstartup. -D 2013-04-06T18:35:07.098 +C Add\s--if,\s--else,\s--endif\sprocessing\sto\smptest.c. +D 2013-04-08T13:13:43.458 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -111,7 +111,7 @@ F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 058a9bc2b0db710d36003ab06dc1618566f27b52 -F mptest/mptest.c 6e23dade7b4cde4801b7a6df29f52a492d82d450 +F mptest/mptest.c b4030a7ef4a2bfd2913062d637ae3f55fb9443c2 F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1047,7 +1047,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P fb7a7773965697855c12193408cb75300decc23c -R 22a5a5addda1bdaa7f5e87ce6cbfc935 +P 15cb0db7583c3a24cbea0c72576047a93fba0801 +R ebe6490efbbe63db101a13263cb552f3 U drh -Z 9527a4645b09957de5c779d28c3eb994 +Z bd1b6a8c139e5f3d36fca35a646773b9 diff --git a/manifest.uuid b/manifest.uuid index aa637d58c4..b0847befc9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -15cb0db7583c3a24cbea0c72576047a93fba0801 \ No newline at end of file +51265acae3088a51ac0ce6ab8731e6e15a48d4ae \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index d2fdb64396..9a19ef2cd6 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -662,7 +662,7 @@ static int extractToken(const char *zIn, int nIn, char *zOut, int nOut){ } /* -** Find the number of characters up to the next "--end" token. +** Find the number of characters up to the start of the next "--end" token. */ static int findEnd(const char *z, int *pnLine){ int n = 0; @@ -672,6 +672,30 @@ static int findEnd(const char *z, int *pnLine){ return n; } +/* +** Find the number of characters up to the first character past the +** of the next "--endif" or "--else" token. Nested --if commands are +** also skipped. +*/ +static int findEndif(const char *z, int stopAtElse, int *pnLine){ + int n = 0; + while( z[n] ){ + int len = tokenLength(z+n, pnLine); + if( (strncmp(z+n,"--endif",7)==0 && isspace(z[n+7])) + || (stopAtElse && strncmp(z+n,"--else",6)==0 && isspace(z[n+6])) + ){ + return n+len; + } + if( strncmp(z+n,"--if",4)==0 && isspace(z[n+4]) ){ + int skip = findEndif(z+n+len, 0, pnLine); + n += skip + len; + }else{ + n += len; + } + } + return n; +} + /* ** Wait for a client process to complete all its tasks */ @@ -712,7 +736,7 @@ static void waitForClient(int iClient, int iTimeout, char *zErrPrefix){ } /* Maximum number of arguments to a --command */ -#define MX_ARG 5 +#define MX_ARG 2 /* ** Run a script. @@ -791,7 +815,7 @@ static void runScript( }else /* - ** --result + ** --reset ** ** Reset accumulated results back to an empty string */ @@ -852,12 +876,49 @@ static void runScript( logMessage("%.*s", len-jj, zScript+ii+jj); }else + /* + ** --if EXPR + ** + ** Skip forward to the next matching --endif or --else if EXPR is false. + */ + if( strcmp(zCmd, "if")==0 ){ + int jj, rc; + sqlite3_stmt *pStmt; + for(jj=4; jj0 ){ startClient(iNewClient); @@ -871,7 +932,7 @@ static void runScript( ** "all" then wait for all clients to complete. Wait no longer than ** TIMEOUT milliseconds (default 10,000) */ - if( strcmp(zCmd, "wait")==0 ){ + if( strcmp(zCmd, "wait")==0 && iClient==0 ){ int iTimeout = nArg>=2 ? atoi(azArg[1]) : 10000; sqlite3_snprintf(sizeof(zError),zError,"line %d of %s\n", prevLine, zFilename); @@ -886,7 +947,7 @@ static void runScript( ** Assign work to a client. Start the client if it is not running ** already. */ - if( strcmp(zCmd, "task")==0 ){ + if( strcmp(zCmd, "task")==0 && iClient==0 ){ int iTarget = atoi(azArg[0]); int iEnd; char *zTask; From 1bf44c7acba2a3bdfb2d2843402cce6a9b1733f0 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Apr 2013 13:48:29 +0000 Subject: [PATCH 106/204] Add the vfsname() and eval() SQL functions to mptest.c. Enhancements to the test/config01.test script. FossilOrigin-Name: 91397a147ce4f67a7ea1182f06a7dda3a96ec465 --- manifest | 14 +++++------ manifest.uuid | 2 +- mptest/config01.test | 24 +++++++++++++++++++ mptest/mptest.c | 55 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3e8ee81047..6766f10a07 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\s--if,\s--else,\s--endif\sprocessing\sto\smptest.c. -D 2013-04-08T13:13:43.458 +C Add\sthe\svfsname()\sand\seval()\sSQL\sfunctions\sto\smptest.c.\s\nEnhancements\sto\sthe\stest/config01.test\sscript. +D 2013-04-08T13:48:29.192 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,8 +110,8 @@ F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 -F mptest/config01.test 058a9bc2b0db710d36003ab06dc1618566f27b52 -F mptest/mptest.c b4030a7ef4a2bfd2913062d637ae3f55fb9443c2 +F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da +F mptest/mptest.c 6614ad2110ce44a78b4745036880275ef939bc6e F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1047,7 +1047,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 15cb0db7583c3a24cbea0c72576047a93fba0801 -R ebe6490efbbe63db101a13263cb552f3 +P 51265acae3088a51ac0ce6ab8731e6e15a48d4ae +R 70ae859a3e60d8d3ce14cc3f23701768 U drh -Z bd1b6a8c139e5f3d36fca35a646773b9 +Z 17c730545a4fcc3406090d39f090bdb3 diff --git a/manifest.uuid b/manifest.uuid index b0847befc9..e40c60c705 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -51265acae3088a51ac0ce6ab8731e6e15a48d4ae \ No newline at end of file +91397a147ce4f67a7ea1182f06a7dda3a96ec465 \ No newline at end of file diff --git a/mptest/config01.test b/mptest/config01.test index be6b570678..75ba2f8fc2 100644 --- a/mptest/config01.test +++ b/mptest/config01.test @@ -1,6 +1,7 @@ /* ** Configure five tasks in different ways, then run tests. */ +--if vfsname() GLOB 'unix' PRAGMA page_size=8192; --task 1 PRAGMA journal_mode=PERSIST; @@ -17,3 +18,26 @@ PRAGMA page_size=8192; PRAGMA journal_mode=OFF; --end --source multiwrite01.test +--wait all +PRAGMA page_size=16384; +VACUUM; +CREATE TABLE pgsz(taskid, sz INTEGER); +--task 1 + INSERT INTO pgsz VALUES(1, eval('PRAGMA page_size')); +--end +--task 2 + INSERT INTO pgsz VALUES(2, eval('PRAGMA page_size')); +--end +--task 3 + INSERT INTO pgsz VALUES(3, eval('PRAGMA page_size')); +--end +--task 4 + INSERT INTO pgsz VALUES(4, eval('PRAGMA page_size')); +--end +--task 5 + INSERT INTO pgsz VALUES(5, eval('PRAGMA page_size')); +--end +--source multiwrite01.test +--wait all +SELECT sz FROM pgsz; +--match 16384 16384 16384 16384 16384 diff --git a/mptest/mptest.c b/mptest/mptest.c index 9a19ef2cd6..8fc13d069a 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -269,6 +269,22 @@ static int clipLength(const char *z){ return n; } +/* +** Auxiliary SQL function to return the name of the VFS +*/ +static void vfsNameFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + char *zVfs = 0; + sqlite3_file_control(db, "main", SQLITE_FCNTL_VFSNAME, &zVfs); + if( zVfs ){ + sqlite3_result_text(context, zVfs, -1, sqlite3_free); + } +} + /* ** Busy handler with a g.iTimeout-millisecond timeout */ @@ -450,6 +466,32 @@ static int evalSql(String *p, const char *zFormat, ...){ return rc; } +/* +** Auxiliary SQL function to recursively evaluate SQL. +*/ +static void evalFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + String res; + char *zErrMsg = 0; + int rc; + memset(&res, 0, sizeof(res)); + rc = sqlite3_exec(db, zSql, evalCallback, &res, &zErrMsg); + if( zErrMsg ){ + sqlite3_result_error(context, zErrMsg, -1); + sqlite3_free(zErrMsg); + }else if( rc ){ + sqlite3_result_error_code(context, rc); + }else{ + sqlite3_result_text(context, res.z, -1, SQLITE_TRANSIENT); + } + stringFree(&res); +} + /* ** Look up the next task for client iClient in the database. ** Return the task script and the task number and mark that @@ -841,6 +883,15 @@ static void runScript( stringReset(&sResult); }else + /* + ** --output + ** + ** Output the result of the previous SQL. + */ + if( strcmp(zCmd, "output")==0 ){ + logMessage("%s", sResult.z); + }else + /* ** --source FILENAME ** @@ -1117,6 +1168,10 @@ int main(int argc, char **argv){ rc = sqlite3_open_v2(g.zDbFile, &g.db, openFlags, g.zVfs); if( rc ) fatalError("cannot open [%s]", g.zDbFile); sqlite3_busy_handler(g.db, busyHandler, 0); + sqlite3_create_function(g.db, "vfsname", 0, SQLITE_UTF8, 0, + vfsNameFunc, 0, 0); + sqlite3_create_function(g.db, "eval", 1, SQLITE_UTF8, 0, + evalFunc, 0, 0); g.iTimeout = DEFAULT_TIMEOUT; if( g.bSqlTrace ) sqlite3_trace(g.db, sqlTraceCallback, 0); if( iClient>0 ){ From 841810cef6e8f7f15b8bfe789e0a77ebbe82459f Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Apr 2013 13:59:11 +0000 Subject: [PATCH 107/204] Fix harmless compiler warnings in mptest.c. FossilOrigin-Name: 59bdbb10ed67bf79e0991e2ad58a28321dedb959 --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 12 +++++++++++- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 6766f10a07..1a6f824d31 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\svfsname()\sand\seval()\sSQL\sfunctions\sto\smptest.c.\s\nEnhancements\sto\sthe\stest/config01.test\sscript. -D 2013-04-08T13:48:29.192 +C Fix\sharmless\scompiler\swarnings\sin\smptest.c. +D 2013-04-08T13:59:11.021 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -111,7 +111,7 @@ F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da -F mptest/mptest.c 6614ad2110ce44a78b4745036880275ef939bc6e +F mptest/mptest.c fb46946df7d6f2e8be8c028b21a580a977326f24 F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1047,7 +1047,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 51265acae3088a51ac0ce6ab8731e6e15a48d4ae -R 70ae859a3e60d8d3ce14cc3f23701768 +P 91397a147ce4f67a7ea1182f06a7dda3a96ec465 +R ede2c5555feb77dd0a5488dbe1f8d502 U drh -Z 17c730545a4fcc3406090d39f090bdb3 +Z 8e0fdcf5a1bb9a273c7156627676f7a5 diff --git a/manifest.uuid b/manifest.uuid index e40c60c705..a4c5a4a477 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -91397a147ce4f67a7ea1182f06a7dda3a96ec465 \ No newline at end of file +59bdbb10ed67bf79e0991e2ad58a28321dedb959 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 8fc13d069a..30449bddfb 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -41,6 +41,9 @@ #include #include +/* Mark a parameter as unused to suppress compiler warnings */ +#define UNUSED_PARAMETER(x) (void)x + /* Global data */ static struct Global { @@ -279,6 +282,8 @@ static void vfsNameFunc( ){ sqlite3 *db = sqlite3_context_db_handle(context); char *zVfs = 0; + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); sqlite3_file_control(db, "main", SQLITE_FCNTL_VFSNAME, &zVfs); if( zVfs ){ sqlite3_result_text(context, zVfs, -1, sqlite3_free); @@ -289,6 +294,7 @@ static void vfsNameFunc( ** Busy handler with a g.iTimeout-millisecond timeout */ static int busyHandler(void *pCD, int count){ + UNUSED_PARAMETER(pCD); if( count*10>g.iTimeout ){ if( g.iTimeout>0 ) errorMessage("timeout after %dms", g.iTimeout); return 0; @@ -301,6 +307,7 @@ static int busyHandler(void *pCD, int count){ ** SQL Trace callback */ static void sqlTraceCallback(void *NotUsed1, const char *zSql){ + UNUSED_PARAMETER(NotUsed1); logMessage("[%.*s]", clipLength(zSql), zSql); } @@ -308,6 +315,7 @@ static void sqlTraceCallback(void *NotUsed1, const char *zSql){ ** SQL error log callback */ static void sqlErrorCallback(void *pArg, int iErrCode, const char *zMsg){ + UNUSED_PARAMETER(pArg); if( (iErrCode&0xff)==SQLITE_SCHEMA && g.iTrace<3 ) return; errorMessage("(errcode=%d) %s", iErrCode, zMsg); } @@ -435,6 +443,7 @@ static void stringAppendTerm(String *p, const char *z){ static int evalCallback(void *pCData, int argc, char **argv, char **azCol){ String *p = (String*)pCData; int i; + UNUSED_PARAMETER(azCol); for(i=0; i Date: Mon, 8 Apr 2013 14:28:33 +0000 Subject: [PATCH 108/204] Add the "mptester" target to the makefiles. Make mptester run with synchronous off by default. FossilOrigin-Name: 1397830bfead3699a7e6b1858ae67ae15259b89e --- Makefile.in | 6 ++++++ Makefile.msc | 5 +++++ main.mk | 5 +++++ manifest | 18 +++++++++--------- manifest.uuid | 2 +- mptest/mptest.c | 21 ++++++++++++++++----- 6 files changed, 42 insertions(+), 15 deletions(-) diff --git a/Makefile.in b/Makefile.in index edd29ae011..0b702def79 100644 --- a/Makefile.in +++ b/Makefile.in @@ -502,6 +502,11 @@ sqlite3$(TEXE): $(TOP)/src/shell.c libsqlite3.la sqlite3.h -o $@ $(TOP)/src/shell.c libsqlite3.la \ $(LIBREADLINE) $(TLIBS) -rpath "$(libdir)" +mptester$(EXE): sqlite3.c $(TOP)/mptest/mptest.c + $(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \ + $(TLIBS) -rpath "$(libdir)" + + # This target creates a directory named "tsrc" and fills it with # copies of all of the C source code and header files needed to # build on the target system. Some of the C source code and header @@ -950,6 +955,7 @@ clean: rm -f shell.c sqlite3ext.h rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c rm -f sqlite-*-output.vsix + rm -f mptest mptest.exe distclean: clean rm -f config.log config.status libtool Makefile sqlite3.pc diff --git a/Makefile.msc b/Makefile.msc index 70be76afa8..8c91e86bc8 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -813,6 +813,10 @@ sqlite3.exe: $(TOP)\src\shell.c libsqlite3.lib $(LIBRESOBJS) sqlite3.h $(TOP)\src\shell.c \ /link $(LTLINKOPTS) $(LTLIBPATHS) libsqlite3.lib $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) +mptester.exe: $(TOP)\mptest\mptest.c libsqlite3.lib $(LIBRESOBJS) sqlite3.h + $(LTLINK) $(TOP)\mptest\mptest.c \ + /link $(LTLINKOPTS) $(LTLIBPATHS) libsqlite3.lib $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) + # This target creates a directory named "tsrc" and fills it with # copies of all of the C source code and header files needed to # build on the target system. Some of the C source code and header @@ -1253,6 +1257,7 @@ clean: del /Q shell.c sqlite3ext.h del /Q sqlite3_analyzer.exe sqlite3_analyzer.exp sqlite3_analyzer.c del /Q sqlite-*-output.vsix + del /Q mptester.exe # Dynamic link library section. # diff --git a/main.mk b/main.mk index 1294121c0a..7ab2bca2ba 100644 --- a/main.mk +++ b/main.mk @@ -365,6 +365,10 @@ sqlite3$(EXE): $(TOP)/src/shell.c libsqlite3.a sqlite3.h $(TOP)/src/shell.c \ libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB) +mptester$(EXE): sqlite3.c $(TOP)/mptest/mptest.c + $(TCCX) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \ + $(TLIBS) $(THREADLIB) + sqlite3.o: sqlite3.c $(TCCX) -c sqlite3.c @@ -626,3 +630,4 @@ clean: rm -f shell.c sqlite3ext.h rm -f sqlite3_analyzer sqlite3_analyzer.exe sqlite3_analyzer.c rm -f sqlite-*-output.vsix + rm -f mptest mptest.exe diff --git a/manifest b/manifest index 1a6f824d31..7acc35ecdc 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Fix\sharmless\scompiler\swarnings\sin\smptest.c. -D 2013-04-08T13:59:11.021 +C Add\sthe\s"mptester"\starget\sto\sthe\smakefiles.\s\sMake\smptester\srun\swith\nsynchronous\soff\sby\sdefault. +D 2013-04-08T14:28:33.142 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 +F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 9ab787d472c9cb8bad29c2c4aeba15b06fba7bc4 +F Makefile.msc a244e5b5ba2493625db95b4d46b79201ea868bfe F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 05c7bd63b96f31cfdef5c766ed91307ac121f5aa @@ -103,7 +103,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk b25a7a0b657409b2dcf9b09d39562703d5df702d +F main.mk 1c26f3cc57d7c357da5218ae2928b7ca5acfecde F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -111,7 +111,7 @@ F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da -F mptest/mptest.c fb46946df7d6f2e8be8c028b21a580a977326f24 +F mptest/mptest.c 739950807a48af97fb3e432256fef96540d0e936 F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1047,7 +1047,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 91397a147ce4f67a7ea1182f06a7dda3a96ec465 -R ede2c5555feb77dd0a5488dbe1f8d502 +P 59bdbb10ed67bf79e0991e2ad58a28321dedb959 +R 1f9c112eec8d81ff124d0eb10d39eea3 U drh -Z 8e0fdcf5a1bb9a273c7156627676f7a5 +Z 523cf56f3e6158426b3e246d7dff8e8c diff --git a/manifest.uuid b/manifest.uuid index a4c5a4a477..dbce6398ba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -59bdbb10ed67bf79e0991e2ad58a28321dedb959 \ No newline at end of file +1397830bfead3699a7e6b1858ae67ae15259b89e \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 30449bddfb..48bc382846 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -35,7 +35,11 @@ */ #include "sqlite3.h" #include -#include +#if defined(_WIN32) +# include +#else +# include +#endif #include #include #include @@ -62,6 +66,7 @@ static struct Global { int nError; /* Number of errors */ int nTest; /* Number of --match operators */ int iTimeout; /* Milliseconds until a busy timeout */ + int bSync; /* Call fsync() */ } g; /* Default timeout */ @@ -604,11 +609,15 @@ static void startClient(int iClient){ if( sqlite3_changes(g.db) ){ #if !defined(_WIN32) char *zSys; + int rc; zSys = sqlite3_mprintf( - "%s \"%s\" --client %d --trace %d %s&", + "%s \"%s\" --client %d --trace %d %s%s&", g.argv0, g.zDbFile, iClient, g.iTrace, - g.bSqlTrace ? "--sqltrace " : ""); - system(zSys); + g.bSqlTrace ? "--sqltrace " : "", + g.bSync ? "--sync " : "" + ); + rc = system(zSys); + if( rc ) errorMessage("system() fails with error code %d", rc); sqlite3_free(zSys); #endif #if defined(_WIN32) @@ -922,8 +931,8 @@ static void runScript( if( g.iTrace ) logMessage("begin script [%s]\n", zNewFile); runScript(0, 0, zNewScript, zNewFile); sqlite3_free(zNewScript); - sqlite3_free(zToDel); if( g.iTrace ) logMessage("end script [%s]\n", zNewFile); + sqlite3_free(zToDel); }else /* @@ -1148,6 +1157,7 @@ int main(int argc, char **argv){ if( zTrace ) g.iTrace = atoi(zTrace); if( findOption(argv+2, &n, "quiet", 0)!=0 ) g.iTrace = 0; g.bSqlTrace = findOption(argv+2, &n, "sqltrace", 0)!=0; + g.bSync = findOption(argv+2, &n, "sync", 0)!=0; if( g.zErrLog ){ g.pErrLog = fopen(g.zErrLog, "a"); }else{ @@ -1184,6 +1194,7 @@ int main(int argc, char **argv){ evalFunc, 0, 0); g.iTimeout = DEFAULT_TIMEOUT; if( g.bSqlTrace ) sqlite3_trace(g.db, sqlTraceCallback, 0); + if( !g.bSync ) trySql("PRAGMA synchronous=OFF"); if( iClient>0 ){ if( n>0 ) unrecognizedArguments(argv[0], n, argv+2); if( g.iTrace ) logMessage("start-client"); From bcb8a868cec66d9edd20e99f7bdb5d2959e2a60d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 8 Apr 2013 15:30:41 +0000 Subject: [PATCH 109/204] Handle the case in os_unix.c where SQLITE_FCNTL_MMAP_LIMIT requests that the mmap limit be set to a value smaller than the current mapping. FossilOrigin-Name: 360473493ec1a7094a2b1c5436f3b70914a6dfdd --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 5 ++++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index c64575a45a..9ff25841fa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\smulti-process\stester\sinto\strunk. -D 2013-04-08T14:41:34.434 +C Handle\sthe\scase\sin\sos_unix.c\swhere\sSQLITE_FCNTL_MMAP_LIMIT\srequests\sthat\sthe\smmap\slimit\sbe\sset\sto\sa\svalue\ssmaller\sthan\sthe\scurrent\smapping. +D 2013-04-08T15:30:41.180 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -164,7 +164,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c f0ecce40d92469d5cc737ae883e776eb3e5c0af5 +F src/os_unix.c 6d584190db2ed442e5c849b7f73dcfbbac7c032e F src/os_win.c 3265df8c762e0b8caf8d02b3352fa8c22e91ae6b F src/pager.c 2e68df46d4086027cb6b527d47a6dedbf1a6b7ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1048,7 +1048,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 320b5295da57023399c7b9eed82eae40cf619803 1397830bfead3699a7e6b1858ae67ae15259b89e -R 518213540ee221b372b2870b99865f42 -U drh -Z 23b5a3829f6928c651f9b0e92776d749 +P ea53c45d79772986d2ad289f4f56f333541747e7 +R 1216631425c14cd22746e029dd206b1f +U dan +Z a5b14b8642cd44330c439704b21c4b8f diff --git a/manifest.uuid b/manifest.uuid index 2449f90305..644f2b2eed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ea53c45d79772986d2ad289f4f56f333541747e7 \ No newline at end of file +360473493ec1a7094a2b1c5436f3b70914a6dfdd \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 75f9236013..1d4c8ad348 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3712,7 +3712,10 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ case SQLITE_FCNTL_MMAP_LIMIT: { i64 newLimit = *(i64*)pArg; *(i64*)pArg = pFile->mmapLimit; - if( newLimit>=0 ) pFile->mmapLimit = newLimit; + if( newLimit>=0 ){ + pFile->mmapLimit = newLimit; + if( newLimitmmapSize ) pFile->mmapSize = newLimit; + } return SQLITE_OK; } #ifdef SQLITE_DEBUG From e5ebd22d992384f8e7fbf94775f695ea6cbe4554 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Apr 2013 15:36:51 +0000 Subject: [PATCH 110/204] Change mptester to avoid reporting SQLITE_BUSY errors if the busy handler is turned off and the tracing level is set low. FossilOrigin-Name: d77407ae6a9e09dea124dd59b14fdbbc36ed3d4e --- manifest | 14 +++++++------- manifest.uuid | 2 +- mptest/mptest.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 9ff25841fa..96187630f4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Handle\sthe\scase\sin\sos_unix.c\swhere\sSQLITE_FCNTL_MMAP_LIMIT\srequests\sthat\sthe\smmap\slimit\sbe\sset\sto\sa\svalue\ssmaller\sthan\sthe\scurrent\smapping. -D 2013-04-08T15:30:41.180 +C Change\smptester\sto\savoid\sreporting\sSQLITE_BUSY\serrors\sif\sthe\sbusy\shandler\nis\sturned\soff\sand\sthe\stracing\slevel\sis\sset\slow. +D 2013-04-08T15:36:51.699 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -112,7 +112,7 @@ F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da F mptest/config02.test 381ec0acbb1e5aab28ca9aa91e17ae9a02bbd2bb -F mptest/mptest.c 739950807a48af97fb3e432256fef96540d0e936 +F mptest/mptest.c 7048030a410707f77845c0fda9c4a223c7069e0b F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1048,7 +1048,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ea53c45d79772986d2ad289f4f56f333541747e7 -R 1216631425c14cd22746e029dd206b1f -U dan -Z a5b14b8642cd44330c439704b21c4b8f +P 360473493ec1a7094a2b1c5436f3b70914a6dfdd +R 138dfe64c0fb20cc8b3e6aefe188c7ba +U drh +Z bbdf55a7e5fdae8e0e462019e357a9d2 diff --git a/manifest.uuid b/manifest.uuid index 644f2b2eed..3944faa67f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -360473493ec1a7094a2b1c5436f3b70914a6dfdd \ No newline at end of file +d77407ae6a9e09dea124dd59b14fdbbc36ed3d4e \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 48bc382846..ca056f76e6 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -322,6 +322,7 @@ static void sqlTraceCallback(void *NotUsed1, const char *zSql){ static void sqlErrorCallback(void *pArg, int iErrCode, const char *zMsg){ UNUSED_PARAMETER(pArg); if( (iErrCode&0xff)==SQLITE_SCHEMA && g.iTrace<3 ) return; + if( g.iTimeout==0 && (iErrCode&0xff)==SQLITE_BUSY && g.iTrace<3 ) return; errorMessage("(errcode=%d) %s", iErrCode, zMsg); } From 023dd903cc2cd8ad87fba3e3b4e8ad6ae5ba19f1 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Apr 2013 17:57:26 +0000 Subject: [PATCH 111/204] Adding tests to the mptester scripts. FossilOrigin-Name: e121adceb1e267e032dcbadb74eaa692a12a65bb --- manifest | 14 +- manifest.uuid | 2 +- mptest/config02.test | 8 ++ mptest/multiwrite01.test | 273 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 284 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 96187630f4..b55b1213e9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\smptester\sto\savoid\sreporting\sSQLITE_BUSY\serrors\sif\sthe\sbusy\shandler\nis\sturned\soff\sand\sthe\stracing\slevel\sis\sset\slow. -D 2013-04-08T15:36:51.699 +C Adding\stests\sto\sthe\smptester\sscripts. +D 2013-04-08T17:57:26.382 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -111,9 +111,9 @@ F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da -F mptest/config02.test 381ec0acbb1e5aab28ca9aa91e17ae9a02bbd2bb +F mptest/config02.test baaee6bba0122f45c895a88c7ac8b62ff77c6b70 F mptest/mptest.c 7048030a410707f77845c0fda9c4a223c7069e0b -F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c +F mptest/multiwrite01.test 16f9bb13640dfe1be1e544d49b16aa9e7503ae2d F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc @@ -1048,7 +1048,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 360473493ec1a7094a2b1c5436f3b70914a6dfdd -R 138dfe64c0fb20cc8b3e6aefe188c7ba +P d77407ae6a9e09dea124dd59b14fdbbc36ed3d4e +R 8e0a5a89df2079b6986569045ce512eb U drh -Z bbdf55a7e5fdae8e0e462019e357a9d2 +Z a3e7b3daf8fd9801ae226170daba9d61 diff --git a/manifest.uuid b/manifest.uuid index 3944faa67f..35124abccb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d77407ae6a9e09dea124dd59b14fdbbc36ed3d4e \ No newline at end of file +e121adceb1e267e032dcbadb74eaa692a12a65bb \ No newline at end of file diff --git a/mptest/config02.test b/mptest/config02.test index 70c3a6cac2..303a5b4d56 100644 --- a/mptest/config02.test +++ b/mptest/config02.test @@ -103,3 +103,11 @@ DELETE FROM pgsz; --wait all SELECT sz FROM pgsz; --match 16384 16384 16384 16384 16384 +PRAGMA auto_vacuum=FULL; +VACUUM; +--source multiwrite01.test +--wait all +PRAGMA auto_vacuum=FULL; +PRAGMA page_size=512; +VACUUM; +--source multiwrite01.test diff --git a/mptest/multiwrite01.test b/mptest/multiwrite01.test index dc71884047..a3cfb0dbb4 100644 --- a/mptest/multiwrite01.test +++ b/mptest/multiwrite01.test @@ -24,6 +24,13 @@ UPDATE t1 SET b='x'||a||'y'; SELECT total(length(b)) FROM t1; --match 247 + SELECT a FROM t1 WHERE b='x17y'; + --match 17 + CREATE INDEX t1b ON t1(b); + SELECT a FROM t1 WHERE b='x17y'; + --match 17 + SELECT a FROM t1 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5; + --match 29 28 27 26 25 --end @@ -49,6 +56,13 @@ UPDATE t2 SET b='x'||a||'y'; SELECT total(length(b)) FROM t2; --match 247 + SELECT a FROM t2 WHERE b='x17y'; + --match 17 + CREATE INDEX t2b ON t2(b); + SELECT a FROM t2 WHERE b='x17y'; + --match 17 + SELECT a FROM t2 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5; + --match 29 28 27 26 25 --end --task 3 @@ -73,6 +87,13 @@ UPDATE t3 SET b='x'||a||'y'; SELECT total(length(b)) FROM t3; --match 247 + SELECT a FROM t3 WHERE b='x17y'; + --match 17 + CREATE INDEX t3b ON t3(b); + SELECT a FROM t3 WHERE b='x17y'; + --match 17 + SELECT a FROM t3 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5; + --match 29 28 27 26 25 --end --task 4 @@ -97,6 +118,13 @@ UPDATE t4 SET b='x'||a||'y'; SELECT total(length(b)) FROM t4; --match 247 + SELECT a FROM t4 WHERE b='x17y'; + --match 17 + CREATE INDEX t4b ON t4(b); + SELECT a FROM t4 WHERE b='x17y'; + --match 17 + SELECT a FROM t4 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5; + --match 29 28 27 26 25 --end --task 5 @@ -121,6 +149,13 @@ UPDATE t5 SET b='x'||a||'y'; SELECT total(length(b)) FROM t5; --match 247 + SELECT a FROM t5 WHERE b='x17y'; + --match 17 + CREATE INDEX t5b ON t5(b); + SELECT a FROM t5 WHERE b='x17y'; + --match 17 + SELECT a FROM t5 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5; + --match 29 28 27 26 25 --end --wait all @@ -134,8 +169,236 @@ SELECT count(*), total(length(b)) FROM t4; --match 64 247 SELECT count(*), total(length(b)) FROM t5; --match 64 247 -DROP TABLE t1; -DROP TABLE t2; -DROP TABLE t3; -DROP TABLE t4; -DROP TABLE t5; + +--task 1 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 5 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 3 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 2 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 4 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--wait all + +--task 5 + DROP INDEX t5b; + --sleep 5 + PRAGMA integrity_check; + --match ok + CREATE INDEX t5b ON t5(b DESC); +--end +--task 3 + DROP INDEX t3b; + --sleep 5 + PRAGMA integrity_check; + --match ok + CREATE INDEX t3b ON t3(b DESC); +--end +--task 1 + DROP INDEX t1b; + --sleep 5 + PRAGMA integrity_check; + --match ok + CREATE INDEX t1b ON t1(b DESC); +--end +--task 2 + DROP INDEX t2b; + --sleep 5 + PRAGMA integrity_check; + --match ok + CREATE INDEX t2b ON t2(b DESC); +--end +--task 4 + DROP INDEX t4b; + --sleep 5 + PRAGMA integrity_check; + --match ok + CREATE INDEX t4b ON t4(b DESC); +--end +--wait all + +--task 1 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 5 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 3 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 2 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 4 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--wait all + +VACUUM; +PRAGMA integrity_check; +--match ok + +--task 1 + UPDATE t1 SET b=randomblob(20000); + --sleep 5 + UPDATE t1 SET b='x'||a||'y'; + SELECT a FROM t1 WHERE b='x63y'; + --match 63 +--end +--task 2 + UPDATE t2 SET b=randomblob(20000); + --sleep 5 + UPDATE t2 SET b='x'||a||'y'; + SELECT a FROM t2 WHERE b='x63y'; + --match 63 +--end +--task 3 + UPDATE t3 SET b=randomblob(20000); + --sleep 5 + UPDATE t3 SET b='x'||a||'y'; + SELECT a FROM t3 WHERE b='x63y'; + --match 63 +--end +--task 4 + UPDATE t4 SET b=randomblob(20000); + --sleep 5 + UPDATE t4 SET b='x'||a||'y'; + SELECT a FROM t4 WHERE b='x63y'; + --match 63 +--end +--task 5 + UPDATE t5 SET b=randomblob(20000); + --sleep 5 + UPDATE t5 SET b='x'||a||'y'; + SELECT a FROM t5 WHERE b='x63y'; + --match 63 +--end +--wait all + +--task 1 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 5 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 3 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 2 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 4 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end From 6adab7a1a870a1af8dfeb18b82957e8e4385b2fa Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Apr 2013 18:58:00 +0000 Subject: [PATCH 112/204] Still more tests using mptester. Added test cases that leave rollback journals that must be recovered by other threads. FossilOrigin-Name: 437b674f00610e2b925dc00ca27b5022e3858994 --- manifest | 18 ++++--- manifest.uuid | 2 +- mptest/config02.test | 7 +++ mptest/crash01.test | 102 +++++++++++++++++++++++++++++++++++++++ mptest/crash02.subtest | 57 ++++++++++++++++++++++ mptest/mptest.c | 20 ++++++-- mptest/multiwrite01.test | 1 + 7 files changed, 193 insertions(+), 14 deletions(-) create mode 100644 mptest/crash01.test create mode 100644 mptest/crash02.subtest diff --git a/manifest b/manifest index b55b1213e9..94bee9a115 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adding\stests\sto\sthe\smptester\sscripts. -D 2013-04-08T17:57:26.382 +C Still\smore\stests\susing\smptester.\s\sAdded\stest\scases\sthat\sleave\srollback\njournals\sthat\smust\sbe\srecovered\sby\sother\sthreads. +D 2013-04-08T18:58:00.322 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -111,9 +111,11 @@ F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da -F mptest/config02.test baaee6bba0122f45c895a88c7ac8b62ff77c6b70 -F mptest/mptest.c 7048030a410707f77845c0fda9c4a223c7069e0b -F mptest/multiwrite01.test 16f9bb13640dfe1be1e544d49b16aa9e7503ae2d +F mptest/config02.test 962913ed2b537d60de4126db7fe54716865cdd22 +F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 +F mptest/crash02.subtest c4647feafa04109e1dc60121a895b6cba8d63ceb +F mptest/mptest.c 76c8cb03f9cc91811907d8f9ccf2a5253bbb229d +F mptest/multiwrite01.test d6cb707062268e65396bd71eb53e9d5edb5519c0 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc @@ -1048,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d77407ae6a9e09dea124dd59b14fdbbc36ed3d4e -R 8e0a5a89df2079b6986569045ce512eb +P e121adceb1e267e032dcbadb74eaa692a12a65bb +R 82dd3ae164ec063cf354b1e10f2e05ba U drh -Z a3e7b3daf8fd9801ae226170daba9d61 +Z f4809a26a66812142b3835cff8ebb4b8 diff --git a/manifest.uuid b/manifest.uuid index 35124abccb..c637c165c7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e121adceb1e267e032dcbadb74eaa692a12a65bb \ No newline at end of file +437b674f00610e2b925dc00ca27b5022e3858994 \ No newline at end of file diff --git a/mptest/config02.test b/mptest/config02.test index 303a5b4d56..e25d94081e 100644 --- a/mptest/config02.test +++ b/mptest/config02.test @@ -15,6 +15,7 @@ PRAGMA page_size=512; PRAGMA mmap_limit=65536; --end --source multiwrite01.test +--source crash02.subtest PRAGMA page_size=1024; VACUUM; CREATE TABLE pgsz(taskid, sz INTEGER); @@ -34,6 +35,7 @@ CREATE TABLE pgsz(taskid, sz INTEGER); INSERT INTO pgsz VALUES(5, eval('PRAGMA page_size')); --end --source multiwrite01.test +--source crash02.subtest --wait all SELECT sz FROM pgsz; --match 1024 1024 1024 1024 1024 @@ -56,6 +58,7 @@ DELETE FROM pgsz; INSERT INTO pgsz VALUES(5, eval('PRAGMA page_size')); --end --source multiwrite01.test +--source crash02.subtest --wait all SELECT sz FROM pgsz; --match 2048 2048 2048 2048 2048 @@ -78,6 +81,7 @@ DELETE FROM pgsz; INSERT INTO pgsz VALUES(5, eval('PRAGMA page_size')); --end --source multiwrite01.test +--source crash02.subtest --wait all SELECT sz FROM pgsz; --match 8192 8192 8192 8192 8192 @@ -100,14 +104,17 @@ DELETE FROM pgsz; INSERT INTO pgsz VALUES(5, eval('PRAGMA page_size')); --end --source multiwrite01.test +--source crash02.subtest --wait all SELECT sz FROM pgsz; --match 16384 16384 16384 16384 16384 PRAGMA auto_vacuum=FULL; VACUUM; --source multiwrite01.test +--source crash02.subtest --wait all PRAGMA auto_vacuum=FULL; PRAGMA page_size=512; VACUUM; --source multiwrite01.test +--source crash02.subtest diff --git a/mptest/crash01.test b/mptest/crash01.test new file mode 100644 index 0000000000..d094d72f29 --- /dev/null +++ b/mptest/crash01.test @@ -0,0 +1,102 @@ +/* Test cases involving incomplete transactions that must be rolled back. +*/ +--task 1 + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + --sleep 1 + INSERT INTO t1 VALUES(1, randomblob(2000)); + INSERT INTO t1 VALUES(2, randomblob(1000)); + --sleep 1 + INSERT INTO t1 SELECT a+2, randomblob(1500) FROM t1; + INSERT INTO t1 SELECT a+4, randomblob(1500) FROM t1; + INSERT INTO t1 SELECT a+8, randomblob(1500) FROM t1; + --sleep 1 + INSERT INTO t1 SELECT a+16, randomblob(1500) FROM t1; + --sleep 1 + INSERT INTO t1 SELECT a+32, randomblob(1500) FROM t1; + SELECT count(*) FROM t1; + --match 64 + SELECT avg(length(b)) FROM t1; + --match 1500.0 + --sleep 2 + UPDATE t1 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t1; + --match 247 + SELECT a FROM t1 WHERE b='x17y'; + --match 17 + CREATE INDEX t1b ON t1(b); + SELECT a FROM t1 WHERE b='x17y'; + --match 17 + SELECT a FROM t1 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5; + --match 29 28 27 26 25 +--end +--wait 1 +--task 2 + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + INSERT INTO t2 SELECT a, b FROM t1; + UPDATE t1 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t2; + --match 247 + SELECT a FROM t2 WHERE b='x17y'; + --match 17 + CREATE INDEX t2b ON t2(b); + SELECT a FROM t2 WHERE b='x17y'; + --match 17 + SELECT a FROM t2 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5; + --match 29 28 27 26 25 +--end +--task 3 + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + INSERT INTO t3 SELECT a, b FROM t1; + UPDATE t1 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t3; + --match 247 + SELECT a FROM t3 WHERE b='x17y'; + --match 17 + CREATE INDEX t3b ON t3(b); + SELECT a FROM t3 WHERE b='x17y'; + --match 17 + SELECT a FROM t3 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5; + --match 29 28 27 26 25 +--end +--task 4 + CREATE TABLE t4(a INTEGER PRIMARY KEY, b); + INSERT INTO t4 SELECT a, b FROM t1; + UPDATE t1 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t4; + --match 247 + SELECT a FROM t4 WHERE b='x17y'; + --match 17 + CREATE INDEX t4b ON t4(b); + SELECT a FROM t4 WHERE b='x17y'; + --match 17 + SELECT a FROM t4 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5; + --match 29 28 27 26 25 +--end +--task 5 + CREATE TABLE t5(a INTEGER PRIMARY KEY, b); + INSERT INTO t5 SELECT a, b FROM t1; + UPDATE t1 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t5; + --match 247 + SELECT a FROM t5 WHERE b='x17y'; + --match 17 + CREATE INDEX t5b ON t5(b); + SELECT a FROM t5 WHERE b='x17y'; + --match 17 + SELECT a FROM t5 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5; + --match 29 28 27 26 25 +--end + +--wait all +/* After the database file has been set up, run the crash2 subscript +** multiple times. */ +--source crash02.subtest +--source crash02.subtest +--source crash02.subtest +--source crash02.subtest +--source crash02.subtest +--source crash02.subtest +--source crash02.subtest +--source crash02.subtest +--source crash02.subtest diff --git a/mptest/crash02.subtest b/mptest/crash02.subtest new file mode 100644 index 0000000000..053a9faf93 --- /dev/null +++ b/mptest/crash02.subtest @@ -0,0 +1,57 @@ +/* +** This script is called from crash01.test and config02.test and perhaps other +** script. After the database file has been set up, make a big rollback +** journal in client 1, then crash client 1. +** Then in the other clients, do an integrity check. +*/ +--task 1 + --sleep 5 + --finish + PRAGMA cache_size=10; + BEGIN; + UPDATE t1 SET b=randomblob(20000); + UPDATE t2 SET b=randomblob(20000); + UPDATE t3 SET b=randomblob(20000); + UPDATE t4 SET b=randomblob(20000); + UPDATE t5 SET b=randomblob(20000); + UPDATE t1 SET b=NULL; + UPDATE t2 SET b=NULL; + UPDATE t3 SET b=NULL; + UPDATE t4 SET b=NULL; + UPDATE t5 SET b=NULL; + --print Task one crashing an incomplete transaction + --exit 1 +--end +--task 2 + SELECT count(*) FROM t1; + --match 64 + --sleep 100 + PRAGMA integrity_check; + --output + --match ok +--end +--task 3 + SELECT count(*) FROM t1; + --match 64 + --sleep 100 + PRAGMA integrity_check; + --output + --match ok +--end +--task 4 + SELECT count(*) FROM t1; + --match 64 + --sleep 100 + PRAGMA integrity_check; + --output + --match ok +--end +--task 5 + SELECT count(*) FROM t1; + --match 64 + --sleep 100 + PRAGMA integrity_check; + --output + --match ok +--end +--wait all diff --git a/mptest/mptest.c b/mptest/mptest.c index ca056f76e6..cf8eb14e1e 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -526,14 +526,14 @@ static int startScript( *pzScript = 0; g.iTimeout = 0; while(1){ - rc = trySql("BEGIN IMMEDIATE"); + rc = trySql("BEGIN EXCLUSIVE"); if( rc==SQLITE_BUSY ){ sqlite3_sleep(10); totalTime += 10; continue; } if( rc!=SQLITE_OK ){ - fatalError("%s\nBEGIN IMMEDIATE", sqlite3_errmsg(g.db)); + fatalError("in startScript: %s", sqlite3_errmsg(g.db)); } if( g.nError || g.nTest ){ runSql("UPDATE counters SET nError=nError+%d, nTest=nTest+%d", @@ -869,13 +869,23 @@ static void runScript( ** Exit this process. If N>0 then exit without shutting down ** SQLite. (In other words, simulate a crash.) */ - if( strcmp(zCmd, "exit")==0 ){ + if( strcmp(zCmd, "exit")==0 && iClient>0 ){ int rc = atoi(azArg[0]); finishScript(iClient, taskId, 1); if( rc==0 ) sqlite3_close(g.db); exit(rc); }else + /* + ** --finish + ** + ** Mark the current task as having finished, even if it is not. + ** This can be used in conjunction with --exit to simulate a crash. + */ + if( strcmp(zCmd, "finish")==0 && iClient>0 ){ + finishScript(iClient, taskId, 1); + }else + /* ** --reset ** @@ -1203,11 +1213,11 @@ int main(int argc, char **argv){ char zTaskName[50]; rc = startScript(iClient, &zScript, &taskId); if( rc==SQLITE_DONE ) break; - if( g.iTrace ) logMessage("begin task %d", taskId); sqlite3_snprintf(sizeof(zTaskName), zTaskName, "client%02d-task-%d", iClient, taskId); + if( g.iTrace ) logMessage("begin %s", zTaskName); runScript(iClient, taskId, zScript, zTaskName); - if( g.iTrace ) logMessage("end task %d", taskId); + if( g.iTrace ) logMessage("end %s", zTaskName); finishScript(iClient, taskId, 0); sqlite3_sleep(10); } diff --git a/mptest/multiwrite01.test b/mptest/multiwrite01.test index a3cfb0dbb4..dc7d62b8f9 100644 --- a/mptest/multiwrite01.test +++ b/mptest/multiwrite01.test @@ -402,3 +402,4 @@ PRAGMA integrity_check; ORDER BY t3.a LIMIT 7 --match 45 46 47 48 49 50 51 --end +--wait all From f90e50f60bd01af442a9b68eb5d808ef6faa4ca7 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Apr 2013 19:13:48 +0000 Subject: [PATCH 113/204] In mptester, handle startScript no-change commits using a retry rather than grabbing an exclusive transaction. FossilOrigin-Name: ae8bb24bfb014bdcb8aaa8159ddae0cd376afe99 --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 13 ++++++++----- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 94bee9a115..535fa7ba3a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Still\smore\stests\susing\smptester.\s\sAdded\stest\scases\sthat\sleave\srollback\njournals\sthat\smust\sbe\srecovered\sby\sother\sthreads. -D 2013-04-08T18:58:00.322 +C In\smptester,\shandle\sstartScript\sno-change\scommits\susing\sa\sretry\srather\nthan\sgrabbing\san\sexclusive\stransaction. +D 2013-04-08T19:13:48.566 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da F mptest/config02.test 962913ed2b537d60de4126db7fe54716865cdd22 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest c4647feafa04109e1dc60121a895b6cba8d63ceb -F mptest/mptest.c 76c8cb03f9cc91811907d8f9ccf2a5253bbb229d +F mptest/mptest.c e31bb60d3de960aa0f7d08af7ae59c6dfecc552b F mptest/multiwrite01.test d6cb707062268e65396bd71eb53e9d5edb5519c0 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e121adceb1e267e032dcbadb74eaa692a12a65bb -R 82dd3ae164ec063cf354b1e10f2e05ba +P 437b674f00610e2b925dc00ca27b5022e3858994 +R 609e885f5484356f11d8798a6d9ad296 U drh -Z f4809a26a66812142b3835cff8ebb4b8 +Z f55865a5fe0b6f2c658b1f5dce1b30e8 diff --git a/manifest.uuid b/manifest.uuid index c637c165c7..b7c4fb39fc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -437b674f00610e2b925dc00ca27b5022e3858994 \ No newline at end of file +ae8bb24bfb014bdcb8aaa8159ddae0cd376afe99 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index cf8eb14e1e..f14a640e8c 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -526,7 +526,7 @@ static int startScript( *pzScript = 0; g.iTimeout = 0; while(1){ - rc = trySql("BEGIN EXCLUSIVE"); + rc = trySql("BEGIN IMMEDIATE"); if( rc==SQLITE_BUSY ){ sqlite3_sleep(10); totalTime += 10; @@ -546,8 +546,8 @@ static int startScript( sqlite3_finalize(pStmt); if( rc==SQLITE_ROW ){ runSql("DELETE FROM client WHERE id=%d", iClient); - runSql("COMMIT"); g.iTimeout = DEFAULT_TIMEOUT; + runSql("COMMIT TRANSACTION;"); return SQLITE_DONE; } pStmt = prepareSql( @@ -557,15 +557,15 @@ static int startScript( rc = sqlite3_step(pStmt); if( rc==SQLITE_ROW ){ int n = sqlite3_column_bytes(pStmt, 0); - *pzScript = sqlite3_malloc( n+ 1 ); + *pzScript = sqlite3_malloc(n+1); strcpy(*pzScript, (const char*)sqlite3_column_text(pStmt, 0)); *pTaskId = taskId = sqlite3_column_int(pStmt, 1); sqlite3_finalize(pStmt); runSql("UPDATE task" " SET starttime=strftime('%%Y-%%m-%%d %%H:%%M:%%f','now')" " WHERE id=%d;", taskId); - runSql("COMMIT;"); g.iTimeout = DEFAULT_TIMEOUT; + runSql("COMMIT TRANSACTION;"); return SQLITE_OK; } sqlite3_finalize(pStmt); @@ -576,7 +576,10 @@ static int startScript( sqlite3_close(g.db); exit(1); } - runSql("COMMIT;"); + while( trySql("COMMIT")==SQLITE_BUSY ){ + sqlite3_sleep(10); + totalTime += 10; + } sqlite3_sleep(100); totalTime += 100; continue; From 188d488409c9d931bb8b59eb86956ec6130017e7 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Apr 2013 20:47:49 +0000 Subject: [PATCH 114/204] Disable the use of memory-mapped I/O if the SQLITE_DISABLE_MMAP macro is defined. Automatically define this macro for OpenBSD and QNX. Other systems are likely to be added to the disabled list over time. FossilOrigin-Name: 8a4314a398c6c6efb25e6993c826b7e60d9bb1f9 --- manifest | 34 +++++++++++++++++----------------- manifest.uuid | 2 +- src/ctime.c | 3 +++ src/os.c | 12 ++++++++++++ src/pager.c | 20 ++++++++++++++++---- src/pragma.c | 3 +++ src/sqliteInt.h | 8 ++++++++ src/sqliteLimit.h | 4 ++++ src/test_config.c | 6 ++++++ test/mmap1.test | 6 ++++-- test/mmap2.test | 4 ++++ test/pageropt.test | 7 ++++++- test/wal.test | 1 + test/wal5.test | 1 + 14 files changed, 86 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 535fa7ba3a..fdbb616c1d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\smptester,\shandle\sstartScript\sno-change\scommits\susing\sa\sretry\srather\nthan\sgrabbing\san\sexclusive\stransaction. -D 2013-04-08T19:13:48.566 +C Disable\sthe\suse\sof\smemory-mapped\sI/O\sif\sthe\sSQLITE_DISABLE_MMAP\smacro\sis\ndefined.\s\sAutomatically\sdefine\sthis\smacro\sfor\sOpenBSD\sand\sQNX.\s\sOther\nsystems\sare\slikely\sto\sbe\sadded\sto\sthe\sdisabled\slist\sover\stime. +D 2013-04-08T20:47:49.018 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -133,7 +133,7 @@ F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c 16658a257bc6a3ca8d8961f574cf61a57e4d6faf +F src/ctime.c 61a53c7e0db5e7ba60cf23b73deaf6779aa715d3 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 @@ -163,18 +163,18 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d +F src/os.c ca679b293a6233327e418fd1dde2cd5db3e90932 F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 6d584190db2ed442e5c849b7f73dcfbbac7c032e F src/os_win.c 3265df8c762e0b8caf8d02b3352fa8c22e91ae6b -F src/pager.c 2e68df46d4086027cb6b527d47a6dedbf1a6b7ec +F src/pager.c eb6d272f4dc1ade4b8f664599cd54a9c95cb2729 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c 682e97f3e3b77fd6c9b569eabfbf4a14c987aca3 +F src/pragma.c 8bbc8c8697d13f76394e592dcecafce3491ed8ba F src/prepare.c 743e484233c51109666d402f470523553b41797c F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -185,8 +185,8 @@ F src/shell.c 319b7791cee6c763b60fde1b590bfaf62613cf37 F src/sqlite.h.in 5fdc866a06ac82fd3c6ccb9f59a100151a36568c F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 -F src/sqliteInt.h 1664dc5ad6f8d4dab871416628aa3271044d66c0 -F src/sqliteLimit.h 1097d2c541147d303e66515040067c6bfcf8cf21 +F src/sqliteInt.h 1c517a89cfdba28084093607ec79af70359c7f9b +F src/sqliteLimit.h 3ea5c07b370b8033c30f66515ccf4e875c252429 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 @@ -203,7 +203,7 @@ F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de F src/test_btree.c 5b89601dcb42a33ba8b820a6b763cc9cb48bac16 -F src/test_config.c ab6605a41f2020912d5e44b8dd4b9fb21bd75ac0 +F src/test_config.c b6f5d680f0cf03d0ae9d6165bd2e447d9b356e4a F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c 8f786bfd0ad48030cf2a06fb1f050e9c60a150d7 @@ -637,8 +637,8 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 146a755bb382677b6f9993c61ddc7732b157d730 -F test/mmap2.test 62dbb5d718e66d654d232116c5a2d96e26a071a5 +F test/mmap1.test f064a9025136cd688c982ccdae81c7b4fac95724 +F test/mmap2.test a5ba639f90b5fc487400a49e158e14e465943e98 F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 @@ -663,7 +663,7 @@ F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f F test/pagerfault.test bba948be0564254b823ec6ca1fd95fbbfb0839a3 F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 -F test/pageropt.test 32cb5a6ed7ccddf8e8c842cb44240bd9340223ce +F test/pageropt.test d6b695e6c12290c406150fc51eec32878d48df4c F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 @@ -961,11 +961,11 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 -F test/wal.test bbd98ac09d8fbcaa80ec233757056eb1bbb7cd95 +F test/wal.test b934b757c4c3776b1e93140a9febc9ae068f9d90 F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c -F test/wal5.test f4d0aee6a2cf09e326ed2459011d396b4fdf661a +F test/wal5.test 13c980d4d8d2d6b558361f19bed4ab31d41a2c7c F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test b3ee739fe8f7586aaebdc2367f477ebcf3e3b034 @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 437b674f00610e2b925dc00ca27b5022e3858994 -R 609e885f5484356f11d8798a6d9ad296 +P ae8bb24bfb014bdcb8aaa8159ddae0cd376afe99 +R 7e782b1e3841573f5a2d86903a4acfcf U drh -Z f55865a5fe0b6f2c658b1f5dce1b30e8 +Z a7d6d975495e51a139482ed48c14db14 diff --git a/manifest.uuid b/manifest.uuid index b7c4fb39fc..387ecfb3dc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ae8bb24bfb014bdcb8aaa8159ddae0cd376afe99 \ No newline at end of file +8a4314a398c6c6efb25e6993c826b7e60d9bb1f9 \ No newline at end of file diff --git a/src/ctime.c b/src/ctime.c index ee6d4cbe2e..450a84045d 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -66,6 +66,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_DISABLE_LFS "DISABLE_LFS", #endif +#ifdef SQLITE_DISABLE_MMAP + "DISABLE_MMAP", +#endif #ifdef SQLITE_ENABLE_ATOMIC_WRITE "ENABLE_ATOMIC_WRITE", #endif diff --git a/src/os.c b/src/os.c index 2ba857e2aa..90cbe5492d 100644 --- a/src/os.c +++ b/src/os.c @@ -141,6 +141,8 @@ int sqlite3OsShmMap( return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } +#if !defined(SQLITE_DISABLE_MMAP) +/* The real implementation of xFetch and xUnfetch */ int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ DO_OS_MALLOC_TEST(id); return id->pMethods->xFetch(id, iOff, iAmt, pp); @@ -148,6 +150,16 @@ int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ return id->pMethods->xUnfetch(id, iOff, p); } +#else +/* No-op stubs to use when memory-mapped I/O is disabled */ +int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + *pp = 0; + return SQLITE_OK; +} +int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ + return SQLITE_OK; +} +#endif /* ** The next group of routines are convenience wrappers around the diff --git a/src/pager.c b/src/pager.c index ae2f10069f..e3ac49cd47 100644 --- a/src/pager.c +++ b/src/pager.c @@ -770,6 +770,16 @@ static const unsigned char aJournalMagic[] = { # define MEMDB pPager->memDb #endif +/* +** The macro USEFETCH is true if we are allowed to use the xFetch and xUnfetch +** interfaces to access the database using memory-mapped I/O. +*/ +#ifdef SQLITE_DISABLE_MMAP +# define USEFETCH(x) 0 +#else +# define USEFETCH(x) ((x)->bUseFetch) +#endif + /* ** The maximum legal page number is (2^31 - 1). */ @@ -3090,7 +3100,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); if( rc!=SQLITE_OK || changed ){ pager_reset(pPager); - if( pPager->bUseFetch ) sqlite3OsUnfetch(pPager->fd, 0, 0); + if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); } return rc; @@ -3356,6 +3366,7 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ ** Invoke SQLITE_FCNTL_MMAP_LIMIT based on the current value of mxMmap. */ static void pagerFixMaplimit(Pager *pPager){ +#if !defined(SQLITE_DISABLE_MMAP) sqlite3_file *fd = pPager->fd; if( isOpen(fd) ){ sqlite3_int64 mx; @@ -3363,6 +3374,7 @@ static void pagerFixMaplimit(Pager *pPager){ mx = pPager->mxMmap; sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT, &mx); } +#endif } /* @@ -5042,7 +5054,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( !pPager->tempFile && ( pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0 - || pPager->bUseFetch + || USEFETCH(pPager) )){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous @@ -5085,7 +5097,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** In this case there may exist a Pager.pMap mapping that appears ** to be the right size but is not actually valid. Avoid this ** possibility by unmapping the db here. */ - if( pPager->bUseFetch ){ + if( USEFETCH(pPager) ){ sqlite3OsUnfetch(pPager->fd, 0, 0); } } @@ -5199,7 +5211,7 @@ int sqlite3PagerAcquire( ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY ** flag was specified by the caller. And so long as the db is not a ** temporary or in-memory database. */ - const int bMmapOk = (pgno!=1 && pPager->bUseFetch + const int bMmapOk = (pgno!=1 && USEFETCH(pPager) && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY)) ); diff --git a/src/pragma.c b/src/pragma.c index 3527c0974c..340e62d0dd 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -774,6 +774,9 @@ void sqlite3Pragma( } mx = -1; if( sqlite3_file_control(db,zDb,SQLITE_FCNTL_MMAP_LIMIT,&mx)==SQLITE_OK ){ +#if defined(SQLITE_DISABLE_MMAP) + mx = 0; +#endif returnSingleInt(pParse, "mmap_limit", mx); } }else diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d894051790..a528e37e44 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -539,6 +539,14 @@ extern const int sqlite3one; # define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&7)==0) #endif +/* +** Disable MMAP on platforms where it is not supported +*/ +#if defined(__OpenBSD__) || defined(__QNXNTO__) +# undef SQLITE_DISABLE_MMAP +# define SQLITE_DISABLE_MMAP 1 +#endif + /* ** An instance of the following structure is used to store the busy-handler diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index d3a3b4ac95..d8f748e6b2 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -210,6 +210,10 @@ /* ** Default maximum size of memory used by xFetch in the VFS. */ +#ifdef SQLITE_DISABLE_MMAP +# undef SQLITE_DEFAULT_MMAP_LIMIT +# define SQLITE_DEFAULT_MMAP_LIMIT 0 +#endif #ifdef __APPLE__ # include # if TARGET_OS_IPHONE diff --git a/src/test_config.c b/src/test_config.c index f3c2f61162..39608dbb64 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -87,6 +87,12 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "lfs", "1", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_DISABLE_MMAP + Tcl_SetVar2(interp, "sqlite_options", "mmap", "0", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "mmap", "1", TCL_GLOBAL_ONLY); +#endif + #if 1 /* def SQLITE_MEMDEBUG */ Tcl_SetVar2(interp, "sqlite_options", "memdebug", "1", TCL_GLOBAL_ONLY); #else diff --git a/test/mmap1.test b/test/mmap1.test index 417a0756b1..9618109933 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -12,6 +12,10 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !mmap { + finish_test + return +} source $testdir/lock_common.tcl set testprefix mmap1 @@ -311,5 +315,3 @@ foreach {tn1 mmap1 mmap2} { finish_test - - diff --git a/test/mmap2.test b/test/mmap2.test index 33830a6943..2c80a0eb7f 100644 --- a/test/mmap2.test +++ b/test/mmap2.test @@ -25,6 +25,10 @@ if {$::tcl_platform(platform)!="unix" || [test_syscall defaultvfs] != "unix"} { finish_test return } +ifcapable !mmap { + finish_test + return +} db close sqlite3_shutdown diff --git a/test/pageropt.test b/test/pageropt.test index 10d3d4917e..a734e39a49 100644 --- a/test/pageropt.test +++ b/test/pageropt.test @@ -87,12 +87,17 @@ do_test pageropt-1.4 { # But if the other thread modifies the database, then the cache # must refill. # +ifcapable mmap { + set x [expr {[permutation]=="nommap" ? 6 : 1}] +} else { + set x 6 +} do_test pageropt-1.5 { db2 eval {CREATE TABLE t2(y)} pagercount_sql { SELECT hex(x) FROM t1 } -} [list [expr {[permutation]=="nommap" ? 6 : 1}] 0 0 $blobcontent] +} [list $x 0 0 $blobcontent] do_test pageropt-1.6 { pagercount_sql { SELECT hex(x) FROM t1 diff --git a/test/wal.test b/test/wal.test index 0539192d25..b997155540 100644 --- a/test/wal.test +++ b/test/wal.test @@ -729,6 +729,7 @@ do_test wal-11.9 { sqlite3_wal db test.db set nWal 39 if {[permutation]=="nommap"} {set nWal 37} +ifcapable !mmap {set nWal 37} do_test wal-11.10 { execsql { PRAGMA cache_size = 10; diff --git a/test/wal5.test b/test/wal5.test index ebd1dbfc31..4a6309886b 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -243,6 +243,7 @@ foreach {testprefix do_wal_checkpoint} { # start of the checkpoint, even though page 2 cannot be written. set nDb 2 if {[permutation]=="no-mmap"} {set nDb 1} + ifcapable !mmap {set nDb 1} do_test 2.3.$tn.8 { file_page_counts } [list $nDb 4 2 4] } From 6e0b6d52da6a6245c97a40997a95f8cb14fed502 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 9 Apr 2013 16:19:20 +0000 Subject: [PATCH 115/204] Add extra #ifndef statements in os_unix.c and os_win.c to make sure the memory mapped I/O really is disabled when SQLITE_DISABLE_MMAP is set. FossilOrigin-Name: c1e2523c9051782569291fff998140f7e0b70b6d --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 12 ++++++++++++ src/os_win.c | 12 ++++++++++++ 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index fdbb616c1d..7f8d4dc923 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sthe\suse\sof\smemory-mapped\sI/O\sif\sthe\sSQLITE_DISABLE_MMAP\smacro\sis\ndefined.\s\sAutomatically\sdefine\sthis\smacro\sfor\sOpenBSD\sand\sQNX.\s\sOther\nsystems\sare\slikely\sto\sbe\sadded\sto\sthe\sdisabled\slist\sover\stime. -D 2013-04-08T20:47:49.018 +C Add\sextra\s#ifndef\sstatements\sin\sos_unix.c\sand\sos_win.c\sto\smake\ssure\sthe\nmemory\smapped\sI/O\sreally\sis\sdisabled\swhen\sSQLITE_DISABLE_MMAP\sis\sset. +D 2013-04-09T16:19:20.624 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,8 +166,8 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c ca679b293a6233327e418fd1dde2cd5db3e90932 F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 6d584190db2ed442e5c849b7f73dcfbbac7c032e -F src/os_win.c 3265df8c762e0b8caf8d02b3352fa8c22e91ae6b +F src/os_unix.c a6e9caadd8956a39153d1bd920e459211a281388 +F src/os_win.c 3aec291c40048653c00a153fee96d3a442ab85ee F src/pager.c eb6d272f4dc1ade4b8f664599cd54a9c95cb2729 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ae8bb24bfb014bdcb8aaa8159ddae0cd376afe99 -R 7e782b1e3841573f5a2d86903a4acfcf +P 8a4314a398c6c6efb25e6993c826b7e60d9bb1f9 +R 815c6131bff16243fe333d79ef346ae3 U drh -Z a7d6d975495e51a139482ed48c14db14 +Z 73d68bfce668b2e7f104132473fd4e42 diff --git a/manifest.uuid b/manifest.uuid index 387ecfb3dc..991d418bcd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8a4314a398c6c6efb25e6993c826b7e60d9bb1f9 \ No newline at end of file +c1e2523c9051782569291fff998140f7e0b70b6d \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 1d4c8ad348..a65f894eb8 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3117,6 +3117,7 @@ static int unixRead( ); #endif +#if !defined(SQLITE_DISABLE_MMAP) /* Deal with as much of this read request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ @@ -3131,6 +3132,7 @@ static int unixRead( offset += nCopy; } } +#endif got = seekAndRead(pFile, offset, pBuf, amt); if( got==amt ){ @@ -3236,6 +3238,7 @@ static int unixWrite( } #endif +#if !defined(SQLITE_DISABLE_MMAP) /* Deal with as much of this write request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ @@ -3250,6 +3253,7 @@ static int unixWrite( offset += nCopy; } } +#endif while( amt>0 && (wrote = seekAndWrite(pFile, offset, pBuf, amt))>0 ){ amt -= wrote; @@ -4525,12 +4529,14 @@ static int unixShmUnmap( */ static void unixUnmapfile(unixFile *pFd){ assert( pFd->nFetchOut==0 ); +#ifndef SQLITE_DISABLE_MMAP if( pFd->pMapRegion ){ osMunmap(pFd->pMapRegion, pFd->mmapOrigsize); pFd->pMapRegion = 0; pFd->mmapSize = 0; pFd->mmapOrigsize = 0; } +#endif } /* @@ -4546,6 +4552,7 @@ static int unixGetPagesize(void){ #endif } +#ifndef SQLITE_DISABLE_MMAP /* ** Attempt to set the size of the memory mapping maintained by file ** descriptor pFd to nNew bytes. Any existing mapping is discarded. @@ -4630,6 +4637,7 @@ static void unixRemapfile( pFd->pMapRegion = (void *)pNew; pFd->mmapSize = pFd->mmapOrigsize = nNew; } +#endif /* ** Memory map or remap the file opened by file-descriptor pFd (if the file @@ -4651,6 +4659,7 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ i64 nMap = nByte; int rc; +#ifndef SQLITE_DISABLE_MMAP assert( nMap>=0 || pFd->nFetchOut==0 ); if( pFd->nFetchOut>0 ) return SQLITE_OK; @@ -4673,6 +4682,7 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ unixUnmapfile(pFd); } } +#endif return SQLITE_OK; } @@ -4693,6 +4703,7 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ unixFile *pFd = (unixFile *)fd; /* The underlying database file */ *pp = 0; +#ifndef SQLITE_DISABLE_MMAP if( pFd->mmapLimit>0 ){ if( pFd->pMapRegion==0 ){ int rc = unixMapfile(pFd, -1); @@ -4703,6 +4714,7 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ pFd->nFetchOut++; } } +#endif return SQLITE_OK; } diff --git a/src/os_win.c b/src/os_win.c index a7966171b5..220ebac268 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2148,6 +2148,7 @@ static int winRead( SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); +#if !defined(SQLITE_DISABLE_MMAP) /* Deal with as much of this read request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ @@ -2162,6 +2163,7 @@ static int winRead( offset += nCopy; } } +#endif #if SQLITE_OS_WINCE if( seekWinFile(pFile, offset) ){ @@ -2212,6 +2214,7 @@ static int winWrite( OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); +#if !defined(SQLITE_DISABLE_MMAP) /* Deal with as much of this write request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ @@ -2226,6 +2229,7 @@ static int winWrite( offset += nCopy; } } +#endif #if SQLITE_OS_WINCE rc = seekWinFile(pFile, offset); @@ -2321,6 +2325,7 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ "winTruncate2", pFile->zPath); } +#if !defined(SQLITE_DISABLE_MMAP) /* If the file was truncated to a size smaller than the currently ** mapped region, reduce the effective mapping size as well. SQLite will ** use read() and write() to access data beyond this point from now on. @@ -2328,6 +2333,7 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ if( pFile->pMapRegion && nBytemmapSize ){ pFile->mmapSize = nByte; } +#endif OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); return rc; @@ -3517,6 +3523,7 @@ shmpage_out: */ static int winUnmapfile(winFile *pFile){ assert( pFile!=0 ); +#if !defined(SQLITE_DISABLE_MMAP) if( pFile->pMapRegion ){ if( !osUnmapViewOfFile(pFile->pMapRegion) ){ pFile->lastErrno = osGetLastError(); @@ -3535,9 +3542,11 @@ static int winUnmapfile(winFile *pFile){ } pFile->hMap = NULL; } +#endif return SQLITE_OK; } +#if !defined(SQLITE_DISABLE_MMAP) /* ** Memory map or remap the file opened by file-descriptor pFd (if the file ** is already mapped, the existing mapping is replaced by the new). Or, if @@ -3625,6 +3634,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ return SQLITE_OK; } +#endif /* !defined(SQLITE_DISABLE_MMAP) */ /* ** If possible, return a pointer to a mapping of file fd starting at offset @@ -3642,6 +3652,7 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ winFile *pFd = (winFile*)fd; /* The underlying database file */ *pp = 0; +#if !defined(SQLITE_DISABLE_MMAP) if( pFd->mmapLimit>0 ){ if( pFd->pMapRegion==0 ){ int rc = winMapfile(pFd, -1); @@ -3652,6 +3663,7 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ pFd->nFetchOut++; } } +#endif return SQLITE_OK; } From ab755ac81a0c4fd17c8de2f777b757ca50070b58 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 9 Apr 2013 18:36:36 +0000 Subject: [PATCH 116/204] Change the rollback journal so that it invokes sqlite3_log() make a record of a recovery, just as the WAL journal does. FossilOrigin-Name: 7cd3f6cd3a39ed1c4bbf9e3508824150632c5bd9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- mptest/mptest.c | 6 +++++- src/pager.c | 9 ++++++++- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 7f8d4dc923..982e5065c0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\s#ifndef\sstatements\sin\sos_unix.c\sand\sos_win.c\sto\smake\ssure\sthe\nmemory\smapped\sI/O\sreally\sis\sdisabled\swhen\sSQLITE_DISABLE_MMAP\sis\sset. -D 2013-04-09T16:19:20.624 +C Change\sthe\srollback\sjournal\sso\sthat\sit\sinvokes\ssqlite3_log()\smake\sa\srecord\nof\sa\srecovery,\sjust\sas\sthe\sWAL\sjournal\sdoes. +D 2013-04-09T18:36:36.048 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da F mptest/config02.test 962913ed2b537d60de4126db7fe54716865cdd22 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest c4647feafa04109e1dc60121a895b6cba8d63ceb -F mptest/mptest.c e31bb60d3de960aa0f7d08af7ae59c6dfecc552b +F mptest/mptest.c c9fc81464dcd2de18ceb5317c5e1b075cb3b9fce F mptest/multiwrite01.test d6cb707062268e65396bd71eb53e9d5edb5519c0 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -168,7 +168,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c a6e9caadd8956a39153d1bd920e459211a281388 F src/os_win.c 3aec291c40048653c00a153fee96d3a442ab85ee -F src/pager.c eb6d272f4dc1ade4b8f664599cd54a9c95cb2729 +F src/pager.c cc60816e677528e4c2e23bbf56ee5c81f4cdfe58 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 8a4314a398c6c6efb25e6993c826b7e60d9bb1f9 -R 815c6131bff16243fe333d79ef346ae3 +P c1e2523c9051782569291fff998140f7e0b70b6d +R 0d15a7f4fe4e9fd889905fd047909cd0 U drh -Z 73d68bfce668b2e7f104132473fd4e42 +Z 5a68f7be27490e18f34eef73cb3a90fe diff --git a/manifest.uuid b/manifest.uuid index 991d418bcd..62d2bcb7d9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c1e2523c9051782569291fff998140f7e0b70b6d \ No newline at end of file +7cd3f6cd3a39ed1c4bbf9e3508824150632c5bd9 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index f14a640e8c..7a8bb08c8b 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -323,7 +323,11 @@ static void sqlErrorCallback(void *pArg, int iErrCode, const char *zMsg){ UNUSED_PARAMETER(pArg); if( (iErrCode&0xff)==SQLITE_SCHEMA && g.iTrace<3 ) return; if( g.iTimeout==0 && (iErrCode&0xff)==SQLITE_BUSY && g.iTrace<3 ) return; - errorMessage("(errcode=%d) %s", iErrCode, zMsg); + if( iErrCode==SQLITE_OK ){ + logMessage("(info) %s", zMsg); + }else{ + errorMessage("(errcode=%d) %s", iErrCode, zMsg); + } } /* diff --git a/src/pager.c b/src/pager.c index e3ac49cd47..d35d91ac12 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2658,6 +2658,7 @@ static int pager_playback(Pager *pPager, int isHot){ int res = 1; /* Value returned by sqlite3OsAccess() */ char *zMaster = 0; /* Name of master journal file if any */ int needPagerReset; /* True to reset page prior to first page rollback */ + int nPlayback = 0; /* Total number of pages restored from journal */ /* Figure out how many records are in the journal. Abort early if ** the journal is empty. @@ -2758,7 +2759,9 @@ static int pager_playback(Pager *pPager, int isHot){ needPagerReset = 0; } rc = pager_playback_one_page(pPager,&pPager->journalOff,0,1,0); - if( rc!=SQLITE_OK ){ + if( rc==SQLITE_OK ){ + nPlayback++; + }else{ if( rc==SQLITE_DONE ){ pPager->journalOff = szJ; break; @@ -2828,6 +2831,10 @@ end_playback: rc = pager_delmaster(pPager, zMaster); testcase( rc!=SQLITE_OK ); } + if( isHot && nPlayback ){ + sqlite3_log(SQLITE_OK, "Recovered %d pages from %s", + nPlayback, pPager->zJournal); + } /* The Pager.sectorSize variable may have been updated while rolling ** back a journal created by a process with a different sector size From 7de96f7fc1e825d6d69dc0475ef412a1f6a82758 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 9 Apr 2013 20:04:09 +0000 Subject: [PATCH 117/204] Limit integrity_check output to 10 lines in mptester scripts. FossilOrigin-Name: 67ee0dc0f4de5dc2ae92fb98932f1fdda3e76bdf --- manifest | 14 +++++++------- manifest.uuid | 2 +- mptest/crash02.subtest | 8 ++++---- mptest/multiwrite01.test | 12 ++++++------ 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 982e5065c0..3bcf945534 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\srollback\sjournal\sso\sthat\sit\sinvokes\ssqlite3_log()\smake\sa\srecord\nof\sa\srecovery,\sjust\sas\sthe\sWAL\sjournal\sdoes. -D 2013-04-09T18:36:36.048 +C Limit\sintegrity_check\soutput\sto\s10\slines\sin\smptester\sscripts. +D 2013-04-09T20:04:09.979 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -113,9 +113,9 @@ F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da F mptest/config02.test 962913ed2b537d60de4126db7fe54716865cdd22 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 -F mptest/crash02.subtest c4647feafa04109e1dc60121a895b6cba8d63ceb +F mptest/crash02.subtest 86641730725895da7baf6443bc26aa1c8e781711 F mptest/mptest.c c9fc81464dcd2de18ceb5317c5e1b075cb3b9fce -F mptest/multiwrite01.test d6cb707062268e65396bd71eb53e9d5edb5519c0 +F mptest/multiwrite01.test 8fa273e97284edaa855f0b0e99d78216e204be15 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c1e2523c9051782569291fff998140f7e0b70b6d -R 0d15a7f4fe4e9fd889905fd047909cd0 +P 7cd3f6cd3a39ed1c4bbf9e3508824150632c5bd9 +R c57dbdff2c71d3b29909dfda1d0fd94f U drh -Z 5a68f7be27490e18f34eef73cb3a90fe +Z 71d9517d85d0a37449eba7c0abca70d4 diff --git a/manifest.uuid b/manifest.uuid index 62d2bcb7d9..0d698f714e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7cd3f6cd3a39ed1c4bbf9e3508824150632c5bd9 \ No newline at end of file +67ee0dc0f4de5dc2ae92fb98932f1fdda3e76bdf \ No newline at end of file diff --git a/mptest/crash02.subtest b/mptest/crash02.subtest index 053a9faf93..320f788a70 100644 --- a/mptest/crash02.subtest +++ b/mptest/crash02.subtest @@ -26,7 +26,7 @@ SELECT count(*) FROM t1; --match 64 --sleep 100 - PRAGMA integrity_check; + PRAGMA integrity_check(10); --output --match ok --end @@ -34,7 +34,7 @@ SELECT count(*) FROM t1; --match 64 --sleep 100 - PRAGMA integrity_check; + PRAGMA integrity_check(10); --output --match ok --end @@ -42,7 +42,7 @@ SELECT count(*) FROM t1; --match 64 --sleep 100 - PRAGMA integrity_check; + PRAGMA integrity_check(10); --output --match ok --end @@ -50,7 +50,7 @@ SELECT count(*) FROM t1; --match 64 --sleep 100 - PRAGMA integrity_check; + PRAGMA integrity_check(10); --output --match ok --end diff --git a/mptest/multiwrite01.test b/mptest/multiwrite01.test index dc7d62b8f9..7fb35a25a7 100644 --- a/mptest/multiwrite01.test +++ b/mptest/multiwrite01.test @@ -225,35 +225,35 @@ SELECT count(*), total(length(b)) FROM t5; --task 5 DROP INDEX t5b; --sleep 5 - PRAGMA integrity_check; + PRAGMA integrity_check(10); --match ok CREATE INDEX t5b ON t5(b DESC); --end --task 3 DROP INDEX t3b; --sleep 5 - PRAGMA integrity_check; + PRAGMA integrity_check(10); --match ok CREATE INDEX t3b ON t3(b DESC); --end --task 1 DROP INDEX t1b; --sleep 5 - PRAGMA integrity_check; + PRAGMA integrity_check(10); --match ok CREATE INDEX t1b ON t1(b DESC); --end --task 2 DROP INDEX t2b; --sleep 5 - PRAGMA integrity_check; + PRAGMA integrity_check(10); --match ok CREATE INDEX t2b ON t2(b DESC); --end --task 4 DROP INDEX t4b; --sleep 5 - PRAGMA integrity_check; + PRAGMA integrity_check(10); --match ok CREATE INDEX t4b ON t4(b DESC); --end @@ -312,7 +312,7 @@ SELECT count(*), total(length(b)) FROM t5; --wait all VACUUM; -PRAGMA integrity_check; +PRAGMA integrity_check(10); --match ok --task 1 From 65eb64765c6b573848028d7a68077e8582bd9677 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 10 Apr 2013 02:56:24 +0000 Subject: [PATCH 118/204] Correct test numbering on several rtree tests. FossilOrigin-Name: 0adf8a3a4f2ca370568b35c9e8ce0e18f18701a2 --- ext/rtree/rtree1.test | 1 + ext/rtree/rtree5.test | 2 +- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ext/rtree/rtree1.test b/ext/rtree/rtree1.test index e3c7d68e82..275b13264f 100644 --- a/ext/rtree/rtree1.test +++ b/ext/rtree/rtree1.test @@ -17,6 +17,7 @@ if {![info exists testdir]} { } source [file join [file dirname [info script]] rtree_util.tcl] source $testdir/tester.tcl +set testprefix rtree1 # Test plan: # diff --git a/ext/rtree/rtree5.test b/ext/rtree/rtree5.test index 8990772356..8ff90b0cb7 100644 --- a/ext/rtree/rtree5.test +++ b/ext/rtree/rtree5.test @@ -61,7 +61,7 @@ do_test rtree5-1.9 { do_test rtree5-1.10 { execsql { SELECT (1<<31)-5, (1<<31)-1, -1*(1<<31), -1*(1<<31)+5 } } {2147483643 2147483647 -2147483648 -2147483643} -do_test rtree5-1.10 { +do_test rtree5-1.11 { execsql { INSERT INTO t1 VALUES(2, (1<<31)-5, (1<<31)-1, -1*(1<<31), -1*(1<<31)+5) } diff --git a/manifest b/manifest index 3bcf945534..6ddc91d8f5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Limit\sintegrity_check\soutput\sto\s10\slines\sin\smptester\sscripts. -D 2013-04-09T20:04:09.979 +C Correct\stest\snumbering\son\sseveral\srtree\stests. +D 2013-04-10T02:56:24.509 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -85,11 +85,11 @@ F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 757abea591d4ff67c0ff4e8f9776aeda86b18c14 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e -F ext/rtree/rtree1.test e474a2b5eff231496dbd073fe67e5fbaf7f444c9 +F ext/rtree/rtree1.test cf679265ecafff494a768ac9c2f43a70915a6290 F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba F ext/rtree/rtree3.test a494da55c30ee0bc9b01a91c80c81b387b22d2dc F ext/rtree/rtree4.test c8fe384f60ebd49540a5fecc990041bf452eb6e0 -F ext/rtree/rtree5.test 9a229678a00f40e6aedb40cb3a07ec5444af892c +F ext/rtree/rtree5.test 6a510494f12454bf57ef28f45bc7764ea279431e F ext/rtree/rtree6.test 3ff9113b4a872fa935309e3511cd9b7cdb4d2472 F ext/rtree/rtree7.test 1fa710b9e6bf997a0c1a537b81be7bb6fded1971 F ext/rtree/rtree8.test 9772e16da71e17e02bdebf0a5188590f289ab37d @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 7cd3f6cd3a39ed1c4bbf9e3508824150632c5bd9 -R c57dbdff2c71d3b29909dfda1d0fd94f -U drh -Z 71d9517d85d0a37449eba7c0abca70d4 +P 67ee0dc0f4de5dc2ae92fb98932f1fdda3e76bdf +R 1390797dd2d0e0c300030d714045b447 +U mistachkin +Z 8596d7b58bb179eaa78f2319025347e8 diff --git a/manifest.uuid b/manifest.uuid index 0d698f714e..ed3822bd21 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -67ee0dc0f4de5dc2ae92fb98932f1fdda3e76bdf \ No newline at end of file +0adf8a3a4f2ca370568b35c9e8ce0e18f18701a2 \ No newline at end of file From 21c9b6bdc9dc939c113628142be2482227384f8c Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 10 Apr 2013 03:06:43 +0000 Subject: [PATCH 119/204] Fix typo in name of mptest executable files in clean targets. FossilOrigin-Name: 4c7d9e1ed8722e0c75b5fdbeffb9030ccef0bd64 --- Makefile.in | 2 +- main.mk | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Makefile.in b/Makefile.in index 0b702def79..486333157e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -955,7 +955,7 @@ clean: rm -f shell.c sqlite3ext.h rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c rm -f sqlite-*-output.vsix - rm -f mptest mptest.exe + rm -f mptester mptester.exe distclean: clean rm -f config.log config.status libtool Makefile sqlite3.pc diff --git a/main.mk b/main.mk index 7ab2bca2ba..2ec009a11d 100644 --- a/main.mk +++ b/main.mk @@ -630,4 +630,4 @@ clean: rm -f shell.c sqlite3ext.h rm -f sqlite3_analyzer sqlite3_analyzer.exe sqlite3_analyzer.c rm -f sqlite-*-output.vsix - rm -f mptest mptest.exe + rm -f mptester mptester.exe diff --git a/manifest b/manifest index 6ddc91d8f5..19b2e38f48 100644 --- a/manifest +++ b/manifest @@ -1,7 +1,7 @@ -C Correct\stest\snumbering\son\sseveral\srtree\stests. -D 2013-04-10T02:56:24.509 +C Fix\stypo\sin\sname\sof\smptest\sexecutable\sfiles\sin\sclean\stargets. +D 2013-04-10T03:06:43.090 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 +F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc a244e5b5ba2493625db95b4d46b79201ea868bfe F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 @@ -103,7 +103,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk 1c26f3cc57d7c357da5218ae2928b7ca5acfecde +F main.mk dcb05737414488383369b93e33cf1a1484c4b6e6 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 67ee0dc0f4de5dc2ae92fb98932f1fdda3e76bdf -R 1390797dd2d0e0c300030d714045b447 +P 0adf8a3a4f2ca370568b35c9e8ce0e18f18701a2 +R 87c959156377d2c80b90d9a959edeb9c U mistachkin -Z 8596d7b58bb179eaa78f2319025347e8 +Z f06e039212bcf2d4f3c2b172131c9afb diff --git a/manifest.uuid b/manifest.uuid index ed3822bd21..f78573fb87 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0adf8a3a4f2ca370568b35c9e8ce0e18f18701a2 \ No newline at end of file +4c7d9e1ed8722e0c75b5fdbeffb9030ccef0bd64 \ No newline at end of file From 4c5298f352c92a723a37d4ccab701f6c28e9f680 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 10 Apr 2013 12:01:21 +0000 Subject: [PATCH 120/204] Use symbolic names for tasks in mptester. FossilOrigin-Name: f0d95afc73f8dbce8943dceb4a14b7de650c8823 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- mptest/crash02.subtest | 14 +++++--------- mptest/mptest.c | 37 +++++++++++++++++++++++++++---------- mptest/multiwrite01.test | 10 +++++----- 5 files changed, 47 insertions(+), 34 deletions(-) diff --git a/manifest b/manifest index 19b2e38f48..a97a348927 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypo\sin\sname\sof\smptest\sexecutable\sfiles\sin\sclean\stargets. -D 2013-04-10T03:06:43.090 +C Use\ssymbolic\snames\sfor\stasks\sin\smptester. +D 2013-04-10T12:01:21.074 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -113,9 +113,9 @@ F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da F mptest/config02.test 962913ed2b537d60de4126db7fe54716865cdd22 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 -F mptest/crash02.subtest 86641730725895da7baf6443bc26aa1c8e781711 -F mptest/mptest.c c9fc81464dcd2de18ceb5317c5e1b075cb3b9fce -F mptest/multiwrite01.test 8fa273e97284edaa855f0b0e99d78216e204be15 +F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 +F mptest/mptest.c d5228be2db65634783b302423704b3928284adb5 +F mptest/multiwrite01.test 81fbc17657964889b60750bd7bbb1deffe8f4d42 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 0adf8a3a4f2ca370568b35c9e8ce0e18f18701a2 -R 87c959156377d2c80b90d9a959edeb9c -U mistachkin -Z f06e039212bcf2d4f3c2b172131c9afb +P 4c7d9e1ed8722e0c75b5fdbeffb9030ccef0bd64 +R f44444aacae5f7c52822e030959d44b9 +U drh +Z f26dfaa28d7c4e704618af941a7b95f6 diff --git a/manifest.uuid b/manifest.uuid index f78573fb87..e0711c6d56 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4c7d9e1ed8722e0c75b5fdbeffb9030ccef0bd64 \ No newline at end of file +f0d95afc73f8dbce8943dceb4a14b7de650c8823 \ No newline at end of file diff --git a/mptest/crash02.subtest b/mptest/crash02.subtest index 320f788a70..86f64ddbe7 100644 --- a/mptest/crash02.subtest +++ b/mptest/crash02.subtest @@ -4,7 +4,7 @@ ** journal in client 1, then crash client 1. ** Then in the other clients, do an integrity check. */ ---task 1 +--task 1 leave-hot-journal --sleep 5 --finish PRAGMA cache_size=10; @@ -22,36 +22,32 @@ --print Task one crashing an incomplete transaction --exit 1 --end ---task 2 +--task 2 integrity_check-2 SELECT count(*) FROM t1; --match 64 --sleep 100 PRAGMA integrity_check(10); - --output --match ok --end ---task 3 +--task 3 integrity_check-3 SELECT count(*) FROM t1; --match 64 --sleep 100 PRAGMA integrity_check(10); - --output --match ok --end ---task 4 +--task 4 integrity_check-4 SELECT count(*) FROM t1; --match 64 --sleep 100 PRAGMA integrity_check(10); - --output --match ok --end ---task 5 +--task 5 integrity_check-5 SELECT count(*) FROM t1; --match 64 --sleep 100 PRAGMA integrity_check(10); - --output --match ok --end --wait all diff --git a/mptest/mptest.c b/mptest/mptest.c index 7a8bb08c8b..6de32b0a05 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -520,7 +520,8 @@ static void evalFunc( static int startScript( int iClient, /* The client number */ char **pzScript, /* Write task script here */ - int *pTaskId /* Write task number here */ + int *pTaskId, /* Write task number here */ + char **pzTaskName /* Name of the task */ ){ sqlite3_stmt *pStmt = 0; int taskId; @@ -555,7 +556,7 @@ static int startScript( return SQLITE_DONE; } pStmt = prepareSql( - "SELECT script, id FROM task" + "SELECT script, id, name FROM task" " WHERE client=%d AND starttime IS NULL" " ORDER BY id LIMIT 1", iClient); rc = sqlite3_step(pStmt); @@ -564,6 +565,7 @@ static int startScript( *pzScript = sqlite3_malloc(n+1); strcpy(*pzScript, (const char*)sqlite3_column_text(pStmt, 0)); *pTaskId = taskId = sqlite3_column_int(pStmt, 1); + *pzTaskName = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 2)); sqlite3_finalize(pStmt); runSql("UPDATE task" " SET starttime=strftime('%%Y-%%m-%%d %%H:%%M:%%f','now')" @@ -804,6 +806,14 @@ static void waitForClient(int iClient, int iTimeout, char *zErrPrefix){ } } +/* Return a pointer to the tail of a filename +*/ +static char *filenameTail(char *z){ + int i, j; + for(i=j=0; z[i]; i++) if( z[i]=='/' ) j = i+1; + return z+j; +} + /* Maximum number of arguments to a --command */ #define MX_ARG 2 @@ -1039,16 +1049,23 @@ static void runScript( int iTarget = atoi(azArg[0]); int iEnd; char *zTask; + char *zTName; iEnd = findEnd(zScript+ii+len, &lineno); if( iTarget<0 ){ errorMessage("line %d of %s: bad client number: %d", prevLine, zFilename, iTarget); }else{ zTask = sqlite3_mprintf("%.*s", iEnd, zScript+ii+len); + if( nArg>1 ){ + zTName = sqlite3_mprintf("%s", azArg[1]); + }else{ + zTName = sqlite3_mprintf("%s:%d", filenameTail(zFilename), prevLine); + } startClient(iTarget); - runSql("INSERT INTO task(client,script)" - " VALUES(%d,'%q')", iTarget, zTask); + runSql("INSERT INTO task(client,script,name)" + " VALUES(%d,'%q',%Q)", iTarget, zTask, zTName); sqlite3_free(zTask); + sqlite3_free(zTName); } iEnd += tokenLength(zScript+ii+len+iEnd, &lineno); len += iEnd; @@ -1217,15 +1234,14 @@ int main(int argc, char **argv){ if( n>0 ) unrecognizedArguments(argv[0], n, argv+2); if( g.iTrace ) logMessage("start-client"); while(1){ - char zTaskName[50]; - rc = startScript(iClient, &zScript, &taskId); + char *zTaskName = 0; + rc = startScript(iClient, &zScript, &taskId, &zTaskName); if( rc==SQLITE_DONE ) break; - sqlite3_snprintf(sizeof(zTaskName), zTaskName, "client%02d-task-%d", - iClient, taskId); - if( g.iTrace ) logMessage("begin %s", zTaskName); + if( g.iTrace ) logMessage("begin %s (%d)", zTaskName, taskId); runScript(iClient, taskId, zScript, zTaskName); - if( g.iTrace ) logMessage("end %s", zTaskName); + if( g.iTrace ) logMessage("end %s (%d)", zTaskName, taskId); finishScript(iClient, taskId, 0); + sqlite3_free(zTaskName); sqlite3_sleep(10); } if( g.iTrace ) logMessage("end-client"); @@ -1239,6 +1255,7 @@ int main(int argc, char **argv){ runSql( "CREATE TABLE task(\n" " id INTEGER PRIMARY KEY,\n" + " name TEXT,\n" " client INTEGER,\n" " starttime DATE,\n" " endtime DATE,\n" diff --git a/mptest/multiwrite01.test b/mptest/multiwrite01.test index 7fb35a25a7..9b3ccb8473 100644 --- a/mptest/multiwrite01.test +++ b/mptest/multiwrite01.test @@ -2,7 +2,7 @@ ** This script sets up five different tasks all writing and updating ** the database at the same time, but each in its own table. */ ---task 1 +--task 1 build-t1 DROP TABLE IF EXISTS t1; CREATE TABLE t1(a INTEGER PRIMARY KEY, b); --sleep 1 @@ -34,7 +34,7 @@ --end ---task 2 +--task 2 build-t2 DROP TABLE IF EXISTS t2; CREATE TABLE t2(a INTEGER PRIMARY KEY, b); --sleep 1 @@ -65,7 +65,7 @@ --match 29 28 27 26 25 --end ---task 3 +--task 3 build-t3 DROP TABLE IF EXISTS t3; CREATE TABLE t3(a INTEGER PRIMARY KEY, b); --sleep 1 @@ -96,7 +96,7 @@ --match 29 28 27 26 25 --end ---task 4 +--task 4 build-t4 DROP TABLE IF EXISTS t4; CREATE TABLE t4(a INTEGER PRIMARY KEY, b); --sleep 1 @@ -127,7 +127,7 @@ --match 29 28 27 26 25 --end ---task 5 +--task 5 build-t5 DROP TABLE IF EXISTS t5; CREATE TABLE t5(a INTEGER PRIMARY KEY, b); --sleep 1 From 56282a5bc75945ef9953f0cab3faff8b86af69f0 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 10 Apr 2013 16:13:38 +0000 Subject: [PATCH 121/204] Add the sqlite3_strglob() interface. FossilOrigin-Name: 41d6ff32a6dd1311dc3aabe5156335a64a886919 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/func.c | 7 +++++++ src/sqlite.h.in | 15 +++++++++++++++ 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a97a348927..5efbde65d6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\ssymbolic\snames\sfor\stasks\sin\smptester. -D 2013-04-10T12:01:21.074 +C Add\sthe\ssqlite3_strglob()\sinterface. +D 2013-04-10T16:13:38.085 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -139,7 +139,7 @@ F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c e16942bd5c8a868ac53287886464a5ed0e72b179 -F src/func.c 48987c025d69399f59a1c2a553cea5da41bf105d +F src/func.c d3fdcff9274bc161152e67ed3f626841c247f4b9 F src/global.c d2494a1cea8f66a2cab8258449df07f8f0ae6330 F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4 F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 @@ -182,7 +182,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c 319b7791cee6c763b60fde1b590bfaf62613cf37 -F src/sqlite.h.in 5fdc866a06ac82fd3c6ccb9f59a100151a36568c +F src/sqlite.h.in 90b4c427a6d67b15a469cfd907405de516e36e10 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 1c517a89cfdba28084093607ec79af70359c7f9b @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 4c7d9e1ed8722e0c75b5fdbeffb9030ccef0bd64 -R f44444aacae5f7c52822e030959d44b9 +P f0d95afc73f8dbce8943dceb4a14b7de650c8823 +R 98c7ddae1a722b08cd7989e28c30adec U drh -Z f26dfaa28d7c4e704618af941a7b95f6 +Z 0b357dfedb1b0f83bb0de889b4353615 diff --git a/manifest.uuid b/manifest.uuid index e0711c6d56..2be4ede130 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f0d95afc73f8dbce8943dceb4a14b7de650c8823 \ No newline at end of file +41d6ff32a6dd1311dc3aabe5156335a64a886919 \ No newline at end of file diff --git a/src/func.c b/src/func.c index 0fce95904f..e40fdad55a 100644 --- a/src/func.c +++ b/src/func.c @@ -693,6 +693,13 @@ static int patternCompare( return *zString==0; } +/* +** The sqlite3_strglob() interface. +*/ +int sqlite3_strglob(const char *zGlobPattern, const char *zString){ + return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, 0)==0; +} + /* ** Count the number of times that the LIKE operator (or GLOB which is ** just a variation of LIKE) gets called. This is used for testing diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 74cecbcf22..8638e5453e 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6837,6 +6837,21 @@ int sqlite3_unlock_notify( int sqlite3_stricmp(const char *, const char *); int sqlite3_strnicmp(const char *, const char *, int); +/* +** CAPI3REF: String Globbing +* +** ^The [sqlite3_strglob(P,X)] interface returns zero if string X matches +** the glob pattern P, and it returns non-zero if string X does not match +** the glob pattern P. ^The definition of glob pattern matching used in +** [sqlite3_strglob(P,X) is the same as for the "X GLOB P" operator in the +** SQL dialect used by SQLite. ^The sqlite3_strglob(P,X) function is case +** sensitive. +** +** Note that this routine returns zero on a match and non-zero if the strings +** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. +*/ +int sqlite3_strglob(const char *zGlob, const char *zStr); + /* ** CAPI3REF: Error Logging Interface ** From d040e764427bfa02bed83e5312a95d2877b182ff Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 10 Apr 2013 23:48:37 +0000 Subject: [PATCH 122/204] Add new primary error codes SQLITE_NOTICE and SQLITE_WARNING for use with sqlite3_log(). Add new extended error codes SQLITE_NOTICE_RECOVER_WAL and SQLITE_NOTICE_RECOVER_ROLLBACK to use with sqlite3_log() messages that occur on each recovery. FossilOrigin-Name: be7d2c5482c41baf000d7fb5dccc31b974e91064 --- manifest | 28 +++++----- manifest.uuid | 2 +- src/pager.c | 2 +- src/sqlite.h.in | 4 ++ src/test1.c | 133 +++++++++++++++++++++++++----------------------- src/test2.c | 54 +++++--------------- src/test3.c | 40 +++------------ src/test4.c | 31 ++--------- src/test7.c | 31 ++--------- src/wal.c | 3 +- test/wal.test | 2 +- 11 files changed, 119 insertions(+), 211 deletions(-) diff --git a/manifest b/manifest index 5efbde65d6..1059d70fbc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssqlite3_strglob()\sinterface. -D 2013-04-10T16:13:38.085 +C Add\snew\sprimary\serror\scodes\sSQLITE_NOTICE\sand\sSQLITE_WARNING\sfor\suse\swith\nsqlite3_log().\s\sAdd\snew\sextended\serror\scodes\sSQLITE_NOTICE_RECOVER_WAL\sand\nSQLITE_NOTICE_RECOVER_ROLLBACK\sto\suse\swith\ssqlite3_log()\smessages\sthat\soccur\non\seach\srecovery. +D 2013-04-10T23:48:37.567 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,7 +168,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c a6e9caadd8956a39153d1bd920e459211a281388 F src/os_win.c 3aec291c40048653c00a153fee96d3a442ab85ee -F src/pager.c cc60816e677528e4c2e23bbf56ee5c81f4cdfe58 +F src/pager.c 28f45e60d9a173368872d6e688e7a848c3926344 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -182,7 +182,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c 319b7791cee6c763b60fde1b590bfaf62613cf37 -F src/sqlite.h.in 90b4c427a6d67b15a469cfd907405de516e36e10 +F src/sqlite.h.in f606903594d314d1b63e677affb1a03e3c9a49c7 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 1c517a89cfdba28084093607ec79af70359c7f9b @@ -190,13 +190,13 @@ F src/sqliteLimit.h 3ea5c07b370b8033c30f66515ccf4e875c252429 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 -F src/test1.c 26e66b839f42c2eed6833f9023e0098f0d863f35 -F src/test2.c 4178056dd1e7d70f954ad8a1e3edb71a2a784daf -F src/test3.c 3c3c2407fa6ec7a19e24ae23f7cb439d0275a60d -F src/test4.c bf9fa9bece01de08e6f5e02314e4af5c13590dfa +F src/test1.c 6784fdacb35c33ba564ef749b62c4718fe515484 +F src/test2.c 29e7154112f7448d64204e8d31179cf497ecf425 +F src/test3.c 96aed72a8e1d542fed127e3e8350ae357712fa82 +F src/test4.c cea2c55110241e4674e66d476d29c914627999f5 F src/test5.c a6d1ac55ac054d0b2b8f37b5e655b6c92645a013 F src/test6.c a437f76f9874d2563352a7e6cd0d43217663c220 -F src/test7.c 2e0781754905c8adc3268d8f0967e7633af58843 +F src/test7.c f4b894b7931f8cf9f5cbf37cfa0727703f526a40 F src/test8.c 58ea1d9698f3947e4662107ef98f429e84ae20e0 F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad @@ -255,7 +255,7 @@ F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c 4fad64071ae120c25f39dcac572d716b9cadeb7f F src/vdbetrace.c a22263ab47f6ba4fcd176515cec1e732866b25f0 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 -F src/wal.c 94b5fed2df988fb12f5bf17256e2840e56957a85 +F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b F src/where.c 4ad2329c439a30ddb915a780f6f80bdffafe3a64 @@ -961,7 +961,7 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 -F test/wal.test b934b757c4c3776b1e93140a9febc9ae068f9d90 +F test/wal.test 20fb8d0f8e65e50eb675994cc3a4f0e692fd4faf F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f0d95afc73f8dbce8943dceb4a14b7de650c8823 -R 98c7ddae1a722b08cd7989e28c30adec +P 41d6ff32a6dd1311dc3aabe5156335a64a886919 +R d4b63b2237f99e77bcb422f5c3bb48b3 U drh -Z 0b357dfedb1b0f83bb0de889b4353615 +Z 6b5c5fede4cd1a096bfdb2b852b5aff1 diff --git a/manifest.uuid b/manifest.uuid index 2be4ede130..088566df4f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -41d6ff32a6dd1311dc3aabe5156335a64a886919 \ No newline at end of file +be7d2c5482c41baf000d7fb5dccc31b974e91064 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index d35d91ac12..6c3376a836 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2832,7 +2832,7 @@ end_playback: testcase( rc!=SQLITE_OK ); } if( isHot && nPlayback ){ - sqlite3_log(SQLITE_OK, "Recovered %d pages from %s", + sqlite3_log(SQLITE_NOTICE_RECOVER_ROLLBACK, "recovered %d pages from %s", nPlayback, pPager->zJournal); } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 8638e5453e..349e36ea92 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -420,6 +420,8 @@ int sqlite3_exec( #define SQLITE_FORMAT 24 /* Auxiliary database format error */ #define SQLITE_RANGE 25 /* 2nd parameter to sqlite3_bind out of range */ #define SQLITE_NOTADB 26 /* File opened that is not a database file */ +#define SQLITE_NOTICE 27 /* Notifications from sqlite3_log() */ +#define SQLITE_WARNING 28 /* Warnings from sqlite3_log() */ #define SQLITE_ROW 100 /* sqlite3_step() has another row ready */ #define SQLITE_DONE 101 /* sqlite3_step() has finished executing */ /* end-of-error-codes */ @@ -490,6 +492,8 @@ int sqlite3_exec( #define SQLITE_CONSTRAINT_TRIGGER (SQLITE_CONSTRAINT | (7<<8)) #define SQLITE_CONSTRAINT_UNIQUE (SQLITE_CONSTRAINT | (8<<8)) #define SQLITE_CONSTRAINT_VTAB (SQLITE_CONSTRAINT | (9<<8)) +#define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) +#define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) /* ** CAPI3REF: Flags For File Open Operations diff --git a/src/test1.c b/src/test1.c index fb340f92d0..63d014cc17 100644 --- a/src/test1.c +++ b/src/test1.c @@ -116,71 +116,76 @@ int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb){ const char *sqlite3TestErrorName(int rc){ const char *zName = 0; - switch( rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; - case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; - case SQLITE_CONSTRAINT_FOREIGNKEY: - zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; - case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; - case SQLITE_CONSTRAINT_PRIMARYKEY: - zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; - case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; - case SQLITE_CONSTRAINT_COMMITHOOK: - zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; - case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; - case SQLITE_CONSTRAINT_FUNCTION: zName = "SQLITE_CONSTRAINT_FUNCTION";break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; - case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; - case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; - case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; - case SQLITE_ROW: zName = "SQLITE_ROW"; break; - case SQLITE_DONE: zName = "SQLITE_DONE"; break; - case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; - case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; - case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; - case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; - case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; - case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; - case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; - case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; - case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; - case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; - case SQLITE_IOERR_BLOCKED: zName = "SQLITE_IOERR_BLOCKED"; break; - case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; - case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; - case SQLITE_IOERR_CHECKRESERVEDLOCK: - zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; - case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; - case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; - case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; - case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; - case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; - default: zName = "SQLITE_Unknown"; break; + int i; + for(i=0; i<2 && zName==0; i++, rc &= 0xff){ + switch( rc ){ + case SQLITE_OK: zName = "SQLITE_OK"; break; + case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; + case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; + case SQLITE_PERM: zName = "SQLITE_PERM"; break; + case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; + case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; + case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; + case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; + case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; + case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; + case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; + case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; + case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; + case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; + case SQLITE_FULL: zName = "SQLITE_FULL"; break; + case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; + case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; + case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; + case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; + case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; + case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; + case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; + case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; + case SQLITE_CONSTRAINT_FOREIGNKEY: + zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; + case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; + case SQLITE_CONSTRAINT_PRIMARYKEY: + zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; + case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; + case SQLITE_CONSTRAINT_COMMITHOOK: + zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; + case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; + case SQLITE_CONSTRAINT_FUNCTION: zName = "SQLITE_CONSTRAINT_FUNCTION";break; + case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; + case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; + case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; + case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; + case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; + case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; + case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; + case SQLITE_ROW: zName = "SQLITE_ROW"; break; + case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; + case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; + case SQLITE_DONE: zName = "SQLITE_DONE"; break; + case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; + case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; + case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; + case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; + case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; + case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; + case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; + case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; + case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; + case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; + case SQLITE_IOERR_BLOCKED: zName = "SQLITE_IOERR_BLOCKED"; break; + case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; + case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; + case SQLITE_IOERR_CHECKRESERVEDLOCK: + zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; + case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; + case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; + case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; + case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; + case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; + } } + if( zName==0 ) zName = "SQLITE_Unknown"; return zName; } #define t1ErrorName sqlite3TestErrorName diff --git a/src/test2.c b/src/test2.c index 8acdf6f4ef..3faec12619 100644 --- a/src/test2.c +++ b/src/test2.c @@ -19,35 +19,7 @@ #include #include -/* -** Interpret an SQLite error number -*/ -static char *errorName(int rc){ - char *zName; - switch( rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - default: zName = "SQLITE_Unknown"; break; - } - return zName; -} +extern const char *sqlite3TestErrorName(int rc); /* ** Page size and reserved size used for testing. @@ -87,7 +59,7 @@ static int pager_open( SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MAIN_DB, pager_test_reiniter); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } sqlite3PagerSetCachesize(pPager, nPage); @@ -119,7 +91,7 @@ static int pager_close( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerClose(pPager); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -146,7 +118,7 @@ static int pager_rollback( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerRollback(pPager); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -173,12 +145,12 @@ static int pager_commit( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerCommitPhaseOne(pPager, 0, 0); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } rc = sqlite3PagerCommitPhaseTwo(pPager); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -205,7 +177,7 @@ static int pager_stmt_begin( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerOpenSavepoint(pPager, 1); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -233,7 +205,7 @@ static int pager_stmt_rollback( rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, 0); sqlite3PagerSavepoint(pPager, SAVEPOINT_RELEASE, 0); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -260,7 +232,7 @@ static int pager_stmt_commit( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_RELEASE, 0); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -353,7 +325,7 @@ static int page_get( rc = sqlite3PagerGet(pPager, pgno, &pPage); } if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf),zBuf,"%p",pPage); @@ -507,7 +479,7 @@ static int page_write( pPage = (DbPage *)sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerWrite(pPage); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } pData = sqlite3PagerGetData(pPage); @@ -556,7 +528,7 @@ static int fake_big_file( (SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_DB), 0 ); if( rc ){ - Tcl_AppendResult(interp, "open failed: ", errorName(rc), 0); + Tcl_AppendResult(interp, "open failed: ", sqlite3TestErrorName(rc), 0); sqlite3_free(zFile); return TCL_ERROR; } @@ -566,7 +538,7 @@ static int fake_big_file( sqlite3OsCloseFree(fd); sqlite3_free(zFile); if( rc ){ - Tcl_AppendResult(interp, "write failed: ", errorName(rc), 0); + Tcl_AppendResult(interp, "write failed: ", sqlite3TestErrorName(rc), 0); return TCL_ERROR; } return TCL_OK; diff --git a/src/test3.c b/src/test3.c index e460c42e46..ecc3a543be 100644 --- a/src/test3.c +++ b/src/test3.c @@ -19,31 +19,7 @@ #include #include -/* -** Interpret an SQLite error number -*/ -static char *errorName(int rc){ - char *zName; - switch( rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - default: zName = "SQLITE_Unknown"; break; - } - return zName; -} +extern const char *sqlite3TestErrorName(int rc); /* ** A bogus sqlite3 connection structure for use in the btree @@ -89,7 +65,7 @@ static int btree_open( SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MAIN_DB); sqlite3_free(zFilename); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } sqlite3BtreeSetCacheSize(pBt, nCache); @@ -119,7 +95,7 @@ static int btree_close( pBt = sqlite3TestTextToPtr(argv[1]); rc = sqlite3BtreeClose(pBt); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } nRefSqlite3--; @@ -156,7 +132,7 @@ static int btree_begin_transaction( rc = sqlite3BtreeBeginTrans(pBt, 1); sqlite3BtreeLeave(pBt); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -250,7 +226,7 @@ static int btree_cursor( sqlite3BtreeLeave(pBt); if( rc ){ ckfree((char *)pCur); - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf), zBuf,"%p", pCur); @@ -285,7 +261,7 @@ static int btree_close_cursor( sqlite3BtreeLeave(pBt); ckfree((char *)pCur); if( rc ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } return SQLITE_OK; @@ -319,7 +295,7 @@ static int btree_next( rc = sqlite3BtreeNext(pCur, &res); sqlite3BtreeLeave(pCur->pBtree); if( rc ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf),zBuf,"%d",res); @@ -354,7 +330,7 @@ static int btree_first( rc = sqlite3BtreeFirst(pCur, &res); sqlite3BtreeLeave(pCur->pBtree); if( rc ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf),zBuf,"%d",res); diff --git a/src/test4.c b/src/test4.c index 5b4c5a1c17..83c56d3d7e 100644 --- a/src/test4.c +++ b/src/test4.c @@ -20,6 +20,8 @@ #include #include +extern const char *sqlite3TestErrorName(int rc); + /* ** Each thread is controlled by an instance of the following ** structure. @@ -372,34 +374,7 @@ static int tcl_thread_result( return TCL_ERROR; } thread_wait(&threadset[i]); - switch( threadset[i].rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; - case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; - case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; - case SQLITE_ROW: zName = "SQLITE_ROW"; break; - case SQLITE_DONE: zName = "SQLITE_DONE"; break; - default: zName = "SQLITE_Unknown"; break; - } + zName = sqlite3TestErrorName(threadset[i].rc); Tcl_AppendResult(interp, zName, 0); return TCL_OK; } diff --git a/src/test7.c b/src/test7.c index 852cd1db5e..707d2c89af 100644 --- a/src/test7.c +++ b/src/test7.c @@ -376,6 +376,8 @@ static int tcl_client_colname( return TCL_OK; } +extern const char *sqlite3TestErrorName(int rc); + /* ** Usage: client_result ID ** @@ -403,34 +405,7 @@ static int tcl_client_result( return TCL_ERROR; } client_wait(&threadset[i]); - switch( threadset[i].rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; - case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; - case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; - case SQLITE_ROW: zName = "SQLITE_ROW"; break; - case SQLITE_DONE: zName = "SQLITE_DONE"; break; - default: zName = "SQLITE_Unknown"; break; - } + zName = sqlite3TestErrorName(threadset[i].rc); Tcl_AppendResult(interp, zName, 0); return TCL_OK; } diff --git a/src/wal.c b/src/wal.c index 881f0a4c78..e642ea2105 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1207,7 +1207,8 @@ finished: ** checkpointing the log file. */ if( pWal->hdr.nPage ){ - sqlite3_log(SQLITE_OK, "Recovered %d frames from WAL file %s", + sqlite3_log(SQLITE_NOTICE_RECOVER_WAL, + "recovered %d frames from WAL file %s", pWal->hdr.mxFrame, pWal->zWalName ); } diff --git a/test/wal.test b/test/wal.test index b997155540..66000d0951 100644 --- a/test/wal.test +++ b/test/wal.test @@ -1515,7 +1515,7 @@ do_test wal-23.3 { set nPage [expr 2+$AUTOVACUUM] do_test wal-23.4 { set ::log -} [list SQLITE_OK "Recovered $nPage frames from WAL file $walfile"] +} [list SQLITE_NOTICE "recovered $nPage frames from WAL file $walfile"] ifcapable autovacuum { From 08d418904a124936ff274240795a471cc6c1af77 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 11 Apr 2013 00:09:44 +0000 Subject: [PATCH 123/204] Enhance multi-process tester integration with the Win32 API. FossilOrigin-Name: 0fdc743583c67a3a017b9ad812c62a5104b9aee7 --- manifest | 14 ++++++------- manifest.uuid | 2 +- mptest/mptest.c | 54 +++++++++++++++++++++++++++---------------------- 3 files changed, 38 insertions(+), 32 deletions(-) diff --git a/manifest b/manifest index 1059d70fbc..6d3e751585 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\sprimary\serror\scodes\sSQLITE_NOTICE\sand\sSQLITE_WARNING\sfor\suse\swith\nsqlite3_log().\s\sAdd\snew\sextended\serror\scodes\sSQLITE_NOTICE_RECOVER_WAL\sand\nSQLITE_NOTICE_RECOVER_ROLLBACK\sto\suse\swith\ssqlite3_log()\smessages\sthat\soccur\non\seach\srecovery. -D 2013-04-10T23:48:37.567 +C Enhance\smulti-process\stester\sintegration\swith\sthe\sWin32\sAPI. +D 2013-04-11T00:09:44.820 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da F mptest/config02.test 962913ed2b537d60de4126db7fe54716865cdd22 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 -F mptest/mptest.c d5228be2db65634783b302423704b3928284adb5 +F mptest/mptest.c 8b6d7878addccbb9a1b28d9dbf2b0b516a3279f1 F mptest/multiwrite01.test 81fbc17657964889b60750bd7bbb1deffe8f4d42 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 41d6ff32a6dd1311dc3aabe5156335a64a886919 -R d4b63b2237f99e77bcb422f5c3bb48b3 -U drh -Z 6b5c5fede4cd1a096bfdb2b852b5aff1 +P be7d2c5482c41baf000d7fb5dccc31b974e91064 +R 255ee63faf4906e3f01e118bd1c7af53 +U mistachkin +Z 56dc3349791505b3e20ff09949e0bcd7 diff --git a/manifest.uuid b/manifest.uuid index 088566df4f..557333d3d0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -be7d2c5482c41baf000d7fb5dccc31b974e91064 \ No newline at end of file +0fdc743583c67a3a017b9ad812c62a5104b9aee7 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 6de32b0a05..0a8a8cedbd 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -36,7 +36,8 @@ #include "sqlite3.h" #include #if defined(_WIN32) -# include +# define WIN32_LEAN_AND_MEAN +# include #else # include #endif @@ -45,6 +46,13 @@ #include #include +/* The suffix to append to the child command lines, if any */ +#if defined(_WIN32) +# define CMDLINE_SUFFIX "" +#else +# define CMDLINE_SUFFIX "&" +#endif + /* Mark a parameter as unused to suppress compiler warnings */ #define UNUSED_PARAMETER(x) (void)x @@ -617,39 +625,37 @@ static int finishScript(int iClient, int taskId, int bShutdown){ static void startClient(int iClient){ runSql("INSERT OR IGNORE INTO client VALUES(%d,0)", iClient); if( sqlite3_changes(g.db) ){ -#if !defined(_WIN32) char *zSys; int rc; zSys = sqlite3_mprintf( - "%s \"%s\" --client %d --trace %d %s%s&", + "%s \"%s\" --client %d --trace %d %s%s%s", g.argv0, g.zDbFile, iClient, g.iTrace, g.bSqlTrace ? "--sqltrace " : "", - g.bSync ? "--sync " : "" + g.bSync ? "--sync " : "", + CMDLINE_SUFFIX ); +#if !defined(_WIN32) rc = system(zSys); if( rc ) errorMessage("system() fails with error code %d", rc); - sqlite3_free(zSys); -#endif -#if defined(_WIN32) - char *argv[10]; - char zClient[20]; - char zTrace[20]; - argv[0] = g.argv0; - argv[1] = g.zDbFile; - argv[2] = "--client"; - sqlite3_snprintf(sizeof(zClient),zClient,"%d",iClient); - argv[3] = zClient; - argv[4] = "--trace"; - sqlite3_snprintf(sizeof(zTrace),zTrace,"%d",g.iTrace); - argv[5] = zTrace; - if( g.bSqlTrace ){ - argv[6] = "--sqltrace"; - argv[7] = 0; - }else{ - argv[6] = 0; +#else + { + STARTUPINFOA startupInfo; + PROCESS_INFORMATION processInfo; + memset(&startupInfo, 0, sizeof(startupInfo)); + startupInfo.cb = sizeof(startupInfo); + memset(&processInfo, 0, sizeof(processInfo)); + rc = CreateProcessA(NULL, zSys, NULL, NULL, FALSE, 0, NULL, NULL, + &startupInfo, &processInfo); + if( rc ){ + CloseHandle(processInfo.hThread); + CloseHandle(processInfo.hProcess); + }else{ + errorMessage("CreateProcessA() fails with error code %lu", + GetLastError()); + } } - _spawnv(_P_NOWAIT, g.argv0, (const char*const*)argv); #endif + sqlite3_free(zSys); } } From fbc7e8845d6c3a0b140306d46c3454ca6ac45288 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 11 Apr 2013 01:16:15 +0000 Subject: [PATCH 124/204] Have the UNIX VFS issue warnings via sqlite3_log() if a database file is renamed or unlinked or linked to more than one name while the file is open. FossilOrigin-Name: e238dcf9189c029fbdcf89339e21d9cdd8fbf2c5 --- manifest | 14 ++++++------- manifest.uuid | 2 +- src/os_unix.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 60 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 6d3e751585..672a059405 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\smulti-process\stester\sintegration\swith\sthe\sWin32\sAPI. -D 2013-04-11T00:09:44.820 +C Have\sthe\sUNIX\sVFS\sissue\swarnings\svia\ssqlite3_log()\sif\sa\sdatabase\sfile\sis\nrenamed\sor\sunlinked\sor\slinked\sto\smore\sthan\sone\sname\swhile\sthe\sfile\sis\sopen. +D 2013-04-11T01:16:15.249 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c ca679b293a6233327e418fd1dde2cd5db3e90932 F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c a6e9caadd8956a39153d1bd920e459211a281388 +F src/os_unix.c 5707fcb125f043e2d3376ea862e8ec83633c5e0e F src/os_win.c 3aec291c40048653c00a153fee96d3a442ab85ee F src/pager.c 28f45e60d9a173368872d6e688e7a848c3926344 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P be7d2c5482c41baf000d7fb5dccc31b974e91064 -R 255ee63faf4906e3f01e118bd1c7af53 -U mistachkin -Z 56dc3349791505b3e20ff09949e0bcd7 +P 0fdc743583c67a3a017b9ad812c62a5104b9aee7 +R 6358d94b9e9fcf447041dc89384ceff9 +U drh +Z 7f7d7e8aee1747123050111170b5d1bc diff --git a/manifest.uuid b/manifest.uuid index 557333d3d0..784964e39b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0fdc743583c67a3a017b9ad812c62a5104b9aee7 \ No newline at end of file +e238dcf9189c029fbdcf89339e21d9cdd8fbf2c5 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index a65f894eb8..778575f7b1 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -280,6 +280,7 @@ struct unixFile { #define UNIXFILE_DELETE 0x20 /* Delete on close */ #define UNIXFILE_URI 0x40 /* Filename might have query parameters */ #define UNIXFILE_NOLOCK 0x80 /* Do no file locking */ +#define UNIXFILE_WARNED 0x0100 /* verifyDbFile() warnings have been issued */ /* ** Include code that is common to all os_*.c files @@ -799,7 +800,6 @@ static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { } - /****************************************************************************** ****************** Begin Unique File ID Utility Used By VxWorks *************** ** @@ -1300,6 +1300,50 @@ static int findInodeInfo( } +/* +** Check a unixFile that is a database. Verify the following: +** +** (1) There is exactly one hard link on the file +** (2) The file is not a symbolic link +** (3) The file has not been renamed or unlinked +** +** Issue sqlite3_log(SQLITE_WARNING,...) messages if anything is not right. +*/ +static void verifyDbFile(unixFile *pFile){ + struct stat buf; + int rc; + if( pFile->ctrlFlags & UNIXFILE_WARNED ){ + /* One or more of the following warnings have already been issued. Do not + ** repeat them so as not to clutter the error log */ + return; + } + rc = osFstat(pFile->h, &buf); + if( rc!=0 ){ + sqlite3_log(SQLITE_WARNING, "cannot fstat db file %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } + if( buf.st_nlink==0 && (pFile->ctrlFlags & UNIXFILE_DELETE)==0 ){ + sqlite3_log(SQLITE_WARNING, "file unlinked while open: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } + if( buf.st_nlink>1 ){ + sqlite3_log(SQLITE_WARNING, "multiple links to file: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } + if( pFile->pInode!=0 + && ((rc = osStat(pFile->zPath, &buf))!=0 + || buf.st_ino!=pFile->pInode->fileId.ino) + ){ + sqlite3_log(SQLITE_WARNING, "file renamed while open: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } +} + + /* ** This routine checks if there is a RESERVED lock held on the specified ** file by this or any other process. If such a lock is held, set *pResOut @@ -1876,6 +1920,7 @@ static int closeUnixFile(sqlite3_file *id){ static int unixClose(sqlite3_file *id){ int rc = SQLITE_OK; unixFile *pFile = (unixFile *)id; + verifyDbFile(pFile); unixUnlock(id, NO_LOCK); unixEnterMutex(); @@ -4539,6 +4584,7 @@ static void unixUnmapfile(unixFile *pFd){ #endif } +#ifndef SQLITE_DISABLE_MMAP /* ** Return the system page size. */ @@ -4551,6 +4597,7 @@ static int unixGetPagesize(void){ return (int)sysconf(_SC_PAGESIZE); #endif } +#endif /* SQLITE_DISABLE_MMAP */ #ifndef SQLITE_DISABLE_MMAP /* @@ -4656,10 +4703,10 @@ static void unixRemapfile( ** code otherwise. */ static int unixMapfile(unixFile *pFd, i64 nByte){ +#ifndef SQLITE_DISABLE_MMAP i64 nMap = nByte; int rc; -#ifndef SQLITE_DISABLE_MMAP assert( nMap>=0 || pFd->nFetchOut==0 ); if( pFd->nFetchOut>0 ) return SQLITE_OK; @@ -4700,7 +4747,9 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ ** release the reference by calling unixUnfetch(). */ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ +#ifndef SQLITE_DISABLE_MMAP unixFile *pFd = (unixFile *)fd; /* The underlying database file */ +#endif *pp = 0; #ifndef SQLITE_DISABLE_MMAP @@ -5222,6 +5271,7 @@ static int fillInUnixFile( }else{ pNew->pMethod = pLockingStyle; OpenCounter(+1); + verifyDbFile(pNew); } return rc; } From e3be8c86277b72d8ce281d347792c543f5084544 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 11 Apr 2013 11:53:45 +0000 Subject: [PATCH 125/204] Show the process-id on log messages from mptester. FossilOrigin-Name: 6748a83dc5c02db37ecd963e678c5c69db142cac --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 13 +++++++++---- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 672a059405..8a66fe58aa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\sthe\sUNIX\sVFS\sissue\swarnings\svia\ssqlite3_log()\sif\sa\sdatabase\sfile\sis\nrenamed\sor\sunlinked\sor\slinked\sto\smore\sthan\sone\sname\swhile\sthe\sfile\sis\sopen. -D 2013-04-11T01:16:15.249 +C Show\sthe\sprocess-id\son\slog\smessages\sfrom\smptester. +D 2013-04-11T11:53:45.288 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da F mptest/config02.test 962913ed2b537d60de4126db7fe54716865cdd22 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 -F mptest/mptest.c 8b6d7878addccbb9a1b28d9dbf2b0b516a3279f1 +F mptest/mptest.c 1a8ca3d21e285d38f3ee06d708e634dda3f05bed F mptest/multiwrite01.test 81fbc17657964889b60750bd7bbb1deffe8f4d42 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 0fdc743583c67a3a017b9ad812c62a5104b9aee7 -R 6358d94b9e9fcf447041dc89384ceff9 +P e238dcf9189c029fbdcf89339e21d9cdd8fbf2c5 +R 961d2f18ca03b287ca1083092d6feb69 U drh -Z 7f7d7e8aee1747123050111170b5d1bc +Z d718ebcef0d40f58f76fbe3a2824e294 diff --git a/manifest.uuid b/manifest.uuid index 784964e39b..68dd000d58 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e238dcf9189c029fbdcf89339e21d9cdd8fbf2c5 \ No newline at end of file +6748a83dc5c02db37ecd963e678c5c69db142cac \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 0a8a8cedbd..210ac6fa7b 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -38,8 +38,11 @@ #if defined(_WIN32) # define WIN32_LEAN_AND_MEAN # include +# include +# define GETPID _getpid #else # include +# define GETPID getpid #endif #include #include @@ -67,7 +70,7 @@ static struct Global { FILE *pErrLog; /* Where to write errors */ char *zLog; /* Name of output log file */ FILE *pLog; /* Where to write log messages */ - char zName[12]; /* Symbolic name of this process */ + char zName[32]; /* Symbolic name of this process */ int taskId; /* Task ID. 0 means supervisor. */ int iTrace; /* Tracing level */ int bSqlTrace; /* True to trace SQL commands */ @@ -331,7 +334,7 @@ static void sqlErrorCallback(void *pArg, int iErrCode, const char *zMsg){ UNUSED_PARAMETER(pArg); if( (iErrCode&0xff)==SQLITE_SCHEMA && g.iTrace<3 ) return; if( g.iTimeout==0 && (iErrCode&0xff)==SQLITE_BUSY && g.iTrace<3 ) return; - if( iErrCode==SQLITE_OK ){ + if( (iErrCode&0xff)==SQLITE_NOTICE ){ logMessage("(info) %s", zMsg); }else{ errorMessage("(errcode=%d) %s", iErrCode, zMsg); @@ -1189,7 +1192,7 @@ int main(int argc, char **argv){ exit(1); } n = argc-2; - sqlite3_snprintf(sizeof(g.zName), g.zName, "mptest"); + sqlite3_snprintf(sizeof(g.zName), g.zName, "%05d.mptest", GETPID()); g.zVfs = findOption(argv+2, &n, "vfs", 1); zClient = findOption(argv+2, &n, "client", 1); g.zErrLog = findOption(argv+2, &n, "errlog", 1); @@ -1209,11 +1212,13 @@ int main(int argc, char **argv){ }else{ g.pLog = stdout; } + sqlite3_config(SQLITE_CONFIG_LOG, sqlErrorCallback, 0); if( zClient ){ iClient = atoi(zClient); if( iClient<1 ) fatalError("illegal client number: %d\n", iClient); - sqlite3_snprintf(sizeof(g.zName), g.zName, "client%02d", iClient); + sqlite3_snprintf(sizeof(g.zName), g.zName, "%05d.client%02d", + GETPID(), iClient); }else{ if( g.iTrace>0 ){ printf("With SQLite " SQLITE_VERSION " " SQLITE_SOURCE_ID "\n" ); From ef8e986bde954308f2bcb951dfccd23c04809fc4 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 11 Apr 2013 13:26:18 +0000 Subject: [PATCH 126/204] Fix pragma code generation so that it always outputs an OP_Trace opcode so that pragmas are shown in sqlite3_trace() output. FossilOrigin-Name: 663f04bd48bc6f302230a22bd22b82bde2353943 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pragma.c | 7 ++++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 8a66fe58aa..4fe8619c70 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Show\sthe\sprocess-id\son\slog\smessages\sfrom\smptester. -D 2013-04-11T11:53:45.288 +C Fix\spragma\scode\sgeneration\sso\sthat\sit\salways\soutputs\san\sOP_Trace\sopcode\sso\nthat\spragmas\sare\sshown\sin\ssqlite3_trace()\soutput. +D 2013-04-11T13:26:18.388 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -174,7 +174,7 @@ F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c 8bbc8c8697d13f76394e592dcecafce3491ed8ba +F src/pragma.c 216d6c24520f7b2e267a2356e5d3269d28690f49 F src/prepare.c 743e484233c51109666d402f470523553b41797c F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e238dcf9189c029fbdcf89339e21d9cdd8fbf2c5 -R 961d2f18ca03b287ca1083092d6feb69 +P 6748a83dc5c02db37ecd963e678c5c69db142cac +R d345752e155752ef7baeb6380027dfce U drh -Z d718ebcef0d40f58f76fbe3a2824e294 +Z c8c7d1ee07a5c8a52f14de0a5a9d5c85 diff --git a/manifest.uuid b/manifest.uuid index 68dd000d58..234c88c0fe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6748a83dc5c02db37ecd963e678c5c69db142cac \ No newline at end of file +663f04bd48bc6f302230a22bd22b82bde2353943 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 340e62d0dd..f4fc73d297 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -319,7 +319,7 @@ void sqlite3Pragma( int rc; /* return value form SQLITE_FCNTL_PRAGMA */ sqlite3 *db = pParse->db; /* The database connection */ Db *pDb; /* The specific database being pragmaed */ - Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(db); /* Prepared statement */ + Vdbe *v = sqlite3GetVdbe(pParse); /* Prepared statement */ if( v==0 ) return; sqlite3VdbeRunOnlyOnce(v); @@ -402,11 +402,12 @@ void sqlite3Pragma( static const VdbeOpList getCacheSize[] = { { OP_Transaction, 0, 0, 0}, /* 0 */ { OP_ReadCookie, 0, 1, BTREE_DEFAULT_CACHE_SIZE}, /* 1 */ - { OP_IfPos, 1, 7, 0}, + { OP_IfPos, 1, 8, 0}, { OP_Integer, 0, 2, 0}, { OP_Subtract, 1, 2, 1}, - { OP_IfPos, 1, 7, 0}, + { OP_IfPos, 1, 8, 0}, { OP_Integer, 0, 1, 0}, /* 6 */ + { OP_Noop, 0, 0, 0}, { OP_ResultRow, 1, 1, 0}, }; int addr; From e82c6c39f41724006ce063c313fe22e78e8b1da4 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 11 Apr 2013 18:28:55 +0000 Subject: [PATCH 127/204] Fix the xCheckReservedLock() method on the windows VFS so that it cannot return a false positive when two or more processes use it at the same time on the same file. Ticket [7ff3120e4fa54abb55] FossilOrigin-Name: dd3510bb20ade173c81c9874c05466011c8a249d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 4fe8619c70..8623e2f65f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\spragma\scode\sgeneration\sso\sthat\sit\salways\soutputs\san\sOP_Trace\sopcode\sso\nthat\spragmas\sare\sshown\sin\ssqlite3_trace()\soutput. -D 2013-04-11T13:26:18.388 +C Fix\sthe\sxCheckReservedLock()\smethod\son\sthe\swindows\sVFS\sso\sthat\sit\scannot\nreturn\sa\sfalse\spositive\swhen\stwo\sor\smore\sprocesses\suse\sit\sat\sthe\ssame\stime\non\sthe\ssame\sfile.\s\sTicket\s[7ff3120e4fa54abb55] +D 2013-04-11T18:28:55.139 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F src/os.c ca679b293a6233327e418fd1dde2cd5db3e90932 F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 5707fcb125f043e2d3376ea862e8ec83633c5e0e -F src/os_win.c 3aec291c40048653c00a153fee96d3a442ab85ee +F src/os_win.c 426563476ac4b0939c1bb2a59c6330e6864db5b0 F src/pager.c 28f45e60d9a173368872d6e688e7a848c3926344 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 6748a83dc5c02db37ecd963e678c5c69db142cac -R d345752e155752ef7baeb6380027dfce +P 663f04bd48bc6f302230a22bd22b82bde2353943 +R bd2e6ca88b98f09bd2a36a3fcc06c9f0 U drh -Z c8c7d1ee07a5c8a52f14de0a5a9d5c85 +Z 2e31b5ec3b1e364139cb9a99bd138f76 diff --git a/manifest.uuid b/manifest.uuid index 234c88c0fe..8f27dd8da9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -663f04bd48bc6f302230a22bd22b82bde2353943 \ No newline at end of file +dd3510bb20ade173c81c9874c05466011c8a249d \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 220ebac268..034f719fbd 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2702,7 +2702,7 @@ static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ rc = 1; OSTRACE(("TEST WR-LOCK %d %d (local)\n", pFile->h, rc)); }else{ - rc = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, RESERVED_BYTE, 0, 1, 0); + rc = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS,RESERVED_BYTE, 0, 1, 0); if( rc ){ winUnlockFile(&pFile->h, RESERVED_BYTE, 0, 1, 0); } From fdd72c9a2fc9b4f1d3eb256830dc055dcdbad4e0 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 11 Apr 2013 21:13:10 +0000 Subject: [PATCH 128/204] For the multi-process tester on Win32, make use of the GetCurrentProcessId API. FossilOrigin-Name: f1b524b9d9ea3db96d54ac55c39f15e6879085bd --- manifest | 14 +++++++------- manifest.uuid | 2 +- mptest/mptest.c | 5 ++--- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 8623e2f65f..347b207be7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sxCheckReservedLock()\smethod\son\sthe\swindows\sVFS\sso\sthat\sit\scannot\nreturn\sa\sfalse\spositive\swhen\stwo\sor\smore\sprocesses\suse\sit\sat\sthe\ssame\stime\non\sthe\ssame\sfile.\s\sTicket\s[7ff3120e4fa54abb55] -D 2013-04-11T18:28:55.139 +C For\sthe\smulti-process\stester\son\sWin32,\smake\suse\sof\sthe\sGetCurrentProcessId\sAPI. +D 2013-04-11T21:13:10.779 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da F mptest/config02.test 962913ed2b537d60de4126db7fe54716865cdd22 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 -F mptest/mptest.c 1a8ca3d21e285d38f3ee06d708e634dda3f05bed +F mptest/mptest.c 53066ffe4ae6f0c4de451cc77985b88d0fee2aec F mptest/multiwrite01.test 81fbc17657964889b60750bd7bbb1deffe8f4d42 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 663f04bd48bc6f302230a22bd22b82bde2353943 -R bd2e6ca88b98f09bd2a36a3fcc06c9f0 -U drh -Z 2e31b5ec3b1e364139cb9a99bd138f76 +P dd3510bb20ade173c81c9874c05466011c8a249d +R 69eb3039bde7025dc2da94e69ac2fbdc +U mistachkin +Z 5b9c93ebe8aafa80e045145422e5fe06 diff --git a/manifest.uuid b/manifest.uuid index 8f27dd8da9..0a9b5274e1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dd3510bb20ade173c81c9874c05466011c8a249d \ No newline at end of file +f1b524b9d9ea3db96d54ac55c39f15e6879085bd \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 210ac6fa7b..33f4a3fa71 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -38,11 +38,8 @@ #if defined(_WIN32) # define WIN32_LEAN_AND_MEAN # include -# include -# define GETPID _getpid #else # include -# define GETPID getpid #endif #include #include @@ -52,8 +49,10 @@ /* The suffix to append to the child command lines, if any */ #if defined(_WIN32) # define CMDLINE_SUFFIX "" +# define GETPID (int)GetCurrentProcessId #else # define CMDLINE_SUFFIX "&" +# define GETPID getpid #endif /* Mark a parameter as unused to suppress compiler warnings */ From 739ee7ffac34ef1b93c4560f34dc272d3864f45f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 12 Apr 2013 01:04:36 +0000 Subject: [PATCH 129/204] In mptester: improve the way that child processes are dispatched. Pass the --vfs option through to children. Log the command used to start child processes when the tracing level is high enough. FossilOrigin-Name: 55718ae3462b2b6e0774d49e1c4c74143bc9e3a5 --- manifest | 14 +++++++------- manifest.uuid | 2 +- mptest/mptest.c | 22 +++++++++++++--------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 347b207be7..1fbc9f7b6e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\sthe\smulti-process\stester\son\sWin32,\smake\suse\sof\sthe\sGetCurrentProcessId\sAPI. -D 2013-04-11T21:13:10.779 +C In\smptester:\simprove\sthe\sway\sthat\schild\sprocesses\sare\sdispatched.\s\sPass\nthe\s--vfs\soption\sthrough\sto\schildren.\s\sLog\sthe\scommand\sused\sto\sstart\nchild\sprocesses\swhen\sthe\stracing\slevel\sis\shigh\senough. +D 2013-04-12T01:04:36.558 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da F mptest/config02.test 962913ed2b537d60de4126db7fe54716865cdd22 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 -F mptest/mptest.c 53066ffe4ae6f0c4de451cc77985b88d0fee2aec +F mptest/mptest.c 2a263e88f18762ac5d7fb56749d2d2ea8e22cb93 F mptest/multiwrite01.test 81fbc17657964889b60750bd7bbb1deffe8f4d42 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P dd3510bb20ade173c81c9874c05466011c8a249d -R 69eb3039bde7025dc2da94e69ac2fbdc -U mistachkin -Z 5b9c93ebe8aafa80e045145422e5fe06 +P f1b524b9d9ea3db96d54ac55c39f15e6879085bd +R 42c876901a232f69407f221f508add46 +U drh +Z f52bc6984c23b912a4265566462deba4 diff --git a/manifest.uuid b/manifest.uuid index 0a9b5274e1..6692706ce6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f1b524b9d9ea3db96d54ac55c39f15e6879085bd \ No newline at end of file +55718ae3462b2b6e0774d49e1c4c74143bc9e3a5 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 33f4a3fa71..e47be5ae6d 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -48,10 +48,8 @@ /* The suffix to append to the child command lines, if any */ #if defined(_WIN32) -# define CMDLINE_SUFFIX "" # define GETPID (int)GetCurrentProcessId #else -# define CMDLINE_SUFFIX "&" # define GETPID getpid #endif @@ -629,14 +627,20 @@ static void startClient(int iClient){ if( sqlite3_changes(g.db) ){ char *zSys; int rc; - zSys = sqlite3_mprintf( - "%s \"%s\" --client %d --trace %d %s%s%s", - g.argv0, g.zDbFile, iClient, g.iTrace, - g.bSqlTrace ? "--sqltrace " : "", - g.bSync ? "--sync " : "", - CMDLINE_SUFFIX - ); + zSys = sqlite3_mprintf("%s \"%s\" --client %d --trace %d", + g.argv0, g.zDbFile, iClient, g.iTrace); + if( g.bSqlTrace ){ + zSys = sqlite3_mprintf("%z --sqltrace", zSys); + } + if( g.bSync ){ + zSys = sqlite3_mprintf("%z --sync", zSys); + } + if( g.zVfs ){ + zSys = sqlite3_mprintf("%z --vfs \"%s\"", zSys, g.zVfs); + } + if( g.iTrace>=2 ) logMessage("system('%q')", zSys); #if !defined(_WIN32) + zSys = sqlite3_mprintf("%z &", zSys); rc = system(zSys); if( rc ) errorMessage("system() fails with error code %d", rc); #else From a8f60381a93d037d39d5dc340f08e3cecb70ec7e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 12 Apr 2013 16:47:27 +0000 Subject: [PATCH 130/204] Change the fts4aux virtual table module so that fts4aux tables created in the temp database may report on fts3/fts4 tables in any attached database. FossilOrigin-Name: 546a187f1361dad340ed8f6c28dd89e6c19f9c33 --- ext/fts3/fts3_aux.c | 29 +++++++++++++++------ manifest | 16 ++++++------ manifest.uuid | 2 +- test/fts3aux1.test | 61 ++++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 88 insertions(+), 20 deletions(-) diff --git a/ext/fts3/fts3_aux.c b/ext/fts3/fts3_aux.c index a2bff2e1d3..9b582fcf8b 100644 --- a/ext/fts3/fts3_aux.c +++ b/ext/fts3/fts3_aux.c @@ -70,17 +70,26 @@ static int fts3auxConnectMethod( UNUSED_PARAMETER(pUnused); - /* The user should specify a single argument - the name of an fts3 table. */ - if( argc!=4 ){ - *pzErr = sqlite3_mprintf( - "wrong number of arguments to fts4aux constructor" - ); - return SQLITE_ERROR; - } + /* The user should invoke this in one of two forms: + ** + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table); + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table-db, fts4-table); + */ + if( argc!=4 && argc!=5 ) goto bad_args; zDb = argv[1]; nDb = (int)strlen(zDb); - zFts3 = argv[3]; + if( argc==5 ){ + if( nDb==4 && 0==sqlite3_strnicmp("temp", zDb, 4) ){ + zDb = argv[3]; + nDb = (int)strlen(zDb); + zFts3 = argv[4]; + }else{ + goto bad_args; + } + }else{ + zFts3 = argv[3]; + } nFts3 = (int)strlen(zFts3); rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA); @@ -103,6 +112,10 @@ static int fts3auxConnectMethod( *ppVtab = (sqlite3_vtab *)p; return SQLITE_OK; + + bad_args: + *pzErr = sqlite3_mprintf("invalid arguments to fts4aux constructor"); + return SQLITE_ERROR; } /* diff --git a/manifest b/manifest index 1fbc9f7b6e..e1fd998e5e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\smptester:\simprove\sthe\sway\sthat\schild\sprocesses\sare\sdispatched.\s\sPass\nthe\s--vfs\soption\sthrough\sto\schildren.\s\sLog\sthe\scommand\sused\sto\sstart\nchild\sprocesses\swhen\sthe\stracing\slevel\sis\shigh\senough. -D 2013-04-12T01:04:36.558 +C Change\sthe\sfts4aux\svirtual\stable\smodule\sso\sthat\sfts4aux\stables\screated\sin\sthe\stemp\sdatabase\smay\sreport\son\sfts3/fts4\stables\sin\sany\sattached\sdatabase. +D 2013-04-12T16:47:27.407 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -58,7 +58,7 @@ F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d F ext/fts3/fts3.c 0eaedfc6d2eb22563ef1d044dcfed93b70ec79f2 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 1e58825246b56259267382d2f9902774c049460a -F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e +F ext/fts3/fts3_aux.c b02632f6dd0e375ce97870206d914ea6d8df5ccd F ext/fts3/fts3_expr.c 6cb4410f87676ae633bd7923bbc78526cb839c4d F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf @@ -478,7 +478,7 @@ F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test e7b80272efcced57d1d087a9da5c690dd7c21fd9 F test/fts3atoken.test fb398ab50aa232489e2a17f9b29d7ad3a3885f36 F test/fts3auto.test 74315a7377403a57ba82a652a33704197fe1e4be -F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 +F test/fts3aux1.test 575cc4b46320d96be10b878c954a0a08573b53b2 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 F test/fts3comp1.test a0f5b16a2df44dd0b15751787130af2183167c0c @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f1b524b9d9ea3db96d54ac55c39f15e6879085bd -R 42c876901a232f69407f221f508add46 -U drh -Z f52bc6984c23b912a4265566462deba4 +P 55718ae3462b2b6e0774d49e1c4c74143bc9e3a5 +R dc56d874edc4dcec111c7be4534352ac +U dan +Z fb78d8a2b290ae6166ab0175cd5b9a72 diff --git a/manifest.uuid b/manifest.uuid index 6692706ce6..e651e4517b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -55718ae3462b2b6e0774d49e1c4c74143bc9e3a5 \ No newline at end of file +546a187f1361dad340ed8f6c28dd89e6c19f9c33 \ No newline at end of file diff --git a/test/fts3aux1.test b/test/fts3aux1.test index adda586353..1688d6b231 100644 --- a/test/fts3aux1.test +++ b/test/fts3aux1.test @@ -354,10 +354,10 @@ do_execsql_test 3.1.1 { do_catchsql_test 3.1.2 { CREATE VIRTUAL TABLE terms2 USING fts4aux; -} {1 {wrong number of arguments to fts4aux constructor}} +} {1 {invalid arguments to fts4aux constructor}} do_catchsql_test 3.1.3 { CREATE VIRTUAL TABLE terms2 USING fts4aux(t2, t2); -} {1 {wrong number of arguments to fts4aux constructor}} +} {1 {invalid arguments to fts4aux constructor}} do_execsql_test 3.2.1 { CREATE VIRTUAL TABLE terms3 USING fts4aux(does_not_exist) @@ -444,7 +444,6 @@ do_plansql_test 4.5 { # odd name (one that requires quoting for use in SQL statements). And that # the argument to the fts4aux constructor is properly dequoted before use. # -# do_execsql_test 5.1 { CREATE VIRTUAL TABLE "abc '!' def" USING fts4(x, y); INSERT INTO "abc '!' def" VALUES('XX', 'YY'); @@ -458,5 +457,61 @@ do_execsql_test 5.2 { SELECT * FROM "%%^^%%"; } {xx * 1 1 xx 0 1 1 yy * 1 1 yy 1 1 1} +#------------------------------------------------------------------------- +# Test that we can create an fts4aux table in the temp database. +# +forcedelete test.db2 +do_execsql_test 6.1 { + CREATE VIRTUAL TABLE ft1 USING fts4(x, y); + INSERT INTO ft1 VALUES('a b', 'c d'); + INSERT INTO ft1 VALUES('e e', 'c d'); + INSERT INTO ft1 VALUES('a a', 'b b'); + CREATE VIRTUAL TABLE temp.aux1 USING fts4aux(main, ft1); + SELECT * FROM aux1; +} { + a * 2 3 a 0 2 3 + b * 2 3 b 0 1 1 b 1 1 2 + c * 2 2 c 1 2 2 + d * 2 2 d 1 2 2 + e * 1 2 e 0 1 2 +} + +do_execsql_test 6.2 { + ATTACH 'test.db2' AS att; + CREATE VIRTUAL TABLE att.ft1 USING fts4(x, y); + INSERT INTO att.ft1 VALUES('v w', 'x y'); + INSERT INTO att.ft1 VALUES('z z', 'x y'); + INSERT INTO att.ft1 VALUES('v v', 'w w'); + CREATE VIRTUAL TABLE temp.aux2 USING fts4aux(att, ft1); + SELECT * FROM aux2; +} { + v * 2 3 v 0 2 3 + w * 2 3 w 0 1 1 w 1 1 2 + x * 2 2 x 1 2 2 + y * 2 2 y 1 2 2 + z * 1 2 z 0 1 2 +} + +foreach {tn q res1 res2} { + 1 { SELECT * FROM %%% WHERE term = 'a' } {a * 2 3 a 0 2 3} {} + 2 { SELECT * FROM %%% WHERE term = 'x' } {} {x * 2 2 x 1 2 2} + + 3 { SELECT * FROM %%% WHERE term >= 'y' } + {} {y * 2 2 y 1 2 2 z * 1 2 z 0 1 2} + + 4 { SELECT * FROM %%% WHERE term <= 'c' } + {a * 2 3 a 0 2 3 b * 2 3 b 0 1 1 b 1 1 2 c * 2 2 c 1 2 2} {} +} { + set sql1 [string map {%%% aux1} $q] + set sql2 [string map {%%% aux2} $q] + + do_execsql_test 7.$tn.1 $sql1 $res1 + do_execsql_test 7.$tn.2 $sql2 $res2 +} + +do_test 8.1 { + catchsql { CREATE VIRTUAL TABLE att.aux3 USING fts4aux(main, ft1) } +} {1 {invalid arguments to fts4aux constructor}} finish_test + From 4b2ac35e5bf760bad76366e44c3ed6eb4448ed41 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 12 Apr 2013 16:53:19 +0000 Subject: [PATCH 131/204] Add a test case for detaching a database referenced by an fts4aux table created in the temp schema. FossilOrigin-Name: 6d6f0592abe237256dbdf30ab0ba0bb0c365dd95 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/fts3aux1.test | 5 +++++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index e1fd998e5e..c4d5a473e1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sfts4aux\svirtual\stable\smodule\sso\sthat\sfts4aux\stables\screated\sin\sthe\stemp\sdatabase\smay\sreport\son\sfts3/fts4\stables\sin\sany\sattached\sdatabase. -D 2013-04-12T16:47:27.407 +C Add\sa\stest\scase\sfor\sdetaching\sa\sdatabase\sreferenced\sby\san\sfts4aux\stable\screated\sin\sthe\stemp\sschema. +D 2013-04-12T16:53:19.544 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -478,7 +478,7 @@ F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test e7b80272efcced57d1d087a9da5c690dd7c21fd9 F test/fts3atoken.test fb398ab50aa232489e2a17f9b29d7ad3a3885f36 F test/fts3auto.test 74315a7377403a57ba82a652a33704197fe1e4be -F test/fts3aux1.test 575cc4b46320d96be10b878c954a0a08573b53b2 +F test/fts3aux1.test 03cec2dea379931c219dd4406817485caa69d1d8 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 F test/fts3comp1.test a0f5b16a2df44dd0b15751787130af2183167c0c @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 55718ae3462b2b6e0774d49e1c4c74143bc9e3a5 -R dc56d874edc4dcec111c7be4534352ac +P 546a187f1361dad340ed8f6c28dd89e6c19f9c33 +R 05383c39b4618f3bdf18eec38f2fc283 U dan -Z fb78d8a2b290ae6166ab0175cd5b9a72 +Z db66ae2fc0950c0cd2f83a29d6b2c3f8 diff --git a/manifest.uuid b/manifest.uuid index e651e4517b..d1d8020d4c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -546a187f1361dad340ed8f6c28dd89e6c19f9c33 \ No newline at end of file +6d6f0592abe237256dbdf30ab0ba0bb0c365dd95 \ No newline at end of file diff --git a/test/fts3aux1.test b/test/fts3aux1.test index 1688d6b231..ef17949fd9 100644 --- a/test/fts3aux1.test +++ b/test/fts3aux1.test @@ -513,5 +513,10 @@ do_test 8.1 { catchsql { CREATE VIRTUAL TABLE att.aux3 USING fts4aux(main, ft1) } } {1 {invalid arguments to fts4aux constructor}} +do_test 8.2 { + execsql {DETACH att} + catchsql { SELECT * FROM aux2 } +} {1 {SQL logic error or missing database}} + finish_test From a3a5bd9b62324bcfafaf2e6d4d98fbc57e1a1d94 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 13 Apr 2013 19:59:58 +0000 Subject: [PATCH 132/204] Only consider AS names from the result set as candidates for resolving identifiers in the WHERE clause if there are no other matches. In the ORDER BY clause, AS names take priority over any column names. Candidate fix for ticket [2500cdb9be]. FossilOrigin-Name: ad53924dcadffb95c6497c46c228c67e8f5370e4 --- manifest | 22 +++++++++++++--------- manifest.uuid | 2 +- src/resolve.c | 18 +++++++++++------- src/sqliteInt.h | 2 ++ test/resolver01.test | 39 +++++++++++++++++++++++++++++++++++++++ test/tkt2822.test | 5 ++++- 6 files changed, 70 insertions(+), 18 deletions(-) create mode 100644 test/resolver01.test diff --git a/manifest b/manifest index c4d5a473e1..3a4b72859e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\scase\sfor\sdetaching\sa\sdatabase\sreferenced\sby\san\sfts4aux\stable\screated\sin\sthe\stemp\sschema. -D 2013-04-12T16:53:19.544 +C Only\sconsider\sAS\snames\sfrom\sthe\sresult\sset\sas\scandidates\sfor\sresolving\nidentifiers\sin\sthe\sWHERE\sclause\sif\sthere\sare\sno\sother\smatches.\s\sIn\sthe\nORDER\sBY\sclause,\sAS\snames\stake\spriority\sover\sany\scolumn\snames.\nCandidate\sfix\sfor\sticket\s[2500cdb9be]. +D 2013-04-13T19:59:58.676 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -178,14 +178,14 @@ F src/pragma.c 216d6c24520f7b2e267a2356e5d3269d28690f49 F src/prepare.c 743e484233c51109666d402f470523553b41797c F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 -F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 +F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c 319b7791cee6c763b60fde1b590bfaf62613cf37 F src/sqlite.h.in f606903594d314d1b63e677affb1a03e3c9a49c7 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 -F src/sqliteInt.h 1c517a89cfdba28084093607ec79af70359c7f9b +F src/sqliteInt.h 7580edf7e7e13af8dff6152666e4ea32535abf8c F src/sqliteLimit.h 3ea5c07b370b8033c30f66515ccf4e875c252429 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -685,6 +685,7 @@ F test/regexp1.test 5cbb6e7122ca51260d71079cf9245b63b8f64e1a F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a F test/releasetest.tcl 06d289d8255794073a58d2850742f627924545ce +F test/resolver01.test d1b487fc567bdb70b5dd09ccaaa877ddc61a233e F test/rollback.test a1b4784b864331eae8b2a98c189efa2a8b11ff07 F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test f777404492adb0e00868fd706a3721328fd3af48 @@ -854,7 +855,7 @@ F test/tkt2686.test 6ee01c9b9e9c48f6d3a1fdd553b1cc4258f903d6 F test/tkt2767.test 569000d842678f9cf2db7e0d1b27cbc9011381b0 F test/tkt2817.test f31839e01f4243cff7399ef654d3af3558cb8d8d F test/tkt2820.test 39940276b3436d125deb7d8ebeee053e4cf13213 -F test/tkt2822.test f391776423a7c0d0949edfce375708bfb0f3141e +F test/tkt2822.test c3589494fba643e039bcf0bfde7554ff6028406d F test/tkt2832.test a9b0b74a02dca166a04d9e37739c414b10929caa F test/tkt2854.test e432965db29e27e16f539b2ba7f502eb2ccc49af F test/tkt2920.test a8737380e4ae6424e00c0273dc12775704efbebf @@ -1050,7 +1051,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 546a187f1361dad340ed8f6c28dd89e6c19f9c33 -R 05383c39b4618f3bdf18eec38f2fc283 -U dan -Z db66ae2fc0950c0cd2f83a29d6b2c3f8 +P 6d6f0592abe237256dbdf30ab0ba0bb0c365dd95 +R b0858137a1ee08b4782056449cba9377 +T *branch * tkt-2500cdb9be +T *sym-tkt-2500cdb9be * +T -sym-trunk * +U drh +Z 489735ec39077e65a3d08cafc2217189 diff --git a/manifest.uuid b/manifest.uuid index d1d8020d4c..ac4feed793 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6d6f0592abe237256dbdf30ab0ba0bb0c365dd95 \ No newline at end of file +ad53924dcadffb95c6497c46c228c67e8f5370e4 \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index 0380138678..9b350caf80 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -388,7 +388,10 @@ static int lookupName( ** Note that the expression in the result set should have already been ** resolved by the time the WHERE clause is resolved. */ - if( cnt==0 && (pEList = pNC->pEList)!=0 && zTab==0 ){ + if( (pEList = pNC->pEList)!=0 + && zTab==0 + && ((pNC->ncFlags & NC_AsMaybe)==0 || cnt==0) + ){ for(j=0; jnExpr; j++){ char *zAs = pEList->a[j].zName; if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){ @@ -479,7 +482,9 @@ static int lookupName( lookupname_end: if( cnt==1 ){ assert( pNC!=0 ); - sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); + if( pExpr->op!=TK_AS ){ + sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); + } /* Increment the nRef value on all name contexts from TopNC up to ** the point where the name matched. */ for(;;){ @@ -1154,11 +1159,10 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** re-evaluated for each reference to it. */ sNC.pEList = p->pEList; - if( sqlite3ResolveExprNames(&sNC, p->pWhere) || - sqlite3ResolveExprNames(&sNC, p->pHaving) - ){ - return WRC_Abort; - } + if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; + sNC.ncFlags |= NC_AsMaybe; + if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; + sNC.ncFlags &= ~NC_AsMaybe; /* The ORDER BY and GROUP BY clauses may not refer to terms in ** outer queries diff --git a/src/sqliteInt.h b/src/sqliteInt.h index a528e37e44..264db39d14 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2079,6 +2079,8 @@ struct NameContext { #define NC_HasAgg 0x02 /* One or more aggregate functions seen */ #define NC_IsCheck 0x04 /* True if resolving names in a CHECK constraint */ #define NC_InAggFunc 0x08 /* True if analyzing arguments to an agg func */ +#define NC_AsMaybe 0x10 /* Resolve to AS terms of the result set only + ** if no other resolution is available */ /* ** An instance of the following structure contains all information diff --git a/test/resolver01.test b/test/resolver01.test new file mode 100644 index 0000000000..3ca6acec47 --- /dev/null +++ b/test/resolver01.test @@ -0,0 +1,39 @@ +# 2013-04-13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file tests features of the name resolver (the component that +# figures out what identifiers in the SQL statement refer to) that +# were fixed by ticket [2500cdb9be] +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_test resolver01-1.1 { + catchsql { + CREATE TABLE t1(x, y); INSERT INTO t1 VALUES(11,22); + CREATE TABLE t2(y, z); INSERT INTO t2 VALUES(33,44); + SELECT 1 AS y FROM t1, t2 ORDER BY y; + } +} {0 1} +do_test resolver01-1.2 { + catchsql { + SELECT 2 AS y FROM t1, t2 ORDER BY y COLLATE nocase; + } +} {0 2} +do_test resolver01-1.3 { + catchsql { + SELECT 3 AS y FROM t1, t2 ORDER BY +y; + } +} {0 3} + + +finish_test diff --git a/test/tkt2822.test b/test/tkt2822.test index d0b16338c6..d3512d3038 100644 --- a/test/tkt2822.test +++ b/test/tkt2822.test @@ -208,12 +208,15 @@ do_test tkt2822-5.4 { # In "ORDER BY +b" the term is now an expression rather than # a label. It therefore matches by rule (3) instead of rule (2). +# +# 2013-04-13: This is busted. Changed to conform to PostgreSQL and +# MySQL and Oracle behavior. # do_test tkt2822-5.5 { execsql { SELECT a AS b FROM t3 ORDER BY +b; } -} {9 1} +} {1 9} # Tests for rule 2 in compound queries # From a1710cc2ea2ed88207b8b4f7ae52c7e95e96b2c4 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 15 Apr 2013 13:10:30 +0000 Subject: [PATCH 133/204] Fix a formatting typo in a comment. No changes to code. FossilOrigin-Name: 3412424990c93d2978e819e6099811f1cdde316d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2c5683ef6b..5428c2214c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Only\sconsider\sAS\snames\sfrom\sthe\sresult\sset\sas\scandidates\sfor\sresolving\nidentifiers\sin\sthe\sWHERE\sclause\sif\sthere\sare\sno\sother\smatches.\nIn\sthe\sORDER\sBY\sclause,\sAS\snames\stake\spriority\sover\sany\scolumn\snames.\nFix\sfor\sticket\s[2500cdb9be]. -D 2013-04-14T23:51:27.821 +C Fix\sa\sformatting\stypo\sin\sa\scomment.\s\sNo\schanges\sto\scode. +D 2013-04-15T13:10:30.554 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -182,7 +182,7 @@ F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c 319b7791cee6c763b60fde1b590bfaf62613cf37 -F src/sqlite.h.in f606903594d314d1b63e677affb1a03e3c9a49c7 +F src/sqlite.h.in 696ecfc764a32b47daaaf0e10438ff2aaa0739e0 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 7580edf7e7e13af8dff6152666e4ea32535abf8c @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 6d6f0592abe237256dbdf30ab0ba0bb0c365dd95 ad53924dcadffb95c6497c46c228c67e8f5370e4 -R b0858137a1ee08b4782056449cba9377 +P d0d5af79a005de30194863c57c5018ee75e03581 +R 18db1cd249206022b16580a94625cafa U drh -Z c608eb98c484fd069203789a0c999b98 +Z 4a78dc5e25d8c90b818007b69b504f95 diff --git a/manifest.uuid b/manifest.uuid index 1cbd0dc9a2..a5a10fb745 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d0d5af79a005de30194863c57c5018ee75e03581 \ No newline at end of file +3412424990c93d2978e819e6099811f1cdde316d \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 349e36ea92..a2e85053d6 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6847,7 +6847,7 @@ int sqlite3_strnicmp(const char *, const char *, int); ** ^The [sqlite3_strglob(P,X)] interface returns zero if string X matches ** the glob pattern P, and it returns non-zero if string X does not match ** the glob pattern P. ^The definition of glob pattern matching used in -** [sqlite3_strglob(P,X) is the same as for the "X GLOB P" operator in the +** [sqlite3_strglob(P,X)] is the same as for the "X GLOB P" operator in the ** SQL dialect used by SQLite. ^The sqlite3_strglob(P,X) function is case ** sensitive. ** From 9b4c59fa1b591e4474ca458a4af239dd9c1d540d Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 15 Apr 2013 17:03:42 +0000 Subject: [PATCH 134/204] Refactoring the mmap interface. The controlling pragma is now "mmap_size" instead of "mmap_limit". Also change SQLITE_CONFIG_MMAP_LIMIT and SQLITE_FCNTL_MMAP_LIMIT to SQLITE_CONFIG_MMAP_SIZE and SQLITE_FCNTL_MMAP_SIZE, respecctively. The default mmap_size is now always 0, meaning that memory mapped I/O is off by default. There is a new compile-time option SQLITE_MAX_MMAP_SIZE that determines a hard upper bound on the mmap_size. Setting SQLITE_MAX_MMAP_SIZE to zero disables the memory-mapped I/O logic and causes it to be omitted from the build. An extra argument is added to SQLITE_CONFIG_MMAP_SIZE that can optionally lower the SQLITE_MAX_MMAP_SIZE at start-time. The SQLITE_MAX_MMAP_SIZE is zero for platforms where we know that it does not work, meaning that it cannot be turned on by mistake on those platforms. FossilOrigin-Name: ea1404a10abd7f68e1f8e0708c8a3199d1f79665 --- manifest | 68 +++++++++++++++++++++--------------------- manifest.uuid | 2 +- mptest/config01.test | 7 +++-- mptest/config02.test | 11 ++++--- src/btree.c | 6 ++-- src/ctime.c | 10 +++---- src/global.c | 3 +- src/main.c | 12 ++++++-- src/os.c | 2 +- src/os_unix.c | 57 ++++++++++++++++++----------------- src/os_win.c | 51 ++++++++++++++++--------------- src/pager.c | 26 ++++++++-------- src/pragma.c | 26 ++++++++-------- src/shell.c | 3 +- src/sqlite.h.in | 36 +++++++++++----------- src/sqliteInt.h | 44 +++++++++++++++++++++++---- src/sqliteLimit.h | 24 --------------- src/test_config.c | 6 ++-- test/dbstatus2.test | 2 +- test/exclusive2.test | 2 +- test/func.test | 2 +- test/incrblob.test | 8 ++--- test/mmap1.test | 26 ++++++++-------- test/pager1.test | 8 ++--- test/pagerfault.test | 2 +- test/pageropt.test | 2 +- test/permutations.test | 4 +-- test/sysfault.test | 2 +- test/wal.test | 2 +- test/wal5.test | 2 +- test/win32lock.test | 2 +- 31 files changed, 245 insertions(+), 213 deletions(-) diff --git a/manifest b/manifest index 5428c2214c..215a7b5881 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sformatting\stypo\sin\sa\scomment.\s\sNo\schanges\sto\scode. -D 2013-04-15T13:10:30.554 +C Refactoring\sthe\smmap\sinterface.\s\sThe\scontrolling\spragma\sis\snow\s"mmap_size"\ninstead\sof\s"mmap_limit".\s\sAlso\schange\sSQLITE_CONFIG_MMAP_LIMIT\sand\nSQLITE_FCNTL_MMAP_LIMIT\sto\sSQLITE_CONFIG_MMAP_SIZE\sand\nSQLITE_FCNTL_MMAP_SIZE,\srespecctively.\s\s\nThe\sdefault\smmap_size\sis\snow\salways\s0,\smeaning\sthat\nmemory\smapped\sI/O\sis\soff\sby\sdefault.\s\sThere\sis\sa\snew\scompile-time\soption\nSQLITE_MAX_MMAP_SIZE\sthat\sdetermines\sa\shard\supper\sbound\son\sthe\smmap_size.\nSetting\sSQLITE_MAX_MMAP_SIZE\sto\szero\sdisables\sthe\smemory-mapped\sI/O\slogic\nand\scauses\sit\sto\sbe\somitted\sfrom\sthe\sbuild.\s\sAn\sextra\sargument\sis\sadded\nto\sSQLITE_CONFIG_MMAP_SIZE\sthat\scan\soptionally\slower\sthe\sSQLITE_MAX_MMAP_SIZE\nat\sstart-time.\sThe\sSQLITE_MAX_MMAP_SIZE\sis\szero\sfor\splatforms\swhere\swe\s\nknow\sthat\sit\sdoes\snot\swork,\smeaning\sthat\sit\scannot\sbe\sturned\son\sby\smistake\non\sthose\splatforms. +D 2013-04-15T17:03:42.119 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,8 +110,8 @@ F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 -F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da -F mptest/config02.test 962913ed2b537d60de4126db7fe54716865cdd22 +F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 +F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 F mptest/mptest.c 2a263e88f18762ac5d7fb56749d2d2ea8e22cb93 @@ -127,20 +127,20 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 5f2c4fe72f663bba837c5d01a732799c1d1c93f8 +F src/btree.c 480a6d255cc4f066029daf23dd54acf152cd0e13 F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c 61a53c7e0db5e7ba60cf23b73deaf6779aa715d3 +F src/ctime.c 726e88b7282e7f348f805437db3222f97a136d35 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c e16942bd5c8a868ac53287886464a5ed0e72b179 F src/func.c d3fdcff9274bc161152e67ed3f626841c247f4b9 -F src/global.c d2494a1cea8f66a2cab8258449df07f8f0ae6330 +F src/global.c 5caf4deab621abb45b4c607aad1bd21c20aac759 F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4 F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -149,7 +149,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 1422eba4aa2b1fb5f7b3aef574752272477d21e2 -F src/main.c 54a841854734b6731c4d026834788cac6a19f3d1 +F src/main.c de47c3e6b68f512b9e64ba2533ee4eecc9d501c1 F src/malloc.c fe085aa851b666b7c375c1ff957643dc20a04bf6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa @@ -163,30 +163,30 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c ca679b293a6233327e418fd1dde2cd5db3e90932 +F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 5707fcb125f043e2d3376ea862e8ec83633c5e0e -F src/os_win.c 426563476ac4b0939c1bb2a59c6330e6864db5b0 -F src/pager.c 28f45e60d9a173368872d6e688e7a848c3926344 +F src/os_unix.c 0a561eae5965c9371300b0419027f5ae9f847af2 +F src/os_win.c 873bbc5dff8eb9f2bf8d9287d29df082eacb8391 +F src/pager.c a55adacb1842b83354198c408e7adde95ecd1189 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c 216d6c24520f7b2e267a2356e5d3269d28690f49 +F src/pragma.c 3eacf001cbf4becbd494f8d82d08fdf1648cf8cb F src/prepare.c 743e484233c51109666d402f470523553b41797c F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 -F src/shell.c 319b7791cee6c763b60fde1b590bfaf62613cf37 -F src/sqlite.h.in 696ecfc764a32b47daaaf0e10438ff2aaa0739e0 +F src/shell.c 06bca04044b312ccadc43bf72385e7440cf60d1b +F src/sqlite.h.in d897fd15b9c74d4e78e4c42319484ceb1649c5d7 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 -F src/sqliteInt.h 7580edf7e7e13af8dff6152666e4ea32535abf8c -F src/sqliteLimit.h 3ea5c07b370b8033c30f66515ccf4e875c252429 +F src/sqliteInt.h 9933ad95b2c1e017f16428e9c0a929867598c2ae +F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 @@ -203,7 +203,7 @@ F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de F src/test_btree.c 5b89601dcb42a33ba8b820a6b763cc9cb48bac16 -F src/test_config.c b6f5d680f0cf03d0ae9d6165bd2e447d9b356e4a +F src/test_config.c 49c99b714c1dab68aa6b37d53681a4278982be8d F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c 8f786bfd0ad48030cf2a06fb1f050e9c60a150d7 @@ -376,7 +376,7 @@ F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/date.test f3228180c87bbe5d39c9397bf001c0095c3821b9 F test/dbstatus.test 207e5b63fcb7b9c3bb8e1fdf38ebd4654ad0e54b -F test/dbstatus2.test f329941d5f4a8bc0ba6ec5735897ef0cf34e2f5f +F test/dbstatus2.test 10418e62b3db5dca070f0c3eef3ea13946f339c2 F test/default.test 6faf23ccb300114924353007795aa9a8ec0aa9dc F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa @@ -410,7 +410,7 @@ F test/eqp.test 46aa946dd55c90635327898275d3e533d23a9845 F test/errmsg.test 050717f1c6a5685de9c79f5f9f6b83d7c592f73a F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test a1b324cb21834a490cd052d409d34789cfef57cb -F test/exclusive2.test 354bdabe299a2546c898dff42f79079ff1590d88 +F test/exclusive2.test 881193eccec225cfed9d7f744b65e57f26adee25 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30 F test/expr.test 67c9fd6f8f829e239dc8b0f4a08a73c08b09196d @@ -515,7 +515,7 @@ F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7 F test/fts4unicode.test 25ccad45896f8e50f6a694cff738a35f798cdb40 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d -F test/func.test a4f24707e6af0bcd4cb5b856af4462db40b047b1 +F test/func.test b0fc34fdc36897769651975a2b0a606312753643 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a F test/fuzz-oss1.test 4912e528ec9cf2f42134456933659d371c9e0d74 @@ -533,7 +533,7 @@ F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test 64f3cc1acde1b9161ccdd8e5bde3daefdb5b2617 F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 -F test/incrblob.test 2100cb8964e4a106e5ca9bf80e2c5c3e8be33f77 +F test/incrblob.test e81846d214f3637622620fbde7cd526781cfe328 F test/incrblob2.test edc3a96e557bd61fb39acc8d2edd43371fbbaa19 F test/incrblob3.test aedbb35ea1b6450c33b98f2b6ed98e5020be8dc7 F test/incrblob4.test 09be37d3dd996a31ea6993bba7837ece549414a8 @@ -637,7 +637,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test f064a9025136cd688c982ccdae81c7b4fac95724 +F test/mmap1.test 0b5802cf64acaa509ea889b3c708196cc6eb9d31 F test/mmap2.test a5ba639f90b5fc487400a49e158e14e465943e98 F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -657,17 +657,17 @@ F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 -F test/pager1.test 134144862f6a98f862dc2edda1b10ff499141bfa +F test/pager1.test 30e63afd425fea12285e9ec5fa1fd000808031f1 F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f -F test/pagerfault.test bba948be0564254b823ec6ca1fd95fbbfb0839a3 +F test/pagerfault.test 8483e65d33d5636e5b7656204bb274d826e728d9 F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 -F test/pageropt.test d6b695e6c12290c406150fc51eec32878d48df4c +F test/pageropt.test 6b8f6a123a5572c195ad4ae40f2987007923bbd6 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test b6cb45ce4d3193b831231025c9d30c77317dd240 +F test/permutations.test d2918e9ee6d0008292e5a70e90eed23b0440ca88 F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -758,7 +758,7 @@ F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test a653783d985108c4912cc64d341ffbbb55ad2806 -F test/sysfault.test 503f72712b2b21cb80dc9899e53c2e39484d0313 +F test/sysfault.test fa776e60bf46bdd3ae69f0b73e46ee3977a58ae6 F test/table.test a59d985ca366e39b17b175f387f9d5db5a18d4e2 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 @@ -962,11 +962,11 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 -F test/wal.test 20fb8d0f8e65e50eb675994cc3a4f0e692fd4faf +F test/wal.test 0b4837cd5e9283c116d30810a6859bed7425a95e F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c -F test/wal5.test 13c980d4d8d2d6b558361f19bed4ab31d41a2c7c +F test/wal5.test 8f888b50f66b78821e61ed0e233ded5de378224b F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test b3ee739fe8f7586aaebdc2367f477ebcf3e3b034 @@ -1004,7 +1004,7 @@ F test/whereD.test 3f3ee93825c94804f1fc91eef2de0d365981759a F test/whereE.test 7bd34945797efef15819368479bacc34215e4e1d F test/whereF.test a0e296643cabe5278379bc1a0aa158cf3c54a1c9 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 -F test/win32lock.test 21810396eee150680cc398970839e6abdcb807b9 +F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F test/zerodamage.test e7f77fded01dfcdf92ac2c5400f1e35d7a21463c F tool/build-all-msvc.bat 74fb6e5cca66ebdb6c9bbafb2f8b802f08146d38 x @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d0d5af79a005de30194863c57c5018ee75e03581 -R 18db1cd249206022b16580a94625cafa +P 3412424990c93d2978e819e6099811f1cdde316d +R aa0c6ea54adf9732b395ebf6e1c98243 U drh -Z 4a78dc5e25d8c90b818007b69b504f95 +Z 71eb761a9109889dcfada6d3231b23c2 diff --git a/manifest.uuid b/manifest.uuid index a5a10fb745..65cc51dd9d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3412424990c93d2978e819e6099811f1cdde316d \ No newline at end of file +ea1404a10abd7f68e1f8e0708c8a3199d1f79665 \ No newline at end of file diff --git a/mptest/config01.test b/mptest/config01.test index 75ba2f8fc2..683ee91143 100644 --- a/mptest/config01.test +++ b/mptest/config01.test @@ -5,11 +5,11 @@ PRAGMA page_size=8192; --task 1 PRAGMA journal_mode=PERSIST; - PRAGMA mmap_limit=0; + PRAGMA mmap_size=0; --end --task 2 PRAGMA journal_mode=TRUNCATE; - PRAGMA mmap_limit=28672; + PRAGMA mmap_size=28672; --end --task 3 PRAGMA journal_mode=MEMORY; @@ -17,6 +17,9 @@ PRAGMA page_size=8192; --task 4 PRAGMA journal_mode=OFF; --end +--task 4 + PRAGMA mmap_size(268435456); +--end --source multiwrite01.test --wait all PRAGMA page_size=16384; diff --git a/mptest/config02.test b/mptest/config02.test index e25d94081e..7d4b27898b 100644 --- a/mptest/config02.test +++ b/mptest/config02.test @@ -3,16 +3,19 @@ */ PRAGMA page_size=512; --task 1 - PRAGMA mmap_limit=0; + PRAGMA mmap_size=0; --end --task 2 - PRAGMA mmap_limit=28672; + PRAGMA mmap_size=28672; --end --task 3 - PRAGMA mmap_limit=8192; + PRAGMA mmap_size=8192; --end --task 4 - PRAGMA mmap_limit=65536; + PRAGMA mmap_size=65536; +--end +--task 5 + PRAGMA mmap_size=268435456; --end --source multiwrite01.test --source crash02.subtest diff --git a/src/btree.c b/src/btree.c index c13f0194f6..616d403681 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1876,7 +1876,7 @@ int sqlite3BtreeOpen( rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, EXTRA_SIZE, flags, vfsFlags, pageReinit); if( rc==SQLITE_OK ){ - sqlite3PagerSetMmapLimit(pBt->pPager, db->mxMmap); + sqlite3PagerSetMmapLimit(pBt->pPager, db->szMmap); rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); } if( rc!=SQLITE_OK ){ @@ -2147,11 +2147,11 @@ int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ ** Change the limit on the amount of the database file that may be ** memory mapped. */ -int sqlite3BtreeSetMmapLimit(Btree *p, sqlite3_int64 mxMmap){ +int sqlite3BtreeSetMmapLimit(Btree *p, sqlite3_int64 szMmap){ BtShared *pBt = p->pBt; assert( sqlite3_mutex_held(p->db->mutex) ); sqlite3BtreeEnter(p); - sqlite3PagerSetMmapLimit(pBt->pPager, mxMmap); + sqlite3PagerSetMmapLimit(pBt->pPager, szMmap); sqlite3BtreeLeave(p); return SQLITE_OK; } diff --git a/src/ctime.c b/src/ctime.c index 450a84045d..d60de40dd4 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -57,8 +57,8 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_DEFAULT_LOCKING_MODE "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), #endif -#ifdef SQLITE_DEFAULT_MMAP_LIMIT - "DEFAULT_MMAP_LIMIT=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_LIMIT), +#ifdef SQLITE_DEFAULT_MMAP_SIZE + "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE), #endif #ifdef SQLITE_DISABLE_DIRSYNC "DISABLE_DIRSYNC", @@ -66,9 +66,6 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_DISABLE_LFS "DISABLE_LFS", #endif -#ifdef SQLITE_DISABLE_MMAP - "DISABLE_MMAP", -#endif #ifdef SQLITE_ENABLE_ATOMIC_WRITE "ENABLE_ATOMIC_WRITE", #endif @@ -153,6 +150,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_LOCK_TRACE "LOCK_TRACE", #endif +#ifdef SQLITE_MAX_MMAP_SIZE + "MAX_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_MMAP_SIZE), +#endif #ifdef SQLITE_MAX_SCHEMA_RETRY "MAX_SCHEMA_RETRY=" CTIMEOPT_VAL(SQLITE_MAX_SCHEMA_RETRY), #endif diff --git a/src/global.c b/src/global.c index aec3958c1f..7b02cf2130 100644 --- a/src/global.c +++ b/src/global.c @@ -156,7 +156,8 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { (void*)0, /* pHeap */ 0, /* nHeap */ 0, 0, /* mnHeap, mxHeap */ - SQLITE_DEFAULT_MMAP_LIMIT, /* mxMmap */ + SQLITE_DEFAULT_MMAP_SIZE, /* szMmap */ + SQLITE_MAX_MMAP_SIZE, /* mxMmap */ (void*)0, /* pScratch */ 0, /* szScratch */ 0, /* nScratch */ diff --git a/src/main.c b/src/main.c index 288d4535f3..062fe40756 100644 --- a/src/main.c +++ b/src/main.c @@ -496,10 +496,16 @@ int sqlite3_config(int op, ...){ } #endif - case SQLITE_CONFIG_MMAP_LIMIT: { + case SQLITE_CONFIG_MMAP_SIZE: { + sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); - if( mxMmap<0 ) mxMmap = SQLITE_DEFAULT_MMAP_LIMIT; + if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ + mxMmap = SQLITE_MAX_MMAP_SIZE; + } sqlite3GlobalConfig.mxMmap = mxMmap; + if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; + if( szMmap>mxMmap) szMmap = mxMmap; + sqlite3GlobalConfig.szMmap = szMmap; break; } @@ -2323,7 +2329,7 @@ static int openDatabase( memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); db->autoCommit = 1; db->nextAutovac = -1; - db->mxMmap = sqlite3GlobalConfig.mxMmap; + db->szMmap = sqlite3GlobalConfig.szMmap; db->nextPagesize = 0; db->flags |= SQLITE_ShortColNames | SQLITE_AutoIndex | SQLITE_EnableTrigger #if SQLITE_DEFAULT_FILE_FORMAT<4 diff --git a/src/os.c b/src/os.c index 90cbe5492d..be2ea4cfc0 100644 --- a/src/os.c +++ b/src/os.c @@ -141,7 +141,7 @@ int sqlite3OsShmMap( return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } -#if !defined(SQLITE_DISABLE_MMAP) +#if SQLITE_MAX_MMAP_SIZE>0 /* The real implementation of xFetch and xUnfetch */ int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ DO_OS_MALLOC_TEST(id); diff --git a/src/os_unix.c b/src/os_unix.c index 778575f7b1..8dfbe61814 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -227,8 +227,8 @@ struct unixFile { int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ int nFetchOut; /* Number of outstanding xFetch refs */ sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ - sqlite3_int64 mmapOrigsize; /* Actual size of mapping at pMapRegion */ - sqlite3_int64 mmapLimit; /* Configured FCNTL_MMAP_LIMIT value */ + sqlite3_int64 mmapSizeActual; /* Actual size of mapping at pMapRegion */ + sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ void *pMapRegion; /* Memory mapped region */ #ifdef __QNXNTO__ int sectorSize; /* Device sector size */ @@ -3162,7 +3162,7 @@ static int unixRead( ); #endif -#if !defined(SQLITE_DISABLE_MMAP) +#if SQLITE_MAX_MMAP_SIZE>0 /* Deal with as much of this read request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ @@ -3283,7 +3283,7 @@ static int unixWrite( } #endif -#if !defined(SQLITE_DISABLE_MMAP) +#if SQLITE_MAX_MMAP_SIZE>0 /* Deal with as much of this write request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ @@ -3678,7 +3678,7 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ } } - if( pFile->mmapLimit>0 && nByte>pFile->mmapSize ){ + if( pFile->mmapSizeMax>0 && nByte>pFile->mmapSize ){ int rc; if( pFile->szChunk<=0 ){ if( robust_ftruncate(pFile->h, nByte) ){ @@ -3758,11 +3758,14 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } - case SQLITE_FCNTL_MMAP_LIMIT: { + case SQLITE_FCNTL_MMAP_SIZE: { i64 newLimit = *(i64*)pArg; - *(i64*)pArg = pFile->mmapLimit; + if( newLimit>sqlite3GlobalConfig.mxMmap ){ + newLimit = sqlite3GlobalConfig.mxMmap; + } + *(i64*)pArg = pFile->mmapSizeMax; if( newLimit>=0 ){ - pFile->mmapLimit = newLimit; + pFile->mmapSizeMax = newLimit; if( newLimitmmapSize ) pFile->mmapSize = newLimit; } return SQLITE_OK; @@ -4574,17 +4577,17 @@ static int unixShmUnmap( */ static void unixUnmapfile(unixFile *pFd){ assert( pFd->nFetchOut==0 ); -#ifndef SQLITE_DISABLE_MMAP +#if SQLITE_MAX_MMAP_SIZE>0 if( pFd->pMapRegion ){ - osMunmap(pFd->pMapRegion, pFd->mmapOrigsize); + osMunmap(pFd->pMapRegion, pFd->mmapSizeActual); pFd->pMapRegion = 0; pFd->mmapSize = 0; - pFd->mmapOrigsize = 0; + pFd->mmapSizeActual = 0; } #endif } -#ifndef SQLITE_DISABLE_MMAP +#if SQLITE_MAX_MMAP_SIZE>0 /* ** Return the system page size. */ @@ -4597,9 +4600,9 @@ static int unixGetPagesize(void){ return (int)sysconf(_SC_PAGESIZE); #endif } -#endif /* SQLITE_DISABLE_MMAP */ +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ -#ifndef SQLITE_DISABLE_MMAP +#if SQLITE_MAX_MMAP_SIZE>0 /* ** Attempt to set the size of the memory mapping maintained by file ** descriptor pFd to nNew bytes. Any existing mapping is discarded. @@ -4608,7 +4611,7 @@ static int unixGetPagesize(void){ ** ** unixFile.pMapRegion ** unixFile.mmapSize -** unixFile.mmapOrigsize +** unixFile.mmapSizeActual ** ** If unsuccessful, an error message is logged via sqlite3_log() and ** the three variables above are zeroed. In this case SQLite should @@ -4622,15 +4625,15 @@ static void unixRemapfile( const char *zErr = "mmap"; int h = pFd->h; /* File descriptor open on db file */ u8 *pOrig = (u8 *)pFd->pMapRegion; /* Pointer to current file mapping */ - i64 nOrig = pFd->mmapOrigsize; /* Size of pOrig region in bytes */ + i64 nOrig = pFd->mmapSizeActual; /* Size of pOrig region in bytes */ u8 *pNew = 0; /* Location of new mapping */ int flags = PROT_READ; /* Flags to pass to mmap() */ assert( pFd->nFetchOut==0 ); assert( nNew>pFd->mmapSize ); - assert( nNew<=pFd->mmapLimit ); + assert( nNew<=pFd->mmapSizeMax ); assert( nNew>0 ); - assert( pFd->mmapOrigsize>=pFd->mmapSize ); + assert( pFd->mmapSizeActual>=pFd->mmapSize ); assert( MAP_FAILED!=0 ); if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; @@ -4679,10 +4682,10 @@ static void unixRemapfile( /* If the mmap() above failed, assume that all subsequent mmap() calls ** will probably fail too. Fall back to using xRead/xWrite exclusively ** in this case. */ - pFd->mmapLimit = 0; + pFd->mmapSizeMax = 0; } pFd->pMapRegion = (void *)pNew; - pFd->mmapSize = pFd->mmapOrigsize = nNew; + pFd->mmapSize = pFd->mmapSizeActual = nNew; } #endif @@ -4703,7 +4706,7 @@ static void unixRemapfile( ** code otherwise. */ static int unixMapfile(unixFile *pFd, i64 nByte){ -#ifndef SQLITE_DISABLE_MMAP +#if SQLITE_MAX_MMAP_SIZE>0 i64 nMap = nByte; int rc; @@ -4718,8 +4721,8 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ } nMap = statbuf.st_size; } - if( nMap>pFd->mmapLimit ){ - nMap = pFd->mmapLimit; + if( nMap>pFd->mmapSizeMax ){ + nMap = pFd->mmapSizeMax; } if( nMap!=pFd->mmapSize ){ @@ -4747,13 +4750,13 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ ** release the reference by calling unixUnfetch(). */ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ -#ifndef SQLITE_DISABLE_MMAP +#if SQLITE_MAX_MMAP_SIZE>0 unixFile *pFd = (unixFile *)fd; /* The underlying database file */ #endif *pp = 0; -#ifndef SQLITE_DISABLE_MMAP - if( pFd->mmapLimit>0 ){ +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->mmapSizeMax>0 ){ if( pFd->pMapRegion==0 ){ int rc = unixMapfile(pFd, -1); if( rc!=SQLITE_OK ) return rc; @@ -5126,7 +5129,7 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; - pNew->mmapLimit = sqlite3GlobalConfig.mxMmap; + pNew->mmapSizeMax = sqlite3GlobalConfig.mxMmap; if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; diff --git a/src/os_win.c b/src/os_win.c index 034f719fbd..e56ccb50c6 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -150,12 +150,12 @@ struct winFile { winceLock local; /* Locks obtained by this instance of winFile */ winceLock *shared; /* Global shared lock memory for the file */ #endif - int nFetchOut; /* Number of outstanding xFetch references */ - HANDLE hMap; /* Handle for accessing memory mapping */ - void *pMapRegion; /* Area memory mapped */ - sqlite3_int64 mmapSize; /* Usable size of mapped region */ - sqlite3_int64 mmapOrigsize; /* Actual size of mapped region */ - sqlite3_int64 mmapLimit; /* Configured FCNTL_MMAP_LIMIT value */ + int nFetchOut; /* Number of outstanding xFetch references */ + HANDLE hMap; /* Handle for accessing memory mapping */ + void *pMapRegion; /* Area memory mapped */ + sqlite3_int64 mmapSize; /* Usable size of mapped region */ + sqlite3_int64 mmapSizeActual; /* Actual size of mapped region */ + sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ }; /* @@ -2148,7 +2148,7 @@ static int winRead( SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); -#if !defined(SQLITE_DISABLE_MMAP) +#if SQLITE_MAX_MMAP_SIZE>0 /* Deal with as much of this read request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ @@ -2214,7 +2214,7 @@ static int winWrite( OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); -#if !defined(SQLITE_DISABLE_MMAP) +#if SQLITE_MAX_MMAP_SIZE>0 /* Deal with as much of this write request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ @@ -2325,7 +2325,7 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ "winTruncate2", pFile->zPath); } -#if !defined(SQLITE_DISABLE_MMAP) +#if SQLITE_MAX_MMAP_SIZE>0 /* If the file was truncated to a size smaller than the currently ** mapped region, reduce the effective mapping size as well. SQLite will ** use read() and write() to access data beyond this point from now on. @@ -2842,10 +2842,13 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } - case SQLITE_FCNTL_MMAP_LIMIT: { + case SQLITE_FCNTL_MMAP_SIZE: { i64 newLimit = *(i64*)pArg; - *(i64*)pArg = pFile->mmapLimit; - if( newLimit>=0 ) pFile->mmapLimit = newLimit; + if( newLimit>sqlite3GlobalConfig.mxMmap ){ + newLimit = sqlite3GlobalConfig.mxMmap; + } + *(i64*)pArg = pFile->mmapSizeMax; + if( newLimit>=0 ) pFile->mmapSizeMax = newLimit; return SQLITE_OK; } } @@ -3523,7 +3526,7 @@ shmpage_out: */ static int winUnmapfile(winFile *pFile){ assert( pFile!=0 ); -#if !defined(SQLITE_DISABLE_MMAP) +#if SQLITE_MAX_MMAP_SIZE>0 if( pFile->pMapRegion ){ if( !osUnmapViewOfFile(pFile->pMapRegion) ){ pFile->lastErrno = osGetLastError(); @@ -3532,7 +3535,7 @@ static int winUnmapfile(winFile *pFile){ } pFile->pMapRegion = 0; pFile->mmapSize = 0; - pFile->mmapOrigsize = 0; + pFile->mmapSizeActual = 0; } if( pFile->hMap!=NULL ){ if( !osCloseHandle(pFile->hMap) ){ @@ -3546,7 +3549,7 @@ static int winUnmapfile(winFile *pFile){ return SQLITE_OK; } -#if !defined(SQLITE_DISABLE_MMAP) +#if SQLITE_MAX_MMAP_SIZE>0 /* ** Memory map or remap the file opened by file-descriptor pFd (if the file ** is already mapped, the existing mapping is replaced by the new). Or, if @@ -3557,7 +3560,7 @@ static int winUnmapfile(winFile *pFile){ ** the mapping to create. Otherwise, if nByte is less than zero, then the ** requested size is the size of the file on disk. The actual size of the ** created mapping is either the requested size or the value configured -** using SQLITE_FCNTL_MMAP_LIMIT, whichever is smaller. +** using SQLITE_FCNTL_MMAP_SIZE, whichever is smaller. ** ** SQLITE_OK is returned if no error occurs (even if the mapping is not ** recreated as a result of outstanding references) or an SQLite error @@ -3576,8 +3579,8 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ return SQLITE_IOERR_FSTAT; } } - if( nMap>pFd->mmapLimit ){ - nMap = pFd->mmapLimit; + if( nMap>pFd->mmapSizeMax ){ + nMap = pFd->mmapSizeMax; } nMap &= ~(sqlite3_int64)(winSysInfo.dwPageSize - 1); @@ -3629,12 +3632,12 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ } pFd->pMapRegion = pNew; pFd->mmapSize = nMap; - pFd->mmapOrigsize = nMap; + pFd->mmapSizeActual = nMap; } return SQLITE_OK; } -#endif /* !defined(SQLITE_DISABLE_MMAP) */ +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ /* ** If possible, return a pointer to a mapping of file fd starting at offset @@ -3652,8 +3655,8 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ winFile *pFd = (winFile*)fd; /* The underlying database file */ *pp = 0; -#if !defined(SQLITE_DISABLE_MMAP) - if( pFd->mmapLimit>0 ){ +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->mmapSizeMax>0 ){ if( pFd->pMapRegion==0 ){ int rc = winMapfile(pFd, -1); if( rc!=SQLITE_OK ) return rc; @@ -4129,8 +4132,8 @@ static int winOpen( pFile->hMap = NULL; pFile->pMapRegion = 0; pFile->mmapSize = 0; - pFile->mmapOrigsize = 0; - pFile->mmapLimit = sqlite3GlobalConfig.mxMmap; + pFile->mmapSizeActual = 0; + pFile->mmapSizeMax = sqlite3GlobalConfig.mxMmap; OpenCounter(+1); return rc; diff --git a/src/pager.c b/src/pager.c index 6c3376a836..96032b2896 100644 --- a/src/pager.c +++ b/src/pager.c @@ -658,7 +658,7 @@ struct Pager { u8 bUseFetch; /* True to use xFetch() */ int nMmapOut; /* Number of mmap pages currently outstanding */ - sqlite3_int64 mxMmap; /* Desired maximum mmap size */ + sqlite3_int64 szMmap; /* Desired maximum mmap size */ PgHdr *pMmapFreelist; /* List of free mmap page headers (pDirty) */ /* ** End of the routinely-changing class members @@ -774,10 +774,10 @@ static const unsigned char aJournalMagic[] = { ** The macro USEFETCH is true if we are allowed to use the xFetch and xUnfetch ** interfaces to access the database using memory-mapped I/O. */ -#ifdef SQLITE_DISABLE_MMAP -# define USEFETCH(x) 0 -#else +#if SQLITE_MAX_MMAP_SIZE>0 # define USEFETCH(x) ((x)->bUseFetch) +#else +# define USEFETCH(x) 0 #endif /* @@ -3370,16 +3370,16 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ } /* -** Invoke SQLITE_FCNTL_MMAP_LIMIT based on the current value of mxMmap. +** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of szMmap. */ static void pagerFixMaplimit(Pager *pPager){ -#if !defined(SQLITE_DISABLE_MMAP) +#if SQLITE_MAX_MMAP_SIZE>0 sqlite3_file *fd = pPager->fd; if( isOpen(fd) ){ - sqlite3_int64 mx; - pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->mxMmap>0; - mx = pPager->mxMmap; - sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT, &mx); + sqlite3_int64 sz; + pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->szMmap>0; + sz = pPager->szMmap; + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_SIZE, &sz); } #endif } @@ -3387,8 +3387,8 @@ static void pagerFixMaplimit(Pager *pPager){ /* ** Change the maximum size of any memory mapping made of the database file. */ -void sqlite3PagerSetMmapLimit(Pager *pPager, sqlite3_int64 mxMmap){ - pPager->mxMmap = mxMmap; +void sqlite3PagerSetMmapLimit(Pager *pPager, sqlite3_int64 szMmap){ + pPager->szMmap = szMmap; pagerFixMaplimit(pPager); } @@ -4766,7 +4766,7 @@ int sqlite3PagerOpen( /* pPager->pBusyHandlerArg = 0; */ pPager->xReiniter = xReinit; /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ - /* pPager->mxMmap = SQLITE_DEFAULT_MMAP_LIMIT // will be set by btree.c */ + /* pPager->szMmap = SQLITE_DEFAULT_MMAP_SIZE // will be set by btree.c */ *ppPager = pPager; return SQLITE_OK; diff --git a/src/pragma.c b/src/pragma.c index f4fc73d297..65efbd8b21 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -746,39 +746,39 @@ void sqlite3Pragma( }else /* - ** PRAGMA [database.]mmap_limit(N) + ** PRAGMA [database.]mmap_size(N) ** ** Used to set mapping size limit. The mapping size limit is ** used to limit the aggregate size of all memory mapped regions of the ** database file. If this parameter is set to zero, then memory mapping ** is not used at all. If N is negative, then the default memory map - ** limit determined by sqlite3_config(SQLITE_CONFIG_MMAP_LIMIT) is set. + ** limit determined by sqlite3_config(SQLITE_CONFIG_MMAP_SIZE) is set. ** The parameter N is measured in bytes. ** ** This value is advisory. The underlying VFS is free to memory map ** as little or as much as it wants. Except, if N is set to 0 then the ** upper layers will never invoke the xFetch interfaces to the VFS. */ - if( sqlite3StrICmp(zLeft,"mmap_limit")==0 ){ - sqlite3_int64 mx; + if( sqlite3StrICmp(zLeft,"mmap_size")==0 ){ + sqlite3_int64 sz; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( zRight ){ int ii; - sqlite3Atoi64(zRight, &mx, 1000, SQLITE_UTF8); - if( mx<0 ) mx = sqlite3GlobalConfig.mxMmap; - if( pId2->n==0 ) db->mxMmap = mx; + sqlite3Atoi64(zRight, &sz, 1000, SQLITE_UTF8); + if( sz<0 ) sz = sqlite3GlobalConfig.szMmap; + if( pId2->n==0 ) db->szMmap = sz; for(ii=db->nDb-1; ii>=0; ii--){ if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ - sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, mx); + sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, sz); } } } - mx = -1; - if( sqlite3_file_control(db,zDb,SQLITE_FCNTL_MMAP_LIMIT,&mx)==SQLITE_OK ){ -#if defined(SQLITE_DISABLE_MMAP) - mx = 0; + sz = -1; + if( sqlite3_file_control(db,zDb,SQLITE_FCNTL_MMAP_SIZE,&sz)==SQLITE_OK ){ +#if SQLITE_MAX_MMAP_SIZE==0 + sz = 0; #endif - returnSingleInt(pParse, "mmap_limit", mx); + returnSingleInt(pParse, "mmap_size", sz); } }else diff --git a/src/shell.c b/src/shell.c index 6a0e047475..7dde2203ec 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3060,7 +3060,8 @@ int main(int argc, char **argv){ sqlite3_multiplex_initialize(0, 1); #endif }else if( strcmp(z,"-mmap")==0 ){ - sqlite3_config(SQLITE_CONFIG_MMAP_LIMIT, integerValue(cmdline_option_value(argc,argv,++i))); + sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); + sqlite3_config(SQLITE_CONFIG_MMAP_SIZE, sz, sz); }else if( strcmp(z,"-vfs")==0 ){ sqlite3_vfs *pVfs = sqlite3_vfs_find(cmdline_option_value(argc,argv,++i)); if( pVfs ){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index a2e85053d6..69f79a0377 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -892,15 +892,15 @@ struct sqlite3_io_methods { ** written into memory obtained from [sqlite3_malloc()]. The caller should ** invoke [sqlite3_free()] on the result to avoid a memory leak. ** -**
  • [[SQLITE_FCNTL_MMAP_LIMIT]] -** The [SQLITE_FCNTL_MMAP_LIMIT] file control is used to query or set the +**
  • [[SQLITE_FCNTL_MMAP_SIZE]] +** The [SQLITE_FCNTL_MMAP_SIZE] file control is used to query or set the ** maximum number of bytes that will be used for memory-mapped I/O. ** The argument is a pointer to a value of type sqlite3_int64 that ** is an advisory maximum number of bytes in the file to memory map. The ** pointer is overwritten with the old value. The limit is not changed if -** the originally pointed to is negative, and so the current limit can be -** queried by passing in a pointer to a negative number. This file-control -** is used internally to implement [PRAGMA mmap_limit]. +** the value originally pointed to is negative, and so the current limit +** can be queried by passing in a pointer to a negative number. This +** file-control is used internally to implement [PRAGMA mmap_size]. ** ** */ @@ -920,7 +920,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_PRAGMA 14 #define SQLITE_FCNTL_BUSYHANDLER 15 #define SQLITE_FCNTL_TEMPFILENAME 16 -#define SQLITE_FCNTL_MMAP_LIMIT 18 +#define SQLITE_FCNTL_MMAP_SIZE 18 /* ** CAPI3REF: Mutex Handle @@ -1650,17 +1650,19 @@ struct sqlite3_mem_methods { ** the connection being passed as the second parameter is being closed. The ** third parameter is passed NULL In this case. ** -** [[SQLITE_CONFIG_MMAP_LIMIT]] -**
    SQLITE_CONFIG_MMAP_LIMIT -**
    The sole argument should be a 64-bit integer (an sqlite3_int64) that -** is the default maximum number of bytes of process address space that -** should be used for accessing each database file using memory mapping. +** [[SQLITE_CONFIG_MMAP_SIZE]] +**
    SQLITE_CONFIG_MMAP_SIZE +**
    SQLITE_CONFIG_MMAP_SIZE takes two 64-bit integer (sqlite3_int64) values +** that are the default mmap size limit (the default setting for +** [PRAGMA mmap_size]) and the maximum allowed mmap size limit. ** The default setting can be overridden by each database connection using -** either the [PRAGMA mmap_limit] command, or by using the -** [SQLITE_FCNTL_MMAP_LIMIT] file control. The value set here overrides the -** compile-time default that is set using [SQLITE_DEFAULT_MMAP_LIMIT]. -** If the argument to this option is negative, then -** the memory map limit is set to the compile-time default. +** either the [PRAGMA mmap_size] command, or by using the +** [SQLITE_FCNTL_MMAP_SIZE] file control. The maximum allowed mmap size +** cannot be changed at run-time. Nor may the maximum allowed mmap size +** exceed the compile-time maximum mmap size set by the +** [SQLITE_MAX_MMAP_SIZE] compile-time option. +** If either argument to this option is negative, then that argument is +** changed to its compile-time default. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -1684,7 +1686,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ #define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ -#define SQLITE_CONFIG_MMAP_LIMIT 22 /* sqlite3_int64 */ +#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ /* ** CAPI3REF: Database Connection Configuration Options diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 264db39d14..92a5061406 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -540,13 +540,46 @@ extern const int sqlite3one; #endif /* -** Disable MMAP on platforms where it is not supported +** Disable MMAP on platforms where it is known to not work */ #if defined(__OpenBSD__) || defined(__QNXNTO__) -# undef SQLITE_DISABLE_MMAP -# define SQLITE_DISABLE_MMAP 1 +# undef SQLITE_MAX_MMAP_SIZE +# define SQLITE_MAX_MMAP_SIZE 0 #endif +/* +** Default maximum size of memory used by memory-mapped I/O in the VFS +*/ +#ifdef __APPLE__ +# include +# if TARGET_OS_IPHONE +# undef SQLITE_MAX_MMAP_SIZE +# define SQLITE_MAX_MMAP_SIZE 0 +# endif +#endif +#ifndef SQLITE_MAX_MMAP_SIZE +# if defined(__linux__) \ + || defined(_WIN32) \ + || (defined(__APPLE__) && defined(__MACH__)) \ + || defined(__sun) +# define SQLITE_MAX_MMAP_SIZE 2147483648 +# else +# define SQLITE_MAX_MMAP_SIZE 0 +# endif +#endif + +/* +** The default MMAP_SIZE is zero on all platforms. Or, even if a larger +** default MMAP_SIZE is specified at compile-time, make sure that it does +** not exceed the maximum mmap size. +*/ +#ifndef SQLITE_DEFAULT_MMAP_SIZE +# define SQLITE_DEFAULT_MMAP_SIZE 0 +#endif +#if SQLITE_DEFAULT_MMAP_SIZE>SQLITE_MAX_MMAP_SIZE +# undef SQLITE_DEFAULT_MMAP_SIZE +# define SQLITE_DEFAULT_MMAP_SIZE SQLITE_MAX_MMAP_SIZE +#endif /* ** An instance of the following structure is used to store the busy-handler @@ -842,7 +875,7 @@ struct sqlite3 { int nDb; /* Number of backends currently in use */ int flags; /* Miscellaneous flags. See below */ i64 lastRowid; /* ROWID of most recent insert (see above) */ - i64 mxMmap; /* Default mmap_limit setting */ + i64 szMmap; /* Default mmap_size setting */ unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ @@ -2516,7 +2549,8 @@ struct Sqlite3Config { void *pHeap; /* Heap storage space */ int nHeap; /* Size of pHeap[] */ int mnReq, mxReq; /* Min and max heap requests sizes */ - sqlite3_int64 mxMmap; /* Maximum mmap() space per open file */ + sqlite3_int64 szMmap; /* mmap() space per open file */ + sqlite3_int64 mxMmap; /* Maximum value for szMmap */ void *pScratch; /* Scratch memory */ int szScratch; /* Size of each scratch buffer */ int nScratch; /* Number of scratch buffers */ diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index d8f748e6b2..c7aee53ceb 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -206,27 +206,3 @@ #ifndef SQLITE_MAX_TRIGGER_DEPTH # define SQLITE_MAX_TRIGGER_DEPTH 1000 #endif - -/* -** Default maximum size of memory used by xFetch in the VFS. -*/ -#ifdef SQLITE_DISABLE_MMAP -# undef SQLITE_DEFAULT_MMAP_LIMIT -# define SQLITE_DEFAULT_MMAP_LIMIT 0 -#endif -#ifdef __APPLE__ -# include -# if TARGET_OS_IPHONE -# define SQLITE_DEFAULT_MMAP_LIMIT 0 -# endif -#endif -#ifndef SQLITE_DEFAULT_MMAP_LIMIT -# if defined(__linux__) \ - || defined(_WIN32) \ - || (defined(__APPLE__) && defined(__MACH__)) \ - || defined(__sun) -# define SQLITE_DEFAULT_MMAP_LIMIT 268435456 /* = 256*1024*1024 */ -# else -# define SQLITE_DEFAULT_MMAP_LIMIT 0 -# endif -#endif diff --git a/src/test_config.c b/src/test_config.c index 39608dbb64..4f61f4bb2b 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -87,10 +87,10 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "lfs", "1", TCL_GLOBAL_ONLY); #endif -#ifdef SQLITE_DISABLE_MMAP - Tcl_SetVar2(interp, "sqlite_options", "mmap", "0", TCL_GLOBAL_ONLY); -#else +#if SQLITE_MAX_MMAP_SIZE>0 Tcl_SetVar2(interp, "sqlite_options", "mmap", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "mmap", "0", TCL_GLOBAL_ONLY); #endif #if 1 /* def SQLITE_MEMDEBUG */ diff --git a/test/dbstatus2.test b/test/dbstatus2.test index c2d9164fa2..2541a1a823 100644 --- a/test/dbstatus2.test +++ b/test/dbstatus2.test @@ -40,7 +40,7 @@ proc db_write {db {reset 0}} { do_test 1.1 { db close sqlite3 db test.db - execsql { PRAGMA mmap_limit = 0 } + execsql { PRAGMA mmap_size = 0 } expr {[file size test.db] / 1024} } 6 diff --git a/test/exclusive2.test b/test/exclusive2.test index d09097671d..54203e3d8f 100644 --- a/test/exclusive2.test +++ b/test/exclusive2.test @@ -28,7 +28,7 @@ ifcapable {!pager_pragmas} { # Tests in this file verify that locking_mode=exclusive causes SQLite to # use cached pages even if the database is changed on disk. This doesn't # work with mmap. -if {[permutation]!="nommap"} { +if {[permutation]=="mmap"} { finish_test return } diff --git a/test/func.test b/test/func.test index 032becc94f..4ab7688461 100644 --- a/test/func.test +++ b/test/func.test @@ -1273,7 +1273,7 @@ do_test func-29.3 { sqlite3_db_status db CACHE_MISS 1 db eval {SELECT typeof(+x) FROM t29 ORDER BY id} } {integer null real blob text} -if {[permutation] == "nommap"} { +if {[permutation] != "mmap"} { do_test func-29.4 { set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] if {$x>100} {set x many} diff --git a/test/incrblob.test b/test/incrblob.test index fa564de0cf..4277e5c4c1 100644 --- a/test/incrblob.test +++ b/test/incrblob.test @@ -123,7 +123,7 @@ foreach AutoVacuumMode [list 0 1] { forcedelete test.db test.db-journal sqlite3 db test.db - execsql "PRAGMA mmap_limit = 0" + execsql "PRAGMA mmap_size = 0" execsql "PRAGMA auto_vacuum = $AutoVacuumMode" do_test incrblob-2.$AutoVacuumMode.1 { @@ -150,7 +150,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db - execsql "PRAGMA mmap_limit = 0" + execsql "PRAGMA mmap_size = 0" # Read the last 20 bytes of the blob via a blob handle. set ::blob [db incrblob blobs v 1] @@ -173,7 +173,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db - execsql "PRAGMA mmap_limit = 0" + execsql "PRAGMA mmap_size = 0" # Write the second-to-last 20 bytes of the blob via a blob handle. # @@ -203,7 +203,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db - execsql { PRAGMA mmap_limit = 0 } + execsql { PRAGMA mmap_size = 0 } execsql { SELECT i FROM blobs } } {45} diff --git a/test/mmap1.test b/test/mmap1.test index 9618109933..0d0e3fd7a4 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -40,17 +40,17 @@ proc register_rblob_code {dbname seed} { }] } -foreach {t mmap_limit nRead c2init} { - 1.1 { PRAGMA mmap_limit = 67108864 } 4 {PRAGMA mmap_limit = 0} - 1.2 { PRAGMA mmap_limit = 53248 } 150 {PRAGMA mmap_limit = 0} - 1.3 { PRAGMA mmap_limit = 0 } 344 {PRAGMA mmap_limit = 0} - 1.4 { PRAGMA mmap_limit = 67108864 } 4 {PRAGMA mmap_limit = 67108864 } - 1.5 { PRAGMA mmap_limit = 53248 } 150 {PRAGMA mmap_limit = 67108864 } - 1.6 { PRAGMA mmap_limit = 0 } 344 {PRAGMA mmap_limit = 67108864 } +foreach {t mmap_size nRead c2init} { + 1.1 { PRAGMA mmap_size = 67108864 } 4 {PRAGMA mmap_size = 0} + 1.2 { PRAGMA mmap_size = 53248 } 150 {PRAGMA mmap_size = 0} + 1.3 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = 0} + 1.4 { PRAGMA mmap_size = 67108864 } 4 {PRAGMA mmap_size = 67108864 } + 1.5 { PRAGMA mmap_size = 53248 } 150 {PRAGMA mmap_size = 67108864 } + 1.6 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = 67108864 } } { do_multiclient_test tn { sql1 {PRAGMA page_size=1024} - sql1 $mmap_limit + sql1 $mmap_size sql2 $c2init code2 [register_rblob_code db2 0] @@ -89,7 +89,7 @@ foreach {t mmap_limit nRead c2init} { } {64 ok 149} # Check that the number of pages read by connection 1 indicates that the - # "PRAGMA mmap_limit" command worked. + # "PRAGMA mmap_size" command worked. do_test $t.$tn.5 { nRead db } $nRead } } @@ -106,7 +106,7 @@ db func rblob rblob do_execsql_test 2.1 { PRAGMA auto_vacuum = 1; - PRAGMA mmap_limit = 67108864; + PRAGMA mmap_size = 67108864; PRAGMA journal_mode = wal; CREATE TABLE t1(a, b, UNIQUE(a, b)); INSERT INTO t1 VALUES(rblob(500), rblob(500)); @@ -254,7 +254,7 @@ do_test 5.5 { } SQLITE_OK #------------------------------------------------------------------------- -# Test various mmap_limit settings. +# Test various mmap_size settings. # foreach {tn1 mmap1 mmap2} { 1 6144 167773 @@ -279,8 +279,8 @@ foreach {tn1 mmap1 mmap2} { code1 [register_rblob_code db 0] code2 [register_rblob_code db2 444] - sql1 "PRAGMA mmap_limit = $mmap1" - sql2 "PRAGMA mmap_limit = $mmap2" + sql1 "PRAGMA mmap_size = $mmap1" + sql2 "PRAGMA mmap_size = $mmap2" do_test $tn1.$tn { for {set i 1} {$i <= 100} {incr i} { diff --git a/test/pager1.test b/test/pager1.test index ab0ec67526..72e47805a4 100644 --- a/test/pager1.test +++ b/test/pager1.test @@ -2530,9 +2530,9 @@ if {$::tcl_platform(platform)=="unix"} { # an error. # foreach {tn pragma strsize} { - 1 { PRAGMA mmap_limit = 0 } 2400 + 1 { PRAGMA mmap_size = 0 } 2400 2 { } 2400 - 3 { PRAGMA mmap_limit = 0 } 4400 + 3 { PRAGMA mmap_size = 0 } 4400 4 { } 4400 } { reset_db @@ -2757,7 +2757,7 @@ do_test 42.1 { db close sqlite3_test_control_pending_byte 0x0010000 sqlite3 db test.db - db eval { PRAGMA mmap_limit = 0 } + db eval { PRAGMA mmap_size = 0 } catchsql { SELECT sum(length(y)) FROM t1 } } {1 {database disk image is malformed}} do_test 42.2 { @@ -2799,7 +2799,7 @@ do_test 43.1 { db close sqlite3 db test.db - db eval { PRAGMA mmap_limit = 0 } + db eval { PRAGMA mmap_size = 0 } db eval { SELECT * FROM t1 } sqlite3_db_status db CACHE_MISS 0 } {0 2 0} diff --git a/test/pagerfault.test b/test/pagerfault.test index 9b005c7a03..23754fa9de 100644 --- a/test/pagerfault.test +++ b/test/pagerfault.test @@ -1282,7 +1282,7 @@ faultsim_save_and_close do_faultsim_test pagerfault-28a -faults oom* -prep { faultsim_restore_and_reopen - execsql { PRAGMA mmap_limit=0 } + execsql { PRAGMA mmap_size=0 } sqlite3 db2 test.db db2 eval { SELECT count(*) FROM t2 } diff --git a/test/pageropt.test b/test/pageropt.test index a734e39a49..7191661ba5 100644 --- a/test/pageropt.test +++ b/test/pageropt.test @@ -88,7 +88,7 @@ do_test pageropt-1.4 { # must refill. # ifcapable mmap { - set x [expr {[permutation]=="nommap" ? 6 : 1}] + set x [expr {[permutation]=="mmap" ? 1 : 6}] } else { set x 6 } diff --git a/test/permutations.test b/test/permutations.test index 3aba9d9b38..9334fc49e6 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -138,10 +138,10 @@ test_suite "veryquick" -prefix "" -description { test_set $allquicktests -exclude *malloc* *ioerr* *fault* ] -test_suite "nommap" -prefix "nomm-" -description { +test_suite "mmap" -prefix "mm-" -description { Similar to veryquick. Except with memory mapping disabled. } -presql { - pragma mmap_limit = 0; + pragma mmap_size = 268435456; } -files [ test_set $allquicktests -exclude *malloc* *ioerr* *fault* -include malloc.test ] diff --git a/test/sysfault.test b/test/sysfault.test index 4ecaa1fabc..92fb534dd0 100644 --- a/test/sysfault.test +++ b/test/sysfault.test @@ -262,7 +262,7 @@ do_faultsim_test 4 -faults vfsfault-* -prep { faultsim_restore_and_reopen file_control_chunksize_test db main 8192 execsql { - PRAGMA mmap_limit = 1000000; + PRAGMA mmap_size = 1000000; } } -body { test_syscall errno mmap EACCES diff --git a/test/wal.test b/test/wal.test index 66000d0951..d9f2145b2e 100644 --- a/test/wal.test +++ b/test/wal.test @@ -728,7 +728,7 @@ do_test wal-11.9 { } {37 1} sqlite3_wal db test.db set nWal 39 -if {[permutation]=="nommap"} {set nWal 37} +if {[permutation]!="mmap"} {set nWal 37} ifcapable !mmap {set nWal 37} do_test wal-11.10 { execsql { diff --git a/test/wal5.test b/test/wal5.test index 4a6309886b..68750f1479 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -242,7 +242,7 @@ foreach {testprefix do_wal_checkpoint} { # However, in mmap() mode, the db is pre-allocated to 2 pages at the # start of the checkpoint, even though page 2 cannot be written. set nDb 2 - if {[permutation]=="no-mmap"} {set nDb 1} + if {[permutation]!="mmap"} {set nDb 1} ifcapable !mmap {set nDb 1} do_test 2.3.$tn.8 { file_page_counts } [list $nDb 4 2 4] } diff --git a/test/win32lock.test b/test/win32lock.test index 50c9f1d5f4..724172065f 100644 --- a/test/win32lock.test +++ b/test/win32lock.test @@ -27,7 +27,7 @@ proc xLog {error_code msg} { lappend ::log $msg } sqlite3 db test.db -db eval {PRAGMA mmap_limit=0} +db eval {PRAGMA mmap_size=0} do_test win32lock-1.1 { db eval { From 5824e05315453704c332f5cf64665932a059260f Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 15 Apr 2013 20:08:27 +0000 Subject: [PATCH 135/204] Expand scope of the SQLITE_MAX_MMAP_SIZE define for the Win32 VFS. FossilOrigin-Name: f4b8faab45e217f227bd7ce65d4741754731b8cd --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 18 +++++++++++++++--- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 215a7b5881..a560dd55b0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactoring\sthe\smmap\sinterface.\s\sThe\scontrolling\spragma\sis\snow\s"mmap_size"\ninstead\sof\s"mmap_limit".\s\sAlso\schange\sSQLITE_CONFIG_MMAP_LIMIT\sand\nSQLITE_FCNTL_MMAP_LIMIT\sto\sSQLITE_CONFIG_MMAP_SIZE\sand\nSQLITE_FCNTL_MMAP_SIZE,\srespecctively.\s\s\nThe\sdefault\smmap_size\sis\snow\salways\s0,\smeaning\sthat\nmemory\smapped\sI/O\sis\soff\sby\sdefault.\s\sThere\sis\sa\snew\scompile-time\soption\nSQLITE_MAX_MMAP_SIZE\sthat\sdetermines\sa\shard\supper\sbound\son\sthe\smmap_size.\nSetting\sSQLITE_MAX_MMAP_SIZE\sto\szero\sdisables\sthe\smemory-mapped\sI/O\slogic\nand\scauses\sit\sto\sbe\somitted\sfrom\sthe\sbuild.\s\sAn\sextra\sargument\sis\sadded\nto\sSQLITE_CONFIG_MMAP_SIZE\sthat\scan\soptionally\slower\sthe\sSQLITE_MAX_MMAP_SIZE\nat\sstart-time.\sThe\sSQLITE_MAX_MMAP_SIZE\sis\szero\sfor\splatforms\swhere\swe\s\nknow\sthat\sit\sdoes\snot\swork,\smeaning\sthat\sit\scannot\sbe\sturned\son\sby\smistake\non\sthose\splatforms. -D 2013-04-15T17:03:42.119 +C Expand\sscope\sof\sthe\sSQLITE_MAX_MMAP_SIZE\sdefine\sfor\sthe\sWin32\sVFS. +D 2013-04-15T20:08:27.655 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 0a561eae5965c9371300b0419027f5ae9f847af2 -F src/os_win.c 873bbc5dff8eb9f2bf8d9287d29df082eacb8391 +F src/os_win.c 673b3e3d1fa3040d8d95a7f1f5e0e553aed56cfb F src/pager.c a55adacb1842b83354198c408e7adde95ecd1189 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3412424990c93d2978e819e6099811f1cdde316d -R aa0c6ea54adf9732b395ebf6e1c98243 -U drh -Z 71eb761a9109889dcfada6d3231b23c2 +P ea1404a10abd7f68e1f8e0708c8a3199d1f79665 +R 4ce5175815a884ac178e2248b6e66ee1 +U mistachkin +Z 55620fd00464e5f458c15f94b0f9c9d2 diff --git a/manifest.uuid b/manifest.uuid index 65cc51dd9d..b2fbd3d393 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ea1404a10abd7f68e1f8e0708c8a3199d1f79665 \ No newline at end of file +f4b8faab45e217f227bd7ce65d4741754731b8cd \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index e56ccb50c6..12eefed8be 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -150,12 +150,14 @@ struct winFile { winceLock local; /* Locks obtained by this instance of winFile */ winceLock *shared; /* Global shared lock memory for the file */ #endif +#if SQLITE_MAX_MMAP_SIZE>0 int nFetchOut; /* Number of outstanding xFetch references */ HANDLE hMap; /* Handle for accessing memory mapping */ void *pMapRegion; /* Area memory mapped */ sqlite3_int64 mmapSize; /* Usable size of mapped region */ sqlite3_int64 mmapSizeActual; /* Actual size of mapped region */ sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ +#endif }; /* @@ -2068,8 +2070,10 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ #endif } +#if SQLITE_MAX_MMAP_SIZE>0 /* Forward references to VFS methods */ static int winUnmapfile(winFile*); +#endif /* ** Close a file. @@ -2093,8 +2097,10 @@ static int winClose(sqlite3_file *id){ OSTRACE(("CLOSE %d\n", pFile->h)); assert( pFile->h!=NULL && pFile->h!=INVALID_HANDLE_VALUE ); +#if SQLITE_MAX_MMAP_SIZE>0 rc = winUnmapfile(pFile); if( rc!=SQLITE_OK ) return rc; +#endif do{ rc = osCloseHandle(pFile->h); @@ -2842,6 +2848,7 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } +#if SQLITE_MAX_MMAP_SIZE>0 case SQLITE_FCNTL_MMAP_SIZE: { i64 newLimit = *(i64*)pArg; if( newLimit>sqlite3GlobalConfig.mxMmap ){ @@ -2851,6 +2858,7 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ if( newLimit>=0 ) pFile->mmapSizeMax = newLimit; return SQLITE_OK; } +#endif } return SQLITE_NOTFOUND; } @@ -3524,9 +3532,9 @@ shmpage_out: /* ** Cleans up the mapped region of the specified file, if any. */ +#if SQLITE_MAX_MMAP_SIZE>0 static int winUnmapfile(winFile *pFile){ assert( pFile!=0 ); -#if SQLITE_MAX_MMAP_SIZE>0 if( pFile->pMapRegion ){ if( !osUnmapViewOfFile(pFile->pMapRegion) ){ pFile->lastErrno = osGetLastError(); @@ -3545,11 +3553,9 @@ static int winUnmapfile(winFile *pFile){ } pFile->hMap = NULL; } -#endif return SQLITE_OK; } -#if SQLITE_MAX_MMAP_SIZE>0 /* ** Memory map or remap the file opened by file-descriptor pFd (if the file ** is already mapped, the existing mapping is replaced by the new). Or, if @@ -3652,7 +3658,9 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ ** release the reference by calling unixUnfetch(). */ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ +#if SQLITE_MAX_MMAP_SIZE>0 winFile *pFd = (winFile*)fd; /* The underlying database file */ +#endif *pp = 0; #if SQLITE_MAX_MMAP_SIZE>0 @@ -3681,6 +3689,7 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ ** may now be invalid and should be unmapped. */ static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ +#if SQLITE_MAX_MMAP_SIZE>0 winFile *pFd = (winFile*)fd; /* The underlying database file */ /* If p==0 (unmap the entire file) then there must be no outstanding @@ -3702,6 +3711,7 @@ static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ } assert( pFd->nFetchOut>=0 ); +#endif return SQLITE_OK; } @@ -4129,11 +4139,13 @@ static int winOpen( } pFile->lastErrno = NO_ERROR; pFile->zPath = zName; +#if SQLITE_MAX_MMAP_SIZE>0 pFile->hMap = NULL; pFile->pMapRegion = 0; pFile->mmapSize = 0; pFile->mmapSizeActual = 0; pFile->mmapSizeMax = sqlite3GlobalConfig.mxMmap; +#endif OpenCounter(+1); return rc; From 0f2340132ff70a7ffd5d36ae256c47e57ec789f1 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 16 Apr 2013 14:52:47 +0000 Subject: [PATCH 136/204] Disable memory-mapped I/O when the codec is enabled. FossilOrigin-Name: 340cca3079cd7aac2f51071435666e9e00ed8bd5 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 3 +++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a560dd55b0..e954f9c948 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Expand\sscope\sof\sthe\sSQLITE_MAX_MMAP_SIZE\sdefine\sfor\sthe\sWin32\sVFS. -D 2013-04-15T20:08:27.655 +C Disable\smemory-mapped\sI/O\swhen\sthe\scodec\sis\senabled. +D 2013-04-16T14:52:47.532 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,7 +168,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 0a561eae5965c9371300b0419027f5ae9f847af2 F src/os_win.c 673b3e3d1fa3040d8d95a7f1f5e0e553aed56cfb -F src/pager.c a55adacb1842b83354198c408e7adde95ecd1189 +F src/pager.c 6c3a8a5d665498b0344395a2c9f82d5abc4cc771 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ea1404a10abd7f68e1f8e0708c8a3199d1f79665 -R 4ce5175815a884ac178e2248b6e66ee1 -U mistachkin -Z 55620fd00464e5f458c15f94b0f9c9d2 +P f4b8faab45e217f227bd7ce65d4741754731b8cd +R 6717674ae7188e500403f1978f39e75b +U drh +Z 9185ac7186177dd4c0f45720166ae532 diff --git a/manifest.uuid b/manifest.uuid index b2fbd3d393..f78765dbdd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f4b8faab45e217f227bd7ce65d4741754731b8cd \ No newline at end of file +340cca3079cd7aac2f51071435666e9e00ed8bd5 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 96032b2896..1c8b5df651 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5220,6 +5220,9 @@ int sqlite3PagerAcquire( ** temporary or in-memory database. */ const int bMmapOk = (pgno!=1 && USEFETCH(pPager) && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY)) +#ifdef SQLITE_HAS_CODEC + && pPager->xCodec==0 +#endif ); assert( pPager->eState>=PAGER_READER ); From 50287f5c0ca7730074da612f209541963e466e9e Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 16 Apr 2013 14:58:52 +0000 Subject: [PATCH 137/204] Remove the unused SQLITE_CURDIR macro. FossilOrigin-Name: 16f7ba4afaa69f6150b4defd4ddf5800f5d23084 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/ctime.c | 3 --- src/os.h | 8 -------- src/test_config.c | 2 +- 5 files changed, 10 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index e954f9c948..7f0f08bfbd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\smemory-mapped\sI/O\swhen\sthe\scodec\sis\senabled. -D 2013-04-16T14:52:47.532 +C Remove\sthe\sunused\sSQLITE_CURDIR\smacro. +D 2013-04-16T14:58:52.577 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -133,7 +133,7 @@ F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c 726e88b7282e7f348f805437db3222f97a136d35 +F src/ctime.c cb3cd5bf69bdf4ff3e1fc8ca3b1f0e405729dd80 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 @@ -164,7 +164,7 @@ F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be -F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 +F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 0a561eae5965c9371300b0419027f5ae9f847af2 F src/os_win.c 673b3e3d1fa3040d8d95a7f1f5e0e553aed56cfb @@ -203,7 +203,7 @@ F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de F src/test_btree.c 5b89601dcb42a33ba8b820a6b763cc9cb48bac16 -F src/test_config.c 49c99b714c1dab68aa6b37d53681a4278982be8d +F src/test_config.c 95bb33e9dcaa340a296c0bf0e0ba3d1a1c8004c0 F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c 8f786bfd0ad48030cf2a06fb1f050e9c60a150d7 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f4b8faab45e217f227bd7ce65d4741754731b8cd -R 6717674ae7188e500403f1978f39e75b +P 340cca3079cd7aac2f51071435666e9e00ed8bd5 +R 189962300d1abed770df02e09aa73052 U drh -Z 9185ac7186177dd4c0f45720166ae532 +Z 5f64b27d98cd9642b02d695bfb0e5a2b diff --git a/manifest.uuid b/manifest.uuid index f78765dbdd..367855f236 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -340cca3079cd7aac2f51071435666e9e00ed8bd5 \ No newline at end of file +16f7ba4afaa69f6150b4defd4ddf5800f5d23084 \ No newline at end of file diff --git a/src/ctime.c b/src/ctime.c index d60de40dd4..e9d023401f 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -48,9 +48,6 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_COVERAGE_TEST "COVERAGE_TEST", #endif -#ifdef SQLITE_CURDIR - "CURDIR", -#endif #ifdef SQLITE_DEBUG "DEBUG", #endif diff --git a/src/os.h b/src/os.h index 82e07ee1cb..070a2ddd17 100644 --- a/src/os.h +++ b/src/os.h @@ -99,14 +99,6 @@ # define SQLITE_OS_WINRT 0 #endif -/* -** When compiled for WinCE or WinRT, there is no concept of the current -** directory. - */ -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT -# define SQLITE_CURDIR 1 -#endif - /* If the SET_FULLSYNC macro is not defined above, then make it ** a no-op */ diff --git a/src/test_config.c b/src/test_config.c index 4f61f4bb2b..534727a080 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -57,7 +57,7 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options","casesensitivelike","0",TCL_GLOBAL_ONLY); #endif -#ifdef SQLITE_CURDIR +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT Tcl_SetVar2(interp, "sqlite_options", "curdir", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "curdir", "0", TCL_GLOBAL_ONLY); From d16d0bc58b44ad97c6c12fd01600400f4776f653 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 16 Apr 2013 18:24:34 +0000 Subject: [PATCH 138/204] Omit sqlite3_compileoption_used() entries that are not explicitly stated on the compiler command-line, except keep SQLITE_THREADSAFE around so that there is always at least one compilation option. FossilOrigin-Name: 109d6767d2626741db1d15ed55e699ac3184a536 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/ctime.c | 20 +++++++++----------- src/sqliteInt.h | 13 ++++++++----- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 7f0f08bfbd..993bd65c98 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sunused\sSQLITE_CURDIR\smacro. -D 2013-04-16T14:58:52.577 +C Omit\ssqlite3_compileoption_used()\sentries\sthat\sare\snot\sexplicitly\sstated\non\sthe\scompiler\scommand-line,\sexcept\skeep\sSQLITE_THREADSAFE\saround\sso\sthat\nthere\sis\salways\sat\sleast\sone\scompilation\soption. +D 2013-04-16T18:24:34.635 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -133,7 +133,7 @@ F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c cb3cd5bf69bdf4ff3e1fc8ca3b1f0e405729dd80 +F src/ctime.c 387589abe92946f47dac6fa1d4479af977b4f11b F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 @@ -185,7 +185,7 @@ F src/shell.c 06bca04044b312ccadc43bf72385e7440cf60d1b F src/sqlite.h.in d897fd15b9c74d4e78e4c42319484ceb1649c5d7 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 -F src/sqliteInt.h 9933ad95b2c1e017f16428e9c0a929867598c2ae +F src/sqliteInt.h 2a83cfec9963372b636b0cabd4b200c1f1074a99 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 340cca3079cd7aac2f51071435666e9e00ed8bd5 -R 189962300d1abed770df02e09aa73052 +P 16f7ba4afaa69f6150b4defd4ddf5800f5d23084 +R 28c9578f8f913595ef0f108427330fb4 U drh -Z 5f64b27d98cd9642b02d695bfb0e5a2b +Z e5270d047eef465302b7108533238f0a diff --git a/manifest.uuid b/manifest.uuid index 367855f236..dd03c38f4f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -16f7ba4afaa69f6150b4defd4ddf5800f5d23084 \ No newline at end of file +109d6767d2626741db1d15ed55e699ac3184a536 \ No newline at end of file diff --git a/src/ctime.c b/src/ctime.c index e9d023401f..804bed704c 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -54,7 +54,7 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_DEFAULT_LOCKING_MODE "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), #endif -#ifdef SQLITE_DEFAULT_MMAP_SIZE +#if defined(SQLITE_DEFAULT_MMAP_SIZE) && !defined(SQLITE_DEFAULT_MMAP_SIZE_xc) "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE), #endif #ifdef SQLITE_DISABLE_DIRSYNC @@ -147,7 +147,7 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_LOCK_TRACE "LOCK_TRACE", #endif -#ifdef SQLITE_MAX_MMAP_SIZE +#if defined(SQLITE_MAX_MMAP_SIZE) && !defined(SQLITE_MAX_MMAP_SIZE_xc) "MAX_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_MMAP_SIZE), #endif #ifdef SQLITE_MAX_SCHEMA_RETRY @@ -207,11 +207,6 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_OMIT_CHECK "OMIT_CHECK", #endif -/* // redundant -** #ifdef SQLITE_OMIT_COMPILEOPTION_DIAGS -** "OMIT_COMPILEOPTION_DIAGS", -** #endif -*/ #ifdef SQLITE_OMIT_COMPLETE "OMIT_COMPLETE", #endif @@ -353,13 +348,13 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_TCL "TCL", #endif -#ifdef SQLITE_TEMP_STORE +#if defined(SQLITE_TEMP_STORE) && !defined(SQLITE_TEMP_STORE_xc) "TEMP_STORE=" CTIMEOPT_VAL(SQLITE_TEMP_STORE), #endif #ifdef SQLITE_TEST "TEST", #endif -#ifdef SQLITE_THREADSAFE +#if defined(SQLITE_THREADSAFE) "THREADSAFE=" CTIMEOPT_VAL(SQLITE_THREADSAFE), #endif #ifdef SQLITE_USE_ALLOCA @@ -385,8 +380,11 @@ int sqlite3_compileoption_used(const char *zOptName){ /* Since ArraySize(azCompileOpt) is normally in single digits, a ** linear search is adequate. No need for a binary search. */ for(i=0; iSQLITE_MAX_MMAP_SIZE # undef SQLITE_DEFAULT_MMAP_SIZE From 99b390827b9f4de28ea2095d8e26047c6041d532 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 17 Apr 2013 12:19:48 +0000 Subject: [PATCH 139/204] Update the command-line shell to give an exit code of 0 when the ".quit" command is run from the command-line. FossilOrigin-Name: 467e218120d63d0f34c447f7324bac1364ac0eac --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 993bd65c98..74612d5403 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\ssqlite3_compileoption_used()\sentries\sthat\sare\snot\sexplicitly\sstated\non\sthe\scompiler\scommand-line,\sexcept\skeep\sSQLITE_THREADSAFE\saround\sso\sthat\nthere\sis\salways\sat\sleast\sone\scompilation\soption. -D 2013-04-16T18:24:34.635 +C Update\sthe\scommand-line\sshell\sto\sgive\san\sexit\scode\sof\s0\swhen\sthe\s".quit"\ncommand\sis\srun\sfrom\sthe\scommand-line. +D 2013-04-17T12:19:48.061 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 -F src/shell.c 06bca04044b312ccadc43bf72385e7440cf60d1b +F src/shell.c a303eb4529ee78a68ee54e8ba6941f76ee5601d2 F src/sqlite.h.in d897fd15b9c74d4e78e4c42319484ceb1649c5d7 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 16f7ba4afaa69f6150b4defd4ddf5800f5d23084 -R 28c9578f8f913595ef0f108427330fb4 +P 109d6767d2626741db1d15ed55e699ac3184a536 +R 11f575757c220b40832c775a36cd7f16 U drh -Z e5270d047eef465302b7108533238f0a +Z 387afe95e75dc05ba67ce426c2ff6b48 diff --git a/manifest.uuid b/manifest.uuid index dd03c38f4f..fd6a44aeb7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -109d6767d2626741db1d15ed55e699ac3184a536 \ No newline at end of file +467e218120d63d0f34c447f7324bac1364ac0eac \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 7dde2203ec..1bb1b377b3 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3166,7 +3166,7 @@ int main(int argc, char **argv){ z = cmdline_option_value(argc,argv,++i); if( z[0]=='.' ){ rc = do_meta_command(z, &data); - if( rc && bail_on_error ) return rc; + if( rc && bail_on_error ) return rc==2 ? 0 : rc; }else{ open_db(&data); rc = shell_exec(data.db, z, shell_callback, &data, &zErrMsg); @@ -3190,6 +3190,7 @@ int main(int argc, char **argv){ */ if( zFirstCmd[0]=='.' ){ rc = do_meta_command(zFirstCmd, &data); + if( rc==2 ) rc = 0; }else{ open_db(&data); rc = shell_exec(data.db, zFirstCmd, shell_callback, &data, &zErrMsg); From 4670f6d5f60e8172d36d663b19353bfef06f0760 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 17 Apr 2013 14:04:52 +0000 Subject: [PATCH 140/204] Improvements to some of the comments that are used to render documentation concerning loadable extensions. No changes to code. FossilOrigin-Name: 76f4e31245fd1676a4520a2f7488bca6eb981e4a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 74612d5403..7d65c5f899 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\scommand-line\sshell\sto\sgive\san\sexit\scode\sof\s0\swhen\sthe\s".quit"\ncommand\sis\srun\sfrom\sthe\scommand-line. -D 2013-04-17T12:19:48.061 +C Improvements\sto\ssome\sof\sthe\scomments\sthat\sare\sused\sto\srender\sdocumentation\nconcerning\sloadable\sextensions.\s\sNo\schanges\sto\scode. +D 2013-04-17T14:04:52.335 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -182,7 +182,7 @@ F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c a303eb4529ee78a68ee54e8ba6941f76ee5601d2 -F src/sqlite.h.in d897fd15b9c74d4e78e4c42319484ceb1649c5d7 +F src/sqlite.h.in 31442751f3a63a538777188b6dfe70aebed338ac F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 2a83cfec9963372b636b0cabd4b200c1f1074a99 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 109d6767d2626741db1d15ed55e699ac3184a536 -R 11f575757c220b40832c775a36cd7f16 +P 467e218120d63d0f34c447f7324bac1364ac0eac +R 8ddf4f94c02f24d9b9c0c2b780ed2098 U drh -Z 387afe95e75dc05ba67ce426c2ff6b48 +Z d6edc377ec44c72133adc32403868eee diff --git a/manifest.uuid b/manifest.uuid index fd6a44aeb7..b12409c708 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -467e218120d63d0f34c447f7324bac1364ac0eac \ No newline at end of file +76f4e31245fd1676a4520a2f7488bca6eb981e4a \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 69f79a0377..f4f02d3eac 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -4218,7 +4218,7 @@ void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); ** the content before returning. ** ** The typedef is necessary to work around problems in certain -** C++ compilers. See ticket #2191. +** C++ compilers. */ typedef void (*sqlite3_destructor_type)(void*); #define SQLITE_STATIC ((sqlite3_destructor_type)0) @@ -5017,7 +5017,7 @@ int sqlite3_table_column_metadata( ** ^This interface loads an SQLite extension library from the named file. ** ** ^The sqlite3_load_extension() interface attempts to load an -** SQLite extension library contained in the file zFile. +** [SQLite extension] library contained in the file zFile. ** ** ^The entry point is zProc. ** ^zProc may be 0, in which case the name of the entry point @@ -5047,11 +5047,11 @@ int sqlite3_load_extension( ** CAPI3REF: Enable Or Disable Extension Loading ** ** ^So as not to open security holes in older applications that are -** unprepared to deal with extension loading, and as a means of disabling -** extension loading while evaluating user-entered SQL, the following API +** unprepared to deal with [extension loading], and as a means of disabling +** [extension loading] while evaluating user-entered SQL, the following API ** is provided to turn the [sqlite3_load_extension()] mechanism on and off. ** -** ^Extension loading is off by default. See ticket #1863. +** ^Extension loading is off by default. ** ^Call the sqlite3_enable_load_extension() routine with onoff==1 ** to turn extension loading on and call it with onoff==0 to turn ** it back off again. @@ -5063,7 +5063,7 @@ int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** ** ^This interface causes the xEntryPoint() function to be invoked for ** each new [database connection] that is created. The idea here is that -** xEntryPoint() is the entry point for a statically linked SQLite extension +** xEntryPoint() is the entry point for a statically linked [SQLite extension] ** that is to be automatically loaded into all new database connections. ** ** ^(Even though the function prototype shows that xEntryPoint() takes From 7a411f441bd10739266df96fb029b0c1068e2278 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 17 Apr 2013 17:33:17 +0000 Subject: [PATCH 141/204] In the command-line shell, allow a dot-command to occur after a multi-line c-style comment. FossilOrigin-Name: e2c94ab930f0e5a6dbe8cdd34ebb8cfeeedca56c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 7d65c5f899..3e8e1903cb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\ssome\sof\sthe\scomments\sthat\sare\sused\sto\srender\sdocumentation\nconcerning\sloadable\sextensions.\s\sNo\schanges\sto\scode. -D 2013-04-17T14:04:52.335 +C In\sthe\scommand-line\sshell,\sallow\sa\sdot-command\sto\soccur\safter\sa\nmulti-line\sc-style\scomment. +D 2013-04-17T17:33:17.457 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 -F src/shell.c a303eb4529ee78a68ee54e8ba6941f76ee5601d2 +F src/shell.c aca9d94653decd4496846dee0c7ba83eaf96a46d F src/sqlite.h.in 31442751f3a63a538777188b6dfe70aebed338ac F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 467e218120d63d0f34c447f7324bac1364ac0eac -R 8ddf4f94c02f24d9b9c0c2b780ed2098 +P 76f4e31245fd1676a4520a2f7488bca6eb981e4a +R 7329574a8fd763912288e6955296b961 U drh -Z d6edc377ec44c72133adc32403868eee +Z d7a8232879b542fa4edbd90644b25d07 diff --git a/manifest.uuid b/manifest.uuid index b12409c708..2d365214d6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -76f4e31245fd1676a4520a2f7488bca6eb981e4a \ No newline at end of file +e2c94ab930f0e5a6dbe8cdd34ebb8cfeeedca56c \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 1bb1b377b3..13a2f70cbc 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2784,6 +2784,10 @@ static int process_input(struct callback_data *p, FILE *in){ free(zSql); zSql = 0; nSql = 0; + }else if( zSql && _all_whitespace(zSql) ){ + free(zSql); + zSql = 0; + nSql = 0; } } if( zSql ){ From 87f9caa85ca7249d13d814993ac6e375d16b8535 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 17 Apr 2013 18:56:16 +0000 Subject: [PATCH 142/204] In the mptester, add --glob and --notglob and --testcase. Make --exit work on the main thread. Enable the load_extension() SQL function. FossilOrigin-Name: c273c171f511475045ef0aa68ecf8e22b8351996 --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 38 +++++++++++++++++++++++++++++++++++++- 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3e8e1903cb..8c393daed2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\scommand-line\sshell,\sallow\sa\sdot-command\sto\soccur\safter\sa\nmulti-line\sc-style\scomment. -D 2013-04-17T17:33:17.457 +C In\sthe\smptester,\sadd\s--glob\sand\s--notglob\sand\s--testcase.\s\sMake\s--exit\swork\non\sthe\smain\sthread.\s\sEnable\sthe\sload_extension()\sSQL\sfunction. +D 2013-04-17T18:56:16.833 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 -F mptest/mptest.c 2a263e88f18762ac5d7fb56749d2d2ea8e22cb93 +F mptest/mptest.c 36c511bf0b562ccadede7cd31208dc1ec75b7ce2 F mptest/multiwrite01.test 81fbc17657964889b60750bd7bbb1deffe8f4d42 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 76f4e31245fd1676a4520a2f7488bca6eb981e4a -R 7329574a8fd763912288e6955296b961 +P e2c94ab930f0e5a6dbe8cdd34ebb8cfeeedca56c +R d244e7c9ac86257619e9ec497e6803a5 U drh -Z d7a8232879b542fa4edbd90644b25d07 +Z 0c9d37cd97d14173f04c5765259a70d1 diff --git a/manifest.uuid b/manifest.uuid index 2d365214d6..6821f27d9a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e2c94ab930f0e5a6dbe8cdd34ebb8cfeeedca56c \ No newline at end of file +c273c171f511475045ef0aa68ecf8e22b8351996 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index e47be5ae6d..8dc95be7bc 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -898,13 +898,24 @@ static void runScript( ** Exit this process. If N>0 then exit without shutting down ** SQLite. (In other words, simulate a crash.) */ - if( strcmp(zCmd, "exit")==0 && iClient>0 ){ + if( strcmp(zCmd, "exit")==0 ){ int rc = atoi(azArg[0]); finishScript(iClient, taskId, 1); if( rc==0 ) sqlite3_close(g.db); exit(rc); }else + /* + ** --testcase NAME + ** + ** Exit this process. If N>0 then exit without shutting down + ** SQLite. (In other words, simulate a crash.) + */ + if( strcmp(zCmd, "testcase")==0 ){ + if( g.iTrace==1 ) logMessage("%.*s", len - 1, zScript+ii); + stringReset(&sResult); + }else + /* ** --finish ** @@ -942,6 +953,30 @@ static void runScript( stringReset(&sResult); }else + /* + ** --glob ANSWER... + ** --notglob ANSWER.... + ** + ** Check to see if output does or does not match the glob pattern + ** ANSWER. + */ + if( strcmp(zCmd, "glob")==0 || strcmp(zCmd, "notglob")==0 ){ + int jj; + char *zAns = zScript+ii; + char *zCopy; + int isGlob = (zCmd[0]=='g'); + for(jj=9-3*isGlob; jj Date: Wed, 17 Apr 2013 19:42:17 +0000 Subject: [PATCH 143/204] Fix the --match command in mptester so that it checks the entire result, not just a prefix of the result. FossilOrigin-Name: 3e3ecad2b5d6c97032f2a9fb920c1b8a16ab8ef5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 8c393daed2..6d229bb880 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\smptester,\sadd\s--glob\sand\s--notglob\sand\s--testcase.\s\sMake\s--exit\swork\non\sthe\smain\sthread.\s\sEnable\sthe\sload_extension()\sSQL\sfunction. -D 2013-04-17T18:56:16.833 +C Fix\sthe\s--match\scommand\sin\smptester\sso\sthat\sit\schecks\sthe\sentire\sresult,\nnot\sjust\sa\sprefix\sof\sthe\sresult. +D 2013-04-17T19:42:17.479 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 -F mptest/mptest.c 36c511bf0b562ccadede7cd31208dc1ec75b7ce2 +F mptest/mptest.c 23365b9f65f220f1206caed7bcee651538a31559 F mptest/multiwrite01.test 81fbc17657964889b60750bd7bbb1deffe8f4d42 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e2c94ab930f0e5a6dbe8cdd34ebb8cfeeedca56c -R d244e7c9ac86257619e9ec497e6803a5 +P c273c171f511475045ef0aa68ecf8e22b8351996 +R c2176482a0665dc72d6ff4f34c638c84 U drh -Z 0c9d37cd97d14173f04c5765259a70d1 +Z 66ee4dc6efdb5b603ad21e244f0428fb diff --git a/manifest.uuid b/manifest.uuid index 6821f27d9a..60b8e3a6e5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c273c171f511475045ef0aa68ecf8e22b8351996 \ No newline at end of file +3e3ecad2b5d6c97032f2a9fb920c1b8a16ab8ef5 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 8dc95be7bc..cb6dd55d91 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -945,7 +945,7 @@ static void runScript( char *zAns = zScript+ii; for(jj=7; jj Date: Thu, 18 Apr 2013 02:55:54 +0000 Subject: [PATCH 144/204] Do not suppress the ORDER BY clause on a virtual table query if an IN constraint is used. Fix for ticket [f69b96e3076e]. Testing done on TH3 using cov1/where37.test. FossilOrigin-Name: 61b2a7be3b9c04bf45bffa93a7d3a480fc5c947a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 19 ++++++++----------- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 6d229bb880..d34134f033 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s--match\scommand\sin\smptester\sso\sthat\sit\schecks\sthe\sentire\sresult,\nnot\sjust\sa\sprefix\sof\sthe\sresult. -D 2013-04-17T19:42:17.479 +C Do\snot\ssuppress\sthe\sORDER\sBY\sclause\son\sa\svirtual\stable\squery\sif\san\sIN\nconstraint\sis\sused.\s\sFix\sfor\sticket\s[f69b96e3076e].\s\sTesting\sdone\son\nTH3\susing\scov1/where37.test. +D 2013-04-18T02:55:54.826 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -258,7 +258,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 4ad2329c439a30ddb915a780f6f80bdffafe3a64 +F src/where.c e63f84e5583133c77573b78d3696dccd97bfd00d F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c273c171f511475045ef0aa68ecf8e22b8351996 -R c2176482a0665dc72d6ff4f34c638c84 +P 3e3ecad2b5d6c97032f2a9fb920c1b8a16ab8ef5 +R 6fada9489bb28405c28c5a46dce1bedb U drh -Z 66ee4dc6efdb5b603ad21e244f0428fb +Z da33d363d10fd9678c928e7a78f01d61 diff --git a/manifest.uuid b/manifest.uuid index 60b8e3a6e5..8101a7d5ae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3e3ecad2b5d6c97032f2a9fb920c1b8a16ab8ef5 \ No newline at end of file +61b2a7be3b9c04bf45bffa93a7d3a480fc5c947a \ No newline at end of file diff --git a/src/where.c b/src/where.c index bd8a7541fa..a0dc868422 100644 --- a/src/where.c +++ b/src/where.c @@ -2275,9 +2275,8 @@ static void bestVirtualIndex(WhereBestIdx *p){ struct sqlite3_index_constraint *pIdxCons; struct sqlite3_index_constraint_usage *pUsage; WhereTerm *pTerm; - int i, j, k; + int i, j; int nOrderBy; - int sortOrder; /* Sort order for IN clauses */ int bAllowIN; /* Allow IN optimizations */ double rCost; @@ -2376,7 +2375,6 @@ static void bestVirtualIndex(WhereBestIdx *p){ return; } - sortOrder = SQLITE_SO_ASC; pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; for(i=0; inConstraint; i++, pIdxCons++){ if( pUsage[i].argvIndex>0 ){ @@ -2391,12 +2389,12 @@ static void bestVirtualIndex(WhereBestIdx *p){ ** repeated in the output. */ break; } - for(k=0; knOrderBy; k++){ - if( pIdxInfo->aOrderBy[k].iColumn==pIdxCons->iColumn ){ - sortOrder = pIdxInfo->aOrderBy[k].desc; - break; - } - } + /* A virtual table that is constrained by an IN clause may not + ** consume the ORDER BY clause because (1) the order of IN terms + ** is not necessarily related to the order of output terms and + ** (2) Multiple outputs from a single IN value will not merge + ** together. */ + pIdxInfo->orderByConsumed = 0; } } } @@ -2426,8 +2424,7 @@ static void bestVirtualIndex(WhereBestIdx *p){ } p->cost.plan.u.pVtabIdx = pIdxInfo; if( pIdxInfo->orderByConsumed ){ - assert( sortOrder==0 || sortOrder==1 ); - p->cost.plan.wsFlags |= WHERE_ORDERED + sortOrder*WHERE_REVERSE; + p->cost.plan.wsFlags |= WHERE_ORDERED; p->cost.plan.nOBSat = nOrderBy; }else{ p->cost.plan.nOBSat = p->i ? p->aLevel[p->i-1].plan.nOBSat : 0; From a956af67e1a34d4a60fe8bddb9496969525a894a Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 18 Apr 2013 03:10:43 +0000 Subject: [PATCH 145/204] Fix a compiler warning in sqlite3_compileoption_used(). FossilOrigin-Name: e9f9d84b223b69d36688cd7a4c7c696bfda8a1ad --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/ctime.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d34134f033..a10eb45721 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\ssuppress\sthe\sORDER\sBY\sclause\son\sa\svirtual\stable\squery\sif\san\sIN\nconstraint\sis\sused.\s\sFix\sfor\sticket\s[f69b96e3076e].\s\sTesting\sdone\son\nTH3\susing\scov1/where37.test. -D 2013-04-18T02:55:54.826 +C Fix\sa\scompiler\swarning\sin\ssqlite3_compileoption_used(). +D 2013-04-18T03:10:43.014 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -133,7 +133,7 @@ F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c 387589abe92946f47dac6fa1d4479af977b4f11b +F src/ctime.c 4262c227bc91cecc61ae37ed3a40f08069cfa267 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3e3ecad2b5d6c97032f2a9fb920c1b8a16ab8ef5 -R 6fada9489bb28405c28c5a46dce1bedb +P 61b2a7be3b9c04bf45bffa93a7d3a480fc5c947a +R 6659de1ffecbe8f368709b1c4cb6dc19 U drh -Z da33d363d10fd9678c928e7a78f01d61 +Z 0f74e436c879f4a5d6bdb52557a8b2af diff --git a/manifest.uuid b/manifest.uuid index 8101a7d5ae..26f13d75e5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -61b2a7be3b9c04bf45bffa93a7d3a480fc5c947a \ No newline at end of file +e9f9d84b223b69d36688cd7a4c7c696bfda8a1ad \ No newline at end of file diff --git a/src/ctime.c b/src/ctime.c index 804bed704c..60595ff88d 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -381,7 +381,7 @@ int sqlite3_compileoption_used(const char *zOptName){ ** linear search is adequate. No need for a binary search. */ for(i=0; i Date: Thu, 18 Apr 2013 15:11:03 +0000 Subject: [PATCH 146/204] Add the --breakpoint and --show-sql-errors commands to mptester. FossilOrigin-Name: d0898fd76a54512894418e53ba28703e250c9ed3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index a10eb45721..519938b65f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scompiler\swarning\sin\ssqlite3_compileoption_used(). -D 2013-04-18T03:10:43.014 +C Add\sthe\s--breakpoint\sand\s--show-sql-errors\scommands\sto\smptester. +D 2013-04-18T15:11:03.053 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 -F mptest/mptest.c 23365b9f65f220f1206caed7bcee651538a31559 +F mptest/mptest.c ce0d99750a9c01624f29b8942241592e94a7d0d1 F mptest/multiwrite01.test 81fbc17657964889b60750bd7bbb1deffe8f4d42 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 61b2a7be3b9c04bf45bffa93a7d3a480fc5c947a -R 6659de1ffecbe8f368709b1c4cb6dc19 +P e9f9d84b223b69d36688cd7a4c7c696bfda8a1ad +R fd7f4162984b752563533589ad89bff8 U drh -Z 0f74e436c879f4a5d6bdb52557a8b2af +Z e0f5854023cf43fccbc89e940a13a42c diff --git a/manifest.uuid b/manifest.uuid index 26f13d75e5..530384dc8c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e9f9d84b223b69d36688cd7a4c7c696bfda8a1ad \ No newline at end of file +d0898fd76a54512894418e53ba28703e250c9ed3 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index cb6dd55d91..ac3d408204 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -71,6 +71,7 @@ static struct Global { int taskId; /* Task ID. 0 means supervisor. */ int iTrace; /* Tracing level */ int bSqlTrace; /* True to trace SQL commands */ + int bIgnoreSqlErrors; /* Ignore errors in SQL statements */ int nError; /* Number of errors */ int nTest; /* Number of --match operators */ int iTimeout; /* Milliseconds until a busy timeout */ @@ -329,6 +330,7 @@ static void sqlTraceCallback(void *NotUsed1, const char *zSql){ */ static void sqlErrorCallback(void *pArg, int iErrCode, const char *zMsg){ UNUSED_PARAMETER(pArg); + if( iErrCode==SQLITE_ERROR && g.bIgnoreSqlErrors ) return; if( (iErrCode&0xff)==SQLITE_SCHEMA && g.iTrace<3 ) return; if( g.iTimeout==0 && (iErrCode&0xff)==SQLITE_BUSY && g.iTrace<3 ) return; if( (iErrCode&0xff)==SQLITE_NOTICE ){ @@ -826,6 +828,30 @@ static char *filenameTail(char *z){ return z+j; } +/* +** Interpret zArg as a boolean value. Return either 0 or 1. +*/ +static int booleanValue(char *zArg){ + int i; + if( zArg==0 ) return 0; + for(i=0; zArg[i]>='0' && zArg[i]<='9'; i++){} + if( i>0 && zArg[i]==0 ) return atoi(zArg); + if( sqlite3_stricmp(zArg, "on")==0 || sqlite3_stricmp(zArg,"yes")==0 ){ + return 1; + } + if( sqlite3_stricmp(zArg, "off")==0 || sqlite3_stricmp(zArg,"no")==0 ){ + return 0; + } + errorMessage("unknown boolean: [%s]", zArg); + return 0; +} + + +/* This routine exists as a convenient place to set a debugger +** breakpoint. +*/ +static void test_breakpoint(void){ static volatile int cnt = 0; cnt++; } + /* Maximum number of arguments to a --command */ #define MX_ARG 2 @@ -1119,6 +1145,26 @@ static void runScript( iBegin = ii+len; }else + /* + ** --breakpoint + ** + ** This command calls "test_breakpoint()" which is a routine provided + ** as a convenient place to set a debugger breakpoint. + */ + if( strcmp(zCmd, "breakpoint")==0 ){ + test_breakpoint(); + }else + + /* + ** --show-sql-errors BOOLEAN + ** + ** Turn display of SQL errors on and off. + */ + if( strcmp(zCmd, "show-sql-errors")==0 ){ + g.bIgnoreSqlErrors = nArg>=1 ? !booleanValue(azArg[0]) : 1; + }else + + /* error */{ errorMessage("line %d of %s: unknown command --%s", prevLine, zFilename, zCmd); From 93c8c457e85e9fec3bd62278c62e77feaf722225 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 18 Apr 2013 20:33:41 +0000 Subject: [PATCH 147/204] Fix an incorrect comment (a copy/paste error) in the mptester. No code changes. FossilOrigin-Name: e94783f36ade43ef9d5b94db278622bcbce4179b --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 519938b65f..e07c960b3c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s--breakpoint\sand\s--show-sql-errors\scommands\sto\smptester. -D 2013-04-18T15:11:03.053 +C Fix\san\sincorrect\scomment\s(a\scopy/paste\serror)\sin\sthe\smptester.\s\sNo\scode\nchanges. +D 2013-04-18T20:33:41.233 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 -F mptest/mptest.c ce0d99750a9c01624f29b8942241592e94a7d0d1 +F mptest/mptest.c 499a74af4be293b7c1c7c3d40f332b67227dd739 F mptest/multiwrite01.test 81fbc17657964889b60750bd7bbb1deffe8f4d42 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e9f9d84b223b69d36688cd7a4c7c696bfda8a1ad -R fd7f4162984b752563533589ad89bff8 +P d0898fd76a54512894418e53ba28703e250c9ed3 +R 489ed5534c8ff7ba356053afa67de654 U drh -Z e0f5854023cf43fccbc89e940a13a42c +Z da35d51e74cda792a97ee9cf150ece03 diff --git a/manifest.uuid b/manifest.uuid index 530384dc8c..3b6aeacff2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d0898fd76a54512894418e53ba28703e250c9ed3 \ No newline at end of file +e94783f36ade43ef9d5b94db278622bcbce4179b \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index ac3d408204..059ae102fa 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -934,8 +934,8 @@ static void runScript( /* ** --testcase NAME ** - ** Exit this process. If N>0 then exit without shutting down - ** SQLite. (In other words, simulate a crash.) + ** Begin a new test case. Announce in the log that the test case + ** has begun. */ if( strcmp(zCmd, "testcase")==0 ){ if( g.iTrace==1 ) logMessage("%.*s", len - 1, zScript+ii); From c288e442e3400e4672f81c1acf86bf2ed61480d1 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 18 Apr 2013 22:56:42 +0000 Subject: [PATCH 148/204] Experiments in making loadable extensions easier to program. The shared library filename tries various operating-system specific extensions if the bare library name does not work. And the entry point is derived from the filename is the legacy entry point name "sqlite3_extension_init" is not found. FossilOrigin-Name: 6cfa1d73d79b9f0be8157f0a37c264dd95b031c8 --- manifest | 17 +++++++----- manifest.uuid | 2 +- src/loadext.c | 74 +++++++++++++++++++++++++++++++++++++++++++------ src/sqlite.h.in | 15 ++++++++-- 4 files changed, 89 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index e07c960b3c..ed0d92dbad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sincorrect\scomment\s(a\scopy/paste\serror)\sin\sthe\smptester.\s\sNo\scode\nchanges. -D 2013-04-18T20:33:41.233 +C Experiments\sin\smaking\sloadable\sextensions\seasier\sto\sprogram.\s\sThe\nshared\slibrary\sfilename\stries\svarious\soperating-system\sspecific\sextensions\nif\sthe\sbare\slibrary\sname\sdoes\snot\swork.\s\sAnd\sthe\sentry\spoint\sis\sderived\nfrom\sthe\sfilename\sis\sthe\slegacy\sentry\spoint\sname\s"sqlite3_extension_init"\nis\snot\sfound. +D 2013-04-18T22:56:42.784 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -148,7 +148,7 @@ F src/insert.c f7cb141e8ce257cb6b15c497f09e4e23d6055599 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b -F src/loadext.c 1422eba4aa2b1fb5f7b3aef574752272477d21e2 +F src/loadext.c c48f7f3f170e502fe0cc20748e03c6e0b5a016c2 F src/main.c de47c3e6b68f512b9e64ba2533ee4eecc9d501c1 F src/malloc.c fe085aa851b666b7c375c1ff957643dc20a04bf6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -182,7 +182,7 @@ F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c aca9d94653decd4496846dee0c7ba83eaf96a46d -F src/sqlite.h.in 31442751f3a63a538777188b6dfe70aebed338ac +F src/sqlite.h.in a291a999d28ebc16cd1fe7fdd010dfe5266ef767 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 2a83cfec9963372b636b0cabd4b200c1f1074a99 @@ -1051,7 +1051,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d0898fd76a54512894418e53ba28703e250c9ed3 -R 489ed5534c8ff7ba356053afa67de654 +P e94783f36ade43ef9d5b94db278622bcbce4179b +R 5eb9b691174dd0ce00c815b019b68747 +T *branch * loadext +T *sym-loadext * +T -sym-trunk * U drh -Z da35d51e74cda792a97ee9cf150ece03 +Z 1611e9ed5f0542b10fe5f2f8888ad81f diff --git a/manifest.uuid b/manifest.uuid index 3b6aeacff2..fc3b50454d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e94783f36ade43ef9d5b94db278622bcbce4179b \ No newline at end of file +6cfa1d73d79b9f0be8157f0a37c264dd95b031c8 \ No newline at end of file diff --git a/src/loadext.c b/src/loadext.c index 76450f2e8b..cdcf6a93b8 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -415,8 +415,23 @@ static int sqlite3LoadExtension( void *handle; int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); char *zErrmsg = 0; + const char *zEntry; + char *zAltEntry = 0; void **aHandle; int nMsg = 300 + sqlite3Strlen30(zFile); + int ii; + + /* Shared library endings to try if zFile cannot be loaded as written */ + static const char *azEndings[] = { +#if SQLITE_OS_WIN + "dll" +#elif defined(__APPLE__) + "dylib" +#else + "so" +#endif + }; + if( pzErrMsg ) *pzErrMsg = 0; @@ -433,11 +448,17 @@ static int sqlite3LoadExtension( return SQLITE_ERROR; } - if( zProc==0 ){ - zProc = "sqlite3_extension_init"; - } + zEntry = zProc ? zProc : "sqlite3_extension_init"; handle = sqlite3OsDlOpen(pVfs, zFile); +#if SQLITE_OS_UNIX || SQLITE_OS_WIN + for(ii=0; ii sqlite3_example_init + ** C:/lib/mathfuncs.dll ==> sqlite3_mathfuncs_init + */ + if( xInit==0 && zProc==0 ){ + int iFile, iEntry, c; + int ncFile = sqlite3Strlen30(zFile); + zAltEntry = sqlite3_malloc(ncFile+30); + if( zAltEntry==0 ){ + sqlite3OsDlClose(pVfs, handle); + return SQLITE_NOMEM; + } + memcpy(zAltEntry, "sqlite3_", 8); + for(iFile=ncFile-1; iFile>=0 && zFile[iFile]!='/'; iFile--){} + iFile++; + if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; + for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ + if( sqlite3Isalpha(c) ){ + zAltEntry[iEntry++] = (char)sqlite3UpperToLower[(unsigned)c]; + } + } + memcpy(zAltEntry+iEntry, "_init", 6); + zEntry = zAltEntry; + xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) + sqlite3OsDlSym(pVfs, handle, zEntry); + } if( xInit==0 ){ if( pzErrMsg ){ - nMsg += sqlite3Strlen30(zProc); + nMsg += sqlite3Strlen30(zEntry); *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); if( zErrmsg ){ sqlite3_snprintf(nMsg, zErrmsg, - "no entry point [%s] in shared library [%s]", zProc,zFile); + "no entry point [%s] in shared library [%s]", zEntry, zFile); sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); } - sqlite3OsDlClose(pVfs, handle); } + sqlite3OsDlClose(pVfs, handle); + sqlite3_free(zAltEntry); return SQLITE_ERROR; - }else if( xInit(db, &zErrmsg, &sqlite3Apis) ){ + } + sqlite3_free(zAltEntry); + if( xInit(db, &zErrmsg, &sqlite3Apis) ){ if( pzErrMsg ){ *pzErrMsg = sqlite3_mprintf("error during initialization: %s", zErrmsg); } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index f4f02d3eac..3c6291eebe 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5017,11 +5017,20 @@ int sqlite3_table_column_metadata( ** ^This interface loads an SQLite extension library from the named file. ** ** ^The sqlite3_load_extension() interface attempts to load an -** [SQLite extension] library contained in the file zFile. +** [SQLite extension] library contained in the file zFile. If +** the file cannot be loaded directly, attempts are made to load +** with various operating-system specific extensions added. +** So for example, if "samplelib" cannot be loaded, then names like +** "samplelib.so" or "samplelib.dylib" or "samplelib.dll" might +** be tried also. ** ** ^The entry point is zProc. -** ^zProc may be 0, in which case the name of the entry point -** defaults to "sqlite3_extension_init". +** ^(zProc may be 0, in which case SQLite will try to come up with an +** entry point name on its own. It first tries "sqlite3_extension_init". +** If that does not work, it constructs a name "sqlite3_X_init" where the +** X is consists of the lower-case equivalent of all ASCII alphabetic +** characters in the filename from the last "/" to the first following +** "." and omitting any initial "lib".)^ ** ^The sqlite3_load_extension() interface returns ** [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. ** ^If an error occurs and pzErrMsg is not 0, then the From 08ef8d7311624b78af522275de4e0d62b799882c Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 19 Apr 2013 01:23:06 +0000 Subject: [PATCH 149/204] Make the SQLITE_EXTENSION_INIT1 and SQLITE_EXTENSION_INIT2 macros into no-ops if the extension is statically linked. FossilOrigin-Name: 86eb3eed4c70730dba0e319f243f8cb4fa8de06c --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/sqlite3ext.h | 13 +++++++++++-- src/test_autoext.c | 2 +- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index ed0d92dbad..d25b1c8d4d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experiments\sin\smaking\sloadable\sextensions\seasier\sto\sprogram.\s\sThe\nshared\slibrary\sfilename\stries\svarious\soperating-system\sspecific\sextensions\nif\sthe\sbare\slibrary\sname\sdoes\snot\swork.\s\sAnd\sthe\sentry\spoint\sis\sderived\nfrom\sthe\sfilename\sis\sthe\slegacy\sentry\spoint\sname\s"sqlite3_extension_init"\nis\snot\sfound. -D 2013-04-18T22:56:42.784 +C Make\sthe\sSQLITE_EXTENSION_INIT1\sand\sSQLITE_EXTENSION_INIT2\smacros\sinto\nno-ops\sif\sthe\sextension\sis\sstatically\slinked. +D 2013-04-19T01:23:06.788 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -184,7 +184,7 @@ F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c aca9d94653decd4496846dee0c7ba83eaf96a46d F src/sqlite.h.in a291a999d28ebc16cd1fe7fdd010dfe5266ef767 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 -F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 +F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 F src/sqliteInt.h 2a83cfec9963372b636b0cabd4b200c1f1074a99 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 @@ -200,7 +200,7 @@ F src/test7.c f4b894b7931f8cf9f5cbf37cfa0727703f526a40 F src/test8.c 58ea1d9698f3947e4662107ef98f429e84ae20e0 F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad -F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e +F src/test_autoext.c 5c95b5d435eaa09d6c0e7d90371c5ca8cd567701 F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de F src/test_btree.c 5b89601dcb42a33ba8b820a6b763cc9cb48bac16 F src/test_config.c 95bb33e9dcaa340a296c0bf0e0ba3d1a1c8004c0 @@ -1051,10 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e94783f36ade43ef9d5b94db278622bcbce4179b -R 5eb9b691174dd0ce00c815b019b68747 -T *branch * loadext -T *sym-loadext * -T -sym-trunk * +P 6cfa1d73d79b9f0be8157f0a37c264dd95b031c8 +R 6c3fdd8d85d98822b9ca8decd52bbb5c U drh -Z 1611e9ed5f0542b10fe5f2f8888ad81f +Z 311c6fc82989346d7dcdfa6c2765eba0 diff --git a/manifest.uuid b/manifest.uuid index fc3b50454d..ee9ed3efc1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6cfa1d73d79b9f0be8157f0a37c264dd95b031c8 \ No newline at end of file +86eb3eed4c70730dba0e319f243f8cb4fa8de06c \ No newline at end of file diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index a465b8eccc..928bb3bad9 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -469,7 +469,16 @@ struct sqlite3_api_routines { #define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2 #endif /* SQLITE_CORE */ -#define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0; -#define SQLITE_EXTENSION_INIT2(v) sqlite3_api = v; +#ifndef SQLITE_CORE + /* This case when the file really is being compiled as a loadable + ** extension */ +# define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; +# define SQLITE_EXTENSION_INIT2(v) sqlite3_api=v; +#else + /* This case when the file is being statically linked into the + ** application */ +# define SQLITE_EXTENSION_INIT1 /*no-op*/ +# define SQLITE_EXTENSION_INIT2(v) (void)v; /* unused parameter */ +#endif #endif /* _SQLITE3EXT_H_ */ diff --git a/src/test_autoext.c b/src/test_autoext.c index 6b1e297ab7..b5013f3173 100644 --- a/src/test_autoext.c +++ b/src/test_autoext.c @@ -15,7 +15,7 @@ #include "sqlite3ext.h" #ifndef SQLITE_OMIT_LOAD_EXTENSION -static SQLITE_EXTENSION_INIT1 +SQLITE_EXTENSION_INIT1 /* ** The sqr() SQL function returns the square of its input value. From cca17c303baade3ce4e823ecfcd89f9f8051e051 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 19 Apr 2013 12:32:52 +0000 Subject: [PATCH 150/204] Add the "warning" command to the TCL test infrastructure. Fix problems with the loadext.test module. FossilOrigin-Name: 514adbbd8cf3e296f55e8f803bddaac8ad8b2c96 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/loadext.test | 16 +++++++++------- test/tester.tcl | 16 ++++++++++++++++ 4 files changed, 33 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 1dd6c2db3b..1110498763 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sextension\sloading\smechanism\sso\sthat\sthe\ssame\ssource\scode\ncan\sbe\sused\sfor\srun-time\sloadable\sextensions\sand\sstatically\slinked\nextensions,\sand\sto\smask\sarchitectural\sdifferences\sin\sshared\slibraries. -D 2013-04-19T02:47:05.486 +C Add\sthe\s"warning"\scommand\sto\sthe\sTCL\stest\sinfrastructure.\s\sFix\sproblems\nwith\sthe\sloadext.test\smodule. +D 2013-04-19T12:32:52.666 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -585,7 +585,7 @@ F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/like.test 0e5412f4dac4a849f613e1ef8b529d56a6e31d08 F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da F test/limit.test cc0ab63385239b63c72452b0e93700bf5e8f0b99 -F test/loadext.test 2b5e249c51c986a5aff1f0950cf7ba30976c8f22 +F test/loadext.test 92e6dfefd1229c3ef4aaabd87419efd8fa57a7a5 F test/loadext2.test 0bcaeb4d81cd5b6e883fdfea3c1bdbe1f173cbca F test/lock.test 87af515b0c4cf928576d0f89946d67d7c265dfb4 F test/lock2.test 5242d8ac4e2d59c403aebff606af449b455aceff @@ -765,7 +765,7 @@ F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptrigger.test 26670ed7a39cf2296a7f0a9e0a1d7bdb7abe936d -F test/tester.tcl 0d95028653ad6782e17400b7a32a046b87c67a48 +F test/tester.tcl 3b08771e6d601612fe62d13787db0e50aac4cf7b F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e94783f36ade43ef9d5b94db278622bcbce4179b 86eb3eed4c70730dba0e319f243f8cb4fa8de06c -R 6c3fdd8d85d98822b9ca8decd52bbb5c +P a85b6ecefa14fcfe6ebfceabf910f00c3721be7d +R 2c3aca07297ff95a6b74db849941a18d U drh -Z 0d2df06bcea7e6410ce186a367eefd17 +Z bfc695f8f3325c7943f46501f9086d37 diff --git a/manifest.uuid b/manifest.uuid index 02b97e09e3..d269cbb390 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a85b6ecefa14fcfe6ebfceabf910f00c3721be7d \ No newline at end of file +514adbbd8cf3e296f55e8f803bddaac8ad8b2c96 \ No newline at end of file diff --git a/test/loadext.test b/test/loadext.test index 72eff12f7a..0d5b841980 100644 --- a/test/loadext.test +++ b/test/loadext.test @@ -139,20 +139,22 @@ do_test loadext-2.1 { sqlite3_load_extension db "${testextension}xx" } msg] list $rc $msg -} [list 1 [format $dlerror_nosuchfile ${testextension}xx]] +} /[list 1 [format $dlerror_nosuchfile ${testextension}xx.*]]/ # Try to load an extension for which the file is not a shared object # do_test loadext-2.2 { - set fd [open "${testextension}xx" w] + set fd [open "./notasharedlib.so" w] + puts $fd blah + close $fd + set fd [open "./notasharedlib.dll" w] puts $fd blah close $fd set rc [catch { - sqlite3_load_extension db "${testextension}xx" + sqlite3_load_extension db "./notasharedlib" } msg] - set expected_error_pattern [format $dlerror_notadll ${testextension}xx] - list $rc [string match $expected_error_pattern $msg] -} [list 1 1] + list $rc $msg +} /[list 1 [format $dlerror_notadll ./notasharedlib.*]]/ # Try to load an extension for which the file is present but the # entry point is not. @@ -196,7 +198,7 @@ do_test loadext-3.2 { regsub {0x[1234567890abcdefABCDEF]*} $res XXX res } set res -} [list 1 [format $dlerror_nosymbol $testextension sqlite3_extension_init]] +} /[list 1 [format $dlerror_nosymbol $testextension sqlite3_.*_init]]/ do_test loadext-3.3 { catchsql { SELECT load_extension($::testextension,'testloadext_init') diff --git a/test/tester.tcl b/test/tester.tcl index d6468f7b14..761a36e355 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -462,6 +462,7 @@ if {0==[info exists ::SLAVE]} { set TC(count) 0 set TC(fail_list) [list] set TC(omit_list) [list] + set TC(warn_list) [list] proc set_test_counter {counter args} { if {[llength $args]} { @@ -496,6 +497,18 @@ proc fail_test {name} { } } +# Remember a warning message to be displayed at the conclusion of all testing +# +proc warning {msg {append 1}} { + puts "Warning: $msg" + set warnList [set_test_counter warn_list] + if {$append} { + lappend warnList $msg + } + set_test_counter warn_list $warnList +} + + # Increment the number of tests run # proc incr_ntest {} { @@ -784,6 +797,9 @@ proc finalize_testing {} { if {$nErr>0} { puts "Failures on these tests: [set_test_counter fail_list]" } + foreach warning [set_test_counter warn_list] { + puts "Warning: $warning" + } run_thread_tests 1 if {[llength $omitList]>0} { puts "Omitted test cases:" From 0c41d22920741a7904ac89de203c463c0b12ec7f Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 22 Apr 2013 02:39:10 +0000 Subject: [PATCH 151/204] Use transitivity to move constraints into the outer loops of a join whenever possible, thereby reducing the amount of work that needs to occur in inner loops. FossilOrigin-Name: 5f4907e1c6230e3dd904bd99e1c48c576c669f63 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 36 ++++++++++++++++++++++++++++++++---- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 1110498763..9cf2da3aad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"warning"\scommand\sto\sthe\sTCL\stest\sinfrastructure.\s\sFix\sproblems\nwith\sthe\sloadext.test\smodule. -D 2013-04-19T12:32:52.666 +C Use\stransitivity\sto\smove\sconstraints\sinto\sthe\souter\sloops\sof\sa\sjoin\swhenever\npossible,\sthereby\sreducing\sthe\samount\sof\swork\sthat\sneeds\sto\soccur\sin\ninner\sloops. +D 2013-04-22T02:39:10.341 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -258,7 +258,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c e63f84e5583133c77573b78d3696dccd97bfd00d +F src/where.c c7c74fe8e354b60dee0810ea8b954c5ccb9e0f3c F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a85b6ecefa14fcfe6ebfceabf910f00c3721be7d -R 2c3aca07297ff95a6b74db849941a18d +P 514adbbd8cf3e296f55e8f803bddaac8ad8b2c96 +R a81fa97db156009318d49bf4c6dae35e U drh -Z bfc695f8f3325c7943f46501f9086d37 +Z 1c3c5b72018db605301d161eada3f156 diff --git a/manifest.uuid b/manifest.uuid index d269cbb390..bd39d46376 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -514adbbd8cf3e296f55e8f803bddaac8ad8b2c96 \ No newline at end of file +5f4907e1c6230e3dd904bd99e1c48c576c669f63 \ No newline at end of file diff --git a/src/where.c b/src/where.c index a0dc868422..24b059e2d5 100644 --- a/src/where.c +++ b/src/where.c @@ -4161,6 +4161,7 @@ static Bitmask codeOneLoopStart( int addrCont; /* Jump here to continue with next cycle */ int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ int iReleaseReg = 0; /* Temp register to free before returning */ + Bitmask newNotReady; /* Return value */ pParse = pWInfo->pParse; v = pParse->pVdbe; @@ -4171,6 +4172,7 @@ static Bitmask codeOneLoopStart( bRev = (pLevel->plan.wsFlags & WHERE_REVERSE)!=0; omitTable = (pLevel->plan.wsFlags & WHERE_IDX_ONLY)!=0 && (wctrlFlags & WHERE_FORCE_TABLE)==0; + VdbeNoopComment((v, "Begin Join Loop %d", iLevel)); /* Create labels for the "break" and "continue" instructions ** for the current loop. Jump to addrBrk to break out of a loop. @@ -4821,7 +4823,7 @@ static Bitmask codeOneLoopStart( pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; } - notReady &= ~getMask(pWC->pMaskSet, iCur); + newNotReady = notReady & ~getMask(pWC->pMaskSet, iCur); /* Insert code to test every subexpression that can be completely ** computed using the current set of tables. @@ -4835,7 +4837,7 @@ static Bitmask codeOneLoopStart( testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & notReady)!=0 ){ + if( (pTerm->prereqAll & newNotReady)!=0 ){ testcase( pWInfo->untestedTerms==0 && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ); pWInfo->untestedTerms = 1; @@ -4850,6 +4852,32 @@ static Bitmask codeOneLoopStart( pTerm->wtFlags |= TERM_CODED; } + /* Insert code to test for implied constraints based on transitivity + ** of the "==" operator. + ** + ** Example: If the WHERE clause contains "t1.a=t2.b" and "t2.b=123" + ** and we are coding the t1 loop and the t2 loop has not yet coded, + ** then we cannot use the "t1.a=t2.b" constraint, but we can code + ** the implied "t1.a=123" constraint. + */ + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE; + WhereTerm *pAlt; + Expr sEq; + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( pTerm->eOperator!=(WO_EQUIV|WO_EQ) ) continue; + if( pTerm->leftCursor!=iCur ) continue; + pE = pTerm->pExpr; + assert( !ExprHasProperty(pE, EP_FromJoin) ); + assert( (pTerm->prereqRight & newNotReady)!=0 ); + pAlt = findTerm(pWC, iCur, pTerm->u.leftColumn, notReady, WO_EQ|WO_IN, 0); + if( pAlt==0 ) continue; + VdbeNoopComment((v, "begin transitive constraint")); + sEq = *pAlt->pExpr; + sEq.pLeft = pE->pLeft; + sqlite3ExprIfFalse(pParse, &sEq, addrCont, SQLITE_JUMPIFNULL); + } + /* For a LEFT OUTER JOIN, generate code that will record the fact that ** at least one row of the right table has matched the left table. */ @@ -4862,7 +4890,7 @@ static Bitmask codeOneLoopStart( testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & notReady)!=0 ){ + if( (pTerm->prereqAll & newNotReady)!=0 ){ assert( pWInfo->untestedTerms ); continue; } @@ -4873,7 +4901,7 @@ static Bitmask codeOneLoopStart( } sqlite3ReleaseTempReg(pParse, iReleaseReg); - return notReady; + return newNotReady; } #if defined(SQLITE_TEST) From b9830a14543c823f6690a70b5b43d8bed409edd9 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 22 Apr 2013 13:51:09 +0000 Subject: [PATCH 152/204] Enhanced documentation for the SQLITE_CONFIG_SQLLOG mechanism and the test_sqllog.c demonstration file. No changes to code. FossilOrigin-Name: 206caf21f6f6963aa38a9aa0f6ea11c7230bb726 --- manifest | 14 +++++------ manifest.uuid | 2 +- src/sqlite.h.in | 8 ++++--- src/test_sqllog.c | 59 +++++++++++++++++++++++++++++++++++++---------- 4 files changed, 60 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 9cf2da3aad..68a8c40938 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\stransitivity\sto\smove\sconstraints\sinto\sthe\souter\sloops\sof\sa\sjoin\swhenever\npossible,\sthereby\sreducing\sthe\samount\sof\swork\sthat\sneeds\sto\soccur\sin\ninner\sloops. -D 2013-04-22T02:39:10.341 +C Enhanced\sdocumentation\sfor\sthe\sSQLITE_CONFIG_SQLLOG\smechanism\sand\sthe\ntest_sqllog.c\sdemonstration\sfile.\s\sNo\schanges\sto\scode. +D 2013-04-22T13:51:09.759 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -182,7 +182,7 @@ F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c aca9d94653decd4496846dee0c7ba83eaf96a46d -F src/sqlite.h.in a291a999d28ebc16cd1fe7fdd010dfe5266ef767 +F src/sqlite.h.in ec279b782bea05db63b8b29481f9642b406004af F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 F src/sqliteInt.h 2a83cfec9963372b636b0cabd4b200c1f1074a99 @@ -229,7 +229,7 @@ F src/test_rtree.c aba603c949766c4193f1068b91c787f57274e0d9 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f F src/test_spellfix.c 56dfa6d583ac34f61af0834d7b58d674e7e18e13 -F src/test_sqllog.c bc50e5afeb7fb50e77b4594e42302df9d05446aa +F src/test_sqllog.c c1c1bbedbcaf82b93d83e4f9dd990e62476a680e F src/test_stat.c d1569c7a4839f13e80187e2c26b2ab4da2d03935 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd F src/test_syscall.c 7e8293e4e6971b0f44c7f7f37b1315a8cc9f6018 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 514adbbd8cf3e296f55e8f803bddaac8ad8b2c96 -R a81fa97db156009318d49bf4c6dae35e +P 5f4907e1c6230e3dd904bd99e1c48c576c669f63 +R 3f851c92fc64d9ebdb6dc226b2109b9c U drh -Z 1c3c5b72018db605301d161eada3f156 +Z 928e878ceb923ac2280cddfe9d967aac diff --git a/manifest.uuid b/manifest.uuid index bd39d46376..77c2db74de 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5f4907e1c6230e3dd904bd99e1c48c576c669f63 \ No newline at end of file +206caf21f6f6963aa38a9aa0f6ea11c7230bb726 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 3c6291eebe..3265b3196b 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1633,12 +1633,12 @@ struct sqlite3_mem_methods { **
    SQLITE_CONFIG_PCACHE and SQLITE_CONFIG_GETPCACHE **
    These options are obsolete and should not be used by new code. ** They are retained for backwards compatibility but are now no-ops. -** +**
    ** ** [[SQLITE_CONFIG_SQLLOG]] **
    SQLITE_CONFIG_SQLLOG **
    This option is only available if sqlite is compiled with the -** SQLITE_ENABLE_SQLLOG pre-processor macro defined. The first argument should +** [SQLITE_ENABLE_SQLLOG] pre-processor macro defined. The first argument should ** be a pointer to a function of type void(*)(void*,sqlite3*,const char*, int). ** The second should be of type (void*). The callback is invoked by the library ** in three separate circumstances, identified by the value passed as the @@ -1648,7 +1648,9 @@ struct sqlite3_mem_methods { ** fourth parameter is 1, then the SQL statement that the third parameter ** points to has just been executed. Or, if the fourth parameter is 2, then ** the connection being passed as the second parameter is being closed. The -** third parameter is passed NULL In this case. +** third parameter is passed NULL In this case. An example of using this +** configuration option can be seen in the "test_sqllog.c" source file in +** the canonical SQLite source tree.
    ** ** [[SQLITE_CONFIG_MMAP_SIZE]] **
    SQLITE_CONFIG_MMAP_SIZE diff --git a/src/test_sqllog.c b/src/test_sqllog.c index 24c675b5e9..4aa68b7c42 100644 --- a/src/test_sqllog.c +++ b/src/test_sqllog.c @@ -13,22 +13,32 @@ ** OVERVIEW ** ** This file contains experimental code used to record data from live -** SQLite applications that may be useful for offline analysis. Specifically: +** SQLite applications that may be useful for offline analysis. +** Specifically, this module can be used to capture the following +** information: ** ** 1) The initial contents of all database files opened by the ** application, and ** ** 2) All SQL statements executed by the application. ** +** The captured information can then be used to run (for example) +** performance analysis looking for slow queries or to look for +** optimization opportunities in either the application or in SQLite +** itself. +** ** USAGE ** ** To use this module, SQLite must be compiled with the SQLITE_ENABLE_SQLLOG -** pre-processor symbol defined and this file linked into the application -** somehow. +** pre-processor symbol defined and this file linked into the application. +** One way to link this file into the application is to append the content +** of this file onto the end of the "sqlite3.c" amalgamation and then +** recompile the application as normal except with the addition of the +** -DSQLITE_ENABLE_SQLLOG option. ** ** At runtime, logging is enabled by setting environment variable ** SQLITE_SQLLOG_DIR to the name of a directory in which to store logged -** data. The directory must already exist. +** data. The logging directory must already exist. ** ** Usually, if the application opens the same database file more than once ** (either by attaching it or by using more than one database handle), only @@ -57,14 +67,16 @@ ** logging proceeds as expected. Errors are logged by calling sqlite3_log(). */ +#ifndef _SQLITE3_H_ #include "sqlite3.h" -#include "stdio.h" -#include "stdlib.h" -#include "string.h" -#include "assert.h" +#endif +#include +#include +#include +#include -#include "sys/types.h" -#include "unistd.h" +#include +#include static int getProcessId(void){ #if SQLITE_OS_WIN return (int)_getpid(); @@ -73,7 +85,7 @@ static int getProcessId(void){ #endif } - +/* Names of environment variables to be used */ #define ENVIRONMENT_VARIABLE1_NAME "SQLITE_SQLLOG_DIR" #define ENVIRONMENT_VARIABLE2_NAME "SQLITE_SQLLOG_REUSE_FILES" @@ -86,6 +98,9 @@ static int getProcessId(void){ */ #define MAX_CONNECTIONS 256 +/* There is one instance of this object for each SQLite database connection +** that is being logged. +*/ struct SLConn { int isErr; /* True if an error has occurred */ sqlite3 *db; /* Connection handle */ @@ -93,7 +108,9 @@ struct SLConn { FILE *fd; /* File descriptor for log file */ }; -struct SLGlobal { +/* This object is a singleton that keeps track of all data loggers. +*/ +static struct SLGlobal { /* Protected by MUTEX_STATIC_MASTER */ sqlite3_mutex *mutex; /* Recursive mutex */ int nConn; /* Size of aConn[] array */ @@ -388,6 +405,24 @@ static void testSqllogStmt(struct SLConn *p, const char *zSql){ /* ** The SQLITE_CONFIG_SQLLOG callback registered by sqlite3_init_sqllog(). +** +** The eType parameter has the following values: +** +** 0: Opening a new database connection. zSql is the name of the +** file being opened. db is a pointer to the newly created database +** connection. +** +** 1: An SQL statement has run to completion. zSql is the text of the +** SQL statement with all parameters expanded to their actual values. +** +** 2: Closing a database connection. zSql is NULL. The db pointer to +** the database connection being closed has already been shut down +** and cannot be used for any further SQL. +** +** The pCtx parameter is a copy of the pointer that was originally passed +** into the sqlite3_config(SQLITE_CONFIG_SQLLOG) statement. In this +** particular implementation, pCtx is always a pointer to the +** sqllogglobal global variable define above. */ static void testSqllog(void *pCtx, sqlite3 *db, const char *zSql, int eType){ struct SLConn *p = 0; From d7a959c69453e8b12976b2189c99b839751b5020 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 22 Apr 2013 15:30:37 +0000 Subject: [PATCH 153/204] Add virtual table module 'fts3tokenize' to fts3. fts3tokenize provides SQL access to FTS tokenizer implementations. FossilOrigin-Name: c5a4b21a15bd8a184380817ff41657acbccb866d --- Makefile.in | 5 + ext/fts3/fts3.c | 3 + ext/fts3/fts3_tokenize_vtab.c | 459 ++++++++++++++++++++++++++++++++++ main.mk | 5 + manifest | 23 +- manifest.uuid | 2 +- test/fts3tok1.test | 125 +++++++++ test/fts3tok_err.test | 49 ++++ tool/mksqlite3c.tcl | 1 + 9 files changed, 661 insertions(+), 11 deletions(-) create mode 100644 ext/fts3/fts3_tokenize_vtab.c create mode 100644 test/fts3tok1.test create mode 100644 test/fts3tok_err.test diff --git a/Makefile.in b/Makefile.in index 486333157e..5c636209d8 100644 --- a/Makefile.in +++ b/Makefile.in @@ -167,6 +167,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ expr.lo fault.lo fkey.lo \ fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \ fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \ + fts3_tokenize_vtab.lo \ fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \ func.lo global.lo hash.lo \ icu.lo insert.lo journal.lo legacy.lo loadext.lo \ @@ -316,6 +317,7 @@ SRC += \ $(TOP)/ext/fts3/fts3_tokenizer.h \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_tokenizer1.c \ + $(TOP)/ext/fts3/fts3_tokenize_vtab.c \ $(TOP)/ext/fts3/fts3_unicode.c \ $(TOP)/ext/fts3/fts3_unicode2.c \ $(TOP)/ext/fts3/fts3_write.c @@ -858,6 +860,9 @@ fts3_tokenizer.lo: $(TOP)/ext/fts3/fts3_tokenizer.c $(HDR) $(EXTHDR) fts3_tokenizer1.lo: $(TOP)/ext/fts3/fts3_tokenizer1.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer1.c +fts3_tokenizer_vtab.lo: $(TOP)/ext/fts3/fts3_tokenizer_vtab.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer_vtab.c + fts3_unicode.lo: $(TOP)/ext/fts3/fts3_unicode.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_unicode.c diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 6b3a7b1565..0c7e07740d 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3593,6 +3593,9 @@ int sqlite3Fts3Init(sqlite3 *db){ rc = sqlite3Fts3InitAux(db); if( rc!=SQLITE_OK ) return rc; + rc = sqlite3Fts3InitTok(db); + if( rc!=SQLITE_OK ) return rc; + sqlite3Fts3SimpleTokenizerModule(&pSimple); sqlite3Fts3PorterTokenizerModule(&pPorter); diff --git a/ext/fts3/fts3_tokenize_vtab.c b/ext/fts3/fts3_tokenize_vtab.c new file mode 100644 index 0000000000..f00fefe7a9 --- /dev/null +++ b/ext/fts3/fts3_tokenize_vtab.c @@ -0,0 +1,459 @@ +/* +** 2013 Apr 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code for the "fts3tokenize" virtual table module. +** An fts3tokenize virtual table is created as follows: +** +** CREATE VIRTUAL TABLE USING fts3tokenize( +** , , ... +** ); +** +** The table created has the following schema: +** +** CREATE TABLE (input, token, start, end, position) +** +** When queried, the query must include a WHERE clause of type: +** +** input = +** +** The virtual table module tokenizes this , using the FTS3 +** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE +** statement and returns one row for each token in the result. With +** fields set as follows: +** +** input: Always set to a copy of +** token: A token from the input. +** start: Byte offset of the token within the input . +** end: Byte offset of the byte immediately following the end of the +** token within the input string. +** pos: Token offset of token within input. +** +*/ +#include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include +#include + +typedef struct Fts3tokTable Fts3tokTable; +typedef struct Fts3tokCursor Fts3tokCursor; + +/* +** Virtual table structure. +*/ +struct Fts3tokTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + const sqlite3_tokenizer_module *pMod; + sqlite3_tokenizer *pTok; +}; + +/* +** Virtual table cursor structure. +*/ +struct Fts3tokCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + char *zInput; /* Input string */ + sqlite3_tokenizer_cursor *pCsr; /* Cursor to iterate through zInput */ + int iRowid; /* Current 'rowid' value */ + const char *zToken; /* Current 'token' value */ + int nToken; /* Size of zToken in bytes */ + int iStart; /* Current 'start' value */ + int iEnd; /* Current 'end' value */ + int iPos; /* Current 'pos' value */ +}; + +/* +** Query FTS for the tokenizer implementation named zName. +*/ +static int fts3tokQueryTokenizer( + sqlite3 *db, + const char *zName, + const sqlite3_tokenizer_module **pp +){ + int rc; + sqlite3_stmt *pStmt; + const char *zSql = "SELECT fts3_tokenizer(?)"; + + *pp = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ + memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); + } + } + + return sqlite3_finalize(pStmt); +} + +/* +** The second argument, argv[], is an array of pointers to nul-terminated +** strings. This function makes a copy of the array and strings into a +** single block of memory. It then dequotes any of the strings that appear +** to be quoted. +** +** If successful, output parameter *pazDequote is set to point at the +** array of dequoted strings and SQLITE_OK is returned. The caller is +** responsible for eventually calling sqlite3_free() to free the array +** in this case. Or, if an error occurs, an SQLite error code is returned. +** The final value of *pazDequote is undefined in this case. +*/ +static int fts3tokDequoteArray( + int argc, /* Number of elements in argv[] */ + const char * const *argv, /* Input array */ + char ***pazDequote /* Output array */ +){ + int rc = SQLITE_OK; /* Return code */ + if( argc==0 ){ + *pazDequote = 0; + }else{ + int i; + int nByte; + char **azDequote; + + for(i=0; ixCreate((nDequote>1 ? nDequote-1 : 0), azArg, &pTok); + } + + if( rc==SQLITE_OK ){ + pTab = (Fts3tokTable *)sqlite3_malloc(sizeof(Fts3tokTable)); + if( pTab==0 ){ + rc = SQLITE_NOMEM; + } + } + + if( rc==SQLITE_OK ){ + memset(pTab, 0, sizeof(Fts3tokTable)); + pTab->pMod = pMod; + pTab->pTok = pTok; + *ppVtab = &pTab->base; + }else{ + if( pTok ){ + pMod->xDestroy(pTok); + } + } + + sqlite3_free(azDequote); + return rc; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts3tokDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3tokTable *pTab = (Fts3tokTable *)pVtab; + + pTab->pMod->xDestroy(pTab->pTok); + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts3tokBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + int i; + + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable + && pInfo->aConstraint[i].iColumn==0 + && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + pInfo->idxNum = 1; + pInfo->aConstraintUsage[i].argvIndex = 1; + pInfo->aConstraintUsage[i].omit = 1; + pInfo->estimatedCost = 1; + return SQLITE_OK; + } + } + + pInfo->idxNum = 0; + assert( pInfo->estimatedCost>1000000.0 ); + + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts3tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3tokCursor *pCsr; + + pCsr = (Fts3tokCursor *)sqlite3_malloc(sizeof(Fts3tokCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts3tokCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Reset the tokenizer cursor passed as the only argument. As if it had +** just been returned by fts3tokOpenMethod(). +*/ +static void fts3tokResetCursor(Fts3tokCursor *pCsr){ + if( pCsr->pCsr ){ + Fts3tokTable *pTab = (Fts3tokTable *)(pCsr->base.pVtab); + pTab->pMod->xClose(pCsr->pCsr); + pCsr->pCsr = 0; + } + sqlite3_free(pCsr->zInput); + pCsr->zInput = 0; + pCsr->zToken = 0; + pCsr->nToken = 0; + pCsr->iStart = 0; + pCsr->iEnd = 0; + pCsr->iPos = 0; + pCsr->iRowid = 0; +} + +/* +** xClose - Close a cursor. +*/ +static int fts3tokCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + + fts3tokResetCursor(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts3tokNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + int rc; /* Return code */ + + pCsr->iRowid++; + rc = pTab->pMod->xNext(pCsr->pCsr, + &pCsr->zToken, &pCsr->nToken, + &pCsr->iStart, &pCsr->iEnd, &pCsr->iPos + ); + + if( rc!=SQLITE_OK ){ + fts3tokResetCursor(pCsr); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + + return rc; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts3tokFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + int rc = SQLITE_ERROR; + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + + fts3tokResetCursor(pCsr); + if( idxNum==1 ){ + const char *zByte = (const char *)sqlite3_value_text(apVal[0]); + int nByte = sqlite3_value_bytes(apVal[0]); + pCsr->zInput = sqlite3_malloc(nByte+1); + if( pCsr->zInput==0 ){ + rc = SQLITE_NOMEM; + }else{ + memcpy(pCsr->zInput, zByte, nByte); + pCsr->zInput[nByte] = 0; + rc = pTab->pMod->xOpen(pTab->pTok, pCsr->zInput, nByte, &pCsr->pCsr); + if( rc==SQLITE_OK ){ + pCsr->pCsr->pTokenizer = pTab->pTok; + } + } + } + + if( rc!=SQLITE_OK ) return rc; + return fts3tokNextMethod(pCursor); +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3tokEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + return (pCsr->zToken==0); +} + +/* +** xColumn - Return a column value. +*/ +static int fts3tokColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + + /* CREATE TABLE x(input, token, start, end, position) */ + switch( iCol ){ + case 0: + sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_text(pCtx, pCsr->zToken, pCsr->nToken, SQLITE_TRANSIENT); + break; + case 2: + sqlite3_result_int(pCtx, pCsr->iStart); + break; + case 3: + sqlite3_result_int(pCtx, pCsr->iEnd); + break; + default: + assert( iCol==4 ); + sqlite3_result_int(pCtx, pCsr->iPos); + break; + } + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3tokRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + *pRowid = (sqlite3_int64)pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts3tok module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +int sqlite3Fts3InitTok(sqlite3 *db){ + static const sqlite3_module fts3tok_module = { + 0, /* iVersion */ + fts3tokConnectMethod, /* xCreate */ + fts3tokConnectMethod, /* xConnect */ + fts3tokBestIndexMethod, /* xBestIndex */ + fts3tokDisconnectMethod, /* xDisconnect */ + fts3tokDisconnectMethod, /* xDestroy */ + fts3tokOpenMethod, /* xOpen */ + fts3tokCloseMethod, /* xClose */ + fts3tokFilterMethod, /* xFilter */ + fts3tokNextMethod, /* xNext */ + fts3tokEofMethod, /* xEof */ + fts3tokColumnMethod, /* xColumn */ + fts3tokRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, 0); + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/main.mk b/main.mk index 2ec009a11d..36a4da0294 100644 --- a/main.mk +++ b/main.mk @@ -55,6 +55,7 @@ LIBOBJ+= alter.o analyze.o attach.o auth.o \ callback.o complete.o ctime.o date.o delete.o expr.o fault.o fkey.o \ fts3.o fts3_aux.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o \ + fts3_tokenize_vtab.o \ fts3_unicode.o fts3_unicode2.o \ fts3_write.o func.o global.o hash.o \ icu.o insert.o journal.o legacy.o loadext.o \ @@ -197,6 +198,7 @@ SRC += \ $(TOP)/ext/fts3/fts3_tokenizer.h \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_tokenizer1.c \ + $(TOP)/ext/fts3/fts3_tokenize_vtab.c \ $(TOP)/ext/fts3/fts3_unicode.c \ $(TOP)/ext/fts3/fts3_unicode2.c \ $(TOP)/ext/fts3/fts3_write.c @@ -516,6 +518,9 @@ fts3_tokenizer.o: $(TOP)/ext/fts3/fts3_tokenizer.c $(HDR) $(EXTHDR) fts3_tokenizer1.o: $(TOP)/ext/fts3/fts3_tokenizer1.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer1.c +fts3_tokenize_vtab.o: $(TOP)/ext/fts3/fts3_tokenize_vtab.c $(HDR) $(EXTHDR) + $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenize_vtab.c + fts3_unicode.o: $(TOP)/ext/fts3/fts3_unicode.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_unicode.c diff --git a/manifest b/manifest index 68a8c40938..9d7bde3b76 100644 --- a/manifest +++ b/manifest @@ -1,7 +1,7 @@ -C Enhanced\sdocumentation\sfor\sthe\sSQLITE_CONFIG_SQLLOG\smechanism\sand\sthe\ntest_sqllog.c\sdemonstration\sfile.\s\sNo\schanges\sto\scode. -D 2013-04-22T13:51:09.759 +C Add\svirtual\stable\smodule\s'fts3tokenize'\sto\sfts3.\sfts3tokenize\sprovides\sSQL\saccess\sto\sFTS\stokenizer\simplementations. +D 2013-04-22T15:30:37.451 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 +F Makefile.in 87591ea5bf7d6ed521ad42d5bc69c124debe11a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc a244e5b5ba2493625db95b4d46b79201ea868bfe F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 @@ -55,7 +55,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 0eaedfc6d2eb22563ef1d044dcfed93b70ec79f2 +F ext/fts3/fts3.c 784aadfb4c2a217c3eb1feaecac924989f29728f F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 1e58825246b56259267382d2f9902774c049460a F ext/fts3/fts3_aux.c b02632f6dd0e375ce97870206d914ea6d8df5ccd @@ -67,6 +67,7 @@ F ext/fts3/fts3_porter.c a465b49fcb8249a755792f87516eff182efa42b3 F ext/fts3/fts3_snippet.c 5fcfcafff46a2a3a63b8e59fcb51987d01c74695 F ext/fts3/fts3_term.c a521f75132f9a495bdca1bdd45949b3191c52763 F ext/fts3/fts3_test.c f9a1a1702db1bfad3e2d0064746eeb808f125489 +F ext/fts3/fts3_tokenize_vtab.c 1019e24b7f12d234c902053ad8e70f43424d360a F ext/fts3/fts3_tokenizer.c bbdc731bc91338050675c6d1da9ab82147391e16 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 @@ -103,7 +104,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk dcb05737414488383369b93e33cf1a1484c4b6e6 +F main.mk e97fbfe4c69060560574e1b59c43f4f88a38a0ec F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -506,6 +507,8 @@ F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0 F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2 F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 +F test/fts3tok1.test 05ff5c57098282bacba7192507702b17f4061295 +F test/fts3tok_err.test 41e5413139c2ef536ffadfcd1584ee50b1665ec0 F test/fts4aa.test 95f448fb02c4a976968b08d1b4ce134e720946ae F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8 F test/fts4content.test 6efc53b4fd03cab167e6998d2b0b7d4b7d419ee6 @@ -1022,7 +1025,7 @@ F tool/mkkeywordhash.c bb52064aa614e1426445e4b2b9b00eeecd23cc79 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 8bce31074e4cbe631bb7676526a048335f4c9f02 -F tool/mksqlite3c.tcl a570a0b0847f5659482087f70e409ac45ce93b91 +F tool/mksqlite3c.tcl a61fe62a2895ca6458c463fccf1211ca1c000fcf F tool/mksqlite3h.tcl ba24038056f51fde07c0079c41885ab85e2cff12 F tool/mksqlite3internalh.tcl 3dca7bb5374cee003379b8cbac73714f610ef795 F tool/mkvsix.tcl 0be7f7a591f1e83f9199cb82911b66668ca484c9 @@ -1051,7 +1054,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5f4907e1c6230e3dd904bd99e1c48c576c669f63 -R 3f851c92fc64d9ebdb6dc226b2109b9c -U drh -Z 928e878ceb923ac2280cddfe9d967aac +P 206caf21f6f6963aa38a9aa0f6ea11c7230bb726 +R 36a7a566fbf2ef5cbf69a5b7a4283f26 +U dan +Z 5ed9f520f4f2e158b055381052a84fbc diff --git a/manifest.uuid b/manifest.uuid index 77c2db74de..c65f3e21d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -206caf21f6f6963aa38a9aa0f6ea11c7230bb726 \ No newline at end of file +c5a4b21a15bd8a184380817ff41657acbccb866d \ No newline at end of file diff --git a/test/fts3tok1.test b/test/fts3tok1.test new file mode 100644 index 0000000000..89f799606b --- /dev/null +++ b/test/fts3tok1.test @@ -0,0 +1,125 @@ +# 2013 April 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the "fts3tokenize" virtual table +# that is part of the FTS3 module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +ifcapable !fts3 { finish_test ; return } +set ::testprefix fts3tok1 + +#------------------------------------------------------------------------- +# Simple test cases. Using the default (simple) tokenizer. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts3tokenize(simple); + CREATE VIRTUAL TABLE t2 USING fts3tokenize(); + CREATE VIRTUAL TABLE t3 USING fts3tokenize(simple, '', 'xyz '); +} + +foreach {tn tbl} {1 t1 2 t2 3 t3} { + do_execsql_test 1.$tn.1 "SELECT * FROM $tbl WHERE input = 'one two three'" { + {one two three} one 0 3 0 + {one two three} two 4 7 1 + {one two three} three 8 13 2 + } + + do_execsql_test 1.$tn.2 " + SELECT token FROM $tbl WHERE input = 'OnE tWo tHrEe' + " { + one two three + } +} + +do_execsql_test 1.4 { + SELECT token FROM t3 WHERE input = '1x2x3x' +} {1 2 3} + +do_execsql_test 1.5 { + SELECT token FROM t1 WHERE input = '1x2x3x' +} {1x2x3x} + +do_execsql_test 1.6 { + SELECT token FROM t3 WHERE input = '1''2x3x' +} {1'2 3} + +do_execsql_test 1.7 { + SELECT token FROM t3 WHERE input = '' +} {} + +do_execsql_test 1.8 { + SELECT token FROM t3 WHERE input = NULL +} {} + +do_execsql_test 1.9 { + SELECT * FROM t3 WHERE input = 123 +} {123 123 0 3 0} + +do_execsql_test 1.10 { + SELECT * FROM t1 WHERE input = 'a b c' AND token = 'b'; +} { + {a b c} b 2 3 1 +} + +do_execsql_test 1.11 { + SELECT * FROM t1 WHERE token = 'b' AND input = 'a b c'; +} { + {a b c} b 2 3 1 +} + +do_execsql_test 1.12 { + SELECT * FROM t1 WHERE input < 'b' AND input = 'a b c'; +} { + {a b c} a 0 1 0 + {a b c} b 2 3 1 + {a b c} c 4 5 2 +} + +do_execsql_test 1.13.1 { + CREATE TABLE c1(x); + INSERT INTO c1(x) VALUES('a b c'); + INSERT INTO c1(x) VALUES('d e f'); +} +breakpoint +do_execsql_test 1.13.2 { + SELECT * FROM c1, t1 WHERE input = x AND c1.rowid=t1.rowid; +} { + {a b c} {a b c} a 0 1 0 + {d e f} {d e f} e 2 3 1 +} + + +#------------------------------------------------------------------------- +# Error cases. +# +do_catchsql_test 2.0 { + CREATE VIRTUAL TABLE tX USING fts3tokenize(nosuchtokenizer); +} {1 {unknown tokenizer: nosuchtokenizer}} + +proc fts3_tokenizer {args} { return 1 } +db function fts3_tokenizer -argcount 1 fts3_tokenizer +do_catchsql_test 2.1 { + CREATE VIRTUAL TABLE tX USING fts3tokenize(simple); +} {1 {vtable constructor failed: tX}} + +db close +sqlite3 db test.db +do_catchsql_test 2.2 { + CREATE VIRTUAL TABLE t4 USING fts3tokenize; + SELECT * FROM t4; +} {1 {SQL logic error or missing database}} + + +finish_test + + diff --git a/test/fts3tok_err.test b/test/fts3tok_err.test new file mode 100644 index 0000000000..aaa7272b7b --- /dev/null +++ b/test/fts3tok_err.test @@ -0,0 +1,49 @@ +# 2013 April 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the "fts3tokenize" virtual table +# that is part of the FTS3 module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +ifcapable !fts3 { finish_test ; return } +set ::testprefix fts3tok_err + + +faultsim_save_and_close +do_faultsim_test fts3tok_err-1 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { CREATE VIRTUAL TABLE t1 USING fts3tokenize("simple"); } +} -test { + faultsim_test_result {0 {}} +} + +do_test fts3tok_err-2.prep { + faultsim_delete_and_reopen + execsql { CREATE VIRTUAL TABLE t1 USING fts3tokenize("simple"); } + faultsim_save_and_close +} {} + +do_faultsim_test fts3tok_err-2 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { SELECT token FROM t1 WHERE input = 'A galaxy far, far away' } +} -test { + faultsim_test_result {0 {a galaxy far far away}} +} + + +finish_test + + diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 7c8a7d54ad..c93bcea44d 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -311,6 +311,7 @@ foreach file { fts3_porter.c fts3_tokenizer.c fts3_tokenizer1.c + fts3_tokenize_vtab.c fts3_write.c fts3_snippet.c fts3_unicode.c From 4d648c777bc9ab75f4f25e31a4f206f469775189 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 22 Apr 2013 17:07:56 +0000 Subject: [PATCH 154/204] Update MSCV makefile for the fts3tokenize virtual table. Clear some harmless compiler warnings in the fts3tokenize code. FossilOrigin-Name: 2b3fed09eb89f1e9d6a5f7487ef526b46eb51490 --- Makefile.msc | 7 ++++++- ext/fts3/fts3Int.h | 3 +++ ext/fts3/fts3_tokenize_vtab.c | 6 +++--- manifest | 18 +++++++++--------- manifest.uuid | 2 +- 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 8c91e86bc8..47f815e677 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -478,7 +478,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ expr.lo fault.lo fkey.lo \ fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \ fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \ - fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \ + fts3_tokenize_vtab.lo fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \ func.lo global.lo hash.lo \ icu.lo insert.lo journal.lo legacy.lo loadext.lo \ main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ @@ -638,6 +638,7 @@ SRC = $(SRC) \ $(TOP)\ext\fts3\fts3_tokenizer.h \ $(TOP)\ext\fts3\fts3_tokenizer.c \ $(TOP)\ext\fts3\fts3_tokenizer1.c \ + $(TOP)\ext\fts3\fts3_tokenize_vtab.c \ $(TOP)\ext\fts3\fts3_unicode.c \ $(TOP)\ext\fts3\fts3_unicode2.c \ $(TOP)\ext\fts3\fts3_write.c @@ -748,6 +749,7 @@ TESTSRC2 = \ $(TOP)\ext\fts3\fts3_aux.c \ $(TOP)\ext\fts3\fts3_expr.c \ $(TOP)\ext\fts3\fts3_tokenizer.c \ + $(TOP)\ext\fts3\fts3_tokenize_vtab.c \ $(TOP)\ext\fts3\fts3_unicode.c \ $(TOP)\ext\fts3\fts3_unicode2.c \ $(TOP)\ext\fts3\fts3_write.c \ @@ -1176,6 +1178,9 @@ fts3_tokenizer.lo: $(TOP)\ext\fts3\fts3_tokenizer.c $(HDR) $(EXTHDR) fts3_tokenizer1.lo: $(TOP)\ext\fts3\fts3_tokenizer1.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_tokenizer1.c +fts3_tokenize_vtab.lo: $(TOP)\ext\fts3\fts3_tokenize_vtab.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_tokenize_vtab.c + fts3_unicode.lo: $(TOP)\ext\fts3\fts3_unicode.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_unicode.c diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 77ca4704e8..a11c18a3b7 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -549,6 +549,9 @@ int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); +/* fts3_tokenize_vtab.c */ +int sqlite3Fts3InitTok(sqlite3*); + /* fts3_unicode2.c (functions generated by parsing unicode text files) */ #ifdef SQLITE_ENABLE_FTS4_UNICODE61 int sqlite3FtsUnicodeFold(int, int); diff --git a/ext/fts3/fts3_tokenize_vtab.c b/ext/fts3/fts3_tokenize_vtab.c index f00fefe7a9..0f2cef83c3 100644 --- a/ext/fts3/fts3_tokenize_vtab.c +++ b/ext/fts3/fts3_tokenize_vtab.c @@ -92,7 +92,7 @@ static int fts3tokQueryTokenizer( sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); if( SQLITE_ROW==sqlite3_step(pStmt) ){ if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ - memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); + memcpy((void*)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); } } @@ -125,7 +125,7 @@ static int fts3tokDequoteArray( char **azDequote; for(i=0; i Date: Mon, 22 Apr 2013 19:56:12 +0000 Subject: [PATCH 155/204] Do not allow a virtual table to cancel the ORDER BY clause unless all outer loops are guaranteed to return no more than one row result. Candidate fix for ticket [ba82a4a41eac1]. FossilOrigin-Name: 49cfa14fceeef2d55b449eb927c283ce6f650c07 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 11 +++++++++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index fe2a260a73..7c003c6acd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sMSCV\smakefile\sfor\sthe\sfts3tokenize\svirtual\stable.\s\sClear\ssome\nharmless\scompiler\swarnings\sin\sthe\sfts3tokenize\scode. -D 2013-04-22T17:07:56.175 +C Do\snot\sallow\sa\svirtual\stable\sto\scancel\sthe\sORDER\sBY\sclause\sunless\sall\nouter\sloops\sare\sguaranteed\sto\sreturn\sno\smore\sthan\sone\srow\sresult.\nCandidate\sfix\sfor\sticket\s[ba82a4a41eac1]. +D 2013-04-22T19:56:12.702 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 87591ea5bf7d6ed521ad42d5bc69c124debe11a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -259,7 +259,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c c7c74fe8e354b60dee0810ea8b954c5ccb9e0f3c +F src/where.c d54e63087b52c309550aa2defdb20ef27add9f9a F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1054,7 +1054,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c5a4b21a15bd8a184380817ff41657acbccb866d -R 8cef88af587ec812976b521330718bb2 +P 2b3fed09eb89f1e9d6a5f7487ef526b46eb51490 +R c15aea6fcd9f9126da1de54ee794b8c9 U drh -Z 6d195d8624ede0ee30827cabebff2ea8 +Z 5c7507bd8ad3ff74a1cf8b492f137aec diff --git a/manifest.uuid b/manifest.uuid index 4f76ff333d..5a0d3a7ebc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2b3fed09eb89f1e9d6a5f7487ef526b46eb51490 \ No newline at end of file +49cfa14fceeef2d55b449eb927c283ce6f650c07 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 24b059e2d5..d70205205c 100644 --- a/src/where.c +++ b/src/where.c @@ -2400,6 +2400,17 @@ static void bestVirtualIndex(WhereBestIdx *p){ } if( i>=pIdxInfo->nConstraint ) break; } + + /* The orderByConsumed signal is only valid if all outer loops collectively + ** generate just a single row of output. + */ + if( pIdxInfo->orderByConsumed ){ + for(i=0; ii; i++){ + if( (p->aLevel[i].plan.wsFlags & WHERE_UNIQUE)==0 ){ + pIdxInfo->orderByConsumed = 0; + } + } + } /* If there is an ORDER BY clause, and the selected virtual table index ** does not satisfy it, increase the cost of the scan accordingly. This From da8caa0b2da85ebaf75fce1b19f5d645246f3eba Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 22 Apr 2013 23:38:50 +0000 Subject: [PATCH 156/204] Fix harmless compiler warnings. FossilOrigin-Name: 1a1cf5aa86734c832d845e07780262a178188d56 --- ext/fts3/fts3_tokenize_vtab.c | 7 ++++++- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/memjournal.c | 4 +++- src/os_unix.c | 1 + src/vdbetrace.c | 22 +++++++++++----------- 6 files changed, 31 insertions(+), 23 deletions(-) diff --git a/ext/fts3/fts3_tokenize_vtab.c b/ext/fts3/fts3_tokenize_vtab.c index 0f2cef83c3..9ec3b32aad 100644 --- a/ext/fts3/fts3_tokenize_vtab.c +++ b/ext/fts3/fts3_tokenize_vtab.c @@ -121,7 +121,7 @@ static int fts3tokDequoteArray( *pazDequote = 0; }else{ int i; - int nByte; + int nByte = 0; char **azDequote; for(i=0; inConstraint; i++){ if( pInfo->aConstraint[i].usable @@ -269,6 +271,7 @@ static int fts3tokBestIndexMethod( */ static int fts3tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ Fts3tokCursor *pCsr; + UNUSED_PARAMETER(pVTab); pCsr = (Fts3tokCursor *)sqlite3_malloc(sizeof(Fts3tokCursor)); if( pCsr==0 ){ @@ -346,6 +349,8 @@ static int fts3tokFilterMethod( int rc = SQLITE_ERROR; Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(nVal); fts3tokResetCursor(pCsr); if( idxNum==1 ){ diff --git a/manifest b/manifest index 7c003c6acd..f702eb82f2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sallow\sa\svirtual\stable\sto\scancel\sthe\sORDER\sBY\sclause\sunless\sall\nouter\sloops\sare\sguaranteed\sto\sreturn\sno\smore\sthan\sone\srow\sresult.\nCandidate\sfix\sfor\sticket\s[ba82a4a41eac1]. -D 2013-04-22T19:56:12.702 +C Fix\sharmless\scompiler\swarnings. +D 2013-04-22T23:38:50.007 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 87591ea5bf7d6ed521ad42d5bc69c124debe11a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -67,7 +67,7 @@ F ext/fts3/fts3_porter.c a465b49fcb8249a755792f87516eff182efa42b3 F ext/fts3/fts3_snippet.c 5fcfcafff46a2a3a63b8e59fcb51987d01c74695 F ext/fts3/fts3_term.c a521f75132f9a495bdca1bdd45949b3191c52763 F ext/fts3/fts3_test.c f9a1a1702db1bfad3e2d0064746eeb808f125489 -F ext/fts3/fts3_tokenize_vtab.c e097c2e19126249b812a4002e76309d5fef3495b +F ext/fts3/fts3_tokenize_vtab.c a29f126b9e6c6a6f1021a8f7440bf125e68af1f9 F ext/fts3/fts3_tokenizer.c bbdc731bc91338050675c6d1da9ab82147391e16 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 @@ -157,7 +157,7 @@ F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa F src/mem2.c e307323e86b5da1853d7111b68fd6b84ad6f09cf F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 F src/mem5.c c2c63b7067570b00bf33d751c39af24182316f7f -F src/memjournal.c 0ebce851677a7ac035ba1512a7e65851b34530c6 +F src/memjournal.c 41a598445c8f249bd9b26ecae700c60dcf34cbf3 F src/mutex.c d3b66a569368015e0fcb1ac15f81c119f504d3bc F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 @@ -167,7 +167,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 0a561eae5965c9371300b0419027f5ae9f847af2 +F src/os_unix.c 5a9ac4a566fb566a2ff9b9e3a9d723075d9d26a7 F src/os_win.c 673b3e3d1fa3040d8d95a7f1f5e0e553aed56cfb F src/pager.c 6c3a8a5d665498b0344395a2c9f82d5abc4cc771 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -254,7 +254,7 @@ F src/vdbeaux.c ecb43014bcd3019e5aa2b5561e5c3a447f007a08 F src/vdbeblob.c 5dc79627775bd9a9b494dd956e26297946417d69 F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c 4fad64071ae120c25f39dcac572d716b9cadeb7f -F src/vdbetrace.c a22263ab47f6ba4fcd176515cec1e732866b25f0 +F src/vdbetrace.c 3ad1b4e92b60c082a02ac563da4a2735cc7d297c F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 @@ -1054,7 +1054,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 2b3fed09eb89f1e9d6a5f7487ef526b46eb51490 -R c15aea6fcd9f9126da1de54ee794b8c9 +P 49cfa14fceeef2d55b449eb927c283ce6f650c07 +R 83c630a254d9cc209b10e1a83d85002b U drh -Z 5c7507bd8ad3ff74a1cf8b492f137aec +Z a7a8588b07b6fce1e57eced049fe41f1 diff --git a/manifest.uuid b/manifest.uuid index 5a0d3a7ebc..fa38fa5b2c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -49cfa14fceeef2d55b449eb927c283ce6f650c07 \ No newline at end of file +1a1cf5aa86734c832d845e07780262a178188d56 \ No newline at end of file diff --git a/src/memjournal.c b/src/memjournal.c index 3e66e215b2..05725948f6 100644 --- a/src/memjournal.c +++ b/src/memjournal.c @@ -230,7 +230,9 @@ static const struct sqlite3_io_methods MemJournalMethods = { 0, /* xShmMap */ 0, /* xShmLock */ 0, /* xShmBarrier */ - 0 /* xShmUnlock */ + 0, /* xShmUnmap */ + 0, /* xFetch */ + 0 /* xUnfetch */ }; /* diff --git a/src/os_unix.c b/src/os_unix.c index 8dfbe61814..7448afe4c7 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4782,6 +4782,7 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ */ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + UNUSED_PARAMETER(iOff); /* If p==0 (unmap the entire file) then there must be no outstanding ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), diff --git a/src/vdbetrace.c b/src/vdbetrace.c index 91554d6717..88935beeac 100644 --- a/src/vdbetrace.c +++ b/src/vdbetrace.c @@ -128,7 +128,7 @@ char *sqlite3VdbeExpandSql( }else if( pVar->flags & MEM_Real ){ sqlite3XPrintf(&out, "%!.15g", pVar->r); }else if( pVar->flags & MEM_Str ){ - int n; /* Number of bytes of the string text to include in output */ + int nOut; /* Number of bytes of the string text to include in output */ #ifndef SQLITE_OMIT_UTF16 u8 enc = ENC(db); Mem utf8; @@ -140,16 +140,16 @@ char *sqlite3VdbeExpandSql( pVar = &utf8; } #endif - n = pVar->n; + nOut = pVar->n; #ifdef SQLITE_TRACE_SIZE_LIMIT if( n>SQLITE_TRACE_SIZE_LIMIT ){ - n = SQLITE_TRACE_SIZE_LIMIT; - while( nn && (pVar->z[n]&0xc0)==0x80 ){ n++; } + nOut = SQLITE_TRACE_SIZE_LIMIT; + while( nOutn && (pVar->z[n]&0xc0)==0x80 ){ n++; } } #endif - sqlite3XPrintf(&out, "'%.*q'", n, pVar->z); + sqlite3XPrintf(&out, "'%.*q'", nOut, pVar->z); #ifdef SQLITE_TRACE_SIZE_LIMIT - if( nn ) sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-n); + if( nOutn ) sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-n); #endif #ifndef SQLITE_OMIT_UTF16 if( enc!=SQLITE_UTF8 ) sqlite3VdbeMemRelease(&utf8); @@ -157,19 +157,19 @@ char *sqlite3VdbeExpandSql( }else if( pVar->flags & MEM_Zero ){ sqlite3XPrintf(&out, "zeroblob(%d)", pVar->u.nZero); }else{ - int n; /* Number of bytes of the blob to include in output */ + int nOut; /* Number of bytes of the blob to include in output */ assert( pVar->flags & MEM_Blob ); sqlite3StrAccumAppend(&out, "x'", 2); - n = pVar->n; + nOut = pVar->n; #ifdef SQLITE_TRACE_SIZE_LIMIT - if( n>SQLITE_TRACE_SIZE_LIMIT ) n = SQLITE_TRACE_SIZE_LIMIT; + if( nOut>SQLITE_TRACE_SIZE_LIMIT ) nOut = SQLITE_TRACE_SIZE_LIMIT; #endif - for(i=0; iz[i]&0xff); } sqlite3StrAccumAppend(&out, "'", 1); #ifdef SQLITE_TRACE_SIZE_LIMIT - if( nn ) sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-n); + if( nOutn ) sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-n); #endif } } From 248f2be91aefce1e595dd1916fd104f8d3b4d680 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 23 Apr 2013 20:10:13 +0000 Subject: [PATCH 157/204] Make "test_regexp.c" into a loadable extension and move it over to ext/misc/regexp.c. Add the "load_static_extension" command for testing purposes. FossilOrigin-Name: 860fc393bff27045e0593c3c51bf5577accd0b79 --- src/test_regexp.c => ext/misc/regexp.c | 73 +++++++++----------------- main.mk | 2 +- manifest | 23 ++++---- manifest.uuid | 2 +- src/tclsqlite.c | 2 - src/test1.c | 48 +++++++++++++++++ test/regexp1.test | 2 +- 7 files changed, 89 insertions(+), 63 deletions(-) rename src/test_regexp.c => ext/misc/regexp.c (94%) diff --git a/src/test_regexp.c b/ext/misc/regexp.c similarity index 94% rename from src/test_regexp.c rename to ext/misc/regexp.c index 829d22ad25..cb39956da5 100644 --- a/src/test_regexp.c +++ b/ext/misc/regexp.c @@ -49,7 +49,17 @@ */ #include #include -#include "sqlite3.h" +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 + +/* +** The following #defines change the names of some functions implemented in +** this file to prevent name collisions with C-library functions of the +** same name. +*/ +#define re_match sqlite3re_match +#define re_compile sqlite3re_compile +#define re_free sqlite3re_free /* The end-of-input character */ #define RE_EOF 0 /* End of input */ @@ -175,7 +185,7 @@ static int re_space_char(int c){ /* Run a compiled regular expression on the zero-terminated input ** string zIn[]. Return true on a match and false if there is no match. */ -int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){ +static int re_match(ReCompiled *pRe, const unsigned char *zIn, int nIn){ ReStateSet aStateSet[2], *pThis, *pNext; ReStateNumber aSpace[100]; ReStateNumber *pToFree; @@ -718,53 +728,20 @@ static void re_sql_func( } /* -** Invoke this routine in order to install the REGEXP function in an +** Invoke this routine to register the regexp() function with the ** SQLite database connection. -** -** Use: -** -** sqlite3_auto_extension(sqlite3_add_regexp_func); -** -** to cause this extension to be automatically loaded into each new -** database connection. */ -int sqlite3_add_regexp_func(sqlite3 *db){ - return sqlite3_create_function(db, "regexp", 2, SQLITE_UTF8, 0, - re_sql_func, 0, 0); -} - - -/***************************** Test Code ***********************************/ -#ifdef SQLITE_TEST -#include -extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); - -/* Implementation of the TCL command: -** -** sqlite3_add_regexp_func $DB -*/ -static int tclSqlite3AddRegexpFunc( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_regexp_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi ){ - sqlite3 *db; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB"); - return TCL_ERROR; - } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; - sqlite3_add_regexp_func(db); - return TCL_OK; + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + rc = sqlite3_create_function(db, "regexp", 2, SQLITE_UTF8, 0, + re_sql_func, 0, 0); + return rc; } - -/* Register the sqlite3_add_regexp_func TCL command with the TCL interpreter. -*/ -int Sqlitetestregexp_Init(Tcl_Interp *interp){ - Tcl_CreateObjCommand(interp, "sqlite3_add_regexp_func", - tclSqlite3AddRegexpFunc, 0, 0); - return TCL_OK; -} -#endif /* SQLITE_TEST */ -/**************************** End Of Test Code *******************************/ diff --git a/main.mk b/main.mk index 36a4da0294..550b8f10f0 100644 --- a/main.mk +++ b/main.mk @@ -256,7 +256,7 @@ TESTSRC = \ $(TOP)/src/test_osinst.c \ $(TOP)/src/test_pcache.c \ $(TOP)/src/test_quota.c \ - $(TOP)/src/test_regexp.c \ + $(TOP)/ext/misc/regexp.c \ $(TOP)/src/test_rtree.c \ $(TOP)/src/test_schema.c \ $(TOP)/src/test_server.c \ diff --git a/manifest b/manifest index f702eb82f2..bf91bd1649 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings. -D 2013-04-22T23:38:50.007 +C Make\s"test_regexp.c"\sinto\sa\sloadable\sextension\sand\smove\sit\sover\sto\next/misc/regexp.c.\s\sAdd\sthe\s"load_static_extension"\scommand\sfor\stesting\npurposes. +D 2013-04-23T20:10:13.518 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 87591ea5bf7d6ed521ad42d5bc69c124debe11a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -83,6 +83,7 @@ F ext/fts3/unicode/mkunicode.tcl 7a9bc018e2962abb79563c5a39fe581fcbf2f675 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 +F ext/misc/regexp.c c0fdb8af86981ff9890d776cfb97fe66297cc3b2 w src/test_regexp.c F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 757abea591d4ff67c0ff4e8f9776aeda86b18c14 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -104,7 +105,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk e97fbfe4c69060560574e1b59c43f4f88a38a0ec +F main.mk 8f410dfbb97943889298a2da779be0f8f3dcb750 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -190,8 +191,8 @@ F src/sqliteInt.h 2a83cfec9963372b636b0cabd4b200c1f1074a99 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e -F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 -F src/test1.c 6784fdacb35c33ba564ef749b62c4718fe515484 +F src/tclsqlite.c fd60f5950535fd880ce32f357fa9da62db65c490 +F src/test1.c 317741fa810b8acbdf849891202175ea762d68e9 F src/test2.c 29e7154112f7448d64204e8d31179cf497ecf425 F src/test3.c 96aed72a8e1d542fed127e3e8350ae357712fa82 F src/test4.c cea2c55110241e4674e66d476d29c914627999f5 @@ -225,7 +226,6 @@ F src/test_osinst.c 90a845c8183013d80eccb1f29e8805608516edba F src/test_pcache.c a5cd24730cb43c5b18629043314548c9169abb00 F src/test_quota.c 1ec82e02fd3643899e9a5de9684515e84641c91f F src/test_quota.h 8761e463b25e75ebc078bd67d70e39b9c817a0cb -F src/test_regexp.c 06ae8138d41a793330f62351283dd6f6f21104f4 F src/test_rtree.c aba603c949766c4193f1068b91c787f57274e0d9 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f @@ -684,7 +684,7 @@ F test/quote.test 215897dbe8de1a6f701265836d6601cc6ed103e6 F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rdonly.test c267d050a1d9a6a321de502b737daf28821a518d -F test/regexp1.test 5cbb6e7122ca51260d71079cf9245b63b8f64e1a +F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a F test/releasetest.tcl 06d289d8255794073a58d2850742f627924545ce @@ -1054,7 +1054,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 49cfa14fceeef2d55b449eb927c283ce6f650c07 -R 83c630a254d9cc209b10e1a83d85002b +P 1a1cf5aa86734c832d845e07780262a178188d56 +R d55f2fee19b165fc6c487ed43598c861 +T *branch * std-ext +T *sym-std-ext * +T -sym-trunk * U drh -Z a7a8588b07b6fce1e57eced049fe41f1 +Z 87544ad6dda5ce727b201c9348b14476 diff --git a/manifest.uuid b/manifest.uuid index fa38fa5b2c..f18da3e0e5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a1cf5aa86734c832d845e07780262a178188d56 \ No newline at end of file +860fc393bff27045e0593c3c51bf5577accd0b79 \ No newline at end of file diff --git a/src/tclsqlite.c b/src/tclsqlite.c index c0c4fb6499..f8d428e7a8 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -3685,7 +3685,6 @@ static void init_all(Tcl_Interp *interp){ extern int SqlitetestSyscall_Init(Tcl_Interp*); extern int Sqlitetestfuzzer_Init(Tcl_Interp*); extern int Sqlitetestwholenumber_Init(Tcl_Interp*); - extern int Sqlitetestregexp_Init(Tcl_Interp*); #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) extern int Sqlitetestfts3_Init(Tcl_Interp *interp); @@ -3730,7 +3729,6 @@ static void init_all(Tcl_Interp *interp){ SqlitetestSyscall_Init(interp); Sqlitetestfuzzer_Init(interp); Sqlitetestwholenumber_Init(interp); - Sqlitetestregexp_Init(interp); #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) Sqlitetestfts3_Init(interp); diff --git a/src/test1.c b/src/test1.c index 63d014cc17..178aba809d 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6045,6 +6045,53 @@ static int optimization_control( return TCL_OK; } +typedef struct sqlite3_api_routines sqlite3_api_routines; +/* +** load_static_extension DB NAME +** +** Load an extension that is statically linked. +*/ +static int tclLoadStaticExtensionCmd( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + extern int sqlite3_regexp_init(sqlite3*,char**,const sqlite3_api_routines*); + static const struct { + const char *zExtName; + int (*pInit)(sqlite3*,char**,const sqlite3_api_routines*); + } aExtension[] = { + { "regexp", sqlite3_regexp_init }, + }; + sqlite3 *db; + const char *zName; + int i, rc; + char *zErrMsg = 0; + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB NAME"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zName = Tcl_GetString(objv[2]); + for(i=0; i=ArraySize(aExtension) ){ + Tcl_AppendResult(interp, "no such extension: ", zName, (char*)0); + return TCL_ERROR; + } + rc = aExtension[i].pInit(db, &zErrMsg, 0); + if( rc!=SQLITE_OK || zErrMsg ){ + Tcl_AppendResult(interp, "initialization of ", zName, " failed: ", zErrMsg, + (char*)0); + sqlite3_free(zErrMsg); + return TCL_ERROR; + } + return TCL_OK; +} + + /* ** Register commands with the TCL interpreter. */ @@ -6266,6 +6313,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ #if SQLITE_OS_UNIX { "getrusage", test_getrusage }, #endif + { "load_static_extension", tclLoadStaticExtensionCmd }, }; static int bitmask_size = sizeof(Bitmask)*8; int i; diff --git a/test/regexp1.test b/test/regexp1.test index b7ec8fdeab..0e63cd98c8 100644 --- a/test/regexp1.test +++ b/test/regexp1.test @@ -16,7 +16,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl do_test regexp1-1.1 { - sqlite3_add_regexp_func db + load_static_extension db regexp db eval { CREATE TABLE t1(x INTEGER PRIMARY KEY, y TEXT); INSERT INTO t1 VALUES(1, 'For since by man came death,'); From ed17167e1dc966bfb871aea8297206991eac2794 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 24 Apr 2013 13:50:09 +0000 Subject: [PATCH 158/204] Fix a simple comment typo. No changes to code. FossilOrigin-Name: f136bd95824dc95b9e6acdc4d55db263ba13fbaa --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f702eb82f2..7f10b643a0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings. -D 2013-04-22T23:38:50.007 +C Fix\sa\ssimple\scomment\stypo.\s\sNo\schanges\sto\scode. +D 2013-04-24T13:50:09.308 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 87591ea5bf7d6ed521ad42d5bc69c124debe11a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 +F src/select.c 8d097454ff56bdda38c4d877757f592a3c823d15 F src/shell.c aca9d94653decd4496846dee0c7ba83eaf96a46d F src/sqlite.h.in ec279b782bea05db63b8b29481f9642b406004af F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 @@ -1054,7 +1054,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 49cfa14fceeef2d55b449eb927c283ce6f650c07 -R 83c630a254d9cc209b10e1a83d85002b +P 1a1cf5aa86734c832d845e07780262a178188d56 +R 52301974d4ec9029bc00a43d1015330a U drh -Z a7a8588b07b6fce1e57eced049fe41f1 +Z d70e5c91a8e7a901cc898018c72d77a1 diff --git a/manifest.uuid b/manifest.uuid index fa38fa5b2c..6407b99609 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a1cf5aa86734c832d845e07780262a178188d56 \ No newline at end of file +f136bd95824dc95b9e6acdc4d55db263ba13fbaa \ No newline at end of file diff --git a/src/select.c b/src/select.c index c7948096c9..1daa5bed36 100644 --- a/src/select.c +++ b/src/select.c @@ -4047,7 +4047,7 @@ int sqlite3Select( pItem->addrFillSub = topAddr+1; VdbeNoopComment((v, "materialize %s", pItem->pTab->zName)); if( pItem->isCorrelated==0 ){ - /* If the subquery is no correlated and if we are not inside of + /* If the subquery is not correlated and if we are not inside of ** a trigger, then we only need to compute the value of the subquery ** once. */ onceAddr = sqlite3CodeOnce(pParse); From aa87f9a68b5ac0305699507574cb2ed074e722b8 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Apr 2013 00:57:10 +0000 Subject: [PATCH 159/204] Make sure the affinity and datatype of sub-subqueries are initialized prior to subqueries as the latter relies on the former. FossilOrigin-Name: 39b4e6ff9316cc78ea88349091e195b8104d1e9e --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/expr.c | 3 ++- src/resolve.c | 2 ++ src/select.c | 3 +++ src/sqliteInt.h | 1 + src/walker.c | 17 ++++++++++++++--- test/selectD.test | 19 +++++++++++++++++++ 8 files changed, 53 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 7f10b643a0..31cc673a2f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\ssimple\scomment\stypo.\s\sNo\schanges\sto\scode. -D 2013-04-24T13:50:09.308 +C Make\ssure\sthe\saffinity\sand\sdatatype\sof\ssub-subqueries\sare\sinitialized\nprior\sto\ssubqueries\sas\sthe\slatter\srelies\son\sthe\sformer. +D 2013-04-25T00:57:10.497 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 87591ea5bf7d6ed521ad42d5bc69c124debe11a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -137,7 +137,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 4262c227bc91cecc61ae37ed3a40f08069cfa267 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 -F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 +F src/expr.c 437c03d5bb4fe3a53ecab3ad0286d6c5260da7ed F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c e16942bd5c8a868ac53287886464a5ed0e72b179 F src/func.c d3fdcff9274bc161152e67ed3f626841c247f4b9 @@ -179,14 +179,14 @@ F src/pragma.c 3eacf001cbf4becbd494f8d82d08fdf1648cf8cb F src/prepare.c 743e484233c51109666d402f470523553b41797c F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 -F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 +F src/resolve.c 83cc2d942ee216bc56956c6e6fadb691c1727fa1 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 8d097454ff56bdda38c4d877757f592a3c823d15 +F src/select.c 6bfbe11e2fef81c5e18d30513ab6c69f171667eb F src/shell.c aca9d94653decd4496846dee0c7ba83eaf96a46d F src/sqlite.h.in ec279b782bea05db63b8b29481f9642b406004af F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h 2a83cfec9963372b636b0cabd4b200c1f1074a99 +F src/sqliteInt.h 3585ea1bb8776baa9f2675e9ef3d9170e7aeda29 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -258,7 +258,7 @@ F src/vdbetrace.c 3ad1b4e92b60c082a02ac563da4a2735cc7d297c F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 -F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b +F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 F src/where.c d54e63087b52c309550aa2defdb20ef27add9f9a F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -719,7 +719,7 @@ F test/select9.test c0ca3cd87a8ebb04de2cb1402c77df55d911a0ea F test/selectA.test 06d1032fa9009314c95394f2ca2e60d9f7ae8532 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 F test/selectC.test 871fb55d884d3de5943c4057ebd22c2459e71977 -F test/selectD.test 03f7c1ea8d5ab3c637cbc30fcbbbac96b988c162 +F test/selectD.test b0f02a04ef7737decb24e08be2c39b9664b43394 F test/server1.test 46803bd3fe8b99b30dbc5ff38ffc756f5c13a118 F test/shared.test 1da9dbad400cee0d93f252ccf76e1ae007a63746 F test/shared2.test 03eb4a8d372e290107d34b6ce1809919a698e879 @@ -1054,7 +1054,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 1a1cf5aa86734c832d845e07780262a178188d56 -R 52301974d4ec9029bc00a43d1015330a +P f136bd95824dc95b9e6acdc4d55db263ba13fbaa +R 35e39801836ac191fc1f292f3b81674a U drh -Z d70e5c91a8e7a901cc898018c72d77a1 +Z 424df927045edc37db2bc00e1032f86c diff --git a/manifest.uuid b/manifest.uuid index 6407b99609..4aeba6cd08 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f136bd95824dc95b9e6acdc4d55db263ba13fbaa \ No newline at end of file +39b4e6ff9316cc78ea88349091e195b8104d1e9e \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index ae6a1dec10..a974c5a61f 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1214,6 +1214,7 @@ static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){ } static int exprIsConst(Expr *p, int initFlag){ Walker w; + memset(&w, 0, sizeof(w)); w.u.i = initFlag; w.xExprCallback = exprNodeIsConstant; w.xSelectCallback = selectNodeIsConstant; @@ -3428,8 +3429,8 @@ void sqlite3ExprCodeConstants(Parse *pParse, Expr *pExpr){ Walker w; if( pParse->cookieGoto ) return; if( OptimizationDisabled(pParse->db, SQLITE_FactorOutConst) ) return; + memset(&w, 0, sizeof(w)); w.xExprCallback = evalConstExpr; - w.xSelectCallback = 0; w.pParse = pParse; sqlite3WalkExpr(&w, pExpr); } diff --git a/src/resolve.c b/src/resolve.c index 9b350caf80..a8e196926c 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -1283,6 +1283,7 @@ int sqlite3ResolveExprNames( #endif savedHasAgg = pNC->ncFlags & NC_HasAgg; pNC->ncFlags &= ~NC_HasAgg; + memset(&w, 0, sizeof(w)); w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; w.pParse = pNC->pParse; @@ -1323,6 +1324,7 @@ void sqlite3ResolveSelectNames( Walker w; assert( p!=0 ); + memset(&w, 0, sizeof(w)); w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; w.pParse = pParse; diff --git a/src/select.c b/src/select.c index 1daa5bed36..a745dc370a 100644 --- a/src/select.c +++ b/src/select.c @@ -3576,6 +3576,7 @@ static int exprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ */ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ Walker w; + memset(&w, 0, sizeof(w)); w.xSelectCallback = selectExpander; w.xExprCallback = exprWalkNoop; w.pParse = pParse; @@ -3634,9 +3635,11 @@ static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ #ifndef SQLITE_OMIT_SUBQUERY Walker w; + memset(&w, 0, sizeof(w)); w.xSelectCallback = selectAddSubqueryTypeInfo; w.xExprCallback = exprWalkNoop; w.pParse = pParse; + w.bSelectDepthFirst = 1; sqlite3WalkSelect(&w, pSelect); #endif } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3722041b0a..876629a8c8 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2588,6 +2588,7 @@ struct Walker { int (*xSelectCallback)(Walker*,Select*); /* Callback for SELECTs */ Parse *pParse; /* Parser context. */ int walkerDepth; /* Number of subqueries */ + u8 bSelectDepthFirst; /* Do subqueries first */ union { /* Extra data for callback */ NameContext *pNC; /* Naming context */ int i; /* Integer value */ diff --git a/src/walker.c b/src/walker.c index eab96ea24d..e71ed2ac48 100644 --- a/src/walker.c +++ b/src/walker.c @@ -113,7 +113,9 @@ int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ /* ** Call sqlite3WalkExpr() for every expression in Select statement p. ** Invoke sqlite3WalkSelect() for subqueries in the FROM clause and -** on the compound select chain, p->pPrior. +** on the compound select chain, p->pPrior. Invoke the xSelectCallback() +** either before or after the walk of expressions and FROM clause, depending +** on whether pWalker->bSelectDepthFirst is false or true, respectively. ** ** Return WRC_Continue under normal conditions. Return WRC_Abort if ** there is an abort request. @@ -127,14 +129,23 @@ int sqlite3WalkSelect(Walker *pWalker, Select *p){ rc = WRC_Continue; pWalker->walkerDepth++; while( p ){ - rc = pWalker->xSelectCallback(pWalker, p); - if( rc ) break; + if( !pWalker->bSelectDepthFirst ){ + rc = pWalker->xSelectCallback(pWalker, p); + if( rc ) break; + } if( sqlite3WalkSelectExpr(pWalker, p) || sqlite3WalkSelectFrom(pWalker, p) ){ pWalker->walkerDepth--; return WRC_Abort; } + if( pWalker->bSelectDepthFirst ){ + rc = pWalker->xSelectCallback(pWalker, p); + /* Depth-first search is currently only used for + ** selectAddSubqueryTypeInfo() and that routine always returns + ** WRC_Continue (0). So the following branch is never taken. */ + if( NEVER(rc) ) break; + } p = p->pPrior; } pWalker->walkerDepth--; diff --git a/test/selectD.test b/test/selectD.test index aa8c328ee9..89f999eb6d 100644 --- a/test/selectD.test +++ b/test/selectD.test @@ -152,4 +152,23 @@ for {set i 1} {$i<=2} {incr i} { } {111 x1 111 x2 222 x3 {}} } +# The following test was added on 2013-04-24 in order to verify that +# the datatypes and affinities of sub-sub-queries are set prior to computing +# the datatypes and affinities of the parent sub-queries because the +# latter computation depends on the former. +# +do_execsql_test selectD-4.1 { + CREATE TABLE t41(a INTEGER PRIMARY KEY, b INTEGER); + CREATE TABLE t42(d INTEGER PRIMARY KEY, e INTEGER); + CREATE TABLE t43(f INTEGER PRIMARY KEY, g INTEGER); + EXPLAIN QUERY PLAN + SELECT * + FROM t41 + LEFT JOIN (SELECT count(*) AS cnt, x1.d + FROM (t42 INNER JOIN t43 ON d=g) AS x1 + WHERE x1.d>5 + GROUP BY x1.d) AS x2 + ON t41.b=x2.d; +} {/.*SEARCH SUBQUERY 1 AS x2 USING AUTOMATIC.*/} + finish_test From 24b6422dcb66ec9ad8828b2a65e6e5afc224be22 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Apr 2013 11:58:36 +0000 Subject: [PATCH 160/204] Make test_wholenumber.c into a loadable extension and move it to ext/misc/wholenumber.c. FossilOrigin-Name: efcc9dd012b5f193324dfc2ee9c2410c16fc1b3b --- .../misc/wholenumber.c | 65 ++++--------------- main.mk | 2 +- manifest | 33 +++++----- manifest.uuid | 2 +- src/tclsqlite.c | 2 - src/test1.c | 4 +- test/8_3_names.test | 4 +- test/analyze7.test | 2 +- test/memdb.test | 2 +- test/tkt-2d1a5c67d.test | 2 +- test/zerodamage.test | 2 +- 11 files changed, 37 insertions(+), 83 deletions(-) rename src/test_wholenumber.c => ext/misc/wholenumber.c (83%) diff --git a/src/test_wholenumber.c b/ext/misc/wholenumber.c similarity index 83% rename from src/test_wholenumber.c rename to ext/misc/wholenumber.c index 7c42d01691..a6d79507c4 100644 --- a/src/test_wholenumber.c +++ b/ext/misc/wholenumber.c @@ -22,7 +22,8 @@ ** ** 1 2 3 4 5 6 7 8 9 */ -#include "sqlite3.h" +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 #include #include @@ -250,62 +251,18 @@ static sqlite3_module wholenumberModule = { #endif /* SQLITE_OMIT_VIRTUALTABLE */ - -/* -** Register the wholenumber virtual table -*/ -int wholenumber_register(sqlite3 *db){ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_wholenumber_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); #ifndef SQLITE_OMIT_VIRTUALTABLE rc = sqlite3_create_module(db, "wholenumber", &wholenumberModule, 0); #endif return rc; } - -#ifdef SQLITE_TEST -#include -/* -** Decode a pointer to an sqlite3 object. -*/ -extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); - -/* -** Register the echo virtual table module. -*/ -static int register_wholenumber_module( - ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int objc, /* Number of arguments */ - Tcl_Obj *CONST objv[] /* Command arguments */ -){ - sqlite3 *db; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB"); - return TCL_ERROR; - } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; - wholenumber_register(db); - return TCL_OK; -} - - -/* -** Register commands with the TCL interpreter. -*/ -int Sqlitetestwholenumber_Init(Tcl_Interp *interp){ - static struct { - char *zName; - Tcl_ObjCmdProc *xProc; - void *clientData; - } aObjCmd[] = { - { "register_wholenumber_module", register_wholenumber_module, 0 }, - }; - int i; - for(i=0; i Date: Thu, 25 Apr 2013 14:31:46 +0000 Subject: [PATCH 161/204] Convert the fuzzer virtual table into a loadable extension and move it to the ext/misc/fuzzer.c file. FossilOrigin-Name: c8c69307f60c1d07ac666ae3797b7e3f286fd491 --- Makefile.in | 10 ++-- Makefile.msc | 11 +++-- src/test_fuzzer.c => ext/misc/fuzzer.c | 68 +++++--------------------- main.mk | 11 +++-- manifest | 28 +++++------ manifest.uuid | 2 +- src/tclsqlite.c | 2 - src/test1.c | 2 + test/fuzzer1.test | 7 +-- test/fuzzerfault.test | 2 +- 10 files changed, 54 insertions(+), 89 deletions(-) rename src/test_fuzzer.c => ext/misc/fuzzer.c (96%) diff --git a/Makefile.in b/Makefile.in index 5c636209d8..597994c15c 100644 --- a/Makefile.in +++ b/Makefile.in @@ -361,7 +361,6 @@ TESTSRC = \ $(TOP)/src/test_devsym.c \ $(TOP)/src/test_fs.c \ $(TOP)/src/test_func.c \ - $(TOP)/src/test_fuzzer.c \ $(TOP)/src/test_hexio.c \ $(TOP)/src/test_init.c \ $(TOP)/src/test_intarray.c \ @@ -373,7 +372,6 @@ TESTSRC = \ $(TOP)/src/test_osinst.c \ $(TOP)/src/test_pcache.c \ $(TOP)/src/test_quota.c \ - $(TOP)/src/test_regexp.c \ $(TOP)/src/test_rtree.c \ $(TOP)/src/test_schema.c \ $(TOP)/src/test_server.c \ @@ -383,11 +381,17 @@ TESTSRC = \ $(TOP)/src/test_tclvar.c \ $(TOP)/src/test_thread.c \ $(TOP)/src/test_vfs.c \ - $(TOP)/src/test_wholenumber.c \ $(TOP)/src/test_wsd.c \ $(TOP)/ext/fts3/fts3_term.c \ $(TOP)/ext/fts3/fts3_test.c +# Statically linked extensions +# +TESTSRC += \ + $(TOP)/ext/misc/fuzzer.c \ + $(TOP)/ext/misc/regexp.c \ + $(TOP)/ext/misc/wholenumber.c + # Source code to the library files needed by the test fixture # TESTSRC2 = \ diff --git a/Makefile.msc b/Makefile.msc index 47f815e677..982a959dc6 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -681,7 +681,6 @@ TESTSRC = \ $(TOP)\src\test_devsym.c \ $(TOP)\src\test_fs.c \ $(TOP)\src\test_func.c \ - $(TOP)\src\test_fuzzer.c \ $(TOP)\src\test_hexio.c \ $(TOP)\src\test_init.c \ $(TOP)\src\test_intarray.c \ @@ -693,7 +692,6 @@ TESTSRC = \ $(TOP)\src\test_osinst.c \ $(TOP)\src\test_pcache.c \ $(TOP)\src\test_quota.c \ - $(TOP)\src\test_regexp.c \ $(TOP)\src\test_rtree.c \ $(TOP)\src\test_schema.c \ $(TOP)\src\test_server.c \ @@ -703,11 +701,18 @@ TESTSRC = \ $(TOP)\src\test_tclvar.c \ $(TOP)\src\test_thread.c \ $(TOP)\src\test_vfs.c \ - $(TOP)\src\test_wholenumber.c \ $(TOP)\src\test_wsd.c \ $(TOP)\ext\fts3\fts3_term.c \ $(TOP)\ext\fts3\fts3_test.c +# Statically linked extensions +# +TESTSRC += \ + $(TOP)\ext\misc\fuzzer.c \ + $(TOP)\ext\misc\regexp.c \ + $(TOP)\ext\misc\wholenumber.c + + # Source code to the library files needed by the test fixture # TESTSRC2 = \ diff --git a/src/test_fuzzer.c b/ext/misc/fuzzer.c similarity index 96% rename from src/test_fuzzer.c rename to ext/misc/fuzzer.c index 10496f2ea7..c0c294b11c 100644 --- a/src/test_fuzzer.c +++ b/ext/misc/fuzzer.c @@ -141,13 +141,14 @@ ** of the strings in the second or third column of the fuzzer data table ** is 50 bytes. The maximum cost on a rule is 1000. */ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 /* If SQLITE_DEBUG is not defined, disable assert statements. */ #if !defined(NDEBUG) && !defined(SQLITE_DEBUG) # define NDEBUG #endif -#include "sqlite3.h" #include #include #include @@ -1155,61 +1156,16 @@ static sqlite3_module fuzzerModule = { #endif /* SQLITE_OMIT_VIRTUALTABLE */ -/* -** Register the fuzzer virtual table -*/ -int fuzzer_register(sqlite3 *db){ - int rc = SQLITE_OK; -#ifndef SQLITE_OMIT_VIRTUALTABLE - rc = sqlite3_create_module(db, "fuzzer", &fuzzerModule, 0); +#ifdef _WIN32 +__declspec(dllexport) #endif +int sqlite3_fuzzer_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + rc = sqlite3_create_module(db, "fuzzer", &fuzzerModule, 0); return rc; } - -#ifdef SQLITE_TEST -#include -/* -** Decode a pointer to an sqlite3 object. -*/ -extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); - -/* -** Register the echo virtual table module. -*/ -static int register_fuzzer_module( - ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int objc, /* Number of arguments */ - Tcl_Obj *CONST objv[] /* Command arguments */ -){ - sqlite3 *db; - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB"); - return TCL_ERROR; - } - getDbPointer(interp, Tcl_GetString(objv[1]), &db); - fuzzer_register(db); - return TCL_OK; -} - - -/* -** Register commands with the TCL interpreter. -*/ -int Sqlitetestfuzzer_Init(Tcl_Interp *interp){ - static struct { - char *zName; - Tcl_ObjCmdProc *xProc; - void *clientData; - } aObjCmd[] = { - { "register_fuzzer_module", register_fuzzer_module, 0 }, - }; - int i; - for(i=0; i Date: Thu, 25 Apr 2013 14:36:28 +0000 Subject: [PATCH 162/204] Fix the MSVC makefile so that it works with the ext/misc/*.c extensions. FossilOrigin-Name: 680822e892f3efdb702eea3b321bc5785239dd56 --- Makefile.msc | 6 +++--- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 982a959dc6..ecdac8c412 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -707,7 +707,7 @@ TESTSRC = \ # Statically linked extensions # -TESTSRC += \ +TESTEXT = \ $(TOP)\ext\misc\fuzzer.c \ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\wholenumber.c @@ -1209,8 +1209,8 @@ rtree.lo: $(TOP)\ext\rtree\rtree.c $(HDR) $(EXTHDR) TESTFIXTURE_FLAGS = -DTCLSH=1 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_SERVER=1 -DSQLITE_PRIVATE="" -DSQLITE_CORE -TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.lib -TESTFIXTURE_SRC1 = sqlite3.c +TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2) libsqlite3.lib +TESTFIXTURE_SRC1 = $(TESTEXT) sqlite3.c !IF $(USE_AMALGAMATION)==0 TESTFIXTURE_SRC = $(TESTSRC) $(TOP)\src\tclsqlite.c $(TESTFIXTURE_SRC0) !ELSE diff --git a/manifest b/manifest index 9878de9b34..355653e47b 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Convert\sthe\sfuzzer\svirtual\stable\sinto\sa\sloadable\sextension\sand\smove\sit\nto\sthe\sext/misc/fuzzer.c\sfile. -D 2013-04-25T14:31:46.866 +C Fix\sthe\sMSVC\smakefile\sso\sthat\sit\sworks\swith\sthe\sext/misc/*.c\sextensions. +D 2013-04-25T14:36:28.444 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 422a8541e3595e4fd4d962a46f52fbe095a31bd2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc ef00ccb017f6da580ab934d096b08da40d81f1a5 +F Makefile.msc 3657134d4304934fd9a2c81ff6a2bd7e2187954b F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 05c7bd63b96f31cfdef5c766ed91307ac121f5aa @@ -83,7 +83,7 @@ F ext/fts3/unicode/mkunicode.tcl 7a9bc018e2962abb79563c5a39fe581fcbf2f675 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 -F ext/misc/fuzzer.c fb64a15af978ae73fa9075b9b1dfbe82b8defc6f w src/test_fuzzer.c +F ext/misc/fuzzer.c fb64a15af978ae73fa9075b9b1dfbe82b8defc6f F ext/misc/regexp.c c0fdb8af86981ff9890d776cfb97fe66297cc3b2 F ext/misc/wholenumber.c ce362368b9381ea48cbd951ade8df867eeeab014 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1054,7 +1054,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P efcc9dd012b5f193324dfc2ee9c2410c16fc1b3b -R 2b859f3329d083b32ee90e4538687497 +P c8c69307f60c1d07ac666ae3797b7e3f286fd491 +R 3a7355d9d00cf8a32d1b35340d234e67 U drh -Z 672d17ed58c8e3fac84edc0141a68b79 +Z 8dc283cf0c63ecf74b33a1f0701aa7d4 diff --git a/manifest.uuid b/manifest.uuid index 6f090d8535..b8ed72b0f8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c8c69307f60c1d07ac666ae3797b7e3f286fd491 \ No newline at end of file +680822e892f3efdb702eea3b321bc5785239dd56 \ No newline at end of file From b7045ab2eda7ad2179d15a5d1ae9a4e807148066 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Apr 2013 14:59:01 +0000 Subject: [PATCH 163/204] Move the test_spellfix.c module to ext/misc/spellfix.c. FossilOrigin-Name: de556add10150140981a2e34b3712e96a7c262e3 --- Makefile.in | 1 + Makefile.msc | 1 + src/test_spellfix.c => ext/misc/spellfix.c | 42 ++++++++-------------- main.mk | 1 + manifest | 26 +++++++------- manifest.uuid | 2 +- src/shell.c | 12 ------- src/test1.c | 2 ++ src/test8.c | 24 ------------- test/spellfix.test | 2 +- 10 files changed, 35 insertions(+), 78 deletions(-) rename src/test_spellfix.c => ext/misc/spellfix.c (99%) diff --git a/Makefile.in b/Makefile.in index 597994c15c..74bf58e31f 100644 --- a/Makefile.in +++ b/Makefile.in @@ -390,6 +390,7 @@ TESTSRC = \ TESTSRC += \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/misc/regexp.c \ + $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/wholenumber.c # Source code to the library files needed by the test fixture diff --git a/Makefile.msc b/Makefile.msc index ecdac8c412..ea8fc10a2e 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -710,6 +710,7 @@ TESTSRC = \ TESTEXT = \ $(TOP)\ext\misc\fuzzer.c \ $(TOP)\ext\misc\regexp.c \ + $(TOP)\ext\misc\spellfix.c \ $(TOP)\ext\misc\wholenumber.c diff --git a/src/test_spellfix.c b/ext/misc/spellfix.c similarity index 99% rename from src/test_spellfix.c rename to ext/misc/spellfix.c index 16376244a3..a8492185b3 100644 --- a/src/test_spellfix.c +++ b/ext/misc/spellfix.c @@ -14,20 +14,17 @@ ** to search a large vocabulary for close matches. See separate ** documentation files (spellfix1.wiki and editdist3.wiki) for details. */ -#if SQLITE_CORE -# include "sqliteInt.h" -#else -# include -# include -# include -# include "sqlite3ext.h" -# include -# define ALWAYS(X) 1 -# define NEVER(X) 0 - typedef unsigned char u8; - typedef unsigned short u16; - SQLITE_EXTENSION_INIT1 -#endif /* !SQLITE_CORE */ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 + +#include +#include +#include +#include +#define ALWAYS(X) 1 +#define NEVER(X) 0 +typedef unsigned char u8; +typedef unsigned short u16; #include /* @@ -2822,21 +2819,13 @@ static int spellfix1Register(sqlite3 *db){ return rc; } -#if SQLITE_CORE || defined(SQLITE_TEST) -/* -** Register the spellfix1 virtual table and its associated functions. -*/ -int sqlite3_spellfix1_register(sqlite3 *db){ - return spellfix1Register(db); -} -#endif - - -#if !SQLITE_CORE /* ** Extension load function. */ -int sqlite3_spellfix1_init( +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_spellfix_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi @@ -2844,4 +2833,3 @@ int sqlite3_spellfix1_init( SQLITE_EXTENSION_INIT2(pApi); return spellfix1Register(db); } -#endif /* !SQLITE_CORE */ diff --git a/main.mk b/main.mk index dd85a9acbe..eb3da5eaee 100644 --- a/main.mk +++ b/main.mk @@ -272,6 +272,7 @@ TESTSRC = \ TESTSRC += \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/misc/regexp.c \ + $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/wholenumber.c diff --git a/manifest b/manifest index 355653e47b..4c3502f113 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Fix\sthe\sMSVC\smakefile\sso\sthat\sit\sworks\swith\sthe\sext/misc/*.c\sextensions. -D 2013-04-25T14:36:28.444 +C Move\sthe\stest_spellfix.c\smodule\sto\sext/misc/spellfix.c. +D 2013-04-25T14:59:01.366 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 422a8541e3595e4fd4d962a46f52fbe095a31bd2 +F Makefile.in 10c635460b6c3a20741d71c3a1b65b0ebec7558b F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 3657134d4304934fd9a2c81ff6a2bd7e2187954b +F Makefile.msc 97a23e910afb8c2d23db0217c343481d59045acb F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 05c7bd63b96f31cfdef5c766ed91307ac121f5aa @@ -85,6 +85,7 @@ F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/misc/fuzzer.c fb64a15af978ae73fa9075b9b1dfbe82b8defc6f F ext/misc/regexp.c c0fdb8af86981ff9890d776cfb97fe66297cc3b2 +F ext/misc/spellfix.c 8bb699116e36cc5e68d7ddf1810b638a3090c744 w src/test_spellfix.c F ext/misc/wholenumber.c ce362368b9381ea48cbd951ade8df867eeeab014 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 757abea591d4ff67c0ff4e8f9776aeda86b18c14 @@ -107,7 +108,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk bf3cf721c54c97e0eb02856d7d67828ac9738c4e +F main.mk a6183110637d782988fdf3735f7d125b19639967 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -185,7 +186,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 -F src/shell.c aca9d94653decd4496846dee0c7ba83eaf96a46d +F src/shell.c 5d527e5d08f05ec2c43ff194ea44bf62b974f4c9 F src/sqlite.h.in ec279b782bea05db63b8b29481f9642b406004af F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 @@ -194,14 +195,14 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 2ecec9937e69bc17560ad886da35195daa7261b8 -F src/test1.c aba32f14372febad3e7c14798b25bdd9fd0b7c61 +F src/test1.c 1c7fa0a36703508130d7ed65325883a9e453bf72 F src/test2.c 29e7154112f7448d64204e8d31179cf497ecf425 F src/test3.c 96aed72a8e1d542fed127e3e8350ae357712fa82 F src/test4.c cea2c55110241e4674e66d476d29c914627999f5 F src/test5.c a6d1ac55ac054d0b2b8f37b5e655b6c92645a013 F src/test6.c a437f76f9874d2563352a7e6cd0d43217663c220 F src/test7.c f4b894b7931f8cf9f5cbf37cfa0727703f526a40 -F src/test8.c 58ea1d9698f3947e4662107ef98f429e84ae20e0 +F src/test8.c f7e729e3e1973f68e6d98f5aa65046e3e2cb0bad F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad F src/test_autoext.c 5c95b5d435eaa09d6c0e7d90371c5ca8cd567701 @@ -230,7 +231,6 @@ F src/test_quota.h 8761e463b25e75ebc078bd67d70e39b9c817a0cb F src/test_rtree.c aba603c949766c4193f1068b91c787f57274e0d9 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f -F src/test_spellfix.c 56dfa6d583ac34f61af0834d7b58d674e7e18e13 F src/test_sqllog.c c1c1bbedbcaf82b93d83e4f9dd990e62476a680e F src/test_stat.c d1569c7a4839f13e80187e2c26b2ab4da2d03935 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd @@ -750,7 +750,7 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/spellfix.test 52ae2680b1247c52b9e2b2116de3fd26a78e6bd2 +F test/spellfix.test a85915ab25af7fcfb0d99cb1951e6ef15e26202c F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test be8d477306006ec696bc86757cfb34bec79447ce F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9 @@ -1054,7 +1054,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c8c69307f60c1d07ac666ae3797b7e3f286fd491 -R 3a7355d9d00cf8a32d1b35340d234e67 +P 680822e892f3efdb702eea3b321bc5785239dd56 +R f6325e95c7057e7590cfe9f4379a5b36 U drh -Z 8dc283cf0c63ecf74b33a1f0701aa7d4 +Z 2d983c030ad222dceb5985301e41e8c4 diff --git a/manifest.uuid b/manifest.uuid index b8ed72b0f8..2730c8b8c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -680822e892f3efdb702eea3b321bc5785239dd56 \ No newline at end of file +de556add10150140981a2e34b3712e96a7c262e3 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 13a2f70cbc..09c3b810c8 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1480,18 +1480,6 @@ static void open_db(struct callback_data *p){ } #ifndef SQLITE_OMIT_LOAD_EXTENSION sqlite3_enable_load_extension(p->db, 1); -#endif -#ifdef SQLITE_ENABLE_REGEXP - { - extern int sqlite3_add_regexp_func(sqlite3*); - sqlite3_add_regexp_func(db); - } -#endif -#ifdef SQLITE_ENABLE_SPELLFIX - { - extern int sqlite3_spellfix1_register(sqlite3*); - sqlite3_spellfix1_register(db); - } #endif } } diff --git a/src/test1.c b/src/test1.c index 328d0a8e45..7c30962b75 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6059,6 +6059,7 @@ static int tclLoadStaticExtensionCmd( ){ extern int sqlite3_fuzzer_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_regexp_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_spellfix_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_wholenumber_init(sqlite3*,char**,const sqlite3_api_routines*); static const struct { const char *zExtName; @@ -6066,6 +6067,7 @@ static int tclLoadStaticExtensionCmd( } aExtension[] = { { "fuzzer", sqlite3_fuzzer_init }, { "regexp", sqlite3_regexp_init }, + { "spellfix", sqlite3_spellfix_init }, { "wholenumber", sqlite3_wholenumber_init }, }; sqlite3 *db; diff --git a/src/test8.c b/src/test8.c index 6bd4e396b8..f7b2e6813d 100644 --- a/src/test8.c +++ b/src/test8.c @@ -1370,29 +1370,6 @@ static int declare_vtab( return TCL_OK; } -#include "test_spellfix.c" - -/* -** Register the spellfix virtual table module. -*/ -static int register_spellfix_module( - ClientData clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - sqlite3 *db; - - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB"); - return TCL_ERROR; - } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; - - sqlite3_spellfix1_register(db); - return TCL_OK; -} - #endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ /* @@ -1406,7 +1383,6 @@ int Sqlitetest8_Init(Tcl_Interp *interp){ void *clientData; } aObjCmd[] = { { "register_echo_module", register_echo_module, 0 }, - { "register_spellfix_module", register_spellfix_module, 0 }, { "sqlite3_declare_vtab", declare_vtab, 0 }, }; int i; diff --git a/test/spellfix.test b/test/spellfix.test index 6fb32b6d37..dbe8bd60e9 100644 --- a/test/spellfix.test +++ b/test/spellfix.test @@ -16,7 +16,7 @@ set testprefix spellfix ifcapable !vtab { finish_test ; return } -register_spellfix_module db +load_static_extension db spellfix set vocab { rabbi rabbit rabbits rabble rabid rabies raccoon raccoons race raced racer From 8416fc7fc7a590451c32a5b1f89180d065d37a00 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Apr 2013 16:42:55 +0000 Subject: [PATCH 164/204] Added the transitive_closure, ieee754, and amatch extensions. FossilOrigin-Name: 84018099c8715b982cd24ce9221f93c7379e8c08 --- Makefile.in | 3 + Makefile.msc | 3 + ext/misc/amatch.c | 1477 +++++++++++++++++++++++++++++++++++++++++++ ext/misc/closure.c | 942 +++++++++++++++++++++++++++ ext/misc/ieee754.c | 131 ++++ ext/misc/regexp.c | 11 +- main.mk | 3 + manifest | 26 +- manifest.uuid | 2 +- src/test1.c | 6 + test/closure01.test | 224 +++++++ 11 files changed, 2815 insertions(+), 13 deletions(-) create mode 100644 ext/misc/amatch.c create mode 100644 ext/misc/closure.c create mode 100644 ext/misc/ieee754.c create mode 100644 test/closure01.test diff --git a/Makefile.in b/Makefile.in index 74bf58e31f..c713daaebc 100644 --- a/Makefile.in +++ b/Makefile.in @@ -388,7 +388,10 @@ TESTSRC = \ # Statically linked extensions # TESTSRC += \ + $(TOP)/ext/misc/amatch.c \ + $(TOP)/ext/misc/closure.c \ $(TOP)/ext/misc/fuzzer.c \ + $(TOP)/ext/misc/ieee754.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/wholenumber.c diff --git a/Makefile.msc b/Makefile.msc index ea8fc10a2e..383368c44c 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -708,7 +708,10 @@ TESTSRC = \ # Statically linked extensions # TESTEXT = \ + $(TOP)\ext\misc\amatch.c \ + $(TOP)\ext\misc\closure.c \ $(TOP)\ext\misc\fuzzer.c \ + $(TOP)\ext\misc\ieee754.c \ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\spellfix.c \ $(TOP)\ext\misc\wholenumber.c diff --git a/ext/misc/amatch.c b/ext/misc/amatch.c new file mode 100644 index 0000000000..f91c84f06e --- /dev/null +++ b/ext/misc/amatch.c @@ -0,0 +1,1477 @@ +/* +** 2013-03-14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code for a demonstration virtual table that finds +** "approximate matches" - strings from a finite set that are nearly the +** same as a single input string. The virtual table is called "amatch". +** +** A amatch virtual table is created like this: +** +** CREATE VIRTUAL TABLE f USING approximate_match( +** vocabulary_table=, -- V +** vocabulary_word=, -- W +** vocabulary_language=, -- L +** edit_distances= +** ); +** +** When it is created, the new amatch table must be supplied with the +** the name of a table V and columns V.W and V.L such that +** +** SELECT W FROM V WHERE L=$language +** +** returns the allowed vocabulary for the match. If the "vocabulary_language" +** or L columnname is left unspecified or is an empty string, then no +** filtering of the vocabulary by language is performed. +** +** For efficiency, it is essential that the vocabulary table be indexed: +** +** CREATE vocab_index ON V(W) +** +** A separate edit-cost-table provides scoring information that defines +** what it means for one string to be "close" to another. +** +** The edit-cost-table must contain exactly four columns (more precisely, +** the statement "SELECT * FROM " must return records +** that consist of four columns). It does not matter what the columns are +** named. +** +** Each row in the edit-cost-table represents a single character +** transformation going from user input to the vocabulary. The leftmost +** column of the row (column 0) contains an integer identifier of the +** language to which the transformation rule belongs (see "MULTIPLE LANGUAGES" +** below). The second column of the row (column 1) contains the input +** character or characters - the characters of user input. The third +** column contains characters as they appear in the vocabulary table. +** And the fourth column contains the integer cost of making the +** transformation. For example: +** +** CREATE TABLE f_data(iLang, cFrom, cTo, Cost); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, '', 'a', 100); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, 'b', '', 87); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, 'o', 'oe', 38); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, 'oe', 'o', 40); +** +** The first row inserted into the edit-cost-table by the SQL script +** above indicates that the cost of having an extra 'a' in the vocabulary +** table that is missing in the user input 100. (All costs are integers. +** Overall cost must not exceed 16777216.) The second INSERT statement +** creates a rule saying that the cost of having a single letter 'b' in +** user input which is missing in the vocabulary table is 87. The third +** INSERT statement mean that the cost of matching an 'o' in user input +** against an 'oe' in the vocabulary table is 38. And so forth. +** +** The following rules are special: +** +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, '?', '', 97); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, '', '?', 98); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, '?', '?', 99); +** +** The '?' to '' rule is the cost of having any single character in the input +** that is not found in the vocabular. The '' to '?' rule is the cost of +** having a character in the vocabulary table that is missing from input. +** And the '?' to '?' rule is the cost of doing an arbitrary character +** substitution. These three generic rules apply across all languages. +** In other words, the iLang field is ignored for the generic substitution +** rules. If more than one cost is given for a generic substitution rule, +** then the lowest cost is used. +** +** Once it has been created, the amatch virtual table can be queried +** as follows: +** +** SELECT word, distance FROM f +** WHERE word MATCH 'abcdefg' +** AND distance<200; +** +** This query outputs the strings contained in the T(F) field that +** are close to "abcdefg" and in order of increasing distance. No string +** is output more than once. If there are multiple ways to transform the +** target string ("abcdefg") into a string in the vocabulary table then +** the lowest cost transform is the one that is returned. In this example, +** the search is limited to strings with a total distance of less than 200. +** +** For efficiency, it is important to put tight bounds on the distance. +** The time and memory space needed to perform this query is exponential +** in the maximum distance. A good rule of thumb is to limit the distance +** to no more than 1.5 or 2 times the maximum cost of any rule in the +** edit-cost-table. +** +** The amatch is a read-only table. Any attempt to DELETE, INSERT, or +** UPDATE on a amatch table will throw an error. +** +** It is important to put some kind of a limit on the amatch output. This +** can be either in the form of a LIMIT clause at the end of the query, +** or better, a "distance +#include +#include +#include +#include + +/* +** Forward declaration of objects used by this implementation +*/ +typedef struct amatch_vtab amatch_vtab; +typedef struct amatch_cursor amatch_cursor; +typedef struct amatch_rule amatch_rule; +typedef struct amatch_word amatch_word; +typedef struct amatch_avl amatch_avl; + + +/***************************************************************************** +** AVL Tree implementation +*/ +/* +** Objects that want to be members of the AVL tree should embedded an +** instance of this structure. +*/ +struct amatch_avl { + amatch_word *pWord; /* Points to the object being stored in the tree */ + char *zKey; /* Key. zero-terminated string. Must be unique */ + amatch_avl *pBefore; /* Other elements less than zKey */ + amatch_avl *pAfter; /* Other elements greater than zKey */ + amatch_avl *pUp; /* Parent element */ + short int height; /* Height of this node. Leaf==1 */ + short int imbalance; /* Height difference between pBefore and pAfter */ +}; + +/* Recompute the amatch_avl.height and amatch_avl.imbalance fields for p. +** Assume that the children of p have correct heights. +*/ +static void amatchAvlRecomputeHeight(amatch_avl *p){ + short int hBefore = p->pBefore ? p->pBefore->height : 0; + short int hAfter = p->pAfter ? p->pAfter->height : 0; + p->imbalance = hBefore - hAfter; /* -: pAfter higher. +: pBefore higher */ + p->height = (hBefore>hAfter ? hBefore : hAfter)+1; +} + +/* +** P B +** / \ / \ +** B Z ==> X P +** / \ / \ +** X Y Y Z +** +*/ +static amatch_avl *amatchAvlRotateBefore(amatch_avl *pP){ + amatch_avl *pB = pP->pBefore; + amatch_avl *pY = pB->pAfter; + pB->pUp = pP->pUp; + pB->pAfter = pP; + pP->pUp = pB; + pP->pBefore = pY; + if( pY ) pY->pUp = pP; + amatchAvlRecomputeHeight(pP); + amatchAvlRecomputeHeight(pB); + return pB; +} + +/* +** P A +** / \ / \ +** X A ==> P Z +** / \ / \ +** Y Z X Y +** +*/ +static amatch_avl *amatchAvlRotateAfter(amatch_avl *pP){ + amatch_avl *pA = pP->pAfter; + amatch_avl *pY = pA->pBefore; + pA->pUp = pP->pUp; + pA->pBefore = pP; + pP->pUp = pA; + pP->pAfter = pY; + if( pY ) pY->pUp = pP; + amatchAvlRecomputeHeight(pP); + amatchAvlRecomputeHeight(pA); + return pA; +} + +/* +** Return a pointer to the pBefore or pAfter pointer in the parent +** of p that points to p. Or if p is the root node, return pp. +*/ +static amatch_avl **amatchAvlFromPtr(amatch_avl *p, amatch_avl **pp){ + amatch_avl *pUp = p->pUp; + if( pUp==0 ) return pp; + if( pUp->pAfter==p ) return &pUp->pAfter; + return &pUp->pBefore; +} + +/* +** Rebalance all nodes starting with p and working up to the root. +** Return the new root. +*/ +static amatch_avl *amatchAvlBalance(amatch_avl *p){ + amatch_avl *pTop = p; + amatch_avl **pp; + while( p ){ + amatchAvlRecomputeHeight(p); + if( p->imbalance>=2 ){ + amatch_avl *pB = p->pBefore; + if( pB->imbalance<0 ) p->pBefore = amatchAvlRotateAfter(pB); + pp = amatchAvlFromPtr(p,&p); + p = *pp = amatchAvlRotateBefore(p); + }else if( p->imbalance<=(-2) ){ + amatch_avl *pA = p->pAfter; + if( pA->imbalance>0 ) p->pAfter = amatchAvlRotateBefore(pA); + pp = amatchAvlFromPtr(p,&p); + p = *pp = amatchAvlRotateAfter(p); + } + pTop = p; + p = p->pUp; + } + return pTop; +} + +/* Search the tree rooted at p for an entry with zKey. Return a pointer +** to the entry or return NULL. +*/ +static amatch_avl *amatchAvlSearch(amatch_avl *p, const char *zKey){ + int c; + while( p && (c = strcmp(zKey, p->zKey))!=0 ){ + p = (c<0) ? p->pBefore : p->pAfter; + } + return p; +} + +/* Find the first node (the one with the smallest key). +*/ +static amatch_avl *amatchAvlFirst(amatch_avl *p){ + if( p ) while( p->pBefore ) p = p->pBefore; + return p; +} + +#if 0 /* NOT USED */ +/* Return the node with the next larger key after p. +*/ +static amatch_avl *amatchAvlNext(amatch_avl *p){ + amatch_avl *pPrev = 0; + while( p && p->pAfter==pPrev ){ + pPrev = p; + p = p->pUp; + } + if( p && pPrev==0 ){ + p = amatchAvlFirst(p->pAfter); + } + return p; +} +#endif + +#if 0 /* NOT USED */ +/* Verify AVL tree integrity +*/ +static int amatchAvlIntegrity(amatch_avl *pHead){ + amatch_avl *p; + if( pHead==0 ) return 1; + if( (p = pHead->pBefore)!=0 ){ + assert( p->pUp==pHead ); + assert( amatchAvlIntegrity(p) ); + assert( strcmp(p->zKey, pHead->zKey)<0 ); + while( p->pAfter ) p = p->pAfter; + assert( strcmp(p->zKey, pHead->zKey)<0 ); + } + if( (p = pHead->pAfter)!=0 ){ + assert( p->pUp==pHead ); + assert( amatchAvlIntegrity(p) ); + assert( strcmp(p->zKey, pHead->zKey)>0 ); + p = amatchAvlFirst(p); + assert( strcmp(p->zKey, pHead->zKey)>0 ); + } + return 1; +} +static int amatchAvlIntegrity2(amatch_avl *pHead){ + amatch_avl *p, *pNext; + for(p=amatchAvlFirst(pHead); p; p=pNext){ + pNext = amatchAvlNext(p); + if( pNext==0 ) break; + assert( strcmp(p->zKey, pNext->zKey)<0 ); + } + return 1; +} +#endif + +/* Insert a new node pNew. Return NULL on success. If the key is not +** unique, then do not perform the insert but instead leave pNew unchanged +** and return a pointer to an existing node with the same key. +*/ +static amatch_avl *amatchAvlInsert(amatch_avl **ppHead, amatch_avl *pNew){ + int c; + amatch_avl *p = *ppHead; + if( p==0 ){ + p = pNew; + pNew->pUp = 0; + }else{ + while( p ){ + c = strcmp(pNew->zKey, p->zKey); + if( c<0 ){ + if( p->pBefore ){ + p = p->pBefore; + }else{ + p->pBefore = pNew; + pNew->pUp = p; + break; + } + }else if( c>0 ){ + if( p->pAfter ){ + p = p->pAfter; + }else{ + p->pAfter = pNew; + pNew->pUp = p; + break; + } + }else{ + return p; + } + } + } + pNew->pBefore = 0; + pNew->pAfter = 0; + pNew->height = 1; + pNew->imbalance = 0; + *ppHead = amatchAvlBalance(p); + /* assert( amatchAvlIntegrity(*ppHead) ); */ + /* assert( amatchAvlIntegrity2(*ppHead) ); */ + return 0; +} + +/* Remove node pOld from the tree. pOld must be an element of the tree or +** the AVL tree will become corrupt. +*/ +static void amatchAvlRemove(amatch_avl **ppHead, amatch_avl *pOld){ + amatch_avl **ppParent; + amatch_avl *pBalance; + /* assert( amatchAvlSearch(*ppHead, pOld->zKey)==pOld ); */ + ppParent = amatchAvlFromPtr(pOld, ppHead); + if( pOld->pBefore==0 && pOld->pAfter==0 ){ + *ppParent = 0; + pBalance = pOld->pUp; + }else if( pOld->pBefore && pOld->pAfter ){ + amatch_avl *pX, *pY; + pX = amatchAvlFirst(pOld->pAfter); + *amatchAvlFromPtr(pX, 0) = pX->pAfter; + if( pX->pAfter ) pX->pAfter->pUp = pX->pUp; + pBalance = pX->pUp; + pX->pAfter = pOld->pAfter; + if( pX->pAfter ){ + pX->pAfter->pUp = pX; + }else{ + assert( pBalance==pOld ); + pBalance = pX; + } + pX->pBefore = pY = pOld->pBefore; + if( pY ) pY->pUp = pX; + pX->pUp = pOld->pUp; + *ppParent = pX; + }else if( pOld->pBefore==0 ){ + *ppParent = pBalance = pOld->pAfter; + pBalance->pUp = pOld->pUp; + }else if( pOld->pAfter==0 ){ + *ppParent = pBalance = pOld->pBefore; + pBalance->pUp = pOld->pUp; + } + *ppHead = amatchAvlBalance(pBalance); + pOld->pUp = 0; + pOld->pBefore = 0; + pOld->pAfter = 0; + /* assert( amatchAvlIntegrity(*ppHead) ); */ + /* assert( amatchAvlIntegrity2(*ppHead) ); */ +} +/* +** End of the AVL Tree implementation +******************************************************************************/ + + +/* +** Various types. +** +** amatch_cost is the "cost" of an edit operation. +** +** amatch_len is the length of a matching string. +** +** amatch_langid is an ruleset identifier. +*/ +typedef int amatch_cost; +typedef signed char amatch_len; +typedef int amatch_langid; + +/* +** Limits +*/ +#define AMATCH_MX_LENGTH 50 /* Maximum length of a rule string */ +#define AMATCH_MX_LANGID 2147483647 /* Maximum rule ID */ +#define AMATCH_MX_COST 1000 /* Maximum single-rule cost */ + +/* +** A match or partial match +*/ +struct amatch_word { + amatch_word *pNext; /* Next on a list of all amatch_words */ + amatch_avl sCost; /* Linkage of this node into the cost tree */ + amatch_avl sWord; /* Linkage of this node into the word tree */ + amatch_cost rCost; /* Cost of the match so far */ + int iSeq; /* Sequence number */ + char zCost[10]; /* Cost key (text rendering of rCost) */ + short int nMatch; /* Input characters matched */ + char zWord[4]; /* Text of the word. Extra space appended as needed */ +}; + +/* +** Each transformation rule is stored as an instance of this object. +** All rules are kept on a linked list sorted by rCost. +*/ +struct amatch_rule { + amatch_rule *pNext; /* Next rule in order of increasing rCost */ + char *zFrom; /* Transform from (a string from user input) */ + amatch_cost rCost; /* Cost of this transformation */ + amatch_langid iLang; /* The langauge to which this rule belongs */ + amatch_len nFrom, nTo; /* Length of the zFrom and zTo strings */ + char zTo[4]; /* Tranform to V.W value (extra space appended) */ +}; + +/* +** A amatch virtual-table object +*/ +struct amatch_vtab { + sqlite3_vtab base; /* Base class - must be first */ + char *zClassName; /* Name of this class. Default: "amatch" */ + char *zDb; /* Name of database. (ex: "main") */ + char *zSelf; /* Name of this virtual table */ + char *zCostTab; /* Name of edit-cost-table */ + char *zVocabTab; /* Name of vocabulary table */ + char *zVocabWord; /* Name of vocabulary table word column */ + char *zVocabLang; /* Name of vocabulary table language column */ + amatch_rule *pRule; /* All active rules in this amatch */ + amatch_cost rIns; /* Generic insertion cost '' -> ? */ + amatch_cost rDel; /* Generic deletion cost ? -> '' */ + amatch_cost rSub; /* Generic substitution cost ? -> ? */ + sqlite3 *db; /* The database connection */ + sqlite3_stmt *pVCheck; /* Query to check zVocabTab */ + int nCursor; /* Number of active cursors */ +}; + +/* A amatch cursor object */ +struct amatch_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_int64 iRowid; /* The rowid of the current word */ + amatch_langid iLang; /* Use this language ID */ + amatch_cost rLimit; /* Maximum cost of any term */ + int nBuf; /* Space allocated for zBuf */ + int oomErr; /* True following an OOM error */ + int nWord; /* Number of amatch_word objects */ + char *zBuf; /* Temp-use buffer space */ + char *zInput; /* Input word to match against */ + amatch_vtab *pVtab; /* The virtual table this cursor belongs to */ + amatch_word *pAllWords; /* List of all amatch_word objects */ + amatch_word *pCurrent; /* Most recent solution */ + amatch_avl *pCost; /* amatch_word objects keyed by iCost */ + amatch_avl *pWord; /* amatch_word objects keyed by zWord */ +}; + +/* +** The two input rule lists are both sorted in order of increasing +** cost. Merge them together into a single list, sorted by cost, and +** return a pointer to the head of that list. +*/ +static amatch_rule *amatchMergeRules(amatch_rule *pA, amatch_rule *pB){ + amatch_rule head; + amatch_rule *pTail; + + pTail = &head; + while( pA && pB ){ + if( pA->rCost<=pB->rCost ){ + pTail->pNext = pA; + pTail = pA; + pA = pA->pNext; + }else{ + pTail->pNext = pB; + pTail = pB; + pB = pB->pNext; + } + } + if( pA==0 ){ + pTail->pNext = pB; + }else{ + pTail->pNext = pA; + } + return head.pNext; +} + +/* +** Statement pStmt currently points to a row in the amatch data table. This +** function allocates and populates a amatch_rule structure according to +** the content of the row. +** +** If successful, *ppRule is set to point to the new object and SQLITE_OK +** is returned. Otherwise, *ppRule is zeroed, *pzErr may be set to point +** to an error message and an SQLite error code returned. +*/ +static int amatchLoadOneRule( + amatch_vtab *p, /* Fuzzer virtual table handle */ + sqlite3_stmt *pStmt, /* Base rule on statements current row */ + amatch_rule **ppRule, /* OUT: New rule object */ + char **pzErr /* OUT: Error message */ +){ + sqlite3_int64 iLang = sqlite3_column_int64(pStmt, 0); + const char *zFrom = (const char *)sqlite3_column_text(pStmt, 1); + const char *zTo = (const char *)sqlite3_column_text(pStmt, 2); + amatch_cost rCost = sqlite3_column_int(pStmt, 3); + + int rc = SQLITE_OK; /* Return code */ + int nFrom; /* Size of string zFrom, in bytes */ + int nTo; /* Size of string zTo, in bytes */ + amatch_rule *pRule = 0; /* New rule object to return */ + + if( zFrom==0 ) zFrom = ""; + if( zTo==0 ) zTo = ""; + nFrom = (int)strlen(zFrom); + nTo = (int)strlen(zTo); + + /* Silently ignore null transformations */ + if( strcmp(zFrom, zTo)==0 ){ + if( zFrom[0]=='?' && zFrom[1]==0 ){ + if( p->rSub==0 || p->rSub>rCost ) p->rSub = rCost; + } + *ppRule = 0; + return SQLITE_OK; + } + + if( rCost<=0 || rCost>AMATCH_MX_COST ){ + *pzErr = sqlite3_mprintf("%s: cost must be between 1 and %d", + p->zClassName, AMATCH_MX_COST + ); + rc = SQLITE_ERROR; + }else + if( nFrom>AMATCH_MX_LENGTH || nTo>AMATCH_MX_LENGTH ){ + *pzErr = sqlite3_mprintf("%s: maximum string length is %d", + p->zClassName, AMATCH_MX_LENGTH + ); + rc = SQLITE_ERROR; + }else + if( iLang<0 || iLang>AMATCH_MX_LANGID ){ + *pzErr = sqlite3_mprintf("%s: iLang must be between 0 and %d", + p->zClassName, AMATCH_MX_LANGID + ); + rc = SQLITE_ERROR; + }else + if( strcmp(zFrom,"")==0 && strcmp(zTo,"?")==0 ){ + if( p->rIns==0 || p->rIns>rCost ) p->rIns = rCost; + }else + if( strcmp(zFrom,"?")==0 && strcmp(zTo,"")==0 ){ + if( p->rDel==0 || p->rDel>rCost ) p->rDel = rCost; + }else + { + pRule = sqlite3_malloc( sizeof(*pRule) + nFrom + nTo ); + if( pRule==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pRule, 0, sizeof(*pRule)); + pRule->zFrom = &pRule->zTo[nTo+1]; + pRule->nFrom = nFrom; + memcpy(pRule->zFrom, zFrom, nFrom+1); + memcpy(pRule->zTo, zTo, nTo+1); + pRule->nTo = nTo; + pRule->rCost = rCost; + pRule->iLang = (int)iLang; + } + } + + *ppRule = pRule; + return rc; +} + +/* +** Free all the content in the edit-cost-table +*/ +static void amatchFreeRules(amatch_vtab *p){ + while( p->pRule ){ + amatch_rule *pRule = p->pRule; + p->pRule = pRule->pNext; + sqlite3_free(pRule); + } + p->pRule = 0; +} + +/* +** Load the content of the amatch data table into memory. +*/ +static int amatchLoadRules( + sqlite3 *db, /* Database handle */ + amatch_vtab *p, /* Virtual amatch table to configure */ + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE_OK; /* Return code */ + char *zSql; /* SELECT used to read from rules table */ + amatch_rule *pHead = 0; + + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", p->zDb, p->zCostTab); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + int rc2; /* finalize() return code */ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("%s: %s", p->zClassName, sqlite3_errmsg(db)); + }else if( sqlite3_column_count(pStmt)!=4 ){ + *pzErr = sqlite3_mprintf("%s: %s has %d columns, expected 4", + p->zClassName, p->zCostTab, sqlite3_column_count(pStmt) + ); + rc = SQLITE_ERROR; + }else{ + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + amatch_rule *pRule = 0; + rc = amatchLoadOneRule(p, pStmt, &pRule, pzErr); + if( pRule ){ + pRule->pNext = pHead; + pHead = pRule; + } + } + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + sqlite3_free(zSql); + + /* All rules are now in a singly linked list starting at pHead. This + ** block sorts them by cost and then sets amatch_vtab.pRule to point to + ** point to the head of the sorted list. + */ + if( rc==SQLITE_OK ){ + unsigned int i; + amatch_rule *pX; + amatch_rule *a[15]; + for(i=0; ipNext; + pX->pNext = 0; + for(i=0; a[i] && ipRule = amatchMergeRules(p->pRule, pX); + }else{ + /* An error has occurred. Setting p->pRule to point to the head of the + ** allocated list ensures that the list will be cleaned up in this case. + */ + assert( p->pRule==0 ); + p->pRule = pHead; + } + + return rc; +} + +/* +** This function converts an SQL quoted string into an unquoted string +** and returns a pointer to a buffer allocated using sqlite3_malloc() +** containing the result. The caller should eventually free this buffer +** using sqlite3_free. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +*/ +static char *amatchDequote(const char *zIn){ + int nIn; /* Size of input string, in bytes */ + char *zOut; /* Output (dequoted) string */ + + nIn = (int)strlen(zIn); + zOut = sqlite3_malloc(nIn+1); + if( zOut ){ + char q = zIn[0]; /* Quote character (if any ) */ + + if( q!='[' && q!= '\'' && q!='"' && q!='`' ){ + memcpy(zOut, zIn, nIn+1); + }else{ + int iOut = 0; /* Index of next byte to write to output */ + int iIn; /* Index of next byte to read from input */ + + if( q=='[' ) q = ']'; + for(iIn=1; iInpVCheck ){ + sqlite3_finalize(p->pVCheck); + p->pVCheck = 0; + } +} + +/* +** Deallocate an amatch_vtab object +*/ +static void amatchFree(amatch_vtab *p){ + if( p ){ + amatchFreeRules(p); + amatchVCheckClear(p); + sqlite3_free(p->zClassName); + sqlite3_free(p->zDb); + sqlite3_free(p->zCostTab); + sqlite3_free(p->zVocabTab); + sqlite3_free(p->zVocabWord); + sqlite3_free(p->zVocabLang); + memset(p, 0, sizeof(*p)); + sqlite3_free(p); + } +} + +/* +** xDisconnect/xDestroy method for the amatch module. +*/ +static int amatchDisconnect(sqlite3_vtab *pVtab){ + amatch_vtab *p = (amatch_vtab*)pVtab; + assert( p->nCursor==0 ); + amatchFree(p); + return SQLITE_OK; +} + +/* +** Check to see if the argument is of the form: +** +** KEY = VALUE +** +** If it is, return a pointer to the first character of VALUE. +** If not, return NULL. Spaces around the = are ignored. +*/ +static const char *amatchValueOfKey(const char *zKey, const char *zStr){ + int nKey = (int)strlen(zKey); + int nStr = (int)strlen(zStr); + int i; + if( nStr module name ("approximate_match") +** argv[1] -> database name +** argv[2] -> table name +** argv[3...] -> arguments +*/ +static int amatchConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + int rc = SQLITE_OK; /* Return code */ + amatch_vtab *pNew = 0; /* New virtual table */ + const char *zModule = argv[0]; + const char *zDb = argv[1]; + const char *zVal; + int i; + + (void)pAux; + *ppVtab = 0; + pNew = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + rc = SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + pNew->zClassName = sqlite3_mprintf("%s", zModule); + if( pNew->zClassName==0 ) goto amatchConnectError; + pNew->zDb = sqlite3_mprintf("%s", zDb); + if( pNew->zDb==0 ) goto amatchConnectError; + pNew->zSelf = sqlite3_mprintf("%s", argv[2]); + if( pNew->zSelf==0 ) goto amatchConnectError; + for(i=3; izVocabTab); + pNew->zVocabTab = amatchDequote(zVal); + if( pNew->zVocabTab==0 ) goto amatchConnectError; + continue; + } + zVal = amatchValueOfKey("vocabulary_word", argv[i]); + if( zVal ){ + sqlite3_free(pNew->zVocabWord); + pNew->zVocabWord = amatchDequote(zVal); + if( pNew->zVocabWord==0 ) goto amatchConnectError; + continue; + } + zVal = amatchValueOfKey("vocabulary_language", argv[i]); + if( zVal ){ + sqlite3_free(pNew->zVocabLang); + pNew->zVocabLang = amatchDequote(zVal); + if( pNew->zVocabLang==0 ) goto amatchConnectError; + continue; + } + zVal = amatchValueOfKey("edit_distances", argv[i]); + if( zVal ){ + sqlite3_free(pNew->zCostTab); + pNew->zCostTab = amatchDequote(zVal); + if( pNew->zCostTab==0 ) goto amatchConnectError; + continue; + } + *pzErr = sqlite3_mprintf("unrecognized argument: [%s]\n", argv[i]); + amatchFree(pNew); + *ppVtab = 0; + return SQLITE_ERROR; + } + rc = SQLITE_OK; + if( pNew->zCostTab==0 ){ + *pzErr = sqlite3_mprintf("no edit_distances table specified"); + rc = SQLITE_ERROR; + }else{ + rc = amatchLoadRules(db, pNew, pzErr); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(word,distance,language," + "command HIDDEN,nword HIDDEN)" + ); +#define AMATCH_COL_WORD 0 +#define AMATCH_COL_DISTANCE 1 +#define AMATCH_COL_LANGUAGE 2 +#define AMATCH_COL_COMMAND 3 +#define AMATCH_COL_NWORD 4 + } + if( rc!=SQLITE_OK ){ + amatchFree(pNew); + } + *ppVtab = &pNew->base; + return rc; + +amatchConnectError: + amatchFree(pNew); + return rc; +} + +/* +** Open a new amatch cursor. +*/ +static int amatchOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + amatch_vtab *p = (amatch_vtab*)pVTab; + amatch_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->pVtab = p; + *ppCursor = &pCur->base; + p->nCursor++; + return SQLITE_OK; +} + +/* +** Free up all the memory allocated by a cursor. Set it rLimit to 0 +** to indicate that it is at EOF. +*/ +static void amatchClearCursor(amatch_cursor *pCur){ + amatch_word *pWord, *pNextWord; + for(pWord=pCur->pAllWords; pWord; pWord=pNextWord){ + pNextWord = pWord->pNext; + sqlite3_free(pWord); + } + pCur->pAllWords = 0; + sqlite3_free(pCur->zInput); + pCur->zInput = 0; + pCur->pCost = 0; + pCur->pWord = 0; + pCur->pCurrent = 0; + pCur->rLimit = 1000000; + pCur->iLang = 0; + pCur->nWord = 0; +} + +/* +** Close a amatch cursor. +*/ +static int amatchClose(sqlite3_vtab_cursor *cur){ + amatch_cursor *pCur = (amatch_cursor *)cur; + amatchClearCursor(pCur); + pCur->pVtab->nCursor--; + sqlite3_free(pCur); + return SQLITE_OK; +} + +/* +** Render a 24-bit unsigned integer as a 4-byte base-64 number. +*/ +static void amatchEncodeInt(int x, char *z){ + static const char a[] = + "0123456789" + "ABCDEFGHIJ" + "KLMNOPQRST" + "UVWXYZ^abc" + "defghijklm" + "nopqrstuvw" + "xyz~"; + z[0] = a[(x>>18)&0x3f]; + z[1] = a[(x>>12)&0x3f]; + z[2] = a[(x>>6)&0x3f]; + z[3] = a[x&0x3f]; +} + +/* +** Write the zCost[] field for a amatch_word object +*/ +static void amatchWriteCost(amatch_word *pWord){ + amatchEncodeInt(pWord->rCost, pWord->zCost); + amatchEncodeInt(pWord->iSeq, pWord->zCost+4); + pWord->zCost[8] = 0; +} + +/* +** Add a new amatch_word object to the queue. +** +** If a prior amatch_word object with the same zWord, and nMatch +** already exists, update its rCost (if the new rCost is less) but +** otherwise leave it unchanged. Do not add a duplicate. +** +** Do nothing if the cost exceeds threshold. +*/ +static void amatchAddWord( + amatch_cursor *pCur, + amatch_cost rCost, + int nMatch, + const char *zWordBase, + const char *zWordTail +){ + amatch_word *pWord; + amatch_avl *pNode; + amatch_avl *pOther; + int nBase, nTail; + char zBuf[4]; + + if( rCost>pCur->rLimit ){ + return; + } + nBase = (int)strlen(zWordBase); + nTail = (int)strlen(zWordTail); + if( nBase+nTail+3>pCur->nBuf ){ + pCur->nBuf = nBase+nTail+100; + pCur->zBuf = sqlite3_realloc(pCur->zBuf, pCur->nBuf); + if( pCur->zBuf==0 ){ + pCur->nBuf = 0; + return; + } + } + amatchEncodeInt(nMatch, zBuf); + memcpy(pCur->zBuf, zBuf+2, 2); + memcpy(pCur->zBuf+2, zWordBase, nBase); + memcpy(pCur->zBuf+2+nBase, zWordTail, nTail+1); + pNode = amatchAvlSearch(pCur->pWord, pCur->zBuf); + if( pNode ){ + pWord = pNode->pWord; + if( pWord->rCost>rCost ){ +#ifdef AMATCH_TRACE_1 + printf("UPDATE [%s][%.*s^%s] %d (\"%s\" \"%s\")\n", + pWord->zWord+2, pWord->nMatch, pCur->zInput, pCur->zInput, + pWord->rCost, pWord->zWord, pWord->zCost); +#endif + amatchAvlRemove(&pCur->pCost, &pWord->sCost); + pWord->rCost = rCost; + amatchWriteCost(pWord); +#ifdef AMATCH_TRACE_1 + printf(" ---> %d (\"%s\" \"%s\")\n", + pWord->rCost, pWord->zWord, pWord->zCost); +#endif + pOther = amatchAvlInsert(&pCur->pCost, &pWord->sCost); + assert( pOther==0 ); (void)pOther; + } + return; + } + pWord = sqlite3_malloc( sizeof(*pWord) + nBase + nTail - 1 ); + if( pWord==0 ) return; + memset(pWord, 0, sizeof(*pWord)); + pWord->rCost = rCost; + pWord->iSeq = pCur->nWord++; + amatchWriteCost(pWord); + pWord->nMatch = nMatch; + pWord->pNext = pCur->pAllWords; + pCur->pAllWords = pWord; + pWord->sCost.zKey = pWord->zCost; + pWord->sCost.pWord = pWord; + pOther = amatchAvlInsert(&pCur->pCost, &pWord->sCost); + assert( pOther==0 ); (void)pOther; + pWord->sWord.zKey = pWord->zWord; + pWord->sWord.pWord = pWord; + strcpy(pWord->zWord, pCur->zBuf); + pOther = amatchAvlInsert(&pCur->pWord, &pWord->sWord); + assert( pOther==0 ); (void)pOther; +#ifdef AMATCH_TRACE_1 + printf("INSERT [%s][%.*s^%s] %d (\"%s\" \"%s\")\n", pWord->zWord+2, + pWord->nMatch, pCur->zInput, pCur->zInput+pWord->nMatch, rCost, + pWord->zWord, pWord->zCost); +#endif +} + +/* +** Advance a cursor to its next row of output +*/ +static int amatchNext(sqlite3_vtab_cursor *cur){ + amatch_cursor *pCur = (amatch_cursor*)cur; + amatch_word *pWord = 0; + amatch_avl *pNode; + int isMatch = 0; + amatch_vtab *p = pCur->pVtab; + int nWord; + int rc; + int i; + const char *zW; + amatch_rule *pRule; + char *zBuf = 0; + char nBuf = 0; + char zNext[8]; + char zNextIn[8]; + int nNextIn; + + if( p->pVCheck==0 ){ + char *zSql; + if( p->zVocabLang && p->zVocabLang[0] ){ + zSql = sqlite3_mprintf( + "SELECT \"%s\" FROM \"%s\"", + " WHERE \"%w\">=?1 AND \"%w\"=?2" + " ORDER BY 1", + p->zVocabWord, p->zVocabTab, + p->zVocabWord, p->zVocabLang + ); + }else{ + zSql = sqlite3_mprintf( + "SELECT \"%s\" FROM \"%s\"" + " WHERE \"%w\">=?1" + " ORDER BY 1", + p->zVocabWord, p->zVocabTab, + p->zVocabWord + ); + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &p->pVCheck, 0); + sqlite3_free(zSql); + if( rc ) return rc; + } + sqlite3_bind_int(p->pVCheck, 2, pCur->iLang); + + do{ + pNode = amatchAvlFirst(pCur->pCost); + if( pNode==0 ){ + pWord = 0; + break; + } + pWord = pNode->pWord; + amatchAvlRemove(&pCur->pCost, &pWord->sCost); + +#ifdef AMATCH_TRACE_1 + printf("PROCESS [%s][%.*s^%s] %d (\"%s\" \"%s\")\n", + pWord->zWord+2, pWord->nMatch, pCur->zInput, pCur->zInput+pWord->nMatch, + pWord->rCost, pWord->zWord, pWord->zCost); +#endif + nWord = (int)strlen(pWord->zWord+2); + if( nWord+20>nBuf ){ + nBuf = nWord+100; + zBuf = sqlite3_realloc(zBuf, nBuf); + if( zBuf==0 ) return SQLITE_NOMEM; + } + strcpy(zBuf, pWord->zWord+2); + zNext[0] = 0; + zNextIn[0] = pCur->zInput[pWord->nMatch]; + if( zNextIn[0] ){ + for(i=1; i<=4 && (pCur->zInput[pWord->nMatch+i]&0xc0)==0x80; i++){ + zNextIn[i] = pCur->zInput[pWord->nMatch+i]; + } + zNextIn[i] = 0; + nNextIn = i; + }else{ + nNextIn = 0; + } + + if( zNextIn[0] && zNextIn[0]!='*' ){ + sqlite3_reset(p->pVCheck); + strcat(zBuf, zNextIn); + sqlite3_bind_text(p->pVCheck, 1, zBuf, nWord+nNextIn, SQLITE_STATIC); + rc = sqlite3_step(p->pVCheck); + if( rc==SQLITE_ROW ){ + zW = (const char*)sqlite3_column_text(p->pVCheck, 0); + if( strncmp(zBuf, zW, nWord+nNextIn)==0 ){ + amatchAddWord(pCur, pWord->rCost, pWord->nMatch+nNextIn, zBuf, ""); + } + } + zBuf[nWord] = 0; + } + + while( 1 ){ + strcpy(zBuf+nWord, zNext); + sqlite3_reset(p->pVCheck); + sqlite3_bind_text(p->pVCheck, 1, zBuf, -1, SQLITE_TRANSIENT); + rc = sqlite3_step(p->pVCheck); + if( rc!=SQLITE_ROW ) break; + zW = (const char*)sqlite3_column_text(p->pVCheck, 0); + strcpy(zBuf+nWord, zNext); + if( strncmp(zW, zBuf, nWord)!=0 ) break; + if( (zNextIn[0]=='*' && zNextIn[1]==0) + || (zNextIn[0]==0 && zW[nWord]==0) + ){ + isMatch = 1; + zNextIn[0] = 0; + nNextIn = 0; + break; + } + zNext[0] = zW[nWord]; + for(i=1; i<=4 && (zW[nWord+i]&0xc0)==0x80; i++){ + zNext[i] = zW[nWord+i]; + } + zNext[i] = 0; + zBuf[nWord] = 0; + if( p->rIns>0 ){ + amatchAddWord(pCur, pWord->rCost+p->rIns, pWord->nMatch, + zBuf, zNext); + } + if( p->rSub>0 ){ + amatchAddWord(pCur, pWord->rCost+p->rSub, pWord->nMatch+nNextIn, + zBuf, zNext); + } + if( p->rIns<0 && p->rSub<0 ) break; + zNext[i-1]++; /* FIX ME */ + } + sqlite3_reset(p->pVCheck); + + if( p->rDel>0 ){ + zBuf[nWord] = 0; + amatchAddWord(pCur, pWord->rCost+p->rDel, pWord->nMatch+nNextIn, + zBuf, ""); + } + + for(pRule=p->pRule; pRule; pRule=pRule->pNext){ + if( pRule->iLang!=pCur->iLang ) continue; + if( strncmp(pRule->zFrom, pCur->zInput+pWord->nMatch, pRule->nFrom)==0 ){ + amatchAddWord(pCur, pWord->rCost+pRule->rCost, + pWord->nMatch+pRule->nFrom, pWord->zWord+2, pRule->zTo); + } + } + }while( !isMatch ); + pCur->pCurrent = pWord; + sqlite3_free(zBuf); + return SQLITE_OK; +} + +/* +** Called to "rewind" a cursor back to the beginning so that +** it starts its output over again. Always called at least once +** prior to any amatchColumn, amatchRowid, or amatchEof call. +*/ +static int amatchFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + amatch_cursor *pCur = (amatch_cursor *)pVtabCursor; + const char *zWord = "*"; + int idx; + + amatchClearCursor(pCur); + idx = 0; + if( idxNum & 1 ){ + zWord = (const char*)sqlite3_value_text(argv[0]); + idx++; + } + if( idxNum & 2 ){ + pCur->rLimit = (amatch_cost)sqlite3_value_int(argv[idx]); + idx++; + } + if( idxNum & 4 ){ + pCur->iLang = (amatch_cost)sqlite3_value_int(argv[idx]); + idx++; + } + pCur->zInput = sqlite3_mprintf("%s", zWord); + if( pCur->zInput==0 ) return SQLITE_NOMEM; + amatchAddWord(pCur, 0, 0, "", ""); + amatchNext(pVtabCursor); + + return SQLITE_OK; +} + +/* +** Only the word and distance columns have values. All other columns +** return NULL +*/ +static int amatchColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + amatch_cursor *pCur = (amatch_cursor*)cur; + switch( i ){ + case AMATCH_COL_WORD: { + sqlite3_result_text(ctx, pCur->pCurrent->zWord+2, -1, SQLITE_STATIC); + break; + } + case AMATCH_COL_DISTANCE: { + sqlite3_result_int(ctx, pCur->pCurrent->rCost); + break; + } + case AMATCH_COL_LANGUAGE: { + sqlite3_result_int(ctx, pCur->iLang); + break; + } + case AMATCH_COL_NWORD: { + sqlite3_result_int(ctx, pCur->nWord); + break; + } + default: { + sqlite3_result_null(ctx); + break; + } + } + return SQLITE_OK; +} + +/* +** The rowid. +*/ +static int amatchRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + amatch_cursor *pCur = (amatch_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** EOF indicator +*/ +static int amatchEof(sqlite3_vtab_cursor *cur){ + amatch_cursor *pCur = (amatch_cursor*)cur; + return pCur->pCurrent==0; +} + +/* +** Search for terms of these forms: +** +** (A) word MATCH $str +** (B1) distance < $value +** (B2) distance <= $value +** (C) language == $language +** +** The distance< and distance<= are both treated as distance<=. +** The query plan number is a bit vector: +** +** bit 1: Term of the form (A) found +** bit 2: Term like (B1) or (B2) found +** bit 3: Term like (C) found +** +** If bit-1 is set, $str is always in filter.argv[0]. If bit-2 is set +** then $value is in filter.argv[0] if bit-1 is clear and is in +** filter.argv[1] if bit-1 is set. If bit-3 is set, then $ruleid is +** in filter.argv[0] if bit-1 and bit-2 are both zero, is in +** filter.argv[1] if exactly one of bit-1 and bit-2 are set, and is in +** filter.argv[2] if both bit-1 and bit-2 are set. +*/ +static int amatchBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int iPlan = 0; + int iDistTerm = -1; + int iLangTerm = -1; + int i; + const struct sqlite3_index_constraint *pConstraint; + + (void)tab; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( (iPlan & 1)==0 + && pConstraint->iColumn==0 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH + ){ + iPlan |= 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + } + if( (iPlan & 2)==0 + && pConstraint->iColumn==1 + && (pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT + || pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE) + ){ + iPlan |= 2; + iDistTerm = i; + } + if( (iPlan & 4)==0 + && pConstraint->iColumn==2 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 4; + pIdxInfo->aConstraintUsage[i].omit = 1; + iLangTerm = i; + } + } + if( iPlan & 2 ){ + pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = 1+((iPlan&1)!=0); + } + if( iPlan & 4 ){ + int idx = 1; + if( iPlan & 1 ) idx++; + if( iPlan & 2 ) idx++; + pIdxInfo->aConstraintUsage[iLangTerm].argvIndex = idx; + } + pIdxInfo->idxNum = iPlan; + if( pIdxInfo->nOrderBy==1 + && pIdxInfo->aOrderBy[0].iColumn==1 + && pIdxInfo->aOrderBy[0].desc==0 + ){ + pIdxInfo->orderByConsumed = 1; + } + pIdxInfo->estimatedCost = (double)10000; + + return SQLITE_OK; +} + +/* +** The xUpdate() method. +** +** This implementation disallows DELETE and UPDATE. The only thing +** allowed is INSERT into the "command" column. +*/ +static int amatchUpdate( + sqlite3_vtab *pVTab, + int argc, + sqlite3_value **argv, + sqlite_int64 *pRowid +){ + amatch_vtab *p = (amatch_vtab*)pVTab; + const unsigned char *zCmd; + (void)pRowid; + if( argc==1 ){ + pVTab->zErrMsg = sqlite3_mprintf("DELETE from %s is not allowed", + p->zSelf); + return SQLITE_ERROR; + } + if( sqlite3_value_type(argv[0])!=SQLITE_NULL ){ + pVTab->zErrMsg = sqlite3_mprintf("UPDATE of %s is not allowed", + p->zSelf); + return SQLITE_ERROR; + } + if( sqlite3_value_type(argv[2+AMATCH_COL_WORD])!=SQLITE_NULL + || sqlite3_value_type(argv[2+AMATCH_COL_DISTANCE])!=SQLITE_NULL + || sqlite3_value_type(argv[2+AMATCH_COL_LANGUAGE])!=SQLITE_NULL + ){ + pVTab->zErrMsg = sqlite3_mprintf( + "INSERT INTO %s allowed for column [command] only", p->zSelf); + return SQLITE_ERROR; + } + zCmd = sqlite3_value_text(argv[2+AMATCH_COL_COMMAND]); + if( zCmd==0 ) return SQLITE_OK; + + return SQLITE_OK; +} + +/* +** A virtual table module that implements the "approximate_match". +*/ +static sqlite3_module amatchModule = { + 0, /* iVersion */ + amatchConnect, /* xCreate */ + amatchConnect, /* xConnect */ + amatchBestIndex, /* xBestIndex */ + amatchDisconnect, /* xDisconnect */ + amatchDisconnect, /* xDestroy */ + amatchOpen, /* xOpen - open a cursor */ + amatchClose, /* xClose - close a cursor */ + amatchFilter, /* xFilter - configure scan constraints */ + amatchNext, /* xNext - advance a cursor */ + amatchEof, /* xEof - check for end of scan */ + amatchColumn, /* xColumn - read data */ + amatchRowid, /* xRowid - read data */ + amatchUpdate, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ +}; + +/* +** Register the amatch virtual table +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_amatch_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Not used */ + rc = sqlite3_create_module(db, "approximate_match", &amatchModule, 0); + return rc; +} diff --git a/ext/misc/closure.c b/ext/misc/closure.c new file mode 100644 index 0000000000..7b3d06492e --- /dev/null +++ b/ext/misc/closure.c @@ -0,0 +1,942 @@ +/* +** 2013-04-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code for a virtual table that finds the transitive +** closure of a parent/child relationship in a real table. The virtual +** table is called "transitive_closure". +** +** A transitive_closure virtual table is created like this: +** +** CREATE VIRTUAL TABLE x USING transitive_closure( +** tablename=, -- T +** idcolumn=, -- X +** parentcolumn= -- P +** ); +** +** When it is created, the new transitive_closure table may be supplied +** with default values for the name of a table T and columns T.X and T.P. +** The T.X and T.P columns must contain integers. The ideal case is for +** T.X to be the INTEGER PRIMARY KEY. The T.P column should reference +** the T.X column. The row referenced by T.P is the parent of the current row. +** +** The tablename, idcolumn, and parentcolumn supplied by the CREATE VIRTUAL +** TABLE statement may be overridden in individual queries by including +** terms like tablename='newtable', idcolumn='id2', or +** parentcolumn='parent3' in the WHERE clause of the query. +** +** For efficiency, it is essential that there be an index on the P column: +** +** CREATE Tidx1 ON T(P) +** +** Suppose a specific instance of the closure table is as follows: +** +** CREATE VIRTUAL TABLE ct1 USING transitive_closure( +** tablename='group', +** idcolumn='groupId', +** parentcolumn='parentId' +** ); +** +** Such an instance of the transitive_closure virtual table would be +** appropriate for walking a tree defined using a table like this, for example: +** +** CREATE TABLE group( +** groupId INTEGER PRIMARY KEY, +** parentId INTEGER REFERENCES group +** ); +** CREATE INDEX group_idx1 ON group(parentId); +** +** The group table above would presumably have other application-specific +** fields. The key point here is that rows of the group table form a +** tree. The purpose of the ct1 virtual table is to easily extract +** branches of that tree. +** +** Once it has been created, the ct1 virtual table can be queried +** as follows: +** +** SELECT * FROM element +** WHERE element.groupId IN (SELECT id FROM ct1 WHERE root=?1); +** +** The above query will return all elements that are part of group ?1 +** or children of group ?1 or grand-children of ?1 and so forth for all +** descendents of group ?1. The same query can be formulated as a join: +** +** SELECT element.* FROM element, ct1 +** WHERE element.groupid=ct1.id +** AND ct1.root=?1; +** +** The depth of the transitive_closure (the number of generations of +** parent/child relations to follow) can be limited by setting "depth" +** column in the WHERE clause. So, for example, the following query +** finds only children and grandchildren but no further descendents: +** +** SELECT element.* FROM element, ct1 +** WHERE element.groupid=ct1.id +** AND ct1.root=?1 +** AND ct1.depth<=2; +** +** The "ct1.depth<=2" term could be a strict equality "ct1.depth=2" in +** order to find only the grandchildren of ?1, not ?1 itself or the +** children of ?1. +** +** The root=?1 term must be supplied in WHERE clause or else the query +** of the ct1 virtual table will return an empty set. The tablename, +** idcolumn, and parentcolumn attributes can be overridden in the WHERE +** clause if desired. So, for example, the ct1 table could be repurposed +** to find ancestors rather than descendents by inverting the roles of +** the idcolumn and parentcolumn: +** +** SELECT element.* FROM element, ct1 +** WHERE element.groupid=ct1.id +** AND ct1.root=?1 +** AND ct1.idcolumn='parentId' +** AND ct1.parentcolumn='groupId'; +** +** Multiple calls to ct1 could be combined. For example, the following +** query finds all elements that "cousins" of groupId ?1. That is to say +** elements where the groupId is a grandchild of the grandparent of ?1. +** (This definition of "cousins" also includes siblings and self.) +** +** SELECT element.* FROM element, ct1 +** WHERE element.groupId=ct1.id +** AND ct1.depth=2 +** AND ct1.root IN (SELECT id FROM ct1 +** WHERE root=?1 +** AND depth=2 +** AND idcolumn='parentId' +** AND parentcolumn='groupId'); +** +** In our example, the group.groupId column is unique and thus the +** subquery will return exactly one row. For that reason, the IN +** operator could be replaced by "=" to get the same result. But +** in the general case where the idcolumn is not unique, an IN operator +** would be required for this kind of query. +** +** Note that because the tablename, idcolumn, and parentcolumn can +** all be specified in the query, it is possible for an application +** to define a single transitive_closure virtual table for use on lots +** of different hierarchy tables. One might say: +** +** CREATE VIRTUAL TABLE temp.closure USING transitive_closure; +** +** As each database connection is being opened. Then the application +** would always have a "closure" virtual table handy to use for querying. +** +** SELECT element.* FROM element, closure +** WHERE element.groupid=ct1.id +** AND closure.root=?1 +** AND closure.tablename='group' +** AND closure.idname='groupId' +** AND closure.parentname='parentId'; +** +** See the documentation at http://www.sqlite.org/loadext.html for information +** on how to compile and use loadable extensions such as this one. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include +#include +#include + +/* +** Forward declaration of objects used by this implementation +*/ +typedef struct closure_vtab closure_vtab; +typedef struct closure_cursor closure_cursor; +typedef struct closure_queue closure_queue; +typedef struct closure_avl closure_avl; + +/***************************************************************************** +** AVL Tree implementation +*/ +/* +** Objects that want to be members of the AVL tree should embedded an +** instance of this structure. +*/ +struct closure_avl { + sqlite3_int64 id; /* Id of this entry in the table */ + int iGeneration; /* Which generation is this entry part of */ + closure_avl *pList; /* A linked list of nodes */ + closure_avl *pBefore; /* Other elements less than id */ + closure_avl *pAfter; /* Other elements greater than id */ + closure_avl *pUp; /* Parent element */ + short int height; /* Height of this node. Leaf==1 */ + short int imbalance; /* Height difference between pBefore and pAfter */ +}; + +/* Recompute the closure_avl.height and closure_avl.imbalance fields for p. +** Assume that the children of p have correct heights. +*/ +static void closureAvlRecomputeHeight(closure_avl *p){ + short int hBefore = p->pBefore ? p->pBefore->height : 0; + short int hAfter = p->pAfter ? p->pAfter->height : 0; + p->imbalance = hBefore - hAfter; /* -: pAfter higher. +: pBefore higher */ + p->height = (hBefore>hAfter ? hBefore : hAfter)+1; +} + +/* +** P B +** / \ / \ +** B Z ==> X P +** / \ / \ +** X Y Y Z +** +*/ +static closure_avl *closureAvlRotateBefore(closure_avl *pP){ + closure_avl *pB = pP->pBefore; + closure_avl *pY = pB->pAfter; + pB->pUp = pP->pUp; + pB->pAfter = pP; + pP->pUp = pB; + pP->pBefore = pY; + if( pY ) pY->pUp = pP; + closureAvlRecomputeHeight(pP); + closureAvlRecomputeHeight(pB); + return pB; +} + +/* +** P A +** / \ / \ +** X A ==> P Z +** / \ / \ +** Y Z X Y +** +*/ +static closure_avl *closureAvlRotateAfter(closure_avl *pP){ + closure_avl *pA = pP->pAfter; + closure_avl *pY = pA->pBefore; + pA->pUp = pP->pUp; + pA->pBefore = pP; + pP->pUp = pA; + pP->pAfter = pY; + if( pY ) pY->pUp = pP; + closureAvlRecomputeHeight(pP); + closureAvlRecomputeHeight(pA); + return pA; +} + +/* +** Return a pointer to the pBefore or pAfter pointer in the parent +** of p that points to p. Or if p is the root node, return pp. +*/ +static closure_avl **closureAvlFromPtr(closure_avl *p, closure_avl **pp){ + closure_avl *pUp = p->pUp; + if( pUp==0 ) return pp; + if( pUp->pAfter==p ) return &pUp->pAfter; + return &pUp->pBefore; +} + +/* +** Rebalance all nodes starting with p and working up to the root. +** Return the new root. +*/ +static closure_avl *closureAvlBalance(closure_avl *p){ + closure_avl *pTop = p; + closure_avl **pp; + while( p ){ + closureAvlRecomputeHeight(p); + if( p->imbalance>=2 ){ + closure_avl *pB = p->pBefore; + if( pB->imbalance<0 ) p->pBefore = closureAvlRotateAfter(pB); + pp = closureAvlFromPtr(p,&p); + p = *pp = closureAvlRotateBefore(p); + }else if( p->imbalance<=(-2) ){ + closure_avl *pA = p->pAfter; + if( pA->imbalance>0 ) p->pAfter = closureAvlRotateBefore(pA); + pp = closureAvlFromPtr(p,&p); + p = *pp = closureAvlRotateAfter(p); + } + pTop = p; + p = p->pUp; + } + return pTop; +} + +/* Search the tree rooted at p for an entry with id. Return a pointer +** to the entry or return NULL. +*/ +static closure_avl *closureAvlSearch(closure_avl *p, sqlite3_int64 id){ + while( p && id!=p->id ){ + p = (idid) ? p->pBefore : p->pAfter; + } + return p; +} + +/* Find the first node (the one with the smallest key). +*/ +static closure_avl *closureAvlFirst(closure_avl *p){ + if( p ) while( p->pBefore ) p = p->pBefore; + return p; +} + +/* Return the node with the next larger key after p. +*/ +closure_avl *closureAvlNext(closure_avl *p){ + closure_avl *pPrev = 0; + while( p && p->pAfter==pPrev ){ + pPrev = p; + p = p->pUp; + } + if( p && pPrev==0 ){ + p = closureAvlFirst(p->pAfter); + } + return p; +} + +/* Insert a new node pNew. Return NULL on success. If the key is not +** unique, then do not perform the insert but instead leave pNew unchanged +** and return a pointer to an existing node with the same key. +*/ +static closure_avl *closureAvlInsert( + closure_avl **ppHead, /* Head of the tree */ + closure_avl *pNew /* New node to be inserted */ +){ + closure_avl *p = *ppHead; + if( p==0 ){ + p = pNew; + pNew->pUp = 0; + }else{ + while( p ){ + if( pNew->idid ){ + if( p->pBefore ){ + p = p->pBefore; + }else{ + p->pBefore = pNew; + pNew->pUp = p; + break; + } + }else if( pNew->id>p->id ){ + if( p->pAfter ){ + p = p->pAfter; + }else{ + p->pAfter = pNew; + pNew->pUp = p; + break; + } + }else{ + return p; + } + } + } + pNew->pBefore = 0; + pNew->pAfter = 0; + pNew->height = 1; + pNew->imbalance = 0; + *ppHead = closureAvlBalance(p); + return 0; +} + +/* Walk the tree can call xDestroy on each node +*/ +static void closureAvlDestroy(closure_avl *p, void (*xDestroy)(closure_avl*)){ + if( p ){ + closureAvlDestroy(p->pBefore, xDestroy); + closureAvlDestroy(p->pAfter, xDestroy); + xDestroy(p); + } +} +/* +** End of the AVL Tree implementation +******************************************************************************/ + +/* +** A closure virtual-table object +*/ +struct closure_vtab { + sqlite3_vtab base; /* Base class - must be first */ + char *zDb; /* Name of database. (ex: "main") */ + char *zSelf; /* Name of this virtual table */ + char *zTableName; /* Name of table holding parent/child relation */ + char *zIdColumn; /* Name of ID column of zTableName */ + char *zParentColumn; /* Name of PARENT column in zTableName */ + sqlite3 *db; /* The database connection */ + int nCursor; /* Number of pending cursors */ +}; + +/* A closure cursor object */ +struct closure_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + closure_vtab *pVtab; /* The virtual table this cursor belongs to */ + char *zTableName; /* Name of table holding parent/child relation */ + char *zIdColumn; /* Name of ID column of zTableName */ + char *zParentColumn; /* Name of PARENT column in zTableName */ + closure_avl *pCurrent; /* Current element of output */ + closure_avl *pClosure; /* The complete closure tree */ +}; + +/* A queue of AVL nodes */ +struct closure_queue { + closure_avl *pFirst; /* Oldest node on the queue */ + closure_avl *pLast; /* Youngest node on the queue */ +}; + +/* +** Add a node to the end of the queue +*/ +static void queuePush(closure_queue *pQueue, closure_avl *pNode){ + pNode->pList = 0; + if( pQueue->pLast ){ + pQueue->pLast->pList = pNode; + }else{ + pQueue->pFirst = pNode; + } + pQueue->pLast = pNode; +} + +/* +** Extract the oldest element (the front element) from the queue. +*/ +static closure_avl *queuePull(closure_queue *pQueue){ + closure_avl *p = pQueue->pFirst; + if( p ){ + pQueue->pFirst = p->pList; + if( pQueue->pFirst==0 ) pQueue->pLast = 0; + } + return p; +} + +/* +** This function converts an SQL quoted string into an unquoted string +** and returns a pointer to a buffer allocated using sqlite3_malloc() +** containing the result. The caller should eventually free this buffer +** using sqlite3_free. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +*/ +static char *closureDequote(const char *zIn){ + int nIn; /* Size of input string, in bytes */ + char *zOut; /* Output (dequoted) string */ + + nIn = (int)strlen(zIn); + zOut = sqlite3_malloc(nIn+1); + if( zOut ){ + char q = zIn[0]; /* Quote character (if any ) */ + + if( q!='[' && q!= '\'' && q!='"' && q!='`' ){ + memcpy(zOut, zIn, nIn+1); + }else{ + int iOut = 0; /* Index of next byte to write to output */ + int iIn; /* Index of next byte to read from input */ + + if( q=='[' ) q = ']'; + for(iIn=1; iInzDb); + sqlite3_free(p->zSelf); + sqlite3_free(p->zTableName); + sqlite3_free(p->zIdColumn); + sqlite3_free(p->zParentColumn); + memset(p, 0, sizeof(*p)); + sqlite3_free(p); + } +} + +/* +** xDisconnect/xDestroy method for the closure module. +*/ +static int closureDisconnect(sqlite3_vtab *pVtab){ + closure_vtab *p = (closure_vtab*)pVtab; + assert( p->nCursor==0 ); + closureFree(p); + return SQLITE_OK; +} + +/* +** Check to see if the argument is of the form: +** +** KEY = VALUE +** +** If it is, return a pointer to the first character of VALUE. +** If not, return NULL. Spaces around the = are ignored. +*/ +static const char *closureValueOfKey(const char *zKey, const char *zStr){ + int nKey = (int)strlen(zKey); + int nStr = (int)strlen(zStr); + int i; + if( nStr module name ("approximate_match") +** argv[1] -> database name +** argv[2] -> table name +** argv[3...] -> arguments +*/ +static int closureConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + int rc = SQLITE_OK; /* Return code */ + closure_vtab *pNew = 0; /* New virtual table */ + const char *zDb = argv[1]; + const char *zVal; + int i; + + (void)pAux; + *ppVtab = 0; + pNew = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + rc = SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + pNew->zDb = sqlite3_mprintf("%s", zDb); + if( pNew->zDb==0 ) goto closureConnectError; + pNew->zSelf = sqlite3_mprintf("%s", argv[2]); + if( pNew->zSelf==0 ) goto closureConnectError; + for(i=3; izTableName); + pNew->zTableName = closureDequote(zVal); + if( pNew->zTableName==0 ) goto closureConnectError; + continue; + } + zVal = closureValueOfKey("idcolumn", argv[i]); + if( zVal ){ + sqlite3_free(pNew->zIdColumn); + pNew->zIdColumn = closureDequote(zVal); + if( pNew->zIdColumn==0 ) goto closureConnectError; + continue; + } + zVal = closureValueOfKey("parentcolumn", argv[i]); + if( zVal ){ + sqlite3_free(pNew->zParentColumn); + pNew->zParentColumn = closureDequote(zVal); + if( pNew->zParentColumn==0 ) goto closureConnectError; + continue; + } + *pzErr = sqlite3_mprintf("unrecognized argument: [%s]\n", argv[i]); + closureFree(pNew); + *ppVtab = 0; + return SQLITE_ERROR; + } + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(id,depth,root HIDDEN,tablename HIDDEN," + "idcolumn HIDDEN,parentcolumn HIDDEN)" + ); +#define CLOSURE_COL_ID 0 +#define CLOSURE_COL_DEPTH 1 +#define CLOSURE_COL_ROOT 2 +#define CLOSURE_COL_TABLENAME 3 +#define CLOSURE_COL_IDCOLUMN 4 +#define CLOSURE_COL_PARENTCOLUMN 5 + if( rc!=SQLITE_OK ){ + closureFree(pNew); + } + *ppVtab = &pNew->base; + return rc; + +closureConnectError: + closureFree(pNew); + return rc; +} + +/* +** Open a new closure cursor. +*/ +static int closureOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + closure_vtab *p = (closure_vtab*)pVTab; + closure_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->pVtab = p; + *ppCursor = &pCur->base; + p->nCursor++; + return SQLITE_OK; +} + +/* +** Free up all the memory allocated by a cursor. Set it rLimit to 0 +** to indicate that it is at EOF. +*/ +static void closureClearCursor(closure_cursor *pCur){ + closureAvlDestroy(pCur->pClosure, (void(*)(closure_avl*))sqlite3_free); + sqlite3_free(pCur->zTableName); + sqlite3_free(pCur->zIdColumn); + sqlite3_free(pCur->zParentColumn); + pCur->zTableName = 0; + pCur->zIdColumn = 0; + pCur->zParentColumn = 0; + pCur->pCurrent = 0; + pCur->pClosure = 0; +} + +/* +** Close a closure cursor. +*/ +static int closureClose(sqlite3_vtab_cursor *cur){ + closure_cursor *pCur = (closure_cursor *)cur; + closureClearCursor(pCur); + pCur->pVtab->nCursor--; + sqlite3_free(pCur); + return SQLITE_OK; +} + +/* +** Advance a cursor to its next row of output +*/ +static int closureNext(sqlite3_vtab_cursor *cur){ + closure_cursor *pCur = (closure_cursor*)cur; + pCur->pCurrent = closureAvlNext(pCur->pCurrent); + return SQLITE_OK; +} + +/* +** Allocate and insert a node +*/ +static int closureInsertNode( + closure_queue *pQueue, /* Add new node to this queue */ + closure_cursor *pCur, /* The cursor into which to add the node */ + sqlite3_int64 id, /* The node ID */ + int iGeneration /* The generation number for this node */ +){ + closure_avl *pNew = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->id = id; + pNew->iGeneration = iGeneration; + closureAvlInsert(&pCur->pClosure, pNew); + queuePush(pQueue, pNew); + return SQLITE_OK; +} + +/* +** Called to "rewind" a cursor back to the beginning so that +** it starts its output over again. Always called at least once +** prior to any closureColumn, closureRowid, or closureEof call. +** +** This routine actually computes the closure. +** +** See the comment at the beginning of closureBestIndex() for a +** description of the meaning of idxNum. The idxStr parameter is +** not used. +*/ +static int closureFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + closure_cursor *pCur = (closure_cursor *)pVtabCursor; + closure_vtab *pVtab = pCur->pVtab; + sqlite3_int64 iRoot; + int mxGen = 999999999; + char *zSql; + sqlite3_stmt *pStmt; + closure_avl *pAvl; + int rc = SQLITE_OK; + const char *zTableName = pVtab->zTableName; + const char *zIdColumn = pVtab->zIdColumn; + const char *zParentColumn = pVtab->zParentColumn; + closure_queue sQueue; + + (void)idxStr; /* Unused parameter */ + (void)argc; /* Unused parameter */ + closureClearCursor(pCur); + memset(&sQueue, 0, sizeof(sQueue)); + if( (idxNum & 1)==0 ){ + /* No root=$root in the WHERE clause. Return an empty set */ + return SQLITE_OK; + } + iRoot = sqlite3_value_int64(argv[0]); + if( (idxNum & 0x000f0)!=0 ){ + mxGen = sqlite3_value_int(argv[(idxNum>>4)&0x0f]); + if( (idxNum & 0x00002)!=0 ) mxGen--; + } + if( (idxNum & 0x00f00)!=0 ){ + zTableName = (const char*)sqlite3_value_text(argv[(idxNum>>8)&0x0f]); + pCur->zTableName = sqlite3_mprintf("%s", zTableName); + } + if( (idxNum & 0x0f000)!=0 ){ + zIdColumn = (const char*)sqlite3_value_text(argv[(idxNum>>12)&0x0f]); + pCur->zIdColumn = sqlite3_mprintf("%s", zIdColumn); + } + if( (idxNum & 0x0f0000)!=0 ){ + zParentColumn = (const char*)sqlite3_value_text(argv[(idxNum>>16)&0x0f]); + pCur->zParentColumn = sqlite3_mprintf("%s", zParentColumn); + } + + zSql = sqlite3_mprintf( + "SELECT \"%w\".\"%w\" FROM \"%w\" WHERE \"%w\".\"%w\"=?1", + zTableName, zIdColumn, zTableName, zTableName, zParentColumn); + if( zSql==0 ){ + return SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pVtab->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + sqlite3_free(pVtab->base.zErrMsg); + pVtab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pVtab->db)); + return rc; + } + } + if( rc==SQLITE_OK ){ + rc = closureInsertNode(&sQueue, pCur, iRoot, 0); + } + while( (pAvl = queuePull(&sQueue))!=0 ){ + if( pAvl->iGeneration>=mxGen ) continue; + sqlite3_bind_int64(pStmt, 1, pAvl->id); + while( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + if( sqlite3_column_type(pStmt,0)==SQLITE_INTEGER ){ + sqlite3_int64 iNew = sqlite3_column_int64(pStmt, 0); + if( closureAvlSearch(pCur->pClosure, iNew)==0 ){ + rc = closureInsertNode(&sQueue, pCur, iNew, pAvl->iGeneration+1); + } + } + } + sqlite3_reset(pStmt); + } + sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + pCur->pCurrent = closureAvlFirst(pCur->pClosure); + } + + return rc; +} + +/* +** Only the word and distance columns have values. All other columns +** return NULL +*/ +static int closureColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + closure_cursor *pCur = (closure_cursor*)cur; + switch( i ){ + case CLOSURE_COL_ID: { + sqlite3_result_int64(ctx, pCur->pCurrent->id); + break; + } + case CLOSURE_COL_DEPTH: { + sqlite3_result_int(ctx, pCur->pCurrent->iGeneration); + break; + } + case CLOSURE_COL_ROOT: { + sqlite3_result_null(ctx); + break; + } + case CLOSURE_COL_TABLENAME: { + sqlite3_result_text(ctx, + pCur->zTableName ? pCur->zTableName : pCur->pVtab->zTableName, + -1, SQLITE_TRANSIENT); + break; + } + case CLOSURE_COL_IDCOLUMN: { + sqlite3_result_text(ctx, + pCur->zIdColumn ? pCur->zIdColumn : pCur->pVtab->zIdColumn, + -1, SQLITE_TRANSIENT); + break; + } + case CLOSURE_COL_PARENTCOLUMN: { + sqlite3_result_text(ctx, + pCur->zParentColumn ? pCur->zParentColumn : pCur->pVtab->zParentColumn, + -1, SQLITE_TRANSIENT); + break; + } + } + return SQLITE_OK; +} + +/* +** The rowid. For the closure table, this is the same as the "id" column. +*/ +static int closureRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + closure_cursor *pCur = (closure_cursor*)cur; + *pRowid = pCur->pCurrent->id; + return SQLITE_OK; +} + +/* +** EOF indicator +*/ +static int closureEof(sqlite3_vtab_cursor *cur){ + closure_cursor *pCur = (closure_cursor*)cur; + return pCur->pCurrent==0; +} + +/* +** Search for terms of these forms: +** +** (A) root = $root +** (B1) depth < $depth +** (B2) depth <= $depth +** (B3) depth = $depth +** (C) tablename = $tablename +** (D) idcolumn = $idcolumn +** (E) parentcolumn = $parentcolumn +** +** +** +** idxNum meaning +** ---------- ------------------------------------------------------ +** 0x00000001 Term of the form (A) found +** 0x00000002 The term of bit-2 is like (B1) +** 0x000000f0 Index in filter.argv[] of $depth. 0 if not used. +** 0x00000f00 Index in filter.argv[] of $tablename. 0 if not used. +** 0x0000f000 Index in filter.argv[] of $idcolumn. 0 if not used +** 0x000f0000 Index in filter.argv[] of $parentcolumn. 0 if not used. +** +** There must be a term of type (A). If there is not, then the index type +** is 0 and the query will return an empty set. +*/ +static int closureBestIndex( + sqlite3_vtab *pTab, /* The virtual table */ + sqlite3_index_info *pIdxInfo /* Information about the query */ +){ + int iPlan = 0; + int i; + int idx = 1; + const struct sqlite3_index_constraint *pConstraint; + closure_vtab *pVtab = (closure_vtab*)pTab; + + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( (iPlan & 1)==0 + && pConstraint->iColumn==CLOSURE_COL_ROOT + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + } + if( (iPlan & 0x0000f0)==0 + && pConstraint->iColumn==CLOSURE_COL_DEPTH + && (pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT + || pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE + || pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ) + ){ + iPlan |= idx<<4; + pIdxInfo->aConstraintUsage[i].argvIndex = ++idx; + if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT ) iPlan |= 0x000002; + } + if( (iPlan & 0x000f00)==0 + && pConstraint->iColumn==CLOSURE_COL_TABLENAME + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= idx<<8; + pIdxInfo->aConstraintUsage[i].argvIndex = ++idx; + pIdxInfo->aConstraintUsage[i].omit = 1; + } + if( (iPlan & 0x00f000)==0 + && pConstraint->iColumn==CLOSURE_COL_IDCOLUMN + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= idx<<12; + pIdxInfo->aConstraintUsage[i].argvIndex = ++idx; + pIdxInfo->aConstraintUsage[i].omit = 1; + } + if( (iPlan & 0x0f0000)==0 + && pConstraint->iColumn==CLOSURE_COL_PARENTCOLUMN + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= idx<<16; + pIdxInfo->aConstraintUsage[i].argvIndex = ++idx; + pIdxInfo->aConstraintUsage[i].omit = 1; + } + } + if( (pVtab->zTableName==0 && (iPlan & 0x000f00)==0) + || (pVtab->zIdColumn==0 && (iPlan & 0x00f000)==0) + || (pVtab->zParentColumn==0 && (iPlan & 0x0f0000)==0) + ){ + /* All of tablename, idcolumn, and parentcolumn must be specified + ** in either the CREATE VIRTUAL TABLE or in the WHERE clause constraints + ** or else the result is an empty set. */ + iPlan = 0; + } + pIdxInfo->idxNum = iPlan; + if( pIdxInfo->nOrderBy==1 + && pIdxInfo->aOrderBy[0].iColumn==CLOSURE_COL_ID + && pIdxInfo->aOrderBy[0].desc==0 + ){ + pIdxInfo->orderByConsumed = 1; + } + pIdxInfo->estimatedCost = (double)10000; + + return SQLITE_OK; +} + +/* +** A virtual table module that implements the "approximate_match". +*/ +static sqlite3_module closureModule = { + 0, /* iVersion */ + closureConnect, /* xCreate */ + closureConnect, /* xConnect */ + closureBestIndex, /* xBestIndex */ + closureDisconnect, /* xDisconnect */ + closureDisconnect, /* xDestroy */ + closureOpen, /* xOpen - open a cursor */ + closureClose, /* xClose - close a cursor */ + closureFilter, /* xFilter - configure scan constraints */ + closureNext, /* xNext - advance a cursor */ + closureEof, /* xEof - check for end of scan */ + closureColumn, /* xColumn - read data */ + closureRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ +}; + +/* +** Register the closure virtual table +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_closure_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; + rc = sqlite3_create_module(db, "transitive_closure", &closureModule, 0); + return rc; +} diff --git a/ext/misc/ieee754.c b/ext/misc/ieee754.c new file mode 100644 index 0000000000..436b11e0c5 --- /dev/null +++ b/ext/misc/ieee754.c @@ -0,0 +1,131 @@ +/* +** 2013-04-17 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements functions for the exact display +** and input of IEEE754 Binary64 floating-point numbers. +** +** ieee754(X) +** ieee754(Y,Z) +** +** In the first form, the value X should be a floating-point number. +** The function will return a string of the form 'ieee754(Y,Z)' where +** Y and Z are integers such that X==Y*pow(w.0,Z). +** +** In the second form, Y and Z are integers which are the mantissa and +** base-2 exponent of a new floating point number. The function returns +** a floating-point value equal to Y*pow(2.0,Z). +** +** Examples: +** +** ieee754(2.0) -> 'ieee754(2,0)' +** ieee754(45.25) -> 'ieee754(181,-2)' +** ieee754(2, 0) -> 2.0 +** ieee754(181, -2) -> 45.25 +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include + +/* +** Implementation of the ieee754() function +*/ +static void ieee754func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + if( argc==1 ){ + sqlite3_int64 m, a; + double r; + int e; + int isNeg; + char zResult[100]; + assert( sizeof(m)==sizeof(r) ); + if( sqlite3_value_type(argv[0])!=SQLITE_FLOAT ) return; + r = sqlite3_value_double(argv[0]); + if( r<0.0 ){ + isNeg = 1; + r = -r; + }else{ + isNeg = 0; + } + memcpy(&a,&r,sizeof(a)); + if( a==0 ){ + e = 0; + m = 0; + }else{ + e = a>>52; + m = a & ((((sqlite3_int64)1)<<52)-1); + m |= ((sqlite3_int64)1)<<52; + while( e<1075 && m>0 && (m&1)==0 ){ + m >>= 1; + e++; + } + if( isNeg ) m = -m; + } + sqlite3_snprintf(sizeof(zResult), zResult, "ieee754(%lld,%d)", + m, e-1075); + sqlite3_result_text(context, zResult, -1, SQLITE_TRANSIENT); + }else if( argc==2 ){ + sqlite3_int64 m, e, a; + double r; + int isNeg = 0; + m = sqlite3_value_int64(argv[0]); + e = sqlite3_value_int64(argv[1]); + if( m<0 ){ + isNeg = 1; + m = -m; + if( m<0 ) return; + }else if( m==0 && e>1000 && e<1000 ){ + sqlite3_result_double(context, 0.0); + return; + } + while( (m>>32)&0xffe00000 ){ + m >>= 1; + e++; + } + while( ((m>>32)&0xfff00000)==0 ){ + m <<= 1; + e--; + } + e += 1075; + if( e<0 ) e = m = 0; + if( e>0x7ff ) m = 0; + a = m & ((((sqlite3_int64)1)<<52)-1); + a |= e<<52; + if( isNeg ) a |= ((sqlite3_int64)1)<<63; + memcpy(&r, &a, sizeof(r)); + sqlite3_result_double(context, r); + } +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ieee_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "ieee754", 1, SQLITE_UTF8, 0, + ieee754func, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "ieee754", 2, SQLITE_UTF8, 0, + ieee754func, 0, 0); + } + return rc; +} diff --git a/ext/misc/regexp.c b/ext/misc/regexp.c index cb39956da5..16fa7d0b96 100644 --- a/ext/misc/regexp.c +++ b/ext/misc/regexp.c @@ -12,7 +12,16 @@ ** ** The code in this file implements a compact but reasonably ** efficient regular-expression matcher for posix extended regular -** expressions against UTF8 text. The following syntax is supported: +** expressions against UTF8 text. +** +** This file is an SQLite extension. It registers a single function +** named "regexp(A,B)" where A is the regular expression and B is the +** string to be matched. By registering this function, SQLite will also +** then implement the "B regexp A" operator. Note that with the function +** the regular expression comes first, but with the operator it comes +** second. +** +** The following regular expression syntax is supported: ** ** X* zero or more occurrences of X ** X+ one or more occurrences of X diff --git a/main.mk b/main.mk index eb3da5eaee..90571b0456 100644 --- a/main.mk +++ b/main.mk @@ -270,7 +270,10 @@ TESTSRC = \ # Extensions to be statically loaded. # TESTSRC += \ + $(TOP)/ext/misc/amatch.c \ + $(TOP)/ext/misc/closure.c \ $(TOP)/ext/misc/fuzzer.c \ + $(TOP)/ext/misc/ieee754.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/wholenumber.c diff --git a/manifest b/manifest index 4c3502f113..1e2aa91074 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Move\sthe\stest_spellfix.c\smodule\sto\sext/misc/spellfix.c. -D 2013-04-25T14:59:01.366 +C Added\sthe\stransitive_closure,\sieee754,\sand\samatch\sextensions. +D 2013-04-25T16:42:55.478 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 10c635460b6c3a20741d71c3a1b65b0ebec7558b +F Makefile.in 38a45083d4df568eefd31d27d67381850f35a016 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 97a23e910afb8c2d23db0217c343481d59045acb +F Makefile.msc d03cde9073d5752d6ddcaa6807a8bdda277e48f2 F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 05c7bd63b96f31cfdef5c766ed91307ac121f5aa @@ -83,9 +83,12 @@ F ext/fts3/unicode/mkunicode.tcl 7a9bc018e2962abb79563c5a39fe581fcbf2f675 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 +F ext/misc/amatch.c 3369b2b544066e620d986f0085d039c77d1ef17f +F ext/misc/closure.c fec0c8537c69843e0b7631d500a14c0527962cd6 F ext/misc/fuzzer.c fb64a15af978ae73fa9075b9b1dfbe82b8defc6f -F ext/misc/regexp.c c0fdb8af86981ff9890d776cfb97fe66297cc3b2 -F ext/misc/spellfix.c 8bb699116e36cc5e68d7ddf1810b638a3090c744 w src/test_spellfix.c +F ext/misc/ieee754.c 2565ce373d842977efe0922dc50b8a41b3289556 +F ext/misc/regexp.c c25c65fe775f5d9801fb8573e36ebe73f2c0c2e0 +F ext/misc/spellfix.c 8bb699116e36cc5e68d7ddf1810b638a3090c744 F ext/misc/wholenumber.c ce362368b9381ea48cbd951ade8df867eeeab014 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 757abea591d4ff67c0ff4e8f9776aeda86b18c14 @@ -108,7 +111,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk a6183110637d782988fdf3735f7d125b19639967 +F main.mk 9535835509ac58a1902ba85ed77bcce9840d62d4 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -195,7 +198,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 2ecec9937e69bc17560ad886da35195daa7261b8 -F src/test1.c 1c7fa0a36703508130d7ed65325883a9e453bf72 +F src/test1.c e9562428421bf0cdbf11bf5f60ab0cdabee45885 F src/test2.c 29e7154112f7448d64204e8d31179cf497ecf425 F src/test3.c 96aed72a8e1d542fed127e3e8350ae357712fa82 F src/test4.c cea2c55110241e4674e66d476d29c914627999f5 @@ -332,6 +335,7 @@ F test/capi3d.test 17b57ca28be3e37e14c2ba8f787d292d84b724a1 F test/capi3e.test f7408dda65c92b9056199fdc180f893015f83dde F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/check.test 2eb93611139a7dfaed3be80067c7dc5ceb5fb287 +F test/closure01.test 6194a899cdbba561d0439c0d6cc7bcdf4fc413e7 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 F test/collate1.test fd02c4d8afc71879c4bb952586389961a21fb0ce F test/collate2.test 04cebe4a033be319d6ddbb3bbc69464e01700b49 @@ -1054,7 +1058,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 680822e892f3efdb702eea3b321bc5785239dd56 -R f6325e95c7057e7590cfe9f4379a5b36 +P de556add10150140981a2e34b3712e96a7c262e3 +R 9481b1895542c76cd21821bef5dac0b1 U drh -Z 2d983c030ad222dceb5985301e41e8c4 +Z 540c4ff1b6a2b03d38328f53c71b1af7 diff --git a/manifest.uuid b/manifest.uuid index 2730c8b8c1..31b1041913 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -de556add10150140981a2e34b3712e96a7c262e3 \ No newline at end of file +84018099c8715b982cd24ce9221f93c7379e8c08 \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 7c30962b75..3cbe2bd543 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6057,7 +6057,10 @@ static int tclLoadStaticExtensionCmd( int objc, Tcl_Obj *CONST objv[] ){ + extern int sqlite3_amatch_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_closure_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_fuzzer_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_ieee_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_regexp_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_spellfix_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_wholenumber_init(sqlite3*,char**,const sqlite3_api_routines*); @@ -6065,7 +6068,10 @@ static int tclLoadStaticExtensionCmd( const char *zExtName; int (*pInit)(sqlite3*,char**,const sqlite3_api_routines*); } aExtension[] = { + { "amatch", sqlite3_amatch_init }, + { "closure", sqlite3_closure_init }, { "fuzzer", sqlite3_fuzzer_init }, + { "ieee754", sqlite3_ieee_init }, { "regexp", sqlite3_regexp_init }, { "spellfix", sqlite3_spellfix_init }, { "wholenumber", sqlite3_wholenumber_init }, diff --git a/test/closure01.test b/test/closure01.test new file mode 100644 index 0000000000..abae85c3ac --- /dev/null +++ b/test/closure01.test @@ -0,0 +1,224 @@ +# 2013-04-25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test cases for transitive_closure virtual table. + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix closure01 + +load_static_extension db closure + +do_execsql_test 1.0 { + BEGIN; + CREATE TABLE t1(x INTEGER PRIMARY KEY, y INTEGER); + CREATE INDEX t1y ON t1(y); + INSERT INTO t1(x) VALUES(1),(2); + INSERT INTO t1(x) SELECT x+2 FROM t1; + INSERT INTO t1(x) SELECT x+4 FROM t1; + INSERT INTO t1(x) SELECT x+8 FROM t1; + INSERT INTO t1(x) SELECT x+16 FROM t1; + INSERT INTO t1(x) SELECT x+32 FROM t1; + INSERT INTO t1(x) SELECT x+64 FROM t1; + INSERT INTO t1(x) SELECT x+128 FROM t1; + INSERT INTO t1(x) SELECT x+256 FROM t1; + INSERT INTO t1(x) SELECT x+512 FROM t1; + INSERT INTO t1(x) SELECT x+1024 FROM t1; + INSERT INTO t1(x) SELECT x+2048 FROM t1; + INSERT INTO t1(x) SELECT x+4096 FROM t1; + INSERT INTO t1(x) SELECT x+8192 FROM t1; + INSERT INTO t1(x) SELECT x+16384 FROM t1; + INSERT INTO t1(x) SELECT x+32768 FROM t1; + INSERT INTO t1(x) SELECT x+65536 FROM t1; + UPDATE t1 SET y=x/2 WHERE x>1; + COMMIT; + CREATE VIRTUAL TABLE cx + USING transitive_closure(tablename=t1, idcolumn=x, parentcolumn=y); +} {} + +# The entire table +do_execsql_test 1.1 { + SELECT count(*), depth FROM cx WHERE root=1 GROUP BY depth ORDER BY 1; +} {/1 0 1 17 2 1 4 2 8 3 16 4 .* 65536 16/} + +# descendents of 32768 +do_execsql_test 1.2 { + SELECT * FROM cx WHERE root=32768 ORDER BY id; +} {32768 0 65536 1 65537 1 131072 2} + +# descendents of 16384 +do_execsql_test 1.3 { + SELECT * FROM cx WHERE root=16384 AND depth<=2 ORDER BY id; +} {16384 0 32768 1 32769 1 65536 2 65537 2 65538 2 65539 2} + +# children of 16384 +do_execsql_test 1.4 { + SELECT id, depth, root, tablename, idcolumn, parentcolumn FROM cx + WHERE root=16384 + AND depth=1 + ORDER BY id; +} {32768 1 {} t1 x y 32769 1 {} t1 x y} + +# great-grandparent of 16384 +do_execsql_test 1.5 { + SELECT id, depth, root, tablename, idcolumn, parentcolumn FROM cx + WHERE root=16384 + AND depth=3 + AND idcolumn='Y' + AND parentcolumn='X'; +} {2048 3 {} t1 Y X} + +# depth<5 +do_execsql_test 1.6 { + SELECT count(*), depth FROM cx WHERE root=1 AND depth<5 + GROUP BY depth ORDER BY 1; +} {1 0 2 1 4 2 8 3 16 4} + +# depth<=5 +do_execsql_test 1.7 { + SELECT count(*), depth FROM cx WHERE root=1 AND depth<=5 + GROUP BY depth ORDER BY 1; +} {1 0 2 1 4 2 8 3 16 4 32 5} + +# depth==5 +do_execsql_test 1.8 { + SELECT count(*), depth FROM cx WHERE root=1 AND depth=5 + GROUP BY depth ORDER BY 1; +} {32 5} + +# depth BETWEEN 3 AND 5 +do_execsql_test 1.9 { + SELECT count(*), depth FROM cx WHERE root=1 AND depth BETWEEN 3 AND 5 + GROUP BY depth ORDER BY 1; +} {8 3 16 4 32 5} + +# depth==5 with min() and max() +do_execsql_test 1.10 { + SELECT count(*), min(id), max(id) FROM cx WHERE root=1 AND depth=5; +} {32 32 63} + +# Create a much smaller table t2 with only 32 elements +db eval { + CREATE TABLE t2(x INTEGER PRIMARY KEY, y INTEGER); + INSERT INTO t2 SELECT x, y FROM t1 WHERE x<32; + CREATE INDEX t2y ON t2(y); + CREATE VIRTUAL TABLE c2 + USING transitive_closure(tablename=t2, idcolumn=x, parentcolumn=y); +} + +# t2 full-table +do_execsql_test 2.1 { + SELECT count(*), min(id), max(id) FROM c2 WHERE root=1; +} {31 1 31} +# t2 root=10 +do_execsql_test 2.2 { + SELECT id FROM c2 WHERE root=10; +} {10 20 21} +# t2 root=11 +do_execsql_test 2.3 { + SELECT id FROM c2 WHERE root=12; +} {12 24 25} +# t2 root IN [10,12] +do_execsql_test 2.4 { + SELECT id FROM c2 WHERE root IN (10,12) ORDER BY id; +} {10 12 20 21 24 25} +# t2 root IN [10,12] (sorted) +do_execsql_test 2.5 { + SELECT id FROM c2 WHERE root IN (10,12) ORDER BY +id; +} {10 12 20 21 24 25} + +# t2 c2up from 20 +do_execsql_test 3.0 { + CREATE VIRTUAL TABLE c2up USING transitive_closure( + tablename = t2, + idcolumn = y, + parentcolumn = x + ); + SELECT id FROM c2up WHERE root=20; +} {1 2 5 10 20} + +# cx as c2up +do_execsql_test 3.1 { + SELECT id FROM cx + WHERE root=20 + AND tablename='t2' + AND idcolumn='y' + AND parentcolumn='x'; +} {1 2 5 10 20} + +# t2 first cousins of 20 +do_execsql_test 3.2 { + SELECT DISTINCT id FROM c2 + WHERE root IN (SELECT id FROM c2up + WHERE root=20 AND depth<=2) + ORDER BY id; +} {5 10 11 20 21 22 23} + +# t2 first cousins of 20 +do_execsql_test 3.3 { + SELECT id FROM c2 + WHERE root=(SELECT id FROM c2up + WHERE root=20 AND depth=2) + AND depth=2 + EXCEPT + SELECT id FROM c2 + WHERE root=(SELECT id FROM c2up + WHERE root=20 AND depth=1) + AND depth<=1 + ORDER BY id; +} {22 23} + +# missing tablename. +do_test 4.1 { + catchsql { + SELECT id FROM cx + WHERE root=20 + AND tablename='t3' + AND idcolumn='y' + AND parentcolumn='x'; + } +} {1 {no such table: t3}} + +# missing idcolumn +do_test 4.2 { + catchsql { + SELECT id FROM cx + WHERE root=20 + AND tablename='t2' + AND idcolumn='xyz' + AND parentcolumn='x'; + } +} {1 {no such column: t2.xyz}} + +# missing parentcolumn +do_test 4.3 { + catchsql { + SELECT id FROM cx + WHERE root=20 + AND tablename='t2' + AND idcolumn='x' + AND parentcolumn='pqr'; + } +} {1 {no such column: t2.pqr}} + +# generic closure +do_execsql_test 5.1 { + CREATE VIRTUAL TABLE temp.closure USING transitive_closure; + SELECT id FROM closure + WHERE root=1 + AND depth=3 + AND tablename='t1' + AND idcolumn='x' + AND parentcolumn='y' + ORDER BY id; +} {8 9 10 11 12 13 14 15} + +finish_test From d9555a7927ad0751e3539987ba4acbf264fa7c1e Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Apr 2013 17:07:26 +0000 Subject: [PATCH 165/204] Add wiki documentation files for the spellfix1 virtual table. FossilOrigin-Name: 381564e91bbf619f99a48b0b7a94ac586cb9ee79 --- ext/misc/editdist3.wiki | 114 ++++++++++ ext/misc/spellfix1.wiki | 464 ++++++++++++++++++++++++++++++++++++++++ manifest | 20 +- manifest.uuid | 2 +- 4 files changed, 590 insertions(+), 10 deletions(-) create mode 100644 ext/misc/editdist3.wiki create mode 100644 ext/misc/spellfix1.wiki diff --git a/ext/misc/editdist3.wiki b/ext/misc/editdist3.wiki new file mode 100644 index 0000000000..494922c5e9 --- /dev/null +++ b/ext/misc/editdist3.wiki @@ -0,0 +1,114 @@ +The editdist3 algorithm + +The editdist3 algorithm is a function that computes the minimum edit distance +(a.k.a. the Levenshtein distance) between two input strings. Features of +editdist3 include: + + * It works with unicode (UTF8) text. + + * A table of insertion, deletion, and substitution costs can be + provided by the application. + + * Multi-character insertsions, deletions, and substitutions can be + enumerated in the cost table. + +

    The COST table

    + +To program the costs of editdist3, create a table such as the following: + +
    +CREATE TABLE editcost(
+  iLang INT,   -- The language ID
+  cFrom TEXT,  -- Convert text from this
+  cTo   TEXT,  -- Convert text into this
+  iCost INT    -- The cost of doing the conversionnn
+);
+
    + +The cost table can be named anything you want - it does not have to be called +"editcost". And the table can contain additional columns. However, it the +table must contain the four columns show above, with exactly the names shown. + +The iLang column is a non-negative integer that identifies a set of costs +appropriate for a particular language. The editdist3 function will only use +a single iLang value for any given edit-distance computation. The default +value is 0. It is recommended that applications that only need to use a +single langauge always use iLang==0 for all entries. + +The iCost column is the numeric cost of transforming cFrom into cTo. This +value should be a non-negative integer, and should probably be less than 100. +The default single-character insertion and deletion costs are 100 and the +default single-character to single-character substitution cost is 150. A +cost of 10000 or more is considered "infinite" and causes the rule to be +ignored. + +The cFrom and cTo columns show edit transformation strings. Either or both +columns may contain more than one character. Or either column (but not both) +may hold an empty string. When cFrom is empty, that is the cost of inserting +cTo. When cTo is empty, that is the cost of deleting cFrom. + +In the spellfix1 algorithm, cFrom is the text as the user entered it and +cTo is the correctly spelled text as it exists in the database. The goal +of the editdist3 algorithm is to determine how close the user-entered text is +to the dictionary text. + +There are three special-case entries in the cost table: + + + + + + +
    cFromcToMeaning
    '''?'The default insertion cost
    '?'''The default deletion cost
    '?''?'The default substitution cost
    + +If any of the special-case entries shows above are omitted, then the +value of 100 is used for insertion and deletion and 150 is used for +substitution. To disable the default insertion, deletion, and/or substitution +set their respective cost to 10000 or more. + +Other entries in the cost table specific transforms for particular characters. +The cost of specific transforms should be less than the default costs, or else +the default costs will take precedence and the specific transforms will never +be used. + +Some example, cost table entries: + +
    +INSERT INTO editcost(iLang, cFrom, cTo, iCost)
+VALUES(0, 'a', 'ä', 5);
+
    + +The rule above says that the letter "a" in user input can be matched against +the letter "ä" in the dictionary with a penalty of 5. + +
    +INSERT INTO editcost(iLang, cFrom, cTo, iCost)
+VALUES(0, 'ss', 'ß', 8);
+
    + +The number of characters in cFrom and cTo do not need to be the same. The +rule above says that "ss" on user input will match "ß" with a penalty of 8. + +

    Experimenting with the editcost3() function

    + +The [./spellfix1.wiki | spellfix1 virtual table] +uses editdist3 if the "edit_cost_table=TABLE" option +is specified as an argument when the spellfix1 virtual table is created. +But editdist3 can also be tested directly using the built-in "editdist3()" +SQL function. The editdist3() SQL function has 3 forms: + + 1. editdist3('TABLENAME'); + 2. editdist3('string1', 'string2'); + 3. editdist3('string1', 'string2', langid); + +The first form loads the edit distance coefficients from a table called +'TABLENAME'. Any prior coefficients are discarded. So when experimenting +with weights and the weight table changes, simply rerun the single-argument +form of editdist3() to reload revised coefficients. Note that the +edit distance +weights used by the editdist3() SQL function are independent from the +weights used by the spellfix1 virtual table. + +The second and third forms return the computed edit distance between strings +'string1' and "string2'. In the second form, an language id of 0 is used. +The language id is specified in the third form. diff --git a/ext/misc/spellfix1.wiki b/ext/misc/spellfix1.wiki new file mode 100644 index 0000000000..288c55dd2e --- /dev/null +++ b/ext/misc/spellfix1.wiki @@ -0,0 +1,464 @@ +The Spellfix1 Virtual Table + +This spellfix1 virtual table is used to search +a large vocabulary for close matches. For example, spellfix1 +can be used to suggest corrections to misspelled words. Or, +it could be used with FTS4 to do full-text search using potentially +misspelled words. + +Create an instance of the spellfix1 virtual table like this: + +
    +CREATE VIRTUAL TABLE demo USING spellfix1;
+
    + +The "spellfix1" term is the name of this module and must be entered as +shown. The "demo" term is the +name of the virtual table you will be creating and can be altered +to suit the needs of your application. The virtual table is initially +empty. In order for the virtual table to be useful, you will need to +populate it with your vocabulary. Suppose you +have a list of words in a table named "big_vocabulary". Then do this: + +
    +INSERT INTO demo(word) SELECT word FROM big_vocabulary;
+
    + +If you intend to use this virtual table in cooperation with an FTS4 +table (for spelling correctly of search terms) then you might extract +the vocabulary using an fts3aux table: + +
    +INSERT INTO demo(word) SELECT term FROM search_aux WHERE col='*';
+
    + +You can also provide the virtual table with a "rank" for each word. +The "rank" is an estimate of how common the word is. Larger numbers +mean the word is more common. If you omit the rank when populating +the table, then a rank of 1 is assumed. But if you have rank +information, you can supply it and the virtual table will show a +slight preference for selecting more commonly used terms. To +populate the rank from an fts4aux table "search_aux" do something +like this: + +
    +INSERT INTO demo(word,rank)
+   SELECT term, documents FROM search_aux WHERE col='*';
+
    + +To query the virtual table, include a MATCH operator in the WHERE +clause. For example: + +
    +SELECT word FROM demo WHERE word MATCH 'kennasaw';
+
    + +Using a dataset of American place names (derived from +[http://geonames.usgs.gov/domestic/download_data.htm]) the query above +returns 20 results beginning with: + +
    +kennesaw
+kenosha
+kenesaw
+kenaga
+keanak
+
    + +If you append the character '*' to the end of the pattern, then +a prefix search is performed. For example: + +
    +SELECT word FROM demo WHERE word MATCH 'kennes*';
+
    + +Yields 20 results beginning with: + +
    +kennesaw
+kennestone
+kenneson
+kenneys
+keanes
+keenes
+
    + +

    Search Refinements

    + +By default, the spellfix1 table returns no more than 20 results. +(It might return less than 20 if there were fewer good matches.) +You can change the upper bound on the number of returned rows by +adding a "top=N" term to the WHERE clause of your query, where N +is the new maximum. For example, to see the 5 best matches: + +
    +SELECT word FROM demo WHERE word MATCH 'kennes*' AND top=5;
+
    + +Each entry in the spellfix1 virtual table is associated with a +a particular language, identified by the integer "langid" column. +The default langid is 0 and if no other actions are taken, the +entire vocabulary is a part of the 0 language. But if your application +needs to operate in multiple languages, then you can specify different +vocabulary items for each language by specifying the langid field +when populating the table. For example: + +
    +INSERT INTO demo(word,langid) SELECT word, 0 FROM en_vocabulary;
+INSERT INTO demo(word,langid) SELECT word, 1 FROM de_vocabulary;
+INSERT INTO demo(word,langid) SELECT word, 2 FROM fr_vocabulary;
+INSERT INTO demo(word,langid) SELECT word, 3 FROM ru_vocabulary;
+INSERT INTO demo(word,langid) SELECT word, 4 FROM cn_vocabulary;
+
    + +After the virtual table has been populated with items from multiple +languages, specify the language of interest using a "langid=N" term +in the WHERE clause of the query: + +
    +SELECT word FROM demo WHERE word MATCH 'hildes*' AND langid=1;
+
    + +Note that if you do not include the "langid=N" term in the WHERE clause, +the search will be against language 0 (English in the example above.) +All spellfix1 searches are against a single language id. There is no +way to search all languages at once. + + +

    Virtual Table Details

    + +The virtual table actually has a unique rowid with seven columns plus five +extra hidden columns. The columns are as follows: + +
    +
    rowid
    +A unique integer number associated with each +vocabulary item in the table. This can be used +as a foreign key on other tables in the database. + +
    word
    +The text of the word that matches the pattern. +Both word and pattern can contains unicode characters +and can be mixed case. + +
    rank
    +This is the rank of the word, as specified in the +original INSERT statement. + + +
    distance
    +This is an edit distance or Levensthein distance going +from the pattern to the word. + +
    langid
    +This is the language-id of the word. All queries are +against a single language-id, which defaults to 0. +For any given query this value is the same on all rows. + +
    score
    +The score is a combination of rank and distance. The +idea is that a lower score is better. The virtual table +attempts to find words with the lowest score and +by default (unless overridden by ORDER BY) returns +results in order of increasing score. + +
    matchlen
    +In a prefix search, the matchlen is the number of characters in +the string that match against the prefix. For a non-prefix search, +this is the same as length(word). + +
    phonehash
    +This column shows the phonetic hash prefix that was used to restrict +the search. For any given query, this column should be the same for +every row. This information is available for diagnostic purposes and +is not normally considered useful in real applications. + +
    top
    +(HIDDEN) For any query, this value is the same on all +rows. It is an integer which is the maximum number of +rows that will be output. The actually number of rows +output might be less than this number, but it will never +be greater. The default value for top is 20, but that +can be changed for each query by including a term of +the form "top=N" in the WHERE clause of the query. + +
    scope
    +(HIDDEN) For any query, this value is the same on all +rows. The scope is a measure of how widely the virtual +table looks for matching words. Smaller values of +scope cause a broader search. The scope is normally +choosen automatically and is capped at 4. Applications +can change the scope by including a term of the form +"scope=N" in the WHERE clause of the query. Increasing +the scope will make the query run faster, but will reduce +the possible corrections. + +
    srchcnt
    +(HIDDEN) For any query, this value is the same on all +rows. This value is an integer which is the number of +of words examined using the edit-distance algorithm to +find the top matches that are ultimately displayed. This +value is for diagnostic use only. + +
    soundslike
    +(HIDDEN) When inserting vocabulary entries, this field +can be set to an spelling that matches what the word +sounds like. See the DEALING WITH UNUSUAL AND DIFFICULT +SPELLINGS section below for details. + +
    command
    +(HIDDEN) The value of the "command" column is always NULL. However, +applications can insert special strings into the "command" column in order +to provoke certain behaviors in the spellfix1 virtual table. +For example, inserting the string 'reset' into the "command" column +will cause the virtual table will reread its edit distance weights +(if there are any). +
    + +

    Algorithm

    + +The spellfix1 virtual table creates a single +shadow table named "%_vocab" (where the % is replaced by the name of +the virtual table; Ex: "demo_vocab" for the "demo" virtual table). +the shadow table contains the following columns: + +
    +
    id
    +The unique id (INTEGER PRIMARY KEY) + +
    rank
    +The rank of word. + +
    langid
    +The language id for this entry. + +
    word
    +The original UTF8 text of the vocabulary word + +
    k1
    +The word transliterated into lower-case ASCII. +There is a standard table of mappings from non-ASCII +characters into ASCII. Examples: "æ" -> "ae", +"þ" -> "th", "ß" -> "ss", "á" -> "a", ... The +accessory function spellfix1_translit(X) will do +the non-ASCII to ASCII mapping. The built-in lower(X) +function will convert to lower-case. Thus: +k1 = lower(spellfix1_translit(word)). + +
    k2
    +This field holds a phonetic code derived from k1. Letters +that have similar sounds are mapped into the same symbol. +For example, all vowels and vowel clusters become the +single symbol "A". And the letters "p", "b", "f", and +"v" all become "B". All nasal sounds are represented +as "N". And so forth. The mapping is base on +ideas found in Soundex, Metaphone, and other +long-standing phonetic matching systems. This key can +be generated by the function spellfix1_phonehash(X). +Hence: k2 = spellfix1_phonehash(k1) +
    + +There is also a function for computing the Wagner edit distance or the +Levenshtein distance between a pattern and a word. This function +is exposed as spellfix1_editdist(X,Y). The edit distance function +returns the "cost" of converting X into Y. Some transformations +cost more than others. Changing one vowel into a different vowel, +for example is relatively cheap, as is doubling a constant, or +omitting the second character of a double-constant. Other transformations +or more expensive. The idea is that the edit distance function returns +a low cost of words that are similar and a higher cost for words +that are futher apart. In this implementation, the maximum cost +of any single-character edit (delete, insert, or substitute) is 100, +with lower costs for some edits (such as transforming vowels). + +The "score" for a comparison is the edit distance between the pattern +and the word, adjusted down by the base-2 logorithm of the word rank. +For example, a match with distance 100 but rank 1000 would have a +score of 122 (= 100 - log2(1000) + 32) where as a match with distance +100 with a rank of 1 would have a score of 131 (100 - log2(1) + 32). +(NB: The constant 32 is added to each score to keep it from going +negative in case the edit distance is zero.) In this way, frequently +used words get a slightly lower cost which tends to move them toward +the top of the list of alternative spellings. + +A straightforward implementation of a spelling corrector would be +to compare the search term against every word in the vocabulary +and select the 20 with the lowest scores. However, there will +typically be hundreds of thousands or millions of words in the +vocabulary, and so this approach is not fast enough. + +Suppose the term that is being spell-corrected is X. To limit +the search space, X is converted to a k2-like key using the +equivalent of: + +
    +   key = spellfix1_phonehash(lower(spellfix1_translit(X)))
+
    + +This key is then limited to "scope" characters. The default scope +value is 4, but an alternative scope can be specified using the +"scope=N" term in the WHERE clause. After the key has been truncated, +the edit distance is run against every term in the vocabulary that +has a k2 value that begins with the abbreviated key. + +For example, suppose the input word is "Paskagula". The phonetic +key is "BACACALA" which is then truncated to 4 characters "BACA". +The edit distance is then run on the 4980 entries (out of +272,597 entries total) of the vocabulary whose k2 values begin with +BACA, yielding "Pascagoula" as the best match. + +Only terms of the vocabulary with a matching langid are searched. +Hence, the same table can contain entries from multiple languages +and only the requested language will be used. The default langid +is 0. + +

    Configurable Edit Distance

    + +The built-in Wagner edit-distance function with fixed weights can be +replaced by the [./editdist3.wiki | editdist3()] edit-distance function +with application-defined weights and support for unicode, by specifying +the "edit_cost_table=TABLENAME" parameter to the spellfix1 module +when the virtual table is created. +For example: + +
    +CREATE VIRTUAL TABLE demo2 USING spellfix1(edit_cost_table=APPCOST);
+
    + +In the example above, the APPCOST table would be interrogated to find +the edit distance coefficients. It is the presence of the "edit_cost_table=" +parameter to the spellfix1 module name that causes editdist3() to be used +in place of the built-in edit distance function. + +The edit distance coefficients are normally read from the APPCOST table +once and there after stored in memory. Hence, run-time changes to the +APPCOST table will not normally effect the edit distance results. +However, inserting the special string 'reset' into the "command" column of the +virtual table causes the edit distance coefficients to be reread the +APPCOST table. Hence, applications should run a SQL statement similar +to the following when changes to the APPCOST table occur: + +
    +INSERT INTO demo2(command) VALUES('reset'); +
    + +The tables used for edit distance costs can be changed using a command +like the following: + +
    +INSERT INTO demo2(command) VALUES('edit_cost_table=APPCOST2'); +
    + +In the example above, any prior edit distance costs would be discarded and +all future queries would use the costs found in the APPCOST2 table. If the +name of the table specified by the "edit_cost_table" command is "NULL", then +theh built-in Wagner edit-distance function will be used instead of the +editdist3() function in all future queries. + +

    Dealing With Unusual And Difficult Spellings

    + +The algorithm above works quite well for most cases, but there are +exceptions. These exceptions can be dealt with by making additional +entries in the virtual table using the "soundslike" column. + +For example, many words of Greek origin begin with letters "ps" where +the "p" is silent. Ex: psalm, pseudonym, psoriasis, psyche. In +another example, many Scottish surnames can be spelled with an +initial "Mac" or "Mc". Thus, "MacKay" and "McKay" are both pronounced +the same. + +Accommodation can be made for words that are not spelled as they +sound by making additional entries into the virtual table for the +same word, but adding an alternative spelling in the "soundslike" +column. For example, the canonical entry for "psalm" would be this: + +
    +  INSERT INTO demo(word) VALUES('psalm');
+
    + +To enhance the ability to correct the spelling of "salm" into +"psalm", make an addition entry like this: + +
    +  INSERT INTO demo(word,soundslike) VALUES('psalm','salm');
+
    + +It is ok to make multiple entries for the same word as long as +each entry has a different soundslike value. Note that if no +soundslike value is specified, the soundslike defaults to the word +itself. + +Listed below are some cases where it might make sense to add additional +soundslike entries. The specific entries will depend on the application +and the target language. + + * Silent "p" in words beginning with "ps": psalm, psyche + + * Silent "p" in words beginning with "pn": pneumonia, pneumatic + + * Silent "p" in words beginning with "pt": pterodactyl, ptolemaic + + * Silent "d" in words beginning with "dj": djinn, Djikarta + + * Silent "k" in words beginning with "kn": knight, Knuthson + + * Silent "g" in words beginning with "gn": gnarly, gnome, gnat + + * "Mac" versus "Mc" beginning Scottish surnames + + * "Tch" sounds in Slavic words: Tchaikovsky vs. Chaykovsky + + * The letter "j" pronounced like "h" in Spanish: LaJolla + + * Words beginning with "wr" versus "r": write vs. rite + + * Miscellanous problem words such as "debt", "tsetse", + "Nguyen", "Van Nuyes". + +

    Auxiliary Functions

    + +The source code module that implements the spellfix1 virtual table also +implements several SQL functions that might be useful to applications +that employ spellfix1 or for testing or diagnostic work while developing +applications that use spellfix1. The following auxiliary functions are +available: + +
    +
    editdist3(P,W)
    editdist2(P,W,L)
    editdist3(T)
    +These routines provide direct access to the version of the Wagner +edit-distance function that allows for application-defined weights +on edit operations. The first two forms of this function compare +pattern P against word W and return the edit distance. In the first +function, the langid is assumed to be 0 and in the second, the +langid is given by the L parameter. The third form of this function +reloads edit distance coefficience from the table named by T. + +
    spellfix1_editdist(P,W)
    +This routine provides access to the built-in Wagner edit-distance +function that uses default, fixed costs. The value returned is +the edit distance needed to transform W into P. + +
    spellfix1_phonehash(X)
    +This routine constructs a phonetic hash of the pure ascii input word X +and returns that hash. This routine is used internally by spellfix1 in +order to transform the K1 column of the shadow table into the K2 +column. + +
    spellfix1_scriptcode(X)
    +Given an input string X, this routine attempts to determin the dominant +script of that input and returns the ISO-15924 numeric code for that +script. The current implementation understands the following scripts: +
      +
    • 215 - Latin +
    • 220 - Cyrillic +
    • 200 - Greek +
    +Additional language codes might be added in future releases. + +
    spellfix1_translit(X)
    +This routine transliterates unicode text into pure ascii, returning +the pure ascii representation of the input text X. This is the function +that is used internally to transform vocabulary words into the K1 +column of the shadow table. + +
    diff --git a/manifest b/manifest index 89fc7d396e..473037f91a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sstd-ext\sbranch\sinto\strunk.\s\sThis\smerge\sadds\sseveral\snew\sextensions\nto\sthe\sext/misc\sfolder,\sincluding\stransitive_closure,\sieee754,\sand\samatch,\nand\sit\sconvers\ssome\solder\ssrc/test_*.c\sfile\sinto\sextensions\sin\sthe\sext/misc\nfolder. -D 2013-04-25T16:52:19.390 +C Add\swiki\sdocumentation\sfiles\sfor\sthe\sspellfix1\svirtual\stable. +D 2013-04-25T17:07:26.477 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 38a45083d4df568eefd31d27d67381850f35a016 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -85,11 +85,13 @@ F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/misc/amatch.c 3369b2b544066e620d986f0085d039c77d1ef17f F ext/misc/closure.c fec0c8537c69843e0b7631d500a14c0527962cd6 -F ext/misc/fuzzer.c fb64a15af978ae73fa9075b9b1dfbe82b8defc6f w src/test_fuzzer.c +F ext/misc/editdist3.wiki 06100a0c558921a563cbc40e0d0151902b1eef6d +F ext/misc/fuzzer.c fb64a15af978ae73fa9075b9b1dfbe82b8defc6f F ext/misc/ieee754.c 2565ce373d842977efe0922dc50b8a41b3289556 -F ext/misc/regexp.c c25c65fe775f5d9801fb8573e36ebe73f2c0c2e0 w src/test_regexp.c -F ext/misc/spellfix.c 8bb699116e36cc5e68d7ddf1810b638a3090c744 w src/test_spellfix.c -F ext/misc/wholenumber.c ce362368b9381ea48cbd951ade8df867eeeab014 w src/test_wholenumber.c +F ext/misc/regexp.c c25c65fe775f5d9801fb8573e36ebe73f2c0c2e0 +F ext/misc/spellfix.c 8bb699116e36cc5e68d7ddf1810b638a3090c744 +F ext/misc/spellfix1.wiki dd1830444c14cf0f54dd680cc044df2ace2e9d09 +F ext/misc/wholenumber.c ce362368b9381ea48cbd951ade8df867eeeab014 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 757abea591d4ff67c0ff4e8f9776aeda86b18c14 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1058,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 39b4e6ff9316cc78ea88349091e195b8104d1e9e 84018099c8715b982cd24ce9221f93c7379e8c08 -R dffac967ebf5545ada46cf15534fed7e +P bbe607c7d17c50b667990360e2ccfab8dd22f161 +R 7f8f6b4f826c162e581af7818370c158 U drh -Z d8995ab3eecb48f05ff755314fd194c9 +Z c739a22e46c39eecc65136d2af6019ff diff --git a/manifest.uuid b/manifest.uuid index 33791ed9df..84c8f03dc9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bbe607c7d17c50b667990360e2ccfab8dd22f161 \ No newline at end of file +381564e91bbf619f99a48b0b7a94ac586cb9ee79 \ No newline at end of file From e67f418d10ba3a1823bf68713bc0f04a9880fab3 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Apr 2013 17:27:08 +0000 Subject: [PATCH 166/204] Fix the tool/build-shell.sh script to remove references to files that are now loadable extensions. FossilOrigin-Name: aabeea98f53edde68f484f1794ae70789dac3889 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/build-shell.sh | 4 ---- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 473037f91a..113cfb711f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\swiki\sdocumentation\sfiles\sfor\sthe\sspellfix1\svirtual\stable. -D 2013-04-25T17:07:26.477 +C Fix\sthe\stool/build-shell.sh\sscript\sto\sremove\sreferences\sto\sfiles\sthat\sare\nnow\sloadable\sextensions. +D 2013-04-25T17:27:08.281 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 38a45083d4df568eefd31d27d67381850f35a016 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1017,7 +1017,7 @@ F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F test/zerodamage.test 209d7ed441f44cc5299e4ebffbef06fd5aabfefd F tool/build-all-msvc.bat 74fb6e5cca66ebdb6c9bbafb2f8b802f08146d38 x -F tool/build-shell.sh a9c34a606e2e522ba9eeca1e07090f67dce8c912 +F tool/build-shell.sh 950f47c6174f1eea171319438b93ba67ff5bf367 F tool/checkSpacing.c 810e51703529a204fc4e1eb060e9ab663e3c06d2 F tool/diffdb.c 7524b1b5df217c20cd0431f6789851a4e0cb191b F tool/extract.c 054069d81b095fbdc189a6f5d4466e40380505e2 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P bbe607c7d17c50b667990360e2ccfab8dd22f161 -R 7f8f6b4f826c162e581af7818370c158 +P 381564e91bbf619f99a48b0b7a94ac586cb9ee79 +R dfc5d9e4a45d231ac8d29e1b5d10bc7e U drh -Z c739a22e46c39eecc65136d2af6019ff +Z 8c5379cfdd83f28d6a14d88089c63be6 diff --git a/manifest.uuid b/manifest.uuid index 84c8f03dc9..6111651bb6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -381564e91bbf619f99a48b0b7a94ac586cb9ee79 \ No newline at end of file +aabeea98f53edde68f484f1794ae70789dac3889 \ No newline at end of file diff --git a/tool/build-shell.sh b/tool/build-shell.sh index cd2838ebd6..6a48299d73 100644 --- a/tool/build-shell.sh +++ b/tool/build-shell.sh @@ -15,12 +15,8 @@ gcc -o sqlite3 -g -Os -I. \ -DSQLITE_ENABLE_STAT3 \ -DSQLITE_ENABLE_FTS4 \ -DSQLITE_ENABLE_RTREE \ - -DSQLITE_ENABLE_REGEXP \ - -DSQLITE_ENABLE_SPELLFIX -DSQLITE_CORE=1 \ -DHAVE_READLINE \ -DHAVE_USLEEP=1 \ ../sqlite/src/shell.c \ - ../sqlite/src/test_regexp.c \ - ../sqlite/src/test_spellfix.c \ ../sqlite/src/test_vfstrace.c \ sqlite3.c -ldl -lreadline -lncurses From ea41dc44c579fe48b3130c43ef040e07ca2875b0 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Apr 2013 19:31:33 +0000 Subject: [PATCH 167/204] Added the nextchar.c extension. Minor changes to the spellfix.c extension so that it can be appended to an amalgamation and compiled without duplicating symbols. FossilOrigin-Name: 56b9a417f5451631f11c5206d625f11472ee65f9 --- Makefile.in | 1 + Makefile.msc | 1 + ext/misc/nextchar.c | 265 ++++++++++++++++++++++++++++++++++++++++++++ ext/misc/spellfix.c | 20 ++-- main.mk | 1 + manifest | 23 ++-- manifest.uuid | 2 +- src/test1.c | 42 +++---- test/spellfix.test | 22 +++- 9 files changed, 336 insertions(+), 41 deletions(-) create mode 100644 ext/misc/nextchar.c diff --git a/Makefile.in b/Makefile.in index c713daaebc..068da1a212 100644 --- a/Makefile.in +++ b/Makefile.in @@ -392,6 +392,7 @@ TESTSRC += \ $(TOP)/ext/misc/closure.c \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/misc/ieee754.c \ + $(TOP)/ext/misc/nextchar.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/wholenumber.c diff --git a/Makefile.msc b/Makefile.msc index 383368c44c..1c86264804 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -712,6 +712,7 @@ TESTEXT = \ $(TOP)\ext\misc\closure.c \ $(TOP)\ext\misc\fuzzer.c \ $(TOP)\ext\misc\ieee754.c \ + $(TOP)\ext\misc\nextchar.c \ $(TOP)\ext\misc\regexp.c \ $(TOP)\ext\misc\spellfix.c \ $(TOP)\ext\misc\wholenumber.c diff --git a/ext/misc/nextchar.c b/ext/misc/nextchar.c new file mode 100644 index 0000000000..e063043e04 --- /dev/null +++ b/ext/misc/nextchar.c @@ -0,0 +1,265 @@ +/* +** 2013-02-28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code to implement the next_char(A,T,F,W) SQL function. +** +** The next_char(A,T,F,H) function finds all valid "next" characters for +** string A given the vocabulary in T.F. The T.F field should be indexed. +** If the W value exists and is a non-empty string, then it is an SQL +** expression that limits the entries in T.F that will be considered. +** +** For example, suppose an application has a dictionary like this: +** +** CREATE TABLE dictionary(word TEXT UNIQUE); +** +** Further suppose that for user keypad entry, it is desired to disable +** (gray out) keys that are not valid as the next character. If the +** the user has previously entered (say) 'cha' then to find all allowed +** next characters (and thereby determine when keys should not be grayed +** out) run the following query: +** +** SELECT next_char('cha','dictionary','word'); +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include + +/* +** A structure to hold context of the next_char() computation across +** nested function calls. +*/ +typedef struct nextCharContext nextCharContext; +struct nextCharContext { + sqlite3 *db; /* Database connection */ + sqlite3_stmt *pStmt; /* Prepared statement used to query */ + const unsigned char *zPrefix; /* Prefix to scan */ + int nPrefix; /* Size of zPrefix in bytes */ + int nAlloc; /* Space allocated to aResult */ + int nUsed; /* Space used in aResult */ + unsigned int *aResult; /* Array of next characters */ + int mallocFailed; /* True if malloc fails */ + int otherError; /* True for any other failure */ +}; + +/* +** Append a result character if the character is not already in the +** result. +*/ +static void nextCharAppend(nextCharContext *p, unsigned c){ + int i; + for(i=0; inUsed; i++){ + if( p->aResult[i]==c ) return; + } + if( p->nUsed+1 > p->nAlloc ){ + unsigned int *aNew; + int n = p->nAlloc*2 + 30; + aNew = sqlite3_realloc(p->aResult, n*sizeof(unsigned int)); + if( aNew==0 ){ + p->mallocFailed = 1; + return; + }else{ + p->aResult = aNew; + p->nAlloc = n; + } + } + p->aResult[p->nUsed++] = c; +} + +/* +** Write a character into z[] as UTF8. Return the number of bytes needed +** to hold the character +*/ +static int writeUtf8(unsigned char *z, unsigned c){ + if( c<0x00080 ){ + z[0] = (unsigned char)(c&0xff); + return 1; + } + if( c<0x00800 ){ + z[0] = 0xC0 + (unsigned char)((c>>6)&0x1F); + z[1] = 0x80 + (unsigned char)(c & 0x3F); + return 2; + } + if( c<0x10000 ){ + z[0] = 0xE0 + (unsigned char)((c>>12)&0x0F); + z[1] = 0x80 + (unsigned char)((c>>6) & 0x3F); + z[2] = 0x80 + (unsigned char)(c & 0x3F); + return 3; + } + z[0] = 0xF0 + (unsigned char)((c>>18) & 0x07); + z[1] = 0x80 + (unsigned char)((c>>12) & 0x3F); + z[2] = 0x80 + (unsigned char)((c>>6) & 0x3F); + z[3] = 0x80 + (unsigned char)(c & 0x3F); + return 4; +} + +/* +** Read a UTF8 character out of z[] and write it into *pOut. Return +** the number of bytes in z[] that were used to construct the character. +*/ +static int readUtf8(const unsigned char *z, unsigned *pOut){ + static const unsigned char validBits[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, + }; + unsigned c = z[0]; + if( c<0xc0 ){ + *pOut = c; + return 1; + }else{ + int n = 1; + c = validBits[c-0xc0]; + while( (z[n] & 0xc0)==0x80 ){ + c = (c<<6) + (0x3f & z[n++]); + } + if( c<0x80 || (c&0xFFFFF800)==0xD800 || (c&0xFFFFFFFE)==0xFFFE ){ + c = 0xFFFD; + } + *pOut = c; + return n; + } +} + +/* +** The nextCharContext structure has been set up. Add all "next" characters +** to the result set. +*/ +static void findNextChars(nextCharContext *p){ + unsigned cPrev = 0; + unsigned char zPrev[8]; + int n, rc; + + for(;;){ + sqlite3_bind_text(p->pStmt, 1, (char*)p->zPrefix, p->nPrefix, + SQLITE_STATIC); + n = writeUtf8(zPrev, cPrev+1); + sqlite3_bind_text(p->pStmt, 2, (char*)zPrev, n, SQLITE_STATIC); + rc = sqlite3_step(p->pStmt); + if( rc==SQLITE_DONE ){ + sqlite3_reset(p->pStmt); + return; + }else if( rc!=SQLITE_ROW ){ + p->otherError = rc; + return; + }else{ + const unsigned char *zOut = sqlite3_column_text(p->pStmt, 0); + unsigned cNext; + n = readUtf8(zOut+p->nPrefix, &cNext); + sqlite3_reset(p->pStmt); + nextCharAppend(p, cNext); + cPrev = cNext; + if( p->mallocFailed ) return; + } + } +} + + +/* +** next_character(A,T,F,W) +** +** Return a string composted of all next possible characters after +** A for elements of T.F. If W is supplied, then it is an SQL expression +** that limits the elements in T.F that are considered. +*/ +static void nextCharFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + nextCharContext c; + const unsigned char *zTable = sqlite3_value_text(argv[1]); + const unsigned char *zField = sqlite3_value_text(argv[2]); + const unsigned char *zWhere; + char *zSql; + int rc; + + memset(&c, 0, sizeof(c)); + c.db = sqlite3_context_db_handle(context); + c.zPrefix = sqlite3_value_text(argv[0]); + c.nPrefix = sqlite3_value_bytes(argv[0]); + if( zTable==0 || zField==0 || c.zPrefix==0 ) return; + if( argc<4 + || (zWhere = sqlite3_value_text(argv[3]))==0 + || zWhere[0]==0 + ){ + zSql = sqlite3_mprintf( + "SELECT \"%w\" FROM \"%w\"" + " WHERE \"%w\">=(?1 || ?2)" + " AND \"%w\"<=(?1 || char(1114111))" /* 1114111 == 0x10ffff */ + " ORDER BY 1 ASC LIMIT 1", + zField, zTable, zField, zField); + }else{ + zSql = sqlite3_mprintf( + "SELECT \"%w\" FROM \"%w\"" + " WHERE \"%w\">=(?1 || ?2)" + " AND \"%w\"<=(?1 || char(1114111))" /* 1114111 == 0x10ffff */ + " AND (%s)" + " ORDER BY 1 ASC LIMIT 1", + zField, zTable, zField, zField, zWhere); + } + if( zSql==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + rc = sqlite3_prepare_v2(c.db, zSql, -1, &c.pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + sqlite3_result_error(context, sqlite3_errmsg(c.db), -1); + return; + } + findNextChars(&c); + if( c.mallocFailed ){ + sqlite3_result_error_nomem(context); + }else{ + unsigned char *pRes; + pRes = sqlite3_malloc( c.nUsed*4 + 1 ); + if( pRes==0 ){ + sqlite3_result_error_nomem(context); + }else{ + int i; + int n = 0; + for(i=0; i -#include -#include -#include -#define ALWAYS(X) 1 -#define NEVER(X) 0 -typedef unsigned char u8; -typedef unsigned short u16; -#include +#ifndef SQLITE_AMALGAMATION +# include +# include +# include +# include +# define ALWAYS(X) 1 +# define NEVER(X) 0 + typedef unsigned char u8; + typedef unsigned short u16; +# include +#endif /* ** Character classes for ASCII characters: diff --git a/main.mk b/main.mk index 90571b0456..30add69e29 100644 --- a/main.mk +++ b/main.mk @@ -274,6 +274,7 @@ TESTSRC += \ $(TOP)/ext/misc/closure.c \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/misc/ieee754.c \ + $(TOP)/ext/misc/nextchar.c \ $(TOP)/ext/misc/regexp.c \ $(TOP)/ext/misc/spellfix.c \ $(TOP)/ext/misc/wholenumber.c diff --git a/manifest b/manifest index 113cfb711f..a3d9885853 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Fix\sthe\stool/build-shell.sh\sscript\sto\sremove\sreferences\sto\sfiles\sthat\sare\nnow\sloadable\sextensions. -D 2013-04-25T17:27:08.281 +C Added\sthe\snextchar.c\sextension.\s\sMinor\schanges\sto\sthe\sspellfix.c\sextension\nso\sthat\sit\scan\sbe\sappended\sto\san\samalgamation\sand\scompiled\swithout\sduplicating\nsymbols. +D 2013-04-25T19:31:33.149 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 38a45083d4df568eefd31d27d67381850f35a016 +F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc d03cde9073d5752d6ddcaa6807a8bdda277e48f2 +F Makefile.msc 8f4ee0dab220a5276d5da61149dfd6cd5d1dd5b8 F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 05c7bd63b96f31cfdef5c766ed91307ac121f5aa @@ -88,8 +88,9 @@ F ext/misc/closure.c fec0c8537c69843e0b7631d500a14c0527962cd6 F ext/misc/editdist3.wiki 06100a0c558921a563cbc40e0d0151902b1eef6d F ext/misc/fuzzer.c fb64a15af978ae73fa9075b9b1dfbe82b8defc6f F ext/misc/ieee754.c 2565ce373d842977efe0922dc50b8a41b3289556 +F ext/misc/nextchar.c 1131e2b36116ffc6fe6b2e3464bfdace27978b1e F ext/misc/regexp.c c25c65fe775f5d9801fb8573e36ebe73f2c0c2e0 -F ext/misc/spellfix.c 8bb699116e36cc5e68d7ddf1810b638a3090c744 +F ext/misc/spellfix.c e323eebb877d735bc64404c16a6d758ab17a0b7a F ext/misc/spellfix1.wiki dd1830444c14cf0f54dd680cc044df2ace2e9d09 F ext/misc/wholenumber.c ce362368b9381ea48cbd951ade8df867eeeab014 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -113,7 +114,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk 9535835509ac58a1902ba85ed77bcce9840d62d4 +F main.mk 1b25be82452366abc27cc9ab2acf3244a773d5a1 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -200,7 +201,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 2ecec9937e69bc17560ad886da35195daa7261b8 -F src/test1.c e9562428421bf0cdbf11bf5f60ab0cdabee45885 +F src/test1.c 2b0ec224983403312a4d1db8546e1e1c45694251 F src/test2.c 29e7154112f7448d64204e8d31179cf497ecf425 F src/test3.c 96aed72a8e1d542fed127e3e8350ae357712fa82 F src/test4.c cea2c55110241e4674e66d476d29c914627999f5 @@ -756,7 +757,7 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/spellfix.test a85915ab25af7fcfb0d99cb1951e6ef15e26202c +F test/spellfix.test bea537caf587df30d430c2c6a8fe9f64b8712834 F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test be8d477306006ec696bc86757cfb34bec79447ce F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9 @@ -1060,7 +1061,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 381564e91bbf619f99a48b0b7a94ac586cb9ee79 -R dfc5d9e4a45d231ac8d29e1b5d10bc7e +P aabeea98f53edde68f484f1794ae70789dac3889 +R 77b136ceb7e3bfd54c75649b938a3e12 U drh -Z 8c5379cfdd83f28d6a14d88089c63be6 +Z fb4502a56cfa201ec6c890343c386675 diff --git a/manifest.uuid b/manifest.uuid index 6111651bb6..7c24934462 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aabeea98f53edde68f484f1794ae70789dac3889 \ No newline at end of file +56b9a417f5451631f11c5206d625f11472ee65f9 \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 3cbe2bd543..34370d48e4 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6047,9 +6047,9 @@ static int optimization_control( typedef struct sqlite3_api_routines sqlite3_api_routines; /* -** load_static_extension DB NAME +** load_static_extension DB NAME ... ** -** Load an extension that is statically linked. +** Load one or more statically linked extensions. */ static int tclLoadStaticExtensionCmd( void * clientData, @@ -6061,6 +6061,7 @@ static int tclLoadStaticExtensionCmd( extern int sqlite3_closure_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_fuzzer_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_ieee_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_nextchar_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_regexp_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_spellfix_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_wholenumber_init(sqlite3*,char**,const sqlite3_api_routines*); @@ -6072,33 +6073,36 @@ static int tclLoadStaticExtensionCmd( { "closure", sqlite3_closure_init }, { "fuzzer", sqlite3_fuzzer_init }, { "ieee754", sqlite3_ieee_init }, + { "nextchar", sqlite3_nextchar_init }, { "regexp", sqlite3_regexp_init }, { "spellfix", sqlite3_spellfix_init }, { "wholenumber", sqlite3_wholenumber_init }, }; sqlite3 *db; const char *zName; - int i, rc; + int i, j, rc; char *zErrMsg = 0; - if( objc!=3 ){ - Tcl_WrongNumArgs(interp, 1, objv, "DB NAME"); + if( objc<3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB NAME ..."); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; - zName = Tcl_GetString(objv[2]); - for(i=0; i=ArraySize(aExtension) ){ - Tcl_AppendResult(interp, "no such extension: ", zName, (char*)0); - return TCL_ERROR; - } - rc = aExtension[i].pInit(db, &zErrMsg, 0); - if( rc!=SQLITE_OK || zErrMsg ){ - Tcl_AppendResult(interp, "initialization of ", zName, " failed: ", zErrMsg, - (char*)0); - sqlite3_free(zErrMsg); - return TCL_ERROR; + for(j=2; j=ArraySize(aExtension) ){ + Tcl_AppendResult(interp, "no such extension: ", zName, (char*)0); + return TCL_ERROR; + } + rc = aExtension[i].pInit(db, &zErrMsg, 0); + if( rc!=SQLITE_OK || zErrMsg ){ + Tcl_AppendResult(interp, "initialization of ", zName, " failed: ", zErrMsg, + (char*)0); + sqlite3_free(zErrMsg); + return TCL_ERROR; + } } return TCL_OK; } diff --git a/test/spellfix.test b/test/spellfix.test index dbe8bd60e9..dfa487a1b0 100644 --- a/test/spellfix.test +++ b/test/spellfix.test @@ -16,7 +16,7 @@ set testprefix spellfix ifcapable !vtab { finish_test ; return } -load_static_extension db spellfix +load_static_extension db spellfix nextchar set vocab { rabbi rabbit rabbits rabble rabid rabies raccoon raccoons race raced racer @@ -84,6 +84,26 @@ foreach {tn word res} { } $res } +# Tests of the next_char function. +# +do_test 1.10 { + db eval { + CREATE TABLE vocab(w TEXT PRIMARY KEY); + INSERT INTO vocab SELECT word FROM t1; + } +} {} +do_execsql_test 1.11 { + SELECT next_char('re','vocab','w'); +} {a} +do_execsql_test 1.12 { + SELECT next_char('r','vocab','w'); +} {ae} +do_execsql_test 1.13 { + SELECT next_char('','vocab','w'); +} {r} +do_test 1.14 { + catchsql {SELECT next_char('','xyzzy','a')} +} {1 {no such table: xyzzy}} do_execsql_test 2.1 { CREATE VIRTUAL TABLE t2 USING spellfix1; From aa29c86e49377af479d697e4e046d55a3cf09265 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 25 Apr 2013 20:34:02 +0000 Subject: [PATCH 168/204] Rebalance FTS expressions after parsing to limit recursion during evaluation. Avoid recursion when deleting FTS expression trees. Enforce a limit on the depth of an expression tree. FossilOrigin-Name: f968d43f80cc2f236e7d09ba1e8278343e2b6976 --- ext/fts3/fts3_expr.c | 323 ++++++++++++++++++++++++++++++++++++----- manifest | 20 ++- manifest.uuid | 2 +- test/fts3expr3.test | 202 ++++++++++++++++++++++++++ test/permutations.test | 1 + 5 files changed, 504 insertions(+), 44 deletions(-) create mode 100644 test/fts3expr3.test diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 04f38483a3..c767d7e71e 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -640,8 +640,10 @@ static int fts3ExprParse( } pNot->eType = FTSQUERY_NOT; pNot->pRight = p; + p->pParent = pNot; if( pNotBranch ){ pNot->pLeft = pNotBranch; + pNotBranch->pParent = pNot; } pNotBranch = pNot; p = pPrev; @@ -729,6 +731,7 @@ static int fts3ExprParse( pIter = pIter->pLeft; } pIter->pLeft = pRet; + pRet->pParent = pIter; pRet = pNotBranch; } } @@ -745,6 +748,223 @@ exprparse_out: return rc; } +/* +** Return SQLITE_ERROR if the maximum depth of the expression tree passed +** as the only argument is more than nMaxDepth. +*/ +static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){ + int rc = SQLITE_OK; + if( p ){ + if( nMaxDepth==0 ){ + rc = SQLITE_ERROR; + }else{ + rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(p->pRight, nMaxDepth-1); + } + } + } + return rc; +} + +/* +** This function attempts to transform the expression tree at (*pp) to +** an equivalent but more balanced form. The tree is modified in place. +** If successful, SQLITE_OK is returned and (*pp) set to point to the +** new root expression node. +** +** nMaxDepth is the maximum allowable depth of the balanced sub-tree. +** +** Otherwise, if an error occurs, an SQLite error code is returned and +** expression (*pp) freed. +*/ +static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ + int rc = SQLITE_OK; /* Return code */ + Fts3Expr *pRoot = *pp; /* Initial root node */ + Fts3Expr *pFree = 0; /* List of free nodes. Linked by pParent. */ + int eType = pRoot->eType; /* Type of node in this tree */ + + if( nMaxDepth==0 ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK && (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){ + Fts3Expr **apLeaf; + apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth); + if( 0==apLeaf ){ + rc = SQLITE_NOMEM; + }else{ + memset(apLeaf, 0, sizeof(Fts3Expr *) * nMaxDepth); + } + + if( rc==SQLITE_OK ){ + int i; + Fts3Expr *p; + + /* Set $p to point to the left-most leaf in the tree of eType nodes. */ + for(p=pRoot; p->eType==eType; p=p->pLeft){ + assert( p->pParent==0 || p->pParent->pLeft==p ); + assert( p->pLeft && p->pRight ); + } + + /* This loop runs once for each leaf in the tree of eType nodes. */ + while( 1 ){ + int iLvl; + Fts3Expr *pParent = p->pParent; /* Current parent of p */ + + assert( pParent==0 || pParent->pLeft==p ); + p->pParent = 0; + if( pParent ){ + pParent->pLeft = 0; + }else{ + pRoot = 0; + } + rc = fts3ExprBalance(&p, nMaxDepth-1); + if( rc!=SQLITE_OK ) break; + + for(iLvl=0; p && iLvlpLeft = apLeaf[iLvl]; + pFree->pRight = p; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + apLeaf[iLvl] = 0; + } + } + if( p ){ + sqlite3Fts3ExprFree(p); + rc = SQLITE_ERROR; + break; + } + + /* If that was the last leaf node, break out of the loop */ + if( pParent==0 ) break; + + /* Set $p to point to the next leaf in the tree of eType nodes */ + for(p=pParent->pRight; p->eType==eType; p=p->pLeft); + + /* Remove pParent from the original tree. */ + assert( pParent->pParent==0 || pParent->pParent->pLeft==pParent ); + pParent->pRight->pParent = pParent->pParent; + if( pParent->pParent ){ + pParent->pParent->pLeft = pParent->pRight; + }else{ + assert( pParent==pRoot ); + pRoot = pParent->pRight; + } + + /* Link pParent into the free node list. It will be used as an + ** internal node of the new tree. */ + pParent->pParent = pFree; + pFree = pParent; + } + + if( rc==SQLITE_OK ){ + p = 0; + for(i=0; ipParent = 0; + }else{ + pFree->pRight = p; + pFree->pLeft = apLeaf[i]; + pFree->pLeft->pParent = pFree; + pFree->pRight->pParent = pFree; + + p = pFree; + pFree = pFree->pParent; + p->pParent = 0; + } + } + } + pRoot = p; + }else{ + /* An error occurred. Delete the contents of the apLeaf[] array + ** and pFree list. Everything else is cleaned up by the call to + ** sqlite3Fts3ExprFree(pRoot) below. */ + Fts3Expr *pDel; + for(i=0; ipParent; + sqlite3_free(pDel); + } + } + + assert( pFree==0 ); + sqlite3_free( apLeaf ); + } + } + + if( rc!=SQLITE_OK ){ + sqlite3Fts3ExprFree(pRoot); + pRoot = 0; + } + *pp = pRoot; + return rc; +} + +/* +** This function is similar to sqlite3Fts3ExprParse(), with the following +** differences: +** +** 1. It does not do expression rebalancing. +** 2. It does not check that the expression does not exceed the +** maximum allowable depth. +** 3. Even if it fails, *ppExpr may still be set to point to an +** expression tree. It should be deleted using sqlite3Fts3ExprFree() +** in this case. +*/ +static int fts3ExprParseUnbalanced( + sqlite3_tokenizer *pTokenizer, /* Tokenizer module */ + int iLangid, /* Language id for tokenizer */ + char **azCol, /* Array of column names for fts3 table */ + int bFts4, /* True to allow FTS4-only syntax */ + int nCol, /* Number of entries in azCol[] */ + int iDefaultCol, /* Default column to query */ + const char *z, int n, /* Text of MATCH query */ + Fts3Expr **ppExpr /* OUT: Parsed query structure */ +){ + static const int MAX_EXPR_DEPTH = 12; + int nParsed; + int rc; + ParseContext sParse; + + memset(&sParse, 0, sizeof(ParseContext)); + sParse.pTokenizer = pTokenizer; + sParse.iLangid = iLangid; + sParse.azCol = (const char **)azCol; + sParse.nCol = nCol; + sParse.iDefaultCol = iDefaultCol; + sParse.bFts4 = bFts4; + if( z==0 ){ + *ppExpr = 0; + return SQLITE_OK; + } + if( n<0 ){ + n = (int)strlen(z); + } + rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed); + assert( rc==SQLITE_OK || *ppExpr==0 ); + + /* Check for mismatched parenthesis */ + if( rc==SQLITE_OK && sParse.nNest ){ + rc = SQLITE_ERROR; + } + + return rc; +} + /* ** Parameters z and n contain a pointer to and length of a buffer containing ** an fts3 query expression, respectively. This function attempts to parse the @@ -779,29 +999,20 @@ int sqlite3Fts3ExprParse( const char *z, int n, /* Text of MATCH query */ Fts3Expr **ppExpr /* OUT: Parsed query structure */ ){ - int nParsed; - int rc; - ParseContext sParse; - - memset(&sParse, 0, sizeof(ParseContext)); - sParse.pTokenizer = pTokenizer; - sParse.iLangid = iLangid; - sParse.azCol = (const char **)azCol; - sParse.nCol = nCol; - sParse.iDefaultCol = iDefaultCol; - sParse.bFts4 = bFts4; - if( z==0 ){ - *ppExpr = 0; - return SQLITE_OK; + static const int MAX_EXPR_DEPTH = 12; + int rc = fts3ExprParseUnbalanced( + pTokenizer, iLangid, azCol, bFts4, nCol, iDefaultCol, z, n, ppExpr + ); + + /* Rebalance the expression. And check that its depth does not exceed + ** MAX_EXPR_DEPTH. */ + if( rc==SQLITE_OK && *ppExpr ){ + rc = fts3ExprBalance(ppExpr, MAX_EXPR_DEPTH); + if( rc==SQLITE_OK ){ + rc = fts3ExprCheckDepth(*ppExpr, MAX_EXPR_DEPTH); + } } - if( n<0 ){ - n = (int)strlen(z); - } - rc = fts3ExprParse(&sParse, z, n, ppExpr, &nParsed); - - /* Check for mismatched parenthesis */ - if( rc==SQLITE_OK && sParse.nNest ){ - rc = SQLITE_ERROR; + if( rc!=SQLITE_OK ){ sqlite3Fts3ExprFree(*ppExpr); *ppExpr = 0; } @@ -810,16 +1021,40 @@ int sqlite3Fts3ExprParse( } /* -** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse(). +** Free a single node of an expression tree. */ -void sqlite3Fts3ExprFree(Fts3Expr *p){ - if( p ){ - assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); - sqlite3Fts3ExprFree(p->pLeft); - sqlite3Fts3ExprFree(p->pRight); - sqlite3Fts3EvalPhraseCleanup(p->pPhrase); - sqlite3_free(p->aMI); - sqlite3_free(p); +static void fts3FreeExprNode(Fts3Expr *p){ + assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); + sqlite3Fts3EvalPhraseCleanup(p->pPhrase); + sqlite3_free(p->aMI); + sqlite3_free(p); +} + +/* +** Free a parsed fts3 query expression allocated by sqlite3Fts3ExprParse(). +** +** This function would be simpler if it recursively called itself. But +** that would mean passing a sufficiently large expression to ExprParse() +** could cause a stack overflow. +*/ +void sqlite3Fts3ExprFree(Fts3Expr *pDel){ + Fts3Expr *p; + assert( pDel==0 || pDel->pParent==0 ); + for(p=pDel; p && (p->pLeft||p->pRight); p=(p->pLeft ? p->pLeft : p->pRight)){ + assert( p->pParent==0 || p==p->pParent->pRight || p==p->pParent->pLeft ); + } + while( p ){ + Fts3Expr *pParent = p->pParent; + fts3FreeExprNode(p); + if( pParent && p==pParent->pLeft && pParent->pRight ){ + p = pParent->pRight; + while( p && (p->pLeft || p->pRight) ){ + assert( p==p->pParent->pRight || p==p->pParent->pLeft ); + p = (p->pLeft ? p->pLeft : p->pRight); + } + }else{ + p = pParent; + } } } @@ -871,6 +1106,9 @@ static int queryTestTokenizer( ** the returned expression text and then freed using sqlite3_free(). */ static char *exprToString(Fts3Expr *pExpr, char *zBuf){ + if( pExpr==0 ){ + return sqlite3_mprintf(""); + } switch( pExpr->eType ){ case FTSQUERY_PHRASE: { Fts3Phrase *pPhrase = pExpr->pPhrase; @@ -978,10 +1216,19 @@ static void fts3ExprTest( azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]); } - rc = sqlite3Fts3ExprParse( - pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr - ); + if( sqlite3_user_data(context) ){ + rc = sqlite3Fts3ExprParse( + pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr + ); + assert( rc==SQLITE_OK || pExpr==0 ); + }else{ + rc = fts3ExprParseUnbalanced( + pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr + ); + } + if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM ){ + sqlite3Fts3ExprFree(pExpr); sqlite3_result_error(context, "Error parsing expression", -1); }else if( rc==SQLITE_NOMEM || !(zBuf = exprToString(pExpr, 0)) ){ sqlite3_result_error_nomem(context); @@ -1004,9 +1251,15 @@ exprtest_out: ** with database connection db. */ int sqlite3Fts3ExprInitTestInterface(sqlite3* db){ - return sqlite3_create_function( + int rc = sqlite3_create_function( db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0 ); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", + -1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0 + ); + } + return rc; } #endif diff --git a/manifest b/manifest index f702eb82f2..3441db13bd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings. -D 2013-04-22T23:38:50.007 +C Rebalance\sFTS\sexpressions\safter\sparsing\sto\slimit\srecursion\sduring\sevaluation.\sAvoid\srecursion\swhen\sdeleting\sFTS\sexpression\strees.\sEnforce\sa\slimit\son\sthe\sdepth\sof\san\sexpression\stree. +D 2013-04-25T20:34:02.667 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 87591ea5bf7d6ed521ad42d5bc69c124debe11a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -59,7 +59,7 @@ F ext/fts3/fts3.c 784aadfb4c2a217c3eb1feaecac924989f29728f F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 352c8a83ee4c6a14ced1759a39dd890ab947cbe0 F ext/fts3/fts3_aux.c b02632f6dd0e375ce97870206d914ea6d8df5ccd -F ext/fts3/fts3_expr.c 6cb4410f87676ae633bd7923bbc78526cb839c4d +F ext/fts3/fts3_expr.c a01c0a2e00d0e848f7cdda3b4114fe03b560f59b F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5 @@ -494,6 +494,7 @@ F test/fts3drop.test 1b906e293d6773812587b3dc458cb9e8f3f0c297 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a +F test/fts3expr3.test aed4e71aa40ac1dcba860d1c43e7e23aa812eb1c F test/fts3fault.test cb72dccb0a3b9f730f16c5240f3fcb9303eb1660 F test/fts3fault2.test 3198eef2804deea7cac8403e771d9cbcb752d887 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 @@ -670,7 +671,7 @@ F test/pageropt.test 6b8f6a123a5572c195ad4ae40f2987007923bbd6 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test d2918e9ee6d0008292e5a70e90eed23b0440ca88 +F test/permutations.test 3d0bab9c49c1ec08b868059e30a3e1956f2162e2 F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -1054,7 +1055,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 49cfa14fceeef2d55b449eb927c283ce6f650c07 -R 83c630a254d9cc209b10e1a83d85002b -U drh -Z a7a8588b07b6fce1e57eced049fe41f1 +P 1a1cf5aa86734c832d845e07780262a178188d56 +R 821db2ca4e9de1c38e42323dffba1c12 +T *branch * fts3-expr-rebalance +T *sym-fts3-expr-rebalance * +T -sym-trunk * +U dan +Z 130034041e106884ea2e83109b05e05e diff --git a/manifest.uuid b/manifest.uuid index fa38fa5b2c..7824179050 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a1cf5aa86734c832d845e07780262a178188d56 \ No newline at end of file +f968d43f80cc2f236e7d09ba1e8278343e2b6976 \ No newline at end of file diff --git a/test/fts3expr3.test b/test/fts3expr3.test new file mode 100644 index 0000000000..51a6adf58e --- /dev/null +++ b/test/fts3expr3.test @@ -0,0 +1,202 @@ +# 2009 January 1 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the part of the FTS3 expression +# parser that rebalances large expressions. +# +# $Id: fts3expr2.test,v 1.2 2009/06/05 17:09:12 drh Exp $ +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set ::testprefix fts3expr3 + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +set sqlite_fts3_enable_parentheses 1 + +proc strip_phrase_data {L} { + if {[lindex $L 0] eq "PHRASE"} { + return [list P [lrange $L 3 end]] + } + return [list \ + [lindex $L 0] \ + [strip_phrase_data [lindex $L 1]] \ + [strip_phrase_data [lindex $L 2]] \ + ] +} +proc test_fts3expr2 {expr} { + strip_phrase_data [ + db one {SELECT fts3_exprtest_rebalance('simple', $expr, 'a', 'b', 'c')} + ] +} + +proc balanced_exprtree_structure {nEntry} { + set L [list] + for {set i 1} {$i <= $nEntry} {incr i} { + lappend L xxx + } + while {[llength $L] > 1} { + set N [list] + if {[llength $L] % 2} { + foreach {a b} [lrange $L 0 end-1] { lappend N [list AND $a $b] } + lappend N [lindex $L end] + } else { + foreach {a b} $L { lappend N [list AND $a $b] } + } + set L $N + } + return [lindex $L 0] +} + +proc balanced_and_tree {nEntry} { + set query [balanced_exprtree_structure $nEntry] + if {$query == "xxx"} { + return "P 1" + } + for {set i 1} {$i <= $nEntry} {incr i} { + regsub xxx $query "{P $i}" query + } + return $query +} + +proc random_tree_structure {nEntry bParen op} { + set query xxx + for {set i 1} {$i < $nEntry} {incr i} { + set x1 [expr int(rand()*4.0)] + set x2 [expr int(rand()*2.0)] + if {$x1==0 && $bParen} { + set query "($query)" + } + if {$x2} { + set query "xxx $op $query" + } else { + set query "$query $op xxx" + } + } + return $query +} + +proc random_and_query {nEntry {bParen 0}} { + set query [random_tree_structure $nEntry $bParen AND] + for {set i 1} {$i <= $nEntry} {incr i} { + regsub xxx $query $i query + } + return $query +} + +proc random_or_query {nEntry} { + set query [random_tree_structure $nEntry 1 OR] + for {set i 1} {$i <= $nEntry} {incr i} { + regsub xxx $query $i query + } + return $query +} + +proc random_andor_query {nEntry} { + set query [random_tree_structure $nEntry 1 AND] + for {set i 1} {$i <= $nEntry} {incr i} { + regsub xxx $query "([random_or_query $nEntry])" query + } + return $query +} + +proc balanced_andor_tree {nEntry} { + set tree [balanced_exprtree_structure $nEntry] + set node "{[balanced_and_tree $nEntry]}" + regsub -all AND $node OR node + regsub -all xxx $tree $node tree + return $tree +} + +# Test that queries like "1 AND 2 AND 3 AND 4..." are transformed to +# balanced trees by FTS. +# +for {set i 1} {$i < 100} {incr i} { + do_test 1.$i { + test_fts3expr2 [random_and_query $i] + } [balanced_and_tree $i] +} + +# Same again, except with parenthesis inserted at arbitrary points. +# +for {set i 1} {$i < 100} {incr i} { + do_test 2.$i { + test_fts3expr2 [random_and_query $i 1] + } [balanced_and_tree $i] +} + +# Now attempt to balance two AND trees joined by an OR. +# +for {set i 1} {$i < 100} {incr i} { + do_test 3.$i { + test_fts3expr2 "[random_and_query $i 1] OR [random_and_query $i 1]" + } [list OR [balanced_and_tree $i] [balanced_and_tree $i]] +} + +# Try trees of AND nodes with leaves that are themselves trees of OR nodes. +# +for {set i 2} {$i < 32} {incr i} { + do_test 3.$i { + test_fts3expr2 [random_andor_query $i] + } [balanced_andor_tree $i] +} + +# These exceed the depth limit. +# +for {set i 33} {$i < 40} {incr i} { + do_test 3.$i { + list [catch {test_fts3expr2 [random_andor_query $i]} msg] $msg + } {1 {Error parsing expression}} +} + +# This also exceeds the depth limit. +# +do_test 4.1 { + set q "1" + for {set i 2} {$i < 5000} {incr i} { + append q " AND $i" + } + list [catch {test_fts3expr2 $q} msg] $msg +} {1 {Error parsing expression}} + +proc create_toggle_tree {nDepth} { + if {$nDepth == 0} { return xxx } + set nNew [expr $nDepth-1] + if {$nDepth % 2} { + return "([create_toggle_tree $nNew]) OR ([create_toggle_tree $nNew])" + } + return "([create_toggle_tree $nNew]) AND ([create_toggle_tree $nNew])" +} + +do_test 4.2 { + list [catch {test_fts3expr2 [create_toggle_tree 17]} msg] $msg +} {1 {Error parsing expression}} + +set query [random_andor_query 12] +set result [balanced_andor_tree 12] +do_faultsim_test fts3expr3-fault-1 -faults oom-* -body { + test_fts3expr2 $::query +} -test { + faultsim_test_result [list 0 $::result] +} + +set sqlite_fts3_enable_parentheses 0 +finish_test + + + + diff --git a/test/permutations.test b/test/permutations.test index 9334fc49e6..3a3be27aff 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -186,6 +186,7 @@ test_suite "fts3" -prefix "" -description { fts3ak.test fts3al.test fts3am.test fts3an.test fts3ao.test fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test + fts3expr3.test fts3near.test fts3query.test fts3shared.test fts3snippet.test fts3sort.test fts3fault.test fts3malloc.test fts3matchinfo.test From 2bba8c249a154a3c8e38032169acfe29a3a5bf86 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 26 Apr 2013 12:08:29 +0000 Subject: [PATCH 169/204] Reduce the default SQLITE_MAX_MMAP_SIZE slightly so that it fits in a signed 32-bit integer. FossilOrigin-Name: 460752b8575320163d2659bb7ff24aff41e2bb66 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqliteInt.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a3d9885853..41113c30a7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Added\sthe\snextchar.c\sextension.\s\sMinor\schanges\sto\sthe\sspellfix.c\sextension\nso\sthat\sit\scan\sbe\sappended\sto\san\samalgamation\sand\scompiled\swithout\sduplicating\nsymbols. -D 2013-04-25T19:31:33.149 +C Reduce\sthe\sdefault\sSQLITE_MAX_MMAP_SIZE\sslightly\sso\sthat\sit\sfits\sin\sa\nsigned\s32-bit\sinteger. +D 2013-04-26T12:08:29.821 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -196,7 +196,7 @@ F src/shell.c 5d527e5d08f05ec2c43ff194ea44bf62b974f4c9 F src/sqlite.h.in ec279b782bea05db63b8b29481f9642b406004af F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h 3585ea1bb8776baa9f2675e9ef3d9170e7aeda29 +F src/sqliteInt.h de835c584032769461c123a564381f9808542c0e F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1061,7 +1061,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P aabeea98f53edde68f484f1794ae70789dac3889 -R 77b136ceb7e3bfd54c75649b938a3e12 +P 56b9a417f5451631f11c5206d625f11472ee65f9 +R 83175c7cd0d9ce60e1daa5a05331eb65 U drh -Z fb4502a56cfa201ec6c890343c386675 +Z 44151e6d508d016989d25b422911b94b diff --git a/manifest.uuid b/manifest.uuid index 7c24934462..8936a5bdc4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -56b9a417f5451631f11c5206d625f11472ee65f9 \ No newline at end of file +460752b8575320163d2659bb7ff24aff41e2bb66 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 876629a8c8..b041b3fb6e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -563,7 +563,7 @@ extern const int sqlite3one; || defined(_WIN32) \ || (defined(__APPLE__) && defined(__MACH__)) \ || defined(__sun) -# define SQLITE_MAX_MMAP_SIZE 2147483648 +# define SQLITE_MAX_MMAP_SIZE 0x7fff0000 /* 2147418112 */ # else # define SQLITE_MAX_MMAP_SIZE 0 # endif From 6f77140f78e7c871cdbdff52639cb9a850159f21 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 26 Apr 2013 13:14:19 +0000 Subject: [PATCH 170/204] Fix harmless compiler warnings in the FTS expression parser. FossilOrigin-Name: 3c78af8c535e16518f18733325f4cd1df7ec8282 --- ext/fts3/fts3_expr.c | 3 +-- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index c767d7e71e..b28ab68e4b 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -895,7 +895,7 @@ static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ for(i=0; ipParent; sqlite3_free(pDel); } @@ -935,7 +935,6 @@ static int fts3ExprParseUnbalanced( const char *z, int n, /* Text of MATCH query */ Fts3Expr **ppExpr /* OUT: Parsed query structure */ ){ - static const int MAX_EXPR_DEPTH = 12; int nParsed; int rc; ParseContext sParse; diff --git a/manifest b/manifest index 31e42e1348..e150e60d93 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges. -D 2013-04-26T06:58:06.233 +C Fix\sharmless\scompiler\swarnings\sin\sthe\sFTS\sexpression\sparser. +D 2013-04-26T13:14:19.396 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -59,7 +59,7 @@ F ext/fts3/fts3.c 784aadfb4c2a217c3eb1feaecac924989f29728f F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 352c8a83ee4c6a14ced1759a39dd890ab947cbe0 F ext/fts3/fts3_aux.c b02632f6dd0e375ce97870206d914ea6d8df5ccd -F ext/fts3/fts3_expr.c a01c0a2e00d0e848f7cdda3b4114fe03b560f59b +F ext/fts3/fts3_expr.c 4021d21aadebb50499c482f6b7b718e4b93113e6 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5 @@ -86,13 +86,13 @@ F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/misc/amatch.c 3369b2b544066e620d986f0085d039c77d1ef17f F ext/misc/closure.c fec0c8537c69843e0b7631d500a14c0527962cd6 F ext/misc/editdist3.wiki 06100a0c558921a563cbc40e0d0151902b1eef6d -F ext/misc/fuzzer.c fb64a15af978ae73fa9075b9b1dfbe82b8defc6f w src/test_fuzzer.c +F ext/misc/fuzzer.c fb64a15af978ae73fa9075b9b1dfbe82b8defc6f F ext/misc/ieee754.c 2565ce373d842977efe0922dc50b8a41b3289556 F ext/misc/nextchar.c 1131e2b36116ffc6fe6b2e3464bfdace27978b1e -F ext/misc/regexp.c c25c65fe775f5d9801fb8573e36ebe73f2c0c2e0 w src/test_regexp.c -F ext/misc/spellfix.c e323eebb877d735bc64404c16a6d758ab17a0b7a w src/test_spellfix.c +F ext/misc/regexp.c c25c65fe775f5d9801fb8573e36ebe73f2c0c2e0 +F ext/misc/spellfix.c e323eebb877d735bc64404c16a6d758ab17a0b7a F ext/misc/spellfix1.wiki dd1830444c14cf0f54dd680cc044df2ace2e9d09 -F ext/misc/wholenumber.c ce362368b9381ea48cbd951ade8df867eeeab014 w src/test_wholenumber.c +F ext/misc/wholenumber.c ce362368b9381ea48cbd951ade8df867eeeab014 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 757abea591d4ff67c0ff4e8f9776aeda86b18c14 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1062,7 +1062,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f968d43f80cc2f236e7d09ba1e8278343e2b6976 56b9a417f5451631f11c5206d625f11472ee65f9 -R 70b7c4a8fc1b35fd998a4752a3b57b39 -U dan -Z 98b8c0af1d54867f961d6716e238405d +P 4d08e74d34e82f3be588049c9576a5c1008435e7 +R a62e327052722b023f8b56bfc3226b16 +U drh +Z 0ddc594c1b1b9290fe656a6228646bb0 diff --git a/manifest.uuid b/manifest.uuid index 6ecc568665..157b4bd510 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4d08e74d34e82f3be588049c9576a5c1008435e7 \ No newline at end of file +3c78af8c535e16518f18733325f4cd1df7ec8282 \ No newline at end of file From 9ea88b2b484b0d80ccc0e194e70a03c545f3fdf5 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 26 Apr 2013 15:55:57 +0000 Subject: [PATCH 171/204] Update documentation with new hyperlinks. No changes to code. FossilOrigin-Name: 640eb54ad6aac9bc7109cba167389a9bcec3f21e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 12 +++++++++--- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 914f1bde6e..59fe24e66e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rebalance\sFTS\sexpressions\safter\sparsing\sto\slimit\srecursion\sduring\sevaluation.\sAvoid\srecursion\swhen\sdeleting\sFTS\sexpression\strees.\sEnforce\sa\slimit\s(currently\s12)\son\sthe\sdepth\sof\san\sexpression\stree. -D 2013-04-26T14:13:15.156 +C Update\sdocumentation\swith\snew\shyperlinks.\s\sNo\schanges\sto\scode. +D 2013-04-26T15:55:57.095 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -193,7 +193,7 @@ F src/resolve.c 83cc2d942ee216bc56956c6e6fadb691c1727fa1 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 6bfbe11e2fef81c5e18d30513ab6c69f171667eb F src/shell.c 5d527e5d08f05ec2c43ff194ea44bf62b974f4c9 -F src/sqlite.h.in ec279b782bea05db63b8b29481f9642b406004af +F src/sqlite.h.in f5efcffff16619129bcb92ac352a77fa758f86c1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 F src/sqliteInt.h de835c584032769461c123a564381f9808542c0e @@ -1062,7 +1062,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 460752b8575320163d2659bb7ff24aff41e2bb66 2648966f17bc1b783ef6d3b2368c613f6e02945e -R 3adacbaf5ab67ae1f94478e13972158e -U dan -Z 40f0df5151b2a7d6fa4a01e133941564 +P 49d23ef61f9ce2ffe13237b51a0e01b0b46ba96b +R 9dfdc1bbf9c2e5197fa469091ec011ea +U drh +Z 247eb74d6133c2eb692e1fce731d0d35 diff --git a/manifest.uuid b/manifest.uuid index 64a0465053..de2ac0cb31 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -49d23ef61f9ce2ffe13237b51a0e01b0b46ba96b \ No newline at end of file +640eb54ad6aac9bc7109cba167389a9bcec3f21e \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 3265b3196b..dd5f4d9bae 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1587,7 +1587,9 @@ struct sqlite3_mem_methods { ** page cache implementation into that object.)^
    • ** ** [[SQLITE_CONFIG_LOG]]
    SQLITE_CONFIG_LOG
    -**
    ^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a +**
    The SQLITE_CONFIG_LOG option is used to configure the SQLite +** global [error log]. +** [^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a ** function with a call signature of void(*)(void*,int,const char*), ** and a pointer to void. ^If the function pointer is not NULL, it is ** invoked by [sqlite3_log()] to process each logging event. ^If the @@ -2522,6 +2524,9 @@ int sqlite3_set_authorizer( ** as each triggered subprogram is entered. The callbacks for triggers ** contain a UTF-8 SQL comment that identifies the trigger.)^ ** +** The [SQLITE_TRACE_SIZE_LIMIT] compile-time option can be used to limit +** the length of [bound parameter] expansion in the output of sqlite3_trace(). +** ** ^The callback function registered by sqlite3_profile() is invoked ** as each SQL statement finishes. ^The profile callback contains ** the original statement text and an estimate of wall-clock time @@ -3060,7 +3065,8 @@ int sqlite3_limit(sqlite3*, int id, int newVal); **
  • ** ^If the database schema changes, instead of returning [SQLITE_SCHEMA] as it ** always used to do, [sqlite3_step()] will automatically recompile the SQL -** statement and try to run it again. +** statement and try to run it again. As many as [SQLITE_MAX_SCHEMA_RETRY] +** retries will occur before sqlite3_step() gives up and returns an error. **
    • ** **
  • @@ -6872,7 +6878,7 @@ int sqlite3_strglob(const char *zGlob, const char *zStr); /* ** CAPI3REF: Error Logging Interface ** -** ^The [sqlite3_log()] interface writes a message into the error log +** ^The [sqlite3_log()] interface writes a message into the [error log] ** established by the [SQLITE_CONFIG_LOG] option to [sqlite3_config()]. ** ^If logging is enabled, the zFormat string and subsequent arguments are ** used with [sqlite3_snprintf()] to generate the final output string. From 47a2b4a0b0067bcae3960bb07e616589d509def2 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 26 Apr 2013 16:09:29 +0000 Subject: [PATCH 172/204] Avoid using posix_fallocate() in WAL mode, as it is not supported by all file-systems. FossilOrigin-Name: 9c7523dabf4aee609287732ce787c9b9a9087e7f --- manifest | 15 ++++--- manifest.uuid | 2 +- src/os_unix.c | 115 +++++++++++++++++++++++++++++--------------------- 3 files changed, 78 insertions(+), 54 deletions(-) diff --git a/manifest b/manifest index 914f1bde6e..72d6bee5a4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rebalance\sFTS\sexpressions\safter\sparsing\sto\slimit\srecursion\sduring\sevaluation.\sAvoid\srecursion\swhen\sdeleting\sFTS\sexpression\strees.\sEnforce\sa\slimit\s(currently\s12)\son\sthe\sdepth\sof\san\sexpression\stree. -D 2013-04-26T14:13:15.156 +C Avoid\susing\sposix_fallocate()\sin\sWAL\smode,\sas\sit\sis\snot\ssupported\sby\sall\sfile-systems. +D 2013-04-26T16:09:29.947 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -177,7 +177,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 5a9ac4a566fb566a2ff9b9e3a9d723075d9d26a7 +F src/os_unix.c f86cd628ffb9ccf3adcc5c15b02c00a64eaf598e F src/os_win.c 673b3e3d1fa3040d8d95a7f1f5e0e553aed56cfb F src/pager.c 6c3a8a5d665498b0344395a2c9f82d5abc4cc771 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1062,7 +1062,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 460752b8575320163d2659bb7ff24aff41e2bb66 2648966f17bc1b783ef6d3b2368c613f6e02945e -R 3adacbaf5ab67ae1f94478e13972158e +P 49d23ef61f9ce2ffe13237b51a0e01b0b46ba96b +R c26b6b9eda44a997204fb8564cbd02a4 +T *branch * avoid-fallocate +T *sym-avoid-fallocate * +T -sym-trunk * U dan -Z 40f0df5151b2a7d6fa4a01e133941564 +Z 8ca670519b4ace84695b56a8d0f2a643 diff --git a/manifest.uuid b/manifest.uuid index 64a0465053..a357f425fc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -49d23ef61f9ce2ffe13237b51a0e01b0b46ba96b \ No newline at end of file +9c7523dabf4aee609287732ce787c9b9a9087e7f \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 7448afe4c7..1200ff352b 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3193,6 +3193,51 @@ static int unixRead( } } +/* +** Attempt to seek the file-descriptor passed as the first argument to +** absolute offset iOff, then attempt to write nBuf bytes of data from +** pBuf to it. If an error occurs, return -1 and set *piErrno. Otherwise, +** return the actual number of bytes written (which may be less than +** nBuf). +*/ +static int seekAndWriteFd( + int fd, /* File descriptor to write to */ + i64 iOff, /* File offset to begin writing at */ + const void *pBuf, /* Copy data from this buffer to the file */ + int nBuf, /* Size of buffer pBuf in bytes */ + int *piErrno /* OUT: Error number if error occurs */ +){ + int rc = 0; /* Value returned by system call */ + + assert( nBuf==(nBuf&0x1ffff) ); + nBuf &= 0x1ffff; + TIMER_START; + +#if defined(USE_PREAD) + do{ rc = osPwrite(fd, pBuf, nBuf, iOff); }while( rc<0 && errno==EINTR ); +#elif defined(USE_PREAD64) + do{ rc = osPwrite64(fd, pBuf, nBuf, iOff);}while( rc<0 && errno==EINTR); +#else + do{ + i64 iSeek = lseek(fd, iOff, SEEK_SET); + SimulateIOError( iSeek-- ); + + if( iSeek!=iOff ){ + if( piErrno ) *piErrno = (iSeek==-1 ? errno : 0); + return -1; + } + rc = osWrite(fd, pBuf, nBuf); + }while( rc<0 && errno==EINTR ); +#endif + + TIMER_END; + OSTRACE(("WRITE %-3d %5d %7lld %llu\n", fd, rc, iOff, TIMER_ELAPSED)); + + if( rc<0 && piErrno ) *piErrno = errno; + return rc; +} + + /* ** Seek to the offset in id->offset then read cnt bytes into pBuf. ** Return the number of bytes actually read. Update the offset. @@ -3201,39 +3246,7 @@ static int unixRead( ** is set before returning. */ static int seekAndWrite(unixFile *id, i64 offset, const void *pBuf, int cnt){ - int got; -#if (!defined(USE_PREAD) && !defined(USE_PREAD64)) - i64 newOffset; -#endif - assert( cnt==(cnt&0x1ffff) ); - cnt &= 0x1ffff; - TIMER_START; -#if defined(USE_PREAD) - do{ got = osPwrite(id->h, pBuf, cnt, offset); }while( got<0 && errno==EINTR ); -#elif defined(USE_PREAD64) - do{ got = osPwrite64(id->h, pBuf, cnt, offset);}while( got<0 && errno==EINTR); -#else - do{ - newOffset = lseek(id->h, offset, SEEK_SET); - SimulateIOError( newOffset-- ); - if( newOffset!=offset ){ - if( newOffset == -1 ){ - ((unixFile*)id)->lastErrno = errno; - }else{ - ((unixFile*)id)->lastErrno = 0; - } - return -1; - } - got = osWrite(id->h, pBuf, cnt); - }while( got<0 && errno==EINTR ); -#endif - TIMER_END; - if( got<0 ){ - ((unixFile*)id)->lastErrno = errno; - } - - OSTRACE(("WRITE %-3d %5d %7lld %llu\n", id->h, got, offset, TIMER_ELAPSED)); - return got; + return seekAndWriteFd(id->h, offset, pBuf, cnt, &id->lastErrno); } @@ -4322,24 +4335,32 @@ static int unixShmMap( if( sStat.st_sizeh, sStat.st_size, nByte)!=0 ){ - rc = unixLogError(SQLITE_IOERR_SHMSIZE, "fallocate", - pShmNode->zFilename); + if( !bExtend ){ goto shmpage_out; } -#else - if( robust_ftruncate(pShmNode->h, nByte) ){ - rc = unixLogError(SQLITE_IOERR_SHMSIZE, "ftruncate", - pShmNode->zFilename); - goto shmpage_out; + + /* Alternatively, if bExtend is true, extend the file. Do this by + ** writing a single byte to the end of each (OS) page being + ** allocated or extended. Technically, we need only write to the + ** last page in order to extend the file. But writing to all new + ** pages forces the OS to allocate them immediately, which reduces + ** the chances of SIGBUS while accessing the mapped region later on. + */ + else{ + static const int pgsz = 4096; + int iPg; + + /* Write to the last byte of each newly allocated or extended page */ + assert( (nByte % pgsz)==0 ); + for(iPg=(sStat.st_size/pgsz); iPg<(nByte/pgsz); iPg++){ + if( seekAndWriteFd(pShmNode->h, iPg*pgsz + pgsz-1, "", 1, 0)!=1 ){ + const char *zFile = pShmNode->zFilename; + rc = unixLogError(SQLITE_IOERR_SHMSIZE, "write", zFile); + goto shmpage_out; + } + } } -#endif } } From 8e4714b3033abfe8e48ffac1845166de42d4ddc3 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 26 Apr 2013 18:36:58 +0000 Subject: [PATCH 173/204] Avoid unnecessarily reseting the pager cache after committing a transaction that takes advantage of the SQLITE_IOCAP_ATOMIC related optimization. FossilOrigin-Name: c47144e98c0a0f9e09780c945de10c57b6a495ea --- manifest | 14 +++++------ manifest.uuid | 2 +- src/pager.c | 5 ++++ test/io.test | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index de4051d566..c5c2b1f688 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\susing\sposix_fallocate()\sin\sWAL\smode,\sas\sit\sis\snot\ssupported\sby\sall\sfile-systems. -D 2013-04-26T17:00:52.665 +C Avoid\sunnecessarily\sreseting\sthe\spager\scache\safter\scommitting\sa\stransaction\sthat\stakes\sadvantage\sof\sthe\sSQLITE_IOCAP_ATOMIC\srelated\soptimization. +D 2013-04-26T18:36:58.055 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -179,7 +179,7 @@ F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c f86cd628ffb9ccf3adcc5c15b02c00a64eaf598e F src/os_win.c 673b3e3d1fa3040d8d95a7f1f5e0e553aed56cfb -F src/pager.c 6c3a8a5d665498b0344395a2c9f82d5abc4cc771 +F src/pager.c 4a9160d268977e56ae2df90182050ab30fca715d F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -571,7 +571,7 @@ F test/instr.test a34e1d46a9eefb098a7167ef0e730a4a3d82fba0 F test/intarray.test 066b7d7ac38d25bf96f87f1b017bfc687551cdd4 F test/interrupt.test dfe9a67a94b0b2d8f70545ba1a6cca10780d71cc F test/intpkey.test 7af30f6ae852d8d1c2b70e4bf1551946742e92d8 -F test/io.test a4be25a446a99a0604ceecc039ee7363c56e4185 +F test/io.test b90105d295225a712e2e9cb685876101712058b5 F test/ioerr.test 40bb2cfcab63fb6aa7424cd97812a84bc16b5fb8 F test/ioerr2.test 9d71166f8466eda510f1af6137bdabaa82b5408d F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd @@ -1062,7 +1062,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 640eb54ad6aac9bc7109cba167389a9bcec3f21e 9c7523dabf4aee609287732ce787c9b9a9087e7f -R b4ea9ecffc738493e7005d865dad4b9a +P 1bbb4be1a25947f75b2b0c6f368199016b6f7de3 +R 50c4e22fb69fce6bfb1bbec0cac61356 U dan -Z 8914d43d4bd2278afd40d03ae3982d74 +Z 042ff5429ace151fb0c673232d14fee9 diff --git a/manifest.uuid b/manifest.uuid index 86ca24f5f4..58806060f6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1bbb4be1a25947f75b2b0c6f368199016b6f7de3 \ No newline at end of file +c47144e98c0a0f9e09780c945de10c57b6a495ea \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 1c8b5df651..dffcf602d7 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5927,6 +5927,11 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ pPager->aStat[PAGER_STAT_WRITE]++; } if( rc==SQLITE_OK ){ + /* Update the pager's copy of the change-counter. Otherwise, the + ** next time a read transaction is opened the cache will be + ** flushed (as the change-counter values will not match). */ + const void *pCopy = (const void *)&((const char *)zBuf)[24]; + memcpy(&pPager->dbFileVers, pCopy, sizeof(pPager->dbFileVers)); pPager->changeCountDone = 1; } }else{ diff --git a/test/io.test b/test/io.test index bf4d157275..f9583e0e33 100644 --- a/test/io.test +++ b/test/io.test @@ -38,6 +38,10 @@ sqlite3 db test.db -vfs devsym # # io-5.* - Test that the default page size is selected and used # correctly. +# +# io-6.* - Test that the pager-cache is not being flushed unnecessarily +# after a transaction that uses the special atomic-write path +# is committed. # set ::nWrite 0 @@ -565,5 +569,67 @@ foreach {char sectorsize pgsize} { } $pgsize } +#---------------------------------------------------------------------- +# +do_test io-6.1 { + db close + sqlite3_simulate_device -char atomic + forcedelete test.db + sqlite3 db test.db -vfs devsym + execsql { + PRAGMA mmap_size = 0; + PRAGMA page_size = 1024; + CREATE TABLE t1(x); + CREATE TABLE t2(x); + CREATE TABLE t3(x); + CREATE INDEX i3 ON t3(x); + INSERT INTO t3 VALUES(randomblob(100)); + INSERT INTO t3 SELECT randomblob(100) FROM t3; + INSERT INTO t3 SELECT randomblob(100) FROM t3; + INSERT INTO t3 SELECT randomblob(100) FROM t3; + INSERT INTO t3 SELECT randomblob(100) FROM t3; + INSERT INTO t3 SELECT randomblob(100) FROM t3; + INSERT INTO t3 SELECT randomblob(100) FROM t3; + INSERT INTO t3 SELECT randomblob(100) FROM t3; + INSERT INTO t3 SELECT randomblob(100) FROM t3; + INSERT INTO t3 SELECT randomblob(100) FROM t3; + INSERT INTO t3 SELECT randomblob(100) FROM t3; + INSERT INTO t3 SELECT randomblob(100) FROM t3; + } + + db_save_and_close +} {} + +foreach {tn sql} { + 1 { BEGIN; + INSERT INTO t1 VALUES('123'); + INSERT INTO t2 VALUES('456'); + COMMIT; + } + 2 { BEGIN; + INSERT INTO t1 VALUES('123'); + COMMIT; + } +} { + db_restore + sqlite3 db test.db -vfs devsym + execsql { + PRAGMA mmap_size = 0; + SELECT x FROM t3 ORDER BY rowid; + SELECT x FROM t3 ORDER BY x; + } + do_execsql_test 6.2.$tn.1 { PRAGMA integrity_check } {ok} + do_execsql_test 6.2.$tn.2 $sql + + # Corrupt the database file on disk. This should not matter for the + # purposes of the following "PRAGMA integrity_check", as the entire + # database should be cached in the pager-cache. If corruption is + # reported, it indicates that executing $sql caused the pager cache + # to be flushed. Which is a bug. + hexio_write test.db [expr 1024 * 5] [string repeat 00 2048] + do_execsql_test 6.2.$tn.3 { PRAGMA integrity_check } {ok} +} + sqlite3_simulate_device -char {} -sectorsize 0 finish_test + From a13090fee950b1b34abdb532b18315b30758abe1 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 26 Apr 2013 19:33:34 +0000 Subject: [PATCH 174/204] Fix a formatting typo in a comment. No changes to code. FossilOrigin-Name: 7a97226ffe174349e7113340f5354c4e44bd9738 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index c5c2b1f688..0536079cee 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sunnecessarily\sreseting\sthe\spager\scache\safter\scommitting\sa\stransaction\sthat\stakes\sadvantage\sof\sthe\sSQLITE_IOCAP_ATOMIC\srelated\soptimization. -D 2013-04-26T18:36:58.055 +C Fix\sa\sformatting\stypo\sin\sa\scomment.\s\sNo\schanges\sto\scode. +D 2013-04-26T19:33:34.383 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -193,7 +193,7 @@ F src/resolve.c 83cc2d942ee216bc56956c6e6fadb691c1727fa1 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 6bfbe11e2fef81c5e18d30513ab6c69f171667eb F src/shell.c 5d527e5d08f05ec2c43ff194ea44bf62b974f4c9 -F src/sqlite.h.in f5efcffff16619129bcb92ac352a77fa758f86c1 +F src/sqlite.h.in e6c241170da854abf6f15ec7e6c565348b690169 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 F src/sqliteInt.h de835c584032769461c123a564381f9808542c0e @@ -1062,7 +1062,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 1bbb4be1a25947f75b2b0c6f368199016b6f7de3 -R 50c4e22fb69fce6bfb1bbec0cac61356 -U dan -Z 042ff5429ace151fb0c673232d14fee9 +P c47144e98c0a0f9e09780c945de10c57b6a495ea +R 9aa1f84a471b601353c6669ae1266379 +U drh +Z 18431ca135b1f0070cd379da7401fa7c diff --git a/manifest.uuid b/manifest.uuid index 58806060f6..89ec5fbc80 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c47144e98c0a0f9e09780c945de10c57b6a495ea \ No newline at end of file +7a97226ffe174349e7113340f5354c4e44bd9738 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index dd5f4d9bae..5eb74a8756 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1589,7 +1589,7 @@ struct sqlite3_mem_methods { ** [[SQLITE_CONFIG_LOG]]
    SQLITE_CONFIG_LOG
    **
    The SQLITE_CONFIG_LOG option is used to configure the SQLite ** global [error log]. -** [^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a +** (^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a ** function with a call signature of void(*)(void*,int,const char*), ** and a pointer to void. ^If the function pointer is not NULL, it is ** invoked by [sqlite3_log()] to process each logging event. ^If the From c5797545de1b6bd30acc640443069a612e1b6194 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 27 Apr 2013 12:13:29 +0000 Subject: [PATCH 175/204] Untested fix for building on VxWorks. FossilOrigin-Name: f14d55cf358b0392d3b8cd61dc85f43a610a8edf --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 0536079cee..b47b06dfb4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sformatting\stypo\sin\sa\scomment.\s\sNo\schanges\sto\scode. -D 2013-04-26T19:33:34.383 +C Untested\sfix\sfor\sbuilding\son\sVxWorks. +D 2013-04-27T12:13:29.526 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -177,7 +177,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c f86cd628ffb9ccf3adcc5c15b02c00a64eaf598e +F src/os_unix.c 5a214c5431fd005dbb3b8bbaaa306433e8e9b48f F src/os_win.c 673b3e3d1fa3040d8d95a7f1f5e0e553aed56cfb F src/pager.c 4a9160d268977e56ae2df90182050ab30fca715d F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1062,7 +1062,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c47144e98c0a0f9e09780c945de10c57b6a495ea -R 9aa1f84a471b601353c6669ae1266379 +P 7a97226ffe174349e7113340f5354c4e44bd9738 +R 45f16684eb908e3d81d1d0d938ab9d11 U drh -Z 18431ca135b1f0070cd379da7401fa7c +Z 3089ee3240913ca0c76196e0917eb228 diff --git a/manifest.uuid b/manifest.uuid index 89ec5fbc80..8005ea1784 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7a97226ffe174349e7113340f5354c4e44bd9738 \ No newline at end of file +f14d55cf358b0392d3b8cd61dc85f43a610a8edf \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 1200ff352b..0074412410 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5287,9 +5287,8 @@ static int fillInUnixFile( if( h>=0 ) robust_close(pNew, h, __LINE__); h = -1; osUnlink(zFilename); - isDelete = 0; + pNew->ctrlFlags |= UNIXFILE_DELETE; } - if( isDelete ) pNew->ctrlFlags |= UNIXFILE_DELETE; #endif if( rc!=SQLITE_OK ){ if( h>=0 ) robust_close(pNew, h, __LINE__); From 015db9c85995e1661079afd11fb7c45a95ceee7e Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 27 Apr 2013 18:06:40 +0000 Subject: [PATCH 176/204] Remove spellfix virtual table documentation from the source tree. Reference the separate documentation on the website instead. FossilOrigin-Name: adcf78909ff9064b6e3c4dd15ccd3245c8cf270b --- ext/misc/editdist3.wiki | 114 ---------- ext/misc/spellfix.c | 2 +- ext/misc/spellfix1.wiki | 464 ---------------------------------------- manifest | 14 +- manifest.uuid | 2 +- 5 files changed, 8 insertions(+), 588 deletions(-) delete mode 100644 ext/misc/editdist3.wiki delete mode 100644 ext/misc/spellfix1.wiki diff --git a/ext/misc/editdist3.wiki b/ext/misc/editdist3.wiki deleted file mode 100644 index 494922c5e9..0000000000 --- a/ext/misc/editdist3.wiki +++ /dev/null @@ -1,114 +0,0 @@ -The editdist3 algorithm - -The editdist3 algorithm is a function that computes the minimum edit distance -(a.k.a. the Levenshtein distance) between two input strings. Features of -editdist3 include: - - * It works with unicode (UTF8) text. - - * A table of insertion, deletion, and substitution costs can be - provided by the application. - - * Multi-character insertsions, deletions, and substitutions can be - enumerated in the cost table. - -

    The COST table

    - -To program the costs of editdist3, create a table such as the following: - -
    -CREATE TABLE editcost(
-  iLang INT,   -- The language ID
-  cFrom TEXT,  -- Convert text from this
-  cTo   TEXT,  -- Convert text into this
-  iCost INT    -- The cost of doing the conversionnn
-);
-
    - -The cost table can be named anything you want - it does not have to be called -"editcost". And the table can contain additional columns. However, it the -table must contain the four columns show above, with exactly the names shown. - -The iLang column is a non-negative integer that identifies a set of costs -appropriate for a particular language. The editdist3 function will only use -a single iLang value for any given edit-distance computation. The default -value is 0. It is recommended that applications that only need to use a -single langauge always use iLang==0 for all entries. - -The iCost column is the numeric cost of transforming cFrom into cTo. This -value should be a non-negative integer, and should probably be less than 100. -The default single-character insertion and deletion costs are 100 and the -default single-character to single-character substitution cost is 150. A -cost of 10000 or more is considered "infinite" and causes the rule to be -ignored. - -The cFrom and cTo columns show edit transformation strings. Either or both -columns may contain more than one character. Or either column (but not both) -may hold an empty string. When cFrom is empty, that is the cost of inserting -cTo. When cTo is empty, that is the cost of deleting cFrom. - -In the spellfix1 algorithm, cFrom is the text as the user entered it and -cTo is the correctly spelled text as it exists in the database. The goal -of the editdist3 algorithm is to determine how close the user-entered text is -to the dictionary text. - -There are three special-case entries in the cost table: - - - - - - -
    cFromcToMeaning
    '''?'The default insertion cost
    '?'''The default deletion cost
    '?''?'The default substitution cost
    - -If any of the special-case entries shows above are omitted, then the -value of 100 is used for insertion and deletion and 150 is used for -substitution. To disable the default insertion, deletion, and/or substitution -set their respective cost to 10000 or more. - -Other entries in the cost table specific transforms for particular characters. -The cost of specific transforms should be less than the default costs, or else -the default costs will take precedence and the specific transforms will never -be used. - -Some example, cost table entries: - -
    -INSERT INTO editcost(iLang, cFrom, cTo, iCost)
-VALUES(0, 'a', 'ä', 5);
-
    - -The rule above says that the letter "a" in user input can be matched against -the letter "ä" in the dictionary with a penalty of 5. - -
    -INSERT INTO editcost(iLang, cFrom, cTo, iCost)
-VALUES(0, 'ss', 'ß', 8);
-
    - -The number of characters in cFrom and cTo do not need to be the same. The -rule above says that "ss" on user input will match "ß" with a penalty of 8. - -

    Experimenting with the editcost3() function

    - -The [./spellfix1.wiki | spellfix1 virtual table] -uses editdist3 if the "edit_cost_table=TABLE" option -is specified as an argument when the spellfix1 virtual table is created. -But editdist3 can also be tested directly using the built-in "editdist3()" -SQL function. The editdist3() SQL function has 3 forms: - - 1. editdist3('TABLENAME'); - 2. editdist3('string1', 'string2'); - 3. editdist3('string1', 'string2', langid); - -The first form loads the edit distance coefficients from a table called -'TABLENAME'. Any prior coefficients are discarded. So when experimenting -with weights and the weight table changes, simply rerun the single-argument -form of editdist3() to reload revised coefficients. Note that the -edit distance -weights used by the editdist3() SQL function are independent from the -weights used by the spellfix1 virtual table. - -The second and third forms return the computed edit distance between strings -'string1' and "string2'. In the second form, an language id of 0 is used. -The language id is specified in the third form. diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index e0a7d1f374..c368b34e8f 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -12,7 +12,7 @@ ** ** This module implements the spellfix1 VIRTUAL TABLE that can be used ** to search a large vocabulary for close matches. See separate -** documentation files (spellfix1.wiki and editdist3.wiki) for details. +** documentation (http://www.sqlite.org/spellfix1.html) for details. */ #include "sqlite3ext.h" SQLITE_EXTENSION_INIT1 diff --git a/ext/misc/spellfix1.wiki b/ext/misc/spellfix1.wiki deleted file mode 100644 index 288c55dd2e..0000000000 --- a/ext/misc/spellfix1.wiki +++ /dev/null @@ -1,464 +0,0 @@ -The Spellfix1 Virtual Table - -This spellfix1 virtual table is used to search -a large vocabulary for close matches. For example, spellfix1 -can be used to suggest corrections to misspelled words. Or, -it could be used with FTS4 to do full-text search using potentially -misspelled words. - -Create an instance of the spellfix1 virtual table like this: - -
    -CREATE VIRTUAL TABLE demo USING spellfix1;
-
    - -The "spellfix1" term is the name of this module and must be entered as -shown. The "demo" term is the -name of the virtual table you will be creating and can be altered -to suit the needs of your application. The virtual table is initially -empty. In order for the virtual table to be useful, you will need to -populate it with your vocabulary. Suppose you -have a list of words in a table named "big_vocabulary". Then do this: - -
    -INSERT INTO demo(word) SELECT word FROM big_vocabulary;
-
    - -If you intend to use this virtual table in cooperation with an FTS4 -table (for spelling correctly of search terms) then you might extract -the vocabulary using an fts3aux table: - -
    -INSERT INTO demo(word) SELECT term FROM search_aux WHERE col='*';
-
    - -You can also provide the virtual table with a "rank" for each word. -The "rank" is an estimate of how common the word is. Larger numbers -mean the word is more common. If you omit the rank when populating -the table, then a rank of 1 is assumed. But if you have rank -information, you can supply it and the virtual table will show a -slight preference for selecting more commonly used terms. To -populate the rank from an fts4aux table "search_aux" do something -like this: - -
    -INSERT INTO demo(word,rank)
-   SELECT term, documents FROM search_aux WHERE col='*';
-
    - -To query the virtual table, include a MATCH operator in the WHERE -clause. For example: - -
    -SELECT word FROM demo WHERE word MATCH 'kennasaw';
-
    - -Using a dataset of American place names (derived from -[http://geonames.usgs.gov/domestic/download_data.htm]) the query above -returns 20 results beginning with: - -
    -kennesaw
-kenosha
-kenesaw
-kenaga
-keanak
-
    - -If you append the character '*' to the end of the pattern, then -a prefix search is performed. For example: - -
    -SELECT word FROM demo WHERE word MATCH 'kennes*';
-
    - -Yields 20 results beginning with: - -
    -kennesaw
-kennestone
-kenneson
-kenneys
-keanes
-keenes
-
    - -

    Search Refinements

    - -By default, the spellfix1 table returns no more than 20 results. -(It might return less than 20 if there were fewer good matches.) -You can change the upper bound on the number of returned rows by -adding a "top=N" term to the WHERE clause of your query, where N -is the new maximum. For example, to see the 5 best matches: - -
    -SELECT word FROM demo WHERE word MATCH 'kennes*' AND top=5;
-
    - -Each entry in the spellfix1 virtual table is associated with a -a particular language, identified by the integer "langid" column. -The default langid is 0 and if no other actions are taken, the -entire vocabulary is a part of the 0 language. But if your application -needs to operate in multiple languages, then you can specify different -vocabulary items for each language by specifying the langid field -when populating the table. For example: - -
    -INSERT INTO demo(word,langid) SELECT word, 0 FROM en_vocabulary;
-INSERT INTO demo(word,langid) SELECT word, 1 FROM de_vocabulary;
-INSERT INTO demo(word,langid) SELECT word, 2 FROM fr_vocabulary;
-INSERT INTO demo(word,langid) SELECT word, 3 FROM ru_vocabulary;
-INSERT INTO demo(word,langid) SELECT word, 4 FROM cn_vocabulary;
-
    - -After the virtual table has been populated with items from multiple -languages, specify the language of interest using a "langid=N" term -in the WHERE clause of the query: - -
    -SELECT word FROM demo WHERE word MATCH 'hildes*' AND langid=1;
-
    - -Note that if you do not include the "langid=N" term in the WHERE clause, -the search will be against language 0 (English in the example above.) -All spellfix1 searches are against a single language id. There is no -way to search all languages at once. - - -

    Virtual Table Details

    - -The virtual table actually has a unique rowid with seven columns plus five -extra hidden columns. The columns are as follows: - -
    -
    rowid
    -A unique integer number associated with each -vocabulary item in the table. This can be used -as a foreign key on other tables in the database. - -
    word
    -The text of the word that matches the pattern. -Both word and pattern can contains unicode characters -and can be mixed case. - -
    rank
    -This is the rank of the word, as specified in the -original INSERT statement. - - -
    distance
    -This is an edit distance or Levensthein distance going -from the pattern to the word. - -
    langid
    -This is the language-id of the word. All queries are -against a single language-id, which defaults to 0. -For any given query this value is the same on all rows. - -
    score
    -The score is a combination of rank and distance. The -idea is that a lower score is better. The virtual table -attempts to find words with the lowest score and -by default (unless overridden by ORDER BY) returns -results in order of increasing score. - -
    matchlen
    -In a prefix search, the matchlen is the number of characters in -the string that match against the prefix. For a non-prefix search, -this is the same as length(word). - -
    phonehash
    -This column shows the phonetic hash prefix that was used to restrict -the search. For any given query, this column should be the same for -every row. This information is available for diagnostic purposes and -is not normally considered useful in real applications. - -
    top
    -(HIDDEN) For any query, this value is the same on all -rows. It is an integer which is the maximum number of -rows that will be output. The actually number of rows -output might be less than this number, but it will never -be greater. The default value for top is 20, but that -can be changed for each query by including a term of -the form "top=N" in the WHERE clause of the query. - -
    scope
    -(HIDDEN) For any query, this value is the same on all -rows. The scope is a measure of how widely the virtual -table looks for matching words. Smaller values of -scope cause a broader search. The scope is normally -choosen automatically and is capped at 4. Applications -can change the scope by including a term of the form -"scope=N" in the WHERE clause of the query. Increasing -the scope will make the query run faster, but will reduce -the possible corrections. - -
    srchcnt
    -(HIDDEN) For any query, this value is the same on all -rows. This value is an integer which is the number of -of words examined using the edit-distance algorithm to -find the top matches that are ultimately displayed. This -value is for diagnostic use only. - -
    soundslike
    -(HIDDEN) When inserting vocabulary entries, this field -can be set to an spelling that matches what the word -sounds like. See the DEALING WITH UNUSUAL AND DIFFICULT -SPELLINGS section below for details. - -
    command
    -(HIDDEN) The value of the "command" column is always NULL. However, -applications can insert special strings into the "command" column in order -to provoke certain behaviors in the spellfix1 virtual table. -For example, inserting the string 'reset' into the "command" column -will cause the virtual table will reread its edit distance weights -(if there are any). -
    - -

    Algorithm

    - -The spellfix1 virtual table creates a single -shadow table named "%_vocab" (where the % is replaced by the name of -the virtual table; Ex: "demo_vocab" for the "demo" virtual table). -the shadow table contains the following columns: - -
    -
    id
    -The unique id (INTEGER PRIMARY KEY) - -
    rank
    -The rank of word. - -
    langid
    -The language id for this entry. - -
    word
    -The original UTF8 text of the vocabulary word - -
    k1
    -The word transliterated into lower-case ASCII. -There is a standard table of mappings from non-ASCII -characters into ASCII. Examples: "æ" -> "ae", -"þ" -> "th", "ß" -> "ss", "á" -> "a", ... The -accessory function spellfix1_translit(X) will do -the non-ASCII to ASCII mapping. The built-in lower(X) -function will convert to lower-case. Thus: -k1 = lower(spellfix1_translit(word)). - -
    k2
    -This field holds a phonetic code derived from k1. Letters -that have similar sounds are mapped into the same symbol. -For example, all vowels and vowel clusters become the -single symbol "A". And the letters "p", "b", "f", and -"v" all become "B". All nasal sounds are represented -as "N". And so forth. The mapping is base on -ideas found in Soundex, Metaphone, and other -long-standing phonetic matching systems. This key can -be generated by the function spellfix1_phonehash(X). -Hence: k2 = spellfix1_phonehash(k1) -
    - -There is also a function for computing the Wagner edit distance or the -Levenshtein distance between a pattern and a word. This function -is exposed as spellfix1_editdist(X,Y). The edit distance function -returns the "cost" of converting X into Y. Some transformations -cost more than others. Changing one vowel into a different vowel, -for example is relatively cheap, as is doubling a constant, or -omitting the second character of a double-constant. Other transformations -or more expensive. The idea is that the edit distance function returns -a low cost of words that are similar and a higher cost for words -that are futher apart. In this implementation, the maximum cost -of any single-character edit (delete, insert, or substitute) is 100, -with lower costs for some edits (such as transforming vowels). - -The "score" for a comparison is the edit distance between the pattern -and the word, adjusted down by the base-2 logorithm of the word rank. -For example, a match with distance 100 but rank 1000 would have a -score of 122 (= 100 - log2(1000) + 32) where as a match with distance -100 with a rank of 1 would have a score of 131 (100 - log2(1) + 32). -(NB: The constant 32 is added to each score to keep it from going -negative in case the edit distance is zero.) In this way, frequently -used words get a slightly lower cost which tends to move them toward -the top of the list of alternative spellings. - -A straightforward implementation of a spelling corrector would be -to compare the search term against every word in the vocabulary -and select the 20 with the lowest scores. However, there will -typically be hundreds of thousands or millions of words in the -vocabulary, and so this approach is not fast enough. - -Suppose the term that is being spell-corrected is X. To limit -the search space, X is converted to a k2-like key using the -equivalent of: - -
    -   key = spellfix1_phonehash(lower(spellfix1_translit(X)))
-
    - -This key is then limited to "scope" characters. The default scope -value is 4, but an alternative scope can be specified using the -"scope=N" term in the WHERE clause. After the key has been truncated, -the edit distance is run against every term in the vocabulary that -has a k2 value that begins with the abbreviated key. - -For example, suppose the input word is "Paskagula". The phonetic -key is "BACACALA" which is then truncated to 4 characters "BACA". -The edit distance is then run on the 4980 entries (out of -272,597 entries total) of the vocabulary whose k2 values begin with -BACA, yielding "Pascagoula" as the best match. - -Only terms of the vocabulary with a matching langid are searched. -Hence, the same table can contain entries from multiple languages -and only the requested language will be used. The default langid -is 0. - -

    Configurable Edit Distance

    - -The built-in Wagner edit-distance function with fixed weights can be -replaced by the [./editdist3.wiki | editdist3()] edit-distance function -with application-defined weights and support for unicode, by specifying -the "edit_cost_table=TABLENAME" parameter to the spellfix1 module -when the virtual table is created. -For example: - -
    -CREATE VIRTUAL TABLE demo2 USING spellfix1(edit_cost_table=APPCOST);
-
    - -In the example above, the APPCOST table would be interrogated to find -the edit distance coefficients. It is the presence of the "edit_cost_table=" -parameter to the spellfix1 module name that causes editdist3() to be used -in place of the built-in edit distance function. - -The edit distance coefficients are normally read from the APPCOST table -once and there after stored in memory. Hence, run-time changes to the -APPCOST table will not normally effect the edit distance results. -However, inserting the special string 'reset' into the "command" column of the -virtual table causes the edit distance coefficients to be reread the -APPCOST table. Hence, applications should run a SQL statement similar -to the following when changes to the APPCOST table occur: - -
    -INSERT INTO demo2(command) VALUES('reset'); -
    - -The tables used for edit distance costs can be changed using a command -like the following: - -
    -INSERT INTO demo2(command) VALUES('edit_cost_table=APPCOST2'); -
    - -In the example above, any prior edit distance costs would be discarded and -all future queries would use the costs found in the APPCOST2 table. If the -name of the table specified by the "edit_cost_table" command is "NULL", then -theh built-in Wagner edit-distance function will be used instead of the -editdist3() function in all future queries. - -

    Dealing With Unusual And Difficult Spellings

    - -The algorithm above works quite well for most cases, but there are -exceptions. These exceptions can be dealt with by making additional -entries in the virtual table using the "soundslike" column. - -For example, many words of Greek origin begin with letters "ps" where -the "p" is silent. Ex: psalm, pseudonym, psoriasis, psyche. In -another example, many Scottish surnames can be spelled with an -initial "Mac" or "Mc". Thus, "MacKay" and "McKay" are both pronounced -the same. - -Accommodation can be made for words that are not spelled as they -sound by making additional entries into the virtual table for the -same word, but adding an alternative spelling in the "soundslike" -column. For example, the canonical entry for "psalm" would be this: - -
    -  INSERT INTO demo(word) VALUES('psalm');
-
    - -To enhance the ability to correct the spelling of "salm" into -"psalm", make an addition entry like this: - -
    -  INSERT INTO demo(word,soundslike) VALUES('psalm','salm');
-
    - -It is ok to make multiple entries for the same word as long as -each entry has a different soundslike value. Note that if no -soundslike value is specified, the soundslike defaults to the word -itself. - -Listed below are some cases where it might make sense to add additional -soundslike entries. The specific entries will depend on the application -and the target language. - - * Silent "p" in words beginning with "ps": psalm, psyche - - * Silent "p" in words beginning with "pn": pneumonia, pneumatic - - * Silent "p" in words beginning with "pt": pterodactyl, ptolemaic - - * Silent "d" in words beginning with "dj": djinn, Djikarta - - * Silent "k" in words beginning with "kn": knight, Knuthson - - * Silent "g" in words beginning with "gn": gnarly, gnome, gnat - - * "Mac" versus "Mc" beginning Scottish surnames - - * "Tch" sounds in Slavic words: Tchaikovsky vs. Chaykovsky - - * The letter "j" pronounced like "h" in Spanish: LaJolla - - * Words beginning with "wr" versus "r": write vs. rite - - * Miscellanous problem words such as "debt", "tsetse", - "Nguyen", "Van Nuyes". - -

    Auxiliary Functions

    - -The source code module that implements the spellfix1 virtual table also -implements several SQL functions that might be useful to applications -that employ spellfix1 or for testing or diagnostic work while developing -applications that use spellfix1. The following auxiliary functions are -available: - -
    -
    editdist3(P,W)
    editdist2(P,W,L)
    editdist3(T)
    -These routines provide direct access to the version of the Wagner -edit-distance function that allows for application-defined weights -on edit operations. The first two forms of this function compare -pattern P against word W and return the edit distance. In the first -function, the langid is assumed to be 0 and in the second, the -langid is given by the L parameter. The third form of this function -reloads edit distance coefficience from the table named by T. - -
    spellfix1_editdist(P,W)
    -This routine provides access to the built-in Wagner edit-distance -function that uses default, fixed costs. The value returned is -the edit distance needed to transform W into P. - -
    spellfix1_phonehash(X)
    -This routine constructs a phonetic hash of the pure ascii input word X -and returns that hash. This routine is used internally by spellfix1 in -order to transform the K1 column of the shadow table into the K2 -column. - -
    spellfix1_scriptcode(X)
    -Given an input string X, this routine attempts to determin the dominant -script of that input and returns the ISO-15924 numeric code for that -script. The current implementation understands the following scripts: -
      -
    • 215 - Latin -
    • 220 - Cyrillic -
    • 200 - Greek -
    -Additional language codes might be added in future releases. - -
    spellfix1_translit(X)
    -This routine transliterates unicode text into pure ascii, returning -the pure ascii representation of the input text X. This is the function -that is used internally to transform vocabulary words into the K1 -column of the shadow table. - -
    diff --git a/manifest b/manifest index b47b06dfb4..16c565ef04 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Untested\sfix\sfor\sbuilding\son\sVxWorks. -D 2013-04-27T12:13:29.526 +C Remove\sspellfix\svirtual\stable\sdocumentation\sfrom\sthe\ssource\stree.\nReference\sthe\sseparate\sdocumentation\son\sthe\swebsite\sinstead. +D 2013-04-27T18:06:40.561 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -85,13 +85,11 @@ F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/misc/amatch.c 3369b2b544066e620d986f0085d039c77d1ef17f F ext/misc/closure.c fec0c8537c69843e0b7631d500a14c0527962cd6 -F ext/misc/editdist3.wiki 06100a0c558921a563cbc40e0d0151902b1eef6d F ext/misc/fuzzer.c fb64a15af978ae73fa9075b9b1dfbe82b8defc6f F ext/misc/ieee754.c 2565ce373d842977efe0922dc50b8a41b3289556 F ext/misc/nextchar.c 1131e2b36116ffc6fe6b2e3464bfdace27978b1e F ext/misc/regexp.c c25c65fe775f5d9801fb8573e36ebe73f2c0c2e0 -F ext/misc/spellfix.c e323eebb877d735bc64404c16a6d758ab17a0b7a -F ext/misc/spellfix1.wiki dd1830444c14cf0f54dd680cc044df2ace2e9d09 +F ext/misc/spellfix.c f9d24a2b2617cee143b7841b453e4e1fd8f189cc F ext/misc/wholenumber.c ce362368b9381ea48cbd951ade8df867eeeab014 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 757abea591d4ff67c0ff4e8f9776aeda86b18c14 @@ -1062,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 7a97226ffe174349e7113340f5354c4e44bd9738 -R 45f16684eb908e3d81d1d0d938ab9d11 +P f14d55cf358b0392d3b8cd61dc85f43a610a8edf +R 1d8237b86dffdd5327eb650dcb291120 U drh -Z 3089ee3240913ca0c76196e0917eb228 +Z 369cfa66c298e6ca3c71b08895115463 diff --git a/manifest.uuid b/manifest.uuid index 8005ea1784..edb252e518 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f14d55cf358b0392d3b8cd61dc85f43a610a8edf \ No newline at end of file +adcf78909ff9064b6e3c4dd15ccd3245c8cf270b \ No newline at end of file From f2c1c99f15ad1a14d38e0674c86d467363c99787 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sun, 28 Apr 2013 01:44:43 +0000 Subject: [PATCH 177/204] Initial work on reforming usage of OSTRACE by the Windows VFS. FossilOrigin-Name: b347b5d1a01301f44e6fd20ee03a9529ffd25788 --- manifest | 21 +++--- manifest.uuid | 2 +- src/main.c | 80 +++++++++++++++++++++ src/os_win.c | 187 ++++++++++++++++++++++++++++++++++++++---------- src/sqliteInt.h | 1 + 5 files changed, 242 insertions(+), 49 deletions(-) diff --git a/manifest b/manifest index 16c565ef04..3fd4aeab7b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sspellfix\svirtual\stable\sdocumentation\sfrom\sthe\ssource\stree.\nReference\sthe\sseparate\sdocumentation\son\sthe\swebsite\sinstead. -D 2013-04-27T18:06:40.561 +C Initial\swork\son\sreforming\susage\sof\sOSTRACE\sby\sthe\sWindows\sVFS. +D 2013-04-28T01:44:43.497 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -158,7 +158,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c c48f7f3f170e502fe0cc20748e03c6e0b5a016c2 -F src/main.c de47c3e6b68f512b9e64ba2533ee4eecc9d501c1 +F src/main.c d89118abab8becbefd22e88f14dccbff4aef0aa6 F src/malloc.c fe085aa851b666b7c375c1ff957643dc20a04bf6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa @@ -176,7 +176,7 @@ F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 5a214c5431fd005dbb3b8bbaaa306433e8e9b48f -F src/os_win.c 673b3e3d1fa3040d8d95a7f1f5e0e553aed56cfb +F src/os_win.c 79bf8963cfb91a544b03c4be07b67488f892e929 F src/pager.c 4a9160d268977e56ae2df90182050ab30fca715d F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -194,7 +194,7 @@ F src/shell.c 5d527e5d08f05ec2c43ff194ea44bf62b974f4c9 F src/sqlite.h.in e6c241170da854abf6f15ec7e6c565348b690169 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h de835c584032769461c123a564381f9808542c0e +F src/sqliteInt.h 1a2b576d605505f6584b0ed6c59f3e85eb8ff5dc F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1060,7 +1060,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f14d55cf358b0392d3b8cd61dc85f43a610a8edf -R 1d8237b86dffdd5327eb650dcb291120 -U drh -Z 369cfa66c298e6ca3c71b08895115463 +P adcf78909ff9064b6e3c4dd15ccd3245c8cf270b +R f41b19fed2521384396fcf0a1effb849 +T *branch * winOsTrace +T *sym-winOsTrace * +T -sym-trunk * +U mistachkin +Z fd47dbd9998a9c91c0e16a40715c418b diff --git a/manifest.uuid b/manifest.uuid index edb252e518..1d843c60fd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -adcf78909ff9064b6e3c4dd15ccd3245c8cf270b \ No newline at end of file +b347b5d1a01301f44e6fd20ee03a9529ffd25788 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 062fe40756..57f2d06b79 100644 --- a/src/main.c +++ b/src/main.c @@ -1029,6 +1029,86 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ } } +/* +** Return a static string containing the name corresponding to the error code +** specified in the argument. +*/ +const char *sqlite3ErrName(int rc){ + const char *zName = 0; + int i; + for(i=0; i<2 && zName==0; i++, rc &= 0xff){ + switch( rc ){ + case SQLITE_OK: zName = "SQLITE_OK"; break; + case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; + case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; + case SQLITE_PERM: zName = "SQLITE_PERM"; break; + case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; + case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; + case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; + case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; + case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; + case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; + case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; + case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; + case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; + case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; + case SQLITE_FULL: zName = "SQLITE_FULL"; break; + case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; + case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; + case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; + case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; + case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; + case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; + case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; + case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; + case SQLITE_CONSTRAINT_FOREIGNKEY: + zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; + case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; + case SQLITE_CONSTRAINT_PRIMARYKEY: + zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; + case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; + case SQLITE_CONSTRAINT_COMMITHOOK: + zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; + case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; + case SQLITE_CONSTRAINT_FUNCTION: + zName = "SQLITE_CONSTRAINT_FUNCTION"; break; + case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; + case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; + case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; + case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; + case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; + case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; + case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; + case SQLITE_ROW: zName = "SQLITE_ROW"; break; + case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; + case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; + case SQLITE_DONE: zName = "SQLITE_DONE"; break; + case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; + case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; + case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; + case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; + case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; + case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; + case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; + case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; + case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; + case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; + case SQLITE_IOERR_BLOCKED: zName = "SQLITE_IOERR_BLOCKED"; break; + case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; + case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; + case SQLITE_IOERR_CHECKRESERVEDLOCK: + zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; + case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; + case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; + case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; + case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; + case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; + } + } + if( zName==0 ) zName = "SQLITE_Unknown"; + return zName; +} + /* ** Return a static string that describes the kind of error specified in the ** argument. diff --git a/src/os_win.c b/src/os_win.c index 12eefed8be..6196398023 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2027,6 +2027,8 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ DWORD dwRet; /* Value returned by SetFilePointer() */ DWORD lastErrno; /* Value returned by GetLastError() */ + OSTRACE(("SEEK file=%p, offset=%lld\n", pFile->h, iOffset)); + upperBits = (LONG)((iOffset>>32) & 0x7fffffff); lowerBits = (LONG)(iOffset & 0xffffffff); @@ -2044,9 +2046,11 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ pFile->lastErrno = lastErrno; winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, "seekWinFile", pFile->zPath); + OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); return 1; } + OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); return 0; #else /* @@ -2063,9 +2067,11 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ pFile->lastErrno = osGetLastError(); winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, "seekWinFile", pFile->zPath); + OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); return 1; } + OSTRACE(("SEEK file=%p, rc=SQLITE_OK\n", pFile->h)); return 0; #endif } @@ -2094,8 +2100,8 @@ static int winClose(sqlite3_file *id){ #ifndef SQLITE_OMIT_WAL assert( pFile->pShm==0 ); #endif - OSTRACE(("CLOSE %d\n", pFile->h)); assert( pFile->h!=NULL && pFile->h!=INVALID_HANDLE_VALUE ); + OSTRACE(("CLOSE file=%p\n", pFile->h)); #if SQLITE_MAX_MMAP_SIZE>0 rc = winUnmapfile(pFile); @@ -2121,11 +2127,11 @@ static int winClose(sqlite3_file *id){ sqlite3_free(pFile->zDeleteOnClose); } #endif - OSTRACE(("CLOSE %d %s\n", pFile->h, rc ? "ok" : "failed")); if( rc ){ pFile->h = NULL; } OpenCounter(-1); + OSTRACE(("CLOSE file=%p, rc=%s\n", pFile->h, rc ? "ok" : "failed")); return rc ? SQLITE_OK : winLogError(SQLITE_IOERR_CLOSE, osGetLastError(), "winClose", pFile->zPath); @@ -2152,7 +2158,8 @@ static int winRead( assert( id!=0 ); assert( amt>0 ); SimulateIOError(return SQLITE_IOERR_READ); - OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); + OSTRACE(("READ file=%p, buffer=%p, amount=%d, offset=%lld, lock=%d\n", + pFile->h, pBuf, amt, offset, pFile->locktype)); #if SQLITE_MAX_MMAP_SIZE>0 /* Deal with as much of this read request as possible by transfering @@ -2160,6 +2167,7 @@ static int winRead( if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + OSTRACE(("READ-MMAP file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; }else{ int nCopy = (int)(pFile->mmapSize - offset); @@ -2173,6 +2181,7 @@ static int winRead( #if SQLITE_OS_WINCE if( seekWinFile(pFile, offset) ){ + OSTRACE(("READ file=%p, rc=SQLITE_FULL\n", pFile->h)); return SQLITE_FULL; } while( !osReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ @@ -2186,6 +2195,7 @@ static int winRead( DWORD lastErrno; if( retryIoerr(&nRetry, &lastErrno) ) continue; pFile->lastErrno = lastErrno; + OSTRACE(("READ file=%p, rc=SQLITE_IOERR_READ\n", pFile->h)); return winLogError(SQLITE_IOERR_READ, pFile->lastErrno, "winRead", pFile->zPath); } @@ -2193,9 +2203,11 @@ static int winRead( if( nRead<(DWORD)amt ){ /* Unread parts of the buffer must be zero-filled */ memset(&((char*)pBuf)[nRead], 0, amt-nRead); + OSTRACE(("READ file=%p, rc=SQLITE_IOERR_SHORT_READ\n", pFile->h)); return SQLITE_IOERR_SHORT_READ; } + OSTRACE(("READ file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } @@ -2218,7 +2230,8 @@ static int winWrite( SimulateIOError(return SQLITE_IOERR_WRITE); SimulateDiskfullError(return SQLITE_FULL); - OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); + OSTRACE(("WRITE file=%p, buffer=%p, amount=%d, offset=%lld, lock=%d\n", + pFile->h, pBuf, amt, offset, pFile->locktype)); #if SQLITE_MAX_MMAP_SIZE>0 /* Deal with as much of this write request as possible by transfering @@ -2226,6 +2239,7 @@ static int winWrite( if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + OSTRACE(("WRITE-MMAP file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; }else{ int nCopy = (int)(pFile->mmapSize - offset); @@ -2288,13 +2302,16 @@ static int winWrite( if( rc ){ if( ( pFile->lastErrno==ERROR_HANDLE_DISK_FULL ) || ( pFile->lastErrno==ERROR_DISK_FULL )){ + OSTRACE(("WRITE file=%p, rc=SQLITE_FULL\n", pFile->h)); return SQLITE_FULL; } + OSTRACE(("WRITE file=%p, rc=SQLITE_IOERR_WRITE\n", pFile->h)); return winLogError(SQLITE_IOERR_WRITE, pFile->lastErrno, "winWrite", pFile->zPath); }else{ logIoerr(nRetry); } + OSTRACE(("WRITE file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } @@ -2307,9 +2324,9 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ DWORD lastErrno; assert( pFile ); - - OSTRACE(("TRUNCATE %d %lld\n", pFile->h, nByte)); SimulateIOError(return SQLITE_IOERR_TRUNCATE); + OSTRACE(("TRUNCATE file=%d, size=%lld, lock=%d\n", + pFile->h, nByte, pFile->locktype)); /* If the user has configured a chunk-size for this file, truncate the ** file so that it consists of an integer number of chunks (i.e. the @@ -2341,7 +2358,7 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ } #endif - OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); + OSTRACE(("TRUNCATE file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); return rc; } @@ -2381,13 +2398,14 @@ static int winSync(sqlite3_file *id, int flags){ || (flags&0x0F)==SQLITE_SYNC_FULL ); - OSTRACE(("SYNC %d lock=%d\n", pFile->h, pFile->locktype)); - /* Unix cannot, but some systems may return SQLITE_FULL from here. This ** line is to test that doing so does not cause any problems. */ SimulateDiskfullError( return SQLITE_FULL ); + OSTRACE(("SYNC file=%p, flags=%x, lock=%d\n", + pFile->h, flags, pFile->locktype)); + #ifndef SQLITE_TEST UNUSED_PARAMETER(flags); #else @@ -2406,9 +2424,11 @@ static int winSync(sqlite3_file *id, int flags){ rc = osFlushFileBuffers(pFile->h); SimulateIOError( rc=FALSE ); if( rc ){ + OSTRACE(("SYNC file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; }else{ pFile->lastErrno = osGetLastError(); + OSTRACE(("SYNC file=%p, rc=SQLITE_IOERR_FSYNC\n", pFile->h)); return winLogError(SQLITE_IOERR_FSYNC, pFile->lastErrno, "winSync", pFile->zPath); } @@ -2423,7 +2443,10 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ int rc = SQLITE_OK; assert( id!=0 ); + assert( pSize!=0 ); SimulateIOError(return SQLITE_IOERR_FSTAT); + OSTRACE(("SIZE file=%p, pSize=%p\n", pFile->h, pSize)); + #if SQLITE_OS_WINRT { FILE_STANDARD_INFO info; @@ -2452,6 +2475,8 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ } } #endif + OSTRACE(("SIZE file=%p, pSize=%p, *pSize=%lld, rc=%s\n", + pFile->h, pSize, *pSize, sqlite3ErrName(rc))); return rc; } @@ -2493,6 +2518,7 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ */ static int getReadLock(winFile *pFile){ int res; + OSTRACE(("READ-LOCK file=%p, lock=%d\n", pFile->h, pFile->locktype)); if( isNT() ){ #if SQLITE_OS_WINCE /* @@ -2518,6 +2544,7 @@ static int getReadLock(winFile *pFile){ pFile->lastErrno = osGetLastError(); /* No need to log a failure to lock */ } + OSTRACE(("READ-LOCK file=%p, rc=%s\n", pFile->h, sqlite3ErrName(res))); return res; } @@ -2527,6 +2554,7 @@ static int getReadLock(winFile *pFile){ static int unlockReadLock(winFile *pFile){ int res; DWORD lastErrno; + OSTRACE(("READ-UNLOCK file=%p, lock=%d\n", pFile->h, pFile->locktype)); if( isNT() ){ res = winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); } @@ -2540,6 +2568,7 @@ static int unlockReadLock(winFile *pFile){ winLogError(SQLITE_IOERR_UNLOCK, pFile->lastErrno, "unlockReadLock", pFile->zPath); } + OSTRACE(("READ-UNLOCK file=%p, rc=%s\n", pFile->h, sqlite3ErrName(res))); return res; } @@ -2578,14 +2607,15 @@ static int winLock(sqlite3_file *id, int locktype){ DWORD lastErrno = NO_ERROR; assert( id!=0 ); - OSTRACE(("LOCK %d %d was %d(%d)\n", - pFile->h, locktype, pFile->locktype, pFile->sharedLockByte)); + OSTRACE(("LOCK file=%p, oldLock=%d(%d), newLock=%d\n", + pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); /* If there is already a lock of this type or more restrictive on the ** OsFile, do nothing. Don't use the end_lock: exit path, as ** sqlite3OsEnterMutex() hasn't been called yet. */ if( pFile->locktype>=locktype ){ + OSTRACE(("LOCK-HELD file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } @@ -2613,7 +2643,8 @@ static int winLock(sqlite3_file *id, int locktype){ ** If you are using this code as a model for alternative VFSes, do not ** copy this retry logic. It is a hack intended for Windows only. */ - OSTRACE(("could not get a PENDING lock. cnt=%d\n", cnt)); + OSTRACE(("LOCK-PENDING-FAIL file=%p, count=%d, rc=%s\n", + pFile->h, cnt, sqlite3ErrName(res))); if( cnt ) sqlite3_win32_sleep(1); } gotPendingLock = res; @@ -2658,14 +2689,12 @@ static int winLock(sqlite3_file *id, int locktype){ if( locktype==EXCLUSIVE_LOCK && res ){ assert( pFile->locktype>=SHARED_LOCK ); res = unlockReadLock(pFile); - OSTRACE(("unreadlock = %d\n", res)); res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, SHARED_FIRST, 0, SHARED_SIZE, 0); if( res ){ newLocktype = EXCLUSIVE_LOCK; }else{ lastErrno = osGetLastError(); - OSTRACE(("error-code = %d\n", lastErrno)); getReadLock(pFile); } } @@ -2683,12 +2712,14 @@ static int winLock(sqlite3_file *id, int locktype){ if( res ){ rc = SQLITE_OK; }else{ - OSTRACE(("LOCK FAILED %d trying for %d but got %d\n", pFile->h, - locktype, newLocktype)); + OSTRACE(("LOCK-FAIL file=%p, wanted=%d, got=%d\n", + pFile->h, locktype, newLocktype)); pFile->lastErrno = lastErrno; rc = SQLITE_BUSY; } pFile->locktype = (u8)newLocktype; + OSTRACE(("LOCK file=%p, lock=%d, rc=%s\n", + pFile->h, pFile->locktype, sqlite3ErrName(rc))); return rc; } @@ -2702,20 +2733,23 @@ static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ winFile *pFile = (winFile*)id; SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + OSTRACE(("TEST-WR-LOCK file=%p, pResOut=%p\n", pFile->h, pResOut)); assert( id!=0 ); if( pFile->locktype>=RESERVED_LOCK ){ rc = 1; - OSTRACE(("TEST WR-LOCK %d %d (local)\n", pFile->h, rc)); + OSTRACE(("TEST-WR-LOCK file=%p, rc=%d (local)\n", pFile->h, rc)); }else{ rc = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS,RESERVED_BYTE, 0, 1, 0); if( rc ){ winUnlockFile(&pFile->h, RESERVED_BYTE, 0, 1, 0); } rc = !rc; - OSTRACE(("TEST WR-LOCK %d %d (remote)\n", pFile->h, rc)); + OSTRACE(("TEST-WR-LOCK file=%p, rc=%d (remote)\n", pFile->h, rc)); } *pResOut = rc; + OSTRACE(("TEST-WR-LOCK file=%p, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", + pFile->h, pResOut, *pResOut)); return SQLITE_OK; } @@ -2736,8 +2770,8 @@ static int winUnlock(sqlite3_file *id, int locktype){ int rc = SQLITE_OK; assert( pFile!=0 ); assert( locktype<=SHARED_LOCK ); - OSTRACE(("UNLOCK %d to %d was %d(%d)\n", pFile->h, locktype, - pFile->locktype, pFile->sharedLockByte)); + OSTRACE(("UNLOCK file=%p, oldLock=%d(%d), newLock=%d\n", + pFile->h, pFile->locktype, pFile->sharedLockByte, locktype)); type = pFile->locktype; if( type>=EXCLUSIVE_LOCK ){ winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); @@ -2758,6 +2792,8 @@ static int winUnlock(sqlite3_file *id, int locktype){ winUnlockFile(&pFile->h, PENDING_BYTE, 0, 1, 0); } pFile->locktype = (u8)locktype; + OSTRACE(("UNLOCK file=%p, lock=%d, rc=%s\n", + pFile->h, pFile->locktype, sqlite3ErrName(rc))); return rc; } @@ -2785,17 +2821,21 @@ static int getTempname(int nBuf, char *zBuf); */ static int winFileControl(sqlite3_file *id, int op, void *pArg){ winFile *pFile = (winFile*)id; + OSTRACE(("FCNTL file=%p, op=%d, pArg=%p\n", pFile->h, op, pArg)); switch( op ){ case SQLITE_FCNTL_LOCKSTATE: { *(int*)pArg = pFile->locktype; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_LAST_ERRNO: { *(int*)pArg = (int)pFile->lastErrno; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_CHUNK_SIZE: { pFile->szChunk = *(int *)pArg; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_SIZE_HINT: { @@ -2810,20 +2850,25 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ SimulateIOErrorBenign(0); } } + OSTRACE(("FCNTL file=%p, rc=%s\n", pFile->h, sqlite3ErrName(rc))); return rc; } + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_PERSIST_WAL: { winModeBit(pFile, WINFILE_PERSIST_WAL, (int*)pArg); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_POWERSAFE_OVERWRITE: { winModeBit(pFile, WINFILE_PSOW, (int*)pArg); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_VFSNAME: { *(char**)pArg = sqlite3_mprintf("win32"); + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_WIN32_AV_RETRY: { @@ -2838,6 +2883,7 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ }else{ a[1] = win32IoerrRetryDelay; } + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } case SQLITE_FCNTL_TEMPFILENAME: { @@ -2846,6 +2892,7 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ getTempname(pFile->pVfs->mxPathname, zTFile); *(char**)pArg = zTFile; } + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } #if SQLITE_MAX_MMAP_SIZE>0 @@ -2856,10 +2903,12 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ } *(i64*)pArg = pFile->mmapSizeMax; if( newLimit>=0 ) pFile->mmapSizeMax = newLimit; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } #endif } + OSTRACE(("FCNTL file=%p, rc=SQLITE_NOTFOUND\n", pFile->h)); return SQLITE_NOTFOUND; } @@ -3020,6 +3069,9 @@ static int winShmSystemLock( /* Access to the winShmNode object is serialized by the caller */ assert( sqlite3_mutex_held(pFile->mutex) || pFile->nRef==0 ); + OSTRACE(("SHM-LOCK file=%p, lock=%d, offset=%d, size=%d\n", + pFile->hFile.h, lockType, ofst, nByte)); + /* Release/Acquire the system-level lock */ if( lockType==_SHM_UNLCK ){ rc = winUnlockFile(&pFile->hFile.h, ofst, 0, nByte, 0); @@ -3037,11 +3089,9 @@ static int winShmSystemLock( rc = SQLITE_BUSY; } - OSTRACE(("SHM-LOCK %d %s %s 0x%08lx\n", - pFile->hFile.h, - rc==SQLITE_OK ? "ok" : "failed", - lockType==_SHM_UNLCK ? "UnlockFileEx" : "LockFileEx", - pFile->lastErrno)); + OSTRACE(("SHM-LOCK file=%p, func=%s, errno=%d, rc=%s\n", pFile->hFile.h, + (lockType == _SHM_UNLCK) ? "winUnlockFile" : "winLockFile", + pFile->lastErrno, sqlite3ErrName(rc))); return rc; } @@ -3061,6 +3111,8 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ winShmNode *p; BOOL bRc; assert( winShmMutexHeld() ); + OSTRACE(("SHM-PURGE pid=%d, deleteFlag=%d\n", + (int)osGetCurrentProcessId(), deleteFlag)); pp = &winShmNodeList; while( (p = *pp)!=0 ){ if( p->nRef==0 ){ @@ -3068,11 +3120,11 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ if( p->mutex ) sqlite3_mutex_free(p->mutex); for(i=0; inRegion; i++){ bRc = osUnmapViewOfFile(p->aRegion[i].pMap); - OSTRACE(("SHM-PURGE pid-%d unmap region=%d %s\n", + OSTRACE(("SHM-PURGE-UNMAP pid=%d, region=%d, rc=%s\n", (int)osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); bRc = osCloseHandle(p->aRegion[i].hMap); - OSTRACE(("SHM-PURGE pid-%d close region=%d %s\n", + OSTRACE(("SHM-PURGE-CLOSE pid=%d, region=%d, rc=%s\n", (int)osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); } @@ -3353,9 +3405,9 @@ static int winShmLock( } } sqlite3_mutex_leave(pShmNode->mutex); - OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x %s\n", - p->id, (int)osGetCurrentProcessId(), p->sharedMask, p->exclMask, - rc ? "failed" : "ok")); + OSTRACE(("SHM-LOCK pid=%d, id=%d, sharedMask=%03x, exclMask=%03x, rc=%s\n", + (int)osGetCurrentProcessId(), p->id, p->sharedMask, p->exclMask, + sqlite3ErrName(rc))); return rc; } @@ -3476,7 +3528,7 @@ static int winShmMap( NULL, PAGE_READWRITE, 0, nByte, NULL ); #endif - OSTRACE(("SHM-MAP pid-%d create region=%d nbyte=%d %s\n", + OSTRACE(("SHM-MAP-CREATE pid=%d, region=%d, size=%d, rc=%s\n", (int)osGetCurrentProcessId(), pShmNode->nRegion, nByte, hMap ? "ok" : "failed")); if( hMap ){ @@ -3491,7 +3543,7 @@ static int winShmMap( 0, iOffset - iOffsetShift, szRegion + iOffsetShift ); #endif - OSTRACE(("SHM-MAP pid-%d map region=%d offset=%d size=%d %s\n", + OSTRACE(("SHM-MAP-MAP pid=%d, region=%d, offset=%d, size=%d, rc=%s\n", (int)osGetCurrentProcessId(), pShmNode->nRegion, iOffset, szRegion, pMap ? "ok" : "failed")); } @@ -3535,9 +3587,17 @@ shmpage_out: #if SQLITE_MAX_MMAP_SIZE>0 static int winUnmapfile(winFile *pFile){ assert( pFile!=0 ); + OSTRACE(("UNMAP-FILE pid=%d, pFile=%p, hMap=%p, pMapRegion=%p, " + "mmapSize=%lld, mmapSizeActual=%lld, mmapSizeMax=%lld\n", + (int)osGetCurrentProcessId(), pFile, pFile->hMap, + pFile->pMapRegion, pFile->mmapSize, pFile->mmapSizeActual, + pFile->mmapSizeMax)); if( pFile->pMapRegion ){ if( !osUnmapViewOfFile(pFile->pMapRegion) ){ pFile->lastErrno = osGetLastError(); + OSTRACE(("UNMAP-FILE pid=%d, pFile=%p, pMapRegion=%p, " + "rc=SQLITE_IOERR_MMAP\n", (int)osGetCurrentProcessId(), + pFile, pFile->pMapRegion)); return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, "winUnmap1", pFile->zPath); } @@ -3548,11 +3608,15 @@ static int winUnmapfile(winFile *pFile){ if( pFile->hMap!=NULL ){ if( !osCloseHandle(pFile->hMap) ){ pFile->lastErrno = osGetLastError(); + OSTRACE(("UNMAP-FILE pid=%d, pFile=%p, hMap=%p, rc=SQLITE_IOERR_MMAP\n", + (int)osGetCurrentProcessId(), pFile, pFile->hMap)); return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, "winUnmap2", pFile->zPath); } pFile->hMap = NULL; } + OSTRACE(("UNMAP-FILE pid=%d, pFile=%p, rc=SQLITE_OK\n", + (int)osGetCurrentProcessId(), pFile)); return SQLITE_OK; } @@ -3577,11 +3641,16 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ int rc; assert( nMap>=0 || pFd->nFetchOut==0 ); + OSTRACE(("MAP-FILE pid=%d, pFile=%p, size=%lld\n", + (int)osGetCurrentProcessId(), pFd, nByte)); + if( pFd->nFetchOut>0 ) return SQLITE_OK; if( nMap<0 ){ rc = winFileSize((sqlite3_file*)pFd, &nMap); if( rc ){ + OSTRACE(("MAP-FILE pid=%d, pFile=%p, rc=SQLITE_IOERR_FSTAT\n", + (int)osGetCurrentProcessId(), pFd)); return SQLITE_IOERR_FSTAT; } } @@ -3619,6 +3688,8 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ rc = winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, "winMapfile", pFd->zPath); /* Log the error, but continue normal operation using xRead/xWrite */ + OSTRACE(("MAP-FILE-CREATE pid=%d, pFile=%p, rc=SQLITE_IOERR_MMAP\n", + (int)osGetCurrentProcessId(), pFd)); return SQLITE_OK; } assert( (nMap % winSysInfo.dwPageSize)==0 ); @@ -3634,6 +3705,8 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ pFd->lastErrno = osGetLastError(); winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, "winMapfile", pFd->zPath); + OSTRACE(("MAP-FILE-MAP pid=%d, pFile=%p, rc=SQLITE_IOERR_MMAP\n", + (int)osGetCurrentProcessId(), pFd)); return SQLITE_OK; } pFd->pMapRegion = pNew; @@ -3641,6 +3714,8 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ pFd->mmapSizeActual = nMap; } + OSTRACE(("MAP-FILE pid=%d, pFile=%p, rc=SQLITE_OK\n", + (int)osGetCurrentProcessId(), pFd)); return SQLITE_OK; } #endif /* SQLITE_MAX_MMAP_SIZE>0 */ @@ -3663,11 +3738,18 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ #endif *pp = 0; + OSTRACE(("FETCH pid=%d, pFile=%p, offset=%lld, amount=%d, pp=%p\n", + (int)osGetCurrentProcessId(), fd, iOff, nAmt, pp)); + #if SQLITE_MAX_MMAP_SIZE>0 if( pFd->mmapSizeMax>0 ){ if( pFd->pMapRegion==0 ){ int rc = winMapfile(pFd, -1); - if( rc!=SQLITE_OK ) return rc; + if( rc!=SQLITE_OK ){ + OSTRACE(("FETCH pid=%d, pFile=%p, rc=%s\n", + (int)osGetCurrentProcessId(), pFd, sqlite3ErrName(rc))); + return rc; + } } if( pFd->mmapSize >= iOff+nAmt ){ *pp = &((u8 *)pFd->pMapRegion)[iOff]; @@ -3675,6 +3757,9 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ } } #endif + + OSTRACE(("FETCH pid=%d, pFile=%p, pp=%p, *pp=%p, rc=SQLITE_OK\n", + (int)osGetCurrentProcessId(), fd, pp, pp ? *pp : 0)); return SQLITE_OK; } @@ -3700,6 +3785,9 @@ static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ /* If p!=0, it must match the iOff value. */ assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + OSTRACE(("UNFETCH pid=%d, pFile=%p, offset=%lld, p=%p\n", + (int)osGetCurrentProcessId(), pFd, iOff, p)); + if( p ){ pFd->nFetchOut--; }else{ @@ -3712,6 +3800,9 @@ static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ assert( pFd->nFetchOut>=0 ); #endif + + OSTRACE(("UNFETCH pid=%d, pFile=%p, rc=SQLITE_OK\n", + (int)osGetCurrentProcessId(), fd)); return SQLITE_OK; } @@ -3808,6 +3899,7 @@ static int getTempname(int nBuf, char *zBuf){ sqlite3_snprintf(MAX_PATH-30, zTempPath, "%s", zMulti); sqlite3_free(zMulti); }else{ + OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); return SQLITE_IOERR_NOMEM; } } @@ -3821,6 +3913,7 @@ static int getTempname(int nBuf, char *zBuf){ sqlite3_snprintf(MAX_PATH-30, zTempPath, "%s", zUtf8); sqlite3_free(zUtf8); }else{ + OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); return SQLITE_IOERR_NOMEM; } } @@ -3833,6 +3926,7 @@ static int getTempname(int nBuf, char *zBuf){ nTempPath = sqlite3Strlen30(zTempPath); if( (nTempPath + sqlite3Strlen30(SQLITE_TEMP_FILE_PREFIX) + 18) >= nBuf ){ + OSTRACE(("TEMP-FILENAME rc=SQLITE_ERROR\n")); return SQLITE_ERROR; } @@ -3850,8 +3944,8 @@ static int getTempname(int nBuf, char *zBuf){ zBuf[j] = 0; zBuf[j+1] = 0; - OSTRACE(("TEMP FILENAME: %s\n", zBuf)); - return SQLITE_OK; + OSTRACE(("TEMP-FILENAME name=%s, rc=SQLITE_OK\n", zBuf)); + return SQLITE_OK; } /* @@ -3931,6 +4025,9 @@ static int winOpen( )); #endif + OSTRACE(("OPEN name=%s, pFile=%p, flags=%x, pOutFlags=%p\n", + zUtf8Name, id, flags, pOutFlags)); + /* Check the following statements are true: ** ** (a) Exactly one of the READWRITE and READONLY flags must be set, and @@ -3976,6 +4073,8 @@ static int winOpen( memset(zTmpname, 0, MAX_PATH+2); rc = getTempname(MAX_PATH+2, zTmpname); if( rc!=SQLITE_OK ){ + OSTRACE(("OPEN name=%s, rc=%s", + zUtf8Name, sqlite3ErrName(rc))); return rc; } zUtf8Name = zTmpname; @@ -3991,11 +4090,13 @@ static int winOpen( /* Convert the filename to the system encoding. */ zConverted = convertUtf8Filename(zUtf8Name); if( zConverted==0 ){ + OSTRACE(("OPEN name=%s, rc=SQLITE_IOERR_NOMEM", zUtf8Name)); return SQLITE_IOERR_NOMEM; } if( winIsDir(zConverted) ){ sqlite3_free(zConverted); + OSTRACE(("OPEN name=%s, rc=SQLITE_CANTOPEN_ISDIR", zUtf8Name)); return SQLITE_CANTOPEN_ISDIR; } @@ -4086,9 +4187,8 @@ static int winOpen( #endif logIoerr(cnt); - OSTRACE(("OPEN %d %s 0x%lx %s\n", - h, zName, dwDesiredAccess, - h==INVALID_HANDLE_VALUE ? "failed" : "ok")); + OSTRACE(("OPEN file=%p, name=%s, access=%lx, rc=%s\n", h, zUtf8Name, + dwDesiredAccess, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); if( h==INVALID_HANDLE_VALUE ){ pFile->lastErrno = lastErrno; @@ -4118,6 +4218,7 @@ static int winOpen( ){ osCloseHandle(h); sqlite3_free(zConverted); + OSTRACE(("OPEN-CE-LOCK name=%s, rc=%s\n", zName, sqlite3ErrName(rc))); return rc; } if( isTemp ){ @@ -4177,6 +4278,8 @@ static int winDelete( UNUSED_PARAMETER(syncDir); SimulateIOError(return SQLITE_IOERR_DELETE); + OSTRACE(("DELETE name=%s, syncDir=%d\n", zFilename, syncDir)); + zConverted = convertUtf8Filename(zFilename); if( zConverted==0 ){ return SQLITE_IOERR_NOMEM; @@ -4262,7 +4365,7 @@ static int winDelete( logIoerr(cnt); } sqlite3_free(zConverted); - OSTRACE(("DELETE \"%s\" %s\n", zFilename, (rc ? "failed" : "ok" ))); + OSTRACE(("DELETE name=%s, rc=%s\n", zFilename, sqlite3ErrName(rc))); return rc; } @@ -4282,8 +4385,12 @@ static int winAccess( UNUSED_PARAMETER(pVfs); SimulateIOError( return SQLITE_IOERR_ACCESS; ); + OSTRACE(("ACCESS name=%s, flags=%x, pResOut=%p\n", + zFilename, flags, pResOut)); + zConverted = convertUtf8Filename(zFilename); if( zConverted==0 ){ + OSTRACE(("ACCESS name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); return SQLITE_IOERR_NOMEM; } if( isNT() ){ @@ -4334,6 +4441,8 @@ static int winAccess( assert(!"Invalid flags argument"); } *pResOut = rc; + OSTRACE(("ACCESS name=%s, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", + zFilename, pResOut, *pResOut)); return SQLITE_OK; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b041b3fb6e..2007600d85 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3092,6 +3092,7 @@ void sqlite3Error(sqlite3*, int, const char*,...); void *sqlite3HexToBlob(sqlite3*, const char *z, int n); u8 sqlite3HexToInt(int h); int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); +const char *sqlite3ErrName(int); const char *sqlite3ErrStr(int); int sqlite3ReadSchema(Parse *pParse); CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); From e84d8d32199b2ebded61cbd9be55f152c5c07b49 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 29 Apr 2013 03:09:10 +0000 Subject: [PATCH 178/204] More refinements to Windows OSTRACE usage. Replace all usage of sqlite3TestErrorName() with sqlite3ErrName() and add missing return codes. FossilOrigin-Name: 05ba5e23fcc5da416d47a8ad5e667ed6a0c7a34a --- manifest | 49 +++++++++---------- manifest.uuid | 2 +- src/main.c | 57 ++++++++++++++-------- src/os_win.c | 112 +++++++++++++++++++++---------------------- src/test1.c | 100 +++++--------------------------------- src/test2.c | 26 +++++----- src/test3.c | 16 +++---- src/test4.c | 4 +- src/test7.c | 4 +- src/test8.c | 4 +- src/test_async.c | 8 ++-- src/test_backup.c | 10 ++-- src/test_intarray.c | 6 +-- src/test_malloc.c | 22 ++++----- src/test_multiplex.c | 8 ++-- src/test_mutex.c | 12 ++--- src/test_quota.c | 10 ++-- src/test_rtree.c | 8 ++-- src/test_syscall.c | 6 +-- src/test_thread.c | 16 ++++--- 20 files changed, 212 insertions(+), 268 deletions(-) diff --git a/manifest b/manifest index 3fd4aeab7b..f3ffcbc3c7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initial\swork\son\sreforming\susage\sof\sOSTRACE\sby\sthe\sWindows\sVFS. -D 2013-04-28T01:44:43.497 +C More\srefinements\sto\sWindows\sOSTRACE\susage.\s\sReplace\sall\susage\sof\ssqlite3TestErrorName()\swith\ssqlite3ErrName()\sand\sadd\smissing\sreturn\scodes. +D 2013-04-29T03:09:10.647 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -158,7 +158,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c c48f7f3f170e502fe0cc20748e03c6e0b5a016c2 -F src/main.c d89118abab8becbefd22e88f14dccbff4aef0aa6 +F src/main.c 64c76b9acc7381d08ae1ea609852559ec5caa34e F src/malloc.c fe085aa851b666b7c375c1ff957643dc20a04bf6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa @@ -176,7 +176,7 @@ F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 5a214c5431fd005dbb3b8bbaaa306433e8e9b48f -F src/os_win.c 79bf8963cfb91a544b03c4be07b67488f892e929 +F src/os_win.c bf297963be89f9cddab53c3356e79daa3b8240d2 F src/pager.c 4a9160d268977e56ae2df90182050ab30fca715d F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -199,18 +199,18 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 2ecec9937e69bc17560ad886da35195daa7261b8 -F src/test1.c 2b0ec224983403312a4d1db8546e1e1c45694251 -F src/test2.c 29e7154112f7448d64204e8d31179cf497ecf425 -F src/test3.c 96aed72a8e1d542fed127e3e8350ae357712fa82 -F src/test4.c cea2c55110241e4674e66d476d29c914627999f5 +F src/test1.c ab9dd4fc486a2542f57a2ca17d74fc7f28dfd05a +F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35 +F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c +F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df F src/test5.c a6d1ac55ac054d0b2b8f37b5e655b6c92645a013 F src/test6.c a437f76f9874d2563352a7e6cd0d43217663c220 -F src/test7.c f4b894b7931f8cf9f5cbf37cfa0727703f526a40 -F src/test8.c f7e729e3e1973f68e6d98f5aa65046e3e2cb0bad +F src/test7.c 126b886b53f0358b92aba9b81d3fcbfbe9a93cd6 +F src/test8.c 7ee77ea522ae34aa691dfe407139dec80d4fc039 F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 -F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad +F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 F src/test_autoext.c 5c95b5d435eaa09d6c0e7d90371c5ca8cd567701 -F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de +F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e F src/test_btree.c 5b89601dcb42a33ba8b820a6b763cc9cb48bac16 F src/test_config.c 95bb33e9dcaa340a296c0bf0e0ba3d1a1c8004c0 F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 @@ -219,28 +219,28 @@ F src/test_fs.c 8f786bfd0ad48030cf2a06fb1f050e9c60a150d7 F src/test_func.c 3a8dd37c08ab43b76d38eea2836e34a3897bf170 F src/test_hexio.c abfdecb6fa58c354623978efceb088ca18e379cd F src/test_init.c 3cbad7ce525aec925f8fda2192d576d47f0d478a -F src/test_intarray.c 07ddcebe4097d400ffca362770f1d883c112387a +F src/test_intarray.c 87847c71c3c36889c0bcc9c4baf9d31881665d61 F src/test_intarray.h b999bb18d090b8d9d9c49d36ec37ef8f341fe169 F src/test_journal.c f5c0a05b7b3d5930db769b5ee6c3766dc2221a64 F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e -F src/test_malloc.c 6982a357a6a6c24f281b91c89303a5c31075c392 -F src/test_multiplex.c 0decc630b683979cb32d4b965efd90b6e55a786e +F src/test_malloc.c 2855429b8232107b3296409be2a8eb68345290c2 +F src/test_multiplex.c 5d691eeb6cb6aa7888da28eba5e62a9a857d3c0f F src/test_multiplex.h 9b63b95f07acedee425fdfe49a47197c9bf5f9d8 -F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e +F src/test_mutex.c 293042d623ebba969160f471a82aa1551626454f F src/test_onefile.c 0396f220561f3b4eedc450cef26d40c593c69a25 F src/test_osinst.c 90a845c8183013d80eccb1f29e8805608516edba F src/test_pcache.c a5cd24730cb43c5b18629043314548c9169abb00 -F src/test_quota.c 1ec82e02fd3643899e9a5de9684515e84641c91f +F src/test_quota.c 30c64f0ef84734f2231a686df41ed882b0c59bc0 F src/test_quota.h 8761e463b25e75ebc078bd67d70e39b9c817a0cb -F src/test_rtree.c aba603c949766c4193f1068b91c787f57274e0d9 +F src/test_rtree.c 1d764c352b5348bb2869ff5f54aff8eadcb96041 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f F src/test_sqllog.c c1c1bbedbcaf82b93d83e4f9dd990e62476a680e F src/test_stat.c d1569c7a4839f13e80187e2c26b2ab4da2d03935 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd -F src/test_syscall.c 7e8293e4e6971b0f44c7f7f37b1315a8cc9f6018 +F src/test_syscall.c 437abc86e3b37facc8531b7d14d42fddb50f3677 F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa -F src/test_thread.c e286f2173563f2a1747c24bcda6b9d030bf4f4e4 +F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c 8e6087a8b3dcc260282074b0efba14b76311120c F src/test_vfstrace.c 34b544e80ba7fb77be15395a609c669df2e660a2 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 @@ -1060,10 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P adcf78909ff9064b6e3c4dd15ccd3245c8cf270b -R f41b19fed2521384396fcf0a1effb849 -T *branch * winOsTrace -T *sym-winOsTrace * -T -sym-trunk * +P b347b5d1a01301f44e6fd20ee03a9529ffd25788 +R b3ab158a0ee89a318d762b36a446cf00 U mistachkin -Z fd47dbd9998a9c91c0e16a40715c418b +Z 697f524cff7a4e0ba7e5e291adbbfd2e diff --git a/manifest.uuid b/manifest.uuid index 1d843c60fd..e16bd538ae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b347b5d1a01301f44e6fd20ee03a9529ffd25788 \ No newline at end of file +05ba5e23fcc5da416d47a8ad5e667ed6a0c7a34a \ No newline at end of file diff --git a/src/main.c b/src/main.c index 57f2d06b79..9bb48442d0 100644 --- a/src/main.c +++ b/src/main.c @@ -1043,17 +1043,51 @@ const char *sqlite3ErrName(int rc){ case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; case SQLITE_PERM: zName = "SQLITE_PERM"; break; case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; + case SQLITE_ABORT_ROLLBACK: zName = "SQLITE_ABORT_ROLLBACK"; break; case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; + case SQLITE_BUSY_RECOVERY: zName = "SQLITE_BUSY_RECOVERY"; break; case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; + case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; + case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; + case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; + case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; + case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; + case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; + case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; + case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; + case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; + case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; + case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; + case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; + case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; + case SQLITE_IOERR_BLOCKED: zName = "SQLITE_IOERR_BLOCKED"; break; + case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; + case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; + case SQLITE_IOERR_CHECKRESERVEDLOCK: + zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; + case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; + case SQLITE_IOERR_CLOSE: zName = "SQLITE_IOERR_CLOSE"; break; + case SQLITE_IOERR_DIR_CLOSE: zName = "SQLITE_IOERR_DIR_CLOSE"; break; + case SQLITE_IOERR_SHMOPEN: zName = "SQLITE_IOERR_SHMOPEN"; break; + case SQLITE_IOERR_SHMSIZE: zName = "SQLITE_IOERR_SHMSIZE"; break; + case SQLITE_IOERR_SHMLOCK: zName = "SQLITE_IOERR_SHMLOCK"; break; + case SQLITE_IOERR_SHMMAP: zName = "SQLITE_IOERR_SHMMAP"; break; + case SQLITE_IOERR_SEEK: zName = "SQLITE_IOERR_SEEK"; break; + case SQLITE_IOERR_DELETE_NOENT: zName = "SQLITE_IOERR_DELETE_NOENT";break; + case SQLITE_IOERR_MMAP: zName = "SQLITE_IOERR_MMAP"; break; case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; + case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; case SQLITE_FULL: zName = "SQLITE_FULL"; break; case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; + case SQLITE_CANTOPEN_NOTEMPDIR: zName = "SQLITE_CANTOPEN_NOTEMPDIR";break; + case SQLITE_CANTOPEN_ISDIR: zName = "SQLITE_CANTOPEN_ISDIR"; break; + case SQLITE_CANTOPEN_FULLPATH: zName = "SQLITE_CANTOPEN_FULLPATH"; break; case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; @@ -1081,28 +1115,11 @@ const char *sqlite3ErrName(int rc){ case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; case SQLITE_ROW: zName = "SQLITE_ROW"; break; case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; + case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; + case SQLITE_NOTICE_RECOVER_ROLLBACK: + zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; case SQLITE_DONE: zName = "SQLITE_DONE"; break; - case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; - case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; - case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; - case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; - case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; - case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; - case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; - case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; - case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; - case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; - case SQLITE_IOERR_BLOCKED: zName = "SQLITE_IOERR_BLOCKED"; break; - case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; - case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; - case SQLITE_IOERR_CHECKRESERVEDLOCK: - zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; - case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; - case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; - case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; - case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; - case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; } } if( zName==0 ) zName = "SQLITE_Unknown"; diff --git a/src/os_win.c b/src/os_win.c index 6196398023..e308248158 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1523,7 +1523,7 @@ static int getLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ } #endif if( 0 == dwLen ){ - sqlite3_snprintf(nBuf, zBuf, "OsError 0x%x (%u)", lastErrno, lastErrno); + sqlite3_snprintf(nBuf, zBuf, "OsError 0x%lx (%lu)", lastErrno, lastErrno); }else{ /* copy a maximum of nBuf chars to output buffer */ sqlite3_snprintf(nBuf, zBuf, "%s", zOut); @@ -1566,7 +1566,7 @@ static int winLogErrorAtLine( for(i=0; zMsg[i] && zMsg[i]!='\r' && zMsg[i]!='\n'; i++){} zMsg[i] = 0; sqlite3_log(errcode, - "os_win.c:%d: (%d) %s(%s) - %s", + "os_win.c:%d: (%lu) %s(%s) - %s", iLine, lastErrno, zFunc, zPath, zMsg ); @@ -2325,7 +2325,7 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ assert( pFile ); SimulateIOError(return SQLITE_IOERR_TRUNCATE); - OSTRACE(("TRUNCATE file=%d, size=%lld, lock=%d\n", + OSTRACE(("TRUNCATE file=%p, size=%lld, lock=%d\n", pFile->h, nByte, pFile->locktype)); /* If the user has configured a chunk-size for this file, truncate the @@ -3089,9 +3089,9 @@ static int winShmSystemLock( rc = SQLITE_BUSY; } - OSTRACE(("SHM-LOCK file=%p, func=%s, errno=%d, rc=%s\n", pFile->hFile.h, - (lockType == _SHM_UNLCK) ? "winUnlockFile" : "winLockFile", - pFile->lastErrno, sqlite3ErrName(rc))); + OSTRACE(("SHM-LOCK file=%p, func=%s, errno=%lu, rc=%s\n", + pFile->hFile.h, (lockType == _SHM_UNLCK) ? "winUnlockFile" : + "winLockFile", pFile->lastErrno, sqlite3ErrName(rc))); return rc; } @@ -3111,8 +3111,8 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ winShmNode *p; BOOL bRc; assert( winShmMutexHeld() ); - OSTRACE(("SHM-PURGE pid=%d, deleteFlag=%d\n", - (int)osGetCurrentProcessId(), deleteFlag)); + OSTRACE(("SHM-PURGE pid=%lu, deleteFlag=%d\n", + osGetCurrentProcessId(), deleteFlag)); pp = &winShmNodeList; while( (p = *pp)!=0 ){ if( p->nRef==0 ){ @@ -3120,13 +3120,11 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ if( p->mutex ) sqlite3_mutex_free(p->mutex); for(i=0; inRegion; i++){ bRc = osUnmapViewOfFile(p->aRegion[i].pMap); - OSTRACE(("SHM-PURGE-UNMAP pid=%d, region=%d, rc=%s\n", - (int)osGetCurrentProcessId(), i, - bRc ? "ok" : "failed")); + OSTRACE(("SHM-PURGE-UNMAP pid=%lu, region=%d, rc=%s\n", + osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); bRc = osCloseHandle(p->aRegion[i].hMap); - OSTRACE(("SHM-PURGE-CLOSE pid=%d, region=%d, rc=%s\n", - (int)osGetCurrentProcessId(), i, - bRc ? "ok" : "failed")); + OSTRACE(("SHM-PURGE-CLOSE pid=%lu, region=%d, rc=%s\n", + osGetCurrentProcessId(), i, bRc ? "ok" : "failed")); } if( p->hFile.h!=NULL && p->hFile.h!=INVALID_HANDLE_VALUE ){ SimulateIOErrorBenign(1); @@ -3405,8 +3403,8 @@ static int winShmLock( } } sqlite3_mutex_leave(pShmNode->mutex); - OSTRACE(("SHM-LOCK pid=%d, id=%d, sharedMask=%03x, exclMask=%03x, rc=%s\n", - (int)osGetCurrentProcessId(), p->id, p->sharedMask, p->exclMask, + OSTRACE(("SHM-LOCK pid=%lu, id=%d, sharedMask=%03x, exclMask=%03x, rc=%s\n", + osGetCurrentProcessId(), p->id, p->sharedMask, p->exclMask, sqlite3ErrName(rc))); return rc; } @@ -3528,8 +3526,8 @@ static int winShmMap( NULL, PAGE_READWRITE, 0, nByte, NULL ); #endif - OSTRACE(("SHM-MAP-CREATE pid=%d, region=%d, size=%d, rc=%s\n", - (int)osGetCurrentProcessId(), pShmNode->nRegion, nByte, + OSTRACE(("SHM-MAP-CREATE pid=%lu, region=%d, size=%d, rc=%s\n", + osGetCurrentProcessId(), pShmNode->nRegion, nByte, hMap ? "ok" : "failed")); if( hMap ){ int iOffset = pShmNode->nRegion*szRegion; @@ -3543,8 +3541,8 @@ static int winShmMap( 0, iOffset - iOffsetShift, szRegion + iOffsetShift ); #endif - OSTRACE(("SHM-MAP-MAP pid=%d, region=%d, offset=%d, size=%d, rc=%s\n", - (int)osGetCurrentProcessId(), pShmNode->nRegion, iOffset, + OSTRACE(("SHM-MAP-MAP pid=%lu, region=%d, offset=%d, size=%d, rc=%s\n", + osGetCurrentProcessId(), pShmNode->nRegion, iOffset, szRegion, pMap ? "ok" : "failed")); } if( !pMap ){ @@ -3587,17 +3585,16 @@ shmpage_out: #if SQLITE_MAX_MMAP_SIZE>0 static int winUnmapfile(winFile *pFile){ assert( pFile!=0 ); - OSTRACE(("UNMAP-FILE pid=%d, pFile=%p, hMap=%p, pMapRegion=%p, " + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, hMap=%p, pMapRegion=%p, " "mmapSize=%lld, mmapSizeActual=%lld, mmapSizeMax=%lld\n", - (int)osGetCurrentProcessId(), pFile, pFile->hMap, - pFile->pMapRegion, pFile->mmapSize, pFile->mmapSizeActual, - pFile->mmapSizeMax)); + osGetCurrentProcessId(), pFile, pFile->hMap, pFile->pMapRegion, + pFile->mmapSize, pFile->mmapSizeActual, pFile->mmapSizeMax)); if( pFile->pMapRegion ){ if( !osUnmapViewOfFile(pFile->pMapRegion) ){ pFile->lastErrno = osGetLastError(); - OSTRACE(("UNMAP-FILE pid=%d, pFile=%p, pMapRegion=%p, " - "rc=SQLITE_IOERR_MMAP\n", (int)osGetCurrentProcessId(), - pFile, pFile->pMapRegion)); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, pMapRegion=%p, " + "rc=SQLITE_IOERR_MMAP\n", osGetCurrentProcessId(), pFile, + pFile->pMapRegion)); return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, "winUnmap1", pFile->zPath); } @@ -3608,15 +3605,15 @@ static int winUnmapfile(winFile *pFile){ if( pFile->hMap!=NULL ){ if( !osCloseHandle(pFile->hMap) ){ pFile->lastErrno = osGetLastError(); - OSTRACE(("UNMAP-FILE pid=%d, pFile=%p, hMap=%p, rc=SQLITE_IOERR_MMAP\n", - (int)osGetCurrentProcessId(), pFile, pFile->hMap)); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, hMap=%p, rc=SQLITE_IOERR_MMAP\n", + osGetCurrentProcessId(), pFile, pFile->hMap)); return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, "winUnmap2", pFile->zPath); } pFile->hMap = NULL; } - OSTRACE(("UNMAP-FILE pid=%d, pFile=%p, rc=SQLITE_OK\n", - (int)osGetCurrentProcessId(), pFile)); + OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFile)); return SQLITE_OK; } @@ -3641,16 +3638,16 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ int rc; assert( nMap>=0 || pFd->nFetchOut==0 ); - OSTRACE(("MAP-FILE pid=%d, pFile=%p, size=%lld\n", - (int)osGetCurrentProcessId(), pFd, nByte)); + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, size=%lld\n", + osGetCurrentProcessId(), pFd, nByte)); if( pFd->nFetchOut>0 ) return SQLITE_OK; if( nMap<0 ){ rc = winFileSize((sqlite3_file*)pFd, &nMap); if( rc ){ - OSTRACE(("MAP-FILE pid=%d, pFile=%p, rc=SQLITE_IOERR_FSTAT\n", - (int)osGetCurrentProcessId(), pFd)); + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, rc=SQLITE_IOERR_FSTAT\n", + osGetCurrentProcessId(), pFd)); return SQLITE_IOERR_FSTAT; } } @@ -3688,8 +3685,8 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ rc = winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, "winMapfile", pFd->zPath); /* Log the error, but continue normal operation using xRead/xWrite */ - OSTRACE(("MAP-FILE-CREATE pid=%d, pFile=%p, rc=SQLITE_IOERR_MMAP\n", - (int)osGetCurrentProcessId(), pFd)); + OSTRACE(("MAP-FILE-CREATE pid=%lu, pFile=%p, rc=SQLITE_IOERR_MMAP\n", + osGetCurrentProcessId(), pFd)); return SQLITE_OK; } assert( (nMap % winSysInfo.dwPageSize)==0 ); @@ -3705,8 +3702,8 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ pFd->lastErrno = osGetLastError(); winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, "winMapfile", pFd->zPath); - OSTRACE(("MAP-FILE-MAP pid=%d, pFile=%p, rc=SQLITE_IOERR_MMAP\n", - (int)osGetCurrentProcessId(), pFd)); + OSTRACE(("MAP-FILE-MAP pid=%lu, pFile=%p, rc=SQLITE_IOERR_MMAP\n", + osGetCurrentProcessId(), pFd)); return SQLITE_OK; } pFd->pMapRegion = pNew; @@ -3714,8 +3711,8 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ pFd->mmapSizeActual = nMap; } - OSTRACE(("MAP-FILE pid=%d, pFile=%p, rc=SQLITE_OK\n", - (int)osGetCurrentProcessId(), pFd)); + OSTRACE(("MAP-FILE pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), pFd)); return SQLITE_OK; } #endif /* SQLITE_MAX_MMAP_SIZE>0 */ @@ -3730,7 +3727,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ ** value of *pp is undefined in this case. ** ** If this function does return a pointer, the caller must eventually -** release the reference by calling unixUnfetch(). +** release the reference by calling winUnfetch(). */ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ #if SQLITE_MAX_MMAP_SIZE>0 @@ -3738,16 +3735,16 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ #endif *pp = 0; - OSTRACE(("FETCH pid=%d, pFile=%p, offset=%lld, amount=%d, pp=%p\n", - (int)osGetCurrentProcessId(), fd, iOff, nAmt, pp)); + OSTRACE(("FETCH pid=%lu, pFile=%p, offset=%lld, amount=%d, pp=%p\n", + osGetCurrentProcessId(), fd, iOff, nAmt, pp)); #if SQLITE_MAX_MMAP_SIZE>0 if( pFd->mmapSizeMax>0 ){ if( pFd->pMapRegion==0 ){ int rc = winMapfile(pFd, -1); if( rc!=SQLITE_OK ){ - OSTRACE(("FETCH pid=%d, pFile=%p, rc=%s\n", - (int)osGetCurrentProcessId(), pFd, sqlite3ErrName(rc))); + OSTRACE(("FETCH pid=%lu, pFile=%p, rc=%s\n", + osGetCurrentProcessId(), pFd, sqlite3ErrName(rc))); return rc; } } @@ -3758,16 +3755,16 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ } #endif - OSTRACE(("FETCH pid=%d, pFile=%p, pp=%p, *pp=%p, rc=SQLITE_OK\n", - (int)osGetCurrentProcessId(), fd, pp, pp ? *pp : 0)); + OSTRACE(("FETCH pid=%lu, pFile=%p, pp=%p, *pp=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), fd, pp, *pp)); return SQLITE_OK; } /* ** If the third argument is non-NULL, then this function releases a -** reference obtained by an earlier call to unixFetch(). The second +** reference obtained by an earlier call to winFetch(). The second ** argument passed to this function must be the same as the corresponding -** argument that was passed to the unixFetch() invocation. +** argument that was passed to the winFetch() invocation. ** ** Or, if the third argument is NULL, then this function is being called ** to inform the VFS layer that, according to POSIX, any existing mapping @@ -3785,8 +3782,8 @@ static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ /* If p!=0, it must match the iOff value. */ assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); - OSTRACE(("UNFETCH pid=%d, pFile=%p, offset=%lld, p=%p\n", - (int)osGetCurrentProcessId(), pFd, iOff, p)); + OSTRACE(("UNFETCH pid=%lu, pFile=%p, offset=%lld, p=%p\n", + osGetCurrentProcessId(), pFd, iOff, p)); if( p ){ pFd->nFetchOut--; @@ -3801,8 +3798,8 @@ static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ assert( pFd->nFetchOut>=0 ); #endif - OSTRACE(("UNFETCH pid=%d, pFile=%p, rc=SQLITE_OK\n", - (int)osGetCurrentProcessId(), fd)); + OSTRACE(("UNFETCH pid=%lu, pFile=%p, rc=SQLITE_OK\n", + osGetCurrentProcessId(), fd)); return SQLITE_OK; } @@ -4073,8 +4070,7 @@ static int winOpen( memset(zTmpname, 0, MAX_PATH+2); rc = getTempname(MAX_PATH+2, zTmpname); if( rc!=SQLITE_OK ){ - OSTRACE(("OPEN name=%s, rc=%s", - zUtf8Name, sqlite3ErrName(rc))); + OSTRACE(("OPEN name=%s, rc=%s", zUtf8Name, sqlite3ErrName(rc))); return rc; } zUtf8Name = zTmpname; @@ -4212,6 +4208,10 @@ static int winOpen( } } + OSTRACE(("OPEN file=%p, name=%s, access=%lx, pOutFlags=%p, *pOutFlags=%d, " + "rc=%s\n", h, zUtf8Name, dwDesiredAccess, pOutFlags, pOutFlags ? + *pOutFlags : 0, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); + #if SQLITE_OS_WINCE if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB && (rc = winceCreateLock(zName, pFile))!=SQLITE_OK diff --git a/src/test1.c b/src/test1.c index 34370d48e4..59de1db8e5 100644 --- a/src/test1.c +++ b/src/test1.c @@ -113,82 +113,8 @@ int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb){ return TCL_OK; } - -const char *sqlite3TestErrorName(int rc){ - const char *zName = 0; - int i; - for(i=0; i<2 && zName==0; i++, rc &= 0xff){ - switch( rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; - case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; - case SQLITE_CONSTRAINT_FOREIGNKEY: - zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; - case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; - case SQLITE_CONSTRAINT_PRIMARYKEY: - zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; - case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; - case SQLITE_CONSTRAINT_COMMITHOOK: - zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; - case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; - case SQLITE_CONSTRAINT_FUNCTION: zName = "SQLITE_CONSTRAINT_FUNCTION";break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; - case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; - case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; - case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; - case SQLITE_ROW: zName = "SQLITE_ROW"; break; - case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; - case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; - case SQLITE_DONE: zName = "SQLITE_DONE"; break; - case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; - case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; - case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; - case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; - case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; - case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; - case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; - case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; - case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; - case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; - case SQLITE_IOERR_BLOCKED: zName = "SQLITE_IOERR_BLOCKED"; break; - case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; - case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; - case SQLITE_IOERR_CHECKRESERVEDLOCK: - zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; - case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; - case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; - case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; - case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; - case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; - } - } - if( zName==0 ) zName = "SQLITE_Unknown"; - return zName; -} -#define t1ErrorName sqlite3TestErrorName +extern const char *sqlite3ErrName(int); +#define t1ErrorName sqlite3ErrName /* ** Convert an sqlite3_stmt* into an sqlite3*. This depends on the @@ -1733,7 +1659,7 @@ static int test_blob_read( if( rc==SQLITE_OK ){ Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(zBuf, nByte)); }else{ - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); } Tcl_Free((char *)zBuf); @@ -1783,7 +1709,7 @@ static int test_blob_write( } rc = sqlite3_blob_write(pBlob, zBuf, nBuf, iOffset); if( rc!=SQLITE_OK ){ - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); } return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); @@ -1809,7 +1735,7 @@ static int test_blob_reopen( rc = sqlite3_blob_reopen(pBlob, iRowid); if( rc!=SQLITE_OK ){ - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); } return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); @@ -2019,7 +1945,7 @@ static int test_create_function_v2( ); if( rc!=SQLITE_OK ){ Tcl_ResetResult(interp); - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -2695,7 +2621,7 @@ static int test_collate( if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR; if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -3253,7 +3179,7 @@ static int test_bind_text( rc = sqlite3_bind_text(pStmt, idx, value, bytes, SQLITE_TRANSIENT); if( sqlite3TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR; if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } @@ -3301,7 +3227,7 @@ static int test_bind_text16( rc = sqlite3_bind_text16(pStmt, idx, (void *)value, bytes, xDel); if( sqlite3TestErrCode(interp, StmtToDb(pStmt), rc) ) return TCL_ERROR; if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } @@ -4575,7 +4501,7 @@ static int test_busy_timeout( if( getDbPointer(interp, argv[1], &db) ) return TCL_ERROR; if( Tcl_GetInt(interp, argv[2], &ms) ) return TCL_ERROR; rc = sqlite3_busy_timeout(db, ms); - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_OK; } @@ -5096,7 +5022,7 @@ static int file_control_chunksize_test( rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_CHUNK_SIZE, (void *)&nSize); if( rc ){ - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_ERROR; } return TCL_OK; @@ -5133,7 +5059,7 @@ static int file_control_sizehint_test( rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_SIZE_HINT, (void *)&nSize); if( rc ){ - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_ERROR; } return TCL_OK; @@ -5677,7 +5603,7 @@ static void xLogcallback(void *unused, int err, char *zMsg){ Tcl_Obj *pNew = Tcl_DuplicateObj(logcallback.pObj); Tcl_IncrRefCount(pNew); Tcl_ListObjAppendElement( - 0, pNew, Tcl_NewStringObj(sqlite3TestErrorName(err), -1) + 0, pNew, Tcl_NewStringObj(sqlite3ErrName(err), -1) ); Tcl_ListObjAppendElement(0, pNew, Tcl_NewStringObj(zMsg, -1)); Tcl_EvalObjEx(logcallback.pInterp, pNew, TCL_EVAL_GLOBAL|TCL_EVAL_DIRECT); diff --git a/src/test2.c b/src/test2.c index 3faec12619..d130e9d01b 100644 --- a/src/test2.c +++ b/src/test2.c @@ -19,7 +19,7 @@ #include #include -extern const char *sqlite3TestErrorName(int rc); +extern const char *sqlite3ErrName(int); /* ** Page size and reserved size used for testing. @@ -59,7 +59,7 @@ static int pager_open( SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MAIN_DB, pager_test_reiniter); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } sqlite3PagerSetCachesize(pPager, nPage); @@ -91,7 +91,7 @@ static int pager_close( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerClose(pPager); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -118,7 +118,7 @@ static int pager_rollback( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerRollback(pPager); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -145,12 +145,12 @@ static int pager_commit( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerCommitPhaseOne(pPager, 0, 0); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } rc = sqlite3PagerCommitPhaseTwo(pPager); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -177,7 +177,7 @@ static int pager_stmt_begin( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerOpenSavepoint(pPager, 1); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -205,7 +205,7 @@ static int pager_stmt_rollback( rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, 0); sqlite3PagerSavepoint(pPager, SAVEPOINT_RELEASE, 0); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -232,7 +232,7 @@ static int pager_stmt_commit( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_RELEASE, 0); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -325,7 +325,7 @@ static int page_get( rc = sqlite3PagerGet(pPager, pgno, &pPage); } if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf),zBuf,"%p",pPage); @@ -479,7 +479,7 @@ static int page_write( pPage = (DbPage *)sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerWrite(pPage); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } pData = sqlite3PagerGetData(pPage); @@ -528,7 +528,7 @@ static int fake_big_file( (SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_DB), 0 ); if( rc ){ - Tcl_AppendResult(interp, "open failed: ", sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, "open failed: ", sqlite3ErrName(rc), 0); sqlite3_free(zFile); return TCL_ERROR; } @@ -538,7 +538,7 @@ static int fake_big_file( sqlite3OsCloseFree(fd); sqlite3_free(zFile); if( rc ){ - Tcl_AppendResult(interp, "write failed: ", sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, "write failed: ", sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; diff --git a/src/test3.c b/src/test3.c index ecc3a543be..e3ed310c81 100644 --- a/src/test3.c +++ b/src/test3.c @@ -19,7 +19,7 @@ #include #include -extern const char *sqlite3TestErrorName(int rc); +extern const char *sqlite3ErrName(int); /* ** A bogus sqlite3 connection structure for use in the btree @@ -65,7 +65,7 @@ static int btree_open( SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MAIN_DB); sqlite3_free(zFilename); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } sqlite3BtreeSetCacheSize(pBt, nCache); @@ -95,7 +95,7 @@ static int btree_close( pBt = sqlite3TestTextToPtr(argv[1]); rc = sqlite3BtreeClose(pBt); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } nRefSqlite3--; @@ -132,7 +132,7 @@ static int btree_begin_transaction( rc = sqlite3BtreeBeginTrans(pBt, 1); sqlite3BtreeLeave(pBt); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -226,7 +226,7 @@ static int btree_cursor( sqlite3BtreeLeave(pBt); if( rc ){ ckfree((char *)pCur); - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf), zBuf,"%p", pCur); @@ -261,7 +261,7 @@ static int btree_close_cursor( sqlite3BtreeLeave(pBt); ckfree((char *)pCur); if( rc ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } return SQLITE_OK; @@ -295,7 +295,7 @@ static int btree_next( rc = sqlite3BtreeNext(pCur, &res); sqlite3BtreeLeave(pCur->pBtree); if( rc ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf),zBuf,"%d",res); @@ -330,7 +330,7 @@ static int btree_first( rc = sqlite3BtreeFirst(pCur, &res); sqlite3BtreeLeave(pCur->pBtree); if( rc ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), 0); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf),zBuf,"%d",res); diff --git a/src/test4.c b/src/test4.c index 83c56d3d7e..a6375c7cc4 100644 --- a/src/test4.c +++ b/src/test4.c @@ -20,7 +20,7 @@ #include #include -extern const char *sqlite3TestErrorName(int rc); +extern const char *sqlite3ErrName(int); /* ** Each thread is controlled by an instance of the following @@ -374,7 +374,7 @@ static int tcl_thread_result( return TCL_ERROR; } thread_wait(&threadset[i]); - zName = sqlite3TestErrorName(threadset[i].rc); + zName = sqlite3ErrName(threadset[i].rc); Tcl_AppendResult(interp, zName, 0); return TCL_OK; } diff --git a/src/test7.c b/src/test7.c index 707d2c89af..3cd4a224d7 100644 --- a/src/test7.c +++ b/src/test7.c @@ -376,7 +376,7 @@ static int tcl_client_colname( return TCL_OK; } -extern const char *sqlite3TestErrorName(int rc); +extern const char *sqlite3ErrName(int); /* ** Usage: client_result ID @@ -405,7 +405,7 @@ static int tcl_client_result( return TCL_ERROR; } client_wait(&threadset[i]); - zName = sqlite3TestErrorName(threadset[i].rc); + zName = sqlite3ErrName(threadset[i].rc); Tcl_AppendResult(interp, zName, 0); return TCL_OK; } diff --git a/src/test8.c b/src/test8.c index f7b2e6813d..c5739332d4 100644 --- a/src/test8.c +++ b/src/test8.c @@ -1300,7 +1300,7 @@ static sqlite3_module echoModuleV2 = { ** Decode a pointer to an sqlite3 object. */ extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); -extern const char *sqlite3TestErrorName(int rc); +extern const char *sqlite3ErrName(int); static void moduleDestroy(void *p){ sqlite3_free(p); @@ -1340,7 +1340,7 @@ static int register_echo_module( ); } - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; } diff --git a/src/test_async.c b/src/test_async.c index c760eea1fa..b0b943185b 100644 --- a/src/test_async.c +++ b/src/test_async.c @@ -23,8 +23,8 @@ #include "sqlite3.h" #include -/* From test1.c */ -const char *sqlite3TestErrorName(int); +/* From main.c */ +extern const char *sqlite3ErrName(int); struct TestAsyncGlobal { @@ -60,7 +60,7 @@ static int testAsyncInit( rc = sqlite3async_initialize(zParent, isDefault); if( rc!=SQLITE_OK ){ - Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3TestErrorName(rc), -1)); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); return TCL_ERROR; } return TCL_OK; @@ -208,7 +208,7 @@ static int testAsyncControl( } if( rc!=SQLITE_OK ){ - Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3TestErrorName(rc), -1)); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); return TCL_ERROR; } diff --git a/src/test_backup.c b/src/test_backup.c index 272713785a..e967424a29 100644 --- a/src/test_backup.c +++ b/src/test_backup.c @@ -17,9 +17,11 @@ #include #include +/* These functions are implemented in main.c. */ +extern const char *sqlite3ErrName(int); + /* These functions are implemented in test1.c. */ -int getDbPointer(Tcl_Interp *, const char *, sqlite3 **); -const char *sqlite3TestErrorName(int); +extern int getDbPointer(Tcl_Interp *, const char *, sqlite3 **); static int backupTestCmd( ClientData clientData, @@ -70,7 +72,7 @@ static int backupTestCmd( Tcl_DeleteCommand(interp, zCmdName); rc = sqlite3_backup_finish(p); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); break; } @@ -80,7 +82,7 @@ static int backupTestCmd( return TCL_ERROR; } rc = sqlite3_backup_step(p, nPage); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); break; } diff --git a/src/test_intarray.c b/src/test_intarray.c index 6ea7756176..f5c3d9e405 100644 --- a/src/test_intarray.c +++ b/src/test_intarray.c @@ -278,7 +278,7 @@ int sqlite3_intarray_bind( extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); extern void *sqlite3TestTextToPtr(const char*); extern int sqlite3TestMakePointerStr(Tcl_Interp*, char *zPtr, void*); -extern const char *sqlite3TestErrorName(int); +extern const char *sqlite3ErrName(int); /* ** sqlite3_intarray_create DB NAME @@ -309,7 +309,7 @@ static int test_intarray_create( #endif if( rc!=SQLITE_OK ){ assert( pArray==0 ); - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), (char*)0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), (char*)0); return TCL_ERROR; } sqlite3TestMakePointerStr(interp, zPtr, pArray); @@ -352,7 +352,7 @@ static int test_intarray_bind( } rc = sqlite3_intarray_bind(pArray, n, a, sqlite3_free); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, sqlite3TestErrorName(rc), (char*)0); + Tcl_AppendResult(interp, sqlite3ErrName(rc), (char*)0); return TCL_ERROR; } #endif diff --git a/src/test_malloc.c b/src/test_malloc.c index 33de859d27..cf98a8fb21 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -234,14 +234,14 @@ static int faultsimInstall(int install){ #ifdef SQLITE_TEST /* -** This function is implemented in test1.c. Returns a pointer to a static +** This function is implemented in main.c. Returns a pointer to a static ** buffer containing the symbolic SQLite error code that corresponds to ** the least-significant 8-bits of the integer passed as an argument. ** For example: ** -** sqlite3TestErrorName(1) -> "SQLITE_ERROR" +** sqlite3ErrName(1) -> "SQLITE_ERROR" */ -const char *sqlite3TestErrorName(int); +extern const char *sqlite3ErrName(int); /* ** Transform pointers to text and back again @@ -1072,7 +1072,7 @@ static int test_db_config_lookaside( sqlite3 *db; int bufid; static char azBuf[2][10000]; - int getDbPointer(Tcl_Interp*, const char*, sqlite3**); + extern int getDbPointer(Tcl_Interp*, const char*, sqlite3**); if( objc!=5 ){ Tcl_WrongNumArgs(interp, 1, objv, "BUFID SIZE COUNT"); return TCL_ERROR; @@ -1126,7 +1126,7 @@ static int test_config_heap( rc = sqlite3_config(SQLITE_CONFIG_HEAP, zBuf, nByte, nMinAlloc); } - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } @@ -1143,7 +1143,7 @@ static int test_config_error( Tcl_Obj *CONST objv[] ){ sqlite3 *db; - int getDbPointer(Tcl_Interp*, const char*, sqlite3**); + extern int getDbPointer(Tcl_Interp*, const char*, sqlite3**); if( objc!=2 && objc!=1 ){ Tcl_WrongNumArgs(interp, 1, objv, "[DB]"); @@ -1192,7 +1192,7 @@ static int test_config_uri( } rc = sqlite3_config(SQLITE_CONFIG_URI, bOpenUri); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } @@ -1221,7 +1221,7 @@ static int test_config_cis( } rc = sqlite3_config(SQLITE_CONFIG_COVERING_INDEX_SCAN, bUseCis); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } @@ -1335,7 +1335,7 @@ static int test_db_status( int i, op, resetFlag; const char *zOpName; sqlite3 *db; - int getDbPointer(Tcl_Interp*, const char*, sqlite3**); + extern int getDbPointer(Tcl_Interp*, const char*, sqlite3**); static const struct { const char *zName; int op; @@ -1401,7 +1401,7 @@ static int test_install_malloc_faultsim( return TCL_ERROR; } rc = faultsimInstall(isInstall); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } @@ -1419,7 +1419,7 @@ static int test_install_memsys3( const sqlite3_mem_methods *sqlite3MemGetMemsys3(void); rc = sqlite3_config(SQLITE_CONFIG_MALLOC, sqlite3MemGetMemsys3()); #endif - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } diff --git a/src/test_multiplex.c b/src/test_multiplex.c index 18a3a0740f..624541b32a 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -1183,7 +1183,7 @@ int sqlite3_multiplex_shutdown(void){ /***************************** Test Code ***********************************/ #ifdef SQLITE_TEST #include -extern const char *sqlite3TestErrorName(int); +extern const char *sqlite3ErrName(int); /* @@ -1212,7 +1212,7 @@ static int test_multiplex_initialize( /* Call sqlite3_multiplex_initialize() */ rc = sqlite3_multiplex_initialize(zName, makeDefault); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; } @@ -1237,7 +1237,7 @@ static int test_multiplex_shutdown( /* Call sqlite3_multiplex_shutdown() */ rc = sqlite3_multiplex_shutdown(); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; } @@ -1355,7 +1355,7 @@ static int test_multiplex_control( } rc = sqlite3_file_control(db, Tcl_GetString(objv[2]), aSub[idx].op, pArg); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return (rc==SQLITE_OK) ? TCL_OK : TCL_ERROR; } diff --git a/src/test_mutex.c b/src/test_mutex.c index 0bb74375d8..c9b4a29ab7 100644 --- a/src/test_mutex.c +++ b/src/test_mutex.c @@ -19,8 +19,8 @@ #include #include -/* defined in test1.c */ -const char *sqlite3TestErrorName(int); +/* defined in main.c */ +extern const char *sqlite3ErrName(int); /* A countable mutex */ struct sqlite3_mutex { @@ -148,7 +148,7 @@ static int test_shutdown( } rc = sqlite3_shutdown(); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } @@ -169,7 +169,7 @@ static int test_initialize( } rc = sqlite3_initialize(); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } @@ -230,7 +230,7 @@ static int test_install_mutex_counters( g.isInstalled = isInstall; } - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } @@ -354,7 +354,7 @@ static int test_config( } rc = sqlite3_config(i); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_VOLATILE); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); return TCL_OK; } diff --git a/src/test_quota.c b/src/test_quota.c index 58169e17cc..e590996ca4 100644 --- a/src/test_quota.c +++ b/src/test_quota.c @@ -1325,7 +1325,7 @@ struct TclQuotaCallback { Tcl_Obj *pScript; /* Script to be run */ }; -extern const char *sqlite3TestErrorName(int); +extern const char *sqlite3ErrName(int); /* @@ -1407,7 +1407,7 @@ static int test_quota_initialize( /* Call sqlite3_quota_initialize() */ rc = sqlite3_quota_initialize(zName, makeDefault); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; } @@ -1430,7 +1430,7 @@ static int test_quota_shutdown( /* Call sqlite3_quota_shutdown() */ rc = sqlite3_quota_shutdown(); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; } @@ -1485,7 +1485,7 @@ static int test_quota_set( /* Invoke sqlite3_quota_set() */ rc = sqlite3_quota_set(zPattern, iLimit, xCallback, (void*)p, xDestroy); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; } @@ -1511,7 +1511,7 @@ static int test_quota_file( /* Invoke sqlite3_quota_file() */ rc = sqlite3_quota_file(zFilename); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; } diff --git a/src/test_rtree.c b/src/test_rtree.c index d3c9e0cb3d..f54ae9b063 100644 --- a/src/test_rtree.c +++ b/src/test_rtree.c @@ -254,7 +254,7 @@ static int register_cube_geom( UNUSED_PARAMETER(objv); #else extern int getDbPointer(Tcl_Interp*, const char*, sqlite3**); - extern const char *sqlite3TestErrorName(int); + extern const char *sqlite3ErrName(int); sqlite3 *db; int rc; @@ -264,7 +264,7 @@ static int register_cube_geom( } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; rc = sqlite3_rtree_geometry_callback(db, "cube", cube_geom, (void *)&gHere); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); #endif return TCL_OK; } @@ -282,7 +282,7 @@ static int register_circle_geom( UNUSED_PARAMETER(objv); #else extern int getDbPointer(Tcl_Interp*, const char*, sqlite3**); - extern const char *sqlite3TestErrorName(int); + extern const char *sqlite3ErrName(int); sqlite3 *db; int rc; @@ -292,7 +292,7 @@ static int register_circle_geom( } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; rc = sqlite3_rtree_geometry_callback(db, "circle", circle_geom, 0); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), TCL_STATIC); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); #endif return TCL_OK; } diff --git a/src/test_syscall.c b/src/test_syscall.c index 3d117fb2ab..4d62c737af 100644 --- a/src/test_syscall.c +++ b/src/test_syscall.c @@ -78,8 +78,8 @@ #include "sqliteInt.h" #if SQLITE_OS_UNIX -/* From test1.c */ -extern const char *sqlite3TestErrorName(int); +/* From main.c */ +extern const char *sqlite3ErrName(int); #include #include @@ -498,7 +498,7 @@ static int test_syscall_reset( } } if( rc!=SQLITE_OK ){ - Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3TestErrorName(rc), -1)); + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); return TCL_ERROR; } diff --git a/src/test_thread.c b/src/test_thread.c index ae62de8262..2f9363b750 100644 --- a/src/test_thread.c +++ b/src/test_thread.c @@ -60,12 +60,14 @@ static Tcl_ObjCmdProc blocking_prepare_v2_proc; int Sqlitetest1_Init(Tcl_Interp *); int Sqlite3_Init(Tcl_Interp *); +/* Functions from main.c */ +extern const char *sqlite3ErrName(int); + /* Functions from test1.c */ -void *sqlite3TestTextToPtr(const char *); -const char *sqlite3TestErrorName(int); -int getDbPointer(Tcl_Interp *, const char *, sqlite3 **); -int sqlite3TestMakePointerStr(Tcl_Interp *, char *, void *); -int sqlite3TestErrCode(Tcl_Interp *, sqlite3 *, int); +extern void *sqlite3TestTextToPtr(const char *); +extern int getDbPointer(Tcl_Interp *, const char *, sqlite3 **); +extern int sqlite3TestMakePointerStr(Tcl_Interp *, char *, void *); +extern int sqlite3TestErrCode(Tcl_Interp *, sqlite3 *, int); /* ** Handler for events of type EvalEvent. @@ -559,7 +561,7 @@ static int blocking_step_proc( pStmt = (sqlite3_stmt*)sqlite3TestTextToPtr(Tcl_GetString(objv[1])); rc = sqlite3_blocking_step(pStmt); - Tcl_SetResult(interp, (char *)sqlite3TestErrorName(rc), 0); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), 0); return TCL_OK; } @@ -606,7 +608,7 @@ static int blocking_prepare_v2_proc( } if( rc!=SQLITE_OK ){ assert( pStmt==0 ); - sprintf(zBuf, "%s ", (char *)sqlite3TestErrorName(rc)); + sprintf(zBuf, "%s ", (char *)sqlite3ErrName(rc)); Tcl_AppendResult(interp, zBuf, sqlite3_errmsg(db), 0); return TCL_ERROR; } From 4eb9b721515b27a9197fcae481cd2961ab71edcd Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 29 Apr 2013 07:01:23 +0000 Subject: [PATCH 179/204] Update 'fuzzerfault' test for its new module loading command. Fix several test names in 'io.test' and make sure the database gets closed between tests. FossilOrigin-Name: e81e9ca11db09424dd310bbc91686a5daa618cb1 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- test/fuzzerfault.test | 8 ++++---- test/io.test | 2 ++ 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 16c565ef04..3db186dfa5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sspellfix\svirtual\stable\sdocumentation\sfrom\sthe\ssource\stree.\nReference\sthe\sseparate\sdocumentation\son\sthe\swebsite\sinstead. -D 2013-04-27T18:06:40.561 +C Update\s'fuzzerfault'\stest\sfor\sits\snew\smodule\sloading\scommand.\s\sFix\sseveral\stest\snames\sin\s'io.test'\sand\smake\ssure\sthe\sdatabase\sgets\sclosed\sbetween\stests. +D 2013-04-29T07:01:23.073 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -534,7 +534,7 @@ F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 F test/fuzzer1.test 41bd5aa6ae0cf18d06342a4476e3cad98604ae48 -F test/fuzzerfault.test f96773292b82ce40f7e6614f04ab2166c4ed3c28 +F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536 F test/hook.test 45cb22b940c3cc0af616ba7430f666e245711a48 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 5941096407d8c133b9eff15bd3e666624b6cbde3 @@ -569,7 +569,7 @@ F test/instr.test a34e1d46a9eefb098a7167ef0e730a4a3d82fba0 F test/intarray.test 066b7d7ac38d25bf96f87f1b017bfc687551cdd4 F test/interrupt.test dfe9a67a94b0b2d8f70545ba1a6cca10780d71cc F test/intpkey.test 7af30f6ae852d8d1c2b70e4bf1551946742e92d8 -F test/io.test b90105d295225a712e2e9cb685876101712058b5 +F test/io.test 0147ed5fdbce3286d6128f5f7e3b76ee8352d652 F test/ioerr.test 40bb2cfcab63fb6aa7424cd97812a84bc16b5fb8 F test/ioerr2.test 9d71166f8466eda510f1af6137bdabaa82b5408d F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f14d55cf358b0392d3b8cd61dc85f43a610a8edf -R 1d8237b86dffdd5327eb650dcb291120 -U drh -Z 369cfa66c298e6ca3c71b08895115463 +P adcf78909ff9064b6e3c4dd15ccd3245c8cf270b +R 263b9a6fce3f45179cbdd8291a5dbd19 +U mistachkin +Z c1a76ab097fbf09032c65307a2000f74 diff --git a/manifest.uuid b/manifest.uuid index edb252e518..d77da23add 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -adcf78909ff9064b6e3c4dd15ccd3245c8cf270b \ No newline at end of file +e81e9ca11db09424dd310bbc91686a5daa618cb1 \ No newline at end of file diff --git a/test/fuzzerfault.test b/test/fuzzerfault.test index 50450ede7c..6449612a66 100644 --- a/test/fuzzerfault.test +++ b/test/fuzzerfault.test @@ -30,7 +30,7 @@ do_test 1-pre1 { } {} do_faultsim_test 1 -prep { faultsim_restore_and_reopen - register_fuzzer_module db + load_static_extension db fuzzer } -body { execsql { CREATE VIRTUAL TABLE x1 USING fuzzer(x1_rules); @@ -43,7 +43,7 @@ do_faultsim_test 1 -prep { do_test 2-pre1 { faultsim_delete_and_reopen - register_fuzzer_module db + load_static_extension db fuzzer execsql { CREATE TABLE x2_rules(ruleset, cFrom, cTo, cost); INSERT INTO x2_rules VALUES(0, 'a', 'x', 1); @@ -56,7 +56,7 @@ do_test 2-pre1 { do_faultsim_test 2 -prep { faultsim_restore_and_reopen - register_fuzzer_module db + load_static_extension db fuzzer } -body { execsql { SELECT count(*) FROM x2 WHERE word MATCH 'abc'; @@ -78,7 +78,7 @@ do_test 3-pre1 { do_faultsim_test 3 -prep { faultsim_restore_and_reopen - register_fuzzer_module db + load_static_extension db fuzzer } -body { execsql { CREATE VIRTUAL TABLE x1 USING fuzzer(x1_rules); diff --git a/test/io.test b/test/io.test index f9583e0e33..43289567f1 100644 --- a/test/io.test +++ b/test/io.test @@ -16,6 +16,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set ::testprefix io db close sqlite3_simulate_device @@ -628,6 +629,7 @@ foreach {tn sql} { # to be flushed. Which is a bug. hexio_write test.db [expr 1024 * 5] [string repeat 00 2048] do_execsql_test 6.2.$tn.3 { PRAGMA integrity_check } {ok} + db close } sqlite3_simulate_device -char {} -sectorsize 0 From 56749cd1ba216888fd2074882e1b5d382354bb75 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 29 Apr 2013 08:58:00 +0000 Subject: [PATCH 180/204] Adjust a WAL test result due to changes in the corresponding return code name. FossilOrigin-Name: 9272009f7932b3f18006f73776e36b8ef8770d3e --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/wal.test | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 58c96ec2af..a1389464eb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\supdates\sfrom\strunk. -D 2013-04-29T08:56:49.008 +C Adjust\sa\sWAL\stest\sresult\sdue\sto\schanges\sin\sthe\scorresponding\sreturn\scode\sname. +D 2013-04-29T08:58:00.604 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -971,7 +971,7 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 -F test/wal.test 0b4837cd5e9283c116d30810a6859bed7425a95e +F test/wal.test e6dcf26e3e5cce2adb2f2f57eda53bc2c54c580c F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 05ba5e23fcc5da416d47a8ad5e667ed6a0c7a34a e81e9ca11db09424dd310bbc91686a5daa618cb1 -R cbea6b19af175a78e920dc5a0be06f24 +P 3a245a832d4843802c3c57f8cda3e4e1d0d52622 +R 1a463c1687098b31a5ca5a9b6654d1e1 U mistachkin -Z c57c4f0f878083d5f09ceb7503607ae1 +Z 6828b536c59e8064e8c8cfbf273b3447 diff --git a/manifest.uuid b/manifest.uuid index 5d4dbaeefe..b11e58d75d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3a245a832d4843802c3c57f8cda3e4e1d0d52622 \ No newline at end of file +9272009f7932b3f18006f73776e36b8ef8770d3e \ No newline at end of file diff --git a/test/wal.test b/test/wal.test index d9f2145b2e..8ce63cf824 100644 --- a/test/wal.test +++ b/test/wal.test @@ -1515,7 +1515,8 @@ do_test wal-23.3 { set nPage [expr 2+$AUTOVACUUM] do_test wal-23.4 { set ::log -} [list SQLITE_NOTICE "recovered $nPage frames from WAL file $walfile"] +} [list SQLITE_NOTICE_RECOVER_WAL \ + "recovered $nPage frames from WAL file $walfile"] ifcapable autovacuum { From f054396b38ae93ffb831896368b8226e29ae75d0 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 29 Apr 2013 09:17:42 +0000 Subject: [PATCH 181/204] Fix mmap1.test so that it passes on windows as well as unix. FossilOrigin-Name: 52417eac3ecaec2dbbde170334358f5ddbd32501 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/mmap1.test | 18 ++++++++++++------ 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 3db186dfa5..2c3620bed6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\s'fuzzerfault'\stest\sfor\sits\snew\smodule\sloading\scommand.\s\sFix\sseveral\stest\snames\sin\s'io.test'\sand\smake\ssure\sthe\sdatabase\sgets\sclosed\sbetween\stests. -D 2013-04-29T07:01:23.073 +C Fix\smmap1.test\sso\sthat\sit\spasses\son\swindows\sas\swell\sas\sunix. +D 2013-04-29T09:17:42.276 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -646,7 +646,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 0b5802cf64acaa509ea889b3c708196cc6eb9d31 +F test/mmap1.test 8696aa1b0bd88961c2f16af2a3f7a69d701cea50 F test/mmap2.test a5ba639f90b5fc487400a49e158e14e465943e98 F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P adcf78909ff9064b6e3c4dd15ccd3245c8cf270b -R 263b9a6fce3f45179cbdd8291a5dbd19 -U mistachkin -Z c1a76ab097fbf09032c65307a2000f74 +P e81e9ca11db09424dd310bbc91686a5daa618cb1 +R 0b5ca68b3c8f8712e6440b0981a1650e +U dan +Z 3af177977b6ccb1ab6aa9597c01ddcc6 diff --git a/manifest.uuid b/manifest.uuid index d77da23add..b4bcb2b27e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e81e9ca11db09424dd310bbc91686a5daa618cb1 \ No newline at end of file +52417eac3ecaec2dbbde170334358f5ddbd32501 \ No newline at end of file diff --git a/test/mmap1.test b/test/mmap1.test index 0d0e3fd7a4..09b6d95927 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -40,14 +40,20 @@ proc register_rblob_code {dbname seed} { }] } +# For cases 1.1 and 1.4, the number of pages read using xRead() is 4 on +# unix and 9 on windows. The difference is that windows only ever maps +# an integer number of OS pages (i.e. creates mappings that are a multiple +# of 4KB in size). Whereas on unix any sized mapping may be created. +# foreach {t mmap_size nRead c2init} { - 1.1 { PRAGMA mmap_size = 67108864 } 4 {PRAGMA mmap_size = 0} - 1.2 { PRAGMA mmap_size = 53248 } 150 {PRAGMA mmap_size = 0} - 1.3 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = 0} - 1.4 { PRAGMA mmap_size = 67108864 } 4 {PRAGMA mmap_size = 67108864 } - 1.5 { PRAGMA mmap_size = 53248 } 150 {PRAGMA mmap_size = 67108864 } - 1.6 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = 67108864 } + 1.1 { PRAGMA mmap_size = 67108864 } /[49]/ {PRAGMA mmap_size = 0} + 1.2 { PRAGMA mmap_size = 53248 } 150 {PRAGMA mmap_size = 0} + 1.3 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = 0} + 1.4 { PRAGMA mmap_size = 67108864 } /[49]/ {PRAGMA mmap_size = 67108864 } + 1.5 { PRAGMA mmap_size = 53248 } 150 {PRAGMA mmap_size = 67108864 } + 1.6 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = 67108864 } } { + do_multiclient_test tn { sql1 {PRAGMA page_size=1024} sql1 $mmap_size From 181f4f789d503b014bc1ec662fe0ef3383b03094 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 29 Apr 2013 17:12:06 +0000 Subject: [PATCH 182/204] Fix an off-by-one in the code for limiting the depth of FTS expression trees. FossilOrigin-Name: 72ac73189c3577740a77d2ea2fc7118391c0703f --- ext/fts3/fts3_expr.c | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- test/fts3expr3.test | 14 +++++++++++--- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index b28ab68e4b..2cb0f94bcb 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -755,7 +755,7 @@ exprparse_out: static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){ int rc = SQLITE_OK; if( p ){ - if( nMaxDepth==0 ){ + if( nMaxDepth<0 ){ rc = SQLITE_ERROR; }else{ rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1); diff --git a/manifest b/manifest index 2c3620bed6..7a2fc313d4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\smmap1.test\sso\sthat\sit\spasses\son\swindows\sas\swell\sas\sunix. -D 2013-04-29T09:17:42.276 +C Fix\san\soff-by-one\sin\sthe\scode\sfor\slimiting\sthe\sdepth\sof\sFTS\sexpression\strees. +D 2013-04-29T17:12:06.416 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -59,7 +59,7 @@ F ext/fts3/fts3.c 784aadfb4c2a217c3eb1feaecac924989f29728f F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 352c8a83ee4c6a14ced1759a39dd890ab947cbe0 F ext/fts3/fts3_aux.c b02632f6dd0e375ce97870206d914ea6d8df5ccd -F ext/fts3/fts3_expr.c 4021d21aadebb50499c482f6b7b718e4b93113e6 +F ext/fts3/fts3_expr.c 2449c3f31b87c3376b66999cdd10d23be0b0bd42 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5 @@ -499,7 +499,7 @@ F test/fts3drop.test 1b906e293d6773812587b3dc458cb9e8f3f0c297 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a -F test/fts3expr3.test aed4e71aa40ac1dcba860d1c43e7e23aa812eb1c +F test/fts3expr3.test 1bfb762b53a794f990f3dffaae8bbea5736422f7 F test/fts3fault.test cb72dccb0a3b9f730f16c5240f3fcb9303eb1660 F test/fts3fault2.test 3198eef2804deea7cac8403e771d9cbcb752d887 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e81e9ca11db09424dd310bbc91686a5daa618cb1 -R 0b5ca68b3c8f8712e6440b0981a1650e +P 52417eac3ecaec2dbbde170334358f5ddbd32501 +R 9ce2a78bcb94198e11ae7a2e16b98558 U dan -Z 3af177977b6ccb1ab6aa9597c01ddcc6 +Z 54156ad010a8fff228c493eff10ae743 diff --git a/manifest.uuid b/manifest.uuid index b4bcb2b27e..39bbe3f113 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -52417eac3ecaec2dbbde170334358f5ddbd32501 \ No newline at end of file +72ac73189c3577740a77d2ea2fc7118391c0703f \ No newline at end of file diff --git a/test/fts3expr3.test b/test/fts3expr3.test index 51a6adf58e..e3cc2619c6 100644 --- a/test/fts3expr3.test +++ b/test/fts3expr3.test @@ -149,7 +149,7 @@ for {set i 1} {$i < 100} {incr i} { # Try trees of AND nodes with leaves that are themselves trees of OR nodes. # -for {set i 2} {$i < 32} {incr i} { +for {set i 2} {$i < 64} {incr i 4} { do_test 3.$i { test_fts3expr2 [random_andor_query $i] } [balanced_andor_tree $i] @@ -157,7 +157,7 @@ for {set i 2} {$i < 32} {incr i} { # These exceed the depth limit. # -for {set i 33} {$i < 40} {incr i} { +for {set i 65} {$i < 70} {incr i} { do_test 3.$i { list [catch {test_fts3expr2 [random_andor_query $i]} msg] $msg } {1 {Error parsing expression}} @@ -165,13 +165,21 @@ for {set i 33} {$i < 40} {incr i} { # This also exceeds the depth limit. # -do_test 4.1 { + +do_test 4.1.1 { set q "1" for {set i 2} {$i < 5000} {incr i} { append q " AND $i" } list [catch {test_fts3expr2 $q} msg] $msg } {1 {Error parsing expression}} +do_test 4.1.2 { + set q "1" + for {set i 2} {$i < 4000} {incr i} { + append q " AND $i" + } + catch {test_fts3expr2 $q} +} {0} proc create_toggle_tree {nDepth} { if {$nDepth == 0} { return xxx } From 3a01b600d3d17c098e474707eb0afda005b0fd96 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 29 Apr 2013 18:07:37 +0000 Subject: [PATCH 183/204] Improve the error message issued when an FTS query exceeds the maximum allowable tree depth. FossilOrigin-Name: f480b1fe6012f36c59cd0525efdc6df74143ccd0 --- ext/fts3/fts3.c | 8 +++----- ext/fts3/fts3Int.h | 2 +- ext/fts3/fts3_expr.c | 20 ++++++++++++++++---- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 29 insertions(+), 19 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 0c7e07740d..34d1e2acbd 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -2975,14 +2975,12 @@ static int fts3FilterMethod( pCsr->iLangid = 0; if( nVal==2 ) pCsr->iLangid = sqlite3_value_int(apVal[1]); + assert( p->base.zErrMsg==0 ); rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, - p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr + p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr, + &p->base.zErrMsg ); if( rc!=SQLITE_OK ){ - if( rc==SQLITE_ERROR ){ - static const char *zErr = "malformed MATCH expression: [%s]"; - p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery); - } return rc; } diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index a11c18a3b7..1fbb9c7587 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -524,7 +524,7 @@ void sqlite3Fts3Matchinfo(sqlite3_context *, Fts3Cursor *, const char *); /* fts3_expr.c */ int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, - char **, int, int, int, const char *, int, Fts3Expr ** + char **, int, int, int, const char *, int, Fts3Expr **, char ** ); void sqlite3Fts3ExprFree(Fts3Expr *); #ifdef SQLITE_TEST diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 2cb0f94bcb..3ea3f23c96 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -756,7 +756,7 @@ static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){ int rc = SQLITE_OK; if( p ){ if( nMaxDepth<0 ){ - rc = SQLITE_ERROR; + rc = SQLITE_TOOBIG; }else{ rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1); if( rc==SQLITE_OK ){ @@ -841,7 +841,7 @@ static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ } if( p ){ sqlite3Fts3ExprFree(p); - rc = SQLITE_ERROR; + rc = SQLITE_TOOBIG; break; } @@ -996,7 +996,8 @@ int sqlite3Fts3ExprParse( int nCol, /* Number of entries in azCol[] */ int iDefaultCol, /* Default column to query */ const char *z, int n, /* Text of MATCH query */ - Fts3Expr **ppExpr /* OUT: Parsed query structure */ + Fts3Expr **ppExpr, /* OUT: Parsed query structure */ + char **pzErr /* OUT: Error message (sqlite3_malloc) */ ){ static const int MAX_EXPR_DEPTH = 12; int rc = fts3ExprParseUnbalanced( @@ -1011,9 +1012,18 @@ int sqlite3Fts3ExprParse( rc = fts3ExprCheckDepth(*ppExpr, MAX_EXPR_DEPTH); } } + if( rc!=SQLITE_OK ){ sqlite3Fts3ExprFree(*ppExpr); *ppExpr = 0; + if( rc==SQLITE_TOOBIG ){ + *pzErr = sqlite3_mprintf( + "FTS expression tree is too large (maximum depth %d)", MAX_EXPR_DEPTH + ); + rc = SQLITE_ERROR; + }else if( rc==SQLITE_ERROR ){ + *pzErr = sqlite3_mprintf("malformed MATCH expression: [%s]", z); + } } return rc; @@ -1216,10 +1226,12 @@ static void fts3ExprTest( } if( sqlite3_user_data(context) ){ + char *zDummy = 0; rc = sqlite3Fts3ExprParse( - pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr + pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr, &zDummy ); assert( rc==SQLITE_OK || pExpr==0 ); + sqlite3_free(zDummy); }else{ rc = fts3ExprParseUnbalanced( pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr diff --git a/manifest b/manifest index 7a2fc313d4..f3fe09157b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\soff-by-one\sin\sthe\scode\sfor\slimiting\sthe\sdepth\sof\sFTS\sexpression\strees. -D 2013-04-29T17:12:06.416 +C Improve\sthe\serror\smessage\sissued\swhen\san\sFTS\squery\sexceeds\sthe\smaximum\sallowable\stree\sdepth. +D 2013-04-29T18:07:37.241 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -55,11 +55,11 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 784aadfb4c2a217c3eb1feaecac924989f29728f +F ext/fts3/fts3.c 5c3d44d16701cc4bc81ebf0bb9d5bff136d42de0 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 352c8a83ee4c6a14ced1759a39dd890ab947cbe0 +F ext/fts3/fts3Int.h 23ea0a2bb7258d2539376ed60220cce28ba25765 F ext/fts3/fts3_aux.c b02632f6dd0e375ce97870206d914ea6d8df5ccd -F ext/fts3/fts3_expr.c 2449c3f31b87c3376b66999cdd10d23be0b0bd42 +F ext/fts3/fts3_expr.c 44b4a3c4983ddbf1958c4a40468efb4ff2e0549a F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 52417eac3ecaec2dbbde170334358f5ddbd32501 -R 9ce2a78bcb94198e11ae7a2e16b98558 +P 72ac73189c3577740a77d2ea2fc7118391c0703f +R 845f7187fab8dee39261b18defdba512 U dan -Z 54156ad010a8fff228c493eff10ae743 +Z 0f3ad9cc69769334523e8cc24ad50e47 diff --git a/manifest.uuid b/manifest.uuid index 39bbe3f113..eb0b27c97e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -72ac73189c3577740a77d2ea2fc7118391c0703f \ No newline at end of file +f480b1fe6012f36c59cd0525efdc6df74143ccd0 \ No newline at end of file From 10269dc676f699198f4b76728c6e1b27defd2c2c Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 30 Apr 2013 07:54:42 +0000 Subject: [PATCH 184/204] Define the sqlite3ErrName() function only when necessary. More robust handling of unknown return codes. FossilOrigin-Name: e47cd314371c2be6e00d96392b3892a7f3015f98 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/main.c | 11 +++++++++-- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 28b0ee1d4d..0dd8e3a7d5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\smmap\stest\sfix\sfrom\strunk. -D 2013-04-29T09:20:06.908 +C Define\sthe\ssqlite3ErrName()\sfunction\sonly\swhen\snecessary.\sMore\srobust\shandling\sof\sunknown\sreturn\scodes. +D 2013-04-30T07:54:42.804 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -158,7 +158,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c c48f7f3f170e502fe0cc20748e03c6e0b5a016c2 -F src/main.c 64c76b9acc7381d08ae1ea609852559ec5caa34e +F src/main.c 7531758e3167006f55cd65678d9c72a3c1a6759a F src/malloc.c fe085aa851b666b7c375c1ff957643dc20a04bf6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9272009f7932b3f18006f73776e36b8ef8770d3e 52417eac3ecaec2dbbde170334358f5ddbd32501 -R cae271d9506bb2366270e651f8369050 +P 95811877fdcbede4f61269ff1c7a6d9554f669cd +R 4704202ed1648bd727fc4e08e48da493 U mistachkin -Z f51e73792b7552ee4009f7c1e977bae2 +Z df16913f315861e242e8bc99c3932e49 diff --git a/manifest.uuid b/manifest.uuid index 0546659517..0cd8bc27b1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -95811877fdcbede4f61269ff1c7a6d9554f669cd \ No newline at end of file +e47cd314371c2be6e00d96392b3892a7f3015f98 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 9bb48442d0..3b6cd9e821 100644 --- a/src/main.c +++ b/src/main.c @@ -1033,9 +1033,11 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ ** Return a static string containing the name corresponding to the error code ** specified in the argument. */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) || \ + defined(SQLITE_DEBUG_OS_TRACE) const char *sqlite3ErrName(int rc){ const char *zName = 0; - int i; + int i, origRc = rc; for(i=0; i<2 && zName==0; i++, rc &= 0xff){ switch( rc ){ case SQLITE_OK: zName = "SQLITE_OK"; break; @@ -1122,9 +1124,14 @@ const char *sqlite3ErrName(int rc){ case SQLITE_DONE: zName = "SQLITE_DONE"; break; } } - if( zName==0 ) zName = "SQLITE_Unknown"; + if( zName==0 ){ + static char zBuf[50]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "SQLITE_UNKNOWN(%d)", origRc); + zName = zBuf; + } return zName; } +#endif /* ** Return a static string that describes the kind of error specified in the From dd08ca0ff900e25667f42ea2b00dd3399f56e8ec Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 30 Apr 2013 07:58:15 +0000 Subject: [PATCH 185/204] Add the #ifdefs for sqlite3ErrName() to sqliteInt.h as well. FossilOrigin-Name: 400fc4c37ff34e84f5a129a083a567bda840278e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqliteInt.h | 5 +++++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 0dd8e3a7d5..32b927c8f1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Define\sthe\ssqlite3ErrName()\sfunction\sonly\swhen\snecessary.\sMore\srobust\shandling\sof\sunknown\sreturn\scodes. -D 2013-04-30T07:54:42.804 +C Add\sthe\s#ifdefs\sfor\ssqlite3ErrName()\sto\ssqliteInt.h\sas\swell. +D 2013-04-30T07:58:15.668 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -194,7 +194,7 @@ F src/shell.c 5d527e5d08f05ec2c43ff194ea44bf62b974f4c9 F src/sqlite.h.in e6c241170da854abf6f15ec7e6c565348b690169 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 -F src/sqliteInt.h 1a2b576d605505f6584b0ed6c59f3e85eb8ff5dc +F src/sqliteInt.h 4cc782c9a89b3ddd663e7f68af3fa9e5af596f8b F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 95811877fdcbede4f61269ff1c7a6d9554f669cd -R 4704202ed1648bd727fc4e08e48da493 +P e47cd314371c2be6e00d96392b3892a7f3015f98 +R 70f2f4bd9f342d0b45d8fabecab1b397 U mistachkin -Z df16913f315861e242e8bc99c3932e49 +Z 9f1f55618d07d0d16d8083682b6c561d diff --git a/manifest.uuid b/manifest.uuid index 0cd8bc27b1..f21e9ab5ca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e47cd314371c2be6e00d96392b3892a7f3015f98 \ No newline at end of file +400fc4c37ff34e84f5a129a083a567bda840278e \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 2007600d85..5950f237d9 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3092,7 +3092,12 @@ void sqlite3Error(sqlite3*, int, const char*,...); void *sqlite3HexToBlob(sqlite3*, const char *z, int n); u8 sqlite3HexToInt(int h); int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); + +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) || \ + defined(SQLITE_DEBUG_OS_TRACE) const char *sqlite3ErrName(int); +#endif + const char *sqlite3ErrStr(int); int sqlite3ReadSchema(Parse *pParse); CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); From 3c449c6b6b04e49b74a12e9eb47f5799b06bfaee Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 30 Apr 2013 14:06:57 +0000 Subject: [PATCH 186/204] Make sure extra parentheses around subqueries in the FROM clause are harmless. Ticket [28c6e830f239ea5]. FossilOrigin-Name: 1c79569226db3d5a73e65a35877635ea8b478866 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/parse.y | 2 ++ test/where8.test | 16 +++++++++++++++- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index f3fe09157b..46a1307582 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\serror\smessage\sissued\swhen\san\sFTS\squery\sexceeds\sthe\smaximum\sallowable\stree\sdepth. -D 2013-04-29T18:07:37.241 +C Make\ssure\sextra\sparentheses\saround\ssubqueries\sin\sthe\sFROM\sclause\sare\sharmless.\nTicket\s[28c6e830f239ea5]. +D 2013-04-30T14:06:57.299 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -179,7 +179,7 @@ F src/os_unix.c 5a214c5431fd005dbb3b8bbaaa306433e8e9b48f F src/os_win.c 673b3e3d1fa3040d8d95a7f1f5e0e553aed56cfb F src/pager.c 4a9160d268977e56ae2df90182050ab30fca715d F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 -F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 +F src/parse.y 9708365594eea519cdc8504dee425c0a41c79502 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 @@ -1003,7 +1003,7 @@ F test/where4.test e9b9e2f2f98f00379e6031db6a6fca29bae782a2 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b F test/where7.test 5c566388f0cc318b0032ce860f4ac5548e3c265a -F test/where8.test 02619a9bfc6df7b19979a02852bac09c3c99477a +F test/where8.test d9f889e62dccddb9d790b0c84dfc7861e03a162c F test/where8m.test da346596e19d54f0aba35ebade032a7c47d79739 F test/where9.test 0157862ccf0cfdf1a4622cdf697e5e2f09a8de44 F test/whereA.test 24c234263c8fe358f079d5e57d884fb569d2da0a @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 72ac73189c3577740a77d2ea2fc7118391c0703f -R 845f7187fab8dee39261b18defdba512 -U dan -Z 0f3ad9cc69769334523e8cc24ad50e47 +P f480b1fe6012f36c59cd0525efdc6df74143ccd0 +R 741140f0e346ccf75acdb75c67b0b716 +U drh +Z 1f58c2b83ff706000e8e5b26ed1b3eec diff --git a/manifest.uuid b/manifest.uuid index eb0b27c97e..b2d3fed836 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f480b1fe6012f36c59cd0525efdc6df74143ccd0 \ No newline at end of file +1c79569226db3d5a73e65a35877635ea8b478866 \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index abc0e7dc08..8310b26989 100644 --- a/src/parse.y +++ b/src/parse.y @@ -520,7 +520,9 @@ seltablist(A) ::= stl_prefix(X) nm(Y) dbnm(D) as(Z) indexed_opt(I) struct SrcList_item *pOld = F->a; pNew->zName = pOld->zName; pNew->zDatabase = pOld->zDatabase; + pNew->pSelect = pOld->pSelect; pOld->zName = pOld->zDatabase = 0; + pOld->pSelect = 0; } sqlite3SrcListDelete(pParse->db, F); }else{ diff --git a/test/where8.test b/test/where8.test index 3bf1790132..ae2d04a75d 100644 --- a/test/where8.test +++ b/test/where8.test @@ -296,13 +296,27 @@ do_test where8-3.21 { SELECT a, d FROM t1, (t2) WHERE (a=d OR b=e) AND a<5 ORDER BY a } } {1 1 2 2 3 3 4 2 4 4 0 0} +do_test where8-3.21.1 { + execsql_status { + SELECT a, d FROM t1, ((t2)) AS t3 WHERE (a=d OR b=e) AND a<5 ORDER BY a + } +} {1 1 2 2 3 3 4 2 4 4 0 0} +do_test where8-3.21.2 { + execsql_status { + SELECT a, d FROM t1, ((SELECT * FROM t2)) AS t3 WHERE (a=d OR b=e) AND a<5 ORDER BY a + } +} {1 1 2 2 3 3 4 2 4 4 0 0} do_test where8-3.22 { execsql_status { SELECT a, d FROM ((((((t1))), (((t2)))))) WHERE (a=d OR b=e) AND a<5 ORDER BY a } } {1 1 2 2 3 3 4 2 4 4 0 0} - +do_test where8-3.23 { + execsql_status { + SELECT * FROM ((SELECT * FROM t2)) AS t3; + } +} {1 {} I 2 four IV 3 {} IX 4 sixteen XVI 5 {} XXV 6 thirtysix XXXVI 7 fortynine XLIX 8 sixtyeight LXIV 9 eightyone LXXXIX 10 {} C 9 0} #----------------------------------------------------------------------- # The following tests - where8-4.* - verify that adding or removing From 9a1eccb621ad48bc606019a1fb18220069af38af Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 30 Apr 2013 14:25:32 +0000 Subject: [PATCH 187/204] Update the documentation to explain that when the 3rd parameter to sqlite3_bind_text() and friends is NULL the result is to bind a NULL SQL value. Ticket [19b44e35753ba] FossilOrigin-Name: bd92de0e8d922b96513c5d431493800dda7e7562 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 46a1307582..08e82e4973 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sextra\sparentheses\saround\ssubqueries\sin\sthe\sFROM\sclause\sare\sharmless.\nTicket\s[28c6e830f239ea5]. -D 2013-04-30T14:06:57.299 +C Update\sthe\sdocumentation\sto\sexplain\sthat\swhen\sthe\s3rd\sparameter\sto\nsqlite3_bind_text()\sand\sfriends\sis\sNULL\sthe\sresult\sis\sto\sbind\sa\sNULL\nSQL\svalue.\s\sTicket\s[19b44e35753ba] +D 2013-04-30T14:25:32.967 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -191,7 +191,7 @@ F src/resolve.c 83cc2d942ee216bc56956c6e6fadb691c1727fa1 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 6bfbe11e2fef81c5e18d30513ab6c69f171667eb F src/shell.c 5d527e5d08f05ec2c43ff194ea44bf62b974f4c9 -F src/sqlite.h.in e6c241170da854abf6f15ec7e6c565348b690169 +F src/sqlite.h.in 5a5a22a9b192d81a9e5dee00274e3a0484c4afb1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 F src/sqliteInt.h de835c584032769461c123a564381f9808542c0e @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f480b1fe6012f36c59cd0525efdc6df74143ccd0 -R 741140f0e346ccf75acdb75c67b0b716 +P 1c79569226db3d5a73e65a35877635ea8b478866 +R d47a972814ae8bb4214b1f841a420de6 U drh -Z 1f58c2b83ff706000e8e5b26ed1b3eec +Z 88df79255fe6b8c77c4c16bff1d52cd7 diff --git a/manifest.uuid b/manifest.uuid index b2d3fed836..c280f104a7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1c79569226db3d5a73e65a35877635ea8b478866 \ No newline at end of file +bd92de0e8d922b96513c5d431493800dda7e7562 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 5eb74a8756..6608823175 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -3270,6 +3270,9 @@ typedef struct sqlite3_context sqlite3_context; ** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 999). ** ** ^The third argument is the value to bind to the parameter. +** ^If the third parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** or sqlite3_bind_blob() is a NULL pointer then the fourth parameter +** is ignored and the end result is the same as sqlite3_bind_null(). ** ** ^(In those routines that have a fourth argument, its value is the ** number of bytes in the parameter. To be clear: the value is the From 5c10f3b38e3930ec9937f8867d12fe431e3de2bc Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 May 2013 17:22:38 +0000 Subject: [PATCH 188/204] Avoid redundant constraint checking due to transitive constraints. FossilOrigin-Name: 329478cbed06f93652de50abdb31a6b41af02b9e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 08e82e4973..7f57ed1891 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sdocumentation\sto\sexplain\sthat\swhen\sthe\s3rd\sparameter\sto\nsqlite3_bind_text()\sand\sfriends\sis\sNULL\sthe\sresult\sis\sto\sbind\sa\sNULL\nSQL\svalue.\s\sTicket\s[19b44e35753ba] -D 2013-04-30T14:25:32.967 +C Avoid\sredundant\sconstraint\schecking\sdue\sto\stransitive\sconstraints. +D 2013-05-01T17:22:38.014 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -263,7 +263,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c d54e63087b52c309550aa2defdb20ef27add9f9a +F src/where.c e36bbfbf565c7a2ce49c5cd3101ca7310cd379af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 1c79569226db3d5a73e65a35877635ea8b478866 -R d47a972814ae8bb4214b1f841a420de6 +P bd92de0e8d922b96513c5d431493800dda7e7562 +R 98be73d126ba545d3f3d37ca997eaf1d U drh -Z 88df79255fe6b8c77c4c16bff1d52cd7 +Z df56d0b87948c58bd48a65ece4cb954b diff --git a/manifest.uuid b/manifest.uuid index c280f104a7..d118513ae8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bd92de0e8d922b96513c5d431493800dda7e7562 \ No newline at end of file +329478cbed06f93652de50abdb31a6b41af02b9e \ No newline at end of file diff --git a/src/where.c b/src/where.c index d70205205c..62f9af6760 100644 --- a/src/where.c +++ b/src/where.c @@ -4883,6 +4883,7 @@ static Bitmask codeOneLoopStart( assert( (pTerm->prereqRight & newNotReady)!=0 ); pAlt = findTerm(pWC, iCur, pTerm->u.leftColumn, notReady, WO_EQ|WO_IN, 0); if( pAlt==0 ) continue; + if( pAlt->wtFlags & (TERM_CODED) ) continue; VdbeNoopComment((v, "begin transitive constraint")); sEq = *pAlt->pExpr; sEq.pLeft = pE->pLeft; From a309552e2cc3aa3de14c7fd5a6110375d4efb43d Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 May 2013 17:58:35 +0000 Subject: [PATCH 189/204] Do not use a transitive constraint to an IN operator where the RHS is a constant if there exists a direct == operator to another table in an outer loop. FossilOrigin-Name: faedaeace9c7ed9a8aaf96700caee09db0c0c061 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7f57ed1891..a5bd0b436b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sredundant\sconstraint\schecking\sdue\sto\stransitive\sconstraints. -D 2013-05-01T17:22:38.014 +C Do\snot\suse\sa\stransitive\sconstraint\sto\san\sIN\soperator\swhere\sthe\sRHS\sis\sa\nconstant\sif\sthere\sexists\sa\sdirect\s==\soperator\sto\sanother\stable\sin\san\souter\nloop. +D 2013-05-01T17:58:35.871 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -263,7 +263,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c e36bbfbf565c7a2ce49c5cd3101ca7310cd379af +F src/where.c 12d4200eb6ae991cad02367c391db076ac1af1b0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P bd92de0e8d922b96513c5d431493800dda7e7562 -R 98be73d126ba545d3f3d37ca997eaf1d +P 329478cbed06f93652de50abdb31a6b41af02b9e +R cc457983507eee39e1b071292dee5b26 U drh -Z df56d0b87948c58bd48a65ece4cb954b +Z 944723cfd5dfa8d16765eaab2fd82886 diff --git a/manifest.uuid b/manifest.uuid index d118513ae8..f88e8ae7a8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -329478cbed06f93652de50abdb31a6b41af02b9e \ No newline at end of file +faedaeace9c7ed9a8aaf96700caee09db0c0c061 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 62f9af6760..2de894c3e7 100644 --- a/src/where.c +++ b/src/where.c @@ -705,7 +705,7 @@ static WhereTerm *findTerm( continue; } } - if( pTerm->prereqRight==0 ){ + if( pTerm->prereqRight==0 && (pTerm->eOperator&WO_EQ)!=0 ){ pResult = pTerm; goto findTerm_success; }else if( pResult==0 ){ From 4ee09b4bcc491037416597306a53b8508e4dbec3 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 May 2013 19:49:27 +0000 Subject: [PATCH 190/204] Allocate 4 bytes of unused header space for an "Application ID". Add the "PRAGMA application_id" command to set and query this identifier. Add the "magic.txt" file to show how the posix file command might use this application id. FossilOrigin-Name: 28c9e7fdee2471a3026ee05ff591194d5f398131 --- magic.txt | 28 ++++++++++++++++++++++++++++ manifest | 22 +++++++++++++--------- manifest.uuid | 2 +- src/btree.h | 1 + src/pragma.c | 9 +++++++++ test/pragma.test | 10 ++++++++++ tool/showdb.c | 2 +- 7 files changed, 63 insertions(+), 11 deletions(-) create mode 100644 magic.txt diff --git a/magic.txt b/magic.txt new file mode 100644 index 0000000000..22bdae5712 --- /dev/null +++ b/magic.txt @@ -0,0 +1,28 @@ +# This file contains suggested magic(5) text for the unix file(1) +# utility for recognizing SQLite3 databases. +# +# When SQLite is used as an application file format, it is desirable to +# have file(1) recognize the database file as being with the specific +# application. You can set the application_id for a database file +# using: +# +# PRAGMA application_id = INTEGER; +# +# INTEGER can be any signed 32-bit integer. That integer is written as +# a 4-byte big-endian integer into offset 68 of the database header. In +# the tests below, this integer is sometimes rendered as a string. For +# example, instead of "belong =1598444364" we write "string =_FSL" and +# instead of "belong =1598903374" we write "string =_MTN". +# +# The Monotone application used "PRAGMA user_version=1598903374;" to set +# its identifier long before "PRAGMA application_id" became available. +# The user_version is very similar to application_id except that it is +# stored at offset 60 instead of offset 68. The user of +# "PRAGMA application_id" is preferred now. The rules using offset 60 +# for Monotone are for historical compatibility only. +# +0 string =SQLite\ format\ 3 +>68 string =_FSL Fossil repository +>68 belong =0 +>>60 string =_MTN Monotone source repository +>>60 string !_MTN SQLite 3.x database diff --git a/manifest b/manifest index a5bd0b436b..706371cb24 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\suse\sa\stransitive\sconstraint\sto\san\sIN\soperator\swhere\sthe\sRHS\sis\sa\nconstant\sif\sthere\sexists\sa\sdirect\s==\soperator\sto\sanother\stable\sin\san\souter\nloop. -D 2013-05-01T17:58:35.871 +C Allocate\s4\sbytes\sof\sunused\sheader\sspace\sfor\san\s"Application\sID".\s\sAdd\nthe\s"PRAGMA\sapplication_id"\scommand\sto\sset\sand\squery\sthis\sidentifier.\nAdd\sthe\s"magic.txt"\sfile\sto\sshow\show\sthe\sposix\sfile\scommand\smight\suse\nthis\sapplication\sid. +D 2013-05-01T19:49:27.740 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -112,6 +112,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 +F magic.txt 4255db3a17f0da4b3f561a904585a6605fe6a6b8 F main.mk 1b25be82452366abc27cc9ab2acf3244a773d5a1 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f @@ -137,7 +138,7 @@ F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 480a6d255cc4f066029daf23dd54acf152cd0e13 -F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 +F src/btree.h 6fa8a3ff2483d0bb64a9f0105a8cedeac9e00cca F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc @@ -183,7 +184,7 @@ F src/parse.y 9708365594eea519cdc8504dee425c0a41c79502 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c 3eacf001cbf4becbd494f8d82d08fdf1648cf8cb +F src/pragma.c 8779308bc1ea1901c4bc94dfe9a83d436f73f52c F src/prepare.c 743e484233c51109666d402f470523553b41797c F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -677,7 +678,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/permutations.test 3d0bab9c49c1ec08b868059e30a3e1956f2162e2 -F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a +F test/pragma.test 5e7de6c32a5d764f09437d2025f07e4917b9e178 F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 F test/progress.test 5b075c3c790c7b2a61419bc199db87aaf48b8301 @@ -1040,7 +1041,7 @@ F tool/omittest.tcl 4665982e95a6e5c1bd806cf7bc3dea95be422d77 F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 -F tool/showdb.c acd24ea035a3bd2ffe266f1ef8a161812c29b2f0 +F tool/showdb.c 525ecc443578647703051308ad50a93de6ba2c4b F tool/showjournal.c b62cecaab86a4053d944c276bb5232e4d17ece02 F tool/showwal.c 3f7f7da5ec0cba51b1449a75f700493377da57b5 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe @@ -1060,7 +1061,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 329478cbed06f93652de50abdb31a6b41af02b9e -R cc457983507eee39e1b071292dee5b26 +P faedaeace9c7ed9a8aaf96700caee09db0c0c061 +R 4f1b06c15d2f5747c86dc6db3cebcf91 +T *branch * application-id +T *sym-application-id * +T -sym-trunk * U drh -Z 944723cfd5dfa8d16765eaab2fd82886 +Z 76bc7425f1509081256079e29083992d diff --git a/manifest.uuid b/manifest.uuid index f88e8ae7a8..1b206e6fa8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -faedaeace9c7ed9a8aaf96700caee09db0c0c061 \ No newline at end of file +28c9e7fdee2471a3026ee05ff591194d5f398131 \ No newline at end of file diff --git a/src/btree.h b/src/btree.h index 23f6ad7068..ace0f8cd21 100644 --- a/src/btree.h +++ b/src/btree.h @@ -140,6 +140,7 @@ int sqlite3BtreeNewDb(Btree *p); #define BTREE_TEXT_ENCODING 5 #define BTREE_USER_VERSION 6 #define BTREE_INCR_VACUUM 7 +#define BTREE_APPLICATION_ID 8 /* ** Values that may be OR'd together to form the second argument of an diff --git a/src/pragma.c b/src/pragma.c index 65efbd8b21..3056a7d8e2 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1567,6 +1567,11 @@ void sqlite3Pragma( ** PRAGMA [database.]user_version ** PRAGMA [database.]user_version = ** + ** PRAGMA [database.]freelist_count = + ** + ** PRAGMA [database.]application_id + ** PRAGMA [database.]application_id = + ** ** The pragma's schema_version and user_version are used to set or get ** the value of the schema-version and user-version, respectively. Both ** the schema-version and the user-version are 32-bit signed integers @@ -1588,10 +1593,14 @@ void sqlite3Pragma( if( sqlite3StrICmp(zLeft, "schema_version")==0 || sqlite3StrICmp(zLeft, "user_version")==0 || sqlite3StrICmp(zLeft, "freelist_count")==0 + || sqlite3StrICmp(zLeft, "application_id")==0 ){ int iCookie; /* Cookie index. 1 for schema-cookie, 6 for user-cookie. */ sqlite3VdbeUsesBtree(v, iDb); switch( zLeft[0] ){ + case 'a': case 'A': + iCookie = BTREE_APPLICATION_ID; + break; case 'f': case 'F': iCookie = BTREE_FREE_PAGE_COUNT; break; diff --git a/test/pragma.test b/test/pragma.test index 3eb624328e..a6d198eb69 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -936,6 +936,16 @@ proc check_temp_store {} { return "unknown" } +# Application_ID +# +do_test pragma-8.3.1 { + execsql { + PRAGMA application_id; + } +} {0} +do_test pragma-8.3.2 { + execsql {PRAGMA Application_ID(12345); PRAGMA application_id;} +} {12345} # Test temp_store and temp_store_directory pragmas # diff --git a/tool/showdb.c b/tool/showdb.c index dbd79e9586..27424e0920 100644 --- a/tool/showdb.c +++ b/tool/showdb.c @@ -176,7 +176,7 @@ static void print_db_header(void){ print_decode_line(aData, 56, 4, "Text encoding"); print_decode_line(aData, 60, 4, "User version"); print_decode_line(aData, 64, 4, "Incremental-vacuum mode"); - print_decode_line(aData, 68, 4, "meta[7]"); + print_decode_line(aData, 68, 4, "Application ID"); print_decode_line(aData, 72, 4, "meta[8]"); print_decode_line(aData, 76, 4, "meta[9]"); print_decode_line(aData, 80, 4, "meta[10]"); From b8a67ec827b5a5987f447d68d63648bd2045b3cd Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 May 2013 20:36:23 +0000 Subject: [PATCH 191/204] Preserve the application-ID across VACUUM. Updates to the magic number file. FossilOrigin-Name: 4a190bea18e156b6fa4dc9f21c3ad32409049603 --- magic.txt | 11 ++++++----- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/vacuum.c | 1 + 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/magic.txt b/magic.txt index 22bdae5712..90e2dc50a0 100644 --- a/magic.txt +++ b/magic.txt @@ -17,12 +17,13 @@ # The Monotone application used "PRAGMA user_version=1598903374;" to set # its identifier long before "PRAGMA application_id" became available. # The user_version is very similar to application_id except that it is -# stored at offset 60 instead of offset 68. The user of +# stored at offset 68 instead of offset 60. The user of # "PRAGMA application_id" is preferred now. The rules using offset 60 # for Monotone are for historical compatibility only. # 0 string =SQLite\ format\ 3 ->68 string =_FSL Fossil repository ->68 belong =0 ->>60 string =_MTN Monotone source repository ->>60 string !_MTN SQLite 3.x database +>68 belong =0x0f055111 Fossil repository - +>68 belong =0x0f055112 Fossil checkout - +>68 belong =0x0f055113 Fossil global configuration - +>60 belong =0x5f4d544e Monotone source repository - +>0 string =SQLite SQLite3 database diff --git a/manifest b/manifest index 706371cb24..06e860720a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allocate\s4\sbytes\sof\sunused\sheader\sspace\sfor\san\s"Application\sID".\s\sAdd\nthe\s"PRAGMA\sapplication_id"\scommand\sto\sset\sand\squery\sthis\sidentifier.\nAdd\sthe\s"magic.txt"\sfile\sto\sshow\show\sthe\sposix\sfile\scommand\smight\suse\nthis\sapplication\sid. -D 2013-05-01T19:49:27.740 +C Preserve\sthe\sapplication-ID\sacross\sVACUUM.\s\sUpdates\sto\sthe\smagic\snumber\nfile. +D 2013-05-01T20:36:23.594 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -112,7 +112,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F magic.txt 4255db3a17f0da4b3f561a904585a6605fe6a6b8 +F magic.txt 626cd2e726c22440f94d5342e872011990f90401 F main.mk 1b25be82452366abc27cc9ab2acf3244a773d5a1 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f @@ -250,7 +250,7 @@ F src/trigger.c cd95ac64efa60e39faf9b5597443192ff27a22fa F src/update.c a2a5631d618cbe240fc83725fa9e95c56ae0084c F src/utf.c 8d819e2e5104a430fc2005f018db14347c95a38f F src/util.c f566b5138099a2df8533b190d0dcc74b7dfbe0c9 -F src/vacuum.c 2727bdd08847fcb6b2d2da6d14f018910e8645d3 +F src/vacuum.c ddf21cc9577c4cb459d08bee9863a78ec000c5bb F src/vdbe.c 5f0047130f80c7fd0bc41bc51a653b5542c4fbd5 F src/vdbe.h b52887278cb173e66188da84dfab216bea61119d F src/vdbeInt.h c1e830268b75f04a2901dd895b51a637a26c7045 @@ -1061,10 +1061,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P faedaeace9c7ed9a8aaf96700caee09db0c0c061 -R 4f1b06c15d2f5747c86dc6db3cebcf91 -T *branch * application-id -T *sym-application-id * -T -sym-trunk * +P 28c9e7fdee2471a3026ee05ff591194d5f398131 +R eb1c61fca07cfc02ebd6d8ced912d19c U drh -Z 76bc7425f1509081256079e29083992d +Z b916c83ab322021bf56be2467b81fddd diff --git a/manifest.uuid b/manifest.uuid index 1b206e6fa8..f4b80354d1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -28c9e7fdee2471a3026ee05ff591194d5f398131 \ No newline at end of file +4a190bea18e156b6fa4dc9f21c3ad32409049603 \ No newline at end of file diff --git a/src/vacuum.c b/src/vacuum.c index 7ed2478326..4afb2cca64 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -289,6 +289,7 @@ int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ BTREE_DEFAULT_CACHE_SIZE, 0, /* Preserve the default page cache size */ BTREE_TEXT_ENCODING, 0, /* Preserve the text encoding */ BTREE_USER_VERSION, 0, /* Preserve the user version */ + BTREE_APPLICATION_ID, 0, /* Preserve the application id */ }; assert( 1==sqlite3BtreeIsInTrans(pTemp) ); From 8a28a310281f08c81d0f1c1461ba71f6bf32397c Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 May 2013 20:40:46 +0000 Subject: [PATCH 192/204] Fix comments in the magic number file. FossilOrigin-Name: 5a500848d2fa96fc7397e2acb64d5ae6551b5b1e --- magic.txt | 11 ++++------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/magic.txt b/magic.txt index 90e2dc50a0..c99b4e004c 100644 --- a/magic.txt +++ b/magic.txt @@ -9,17 +9,14 @@ # PRAGMA application_id = INTEGER; # # INTEGER can be any signed 32-bit integer. That integer is written as -# a 4-byte big-endian integer into offset 68 of the database header. In -# the tests below, this integer is sometimes rendered as a string. For -# example, instead of "belong =1598444364" we write "string =_FSL" and -# instead of "belong =1598903374" we write "string =_MTN". +# a 4-byte big-endian integer into offset 68 of the database header. # # The Monotone application used "PRAGMA user_version=1598903374;" to set # its identifier long before "PRAGMA application_id" became available. # The user_version is very similar to application_id except that it is -# stored at offset 68 instead of offset 60. The user of -# "PRAGMA application_id" is preferred now. The rules using offset 60 -# for Monotone are for historical compatibility only. +# stored at offset 68 instead of offset 60. The application_id pragma +# is preferred. The rule using offset 60 for Monotone is for historical +# compatibility only. # 0 string =SQLite\ format\ 3 >68 belong =0x0f055111 Fossil repository - diff --git a/manifest b/manifest index 06e860720a..89c31cf5be 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Preserve\sthe\sapplication-ID\sacross\sVACUUM.\s\sUpdates\sto\sthe\smagic\snumber\nfile. -D 2013-05-01T20:36:23.594 +C Fix\scomments\sin\sthe\smagic\snumber\sfile. +D 2013-05-01T20:40:46.379 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -112,7 +112,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F magic.txt 626cd2e726c22440f94d5342e872011990f90401 +F magic.txt 291863ca976425e2e7bf3f775eb98ece4dd120f6 F main.mk 1b25be82452366abc27cc9ab2acf3244a773d5a1 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f @@ -1061,7 +1061,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 28c9e7fdee2471a3026ee05ff591194d5f398131 -R eb1c61fca07cfc02ebd6d8ced912d19c +P 4a190bea18e156b6fa4dc9f21c3ad32409049603 +R 22c224f485fe7b9b09b8cd93def953de U drh -Z b916c83ab322021bf56be2467b81fddd +Z f2d15752a1c82c60eb5f877e525be03c diff --git a/manifest.uuid b/manifest.uuid index f4b80354d1..4ee88a6f3c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4a190bea18e156b6fa4dc9f21c3ad32409049603 \ No newline at end of file +5a500848d2fa96fc7397e2acb64d5ae6551b5b1e \ No newline at end of file From 32c12fe2bb8e4405db53918204d192760655e51a Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 2 May 2013 17:37:31 +0000 Subject: [PATCH 193/204] Minor fixes for compilation with SQLITE_OMIT_WAL defined. FossilOrigin-Name: b81e87e72b976e7157a53a50abc5422e2a6c4c39 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/os_unix.c | 2 +- src/pager.c | 5 ++++- src/wal.h | 1 + 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index a5bd0b436b..3146caf6a8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\suse\sa\stransitive\sconstraint\sto\san\sIN\soperator\swhere\sthe\sRHS\sis\sa\nconstant\sif\sthere\sexists\sa\sdirect\s==\soperator\sto\sanother\stable\sin\san\souter\nloop. -D 2013-05-01T17:58:35.871 +C Minor\sfixes\sfor\scompilation\swith\sSQLITE_OMIT_WAL\sdefined. +D 2013-05-02T17:37:31.421 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -175,9 +175,9 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 5a214c5431fd005dbb3b8bbaaa306433e8e9b48f +F src/os_unix.c 658b180a09a18214d94547f737dbded71667cdab F src/os_win.c 673b3e3d1fa3040d8d95a7f1f5e0e553aed56cfb -F src/pager.c 4a9160d268977e56ae2df90182050ab30fca715d +F src/pager.c 49e23f9898113ddfe90942bdf1c1ef57955d0921 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 9708365594eea519cdc8504dee425c0a41c79502 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -261,7 +261,7 @@ F src/vdbesort.c 4fad64071ae120c25f39dcac572d716b9cadeb7f F src/vdbetrace.c 3ad1b4e92b60c082a02ac563da4a2735cc7d297c F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d -F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 +F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 F src/where.c 12d4200eb6ae991cad02367c391db076ac1af1b0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -1060,7 +1060,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 329478cbed06f93652de50abdb31a6b41af02b9e -R cc457983507eee39e1b071292dee5b26 -U drh -Z 944723cfd5dfa8d16765eaab2fd82886 +P faedaeace9c7ed9a8aaf96700caee09db0c0c061 +R 0cb8383fe74822aeab35f5d514cbdea3 +U dan +Z 177e5be29875bd46bd8a77f94ee092d8 diff --git a/manifest.uuid b/manifest.uuid index f88e8ae7a8..253732acb3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -faedaeace9c7ed9a8aaf96700caee09db0c0c061 \ No newline at end of file +b81e87e72b976e7157a53a50abc5422e2a6c4c39 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 0074412410..6eed674072 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -126,7 +126,7 @@ #include #include #include -#ifndef SQLITE_OMIT_WAL +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 #include #endif diff --git a/src/pager.c b/src/pager.c index dffcf602d7..1c6a84fea4 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2871,10 +2871,13 @@ static int readDbPage(PgHdr *pPg, u32 iFrame){ return SQLITE_OK; } +#ifndef SQLITE_OMIT_WAL if( iFrame ){ /* Try to pull the page from the write-ahead log. */ rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData); - }else{ + }else +#endif + { i64 iOffset = (pgno-1)*(i64)pPager->pageSize; rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); if( rc==SQLITE_IOERR_SHORT_READ ){ diff --git a/src/wal.h b/src/wal.h index ff7624af63..092546354b 100644 --- a/src/wal.h +++ b/src/wal.h @@ -43,6 +43,7 @@ # define sqlite3WalExclusiveMode(y,z) 0 # define sqlite3WalHeapMemory(z) 0 # define sqlite3WalFramesize(z) 0 +# define sqlite3WalFindFrame(x,y,z) 0 #else #define WAL_SAVEPOINT_NDATA 4 From 178e7ee76f077e8e286427ed09d5f925ce2b8d22 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 3 May 2013 20:08:16 +0000 Subject: [PATCH 194/204] Add magic numbers for Bentley Systems application files. FossilOrigin-Name: 9314b08099e7ac99a507a4799f2c6cdd6d597abb --- magic.txt | 2 ++ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/magic.txt b/magic.txt index c99b4e004c..a2ca209e5d 100644 --- a/magic.txt +++ b/magic.txt @@ -22,5 +22,7 @@ >68 belong =0x0f055111 Fossil repository - >68 belong =0x0f055112 Fossil checkout - >68 belong =0x0f055113 Fossil global configuration - +>68 belong =0x42654462 Bentley Systems BeSQLite Database - +>68 belong =0x42654c6e Bentley Systems Localization File - >60 belong =0x5f4d544e Monotone source repository - >0 string =SQLite SQLite3 database diff --git a/manifest b/manifest index 366bb2d497..94a31dd226 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allocate\smeta(8)\sin\sthe\sheader\sas\sthe\s"application\sID"\sand\sadd\sthe\n"PRAGMA\sapplication_id"\scommand\sto\squery\sand\sset\sit. -D 2013-05-03T15:23:24.035 +C Add\smagic\snumbers\sfor\sBentley\sSystems\sapplication\sfiles. +D 2013-05-03T20:08:16.683 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -112,7 +112,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F magic.txt 291863ca976425e2e7bf3f775eb98ece4dd120f6 +F magic.txt 3f820e18c43504b25da40ff4b4cdb66dc4c4907e F main.mk 1b25be82452366abc27cc9ab2acf3244a773d5a1 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f @@ -1061,7 +1061,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b81e87e72b976e7157a53a50abc5422e2a6c4c39 5a500848d2fa96fc7397e2acb64d5ae6551b5b1e -R d15cd031ae2a811b0ed19162e796056a +P b2efe4f225adc5f4c2e3080bf459cc52fff82e18 +R 0b688f16fe96205d735d9d6969fa7bcb U drh -Z 0ae0a427ab8e27618336357d13716971 +Z b278f2182b5b0ccf50dbdcca9690db25 diff --git a/manifest.uuid b/manifest.uuid index 2b2da7ddae..8169773910 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b2efe4f225adc5f4c2e3080bf459cc52fff82e18 \ No newline at end of file +9314b08099e7ac99a507a4799f2c6cdd6d597abb \ No newline at end of file From 2722898c6ce4298d7564277e9aad18ff7aa14dad Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 6 May 2013 13:22:50 +0000 Subject: [PATCH 195/204] Make sure the authorizer callback gets a valid pointer to "ROWID" for the column-name parameter when doing an UPDATE that changes the rowid. Fix for ticket [0eb70d77cb05bb2272]. FossilOrigin-Name: 26a59bb88d4082758eb281b365b57f9a0c059d89 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/update.c | 4 +++- test/auth.test | 23 +++++++++++++++++++++++ 4 files changed, 34 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 94a31dd226..2e943a7503 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smagic\snumbers\sfor\sBentley\sSystems\sapplication\sfiles. -D 2013-05-03T20:08:16.683 +C Make\ssure\sthe\sauthorizer\scallback\sgets\sa\svalid\spointer\sto\s"ROWID"\sfor\sthe\ncolumn-name\sparameter\swhen\sdoing\san\sUPDATE\sthat\schanges\sthe\srowid.\nFix\sfor\sticket\s[0eb70d77cb05bb2272]. +D 2013-05-06T13:22:50.843 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -247,7 +247,7 @@ F src/test_vfstrace.c 34b544e80ba7fb77be15395a609c669df2e660a2 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 1e86210d3976717a19238ea7b047fac481fe8c12 F src/trigger.c cd95ac64efa60e39faf9b5597443192ff27a22fa -F src/update.c a2a5631d618cbe240fc83725fa9e95c56ae0084c +F src/update.c 4c0c6864c4349ba292042e984a56d15985b57f4e F src/utf.c 8d819e2e5104a430fc2005f018db14347c95a38f F src/util.c f566b5138099a2df8533b190d0dcc74b7dfbe0c9 F src/vacuum.c ddf21cc9577c4cb459d08bee9863a78ec000c5bb @@ -293,7 +293,7 @@ F test/attach2.test e54436ed956d3d88bdee61221da59bf3935a0966 F test/attach3.test d89ccfe4fe6e2b5e368d480fcdfe4b496c54cf4e F test/attach4.test 53bf502f17647c6d6c5add46dda6bac8b6f4665c F test/attachmalloc.test 3a4bfca9545bfe906a8d2e622de10fbac5b711b0 -F test/auth.test 1b21145e888130d60a03db0cb829d59df8f29266 +F test/auth.test 4a4c3b034fff7750513520defa910f376c96ab49 F test/auth2.test a2a371aa6df15f8b0c8109b33d3d7f0f73e4c9aa F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5 F test/autoinc.test bd30d372d00045252f6c2e41b5f41455e1975acf @@ -1061,7 +1061,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b2efe4f225adc5f4c2e3080bf459cc52fff82e18 -R 0b688f16fe96205d735d9d6969fa7bcb +P 9314b08099e7ac99a507a4799f2c6cdd6d597abb +R 838a83e273da8f276170acdb509d4aac U drh -Z b278f2182b5b0ccf50dbdcca9690db25 +Z be0fc56bce8babfb180ef06d811f28d4 diff --git a/manifest.uuid b/manifest.uuid index 8169773910..1c3879a604 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9314b08099e7ac99a507a4799f2c6cdd6d597abb \ No newline at end of file +26a59bb88d4082758eb281b365b57f9a0c059d89 \ No newline at end of file diff --git a/src/update.c b/src/update.c index 1125e5971a..3ab1ab2a4b 100644 --- a/src/update.c +++ b/src/update.c @@ -208,6 +208,7 @@ void sqlite3Update( } if( j>=pTab->nCol ){ if( sqlite3IsRowid(pChanges->a[i].zName) ){ + j = -1; chngRowid = 1; pRowidExpr = pChanges->a[i].pExpr; }else{ @@ -220,7 +221,8 @@ void sqlite3Update( { int rc; rc = sqlite3AuthCheck(pParse, SQLITE_UPDATE, pTab->zName, - pTab->aCol[j].zName, db->aDb[iDb].zName); + j<0 ? "ROWID" : pTab->aCol[j].zName, + db->aDb[iDb].zName); if( rc==SQLITE_DENY ){ goto update_cleanup; }else if( rc==SQLITE_IGNORE ){ diff --git a/test/auth.test b/test/auth.test index 190b490003..fd402b1d9e 100644 --- a/test/auth.test +++ b/test/auth.test @@ -2368,6 +2368,29 @@ ifcapable trigger { } {1} } +# Ticket [0eb70d77cb05bb22720]: Invalid pointer passsed to the authorizer +# callback when updating a ROWID. +# +do_test auth-6.1 { + execsql { + CREATE TABLE t6(a,b,c,d,e,f,g,h); + INSERT INTO t6 VALUES(1,2,3,4,5,6,7,8); + } +} {} +set ::authargs [list] +proc auth {args} { + eval lappend ::authargs $args + return SQLITE_OK +} +do_test auth-6.2 { + execsql {UPDATE t6 SET rowID=rowID+100} + set ::authargs +} [list SQLITE_READ t6 ROWID main {} \ + SQLITE_UPDATE t6 ROWID main {} \ +] +do_test auth-6.3 { + execsql {SELECT rowid, * FROM t6} +} {101 1 2 3 4 5 6 7 8} rename proc {} rename proc_real proc From acf9c2b555d81817c175710a8b1721b436107fbc Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 6 May 2013 14:57:48 +0000 Subject: [PATCH 196/204] Add an assert() to fts3_expr.c in order to silence a clang warning. FossilOrigin-Name: d8dc2c5fb554eb5898ddc8d703c6a360af7fe023 --- ext/fts3/fts3_expr.c | 1 + manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 3ea3f23c96..c046d7d5ff 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -875,6 +875,7 @@ static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ p = apLeaf[i]; p->pParent = 0; }else{ + assert( pFree!=0 ); pFree->pRight = p; pFree->pLeft = apLeaf[i]; pFree->pLeft->pParent = pFree; diff --git a/manifest b/manifest index 2e943a7503..ff2b2e1ad8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\sauthorizer\scallback\sgets\sa\svalid\spointer\sto\s"ROWID"\sfor\sthe\ncolumn-name\sparameter\swhen\sdoing\san\sUPDATE\sthat\schanges\sthe\srowid.\nFix\sfor\sticket\s[0eb70d77cb05bb2272]. -D 2013-05-06T13:22:50.843 +C Add\san\sassert()\sto\sfts3_expr.c\sin\sorder\sto\ssilence\sa\sclang\swarning. +D 2013-05-06T14:57:48.666 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -59,7 +59,7 @@ F ext/fts3/fts3.c 5c3d44d16701cc4bc81ebf0bb9d5bff136d42de0 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 23ea0a2bb7258d2539376ed60220cce28ba25765 F ext/fts3/fts3_aux.c b02632f6dd0e375ce97870206d914ea6d8df5ccd -F ext/fts3/fts3_expr.c 44b4a3c4983ddbf1958c4a40468efb4ff2e0549a +F ext/fts3/fts3_expr.c 193d6fc156d744ab548a2ed06c31869e54dac739 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5 @@ -1061,7 +1061,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9314b08099e7ac99a507a4799f2c6cdd6d597abb -R 838a83e273da8f276170acdb509d4aac +P 26a59bb88d4082758eb281b365b57f9a0c059d89 +R 2feea6cda3683eb0bf228ac73380638e U drh -Z be0fc56bce8babfb180ef06d811f28d4 +Z ca81a07ef0e1a560e388493dc60b3a68 diff --git a/manifest.uuid b/manifest.uuid index 1c3879a604..04a316d2aa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -26a59bb88d4082758eb281b365b57f9a0c059d89 \ No newline at end of file +d8dc2c5fb554eb5898ddc8d703c6a360af7fe023 \ No newline at end of file From 151b7d5baddd4ba3fd0a19f00dfb6a7e533c02eb Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 6 May 2013 20:28:54 +0000 Subject: [PATCH 197/204] In the command-line shell, make sure the output of dot-commands goes to the same redirected output stream as the result of queries. Ticket [2f96536e7ff7dba] FossilOrigin-Name: b72d365920dbe45e61d8b1cff708a8858c2b3e33 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 18 +++++++++--------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index ff2b2e1ad8..4ec94916cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sassert()\sto\sfts3_expr.c\sin\sorder\sto\ssilence\sa\sclang\swarning. -D 2013-05-06T14:57:48.666 +C In\sthe\scommand-line\sshell,\smake\ssure\sthe\soutput\sof\sdot-commands\sgoes\sto\nthe\ssame\sredirected\soutput\sstream\sas\sthe\sresult\sof\squeries.\nTicket\s[2f96536e7ff7dba] +D 2013-05-06T20:28:54.574 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -191,7 +191,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 83cc2d942ee216bc56956c6e6fadb691c1727fa1 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 6bfbe11e2fef81c5e18d30513ab6c69f171667eb -F src/shell.c 5d527e5d08f05ec2c43ff194ea44bf62b974f4c9 +F src/shell.c 115bc2fa3ce4e89f5cea49ace173c4c9e68cfcf6 F src/sqlite.h.in 5a5a22a9b192d81a9e5dee00274e3a0484c4afb1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 @@ -1061,7 +1061,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 26a59bb88d4082758eb281b365b57f9a0c059d89 -R 2feea6cda3683eb0bf228ac73380638e +P d8dc2c5fb554eb5898ddc8d703c6a360af7fe023 +R 51c5817bdcc09862ecf820138ddcd721 U drh -Z ca81a07ef0e1a560e388493dc60b3a68 +Z 6618db8b151a5d589fef00beca7c472a diff --git a/manifest.uuid b/manifest.uuid index 04a316d2aa..c315d93ad3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d8dc2c5fb554eb5898ddc8d703c6a360af7fe023 \ No newline at end of file +b72d365920dbe45e61d8b1cff708a8858c2b3e33 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 09c3b810c8..e6dd496bd1 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2413,9 +2413,9 @@ static int do_meta_command(char *zLine, struct callback_data *p){ for(i=0; iout, "%s%-*s", zSp, maxlen, azResult[j] ? azResult[j] : ""); } - printf("\n"); + fprintf(p->out, "\n"); } } for(ii=0; iidb, opt); - printf("%d (0x%08x)\n", rc, rc); + fprintf(p->out, "%d (0x%08x)\n", rc, rc); } else { fprintf(stderr,"Error: testctrl %s takes a single int option\n", azArg[1]); @@ -2485,7 +2485,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ case SQLITE_TESTCTRL_PRNG_RESET: if( nArg==2 ){ rc = sqlite3_test_control(testctrl); - printf("%d (0x%08x)\n", rc, rc); + fprintf(p->out, "%d (0x%08x)\n", rc, rc); } else { fprintf(stderr,"Error: testctrl %s takes no options\n", azArg[1]); } @@ -2496,7 +2496,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( nArg==3 ){ unsigned int opt = (unsigned int)integerValue(azArg[2]); rc = sqlite3_test_control(testctrl, opt); - printf("%d (0x%08x)\n", rc, rc); + fprintf(p->out, "%d (0x%08x)\n", rc, rc); } else { fprintf(stderr,"Error: testctrl %s takes a single unsigned" " int option\n", azArg[1]); @@ -2509,7 +2509,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( nArg==3 ){ int opt = atoi(azArg[2]); rc = sqlite3_test_control(testctrl, opt); - printf("%d (0x%08x)\n", rc, rc); + fprintf(p->out, "%d (0x%08x)\n", rc, rc); } else { fprintf(stderr,"Error: testctrl %s takes a single int option\n", azArg[1]); @@ -2522,7 +2522,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( nArg==3 ){ const char *opt = azArg[2]; rc = sqlite3_test_control(testctrl, opt); - printf("%d (0x%08x)\n", rc, rc); + fprintf(p->out, "%d (0x%08x)\n", rc, rc); } else { fprintf(stderr,"Error: testctrl %s takes a single char * option\n", azArg[1]); @@ -2567,7 +2567,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ }else if( c=='v' && strncmp(azArg[0], "version", n)==0 ){ - printf("SQLite %s %s\n" /*extra-version-info*/, + fprintf(p->out, "SQLite %s %s\n" /*extra-version-info*/, sqlite3_libversion(), sqlite3_sourceid()); }else @@ -2577,7 +2577,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( p->db ){ sqlite3_file_control(p->db, zDbName, SQLITE_FCNTL_VFSNAME, &zVfsName); if( zVfsName ){ - printf("%s\n", zVfsName); + fprintf(p->out, "%s\n", zVfsName); sqlite3_free(zVfsName); } } From 1ba00291a3a72898ca8e381e65bac8162dad5944 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 6 May 2013 21:01:06 +0000 Subject: [PATCH 198/204] In the command-line shell, output the elements of the schema in the order that they appear in the sqlite_master table. This is necessary in some cases to make the schema parsable. FossilOrigin-Name: e5b3cd747bb0b484e38b8611a81925e2cc144435 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 6 ++---- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 4ec94916cd..394b3344de 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\scommand-line\sshell,\smake\ssure\sthe\soutput\sof\sdot-commands\sgoes\sto\nthe\ssame\sredirected\soutput\sstream\sas\sthe\sresult\sof\squeries.\nTicket\s[2f96536e7ff7dba] -D 2013-05-06T20:28:54.574 +C In\sthe\scommand-line\sshell,\soutput\sthe\selements\sof\sthe\sschema\sin\sthe\sorder\nthat\sthey\sappear\sin\sthe\ssqlite_master\stable.\s\sThis\sis\snecessary\sin\ssome\ncases\sto\smake\sthe\sschema\sparsable. +D 2013-05-06T21:01:06.414 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -191,7 +191,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 83cc2d942ee216bc56956c6e6fadb691c1727fa1 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 6bfbe11e2fef81c5e18d30513ab6c69f171667eb -F src/shell.c 115bc2fa3ce4e89f5cea49ace173c4c9e68cfcf6 +F src/shell.c 2109d54f67c815a100abd7dc6a6e25eddb3b97eb F src/sqlite.h.in 5a5a22a9b192d81a9e5dee00274e3a0484c4afb1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 @@ -1061,7 +1061,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d8dc2c5fb554eb5898ddc8d703c6a360af7fe023 -R 51c5817bdcc09862ecf820138ddcd721 +P b72d365920dbe45e61d8b1cff708a8858c2b3e33 +R e0d7d129e376f94da4504f4153569f21 U drh -Z 6618db8b151a5d589fef00beca7c472a +Z e1d10da5e5ea459f8bce59c30cc68749 diff --git a/manifest.uuid b/manifest.uuid index c315d93ad3..1562d7bd87 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b72d365920dbe45e61d8b1cff708a8858c2b3e33 \ No newline at end of file +e5b3cd747bb0b484e38b8611a81925e2cc144435 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index e6dd496bd1..1be2871fed 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2278,8 +2278,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) " "WHERE lower(tbl_name) LIKE shellstatic()" " AND type!='meta' AND sql NOTNULL " - "ORDER BY substr(type,2,1), " - " CASE type WHEN 'view' THEN rowid ELSE name END", + "ORDER BY rowid", callback, &data, &zErrMsg); zShellStatic = 0; } @@ -2290,8 +2289,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ " FROM sqlite_master UNION ALL" " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) " "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%'" - "ORDER BY substr(type,2,1)," - " CASE type WHEN 'view' THEN rowid ELSE name END", + "ORDER BY rowid", callback, &data, &zErrMsg ); } From e8dee7d1ed76ea6604f478aa4f6f2a80ef2f1455 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 7 May 2013 12:16:48 +0000 Subject: [PATCH 199/204] Modify the fts3tokenize table implementation so that it does not use the SQL function fts3_tokenizer. The user may have installed an authorizer callback that prohibits this. FossilOrigin-Name: 0ba67b64de258883e4c43db09e131bb67083855e --- ext/fts3/fts3.c | 7 +++--- ext/fts3/fts3Int.h | 2 +- ext/fts3/fts3_tokenize_vtab.c | 43 ++++++++++++++--------------------- manifest | 20 ++++++++-------- manifest.uuid | 2 +- test/fts3tok1.test | 8 ------- 6 files changed, 33 insertions(+), 49 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 34d1e2acbd..c00a13f5a1 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3591,9 +3591,6 @@ int sqlite3Fts3Init(sqlite3 *db){ rc = sqlite3Fts3InitAux(db); if( rc!=SQLITE_OK ) return rc; - rc = sqlite3Fts3InitTok(db); - if( rc!=SQLITE_OK ) return rc; - sqlite3Fts3SimpleTokenizerModule(&pSimple); sqlite3Fts3PorterTokenizerModule(&pPorter); @@ -3647,9 +3644,13 @@ int sqlite3Fts3Init(sqlite3 *db){ db, "fts4", &fts3Module, (void *)pHash, 0 ); } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3InitTok(db, (void *)pHash); + } return rc; } + /* An error has occurred. Delete the hash table and return the error code. */ assert( rc!=SQLITE_OK ); if( pHash ){ diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 1fbb9c7587..b19064cd3b 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -550,7 +550,7 @@ int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); /* fts3_tokenize_vtab.c */ -int sqlite3Fts3InitTok(sqlite3*); +int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); /* fts3_unicode2.c (functions generated by parsing unicode text files) */ #ifdef SQLITE_ENABLE_FTS4_UNICODE61 diff --git a/ext/fts3/fts3_tokenize_vtab.c b/ext/fts3/fts3_tokenize_vtab.c index 9ec3b32aad..8528bda0bb 100644 --- a/ext/fts3/fts3_tokenize_vtab.c +++ b/ext/fts3/fts3_tokenize_vtab.c @@ -76,27 +76,22 @@ struct Fts3tokCursor { */ static int fts3tokQueryTokenizer( sqlite3 *db, + Fts3Hash *pHash, const char *zName, - const sqlite3_tokenizer_module **pp + const sqlite3_tokenizer_module **pp, + char **pzErr ){ - int rc; - sqlite3_stmt *pStmt; - const char *zSql = "SELECT fts3_tokenizer(?)"; + sqlite3_tokenizer_module *p; + int nName = strlen(zName); - *pp = 0; - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - if( rc!=SQLITE_OK ){ - return rc; + p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); + if( !p ){ + *pzErr = sqlite3_mprintf("unknown tokenizer: %s", zName); + return SQLITE_ERROR; } - sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ - memcpy((void*)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); - } - } - - return sqlite3_finalize(pStmt); + *pp = p; + return SQLITE_OK; } /* @@ -163,7 +158,7 @@ static int fts3tokDequoteArray( */ static int fts3tokConnectMethod( sqlite3 *db, /* Database connection */ - void *pUnused, /* Unused */ + void *pHash, /* Hash table of tokenizers */ int argc, /* Number of elements in argv array */ const char * const *argv, /* xCreate/xConnect argument array */ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ @@ -175,7 +170,6 @@ static int fts3tokConnectMethod( int rc; char **azDequote = 0; int nDequote; - UNUSED_PARAMETER(pUnused); rc = sqlite3_declare_vtab(db, FTS3_TOK_SCHEMA); if( rc!=SQLITE_OK ) return rc; @@ -190,14 +184,11 @@ static int fts3tokConnectMethod( }else{ zModule = azDequote[0]; } - rc = fts3tokQueryTokenizer(db, zModule, &pMod); + rc = fts3tokQueryTokenizer(db, (Fts3Hash *)pHash, zModule, &pMod, pzErr); } - if( rc!=SQLITE_OK ){ - *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); - }else if( pMod==0 ){ - rc = SQLITE_ERROR; - }else{ + assert( (rc==SQLITE_OK)==(pMod!=0) ); + if( rc==SQLITE_OK ){ const char * const *azArg = (const char * const *)&azDequote[1]; rc = pMod->xCreate((nDequote>1 ? nDequote-1 : 0), azArg, &pTok); } @@ -429,7 +420,7 @@ static int fts3tokRowidMethod( ** Register the fts3tok module with database connection db. Return SQLITE_OK ** if successful or an error code if sqlite3_create_module() fails. */ -int sqlite3Fts3InitTok(sqlite3 *db){ +int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ static const sqlite3_module fts3tok_module = { 0, /* iVersion */ fts3tokConnectMethod, /* xCreate */ @@ -457,7 +448,7 @@ int sqlite3Fts3InitTok(sqlite3 *db){ }; int rc; /* Return code */ - rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, 0); + rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, (void*)pHash); return rc; } diff --git a/manifest b/manifest index 394b3344de..6479021982 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\scommand-line\sshell,\soutput\sthe\selements\sof\sthe\sschema\sin\sthe\sorder\nthat\sthey\sappear\sin\sthe\ssqlite_master\stable.\s\sThis\sis\snecessary\sin\ssome\ncases\sto\smake\sthe\sschema\sparsable. -D 2013-05-06T21:01:06.414 +C Modify\sthe\sfts3tokenize\stable\simplementation\sso\sthat\sit\sdoes\snot\suse\sthe\sSQL\sfunction\sfts3_tokenizer.\sThe\suser\smay\shave\sinstalled\san\sauthorizer\scallback\sthat\sprohibits\sthis. +D 2013-05-07T12:16:48.845 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -55,9 +55,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 5c3d44d16701cc4bc81ebf0bb9d5bff136d42de0 +F ext/fts3/fts3.c 4bc160e6ff9ab5456b600f389f8941485ea5082f F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 23ea0a2bb7258d2539376ed60220cce28ba25765 +F ext/fts3/fts3Int.h 0b167bed9e63151635620a4f639bc62ac6012cba F ext/fts3/fts3_aux.c b02632f6dd0e375ce97870206d914ea6d8df5ccd F ext/fts3/fts3_expr.c 193d6fc156d744ab548a2ed06c31869e54dac739 F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 @@ -67,7 +67,7 @@ F ext/fts3/fts3_porter.c a465b49fcb8249a755792f87516eff182efa42b3 F ext/fts3/fts3_snippet.c 5fcfcafff46a2a3a63b8e59fcb51987d01c74695 F ext/fts3/fts3_term.c a521f75132f9a495bdca1bdd45949b3191c52763 F ext/fts3/fts3_test.c f9a1a1702db1bfad3e2d0064746eeb808f125489 -F ext/fts3/fts3_tokenize_vtab.c a29f126b9e6c6a6f1021a8f7440bf125e68af1f9 +F ext/fts3/fts3_tokenize_vtab.c 03a428b6221dc0b3f4527b6004266c5db71a2026 F ext/fts3/fts3_tokenizer.c bbdc731bc91338050675c6d1da9ab82147391e16 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 @@ -514,7 +514,7 @@ F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0 F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2 F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 -F test/fts3tok1.test 05ff5c57098282bacba7192507702b17f4061295 +F test/fts3tok1.test 4d9e7401679dc71f6b2f76416309b923210bfdbe F test/fts3tok_err.test 41e5413139c2ef536ffadfcd1584ee50b1665ec0 F test/fts4aa.test 95f448fb02c4a976968b08d1b4ce134e720946ae F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8 @@ -1061,7 +1061,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b72d365920dbe45e61d8b1cff708a8858c2b3e33 -R e0d7d129e376f94da4504f4153569f21 -U drh -Z e1d10da5e5ea459f8bce59c30cc68749 +P e5b3cd747bb0b484e38b8611a81925e2cc144435 +R 0fdf3312f3c095b75e4d2deab47476b0 +U dan +Z 58e39fd3c70070287886f650d61f780d diff --git a/manifest.uuid b/manifest.uuid index 1562d7bd87..5c202a5911 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e5b3cd747bb0b484e38b8611a81925e2cc144435 \ No newline at end of file +0ba67b64de258883e4c43db09e131bb67083855e \ No newline at end of file diff --git a/test/fts3tok1.test b/test/fts3tok1.test index 89f799606b..98e55a06c1 100644 --- a/test/fts3tok1.test +++ b/test/fts3tok1.test @@ -106,15 +106,7 @@ do_catchsql_test 2.0 { CREATE VIRTUAL TABLE tX USING fts3tokenize(nosuchtokenizer); } {1 {unknown tokenizer: nosuchtokenizer}} -proc fts3_tokenizer {args} { return 1 } -db function fts3_tokenizer -argcount 1 fts3_tokenizer do_catchsql_test 2.1 { - CREATE VIRTUAL TABLE tX USING fts3tokenize(simple); -} {1 {vtable constructor failed: tX}} - -db close -sqlite3 db test.db -do_catchsql_test 2.2 { CREATE VIRTUAL TABLE t4 USING fts3tokenize; SELECT * FROM t4; } {1 {SQL logic error or missing database}} From c01b7306ae459f544c074c2f5c92f5aa8baada20 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 May 2013 17:49:08 +0000 Subject: [PATCH 200/204] Make sure the ORDER BY collating sequences are compatible with the comparison collations before using the merge algorithm for compound SELECT statements. Candidate fix for ticket [6709574d2a8d8]. FossilOrigin-Name: fc3630cdef6e2cdbfb4e7b373d1a094753e55016 --- manifest | 20 ++++++---- manifest.uuid | 2 +- src/select.c | 67 ++++++++++++++++++++++++++++++++- test/selectA.test | 20 +++++----- test/selectE.test | 95 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 184 insertions(+), 20 deletions(-) create mode 100644 test/selectE.test diff --git a/manifest b/manifest index 6479021982..404651e365 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\sfts3tokenize\stable\simplementation\sso\sthat\sit\sdoes\snot\suse\sthe\sSQL\sfunction\sfts3_tokenizer.\sThe\suser\smay\shave\sinstalled\san\sauthorizer\scallback\sthat\sprohibits\sthis. -D 2013-05-07T12:16:48.845 +C Make\ssure\sthe\sORDER\sBY\scollating\ssequences\sare\scompatible\swith\sthe\ncomparison\scollations\sbefore\susing\sthe\smerge\salgorithm\sfor\scompound\nSELECT\sstatements.\s\sCandidate\sfix\sfor\sticket\s[6709574d2a8d8]. +D 2013-05-07T17:49:08.546 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -190,7 +190,7 @@ F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 83cc2d942ee216bc56956c6e6fadb691c1727fa1 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 6bfbe11e2fef81c5e18d30513ab6c69f171667eb +F src/select.c a4641882279becc200f2680f55f3e89d4e7c7f78 F src/shell.c 2109d54f67c815a100abd7dc6a6e25eddb3b97eb F src/sqlite.h.in 5a5a22a9b192d81a9e5dee00274e3a0484c4afb1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 @@ -723,10 +723,11 @@ F test/select6.test e76bd10a56988f15726c097a5d5a7966fe82d3b2 F test/select7.test dad6f00f0d49728a879d6eb6451d4752db0b0abe F test/select8.test 391de11bdd52339c30580dabbbbe97e3e9a3c79d F test/select9.test c0ca3cd87a8ebb04de2cb1402c77df55d911a0ea -F test/selectA.test 06d1032fa9009314c95394f2ca2e60d9f7ae8532 +F test/selectA.test 99cf21df033b93033ea4f34aba14a500f48f04fe F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25 F test/selectC.test 871fb55d884d3de5943c4057ebd22c2459e71977 F test/selectD.test b0f02a04ef7737decb24e08be2c39b9664b43394 +F test/selectE.test fc02a1eb04c8eb537091482644b7d778ae8759b7 F test/server1.test 46803bd3fe8b99b30dbc5ff38ffc756f5c13a118 F test/shared.test 1da9dbad400cee0d93f252ccf76e1ae007a63746 F test/shared2.test 03eb4a8d372e290107d34b6ce1809919a698e879 @@ -1061,7 +1062,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e5b3cd747bb0b484e38b8611a81925e2cc144435 -R 0fdf3312f3c095b75e4d2deab47476b0 -U dan -Z 58e39fd3c70070287886f650d61f780d +P 0ba67b64de258883e4c43db09e131bb67083855e +R 298d9b4b04cefe916bd24254e1442751 +T *branch * tkt-6709574 +T *sym-tkt-6709574 * +T -sym-trunk * +U drh +Z 52d856a3284951cf8623e01987d271c7 diff --git a/manifest.uuid b/manifest.uuid index 5c202a5911..2528c2de9e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0ba67b64de258883e4c43db09e131bb67083855e \ No newline at end of file +fc3630cdef6e2cdbfb4e7b373d1a094753e55016 \ No newline at end of file diff --git a/src/select.c b/src/select.c index a745dc370a..f3f1490963 100644 --- a/src/select.c +++ b/src/select.c @@ -3260,6 +3260,69 @@ int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pFrom){ } return SQLITE_OK; } +/* +** Detect compound SELECT statements that use an ORDER BY clause with +** an alternative collating sequence. +** +** SELECT ... FROM t1 EXCEPT SELECT ... FROM t2 ORDER BY .. COLLATE ... +** +** These are rewritten as a subquery: +** +** SELECT * FROM (SELECT ... FROM t1 EXCEPT SELECT ... FROM t2) +** ORDER BY ... COLLATE ... +** +** This transformation is necessary because the multiSelectOrderBy() routine +** above that generates the code for a compound SELECT with an ORDER BY clause +** uses a merge algorithm that requires the same collating sequence on the +** result columns as on the ORDER BY clause. See ticket +** http://www.sqlite.org/src/info/6709574d2a +** +** This transformation is only needed for EXCEPT, INTERSECT, and UNION. +** The UNION ALL operator works fine with multiSelectOrderBy() even when +** there are COLLATE terms in the ORDER BY. +*/ +static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ + int i; + Select *pNew; + Select *pX; + sqlite3 *db; + struct ExprList_item *a; + SrcList *pNewSrc; + Parse *pParse; + Token dummy; + + if( p->pPrior==0 ) return WRC_Continue; + if( p->pOrderBy==0 ) return WRC_Continue; + for(pX=p; pX && (pX->op==TK_ALL || pX->op==TK_SELECT); pX=pX->pPrior){} + if( pX==0 ) return WRC_Continue; + a = p->pOrderBy->a; + for(i=p->pOrderBy->nExpr-1; i>=0; i--){ + if( a[i].pExpr->flags & EP_Collate ) break; + } + if( i<0 ) return WRC_Continue; + + /* If we reach this point, that means the transformation is required. */ + + pParse = pWalker->pParse; + db = pParse->db; + pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); + if( pNew==0 ) return WRC_Abort; + memset(&dummy, 0, sizeof(dummy)); + pNewSrc = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&dummy,pNew,0,0); + if( pNewSrc==0 ) return WRC_Abort; + *pNew = *p; + p->pSrc = pNewSrc; + p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ALL, 0)); + p->op = TK_SELECT; + p->pWhere = 0; + pNew->pGroupBy = 0; + pNew->pHaving = 0; + pNew->pOrderBy = 0; + p->pPrior = 0; + pNew->pLimit = 0; + pNew->pOffset = 0; + return WRC_Continue; +} /* ** This routine is a Walker callback for "expanding" a SELECT statement. @@ -3577,10 +3640,12 @@ static int exprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ Walker w; memset(&w, 0, sizeof(w)); - w.xSelectCallback = selectExpander; + w.xSelectCallback = convertCompoundSelectToSubquery; w.xExprCallback = exprWalkNoop; w.pParse = pParse; sqlite3WalkSelect(&w, pSelect); + w.xSelectCallback = selectExpander; + sqlite3WalkSelect(&w, pSelect); } diff --git a/test/selectA.test b/test/selectA.test index 5b8637784b..5fd2288dbf 100644 --- a/test/selectA.test +++ b/test/selectA.test @@ -281,13 +281,13 @@ do_test selectA-2.34 { do_test selectA-2.35 { execsql { SELECT x,y,z FROM t2 UNION SELECT a,b,c FROM t1 - ORDER BY b COLLATE NOCASE,a,c + ORDER BY y COLLATE NOCASE,x,z } } {1 a a 9.9 b B {} C c hello d D abc e e hare m M {} U u 5200000.0 X x -23 Y y mad Z z} do_test selectA-2.36 { execsql { SELECT x,y,z FROM t2 UNION SELECT a,b,c FROM t1 - ORDER BY b COLLATE NOCASE DESC,a,c + ORDER BY y COLLATE NOCASE DESC,x,z } } {mad Z z -23 Y y 5200000.0 X x {} U u hare m M abc e e hello d D {} C c 9.9 b B 1 a a} do_test selectA-2.37 { @@ -311,7 +311,7 @@ do_test selectA-2.39 { do_test selectA-2.40 { execsql { SELECT x,y,z FROM t2 UNION SELECT a,b,c FROM t1 - ORDER BY c COLLATE BINARY DESC,a,b + ORDER BY z COLLATE BINARY DESC,x,y } } {mad Z z -23 Y y 5200000.0 X x {} U u abc e e {} C c 1 a a hare m M hello d D 9.9 b B} do_test selectA-2.41 { @@ -602,7 +602,7 @@ do_test selectA-2.85 { do_test selectA-2.86 { execsql { SELECT x,y,z FROM t2 UNION SELECT a,b,c FROM t3 - ORDER BY b COLLATE NOCASE,a,c + ORDER BY y COLLATE NOCASE,x,z } } {1 a a 9.9 b B {} C c hello d D abc e e hare m M {} U u 5200000.0 X x -23 Y y mad Z z} do_test selectA-2.87 { @@ -632,7 +632,7 @@ do_test selectA-2.90 { do_test selectA-2.91 { execsql { SELECT x,y,z FROM t2 UNION SELECT a,b,c FROM t3 - ORDER BY c COLLATE BINARY DESC,a,b + ORDER BY z COLLATE BINARY DESC,x,y } } {mad Z z -23 Y y 5200000.0 X x {} U u abc e e {} C c 1 a a hare m M hello d D 9.9 b B} do_test selectA-2.92 { @@ -893,13 +893,13 @@ do_test selectA-3.34 { do_test selectA-3.35 { execsql { SELECT x,y,z FROM t2 UNION SELECT a,b,c FROM t1 - ORDER BY b COLLATE NOCASE,a,c + ORDER BY y COLLATE NOCASE,x,z } } {1 a a 9.9 b B {} C c hello d D abc e e hare m M {} U u 5200000.0 X x -23 Y y mad Z z} do_test selectA-3.36 { execsql { SELECT x,y,z FROM t2 UNION SELECT a,b,c FROM t1 - ORDER BY b COLLATE NOCASE DESC,a,c + ORDER BY y COLLATE NOCASE DESC,x,z } } {mad Z z -23 Y y 5200000.0 X x {} U u hare m M abc e e hello d D {} C c 9.9 b B 1 a a} do_test selectA-3.37 { @@ -923,7 +923,7 @@ do_test selectA-3.39 { do_test selectA-3.40 { execsql { SELECT x,y,z FROM t2 UNION SELECT a,b,c FROM t1 - ORDER BY c COLLATE BINARY DESC,a,b + ORDER BY z COLLATE BINARY DESC,x,y } } {mad Z z -23 Y y 5200000.0 X x {} U u abc e e {} C c 1 a a hare m M hello d D 9.9 b B} do_test selectA-3.41 { @@ -1214,7 +1214,7 @@ do_test selectA-3.85 { do_test selectA-3.86 { execsql { SELECT x,y,z FROM t2 UNION SELECT a,b,c FROM t3 - ORDER BY b COLLATE NOCASE,a,c + ORDER BY y COLLATE NOCASE,x,z } } {1 a a 9.9 b B {} C c hello d D abc e e hare m M {} U u 5200000.0 X x -23 Y y mad Z z} do_test selectA-3.87 { @@ -1244,7 +1244,7 @@ do_test selectA-3.90 { do_test selectA-3.91 { execsql { SELECT x,y,z FROM t2 UNION SELECT a,b,c FROM t3 - ORDER BY c COLLATE BINARY DESC,a,b + ORDER BY z COLLATE BINARY DESC,x,y } } {mad Z z -23 Y y 5200000.0 X x {} U u abc e e {} C c 1 a a hare m M hello d D 9.9 b B} do_test selectA-3.92 { diff --git a/test/selectE.test b/test/selectE.test new file mode 100644 index 0000000000..d7592bbbc5 --- /dev/null +++ b/test/selectE.test @@ -0,0 +1,95 @@ +# 2013-05-07 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for compound SELECT statements +# that have ORDER BY clauses with collating sequences that differ +# from the collating sequence used for comparison in the compound. +# +# Ticket 6709574d2a8d8b9be3a9cb1afbf4ff2de48ea4e7: +# drh added on 2013-05-06 15:21:16: +# +# In the code shown below (which is intended to be run from the +# sqlite3.exe command-line tool) the three SELECT statements should all +# generate the same answer. But the third one does not. It is as if the +# COLLATE clause on the ORDER BY somehow got pulled into the EXCEPT +# operator. Note that the ".print" commands are instructions to the +# sqlite3.exe shell program to output delimiter lines so that you can more +# easily tell where the output of one query ends and the next query +# begins. +# +# CREATE TABLE t1(a); +# INSERT INTO t1 VALUES('abc'),('def'); +# CREATE TABLE t2(a); +# INSERT INTO t2 VALUES('DEF'); +# +# SELECT a FROM t1 EXCEPT SELECT a FROM t2 ORDER BY a; +# .print ----- +# SELECT a FROM (SELECT a FROM t1 EXCEPT SELECT a FROM t2) +# ORDER BY a COLLATE nocase; +# .print ----- +# SELECT a FROM t1 EXCEPT SELECT a FROM t2 ORDER BY a COLLATE nocase; +# +# Bisecting shows that this problem was introduced in SQLite version 3.6.0 +# by check-in [8bbfa97837a74ef] on 2008-06-15. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_test selectE-1.0 { + db eval { + CREATE TABLE t1(a); + INSERT INTO t1 VALUES('abc'),('def'),('ghi'); + CREATE TABLE t2(a); + INSERT INTO t2 VALUES('DEF'),('abc'); + CREATE TABLE t3(a); + INSERT INTO t3 VALUES('def'),('jkl'); + + SELECT a FROM t1 EXCEPT SELECT a FROM t2 + ORDER BY a COLLATE nocase; + } +} {def ghi} +do_test selectE-1.1 { + db eval { + SELECT a FROM t2 EXCEPT SELECT a FROM t3 + ORDER BY a COLLATE nocase; + } +} {abc DEF} +do_test selectE-1.2 { + db eval { + SELECT a FROM t2 EXCEPT SELECT a FROM t3 + ORDER BY a COLLATE binary; + } +} {DEF abc} +do_test selectE-1.3 { + db eval { + SELECT a FROM t2 EXCEPT SELECT a FROM t3 + ORDER BY a; + } +} {DEF abc} + +do_test selectE-2.1 { + db eval { + DELETE FROM t2; + DELETE FROM t3; + INSERT INTO t2 VALUES('ABC'),('def'),('GHI'),('jkl'); + INSERT INTO t3 SELECT lower(a) FROM t2; + SELECT a COLLATE nocase FROM t2 EXCEPT SELECT a FROM t3 + ORDER BY 1 + } +} {} +do_test selectE-2.2 { + db eval { + SELECT a COLLATE nocase FROM t2 EXCEPT SELECT a FROM t3 + ORDER BY 1 COLLATE binary + } +} {} + +finish_test From 8c15d99a9a50e15eaa424e0d92cec698cdbaf33b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 May 2013 14:20:28 +0000 Subject: [PATCH 201/204] Fix the wholenumber virtual table so that it returns higher costs for unconstrained usage. Cherrypick from [ceff8955020cd13]. FossilOrigin-Name: 7227b61512a534645c73883c2e37b4e647592e5d --- ext/misc/wholenumber.c | 8 +++++++- manifest | 15 ++++++++------- manifest.uuid | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/ext/misc/wholenumber.c b/ext/misc/wholenumber.c index a6d79507c4..63369c6ac4 100644 --- a/ext/misc/wholenumber.c +++ b/ext/misc/wholenumber.c @@ -218,7 +218,13 @@ static int wholenumberBestIndex( ){ pIdxInfo->orderByConsumed = 1; } - pIdxInfo->estimatedCost = (double)1; + if( (idxNum & 12)==0 ){ + pIdxInfo->estimatedCost = (double)100000000; + }else if( (idxNum & 3)==0 ){ + pIdxInfo->estimatedCost = (double)5; + }else{ + pIdxInfo->estimatedCost = (double)1; + } return SQLITE_OK; } diff --git a/manifest b/manifest index 6479021982..664b5cf818 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\sfts3tokenize\stable\simplementation\sso\sthat\sit\sdoes\snot\suse\sthe\sSQL\sfunction\sfts3_tokenizer.\sThe\suser\smay\shave\sinstalled\san\sauthorizer\scallback\sthat\sprohibits\sthis. -D 2013-05-07T12:16:48.845 +C Fix\sthe\swholenumber\svirtual\stable\sso\sthat\sit\sreturns\shigher\scosts\sfor\nunconstrained\susage.\s\sCherrypick\sfrom\s[ceff8955020cd13]. +D 2013-05-08T14:20:28.350 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -90,7 +90,7 @@ F ext/misc/ieee754.c 2565ce373d842977efe0922dc50b8a41b3289556 F ext/misc/nextchar.c 1131e2b36116ffc6fe6b2e3464bfdace27978b1e F ext/misc/regexp.c c25c65fe775f5d9801fb8573e36ebe73f2c0c2e0 F ext/misc/spellfix.c f9d24a2b2617cee143b7841b453e4e1fd8f189cc -F ext/misc/wholenumber.c ce362368b9381ea48cbd951ade8df867eeeab014 +F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 757abea591d4ff67c0ff4e8f9776aeda86b18c14 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1061,7 +1061,8 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e5b3cd747bb0b484e38b8611a81925e2cc144435 -R 0fdf3312f3c095b75e4d2deab47476b0 -U dan -Z 58e39fd3c70070287886f650d61f780d +P 0ba67b64de258883e4c43db09e131bb67083855e +Q +ceff8955020cd1314bf1ab0af7d075fe2c0863e5 +R 3c92370100000406d8695d9d00d3c30f +U drh +Z 3250570cd81603159ecf8b6de54c9572 diff --git a/manifest.uuid b/manifest.uuid index 5c202a5911..a9e3d58d46 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0ba67b64de258883e4c43db09e131bb67083855e \ No newline at end of file +7227b61512a534645c73883c2e37b4e647592e5d \ No newline at end of file From 92692d32ad129fba340f3f1c7ba50a3491b48fc4 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 May 2013 17:06:28 +0000 Subject: [PATCH 202/204] Fix a couple of harmless compiler warnings in the fts3_tokenize virtual table. FossilOrigin-Name: 1fa8c457394c94864f7584e4c893ec09e685fba4 --- ext/fts3/fts3_tokenize_vtab.c | 5 ++--- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ext/fts3/fts3_tokenize_vtab.c b/ext/fts3/fts3_tokenize_vtab.c index 8528bda0bb..364852ef82 100644 --- a/ext/fts3/fts3_tokenize_vtab.c +++ b/ext/fts3/fts3_tokenize_vtab.c @@ -75,14 +75,13 @@ struct Fts3tokCursor { ** Query FTS for the tokenizer implementation named zName. */ static int fts3tokQueryTokenizer( - sqlite3 *db, Fts3Hash *pHash, const char *zName, const sqlite3_tokenizer_module **pp, char **pzErr ){ sqlite3_tokenizer_module *p; - int nName = strlen(zName); + int nName = (int)strlen(zName); p = (sqlite3_tokenizer_module *)sqlite3Fts3HashFind(pHash, zName, nName+1); if( !p ){ @@ -184,7 +183,7 @@ static int fts3tokConnectMethod( }else{ zModule = azDequote[0]; } - rc = fts3tokQueryTokenizer(db, (Fts3Hash *)pHash, zModule, &pMod, pzErr); + rc = fts3tokQueryTokenizer((Fts3Hash*)pHash, zModule, &pMod, pzErr); } assert( (rc==SQLITE_OK)==(pMod!=0) ); diff --git a/manifest b/manifest index 71317e9b94..e9fbbc3ca6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\sORDER\sBY\scollating\ssequences\sare\scompatible\swith\sthe\ncomparison\scollations\sbefore\susing\sthe\smerge\salgorithm\sfor\scompound\s\nSELECT\sstatements.\s\sFix\sfor\sticket\s[6709574d2a8d8]. -D 2013-05-08T16:57:48.917 +C Fix\sa\scouple\sof\sharmless\scompiler\swarnings\sin\sthe\sfts3_tokenize\svirtual\ntable. +D 2013-05-08T17:06:28.377 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -67,7 +67,7 @@ F ext/fts3/fts3_porter.c a465b49fcb8249a755792f87516eff182efa42b3 F ext/fts3/fts3_snippet.c 5fcfcafff46a2a3a63b8e59fcb51987d01c74695 F ext/fts3/fts3_term.c a521f75132f9a495bdca1bdd45949b3191c52763 F ext/fts3/fts3_test.c f9a1a1702db1bfad3e2d0064746eeb808f125489 -F ext/fts3/fts3_tokenize_vtab.c 03a428b6221dc0b3f4527b6004266c5db71a2026 +F ext/fts3/fts3_tokenize_vtab.c 011170fe9eba5ff062f1a31d3188e00267716706 F ext/fts3/fts3_tokenizer.c bbdc731bc91338050675c6d1da9ab82147391e16 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 @@ -1062,7 +1062,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 7227b61512a534645c73883c2e37b4e647592e5d fc3630cdef6e2cdbfb4e7b373d1a094753e55016 -R dcccb23e4f316b16da66bbbbad7a9346 +P ce853a75068073d6acc8bec0819505a22c4c7e69 +R 693e66fc26b30225620a1dd87e0b79b3 U drh -Z 6134f15a41242cbabed6e0256cc06784 +Z a746215c07dfe5ad61be18a3c8d6b29d diff --git a/manifest.uuid b/manifest.uuid index f381b2208e..00a6e72965 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ce853a75068073d6acc8bec0819505a22c4c7e69 \ No newline at end of file +1fa8c457394c94864f7584e4c893ec09e685fba4 \ No newline at end of file From b3129fa560d12ae44cb0a992a4a4b4980da1e494 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 9 May 2013 14:20:11 +0000 Subject: [PATCH 203/204] Do not move WHERE clause terms inside OR expressions that are contained within an ON clause of a LEFT JOIN. Fix for ticket [f2369304e47167e3e]. FossilOrigin-Name: 1128575d0ab24f7023a0f6e6ce4828b9a09a7c6c --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/where.c | 6 +++++- test/where9.test | 21 +++++++++++++++++++++ 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index e6aa18a66a..3e1372250e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refine\sand\sreform\sall\sWindows\sOSTRACE\smacro\susage.\s\sReplace\sall\susage\sof\ssqlite3TestErrorName()\swith\ssqlite3ErrName()\sand\sadd\smissing\sreturn\scodes. -D 2013-05-09T00:40:13.506 +C Do\snot\smove\sWHERE\sclause\sterms\sinside\sOR\sexpressions\sthat\sare\scontained\nwithin\san\sON\sclause\sof\sa\sLEFT\sJOIN.\s\sFix\sfor\sticket\s[f2369304e47167e3e]. +D 2013-05-09T14:20:11.944 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -264,7 +264,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 12d4200eb6ae991cad02367c391db076ac1af1b0 +F src/where.c 5c4cbc1e5205d8d534c483fa4495c81513b45dea F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1007,7 +1007,7 @@ F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b F test/where7.test 5c566388f0cc318b0032ce860f4ac5548e3c265a F test/where8.test d9f889e62dccddb9d790b0c84dfc7861e03a162c F test/where8m.test da346596e19d54f0aba35ebade032a7c47d79739 -F test/where9.test 0157862ccf0cfdf1a4622cdf697e5e2f09a8de44 +F test/where9.test 1b4387c6eacc9a32b28b4d837c27f857c785d0d8 F test/whereA.test 24c234263c8fe358f079d5e57d884fb569d2da0a F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/whereC.test 13ff5ec0dba407c0e0c075980c75b3275a6774e5 @@ -1062,7 +1062,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 1fa8c457394c94864f7584e4c893ec09e685fba4 400fc4c37ff34e84f5a129a083a567bda840278e -R a3f98c872243f05bc78c5a52c22d5a59 -U mistachkin -Z 68d84fd2c9e0a14b0e1248a70789d361 +P 610425f19008b8b5d4bf027c8b14d97ec3115a54 +R 4023dbf2ca58f8c405419172cf8781f4 +U drh +Z cd99743e7f7615602ad15535e32e4fa6 diff --git a/manifest.uuid b/manifest.uuid index bc648b3196..df6e547ae9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -610425f19008b8b5d4bf027c8b14d97ec3115a54 \ No newline at end of file +1128575d0ab24f7023a0f6e6ce4828b9a09a7c6c \ No newline at end of file diff --git a/src/where.c b/src/where.c index 2de894c3e7..e614f4a6d8 100644 --- a/src/where.c +++ b/src/where.c @@ -4726,6 +4726,10 @@ static Bitmask codeOneLoopStart( ** the "interesting" terms of z - terms that did not originate in the ** ON or USING clause of a LEFT JOIN, and terms that are usable as ** indices. + ** + ** This optimization also only applies if the (x1 OR x2 OR ...) term + ** is not contained in the ON clause of a LEFT JOIN. + ** See ticket http://www.sqlite.org/src/info/f2369304e4 */ if( pWC->nTerm>1 ){ int iTerm; @@ -4747,7 +4751,7 @@ static Bitmask codeOneLoopStart( if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ WhereInfo *pSubWInfo; /* Info for single OR-term scan */ Expr *pOrExpr = pOrTerm->pExpr; - if( pAndExpr ){ + if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){ pAndExpr->pLeft = pOrExpr; pOrExpr = pAndExpr; } diff --git a/test/where9.test b/test/where9.test index a5c1d3935a..1e94fdff78 100644 --- a/test/where9.test +++ b/test/where9.test @@ -893,4 +893,25 @@ do_test where9-8.3 { } } {2 3 4 5 {} {} 5 55 3 4 5 6 2 4 5 55} +# Fix for ticket [f2369304e47167e3e644e2f1fe9736063391d7b7] +# Incorrect results when OR is used in the ON clause of a LEFT JOIN +# +do_test where9-9.1 { + db eval { + CREATE TABLE t91(x); INSERT INTO t91 VALUES(1); + CREATE TABLE t92(y INTEGER PRIMARY KEY,a,b); + INSERT INTO t92 VALUES(1,2,3); + SELECT 1 FROM t91 LEFT JOIN t92 ON a=2 OR b=3; + SELECT 2 FROM t91 LEFT JOIN t92 ON a=2 AND b=3; + SELECT 3 FROM t91 LEFT JOIN t92 ON (a=2 OR b=3) AND y IS NULL; + SELECT 4 FROM t91 LEFT JOIN t92 ON (a=2 AND b=3) AND y IS NULL; + CREATE TEMP TABLE x9 AS SELECT * FROM t91 LEFT JOIN t92 ON a=2 OR b=3; + SELECT 5 FROM x9 WHERE y IS NULL; + SELECT 6 FROM t91 LEFT JOIN t92 ON a=2 OR b=3 WHERE y IS NULL; + SELECT 7 FROM t91 LEFT JOIN t92 ON a=2 AND b=3 WHERE y IS NULL; + SELECT 8 FROM t91 LEFT JOIN t92 ON a=22 OR b=33 WHERE y IS NULL; + SELECT 9 FROM t91 LEFT JOIN t92 ON a=22 AND b=33 WHERE y IS NULL; + } +} {1 2 3 4 8 9} + finish_test From 6cf9d8d63a37c4990838d8ac660529f27ef3cba9 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 9 May 2013 18:12:40 +0000 Subject: [PATCH 204/204] Add assert()s to the implementation of xRead() in the built-in VFSes to verify that the offset parameter is always non-negative. FossilOrigin-Name: cf5c3642247fdd34d87f0368594cd7b8f081636a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 2 ++ src/os_win.c | 1 + 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3e1372250e..525586f02f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\smove\sWHERE\sclause\sterms\sinside\sOR\sexpressions\sthat\sare\scontained\nwithin\san\sON\sclause\sof\sa\sLEFT\sJOIN.\s\sFix\sfor\sticket\s[f2369304e47167e3e]. -D 2013-05-09T14:20:11.944 +C Add\sassert()s\sto\sthe\simplementation\sof\sxRead()\sin\sthe\sbuilt-in\sVFSes\sto\s\nverify\sthat\sthe\soffset\sparameter\sis\salways\snon-negative. +D 2013-05-09T18:12:40.898 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ce81671efd6223d19d4c8c6b88ac2c4134427111 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,8 +176,8 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 658b180a09a18214d94547f737dbded71667cdab -F src/os_win.c bf297963be89f9cddab53c3356e79daa3b8240d2 +F src/os_unix.c 5e0b0ca7594f6707a3ed155528a3ba3318c95e15 +F src/os_win.c 4e2bf0760409aef35e298ff725054e94d834e1a3 F src/pager.c 49e23f9898113ddfe90942bdf1c1ef57955d0921 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 9708365594eea519cdc8504dee425c0a41c79502 @@ -1062,7 +1062,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 610425f19008b8b5d4bf027c8b14d97ec3115a54 -R 4023dbf2ca58f8c405419172cf8781f4 +P 1128575d0ab24f7023a0f6e6ce4828b9a09a7c6c +R 1cad29e868ca8cf8d5b7d354acd90934 U drh -Z cd99743e7f7615602ad15535e32e4fa6 +Z 3b3f34da85296e9879a826be138cb510 diff --git a/manifest.uuid b/manifest.uuid index df6e547ae9..03a8fec162 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1128575d0ab24f7023a0f6e6ce4828b9a09a7c6c \ No newline at end of file +cf5c3642247fdd34d87f0368594cd7b8f081636a \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 6eed674072..30d79f0c50 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3152,6 +3152,8 @@ static int unixRead( unixFile *pFile = (unixFile *)id; int got; assert( id ); + assert( offset>=0 ); + assert( amt>0 ); /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ diff --git a/src/os_win.c b/src/os_win.c index e308248158..66f0703878 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2157,6 +2157,7 @@ static int winRead( assert( id!=0 ); assert( amt>0 ); + assert( offset>=0 ); SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ file=%p, buffer=%p, amount=%d, offset=%lld, lock=%d\n", pFile->h, pBuf, amt, offset, pFile->locktype));