From b2d3de3bf47b7eaf5bb6bf68de5946895bd943af Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 14 Mar 2013 18:34:37 +0000 Subject: [PATCH 001/140] Use mmap() to read from the database file in rollback mode. This branch is unix only for now. FossilOrigin-Name: 6f21d9cbf5d457e63a7282015a89ae785526cf6d --- manifest | 25 ++++---- manifest.uuid | 2 +- src/btree.c | 31 ++++++++++ src/os_unix.c | 4 ++ src/pager.c | 129 ++++++++++++++++++++++++++++++++++++++++- src/pcache.h | 2 + src/sqlite.h.in | 1 + test/permutations.test | 3 +- 8 files changed, 182 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 42eb32ea0f..a61f43717c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\stests\sfor\sticket\s[4dd95f6943]. -D 2013-03-13T07:02:04.083 +C Use\smmap()\sto\sread\sfrom\sthe\sdatabase\sfile\sin\srollback\smode.\sThis\sbranch\sis\sunix\sonly\sfor\snow. +D 2013-03-14T18:34:37.796 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 746c4dafae8565b3be6fb9ce3bb1fa9f1e67cc59 +F src/btree.c c1a956c6762f2a45188c945e1070daec29f5253f F src/btree.h 3ad7964d6c5b1c7bff569aab6adfa075f8bf06cd F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 375e5df716e03b9343c5e1211be3b24e6d6dff05 @@ -160,13 +160,13 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c f6387eef0cf8f6b808738f4f3aa47e6132af0940 +F src/os_unix.c 2a4cd96aabf413f39cf562baebb27aa9993f6f54 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 582f8da52d0bd4b43d3bdaeba0ea7702c1f23702 +F src/pager.c 4e7e66c2959ee43caf8a7000712d6a4121b0888a F src/pager.h 1109a06578ec5574dc2c74cf8d9f69daf36fe3e0 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 -F src/pcache.h 1b5dcc3dc8103d03e625b177023ee67764fa6b7c +F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 F src/pragma.c 9f0ee3d74a7f33eeeff40a4b014fc3abf8182ce2 F src/prepare.c 78cd7ae8cd0d8de8ef8a8b5352fc5f38693a0852 @@ -176,7 +176,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in f2fa32f440dda59ca47e22889966b2a6eb3b491c +F src/sqlite.h.in 9a5c737a1feb4495d351c56883587d4fda52e81e F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 0f8f05ee4db4ba9120b38f7a3992b325698f6e8a @@ -656,7 +656,7 @@ F test/pageropt.test 9191867ed19a2b3db6c42d1b36b6fbc657cd1ab0 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test 360b92859c0af814b3fe10b68746936389606501 +F test/permutations.test eb49937dca270b2c3f62d4c91fc7034ca905b7f1 F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -1038,7 +1038,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 839aa91faf1db7025d90fa3c65e50efb829b053b -R ad71abb6cd4a6ebc6857153126653bad +P 0b452734faa0839c817f040322e7733e423bfce2 +R d4c281f6a900c9b41816881ffb3c2331 +T *branch * experimental-mmap +T *sym-experimental-mmap * +T -sym-trunk * U dan -Z 69b39b8becc09fc4758d1b797bf5f4f2 +Z f16a594407ec9e95c26319dfbc751443 diff --git a/manifest.uuid b/manifest.uuid index 3d8be2e93e..dbbf6a71bc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0b452734faa0839c817f040322e7733e423bfce2 \ No newline at end of file +6f21d9cbf5d457e63a7282015a89ae785526cf6d \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 07ec3fe52a..038126c3f0 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2564,6 +2564,34 @@ int sqlite3BtreeNewDb(Btree *p){ return rc; } +/* +** If the shared-btree passed as the only argument is holding references +** to mmap pages, replace them with read/write pages. Return SQLITE_OK +** if successful, or an error code otherwise. +*/ +static int btreeSwapOutMmap(BtShared *pBt){ + BtCursor *pCsr; + for(pCsr=pBt->pCursor; pCsr; pCsr=pCsr->pNext){ + int i; + for(i=0; i<=pCsr->iPage; i++){ + MemPage *pPg = pCsr->apPage[i]; + if( pPg->pDbPage->flags & PGHDR_MMAP ){ + int rc; + MemPage *pNew = 0; + rc = btreeGetPage(pBt, pPg->pgno, &pNew, 0); + if( rc==SQLITE_OK && i==pCsr->iPage ){ + pCsr->info.pCell = pNew->aData + (pCsr->info.pCell - pPg->aData); + } + pCsr->apPage[i] = pNew; + releasePage(pPg); + if( rc!=SQLITE_OK ) return rc; + } + } + } + + return SQLITE_OK; +} + /* ** Attempt to start a new transaction. A write-transaction ** is started if the second argument is nonzero, otherwise a read- @@ -2670,6 +2698,9 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ rc = SQLITE_READONLY; }else{ rc = sqlite3PagerBegin(pBt->pPager,wrflag>1,sqlite3TempInMemory(p->db)); + if( rc==SQLITE_OK ){ + rc = btreeSwapOutMmap(pBt); + } if( rc==SQLITE_OK ){ rc = newDatabase(pBt); } diff --git a/src/os_unix.c b/src/os_unix.c index 8f094bdc19..89326783d7 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3623,6 +3623,10 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } + case SQLITE_FCNTL_GETFD: { + *(int*)pArg = pFile->h; + return SQLITE_OK; + } #ifdef SQLITE_DEBUG /* The pager calls this method to signal that it has done ** a rollback and that the database is therefore unchanged and diff --git a/src/pager.c b/src/pager.c index 1d84fa2b7e..c362494093 100644 --- a/src/pager.c +++ b/src/pager.c @@ -655,6 +655,11 @@ struct Pager { PagerSavepoint *aSavepoint; /* Array of active savepoints */ int nSavepoint; /* Number of elements in aSavepoint[] */ char dbFileVers[16]; /* Changes whenever database file changes */ + + void *pMap; /* Memory mapped prefix of database file */ + i64 nMap; /* Size of mapping at pMap in bytes */ + int nMmapOut; /* Number of mmap pages currently outstanding */ + PgHdr *pFree; /* List of free mmap page headers (pDirty) */ /* ** End of the routinely-changing class members ***************************************************************************/ @@ -3803,6 +3808,105 @@ static int pagerSyncHotJournal(Pager *pPager){ return rc; } +#include + +/* +** Unmap any mapping of the database file. +*/ +static int pagerUnmap(Pager *pPager){ + if( pPager->pMap ){ + munmap(pPager->pMap, pPager->nMap); + pPager->pMap = 0; + pPager->nMap = 0; + } + return SQLITE_OK; +} + +static int pagerMap(Pager *pPager){ + int rc; + i64 sz = 0; + + assert( pPager->pMap==0 && pPager->nMap==0 ); + + rc = sqlite3OsFileSize(pPager->fd, &sz); + if( rc==SQLITE_OK && sz>0 ){ + int fd; + rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_GETFD, (void *)&fd); + if( rc==SQLITE_OK ){ + void *pMap = mmap(0, sz, PROT_READ, MAP_SHARED, fd, 0); + if( pMap==MAP_FAILED ){ + assert( 0 ); + return SQLITE_IOERR; + } + pPager->pMap = pMap; + pPager->nMap = sz; + } + } + + return rc; +} + +static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ + int rc; + *ppPage = 0; + + assert( pPager->pWal==0 ); + + if( MEMDB==0 && pPager->tempFile==0 ){ + if( pPager->pMap==0 ){ + rc = pagerMap(pPager); + if( rc!=SQLITE_OK ) return rc; + } + + if( pgno!=1 && pPager->pMap && pPager->nMap>=((i64)pgno*pPager->pageSize) ){ + PgHdr *p; + if( pPager->pFree ){ + p = pPager->pFree; + pPager->pFree = p->pDirty; + p->pDirty = 0; + memset(p->pExtra, 0, pPager->nExtra); + }else{ + p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); + if( p==0 ) return SQLITE_NOMEM; + p->pExtra = (void *)&p[1]; + p->flags = PGHDR_MMAP; + p->nRef = 1; + p->pPager = pPager; + } + + assert( p->pExtra==(void *)&p[1] ); + assert( p->pPage==0 ); + assert( p->flags==PGHDR_MMAP ); + assert( p->pPager==pPager ); + assert( p->nRef==1 ); + + p->pData = &((u8 *)pPager->pMap)[(i64)(pgno-1) * pPager->pageSize]; + p->pgno = pgno; + pPager->nMmapOut++; + *ppPage = p; + } + } + + return SQLITE_OK; +} + +static void pagerReleaseMapPage(PgHdr *pPg){ + Pager *pPager = pPg->pPager; + pPager->nMmapOut--; + pPg->pDirty = pPager->pFree; + pPager->pFree = pPg; +} + +static void pagerFreeMapHdrs(Pager *pPager){ + PgHdr *p; + PgHdr *pNext; + for(p=pPager->pFree; p; p=pNext){ + pNext = p->pDirty; + sqlite3_free(p); + } +} + + /* ** Shutdown the page cache. Free all memory and close all files. ** @@ -3823,6 +3927,8 @@ int sqlite3PagerClose(Pager *pPager){ assert( assert_pager_state(pPager) ); disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); + pagerUnmap(pPager); + pagerFreeMapHdrs(pPager); /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL @@ -4966,6 +5072,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( memcmp(pPager->dbFileVers, dbFileVers, sizeof(dbFileVers))!=0 ){ pager_reset(pPager); + pagerUnmap(pPager); } } @@ -5007,7 +5114,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** nothing to rollback, so this routine is a no-op. */ static void pagerUnlockIfUnused(Pager *pPager){ - if( (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ + if( (sqlite3PcacheRefCount(pPager->pPCache)==0) && pPager->nMmapOut==0 ){ pagerUnlockAndRollback(pPager); } } @@ -5083,6 +5190,15 @@ int sqlite3PagerAcquire( if( pPager->errCode!=SQLITE_OK ){ rc = pPager->errCode; }else{ + if( pPager->eState==PAGER_READER && pPager->pWal==0 ){ + rc = pagerAcquireMapPage(pPager, pgno, &pPg); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + if( pPg ){ + *ppPage = pPg; + return SQLITE_OK; + } + } + rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, ppPage); } @@ -5196,7 +5312,11 @@ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ void sqlite3PagerUnref(DbPage *pPg){ if( pPg ){ Pager *pPager = pPg->pPager; - sqlite3PcacheRelease(pPg); + if( pPg->flags & PGHDR_MMAP ){ + pagerReleaseMapPage(pPg); + }else{ + sqlite3PcacheRelease(pPg); + } pagerUnlockIfUnused(pPager); } } @@ -5313,6 +5433,8 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ assert( pPager->eState>=PAGER_READER && pPager->eStatesubjInMemory = (u8)subjInMemory; + pagerUnmap(pPager); + if( ALWAYS(pPager->eState==PAGER_READER) ){ assert( pPager->pInJournal==0 ); @@ -5535,6 +5657,9 @@ int sqlite3PagerWrite(DbPage *pDbPage){ assert( pPager->eState!=PAGER_ERROR ); assert( assert_pager_state(pPager) ); + /* There must not be any outstanding mmap pages at this point */ + assert( pPager->nMmapOut==0 ); + if( nPagePerSector>1 ){ Pgno nPageCount; /* Total number of pages in database file */ Pgno pg1; /* First page of the sector pPg is located on. */ diff --git a/src/pcache.h b/src/pcache.h index b9135fd859..f4d4ad71c1 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -53,6 +53,8 @@ struct PgHdr { #define PGHDR_REUSE_UNLIKELY 0x010 /* A hint that reuse is unlikely */ #define PGHDR_DONT_WRITE 0x020 /* Do not write content to disk */ +#define PGHDR_MMAP 0x040 /* This is an mmap page object */ + /* Initialize and shutdown the page cache subsystem */ int sqlite3PcacheInitialize(void); void sqlite3PcacheShutdown(void); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 0373353d7b..0ddb0dd105 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -900,6 +900,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_PRAGMA 14 #define SQLITE_FCNTL_BUSYHANDLER 15 #define SQLITE_FCNTL_TEMPFILENAME 16 +#define SQLITE_FCNTL_GETFD 17 /* ** CAPI3REF: Mutex Handle diff --git a/test/permutations.test b/test/permutations.test index 711d4e57d3..d9014d77fe 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -135,7 +135,8 @@ test_suite "veryquick" -prefix "" -description { This test suite is the same as the "quick" tests, except that some files that test malloc and IO errors are omitted. } -files [ - test_set $allquicktests -exclude *malloc* *ioerr* *fault* + test_set $allquicktests -exclude *malloc* *ioerr* *fault* \ + multiplex* server1.test shared2.test shared6.test ] test_suite "valgrind" -prefix "" -description { From 11dcd119136b3131f7520e1814232e333ce83d1e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 15 Mar 2013 18:29:18 +0000 Subject: [PATCH 002/140] Allow read-only cursors to use mmap pages even if there is an open write transaction. FossilOrigin-Name: b387e2f9d24dccac1fd040e309f6fc7ec1cfffba --- manifest | 19 ++++------ manifest.uuid | 2 +- src/btree.c | 100 +++++++++++++++++++++++++++----------------------- src/pager.c | 73 ++++++++++++++++++++++++++---------- src/pager.h | 6 +++ 5 files changed, 124 insertions(+), 76 deletions(-) diff --git a/manifest b/manifest index a61f43717c..97b08ff77e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\smmap()\sto\sread\sfrom\sthe\sdatabase\sfile\sin\srollback\smode.\sThis\sbranch\sis\sunix\sonly\sfor\snow. -D 2013-03-14T18:34:37.796 +C Allow\sread-only\scursors\sto\suse\smmap\spages\seven\sif\sthere\sis\san\sopen\swrite\stransaction. +D 2013-03-15T18:29:18.146 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c c1a956c6762f2a45188c945e1070daec29f5253f +F src/btree.c 934921ec91456c264f61ce671ca62cb826af977a F src/btree.h 3ad7964d6c5b1c7bff569aab6adfa075f8bf06cd F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 375e5df716e03b9343c5e1211be3b24e6d6dff05 @@ -162,8 +162,8 @@ F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 2a4cd96aabf413f39cf562baebb27aa9993f6f54 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 4e7e66c2959ee43caf8a7000712d6a4121b0888a -F src/pager.h 1109a06578ec5574dc2c74cf8d9f69daf36fe3e0 +F src/pager.c 734578514314a3d923fa1399c9722acbcc4f7e6b +F src/pager.h 81ac95f4fcfe21981f495146f6d7f2fe51afd110 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 @@ -1038,10 +1038,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 0b452734faa0839c817f040322e7733e423bfce2 -R d4c281f6a900c9b41816881ffb3c2331 -T *branch * experimental-mmap -T *sym-experimental-mmap * -T -sym-trunk * +P 6f21d9cbf5d457e63a7282015a89ae785526cf6d +R 94aae53403ccd6ebc7d159fa0d23eb33 U dan -Z f16a594407ec9e95c26319dfbc751443 +Z 133f9be54c97c4ee392fb0706f346b90 diff --git a/manifest.uuid b/manifest.uuid index dbbf6a71bc..a20f3a4f02 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6f21d9cbf5d457e63a7282015a89ae785526cf6d \ No newline at end of file +b387e2f9d24dccac1fd040e309f6fc7ec1cfffba \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 038126c3f0..5104625c35 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1569,13 +1569,17 @@ static int btreeGetPage( BtShared *pBt, /* The btree */ Pgno pgno, /* Number of the page to fetch */ MemPage **ppPage, /* Return the page in this parameter */ - int noContent /* Do not load page content if true */ + int noContent, /* Do not load page content if true */ + int bReadonly /* True if a read-only (mmap) page is ok */ ){ int rc; DbPage *pDbPage; + int flags = (noContent ? PAGER_ACQUIRE_NOCONTENT : 0) + | (bReadonly ? PAGER_ACQUIRE_READONLY : 0); + assert( noContent==0 || bReadonly==0 ); assert( sqlite3_mutex_held(pBt->mutex) ); - rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, noContent); + rc = sqlite3PagerAcquire(pBt->pPager, pgno, (DbPage**)&pDbPage, flags); if( rc ) return rc; *ppPage = btreePageFromDbPage(pDbPage, pgno, pBt); return SQLITE_OK; @@ -1618,9 +1622,10 @@ u32 sqlite3BtreeLastPage(Btree *p){ ** may remain unchanged, or it may be set to an invalid value. */ static int getAndInitPage( - BtShared *pBt, /* The database file */ - Pgno pgno, /* Number of the page to get */ - MemPage **ppPage /* Write the page pointer here */ + BtShared *pBt, /* The database file */ + Pgno pgno, /* Number of the page to get */ + MemPage **ppPage, /* Write the page pointer here */ + int bReadonly /* True if a read-only (mmap) page is ok */ ){ int rc; assert( sqlite3_mutex_held(pBt->mutex) ); @@ -1628,7 +1633,7 @@ static int getAndInitPage( if( pgno>btreePagecount(pBt) ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = btreeGetPage(pBt, pgno, ppPage, 0); + rc = btreeGetPage(pBt, pgno, ppPage, 0, bReadonly); if( rc==SQLITE_OK ){ rc = btreeInitPage(*ppPage); if( rc!=SQLITE_OK ){ @@ -2350,7 +2355,7 @@ static int lockBtree(BtShared *pBt){ assert( pBt->pPage1==0 ); rc = sqlite3PagerSharedLock(pBt->pPager); if( rc!=SQLITE_OK ) return rc; - rc = btreeGetPage(pBt, 1, &pPage1, 0); + rc = btreeGetPage(pBt, 1, &pPage1, 0, 0); if( rc!=SQLITE_OK ) return rc; /* Do some checking to help insure the file we opened really is @@ -2565,31 +2570,30 @@ int sqlite3BtreeNewDb(Btree *p){ } /* -** If the shared-btree passed as the only argument is holding references -** to mmap pages, replace them with read/write pages. Return SQLITE_OK -** if successful, or an error code otherwise. +** Ensure that any root page references held by open cursors are not +** mmap pages. */ static int btreeSwapOutMmap(BtShared *pBt){ - BtCursor *pCsr; - for(pCsr=pBt->pCursor; pCsr; pCsr=pCsr->pNext){ - int i; - for(i=0; i<=pCsr->iPage; i++){ - MemPage *pPg = pCsr->apPage[i]; + int rc = SQLITE_OK; /* Return code */ + BtCursor *pCsr; /* Used to iterate through all open cursors */ + + for(pCsr=pBt->pCursor; pCsr && rc==SQLITE_OK; pCsr=pCsr->pNext){ + if( pCsr->iPage>=0 ){ + MemPage *pPg = pCsr->apPage[0]; if( pPg->pDbPage->flags & PGHDR_MMAP ){ - int rc; MemPage *pNew = 0; - rc = btreeGetPage(pBt, pPg->pgno, &pNew, 0); - if( rc==SQLITE_OK && i==pCsr->iPage ){ + rc = btreeGetPage(pBt, pPg->pgno, &pNew, 0, 0); + if( rc==SQLITE_OK && pCsr->iPage==0 ){ pCsr->info.pCell = pNew->aData + (pCsr->info.pCell - pPg->aData); } - pCsr->apPage[i] = pNew; + pCsr->apPage[0] = pNew; releasePage(pPg); if( rc!=SQLITE_OK ) return rc; } } } - return SQLITE_OK; + return rc; } /* @@ -2940,7 +2944,7 @@ static int relocatePage( ** iPtrPage. */ if( eType!=PTRMAP_ROOTPAGE ){ - rc = btreeGetPage(pBt, iPtrPage, &pPtrPage, 0); + rc = btreeGetPage(pBt, iPtrPage, &pPtrPage, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -3024,7 +3028,7 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ u8 eMode = BTALLOC_ANY; /* Mode parameter for allocateBtreePage() */ Pgno iNear = 0; /* nearby parameter for allocateBtreePage() */ - rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0); + rc = btreeGetPage(pBt, iLastPg, &pLastPg, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -3116,8 +3120,11 @@ int sqlite3BtreeIncrVacuum(Btree *p){ if( nOrig0 ){ - invalidateAllOverflowCache(pBt); - rc = incrVacuumStep(pBt, nFin, nOrig, 0); + rc = saveAllCursors(pBt, 0, 0); + if( rc==SQLITE_OK ){ + invalidateAllOverflowCache(pBt); + rc = incrVacuumStep(pBt, nFin, nOrig, 0); + } if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); put4byte(&pBt->pPage1->aData[28], pBt->nPage); @@ -3438,7 +3445,7 @@ int sqlite3BtreeRollback(Btree *p, int tripCode){ /* The rollback may have destroyed the pPage1->aData value. So ** call btreeGetPage() on page 1 again to make ** sure pPage1->aData is set correctly. */ - if( btreeGetPage(pBt, 1, &pPage1, 0)==SQLITE_OK ){ + if( btreeGetPage(pBt, 1, &pPage1, 0, 0)==SQLITE_OK ){ int nPage = get4byte(28+(u8*)pPage1->aData); testcase( nPage==0 ); if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); @@ -3872,7 +3879,7 @@ static int getOverflowPage( assert( next==0 || rc==SQLITE_DONE ); if( rc==SQLITE_OK ){ - rc = btreeGetPage(pBt, ovfl, &pPage, 0); + rc = btreeGetPage(pBt, ovfl, &pPage, 0, (ppPage==0)); assert( rc==SQLITE_OK || pPage==0 ); if( rc==SQLITE_OK ){ next = get4byte(pPage->aData); @@ -4093,7 +4100,9 @@ static int accessPayload( { DbPage *pDbPage; - rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage); + rc = sqlite3PagerAcquire(pBt->pPager, nextPage, &pDbPage, + (eOp==0 ? PAGER_ACQUIRE_READONLY : 0) + ); if( rc==SQLITE_OK ){ aPayload = sqlite3PagerGetData(pDbPage); nextPage = get4byte(aPayload); @@ -4272,10 +4281,11 @@ static int moveToChild(BtCursor *pCur, u32 newPgno){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPageiPage>=0 ); if( pCur->iPage>=(BTCURSOR_MAX_DEPTH-1) ){ return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, newPgno, &pNewPage); + rc = getAndInitPage(pBt, newPgno, &pNewPage, (pCur->wrFlag==0)); if( rc ) return rc; pCur->apPage[i+1] = pNewPage; pCur->aiIdx[i+1] = 0; @@ -4392,7 +4402,7 @@ static int moveToRoot(BtCursor *pCur){ pCur->eState = CURSOR_INVALID; return SQLITE_OK; }else{ - rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0]); + rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0], 0); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; @@ -5006,7 +5016,7 @@ static int allocateBtreePage( if( iTrunk>mxPage ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0, 0); } if( rc ){ pTrunk = 0; @@ -5070,7 +5080,7 @@ static int allocateBtreePage( goto end_allocate_page; } testcase( iNewTrunk==mxPage ); - rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0); + rc = btreeGetPage(pBt, iNewTrunk, &pNewTrunk, 0, 0); if( rc!=SQLITE_OK ){ goto end_allocate_page; } @@ -5150,7 +5160,7 @@ static int allocateBtreePage( } put4byte(&aData[4], k-1); noContent = !btreeGetHasContent(pBt, *pPgno); - rc = btreeGetPage(pBt, *pPgno, ppPage, noContent); + rc = btreeGetPage(pBt, *pPgno, ppPage, noContent, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ @@ -5198,7 +5208,7 @@ static int allocateBtreePage( MemPage *pPg = 0; TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, pBt->nPage, &pPg, bNoContent); + rc = btreeGetPage(pBt, pBt->nPage, &pPg, bNoContent, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pPg->pDbPage); releasePage(pPg); @@ -5212,7 +5222,7 @@ static int allocateBtreePage( *pPgno = pBt->nPage; assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); - rc = btreeGetPage(pBt, *pPgno, ppPage, bNoContent); + rc = btreeGetPage(pBt, *pPgno, ppPage, bNoContent, 0); if( rc ) return rc; rc = sqlite3PagerWrite((*ppPage)->pDbPage); if( rc!=SQLITE_OK ){ @@ -5280,7 +5290,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ /* If the secure_delete option is enabled, then ** always fully overwrite deleted information with zeros. */ - if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0) ) + if( (!pPage && ((rc = btreeGetPage(pBt, iPage, &pPage, 0, 0))!=0) ) || ((rc = sqlite3PagerWrite(pPage->pDbPage))!=0) ){ goto freepage_out; @@ -5307,7 +5317,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ u32 nLeaf; /* Initial number of leaf cells on trunk page */ iTrunk = get4byte(&pPage1->aData[32]); - rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); + rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0, 0); if( rc!=SQLITE_OK ){ goto freepage_out; } @@ -5353,7 +5363,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ ** first trunk in the free-list is full. Either way, the page being freed ** will become the new first trunk page in the free-list. */ - if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0)) ){ + if( pPage==0 && SQLITE_OK!=(rc = btreeGetPage(pBt, iPage, &pPage, 0, 0)) ){ goto freepage_out; } rc = sqlite3PagerWrite(pPage->pDbPage); @@ -6154,7 +6164,7 @@ static int balance_nonroot( } pgno = get4byte(pRight); while( 1 ){ - rc = getAndInitPage(pBt, pgno, &apOld[i]); + rc = getAndInitPage(pBt, pgno, &apOld[i], 0); if( rc ){ memset(apOld, 0, (i+1)*sizeof(MemPage*)); goto balance_cleanup; @@ -7245,7 +7255,7 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ releasePage(pPageMove); /* Move the page currently at pgnoRoot to pgnoMove. */ - rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -7266,7 +7276,7 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ if( rc!=SQLITE_OK ){ return rc; } - rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0); + rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -7342,7 +7352,7 @@ static int clearDatabasePage( return SQLITE_CORRUPT_BKPT; } - rc = getAndInitPage(pBt, pgno, &pPage); + rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; for(i=0; inCell; i++){ pCell = findCell(pPage, i); @@ -7444,7 +7454,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return SQLITE_LOCKED_SHAREDCACHE; } - rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); + rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0, 0); if( rc ) return rc; rc = sqlite3BtreeClearTable(p, iTable, 0); if( rc ){ @@ -7479,7 +7489,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ */ MemPage *pMove; releasePage(pPage); - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0, 0); if( rc!=SQLITE_OK ){ return rc; } @@ -7489,7 +7499,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ return rc; } pMove = 0; - rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0); + rc = btreeGetPage(pBt, maxRootPgno, &pMove, 0, 0); freePage(pMove, &rc); releasePage(pMove); if( rc!=SQLITE_OK ){ @@ -7901,7 +7911,7 @@ static int checkTreePage( usableSize = pBt->usableSize; if( iPage==0 ) return 0; if( checkRef(pCheck, iPage, zParentContext) ) return 0; - if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){ + if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0, 0))!=0 ){ checkAppendMsg(pCheck, zContext, "unable to get the page. error code=%d", rc); return 0; diff --git a/src/pager.c b/src/pager.c index c362494093..8470e7e35e 100644 --- a/src/pager.c +++ b/src/pager.c @@ -658,6 +658,7 @@ struct Pager { void *pMap; /* Memory mapped prefix of database file */ i64 nMap; /* Size of mapping at pMap in bytes */ + i64 nMapValid; /* Bytes at pMap known to be valid */ int nMmapOut; /* Number of mmap pages currently outstanding */ PgHdr *pFree; /* List of free mmap page headers (pDirty) */ /* @@ -2512,6 +2513,9 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ if( rc==SQLITE_OK && currentSize!=newSize ){ if( currentSize>newSize ){ rc = sqlite3OsTruncate(pPager->fd, newSize); + if( newSizenMapValid ){ + pPager->nMapValid = newSize; + } }else if( (currentSize+szPage)<=newSize ){ char *pTmp = pPager->pTmpSpace; memset(pTmp, 0, szPage); @@ -3818,6 +3822,7 @@ static int pagerUnmap(Pager *pPager){ munmap(pPager->pMap, pPager->nMap); pPager->pMap = 0; pPager->nMap = 0; + pPager->nMapValid = 0; } return SQLITE_OK; } @@ -3839,7 +3844,7 @@ static int pagerMap(Pager *pPager){ return SQLITE_IOERR; } pPager->pMap = pMap; - pPager->nMap = sz; + pPager->nMapValid = pPager->nMap = sz; } } @@ -3858,7 +3863,9 @@ static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ if( rc!=SQLITE_OK ) return rc; } - if( pgno!=1 && pPager->pMap && pPager->nMap>=((i64)pgno*pPager->pageSize) ){ + if( pgno!=1 && pPager->pMap + && pPager->nMapValid>=((i64)pgno*pPager->pageSize) + ){ PgHdr *p; if( pPager->pFree ){ p = pPager->pFree; @@ -5035,9 +5042,11 @@ int sqlite3PagerSharedLock(Pager *pPager){ ); } - if( !pPager->tempFile - && (pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0) - ){ + if( !pPager->tempFile && ( + pPager->pBackup + || sqlite3PcachePagecount(pPager->pPCache)>0 + || pPager->pMap + )){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous ** read or write transaction). Check to see if the database @@ -5072,6 +5081,15 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( memcmp(pPager->dbFileVers, dbFileVers, sizeof(dbFileVers))!=0 ){ pager_reset(pPager); + + /* Unmap the database file. It is possible that external processes + ** may have truncated the database file and then extended it back + ** to its original size while this process was not holding a lock. + ** In this case there may exist a Pager.pMap mapping that appears + ** to be the right size but is not actually valid. Avoid this + ** possibility by unmapping the db here. */ + pagerUnmap(pPager); + }else if( ((i64)nPage*pPager->pageSize)!=pPager->nMap ){ pagerUnmap(pPager); } } @@ -5173,10 +5191,20 @@ int sqlite3PagerAcquire( Pager *pPager, /* The pager open on the database file */ Pgno pgno, /* Page number to fetch */ DbPage **ppPage, /* Write a pointer to the page here */ - int noContent /* Do not bother reading content from disk if true */ + int flags /* PAGER_ACQUIRE_XXX flags */ ){ - int rc; - PgHdr *pPg; + int rc = SQLITE_OK; + PgHdr *pPg = 0; + const int noContent = (flags & PAGER_ACQUIRE_NOCONTENT); + + /* It is acceptable to use a read-only (mmap) page for any page except + ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY + ** flag was specified by the caller. And so long as the db is not a + ** temporary or in-memory database. */ + const int bMmapOk = ( + (pgno!=1 && pPager->pWal==0 && !pPager->tempFile && !MEMDB) + && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY)) + ); assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); @@ -5190,12 +5218,23 @@ int sqlite3PagerAcquire( if( pPager->errCode!=SQLITE_OK ){ rc = pPager->errCode; }else{ - if( pPager->eState==PAGER_READER && pPager->pWal==0 ){ - rc = pagerAcquireMapPage(pPager, pgno, &pPg); - if( rc!=SQLITE_OK ) goto pager_acquire_err; - if( pPg ){ - *ppPage = pPg; - return SQLITE_OK; + + if( bMmapOk ){ + if( pPager->pMap==0 ) rc = pagerMap(pPager); + if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){ + if( pPager->eState>PAGER_READER ){ + (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); + } + if( pPg==0 ){ + rc = pagerAcquireMapPage(pPager, pgno, &pPg); + } + if( pPg ){ + assert( rc==SQLITE_OK ); + *ppPage = pPg; + return SQLITE_OK; + }else if( rc!=SQLITE_OK ){ + goto pager_acquire_err; + } } } @@ -5433,8 +5472,6 @@ int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ assert( pPager->eState>=PAGER_READER && pPager->eStatesubjInMemory = (u8)subjInMemory; - pagerUnmap(pPager); - if( ALWAYS(pPager->eState==PAGER_READER) ){ assert( pPager->pInJournal==0 ); @@ -5653,13 +5690,11 @@ int sqlite3PagerWrite(DbPage *pDbPage){ Pager *pPager = pPg->pPager; Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); + assert( (pPg->flags & PGHDR_MMAP)==0 ); assert( pPager->eState>=PAGER_WRITER_LOCKED ); assert( pPager->eState!=PAGER_ERROR ); assert( assert_pager_state(pPager) ); - /* There must not be any outstanding mmap pages at this point */ - assert( pPager->nMmapOut==0 ); - if( nPagePerSector>1 ){ Pgno nPageCount; /* Total number of pages in database file */ Pgno pg1; /* First page of the sector pPg is located on. */ diff --git a/src/pager.h b/src/pager.h index 90f8e6af78..94957dac60 100644 --- a/src/pager.h +++ b/src/pager.h @@ -78,6 +78,12 @@ typedef struct PgHdr DbPage; #define PAGER_JOURNALMODE_MEMORY 4 /* In-memory journal file */ #define PAGER_JOURNALMODE_WAL 5 /* Use write-ahead logging */ +/* +** Flags that make up the mask passed to sqlite3PagerAcquire(). +*/ +#define PAGER_ACQUIRE_NOCONTENT 0x01 /* Do not load data from disk */ +#define PAGER_ACQUIRE_READONLY 0x02 /* Read-only page is acceptable */ + /* ** The remainder of this file contains the declarations of the functions ** that make up the Pager sub-system API. See source code comments for From 7af6547a5afde7c5b48058d948df35857ba22a40 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 15 Mar 2013 19:13:42 +0000 Subject: [PATCH 003/140] Fix a dropped error code in pager.c. FossilOrigin-Name: 022fdc986b33701abfd39621072ac3d9f9f7d43e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 9 ++++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 97b08ff77e..ee60425958 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sread-only\scursors\sto\suse\smmap\spages\seven\sif\sthere\sis\san\sopen\swrite\stransaction. -D 2013-03-15T18:29:18.146 +C Fix\sa\sdropped\serror\scode\sin\spager.c. +D 2013-03-15T19:13:42.183 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 2a4cd96aabf413f39cf562baebb27aa9993f6f54 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 734578514314a3d923fa1399c9722acbcc4f7e6b +F src/pager.c bdbcfe676cda9295572e4ce95a3fae827082f9f0 F src/pager.h 81ac95f4fcfe21981f495146f6d7f2fe51afd110 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1038,7 +1038,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 6f21d9cbf5d457e63a7282015a89ae785526cf6d -R 94aae53403ccd6ebc7d159fa0d23eb33 +P b387e2f9d24dccac1fd040e309f6fc7ec1cfffba +R 53728d59c72e59e92889436c2f9b085b U dan -Z 133f9be54c97c4ee392fb0706f346b90 +Z 3a434ac42175412fea267939fd915765 diff --git a/manifest.uuid b/manifest.uuid index a20f3a4f02..9f9d488fb7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b387e2f9d24dccac1fd040e309f6fc7ec1cfffba \ No newline at end of file +022fdc986b33701abfd39621072ac3d9f9f7d43e \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 8470e7e35e..23e7a67411 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5220,7 +5220,9 @@ int sqlite3PagerAcquire( }else{ if( bMmapOk ){ - if( pPager->pMap==0 ) rc = pagerMap(pPager); + if( pPager->pMap==0 ){ + rc = pagerMap(pPager); + } if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){ if( pPager->eState>PAGER_READER ){ (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); @@ -5232,10 +5234,11 @@ int sqlite3PagerAcquire( assert( rc==SQLITE_OK ); *ppPage = pPg; return SQLITE_OK; - }else if( rc!=SQLITE_OK ){ - goto pager_acquire_err; } } + if( rc!=SQLITE_OK ){ + goto pager_acquire_err; + } } rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, ppPage); From a72014faf50889400af55631d4556a0adcffb593 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 16 Mar 2013 20:19:21 +0000 Subject: [PATCH 004/140] When possible, use memcpy() to and from the mapped region instead of xWrite() and xRead(). FossilOrigin-Name: f8ca5622d99bedca957caa9ad311d798f63b3ce9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 41 +++++++++++++++++++++++++++++++++-------- src/test1.c | 24 ++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index ee60425958..a6549b18c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sdropped\serror\scode\sin\spager.c. -D 2013-03-15T19:13:42.183 +C When\spossible,\suse\smemcpy()\sto\sand\sfrom\sthe\smapped\sregion\sinstead\sof\sxWrite()\sand\sxRead(). +D 2013-03-16T20:19:21.766 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 2a4cd96aabf413f39cf562baebb27aa9993f6f54 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c bdbcfe676cda9295572e4ce95a3fae827082f9f0 +F src/pager.c 495c5344392d5932ea5072f20bfbd8a58cf19d67 F src/pager.h 81ac95f4fcfe21981f495146f6d7f2fe51afd110 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -184,7 +184,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 3213f3101e3b85f047d6e389da5a53d76d3d7540 -F src/test1.c ff3e68eedfbd858c9b89cf03e3db233cd29be1d0 +F src/test1.c 3dac8d76be8852d65ff8b9ce4b50ed08b999ed59 F src/test2.c 4178056dd1e7d70f954ad8a1e3edb71a2a784daf F src/test3.c 3c3c2407fa6ec7a19e24ae23f7cb439d0275a60d F src/test4.c bf9fa9bece01de08e6f5e02314e4af5c13590dfa @@ -1038,7 +1038,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b387e2f9d24dccac1fd040e309f6fc7ec1cfffba -R 53728d59c72e59e92889436c2f9b085b +P 022fdc986b33701abfd39621072ac3d9f9f7d43e +R e9d1b00daf0ae79f0e154b8607fa8ad8 U dan -Z 3a434ac42175412fea267939fd915765 +Z 82853c015b40f8bb1665068f9eea77d4 diff --git a/manifest.uuid b/manifest.uuid index 9f9d488fb7..500dcc6707 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -022fdc986b33701abfd39621072ac3d9f9f7d43e \ No newline at end of file +f8ca5622d99bedca957caa9ad311d798f63b3ce9 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 23e7a67411..d9b9ed5052 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2865,9 +2865,13 @@ static int readDbPage(PgHdr *pPg){ } if( rc==SQLITE_OK && !isInWal ){ i64 iOffset = (pgno-1)*(i64)pPager->pageSize; - rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); - if( rc==SQLITE_IOERR_SHORT_READ ){ - rc = SQLITE_OK; + if( pPager->pMap && pPager->nMapValid>=iOffset+pPager->pageSize ){ + memcpy(pPg->pData, &((u8 *)(pPager->pMap))[iOffset], pPager->pageSize); + }else{ + rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; + } } } @@ -3834,13 +3838,14 @@ static int pagerMap(Pager *pPager){ assert( pPager->pMap==0 && pPager->nMap==0 ); rc = sqlite3OsFileSize(pPager->fd, &sz); + sz = sz & ~(4096-1); + if( rc==SQLITE_OK && sz>0 ){ int fd; rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_GETFD, (void *)&fd); if( rc==SQLITE_OK ){ - void *pMap = mmap(0, sz, PROT_READ, MAP_SHARED, fd, 0); + void *pMap = mmap(0, sz, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); if( pMap==MAP_FAILED ){ - assert( 0 ); return SQLITE_IOERR; } pPager->pMap = pMap; @@ -3851,6 +3856,22 @@ static int pagerMap(Pager *pPager){ return rc; } +static int pagerRemap(Pager *pPager, Pgno nPage){ + i64 sz = (i64)nPage * pPager->pageSize; + sz = sz & ~(4096-1); + + if( pPager->nMap!=sz ){ + void *pMap = mremap(pPager->pMap, pPager->nMap, sz, MREMAP_MAYMOVE); + if( pMap==MAP_FAILED ){ + return SQLITE_IOERR; + } + pPager->pMap = pMap; + pPager->nMapValid = pPager->nMap = sz; + } + + return SQLITE_OK; +} + static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ int rc; *ppPage = 0; @@ -4225,7 +4246,11 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); /* Write out the page data. */ - rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); + if( pPager->nMapValid>=(offset+pPager->pageSize) ){ + memcpy(&((u8 *)(pPager->pMap))[offset], pData, pPager->pageSize); + }else{ + rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); + } /* If page 1 was just written, update Pager.dbFileVers to match ** the value now stored in the database file. If writing this @@ -5089,8 +5114,8 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** to be the right size but is not actually valid. Avoid this ** possibility by unmapping the db here. */ pagerUnmap(pPager); - }else if( ((i64)nPage*pPager->pageSize)!=pPager->nMap ){ - pagerUnmap(pPager); + }else if( pPager->pMap ){ + pagerRemap(pPager, nPage); } } diff --git a/src/test1.c b/src/test1.c index 0bace84420..a89b874a3b 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5844,6 +5844,29 @@ static int test_test_control( return TCL_OK; } +#include +#include + +static int test_getrusage( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + char buf[1024]; + struct rusage r; + memset(&r, 0, sizeof(r)); + getrusage(RUSAGE_SELF, &r); + + sprintf(buf, "ru_utime=%d.%06d ru_stime=%d.%06d ru_minflt=%d ru_majflt=%d", + r.ru_utime.tv_sec, r.ru_utime.tv_usec, + r.ru_stime.tv_sec, r.ru_stime.tv_usec, + r.ru_minflt, r.ru_majflt + ); + Tcl_SetObjResult(interp, Tcl_NewStringObj(buf, -1)); + return TCL_OK; +} + #if SQLITE_OS_WIN /* ** Information passed from the main thread into the windows file locker @@ -6233,6 +6256,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "print_explain_query_plan", test_print_eqp, 0 }, #endif { "sqlite3_test_control", test_test_control }, + { "getrusage", test_getrusage }, }; static int bitmask_size = sizeof(Bitmask)*8; int i; From 5d8a137218d43070a2c635900d1ae7d7a7ef0f69 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 19 Mar 2013 19:28:06 +0000 Subject: [PATCH 005/140] Add the sqlite3_io_methods.xMremap() method to the VFS interface. Also "PRAGMA mmap_size". FossilOrigin-Name: 6183f1bd86ceed76d22d9762f3d7eb33262c62d1 --- manifest | 42 +++++----- manifest.uuid | 2 +- src/btree.c | 13 +++ src/btree.h | 1 + src/os.c | 3 + src/os.h | 1 + src/os_unix.c | 41 ++++++++- src/pager.c | 183 ++++++++++++++++++++++++----------------- src/pragma.c | 24 ++++++ src/sqlite.h.in | 21 +++++ src/sqliteInt.h | 1 + src/test1.c | 6 +- test/dbstatus2.test | 1 + test/exclusive2.test | 8 ++ test/func.test | 12 +-- test/incrblob.test | 4 + test/pageropt.test | 2 +- test/permutations.test | 8 ++ 18 files changed, 263 insertions(+), 110 deletions(-) diff --git a/manifest b/manifest index a6549b18c2..45c3ee6fc5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\spossible,\suse\smemcpy()\sto\sand\sfrom\sthe\smapped\sregion\sinstead\sof\sxWrite()\sand\sxRead(). -D 2013-03-16T20:19:21.766 +C Add\sthe\ssqlite3_io_methods.xMremap()\smethod\sto\sthe\sVFS\sinterface.\sAlso\s"PRAGMA\smmap_size". +D 2013-03-19T19:28:06.473 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,8 +121,8 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 934921ec91456c264f61ce671ca62cb826af977a -F src/btree.h 3ad7964d6c5b1c7bff569aab6adfa075f8bf06cd +F src/btree.c d4d551f05a555926a7c0f49c2e263f7ee2b1c59f +F src/btree.h d3259057a38494c4c4358e377032158c762e3d8b F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 375e5df716e03b9343c5e1211be3b24e6d6dff05 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc @@ -157,18 +157,18 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c -F src/os.h 027491c77d2404c0a678bb3fb06286f331eb9b57 +F src/os.c 81a82a736b8a461a656f9b3e401a39768fe73a79 +F src/os.h 4681261aa24a9d2187aaf4cb963880e6cddb1f48 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 2a4cd96aabf413f39cf562baebb27aa9993f6f54 +F src/os_unix.c 0c7b0d076f2ac6279b0b280a26bcae8c89f36f4f F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 495c5344392d5932ea5072f20bfbd8a58cf19d67 +F src/pager.c 78b65bf9685bf21b787ce2a7389c2b96102942dc F src/pager.h 81ac95f4fcfe21981f495146f6d7f2fe51afd110 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c 9f0ee3d74a7f33eeeff40a4b014fc3abf8182ce2 +F src/pragma.c 86c8088ac6a12d3f3be5f7394542651f03fa9a38 F src/prepare.c 78cd7ae8cd0d8de8ef8a8b5352fc5f38693a0852 F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -176,15 +176,15 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in 9a5c737a1feb4495d351c56883587d4fda52e81e +F src/sqlite.h.in fd75f5bcf479b315b1c717fa1d07f018bd919f79 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 -F src/sqliteInt.h 0f8f05ee4db4ba9120b38f7a3992b325698f6e8a +F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 3213f3101e3b85f047d6e389da5a53d76d3d7540 -F src/test1.c 3dac8d76be8852d65ff8b9ce4b50ed08b999ed59 +F src/test1.c 39378e3e14a8162e29dc90d1e05399d12e8a569e F src/test2.c 4178056dd1e7d70f954ad8a1e3edb71a2a784daf F src/test3.c 3c3c2407fa6ec7a19e24ae23f7cb439d0275a60d F src/test4.c bf9fa9bece01de08e6f5e02314e4af5c13590dfa @@ -369,7 +369,7 @@ F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/date.test f3228180c87bbe5d39c9397bf001c0095c3821b9 F test/dbstatus.test 207e5b63fcb7b9c3bb8e1fdf38ebd4654ad0e54b -F test/dbstatus2.test bf7396af964b89e39435babbcdf296ae8fc5f10a +F test/dbstatus2.test 10418e62b3db5dca070f0c3eef3ea13946f339c2 F test/default.test 6faf23ccb300114924353007795aa9a8ec0aa9dc F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa @@ -403,7 +403,7 @@ F test/eqp.test 46aa946dd55c90635327898275d3e533d23a9845 F test/errmsg.test 050717f1c6a5685de9c79f5f9f6b83d7c592f73a F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test a1b324cb21834a490cd052d409d34789cfef57cb -F test/exclusive2.test 372be98f6de44dd78734e364b7b626ea211761a6 +F test/exclusive2.test 881193eccec225cfed9d7f744b65e57f26adee25 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30 F test/expr.test 67c9fd6f8f829e239dc8b0f4a08a73c08b09196d @@ -508,7 +508,7 @@ F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7 F test/fts4unicode.test 25ccad45896f8e50f6a694cff738a35f798cdb40 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d -F test/func.test b058483c17952eff7797b837bbb61e27e6b05606 +F test/func.test b0fc34fdc36897769651975a2b0a606312753643 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a F test/fuzz-oss1.test 4912e528ec9cf2f42134456933659d371c9e0d74 @@ -526,7 +526,7 @@ F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test 64f3cc1acde1b9161ccdd8e5bde3daefdb5b2617 F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 -F test/incrblob.test 34765fa6fb5d8e0f256fc7d6497c04b205398849 +F test/incrblob.test bf210bea512474d4e1d94fbb9b0fcb386cd65dea F test/incrblob2.test edc3a96e557bd61fb39acc8d2edd43371fbbaa19 F test/incrblob3.test aedbb35ea1b6450c33b98f2b6ed98e5020be8dc7 F test/incrblob4.test 09be37d3dd996a31ea6993bba7837ece549414a8 @@ -652,11 +652,11 @@ F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f F test/pagerfault.test 452f2cc23e3bfcfa935f4442aec1da4fe1dc0442 F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 -F test/pageropt.test 9191867ed19a2b3db6c42d1b36b6fbc657cd1ab0 +F test/pageropt.test 290cd59782b1890f02bb33795571facfc5ccac43 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test eb49937dca270b2c3f62d4c91fc7034ca905b7f1 +F test/permutations.test 694f4a2667242bab49cce05c54c2adfcc2727d9e F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -1038,7 +1038,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 022fdc986b33701abfd39621072ac3d9f9f7d43e -R e9d1b00daf0ae79f0e154b8607fa8ad8 +P f8ca5622d99bedca957caa9ad311d798f63b3ce9 +R 1add721bf7e9669d3b4d4c28d41bb13b U dan -Z 82853c015b40f8bb1665068f9eea77d4 +Z 3a3766130d03f0bab6c4936a9c26c7a5 diff --git a/manifest.uuid b/manifest.uuid index 500dcc6707..8d198c909c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8ca5622d99bedca957caa9ad311d798f63b3ce9 \ No newline at end of file +6183f1bd86ceed76d22d9762f3d7eb33262c62d1 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 5104625c35..2b7872fa8c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2130,6 +2130,19 @@ int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ return SQLITE_OK; } +/* +** Change the limit on the amount of the database file that may be +** memory mapped. +*/ +int sqlite3BtreeSetMmapSize(Btree *p, int nMap){ + BtShared *pBt = p->pBt; + assert( sqlite3_mutex_held(p->db->mutex) ); + sqlite3BtreeEnter(p); + sqlite3PagerSetMmapsize(pBt->pPager, nMap); + sqlite3BtreeLeave(p); + return SQLITE_OK; +} + /* ** Change the way data is synced to disk in order to increase or decrease ** how well the database resists damage due to OS crashes and power diff --git a/src/btree.h b/src/btree.h index d4c9fe37d7..f2cca2fc71 100644 --- a/src/btree.h +++ b/src/btree.h @@ -63,6 +63,7 @@ int sqlite3BtreeOpen( int sqlite3BtreeClose(Btree*); int sqlite3BtreeSetCacheSize(Btree*,int); +int sqlite3BtreeSetMmapSize(Btree*, int); int sqlite3BtreeSetSafetyLevel(Btree*,int,int,int); int sqlite3BtreeSyncDisabled(Btree*); int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); diff --git a/src/os.c b/src/os.c index b5e918a727..45d8e0c121 100644 --- a/src/os.c +++ b/src/os.c @@ -140,6 +140,9 @@ int sqlite3OsShmMap( DO_OS_MALLOC_TEST(id); return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } +int sqlite3OsMremap(sqlite3_file *id, i64 iOff, i64 nOld, i64 nNew, void **pp){ + return id->pMethods->xMremap(id, iOff, nOld, nNew, pp); +} /* ** The next group of routines are convenience wrappers around the diff --git a/src/os.h b/src/os.h index 1ec7d4ba11..76a60effdb 100644 --- a/src/os.h +++ b/src/os.h @@ -259,6 +259,7 @@ int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); +int sqlite3OsMremap(sqlite3_file *id, i64, i64, i64, void **); /* diff --git a/src/os_unix.c b/src/os_unix.c index 89326783d7..f37f2404ee 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4429,6 +4429,42 @@ static int unixShmUnmap( # define unixShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ +/* +** Map, remap or unmap part of the database file. +*/ +static int unixMremap( + sqlite3_file *fd, /* Main database file */ + sqlite3_int64 iOff, /* Offset to start mapping at */ + sqlite3_int64 nOld, /* Size of old mapping, or zero */ + sqlite3_int64 nNew, /* Size of new mapping, or zero */ + void **ppMap /* IN/OUT: Old/new mappings */ +){ + unixFile *p = (unixFile *)fd; /* The underlying database file */ + int rc = SQLITE_OK; /* Return code */ + void *pNew = 0; /* New mapping */ + + assert( iOff==0 ); + + if( nOld!=0 ){ + void *pOld = *ppMap; + munmap(pOld, nOld); + } + + if( nNew>0 ){ + int flags = PROT_READ; + if( (p->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; + nNew = (nNew+4095) & ~(i64)((1 << 12)-1); + pNew = mmap(0, nNew, flags, MAP_SHARED, p->h, iOff); + if( pNew==MAP_FAILED ){ + pNew = 0; + rc = SQLITE_IOERR; + } + } + + *ppMap = pNew; + return rc; +} + /* ** Here ends the implementation of all sqlite3_file methods. ** @@ -4487,7 +4523,8 @@ static const sqlite3_io_methods METHOD = { \ unixShmMap, /* xShmMap */ \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ - unixShmUnmap /* xShmUnmap */ \ + unixShmUnmap, /* xShmUnmap */ \ + unixMremap, /* xMremap */ \ }; \ static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ @@ -4504,7 +4541,7 @@ static const sqlite3_io_methods *(*const FINDER)(const char*,unixFile *p) \ IOMETHODS( posixIoFinder, /* Finder function name */ posixIoMethods, /* sqlite3_io_methods object name */ - 2, /* shared memory is enabled */ + 3, /* shared memory and mmap are enabled */ unixClose, /* xClose method */ unixLock, /* xLock method */ unixUnlock, /* xUnlock method */ diff --git a/src/pager.c b/src/pager.c index d9b9ed5052..9f7b349382 100644 --- a/src/pager.c +++ b/src/pager.c @@ -659,8 +659,11 @@ struct Pager { void *pMap; /* Memory mapped prefix of database file */ i64 nMap; /* Size of mapping at pMap in bytes */ i64 nMapValid; /* Bytes at pMap known to be valid */ + i64 nMapLimit; /* Maximum permitted mapping size */ + int nMapCfgLimit; /* Configured limit value */ int nMmapOut; /* Number of mmap pages currently outstanding */ PgHdr *pFree; /* List of free mmap page headers (pDirty) */ + int bMapResize; /* Check if the mapping should be resized */ /* ** End of the routinely-changing class members ***************************************************************************/ @@ -3356,6 +3359,35 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ sqlite3PcacheSetCachesize(pPager->pPCache, mxPage); } +/* +** Set Pager.nMapLimit, the maximum permitted mapping size, based on the +** current values of Pager.nMapCfgLimit and Pager.pageSize. +** +** If this connection should not use mmap at all, set nMapLimit to zero. +*/ +static void pagerFixMaplimit(Pager *pPager){ + if( isOpen(pPager->fd)==0 + || pPager->fd->pMethods->iVersion<3 + || pPager->fd->pMethods->xMremap==0 + || pPager->tempFile + || pPager->pWal + ){ + pPager->nMapLimit = 0; + }else if( pPager->nMapCfgLimit<0 ){ + pPager->nMapLimit = (i64)pPager->nMapCfgLimit * -1024; + }else{ + pPager->nMapLimit = (i64)pPager->nMapCfgLimit * pPager->pageSize; + } +} + +/* +** Change the maximum size of any memory mapping made of the database file. +*/ +void sqlite3PagerSetMmapsize(Pager *pPager, int nMap){ + pPager->nMapCfgLimit = nMap; + pagerFixMaplimit(pPager); +} + /* ** Free as much memory as possible from the pager. */ @@ -3591,6 +3623,7 @@ int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ assert( nReserve>=0 && nReserve<1000 ); pPager->nReserve = (i16)nReserve; pagerReportSize(pPager); + pagerFixMaplimit(pPager); } return rc; } @@ -3816,108 +3849,100 @@ static int pagerSyncHotJournal(Pager *pPager){ return rc; } -#include - /* -** Unmap any mapping of the database file. +** Unmap any memory mapping of the database file. */ static int pagerUnmap(Pager *pPager){ if( pPager->pMap ){ - munmap(pPager->pMap, pPager->nMap); - pPager->pMap = 0; + sqlite3OsMremap(pPager->fd, 0, pPager->nMap, 0, &pPager->pMap); pPager->nMap = 0; pPager->nMapValid = 0; } return SQLITE_OK; } +/* +** Create, or recreate, the memory mapping of the database file. +*/ static int pagerMap(Pager *pPager){ - int rc; - i64 sz = 0; + int rc = SQLITE_OK; /* Return code */ + Pgno nPg; /* Size of mapping to request in pages */ + i64 sz; /* Size of mapping to request in bytes */ - assert( pPager->pMap==0 && pPager->nMap==0 ); + assert( pPager->pWal==0 && isOpen(pPager->fd) && pPager->tempFile==0 ); + assert( pPager->pMap==0 || pPager->nMap>0 ); + assert( pPager->eState>=1 ); + assert( pPager->nMmapOut==0 ); + assert( pPager->nMapLimit>0 ); - rc = sqlite3OsFileSize(pPager->fd, &sz); - sz = sz & ~(4096-1); + /* Figure out how large a mapping to request. Set variable sz to this + ** value in bytes. */ + nPg = (pPager->eState==1) ? pPager->dbSize : pPager->dbFileSize; + sz = (i64)nPg * pPager->pageSize; + if( sz>pPager->nMapLimit ) sz = pPager->nMapLimit; - if( rc==SQLITE_OK && sz>0 ){ - int fd; - rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_GETFD, (void *)&fd); + if( sz!=pPager->nMapValid ){ + rc = sqlite3OsMremap(pPager->fd, 0, pPager->nMap, sz, &pPager->pMap); if( rc==SQLITE_OK ){ - void *pMap = mmap(0, sz, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0); - if( pMap==MAP_FAILED ){ - return SQLITE_IOERR; - } - pPager->pMap = pMap; - pPager->nMapValid = pPager->nMap = sz; + assert( pPager->pMap!=0 ); + pPager->nMap = sz; + }else{ + assert( pPager->pMap==0 ); + pPager->nMap = 0; } + pPager->nMapValid = pPager->nMap; } + pPager->bMapResize = 0; return rc; } -static int pagerRemap(Pager *pPager, Pgno nPage){ - i64 sz = (i64)nPage * pPager->pageSize; - sz = sz & ~(4096-1); - - if( pPager->nMap!=sz ){ - void *pMap = mremap(pPager->pMap, pPager->nMap, sz, MREMAP_MAYMOVE); - if( pMap==MAP_FAILED ){ - return SQLITE_IOERR; - } - pPager->pMap = pMap; - pPager->nMapValid = pPager->nMap = sz; - } - - return SQLITE_OK; -} - +/* +** Obtain a reference to a memory mapped page object for page number pgno. +** The caller must ensure that page pgno lies within the currently mapped +** region. If successful, set *ppPage to point to the new page reference +** and return SQLITE_OK. Otherwise, return an SQLite error code and set +** *ppPage to zero. +** +** Page references obtained by calling this function should be released +** by calling pagerReleaseMapPage(). +*/ static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ - int rc; - *ppPage = 0; + PgHdr *p; /* Memory mapped page to return */ - assert( pPager->pWal==0 ); - - if( MEMDB==0 && pPager->tempFile==0 ){ - if( pPager->pMap==0 ){ - rc = pagerMap(pPager); - if( rc!=SQLITE_OK ) return rc; - } - - if( pgno!=1 && pPager->pMap - && pPager->nMapValid>=((i64)pgno*pPager->pageSize) - ){ - PgHdr *p; - if( pPager->pFree ){ - p = pPager->pFree; - pPager->pFree = p->pDirty; - p->pDirty = 0; - memset(p->pExtra, 0, pPager->nExtra); - }else{ - p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); - if( p==0 ) return SQLITE_NOMEM; - p->pExtra = (void *)&p[1]; - p->flags = PGHDR_MMAP; - p->nRef = 1; - p->pPager = pPager; - } - - assert( p->pExtra==(void *)&p[1] ); - assert( p->pPage==0 ); - assert( p->flags==PGHDR_MMAP ); - assert( p->pPager==pPager ); - assert( p->nRef==1 ); - - p->pData = &((u8 *)pPager->pMap)[(i64)(pgno-1) * pPager->pageSize]; - p->pgno = pgno; - pPager->nMmapOut++; - *ppPage = p; + if( pPager->pFree ){ + *ppPage = p = pPager->pFree; + pPager->pFree = p->pDirty; + p->pDirty = 0; + memset(p->pExtra, 0, pPager->nExtra); + }else{ + *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); + if( p==0 ){ + return SQLITE_NOMEM; } + p->pExtra = (void *)&p[1]; + p->flags = PGHDR_MMAP; + p->nRef = 1; + p->pPager = pPager; } + assert( p->pExtra==(void *)&p[1] ); + assert( p->pPage==0 ); + assert( p->flags==PGHDR_MMAP ); + assert( p->pPager==pPager ); + assert( p->nRef==1 ); + + p->pData = &((u8 *)pPager->pMap)[(i64)(pgno-1) * pPager->pageSize]; + p->pgno = pgno; + pPager->nMmapOut++; + return SQLITE_OK; } +/* +** Release a reference to page pPg. pPg must have been returned by an +** earlier call to pagerAcquireMapPage(). +*/ static void pagerReleaseMapPage(PgHdr *pPg){ Pager *pPager = pPg->pPager; pPager->nMmapOut--; @@ -3925,6 +3950,9 @@ static void pagerReleaseMapPage(PgHdr *pPg){ pPager->pFree = pPg; } +/* +** Free all PgHdr objects stored in the Pager.pFree list. +*/ static void pagerFreeMapHdrs(Pager *pPager){ PgHdr *p; PgHdr *pNext; @@ -5115,7 +5143,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** possibility by unmapping the db here. */ pagerUnmap(pPager); }else if( pPager->pMap ){ - pagerRemap(pPager, nPage); + pPager->bMapResize = 1; } } @@ -5226,8 +5254,7 @@ int sqlite3PagerAcquire( ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY ** flag was specified by the caller. And so long as the db is not a ** temporary or in-memory database. */ - const int bMmapOk = ( - (pgno!=1 && pPager->pWal==0 && !pPager->tempFile && !MEMDB) + const int bMmapOk = (pPager->nMapLimit>0 && pgno!=1 && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY)) ); @@ -5245,7 +5272,7 @@ int sqlite3PagerAcquire( }else{ if( bMmapOk ){ - if( pPager->pMap==0 ){ + if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){ rc = pagerMap(pPager); } if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){ @@ -7014,6 +7041,7 @@ static int pagerOpenWal(Pager *pPager){ pPager->journalSizeLimit, &pPager->pWal ); } + pagerFixMaplimit(pPager); return rc; } @@ -7104,6 +7132,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ rc = sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; + pagerFixMaplimit(pPager); } } return rc; diff --git a/src/pragma.c b/src/pragma.c index f6dadbd4a8..9b31797d3f 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -744,6 +744,30 @@ void sqlite3Pragma( } }else + /* + ** PRAGMA [database.]mmap_size + ** PRAGMA [database.]mmap_size=N + ** + ** Used to set or query the mapping size limit. The mapping size limit is + ** used to limit the aggregate size of all memory mapped regions of the + ** database file. If this parameter is set to zero, then memory mapping + ** is not used at all. If it is set to a positive value, then it is + ** interpreted as a maximum size in pages. If set to less than zero, then + ** the absolute value is interpreted as a size limit in KB. + ** + ** The default value is zero (do not use memory mapped IO). + */ + if( sqlite3StrICmp(zLeft,"mmap_size")==0 ){ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + if( !zRight ){ + returnSingleInt(pParse, "mmap_size", pDb->pSchema->mmap_size); + }else{ + int size = sqlite3Atoi(zRight); + pDb->pSchema->mmap_size = size; + sqlite3BtreeSetMmapSize(pDb->pBt, pDb->pSchema->mmap_size); + } + }else + /* ** PRAGMA temp_store ** PRAGMA temp_store = "default"|"memory"|"file" diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 0ddb0dd105..3fa30e01de 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -706,6 +706,24 @@ struct sqlite3_file { ** fails to zero-fill short reads might seem to work. However, ** failure to zero-fill short reads will eventually lead to ** database corruption. +** +** Assuming parameter nNew is non-zero, the xMremap method should attempt +** to memory map a region nNew bytes in size starting at offset iOffset +** of the file. If successful, it should set *ppMap to point to the +** mapping and return SQLITE_OK. If the file is opened for read-write +** access, then the mapping should also be read-write. +** +** If nOld is non-zero, then the initial value of *ppMap points to a +** mapping returned by a previous call to xMremap. The existing mapping +** is nOld bytes in size and starts at offset iOffset of the file. In +** this case the xMremap method is expected to unmap the existing mapping +** and overwrite *ppMap with the pointer to the new mapping. If nOld is +** zero, then the initial value of *ppMap is undefined. +** +** If nNew is zero, then no new mapping should be created. Any old +** mapping must still be unmapped if nOld is non-zero. If the nOld +** parameter is non-zero, then the existing mapping is always unmapped - +** even if an error occurs. */ typedef struct sqlite3_io_methods sqlite3_io_methods; struct sqlite3_io_methods { @@ -728,6 +746,9 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ + int (*xMremap)(sqlite3_file *fd, + sqlite3_int64 iOff, sqlite3_int64 nOld, sqlite3_int64 nNew, void **ppMap); + /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 129c4c5ea5..59483bc87b 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -746,6 +746,7 @@ struct Schema { u8 enc; /* Text encoding used by this database */ u16 flags; /* Flags associated with this schema */ int cache_size; /* Number of pages to use in the cache */ + int mmap_size; /* Number of pages to memory map */ }; /* diff --git a/src/test1.c b/src/test1.c index a89b874a3b..500d60e0fe 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5859,9 +5859,9 @@ static int test_getrusage( getrusage(RUSAGE_SELF, &r); sprintf(buf, "ru_utime=%d.%06d ru_stime=%d.%06d ru_minflt=%d ru_majflt=%d", - r.ru_utime.tv_sec, r.ru_utime.tv_usec, - r.ru_stime.tv_sec, r.ru_stime.tv_usec, - r.ru_minflt, r.ru_majflt + (int)r.ru_utime.tv_sec, (int)r.ru_utime.tv_usec, + (int)r.ru_stime.tv_sec, (int)r.ru_stime.tv_usec, + (int)r.ru_minflt, (int)r.ru_majflt ); Tcl_SetObjResult(interp, Tcl_NewStringObj(buf, -1)); return TCL_OK; diff --git a/test/dbstatus2.test b/test/dbstatus2.test index 18bb0870bd..2541a1a823 100644 --- a/test/dbstatus2.test +++ b/test/dbstatus2.test @@ -40,6 +40,7 @@ proc db_write {db {reset 0}} { do_test 1.1 { db close sqlite3 db test.db + execsql { PRAGMA mmap_size = 0 } expr {[file size test.db] / 1024} } 6 diff --git a/test/exclusive2.test b/test/exclusive2.test index 2208da5101..54203e3d8f 100644 --- a/test/exclusive2.test +++ b/test/exclusive2.test @@ -25,6 +25,14 @@ ifcapable {!pager_pragmas} { return } +# Tests in this file verify that locking_mode=exclusive causes SQLite to +# use cached pages even if the database is changed on disk. This doesn't +# work with mmap. +if {[permutation]=="mmap"} { + finish_test + return +} + # This module does not work right if the cache spills at unexpected # moments. So disable the soft-heap-limit. # diff --git a/test/func.test b/test/func.test index f09ff49805..4ab7688461 100644 --- a/test/func.test +++ b/test/func.test @@ -1273,11 +1273,13 @@ do_test func-29.3 { sqlite3_db_status db CACHE_MISS 1 db eval {SELECT typeof(+x) FROM t29 ORDER BY id} } {integer null real blob text} -do_test func-29.4 { - set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] - if {$x>100} {set x many} - set x -} {many} +if {[permutation] != "mmap"} { + do_test func-29.4 { + set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] + if {$x>100} {set x many} + set x + } {many} +} do_test func-29.5 { db close sqlite3 db test.db diff --git a/test/incrblob.test b/test/incrblob.test index 7cc99dd983..f51f5852d4 100644 --- a/test/incrblob.test +++ b/test/incrblob.test @@ -123,6 +123,7 @@ foreach AutoVacuumMode [list 0 1] { forcedelete test.db test.db-journal sqlite3 db test.db + execsql "PRAGMA mmap_size = 0" execsql "PRAGMA auto_vacuum = $AutoVacuumMode" do_test incrblob-2.$AutoVacuumMode.1 { @@ -149,6 +150,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db + execsql "PRAGMA mmap_size = 0" # Read the last 20 bytes of the blob via a blob handle. set ::blob [db incrblob blobs v 1] @@ -171,6 +173,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db + execsql "PRAGMA mmap_size = 0" # Write the second-to-last 20 bytes of the blob via a blob handle. # @@ -200,6 +203,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db + execsql { PRAGMA mmap_size = 0 } execsql { SELECT i FROM blobs } } {45} diff --git a/test/pageropt.test b/test/pageropt.test index 82311965a5..de39bbaf05 100644 --- a/test/pageropt.test +++ b/test/pageropt.test @@ -92,7 +92,7 @@ do_test pageropt-1.5 { pagercount_sql { SELECT hex(x) FROM t1 } -} [list 6 0 0 $blobcontent] +} [list [expr {[permutation]=="mmap" ? 1 : 6}] 0 0 $blobcontent] do_test pageropt-1.6 { pagercount_sql { SELECT hex(x) FROM t1 diff --git a/test/permutations.test b/test/permutations.test index d9014d77fe..fd05b58e4f 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -134,6 +134,14 @@ test_suite "veryquick" -prefix "" -description { "Very" quick test suite. Runs in less than 5 minutes on a workstation. This test suite is the same as the "quick" tests, except that some files that test malloc and IO errors are omitted. +} -files [ + test_set $allquicktests -exclude *malloc* *ioerr* *fault* +] + +test_suite "mmap" -prefix "mm-" -description { + Similar to veryquick. Except with memory mapping enabled. +} -presql { + pragma mmap_size = -65536; } -files [ test_set $allquicktests -exclude *malloc* *ioerr* *fault* \ multiplex* server1.test shared2.test shared6.test From eecc3983741f501203faf8404eca56332418696e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 20 Mar 2013 10:07:43 +0000 Subject: [PATCH 006/140] Add test file mmap1.test. FossilOrigin-Name: aee1f53a74e636776cbbc11bdd5516432ad50533 --- manifest | 11 ++++--- manifest.uuid | 2 +- test/mmap1.test | 84 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 test/mmap1.test diff --git a/manifest b/manifest index 45c3ee6fc5..1a8ba2228d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssqlite3_io_methods.xMremap()\smethod\sto\sthe\sVFS\sinterface.\sAlso\s"PRAGMA\smmap_size". -D 2013-03-19T19:28:06.473 +C Add\stest\sfile\smmap1.test. +D 2013-03-20T10:07:43.411 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -630,6 +630,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 +F test/mmap1.test 46ff4038b6d300bb9b39d3f89ea1aef3dff9b88f F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 @@ -1038,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f8ca5622d99bedca957caa9ad311d798f63b3ce9 -R 1add721bf7e9669d3b4d4c28d41bb13b +P 6183f1bd86ceed76d22d9762f3d7eb33262c62d1 +R 1501a2a578b96ab1d025c2aefaaa8f35 U dan -Z 3a3766130d03f0bab6c4936a9c26c7a5 +Z a7aa76ce5b116a8f55025f0891a0dfbb diff --git a/manifest.uuid b/manifest.uuid index 8d198c909c..68ed5bb6a8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6183f1bd86ceed76d22d9762f3d7eb33262c62d1 \ No newline at end of file +aee1f53a74e636776cbbc11bdd5516432ad50533 \ No newline at end of file diff --git a/test/mmap1.test b/test/mmap1.test new file mode 100644 index 0000000000..343ad230b1 --- /dev/null +++ b/test/mmap1.test @@ -0,0 +1,84 @@ +# 2013 March 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set testprefix mmap1 + +proc nRead {db} { + set bt [btree_from_db $db] + db_enter $db + array set stats [btree_pager_stats $bt] + db_leave $db + return $stats(read) +} + +foreach {t mmap_size nRead} { + 1 { PRAGMA mmap_size = -65536 } 4 + 2 { PRAGMA mmap_size = -50 } 156 + 3 { PRAGMA mmap_size = 0 } 344 +} { + do_multiclient_test tn { + sql1 $mmap_size + + code2 { + set ::rcnt 0 + proc rblob {n} { + set ::rcnt [expr (($::rcnt << 3) + $::rcnt + 456) & 0xFFFFFFFF] + set str [format %.8x [expr $::rcnt ^ 0xbdf20da3]] + string range [string repeat $str [expr $n/4]] 1 $n + } + db2 func rblob rblob + } + + sql2 { + PRAGMA auto_vacuum = 1; + CREATE TABLE t1(a, b, UNIQUE(a, b)); + INSERT INTO t1 VALUES(rblob(500), rblob(500)); + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 2 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 4 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 8 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 32 + } + do_test $t.$tn.1 { + sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count" + } {32 ok 77} + + # Have connection 2 shrink the file. Check connection 1 can still read it. + sql2 { DELETE FROM t1 WHERE rowid%2; } + do_test $t.$tn.2 { + sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count" + } {16 ok 42} + + # Have connection 2 grow the file. Check connection 1 can still read it. + sql2 { INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1 } + do_test $t.$tn.3 { + sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count" + } {32 ok 79} + + # Have connection 2 grow the file again. Check connection 1 is still ok. + sql2 { INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1 } + do_test $t.$tn.4 { + sql1 "SELECT count(*) FROM t1; PRAGMA integrity_check ; PRAGMA page_count" + } {64 ok 149} + + # Check that the number of pages read by connection 1 indicates that the + # "PRAGMA mmap_size" command worked. + do_test $t.$tn.5 { nRead db } $nRead + } +} + + +finish_test + From eb97b29345592cf094b6639a985f9253653a5063 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 20 Mar 2013 14:26:59 +0000 Subject: [PATCH 007/140] When possible, use memory mapping when appending new pages to a database file. FossilOrigin-Name: 14135da3cdbafd699563a29608f32347cda28338 --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/os.c | 11 +++++++++-- src/os.h | 2 +- src/os_unix.c | 33 ++++++++++++++++++++++++++++++--- src/pager.c | 18 +++++++++++++----- src/pager.h | 1 + src/sqlite.h.in | 5 ++++- 8 files changed, 70 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 1a8ba2228d..d519ad3e2a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\sfile\smmap1.test. -D 2013-03-20T10:07:43.411 +C When\spossible,\suse\smemory\smapping\swhen\sappending\snew\spages\sto\sa\sdatabase\sfile. +D 2013-03-20T14:26:59.370 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,13 +157,13 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c 81a82a736b8a461a656f9b3e401a39768fe73a79 -F src/os.h 4681261aa24a9d2187aaf4cb963880e6cddb1f48 +F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 +F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 0c7b0d076f2ac6279b0b280a26bcae8c89f36f4f +F src/os_unix.c aedd47c2145edd49051452afbe82fa6f54a49b2a F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 78b65bf9685bf21b787ce2a7389c2b96102942dc -F src/pager.h 81ac95f4fcfe21981f495146f6d7f2fe51afd110 +F src/pager.c d59af9a70aa2d7222b127351fa3cbe70660e4150 +F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 @@ -176,7 +176,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in fd75f5bcf479b315b1c717fa1d07f018bd919f79 +F src/sqlite.h.in d63c7fb5832287af7e8b903c4a4c30c90414876f F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 6183f1bd86ceed76d22d9762f3d7eb33262c62d1 -R 1501a2a578b96ab1d025c2aefaaa8f35 +P aee1f53a74e636776cbbc11bdd5516432ad50533 +R 041b09b1e247c8054516280d9526a179 U dan -Z a7aa76ce5b116a8f55025f0891a0dfbb +Z 8c309d7c56b08fcba504ca346091b196 diff --git a/manifest.uuid b/manifest.uuid index 68ed5bb6a8..f40916822a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -aee1f53a74e636776cbbc11bdd5516432ad50533 \ No newline at end of file +14135da3cdbafd699563a29608f32347cda28338 \ No newline at end of file diff --git a/src/os.c b/src/os.c index 45d8e0c121..38757dcdb1 100644 --- a/src/os.c +++ b/src/os.c @@ -140,8 +140,15 @@ int sqlite3OsShmMap( DO_OS_MALLOC_TEST(id); return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } -int sqlite3OsMremap(sqlite3_file *id, i64 iOff, i64 nOld, i64 nNew, void **pp){ - return id->pMethods->xMremap(id, iOff, nOld, nNew, pp); +int sqlite3OsMremap( + sqlite3_file *id, /* Database file handle */ + int flags, /* SQLITE_MREMAP_XXX flags */ + i64 iOff, /* Offset at which mapping(s) start */ + i64 nOld, /* Size of old mapping */ + i64 nNew, /* Size of requested mapping */ + void **pp /* IN/OUT: Pointer to mapped region */ +){ + return id->pMethods->xMremap(id, flags, iOff, nOld, nNew, pp); } /* diff --git a/src/os.h b/src/os.h index 76a60effdb..f08b92dcf0 100644 --- a/src/os.h +++ b/src/os.h @@ -259,7 +259,7 @@ int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); -int sqlite3OsMremap(sqlite3_file *id, i64, i64, i64, void **); +int sqlite3OsMremap(sqlite3_file *id, int, i64, i64, i64, void **); /* diff --git a/src/os_unix.c b/src/os_unix.c index f37f2404ee..1c26c6badf 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4434,6 +4434,7 @@ static int unixShmUnmap( */ static int unixMremap( sqlite3_file *fd, /* Main database file */ + int flags, /* Mask of SQLITE_MREMAP_XXX flags */ sqlite3_int64 iOff, /* Offset to start mapping at */ sqlite3_int64 nOld, /* Size of old mapping, or zero */ sqlite3_int64 nNew, /* Size of new mapping, or zero */ @@ -4442,8 +4443,35 @@ static int unixMremap( unixFile *p = (unixFile *)fd; /* The underlying database file */ int rc = SQLITE_OK; /* Return code */ void *pNew = 0; /* New mapping */ + i64 nRnd; /* nNew rounded up to 4096 */ assert( iOff==0 ); + nRnd = (nNew+4095) & ~(i64)((1 << 12)-1); + + /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested + ** mapping (nNew bytes) may be greater than the size of the database file. + ** If this is the case, extend the file on disk using ftruncate(). */ + assert( nNew>0 || (flags & SQLITE_MREMAP_EXTEND)==0 ); + if( flags & SQLITE_MREMAP_EXTEND ){ + struct stat statbuf; /* Low-level file information */ + rc = osFstat(p->h, &statbuf); + if( rc==SQLITE_OK && nNew>statbuf.st_size ){ + rc = robust_ftruncate(p->h, nNew); + } + if( rc!=SQLITE_OK ) return rc; + } + +#if defined(_GNU_SOURCE) && defined(__linux__) + if( nRnd && nOld ){ + void *pOld = *ppMap; + *ppMap = pNew = mremap(pOld, nOld, nNew, MREMAP_MAYMOVE); + if( pNew==MAP_FAILED ){ + *ppMap = 0; + return SQLITE_IOERR_MREMAP; + } + return SQLITE_OK; + } +#endif if( nOld!=0 ){ void *pOld = *ppMap; @@ -4453,11 +4481,10 @@ static int unixMremap( if( nNew>0 ){ int flags = PROT_READ; if( (p->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - nNew = (nNew+4095) & ~(i64)((1 << 12)-1); - pNew = mmap(0, nNew, flags, MAP_SHARED, p->h, iOff); + pNew = mmap(0, nRnd, flags, MAP_SHARED, p->h, iOff); if( pNew==MAP_FAILED ){ pNew = 0; - rc = SQLITE_IOERR; + rc = SQLITE_IOERR_MREMAP; } } diff --git a/src/pager.c b/src/pager.c index 9f7b349382..2848f9ad05 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3854,7 +3854,7 @@ static int pagerSyncHotJournal(Pager *pPager){ */ static int pagerUnmap(Pager *pPager){ if( pPager->pMap ){ - sqlite3OsMremap(pPager->fd, 0, pPager->nMap, 0, &pPager->pMap); + sqlite3OsMremap(pPager->fd, 0, 0, pPager->nMap, 0, &pPager->pMap); pPager->nMap = 0; pPager->nMapValid = 0; } @@ -3864,7 +3864,7 @@ static int pagerUnmap(Pager *pPager){ /* ** Create, or recreate, the memory mapping of the database file. */ -static int pagerMap(Pager *pPager){ +static int pagerMap(Pager *pPager, int bExtend){ int rc = SQLITE_OK; /* Return code */ Pgno nPg; /* Size of mapping to request in pages */ i64 sz; /* Size of mapping to request in bytes */ @@ -3882,7 +3882,8 @@ static int pagerMap(Pager *pPager){ if( sz>pPager->nMapLimit ) sz = pPager->nMapLimit; if( sz!=pPager->nMapValid ){ - rc = sqlite3OsMremap(pPager->fd, 0, pPager->nMap, sz, &pPager->pMap); + int flags = (bExtend ? SQLITE_MREMAP_EXTEND : 0); + rc = sqlite3OsMremap(pPager->fd, flags, 0, pPager->nMap, sz, &pPager->pMap); if( rc==SQLITE_OK ){ assert( pPager->pMap!=0 ); pPager->nMap = sz; @@ -4246,10 +4247,17 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ ** file size will be. */ assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); - if( rc==SQLITE_OK && pPager->dbSize>pPager->dbHintSize ){ + if( rc==SQLITE_OK + && (pList->pDirty ? pPager->dbSize : pList->pgno+1)>pPager->dbHintSize + ){ sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); pPager->dbHintSize = pPager->dbSize; + + if( pPager->nMmapOut==0 && pPager->nMapLimit>0 ){ + pPager->dbFileSize = pPager->dbSize; + pagerMap(pPager, 1); + } } while( rc==SQLITE_OK && pList ){ @@ -5273,7 +5281,7 @@ int sqlite3PagerAcquire( if( bMmapOk ){ if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){ - rc = pagerMap(pPager); + rc = pagerMap(pPager, 0); } if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){ if( pPager->eState>PAGER_READER ){ diff --git a/src/pager.h b/src/pager.h index 94957dac60..81ab30c115 100644 --- a/src/pager.h +++ b/src/pager.h @@ -108,6 +108,7 @@ void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); int sqlite3PagerSetPagesize(Pager*, u32*, int); int sqlite3PagerMaxPageCount(Pager*, int); void sqlite3PagerSetCachesize(Pager*, int); +void sqlite3PagerSetMmapsize(Pager *, int); void sqlite3PagerShrink(Pager*); void sqlite3PagerSetSafetyLevel(Pager*,int,int,int); int sqlite3PagerLockingMode(Pager *, int); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 3fa30e01de..c0a2803787 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -470,6 +470,7 @@ int sqlite3_exec( #define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) #define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) #define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) +#define SQLITE_IOERR_MREMAP (SQLITE_IOERR | (24<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) @@ -746,12 +747,14 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ - int (*xMremap)(sqlite3_file *fd, + int (*xMremap)(sqlite3_file *fd, int flags, sqlite3_int64 iOff, sqlite3_int64 nOld, sqlite3_int64 nNew, void **ppMap); /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; +#define SQLITE_MREMAP_EXTEND 0x0001 /* xMremap call may extend file */ + /* ** CAPI3REF: Standard File Control Opcodes ** From d306e1a3a113bd3905c6b661bcb8176b0bb6a844 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 20 Mar 2013 18:25:49 +0000 Subject: [PATCH 008/140] Optimize the xMremap method in os_unix.c some. FossilOrigin-Name: 9529ed88a71fee02fae72dc86f0669bd6856ff92 --- manifest | 12 +++++----- manifest.uuid | 2 +- src/os_unix.c | 61 ++++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 60 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index d519ad3e2a..c9eb696178 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\spossible,\suse\smemory\smapping\swhen\sappending\snew\spages\sto\sa\sdatabase\sfile. -D 2013-03-20T14:26:59.370 +C Optimize\sthe\sxMremap\smethod\sin\sos_unix.c\ssome. +D 2013-03-20T18:25:49.737 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c aedd47c2145edd49051452afbe82fa6f54a49b2a +F src/os_unix.c c96bdc9e912c0f8226a03cd025566b5a43e1c387 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 F src/pager.c d59af9a70aa2d7222b127351fa3cbe70660e4150 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P aee1f53a74e636776cbbc11bdd5516432ad50533 -R 041b09b1e247c8054516280d9526a179 +P 14135da3cdbafd699563a29608f32347cda28338 +R ae7e97a190e1fdc49e8b167e558562e7 U dan -Z 8c309d7c56b08fcba504ca346091b196 +Z a86246ff854d247b1925a564cb8b08dc diff --git a/manifest.uuid b/manifest.uuid index f40916822a..3037557cd4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -14135da3cdbafd699563a29608f32347cda28338 \ No newline at end of file +9529ed88a71fee02fae72dc86f0669bd6856ff92 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 1c26c6badf..7ab61ed0b3 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4429,6 +4429,16 @@ static int unixShmUnmap( # define unixShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ +/* +** Arguments x and y are both integers. Argument y must be a power of 2. +** Round x up to the nearest integer multiple of y. For example: +** +** ROUNDUP(0, 8) -> 0 +** ROUNDUP(13, 8) -> 16 +** ROUNDUP(32, 8) -> 32 +*/ +#define ROUNDUP(x,y) (((x)+y-1)&~(y-1)) + /* ** Map, remap or unmap part of the database file. */ @@ -4443,10 +4453,10 @@ static int unixMremap( unixFile *p = (unixFile *)fd; /* The underlying database file */ int rc = SQLITE_OK; /* Return code */ void *pNew = 0; /* New mapping */ - i64 nRnd; /* nNew rounded up to 4096 */ + i64 nNewRnd; /* nNew rounded up */ + i64 nOldRnd; /* nOld rounded up */ assert( iOff==0 ); - nRnd = (nNew+4095) & ~(i64)((1 << 12)-1); /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested ** mapping (nNew bytes) may be greater than the size of the database file. @@ -4461,10 +4471,43 @@ static int unixMremap( if( rc!=SQLITE_OK ) return rc; } + /* According to some sources, the effect of changing the size of the + ** underlying file on mapped regions that correspond to the added or + ** removed pages is undefined. However, there is reason to believe that + ** on modern platforms like Linux or OSX, things just work. For example, + ** it is possible to create a mapping larger than the file on disk and + ** extend the file on disk later on. + ** + ** Exploit this on OSX to reduce the number of munmap()/mmap() calls + ** if the file size is changing. In this case all mappings are rounded + ** up to the nearest 4MB. And if a new mapping is requested that has the + ** same rounded size as an old mapping, the old mapping can simply be + ** reused as is. + ** + ** It would be possible to do the above on Linux too. However, Linux has + ** the non-standard mremap() call to resize existing mappings, which can + ** be used instead. */ +#if defined(__APPLE__) + nNewRnd = ROUNDUP(nNew, 4096*1024); + nOldRnd = ROUNDUP(nOld, 4096*1024); +#else + nNewRnd = ROUNDUP(nNew, 4096*1); + nOldRnd = ROUNDUP(nOld, 4096*1); +#endif + + /* On OSX or Linux, reuse the old mapping if it is the right size. */ +#if defined(__APPLE__) || defined(__linux__) + if( nNewRnd==nOldRnd ){ + return SQLITE_OK; + } +#endif + + /* On Linux, if there is both an old and new mapping, resize the old + ** mapping using the non-standard mremap() call. */ #if defined(_GNU_SOURCE) && defined(__linux__) - if( nRnd && nOld ){ + if( nNewRnd && nOldRnd ){ void *pOld = *ppMap; - *ppMap = pNew = mremap(pOld, nOld, nNew, MREMAP_MAYMOVE); + *ppMap = pNew = mremap(pOld, nOldRnd, nNewRnd, MREMAP_MAYMOVE); if( pNew==MAP_FAILED ){ *ppMap = 0; return SQLITE_IOERR_MREMAP; @@ -4473,15 +4516,17 @@ static int unixMremap( } #endif - if( nOld!=0 ){ + /* If we get this far, unmap any old mapping. */ + if( nOldRnd!=0 ){ void *pOld = *ppMap; - munmap(pOld, nOld); + munmap(pOld, nOldRnd); } - if( nNew>0 ){ + /* And, if required, use mmap() to create a new mapping. */ + if( nNewRnd>0 ){ int flags = PROT_READ; if( (p->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - pNew = mmap(0, nRnd, flags, MAP_SHARED, p->h, iOff); + pNew = mmap(0, nNewRnd, flags, MAP_SHARED, p->h, iOff); if( pNew==MAP_FAILED ){ pNew = 0; rc = SQLITE_IOERR_MREMAP; From c71b45e61944f6964b384614f3da737eff2c2dd6 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 21 Mar 2013 14:47:47 +0000 Subject: [PATCH 009/140] Do not use the Linux mremap() call. Use the same strategy for xMremap() as on OSX instead. FossilOrigin-Name: 5ed8ad780c991d2ca44003ee84350fb5e95ad58e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 30 ++++++------------------------ 3 files changed, 13 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index c9eb696178..870e5cce8d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Optimize\sthe\sxMremap\smethod\sin\sos_unix.c\ssome. -D 2013-03-20T18:25:49.737 +C Do\snot\suse\sthe\sLinux\smremap()\scall.\sUse\sthe\ssame\sstrategy\sfor\sxMremap()\sas\son\sOSX\sinstead. +D 2013-03-21T14:47:47.427 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c c96bdc9e912c0f8226a03cd025566b5a43e1c387 +F src/os_unix.c be66c31337361a72227638d6f41c7f2031739642 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 F src/pager.c d59af9a70aa2d7222b127351fa3cbe70660e4150 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 14135da3cdbafd699563a29608f32347cda28338 -R ae7e97a190e1fdc49e8b167e558562e7 +P 9529ed88a71fee02fae72dc86f0669bd6856ff92 +R 7e61f02cabbded666ed9e0992a636a37 U dan -Z a86246ff854d247b1925a564cb8b08dc +Z 67f89f7e2101ab4157bb24e7a7ac302a diff --git a/manifest.uuid b/manifest.uuid index 3037557cd4..f10e3bc724 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9529ed88a71fee02fae72dc86f0669bd6856ff92 \ No newline at end of file +5ed8ad780c991d2ca44003ee84350fb5e95ad58e \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 7ab61ed0b3..153cf5f435 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4478,16 +4478,12 @@ static int unixMremap( ** it is possible to create a mapping larger than the file on disk and ** extend the file on disk later on. ** - ** Exploit this on OSX to reduce the number of munmap()/mmap() calls - ** if the file size is changing. In this case all mappings are rounded - ** up to the nearest 4MB. And if a new mapping is requested that has the - ** same rounded size as an old mapping, the old mapping can simply be - ** reused as is. - ** - ** It would be possible to do the above on Linux too. However, Linux has - ** the non-standard mremap() call to resize existing mappings, which can - ** be used instead. */ -#if defined(__APPLE__) + ** Exploit this on Linux and OSX to reduce the number of munmap()/mmap() + ** calls required if the file size is changing. In this case all mappings + ** are rounded up to the nearest 4MB. And if a new mapping is requested + ** that has the same rounded size as an old mapping, the old mapping can + ** be reused as is. */ +#if defined(__APPLE__) || defined(__linux__) nNewRnd = ROUNDUP(nNew, 4096*1024); nOldRnd = ROUNDUP(nOld, 4096*1024); #else @@ -4502,20 +4498,6 @@ static int unixMremap( } #endif - /* On Linux, if there is both an old and new mapping, resize the old - ** mapping using the non-standard mremap() call. */ -#if defined(_GNU_SOURCE) && defined(__linux__) - if( nNewRnd && nOldRnd ){ - void *pOld = *ppMap; - *ppMap = pNew = mremap(pOld, nOldRnd, nNewRnd, MREMAP_MAYMOVE); - if( pNew==MAP_FAILED ){ - *ppMap = 0; - return SQLITE_IOERR_MREMAP; - } - return SQLITE_OK; - } -#endif - /* If we get this far, unmap any old mapping. */ if( nOldRnd!=0 ){ void *pOld = *ppMap; From 5b068f863228ba1f074e2999a7dc1527e34d9612 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 21 Mar 2013 15:57:32 +0000 Subject: [PATCH 010/140] Fix a problem when opening a write-transaction while there exist read-only b-tree cursors in mmap mode. FossilOrigin-Name: 32e0bbb73609ac3ad096a60f1de3095bc79fb0cc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 870e5cce8d..8819c52a55 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\suse\sthe\sLinux\smremap()\scall.\sUse\sthe\ssame\sstrategy\sfor\sxMremap()\sas\son\sOSX\sinstead. -D 2013-03-21T14:47:47.427 +C Fix\sa\sproblem\swhen\sopening\sa\swrite-transaction\swhile\sthere\sexist\sread-only\sb-tree\scursors\sin\smmap\smode. +D 2013-03-21T15:57:32.562 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c d4d551f05a555926a7c0f49c2e263f7ee2b1c59f +F src/btree.c dd4683fa671f0b2b4bc0831036adaa2a5d1806ee F src/btree.h d3259057a38494c4c4358e377032158c762e3d8b F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 375e5df716e03b9343c5e1211be3b24e6d6dff05 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9529ed88a71fee02fae72dc86f0669bd6856ff92 -R 7e61f02cabbded666ed9e0992a636a37 +P 5ed8ad780c991d2ca44003ee84350fb5e95ad58e +R fc9bd174ea58046e3b5bc29693aabeb6 U dan -Z 67f89f7e2101ab4157bb24e7a7ac302a +Z 811da101d49a049638aae5d61fad1590 diff --git a/manifest.uuid b/manifest.uuid index f10e3bc724..95a0f78145 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5ed8ad780c991d2ca44003ee84350fb5e95ad58e \ No newline at end of file +32e0bbb73609ac3ad096a60f1de3095bc79fb0cc \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 2b7872fa8c..0a51ff0301 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2595,7 +2595,7 @@ static int btreeSwapOutMmap(BtShared *pBt){ MemPage *pPg = pCsr->apPage[0]; if( pPg->pDbPage->flags & PGHDR_MMAP ){ MemPage *pNew = 0; - rc = btreeGetPage(pBt, pPg->pgno, &pNew, 0, 0); + rc = getAndInitPage(pBt, pPg->pgno, &pNew, 0); if( rc==SQLITE_OK && pCsr->iPage==0 ){ pCsr->info.pCell = pNew->aData + (pCsr->info.pCell - pPg->aData); } From 7c49427fc25a0fc1738a4f7062abfea45642fbfc Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 21 Mar 2013 20:00:07 +0000 Subject: [PATCH 011/140] Avoid calling xRead() on a part of the database file that is memory mapped. FossilOrigin-Name: c8eac290a7240d69494bd0dad5ed1fdc2505f703 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 8 ++++++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 8819c52a55..b91c073e14 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swhen\sopening\sa\swrite-transaction\swhile\sthere\sexist\sread-only\sb-tree\scursors\sin\smmap\smode. -D 2013-03-21T15:57:32.562 +C Avoid\scalling\sxRead()\son\sa\spart\sof\sthe\sdatabase\sfile\sthat\sis\smemory\smapped. +D 2013-03-21T20:00:07.703 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c be66c31337361a72227638d6f41c7f2031739642 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c d59af9a70aa2d7222b127351fa3cbe70660e4150 +F src/pager.c 5ee2ec6a10aa3d85548e6f0337260f0b2a670899 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5ed8ad780c991d2ca44003ee84350fb5e95ad58e -R fc9bd174ea58046e3b5bc29693aabeb6 +P 32e0bbb73609ac3ad096a60f1de3095bc79fb0cc +R 78d8196d81dc00ab1ac5a50caf4d157f U dan -Z 811da101d49a049638aae5d61fad1590 +Z e76f59323062e94934c6de1e92e548e8 diff --git a/manifest.uuid b/manifest.uuid index 95a0f78145..0a2c6d8ff4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -32e0bbb73609ac3ad096a60f1de3095bc79fb0cc \ No newline at end of file +c8eac290a7240d69494bd0dad5ed1fdc2505f703 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 2848f9ad05..913ae74c46 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5130,9 +5130,13 @@ int sqlite3PagerSharedLock(Pager *pPager){ rc = pagerPagecount(pPager, &nPage); if( rc ) goto failed; - if( nPage>0 ){ + if( nPage>0 || pPager->pMap ){ IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); - rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); + if( pPager->pMap ){ + memcpy(&dbFileVers, &((u8 *)(pPager->pMap))[24], sizeof(dbFileVers)); + }else{ + rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); + } if( rc!=SQLITE_OK ){ goto failed; } From b26e6c14fe6b89b89c0281319743e488d8bfe6ec Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 21 Mar 2013 20:39:55 +0000 Subject: [PATCH 012/140] Fix cases where xRead() was being used to read from a memory mapped part of the database file. FossilOrigin-Name: 5c9e9df27b9f2c46cd55388a858d4e78ee564975 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 29 +++++++++++++++++++++++------ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index b91c073e14..4b2a8a6d5d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\scalling\sxRead()\son\sa\spart\sof\sthe\sdatabase\sfile\sthat\sis\smemory\smapped. -D 2013-03-21T20:00:07.703 +C Fix\scases\swhere\sxRead()\swas\sbeing\sused\sto\sread\sfrom\sa\smemory\smapped\spart\sof\sthe\sdatabase\sfile. +D 2013-03-21T20:39:55.225 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c be66c31337361a72227638d6f41c7f2031739642 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 5ee2ec6a10aa3d85548e6f0337260f0b2a670899 +F src/pager.c 520001015155efee9599c807dfd38e5fff9c6e36 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 32e0bbb73609ac3ad096a60f1de3095bc79fb0cc -R 78d8196d81dc00ab1ac5a50caf4d157f +P c8eac290a7240d69494bd0dad5ed1fdc2505f703 +R 0cdb7517bc23cd336050741c62097e56 U dan -Z e76f59323062e94934c6de1e92e548e8 +Z 0dd43854f08b06a022f11b1a174cdf13 diff --git a/manifest.uuid b/manifest.uuid index 0a2c6d8ff4..50136e7e1b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c8eac290a7240d69494bd0dad5ed1fdc2505f703 \ No newline at end of file +5c9e9df27b9f2c46cd55388a858d4e78ee564975 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 913ae74c46..0ff894bd87 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2087,6 +2087,24 @@ static void pagerReportSize(Pager *pPager){ # define pagerReportSize(X) /* No-op if we do not support a codec */ #endif +/* +** Write nBuf bytes of data from buffer pBuf to offset iOff of the +** database file. If this part of the database file is memory mapped, +** use memcpy() to do so. Otherwise, call sqlite3OsWrite(). +** +** Return SQLITE_OK if successful, or an SQLite error code if an error +** occurs. +*/ +static int pagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff){ + int rc = SQLITE_OK; + if( pPager->nMapValid>=(iOff+nBuf) ){ + memcpy(&((u8 *)(pPager->pMap))[iOff], pBuf, nBuf); + }else{ + rc = sqlite3OsWrite(pPager->fd, pBuf, nBuf, iOff); + } + return rc; +} + /* ** Read a single page from either the journal file (if isMainJrnl==1) or ** from the sub-journal (if isMainJrnl==0) and playback that page. @@ -2261,7 +2279,7 @@ static int pager_playback_one_page( i64 ofst = (pgno-1)*(i64)pPager->pageSize; testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); assert( !pagerUseWal(pPager) ); - rc = sqlite3OsWrite(pPager->fd, (u8*)aData, pPager->pageSize, ofst); + rc = pagerWriteData(pPager, aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } @@ -3853,6 +3871,7 @@ static int pagerSyncHotJournal(Pager *pPager){ ** Unmap any memory mapping of the database file. */ static int pagerUnmap(Pager *pPager){ + assert( pPager->nMmapOut==0 ); if( pPager->pMap ){ sqlite3OsMremap(pPager->fd, 0, 0, pPager->nMap, 0, &pPager->pMap); pPager->nMap = 0; @@ -4282,11 +4301,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); /* Write out the page data. */ - if( pPager->nMapValid>=(offset+pPager->pageSize) ){ - memcpy(&((u8 *)(pPager->pMap))[offset], pData, pPager->pageSize); - }else{ - rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); - } + pagerWriteData(pPager, pData, pPager->pageSize, offset); /* If page 1 was just written, update Pager.dbFileVers to match ** the value now stored in the database file. If writing this @@ -7086,6 +7101,8 @@ int sqlite3PagerOpenWal( assert( pbOpen==0 || *pbOpen==0 ); assert( pbOpen!=0 || (!pPager->tempFile && !pPager->pWal) ); + pagerUnmap(pPager); + if( !pPager->tempFile && !pPager->pWal ){ if( !sqlite3PagerWalSupported(pPager) ) return SQLITE_CANTOPEN; From 6101d50471a97423032fbffcb72f5b62a28a90ee Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 22 Mar 2013 08:58:38 +0000 Subject: [PATCH 013/140] Add assert statements to os_unix.c to ensure that any mapped region of the database file is not being read or written using the xRead() or xWrite() methods. FossilOrigin-Name: 765615f9fba7c1765eb741cb98a09a28b464ee55 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 12 ++++++++++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 4b2a8a6d5d..7a0b83d9c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scases\swhere\sxRead()\swas\sbeing\sused\sto\sread\sfrom\sa\smemory\smapped\spart\sof\sthe\sdatabase\sfile. -D 2013-03-21T20:39:55.225 +C Add\sassert\sstatements\sto\sos_unix.c\sto\sensure\sthat\sany\smapped\sregion\sof\sthe\sdatabase\sfile\sis\snot\sbeing\sread\sor\swritten\susing\sthe\sxRead()\sor\sxWrite()\smethods. +D 2013-03-22T08:58:38.319 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c be66c31337361a72227638d6f41c7f2031739642 +F src/os_unix.c 18153e6fc03580e95d28fb39d47a4e8b64b36187 F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 F src/pager.c 520001015155efee9599c807dfd38e5fff9c6e36 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c8eac290a7240d69494bd0dad5ed1fdc2505f703 -R 0cdb7517bc23cd336050741c62097e56 +P 5c9e9df27b9f2c46cd55388a858d4e78ee564975 +R eb2eb83f96de836951dbdb14158ccee8 U dan -Z 0dd43854f08b06a022f11b1a174cdf13 +Z 42c0bd3cd106c51e7465866daa07e0a3 diff --git a/manifest.uuid b/manifest.uuid index 50136e7e1b..e0d24b066f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5c9e9df27b9f2c46cd55388a858d4e78ee564975 \ No newline at end of file +765615f9fba7c1765eb741cb98a09a28b464ee55 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 153cf5f435..88f3177021 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -249,6 +249,8 @@ struct unixFile { unsigned char transCntrChng; /* True if the transaction counter changed */ unsigned char dbUpdate; /* True if any part of database file changed */ unsigned char inNormalWrite; /* True if in a normal write operation */ + sqlite3_int64 mmapSize; /* Size of xMremap() */ + void *pMapRegion; /* Area memory mapped */ #endif #ifdef SQLITE_TEST /* In test mode, increase the size of this structure a bit so that @@ -3072,6 +3074,7 @@ static int unixRead( unixFile *pFile = (unixFile *)id; int got; assert( id ); + assert( offset>=pFile->mmapSize ); /* Never read from the mmapped region */ /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ @@ -3154,6 +3157,7 @@ static int unixWrite( int wrote = 0; assert( id ); assert( amt>0 ); + assert( offset>=pFile->mmapSize ); /* Never write into the mmapped region */ /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ @@ -4457,6 +4461,8 @@ static int unixMremap( i64 nOldRnd; /* nOld rounded up */ assert( iOff==0 ); + assert( p->mmapSize==nOld ); + assert( p->pMapRegion==0 || p->pMapRegion==(*ppMap) ); /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested ** mapping (nNew bytes) may be greater than the size of the database file. @@ -4494,6 +4500,7 @@ static int unixMremap( /* On OSX or Linux, reuse the old mapping if it is the right size. */ #if defined(__APPLE__) || defined(__linux__) if( nNewRnd==nOldRnd ){ + VVA_ONLY( p->mmapSize = nNew; ) return SQLITE_OK; } #endif @@ -4502,6 +4509,7 @@ static int unixMremap( if( nOldRnd!=0 ){ void *pOld = *ppMap; munmap(pOld, nOldRnd); + VVA_ONLY( p->mmapSize = 0; p->pMapRegion = 0; ); } /* And, if required, use mmap() to create a new mapping. */ @@ -4511,7 +4519,10 @@ static int unixMremap( pNew = mmap(0, nNewRnd, flags, MAP_SHARED, p->h, iOff); if( pNew==MAP_FAILED ){ pNew = 0; + VVA_ONLY( p->mmapSize = 0; p->pMapRegion = 0; ) rc = SQLITE_IOERR_MREMAP; + }else{ + VVA_ONLY( p->mmapSize = nNew; p->pMapRegion = pNew; ) } } @@ -4846,6 +4857,7 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; + VVA_ONLY( pNew->mmapSize = 0; ) if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; From c00033125d612de5877ea2682cd255dd3199c00c Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 22 Mar 2013 17:46:11 +0000 Subject: [PATCH 014/140] Add a fix for the assert() statements added by the previous commit. FossilOrigin-Name: 19345416ed5e1ab5b0b35993b0b9069c2fb1683b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 10 +++++++++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7a0b83d9c2..1be3bebe19 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sassert\sstatements\sto\sos_unix.c\sto\sensure\sthat\sany\smapped\sregion\sof\sthe\sdatabase\sfile\sis\snot\sbeing\sread\sor\swritten\susing\sthe\sxRead()\sor\sxWrite()\smethods. -D 2013-03-22T08:58:38.319 +C Add\sa\sfix\sfor\sthe\sassert()\sstatements\sadded\sby\sthe\sprevious\scommit. +D 2013-03-22T17:46:11.890 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 18153e6fc03580e95d28fb39d47a4e8b64b36187 +F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 F src/pager.c 520001015155efee9599c807dfd38e5fff9c6e36 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5c9e9df27b9f2c46cd55388a858d4e78ee564975 -R eb2eb83f96de836951dbdb14158ccee8 +P 765615f9fba7c1765eb741cb98a09a28b464ee55 +R 125e508e18383c5f0a5b411a39a78cab U dan -Z 42c0bd3cd106c51e7465866daa07e0a3 +Z 4d1b2a2cebcc3fe5bdd2b5af2205540b diff --git a/manifest.uuid b/manifest.uuid index e0d24b066f..8efe087450 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -765615f9fba7c1765eb741cb98a09a28b464ee55 \ No newline at end of file +19345416ed5e1ab5b0b35993b0b9069c2fb1683b \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 88f3177021..2f03c39670 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3470,6 +3470,14 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ if( pFile->inNormalWrite && nByte==0 ){ pFile->transCntrChng = 1; } + + /* If the file was just truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( nBytemmapSize ){ + pFile->mmapSize = nByte; + } #endif return SQLITE_OK; @@ -4461,7 +4469,7 @@ static int unixMremap( i64 nOldRnd; /* nOld rounded up */ assert( iOff==0 ); - assert( p->mmapSize==nOld ); + /* assert( p->mmapSize==nOld ); */ assert( p->pMapRegion==0 || p->pMapRegion==(*ppMap) ); /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested From 99bd10979abd01877d6c213235c0ad22ab142a8f Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 22 Mar 2013 18:20:14 +0000 Subject: [PATCH 015/140] Allow the database file to be memory mapped in wal mode. FossilOrigin-Name: d190ddabc386bc9654b99e33fb81b2f6e67b54d6 --- manifest | 18 +++++------ manifest.uuid | 2 +- src/pager.c | 88 +++++++++++++++++++++++++++++++++++++++++---------- src/pager.h | 4 +++ src/wal.c | 67 ++++++++++++++++++++------------------- src/wal.h | 7 ++-- 6 files changed, 124 insertions(+), 62 deletions(-) diff --git a/manifest b/manifest index 1be3bebe19..cab9a67764 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sfix\sfor\sthe\sassert()\sstatements\sadded\sby\sthe\sprevious\scommit. -D 2013-03-22T17:46:11.890 +C Allow\sthe\sdatabase\sfile\sto\sbe\smemory\smapped\sin\swal\smode. +D 2013-03-22T18:20:14.143 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,8 +162,8 @@ F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 -F src/pager.c 520001015155efee9599c807dfd38e5fff9c6e36 -F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c +F src/pager.c 12b8ff12519fe529a92884c4cdb99afecb1bea3c +F src/pager.h bbc9170281c9d5d603b2175fdc8ea908e47269a7 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 @@ -249,8 +249,8 @@ F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c c61ca318681c0e7267da8be3abfca8469652a7e9 F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 -F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2 -F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 +F src/wal.c 7fec703bb0770d9faad5190f49897a8bc8eb239f +F src/wal.h d99ce512ac60f9147a0640e9e6fb67dd1057b781 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b F src/where.c bdbbfa7ef4ea04c8d9b09585b45d4717a72f980a F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 765615f9fba7c1765eb741cb98a09a28b464ee55 -R 125e508e18383c5f0a5b411a39a78cab +P 19345416ed5e1ab5b0b35993b0b9069c2fb1683b +R 7243a86b7c03fc5d49450319f9d9b1bf U dan -Z 4d1b2a2cebcc3fe5bdd2b5af2205540b +Z 7b1a30ba76756800b3f32db2afc843bf diff --git a/manifest.uuid b/manifest.uuid index 8efe087450..4f8ecfff16 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -19345416ed5e1ab5b0b35993b0b9069c2fb1683b \ No newline at end of file +d190ddabc386bc9654b99e33fb81b2f6e67b54d6 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 0ff894bd87..660514d508 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2095,7 +2095,7 @@ static void pagerReportSize(Pager *pPager){ ** Return SQLITE_OK if successful, or an SQLite error code if an error ** occurs. */ -static int pagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff){ +int sqlite3PagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff){ int rc = SQLITE_OK; if( pPager->nMapValid>=(iOff+nBuf) ){ memcpy(&((u8 *)(pPager->pMap))[iOff], pBuf, nBuf); @@ -2279,7 +2279,7 @@ static int pager_playback_one_page( i64 ofst = (pgno-1)*(i64)pPager->pageSize; testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); assert( !pagerUseWal(pPager) ); - rc = pagerWriteData(pPager, aData, pPager->pageSize, ofst); + rc = sqlite3PagerWriteData(pPager, aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } @@ -2864,11 +2864,10 @@ end_playback: ** If an IO error occurs, then the IO error is returned to the caller. ** Otherwise, SQLITE_OK is returned. */ -static int readDbPage(PgHdr *pPg){ +static int readDbPage(PgHdr *pPg, u32 iFrame){ Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */ Pgno pgno = pPg->pgno; /* Page number to read */ int rc = SQLITE_OK; /* Return code */ - int isInWal = 0; /* True if page is in log file */ int pgsz = pPager->pageSize; /* Number of bytes to read */ assert( pPager->eState>=PAGER_READER && !MEMDB ); @@ -2880,11 +2879,10 @@ static int readDbPage(PgHdr *pPg){ return SQLITE_OK; } - if( pagerUseWal(pPager) ){ + if( iFrame ){ /* Try to pull the page from the write-ahead log. */ - rc = sqlite3WalRead(pPager->pWal, pgno, &isInWal, pgsz, pPg->pData); - } - if( rc==SQLITE_OK && !isInWal ){ + rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData); + }else{ i64 iOffset = (pgno-1)*(i64)pPager->pageSize; if( pPager->pMap && pPager->nMapValid>=iOffset+pPager->pageSize ){ memcpy(pPg->pData, &((u8 *)(pPager->pMap))[iOffset], pPager->pageSize); @@ -2972,7 +2970,13 @@ static int pagerUndoCallback(void *pCtx, Pgno iPg){ if( sqlite3PcachePageRefcount(pPg)==1 ){ sqlite3PcacheDrop(pPg); }else{ - rc = readDbPage(pPg); + u32 iFrame = 0; + if( pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); + } + if( rc==SQLITE_OK ){ + rc = readDbPage(pPg, iFrame); + } if( rc==SQLITE_OK ){ pPager->xReiniter(pPg); } @@ -3388,7 +3392,6 @@ static void pagerFixMaplimit(Pager *pPager){ || pPager->fd->pMethods->iVersion<3 || pPager->fd->pMethods->xMremap==0 || pPager->tempFile - || pPager->pWal ){ pPager->nMapLimit = 0; }else if( pPager->nMapCfgLimit<0 ){ @@ -3888,9 +3891,9 @@ static int pagerMap(Pager *pPager, int bExtend){ Pgno nPg; /* Size of mapping to request in pages */ i64 sz; /* Size of mapping to request in bytes */ - assert( pPager->pWal==0 && isOpen(pPager->fd) && pPager->tempFile==0 ); + assert( isOpen(pPager->fd) && pPager->tempFile==0 ); assert( pPager->pMap==0 || pPager->nMap>0 ); - assert( pPager->eState>=1 ); + /* assert( pPager->eState>=1 ); */ assert( pPager->nMmapOut==0 ); assert( pPager->nMapLimit>0 ); @@ -4213,6 +4216,46 @@ static int syncJournal(Pager *pPager, int newHdr){ return SQLITE_OK; } +/* +** This is called by the wal.c module at the start of a checkpoint. If the +** checkpoint runs to completion, it will set the database file size to +** szReq bytes. This function performs two tasks: +** +** * If the file is currently less than szReq bytes in size, an +** xFileControl(SQLITE_FNCTL_SIZE_HINT) is issued to inform the OS +** layer of the expected file size, and +** +** * If mmap is being used, then the mapping is extended to szReq +** bytes in size. +** +** SQLITE_OK is returned if successful, or an error code if an error occurs. +*/ +int sqlite3PagerSetFilesize(Pager *pPager, i64 szReq){ + int rc; + sqlite3_int64 sz; + + assert( pPager->eState==PAGER_OPEN ); + assert( pPager->nMmapOut==0 ); + + rc = sqlite3OsFileSize(pPager->fd, &sz); + if( rc==SQLITE_OK ){ + if( sz>szReq ){ + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &sz); + } + } + + if( rc==SQLITE_OK + && pPager->nMapLimit>0 + && pPager->nMapValidnMapValidnMapLimit + ){ + pPager->dbFileSize = (szReq / pPager->pageSize); + rc = pagerMap(pPager, 1); + } + + return rc; +} + /* ** The argument is the first in a linked list of dirty pages connected ** by the PgHdr.pDirty pointer. This function writes each one of the @@ -4275,7 +4318,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ if( pPager->nMmapOut==0 && pPager->nMapLimit>0 ){ pPager->dbFileSize = pPager->dbSize; - pagerMap(pPager, 1); + rc = pagerMap(pPager, 1); } } @@ -4301,7 +4344,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); /* Write out the page data. */ - pagerWriteData(pPager, pData, pPager->pageSize, offset); + rc = sqlite3PagerWriteData(pPager, pData, pPager->pageSize, offset); /* If page 1 was just written, update Pager.dbFileVers to match ** the value now stored in the database file. If writing this @@ -5275,6 +5318,7 @@ int sqlite3PagerAcquire( ){ int rc = SQLITE_OK; PgHdr *pPg = 0; + u32 iFrame = 0; /* Frame to read from WAL file */ const int noContent = (flags & PAGER_ACQUIRE_NOCONTENT); /* It is acceptable to use a read-only (mmap) page for any page except @@ -5287,6 +5331,7 @@ int sqlite3PagerAcquire( assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); + assert( noContent==0 || bMmapOk==0 ); if( pgno==0 ){ return SQLITE_CORRUPT_BKPT; @@ -5298,7 +5343,12 @@ int sqlite3PagerAcquire( rc = pPager->errCode; }else{ - if( bMmapOk ){ + if( bMmapOk && pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } + + if( iFrame==0 && bMmapOk ){ if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){ rc = pagerMap(pPager, 0); } @@ -5378,9 +5428,13 @@ int sqlite3PagerAcquire( memset(pPg->pData, 0, pPager->pageSize); IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ + if( pagerUseWal(pPager) && bMmapOk==0 ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } assert( pPg->pPager==pPager ); pPager->aStat[PAGER_STAT_MISS]++; - rc = readDbPage(pPg); + rc = readDbPage(pPg, iFrame); if( rc!=SQLITE_OK ){ goto pager_acquire_err; } @@ -7063,7 +7117,7 @@ static int pagerOpenWal(Pager *pPager){ ** (e.g. due to malloc() failure), return an error code. */ if( rc==SQLITE_OK ){ - rc = sqlite3WalOpen(pPager->pVfs, + rc = sqlite3WalOpen(pPager->pVfs, pPager, pPager->fd, pPager->zWal, pPager->exclusiveMode, pPager->journalSizeLimit, &pPager->pWal ); diff --git a/src/pager.h b/src/pager.h index 81ab30c115..970b5035bf 100644 --- a/src/pager.h +++ b/src/pager.h @@ -174,6 +174,10 @@ int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); +int sqlite3PagerSetFilesize(Pager *, i64); + +/* Write data to the database file */ +int sqlite3PagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff); #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); diff --git a/src/wal.c b/src/wal.c index 0d7271bf04..8757003d78 100644 --- a/src/wal.c +++ b/src/wal.c @@ -412,6 +412,7 @@ struct Wal { sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */ sqlite3_file *pDbFd; /* File handle for the database file */ sqlite3_file *pWalFd; /* File handle for WAL file */ + Pager *pPager; /* Pager object */ u32 iCallback; /* Value to pass to log callback (or 0) */ i64 mxWalSize; /* Truncate WAL to this size upon reset */ int nWiData; /* Size of array apWiData */ @@ -1251,6 +1252,7 @@ static void walIndexClose(Wal *pWal, int isDelete){ */ int sqlite3WalOpen( sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */ + Pager *pPager, /* Pager object handle */ sqlite3_file *pDbFd, /* The open database file */ const char *zWalName, /* Name of the WAL file */ int bNoShm, /* True to run in heap-memory mode */ @@ -1291,6 +1293,7 @@ int sqlite3WalOpen( pRet->zWalName = zWalName; pRet->syncHeader = 1; pRet->padToSectorBoundary = 1; + pRet->pPager = pPager; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); /* Open file handle on the write-ahead log file. */ @@ -1727,10 +1730,7 @@ static int walCheckpoint( */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); - rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); - if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); - } + rc = sqlite3PagerSetFilesize(pWal->pPager, nReq); } /* Iterate through the contents of the WAL, copying data to the db file. */ @@ -1744,7 +1744,7 @@ static int walCheckpoint( if( rc!=SQLITE_OK ) break; iOffset = (iDbpage-1)*(i64)szPage; testcase( IS_BIG_INT(iOffset) ); - rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); + rc = sqlite3PagerWriteData(pWal->pPager, zBuf, szPage, iOffset); if( rc!=SQLITE_OK ) break; } @@ -2287,19 +2287,17 @@ void sqlite3WalEndReadTransaction(Wal *pWal){ } /* -** Read a page from the WAL, if it is present in the WAL and if the -** current read transaction is configured to use the WAL. +** Search the wal file for page pgno. If found, set *piRead to the frame that +** contains the page. Otherwise, if pgno is not in the wal file, set *piRead +** to zero. ** -** The *pInWal is set to 1 if the requested page is in the WAL and -** has been loaded. Or *pInWal is set to 0 if the page was not in -** the WAL and needs to be read out of the database. +** Return SQLITE_OK if successful, or an error code if an error occurs. If an +** error does occur, the final value of *piRead is undefined. */ -int sqlite3WalRead( +int sqlite3WalFindFrame( Wal *pWal, /* WAL handle */ Pgno pgno, /* Database page number to read data for */ - int *pInWal, /* OUT: True if data is read from WAL */ - int nOut, /* Size of buffer pOut in bytes */ - u8 *pOut /* Buffer to write page data to */ + u32 *piRead /* OUT: Frame number (or zero) */ ){ u32 iRead = 0; /* If !=0, WAL frame to return data from */ u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ @@ -2315,7 +2313,7 @@ int sqlite3WalRead( ** WAL were empty. */ if( iLast==0 || pWal->readLock==0 ){ - *pInWal = 0; + *piRead = 0; return SQLITE_OK; } @@ -2386,26 +2384,31 @@ int sqlite3WalRead( } #endif - /* If iRead is non-zero, then it is the log frame number that contains the - ** required page. Read and return data from the log file. - */ - if( iRead ){ - int sz; - i64 iOffset; - sz = pWal->hdr.szPage; - sz = (sz&0xfe00) + ((sz&0x0001)<<16); - testcase( sz<=32768 ); - testcase( sz>=65536 ); - iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; - *pInWal = 1; - /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ - return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); - } - - *pInWal = 0; + *piRead = iRead; return SQLITE_OK; } +/* +** Read the contents of frame iRead from the wal file into buffer pOut +** (which is nOut bytes in size). Return SQLITE_OK if successful, or an +** error code otherwise. +*/ +int sqlite3WalReadFrame( + Wal *pWal, /* WAL handle */ + u32 iRead, /* Frame to read */ + int nOut, /* Size of buffer pOut in bytes */ + u8 *pOut /* Buffer to write page data to */ +){ + int sz; + i64 iOffset; + sz = pWal->hdr.szPage; + sz = (sz&0xfe00) + ((sz&0x0001)<<16); + testcase( sz<=32768 ); + testcase( sz>=65536 ); + iOffset = walFrameOffset(iRead, sz) + WAL_FRAME_HDRSIZE; + /* testcase( IS_BIG_INT(iOffset) ); // requires a 4GiB WAL */ + return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); +} /* ** Return the size of the database in pages (or zero, if unknown). diff --git a/src/wal.h b/src/wal.h index a848de1527..96d4ff9adf 100644 --- a/src/wal.h +++ b/src/wal.h @@ -31,7 +31,6 @@ # define sqlite3WalClose(w,x,y,z) 0 # define sqlite3WalBeginReadTransaction(y,z) 0 # define sqlite3WalEndReadTransaction(z) -# define sqlite3WalRead(v,w,x,y,z) 0 # define sqlite3WalDbsize(y) 0 # define sqlite3WalBeginWriteTransaction(y) 0 # define sqlite3WalEndWriteTransaction(x) 0 @@ -54,7 +53,8 @@ typedef struct Wal Wal; /* Open and close a connection to a write-ahead log. */ -int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**); +int sqlite3WalOpen( + sqlite3_vfs*, Pager *, sqlite3_file*, const char *, int, i64, Wal**); int sqlite3WalClose(Wal *pWal, int sync_flags, int, u8 *); /* Set the limiting size of a WAL file. */ @@ -71,7 +71,8 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *); void sqlite3WalEndReadTransaction(Wal *pWal); /* Read a page from the write-ahead log, if it is present. */ -int sqlite3WalRead(Wal *pWal, Pgno pgno, int *pInWal, int nOut, u8 *pOut); +int sqlite3WalFindFrame(Wal *, Pgno, u32 *); +int sqlite3WalReadFrame(Wal *, u32, int, u8 *); /* If the WAL is not empty, return the size of the database. */ Pgno sqlite3WalDbsize(Wal *pWal); From aa1d67b1d92564e8360a07ff17c933d513542661 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 22 Mar 2013 19:17:45 +0000 Subject: [PATCH 016/140] Fix a case where the mapped part of a database file may be written during a backup operation. FossilOrigin-Name: c8d67aefff7299dd5c8abeb2a3a52109c0a823ee --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/backup.c | 2 +- test/permutations.test | 2 +- test/speed1p.test | 3 +++ 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index cab9a67764..cb648b797f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sthe\sdatabase\sfile\sto\sbe\smemory\smapped\sin\swal\smode. -D 2013-03-22T18:20:14.143 +C Fix\sa\scase\swhere\sthe\smapped\spart\sof\sa\sdatabase\sfile\smay\sbe\swritten\sduring\sa\sbackup\soperation. +D 2013-03-22T19:17:45.777 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -118,7 +118,7 @@ F src/alter.c f8db986c03eb0bfb221523fc9bbb9d0b70de3168 F src/analyze.c d5f895810e8ff9737c9ec7b76abc3dcff5860335 F src/attach.c ea5247f240e2c08afd608e9beb380814b86655e1 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 -F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 +F src/backup.c 6256400ab8be4a15a2512277d8b214dbd44ff8ce F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c dd4683fa671f0b2b4bc0831036adaa2a5d1806ee @@ -657,7 +657,7 @@ F test/pageropt.test 290cd59782b1890f02bb33795571facfc5ccac43 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test 694f4a2667242bab49cce05c54c2adfcc2727d9e +F test/permutations.test 4381ffc2811884cb4c2063176e4e4a849f236adc F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -730,7 +730,7 @@ F test/softheap1.test c16709a16ad79fa43b32929b2e623d1d117ccf53 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb -F test/speed1p.test c4a469f29f135f4d76c55b1f2a52f36e209466cc +F test/speed1p.test 2577211a9b054ffc973780f5b8708f02a0c7422d F test/speed2.test 53177056baf6556dcbdcf032bbdfc41c1aa74ded F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 19345416ed5e1ab5b0b35993b0b9069c2fb1683b -R 7243a86b7c03fc5d49450319f9d9b1bf +P d190ddabc386bc9654b99e33fb81b2f6e67b54d6 +R d5a852613692dd4e943f0ee647615880 U dan -Z 7b1a30ba76756800b3f32db2afc843bf +Z db771a6d89978ace4423b73d7feafccd diff --git a/manifest.uuid b/manifest.uuid index 4f8ecfff16..caccf2eff1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d190ddabc386bc9654b99e33fb81b2f6e67b54d6 \ No newline at end of file +c8d67aefff7299dd5c8abeb2a3a52109c0a823ee \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 71a8a1a3e7..8939bc67bc 100644 --- a/src/backup.c +++ b/src/backup.c @@ -520,7 +520,7 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); if( rc==SQLITE_OK ){ u8 *zData = sqlite3PagerGetData(pSrcPg); - rc = sqlite3OsWrite(pFile, zData, pgszSrc, iOff); + rc = sqlite3PagerWriteData(pDestPager, zData, pgszSrc, iOff); } sqlite3PagerUnref(pSrcPg); } diff --git a/test/permutations.test b/test/permutations.test index fd05b58e4f..1015066e84 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -144,7 +144,7 @@ test_suite "mmap" -prefix "mm-" -description { pragma mmap_size = -65536; } -files [ test_set $allquicktests -exclude *malloc* *ioerr* *fault* \ - multiplex* server1.test shared2.test shared6.test + multiplex* server1.test shared2.test shared6.test ] test_suite "valgrind" -prefix "" -description { diff --git a/test/speed1p.test b/test/speed1p.test index 915f165354..33be553b00 100644 --- a/test/speed1p.test +++ b/test/speed1p.test @@ -65,6 +65,9 @@ proc number_name {n} { # do_test speed1p-1.0 { execsql { + PRAGMA mmap_size=65536; + PRAGMA journal_mode=wal; + PRAGMA page_size=1024; PRAGMA cache_size=500; PRAGMA locking_mode=EXCLUSIVE; From 23f29ec6284aaa01efdc2abfda257b10139670c8 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 22 Mar 2013 19:56:27 +0000 Subject: [PATCH 017/140] Fix a potential NULL-pointer dereference in btreeSwapOutMmap(). FossilOrigin-Name: e81ccdcd87a1387f7ca08d4d7c899311e2180d32 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index cb648b797f..0a2be06535 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scase\swhere\sthe\smapped\spart\sof\sa\sdatabase\sfile\smay\sbe\swritten\sduring\sa\sbackup\soperation. -D 2013-03-22T19:17:45.777 +C Fix\sa\spotential\sNULL-pointer\sdereference\sin\sbtreeSwapOutMmap(). +D 2013-03-22T19:56:27.789 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 6256400ab8be4a15a2512277d8b214dbd44ff8ce F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c dd4683fa671f0b2b4bc0831036adaa2a5d1806ee +F src/btree.c f5ea8d3d658887b5cae0369ef10a427d7469a768 F src/btree.h d3259057a38494c4c4358e377032158c762e3d8b F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 375e5df716e03b9343c5e1211be3b24e6d6dff05 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d190ddabc386bc9654b99e33fb81b2f6e67b54d6 -R d5a852613692dd4e943f0ee647615880 -U dan -Z db771a6d89978ace4423b73d7feafccd +P c8d67aefff7299dd5c8abeb2a3a52109c0a823ee +R dc31f859ef6cd2199f40f19f84f2b02e +U drh +Z 7f958188506961f9abc2573e8dff7118 diff --git a/manifest.uuid b/manifest.uuid index caccf2eff1..6c200497cb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c8d67aefff7299dd5c8abeb2a3a52109c0a823ee \ No newline at end of file +e81ccdcd87a1387f7ca08d4d7c899311e2180d32 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 0a51ff0301..6a9761e91a 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2593,7 +2593,7 @@ static int btreeSwapOutMmap(BtShared *pBt){ for(pCsr=pBt->pCursor; pCsr && rc==SQLITE_OK; pCsr=pCsr->pNext){ if( pCsr->iPage>=0 ){ MemPage *pPg = pCsr->apPage[0]; - if( pPg->pDbPage->flags & PGHDR_MMAP ){ + if( pPg && pPg->pDbPage->flags & PGHDR_MMAP ){ MemPage *pNew = 0; rc = getAndInitPage(pBt, pPg->pgno, &pNew, 0); if( rc==SQLITE_OK && pCsr->iPage==0 ){ From 7909e54a22a0b466e5990d63a1c01de567deeb8f Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 22 Mar 2013 20:15:31 +0000 Subject: [PATCH 018/140] Update wal mode tests so that they work with the mmap test permutation. FossilOrigin-Name: f7295872a36539d10edaf0c633f935f25cf73657 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- test/permutations.test | 3 +-- test/wal.test | 8 +++++--- test/wal5.test | 10 +++++++++- 5 files changed, 25 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 0a2be06535..868c9e0173 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\spotential\sNULL-pointer\sdereference\sin\sbtreeSwapOutMmap(). -D 2013-03-22T19:56:27.789 +C Update\swal\smode\stests\sso\sthat\sthey\swork\swith\sthe\smmap\stest\spermutation. +D 2013-03-22T20:15:31.365 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -657,7 +657,7 @@ F test/pageropt.test 290cd59782b1890f02bb33795571facfc5ccac43 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test 4381ffc2811884cb4c2063176e4e4a849f236adc +F test/permutations.test cf5d475f024cfd3fbb0255b8e43bf0c6887c6bb4 F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -950,11 +950,11 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 -F test/wal.test a040047d7f2b9f34bc4d597964e5e7c09609c635 +F test/wal.test 5baa9a0e682c0394012aa9a06a3d06cf92655fbf F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c -F test/wal5.test f58ed4b8b542f71c7441da12fbd769d99b362437 +F test/wal5.test 091bf7500e97087f1a87b346c8707e93f42d4f80 F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test b3ee739fe8f7586aaebdc2367f477ebcf3e3b034 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c8d67aefff7299dd5c8abeb2a3a52109c0a823ee -R dc31f859ef6cd2199f40f19f84f2b02e -U drh -Z 7f958188506961f9abc2573e8dff7118 +P e81ccdcd87a1387f7ca08d4d7c899311e2180d32 +R f5bd57e0d254253276d114325fd2ea6d +U dan +Z 0c2e4f926a5fed4f3d574bc2f3c43e20 diff --git a/manifest.uuid b/manifest.uuid index 6c200497cb..a8b241dcad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e81ccdcd87a1387f7ca08d4d7c899311e2180d32 \ No newline at end of file +f7295872a36539d10edaf0c633f935f25cf73657 \ No newline at end of file diff --git a/test/permutations.test b/test/permutations.test index 1015066e84..6e6b5c90f7 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -143,8 +143,7 @@ test_suite "mmap" -prefix "mm-" -description { } -presql { pragma mmap_size = -65536; } -files [ - test_set $allquicktests -exclude *malloc* *ioerr* *fault* \ - multiplex* server1.test shared2.test shared6.test + test_set $allquicktests -exclude *malloc* *ioerr* *fault* ] test_suite "valgrind" -prefix "" -description { diff --git a/test/wal.test b/test/wal.test index 24ce5f86a3..ca7edef952 100644 --- a/test/wal.test +++ b/test/wal.test @@ -727,6 +727,8 @@ do_test wal-11.9 { list [expr [file size test.db]/1024] [log_deleted test.db-wal] } {37 1} sqlite3_wal db test.db +set nWal 37 +if {[permutation]=="mmap"} {set nWal 39} do_test wal-11.10 { execsql { PRAGMA cache_size = 10; @@ -735,7 +737,7 @@ do_test wal-11.10 { SELECT count(*) FROM t1; } list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [wal_file_size 37 1024]] +} [list 37 [wal_file_size $nWal 1024]] do_test wal-11.11 { execsql { SELECT count(*) FROM t1; @@ -745,7 +747,7 @@ do_test wal-11.11 { } {32 16} do_test wal-11.12 { list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [wal_file_size 37 1024]] +} [list 37 [wal_file_size $nWal 1024]] do_test wal-11.13 { execsql { INSERT INTO t1 VALUES( blob(900) ); @@ -755,7 +757,7 @@ do_test wal-11.13 { } {17 ok} do_test wal-11.14 { list [expr [file size test.db]/1024] [file size test.db-wal] -} [list 37 [wal_file_size 37 1024]] +} [list 37 [wal_file_size $nWal 1024]] #------------------------------------------------------------------------- diff --git a/test/wal5.test b/test/wal5.test index 6eceed5e59..d8bf541a4b 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -235,7 +235,15 @@ foreach {testprefix do_wal_checkpoint} { do_test 2.3.$tn.5 { sql1 { INSERT INTO t2 VALUES(3, 4) } } {} do_test 2.3.$tn.6 { file_page_counts } {1 4 1 4} do_test 2.3.$tn.7 { code1 { do_wal_checkpoint db -mode full } } {1 4 3} - do_test 2.3.$tn.8 { file_page_counts } {1 4 2 4} + + # The checkpoint above only writes page 1 of the db file. The other + # page (page 2) is locked by the read-transaction opened by the + # [sql2] commmand above. So normally, the db is 1 page in size here. + # However, in mmap() mode, the db is pre-allocated to 2 pages at the + # start of the checkpoint, even though page 2 cannot be written. + set nDb 1 + if {[permutation]=="mmap"} {set nDb 2} + do_test 2.3.$tn.8 { file_page_counts } [list $nDb 4 2 4] } # Check that checkpoints block on the correct locks. And respond correctly From daf9a5a402fdcd69fce93e84bc61b46dcf9803e2 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 23 Mar 2013 09:56:39 +0000 Subject: [PATCH 019/140] Preliminary changes to support the xMremap VFS method on Windows. FossilOrigin-Name: 75a85a1c6ac59f2d021c28fa2161afb1118deea4 --- manifest | 16 ++-- manifest.uuid | 2 +- src/os_win.c | 201 +++++++++++++++++++++++++++++++++++++++++++++++--- src/test1.c | 4 + 4 files changed, 203 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 868c9e0173..b81d7d1140 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\swal\smode\stests\sso\sthat\sthey\swork\swith\sthe\smmap\stest\spermutation. -D 2013-03-22T20:15:31.365 +C Preliminary\schanges\sto\ssupport\sthe\sxMremap\sVFS\smethod\son\sWindows. +D 2013-03-23T09:56:39.403 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf -F src/os_win.c f7da4dc0a2545c0a430080380809946ae4d676d6 +F src/os_win.c 46d2bbad7a65d2867ec0e9afb4aa28478402e9dc F src/pager.c 12b8ff12519fe529a92884c4cdb99afecb1bea3c F src/pager.h bbc9170281c9d5d603b2175fdc8ea908e47269a7 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -184,7 +184,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 3213f3101e3b85f047d6e389da5a53d76d3d7540 -F src/test1.c 39378e3e14a8162e29dc90d1e05399d12e8a569e +F src/test1.c 26e66b839f42c2eed6833f9023e0098f0d863f35 F src/test2.c 4178056dd1e7d70f954ad8a1e3edb71a2a784daf F src/test3.c 3c3c2407fa6ec7a19e24ae23f7cb439d0275a60d F src/test4.c bf9fa9bece01de08e6f5e02314e4af5c13590dfa @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e81ccdcd87a1387f7ca08d4d7c899311e2180d32 -R f5bd57e0d254253276d114325fd2ea6d -U dan -Z 0c2e4f926a5fed4f3d574bc2f3c43e20 +P f7295872a36539d10edaf0c633f935f25cf73657 +R 33ae184619389f0a2e2f45e36bcba41c +U mistachkin +Z 7a5b6171912da74b28162981ae14cd6f diff --git a/manifest.uuid b/manifest.uuid index a8b241dcad..ade491dce4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f7295872a36539d10edaf0c633f935f25cf73657 \ No newline at end of file +75a85a1c6ac59f2d021c28fa2161afb1118deea4 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 970a94b5e8..fd0c8aeb89 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -150,11 +150,15 @@ struct winFile { winceLock local; /* Locks obtained by this instance of winFile */ winceLock *shared; /* Global shared lock memory for the file */ #endif + HANDLE hMap; /* Handle for accessing memory mapping */ + void *pMapRegion; /* Area memory mapped */ + sqlite3_int64 mmapSize; /* Size of xMremap() */ }; /* ** Allowed values for winFile.ctrlFlags */ +#define WINFILE_RDONLY 0x02 /* Connection is read only */ #define WINFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ #define WINFILE_PSOW 0x10 /* SQLITE_IOCAP_POWERSAFE_OVERWRITE */ @@ -2061,6 +2065,9 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ #endif } +/* Forward references to VFS methods */ +static int winUnmap(sqlite3_file *); + /* ** Close a file. ** @@ -2082,6 +2089,10 @@ static int winClose(sqlite3_file *id){ #endif OSTRACE(("CLOSE %d\n", pFile->h)); assert( pFile->h!=NULL && pFile->h!=INVALID_HANDLE_VALUE ); + + rc = winUnmap(id); + if( rc!=SQLITE_OK ) return rc; + do{ rc = osCloseHandle(pFile->h); /* SimulateIOError( rc=0; cnt=MX_CLOSE_ATTEMPT; ); */ @@ -2130,6 +2141,8 @@ static int winRead( int nRetry = 0; /* Number of retrys */ assert( id!=0 ); + assert( amt>0 ); + assert( offset>=pFile->mmapSize ); /* Never read from the mmapped region */ SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); @@ -2176,6 +2189,7 @@ static int winWrite( int nRetry = 0; /* Number of retries */ assert( amt>0 ); + assert( offset>=pFile->mmapSize ); /* Never write into the mmapped region */ assert( pFile ); SimulateIOError(return SQLITE_IOERR_WRITE); SimulateDiskfullError(return SQLITE_FULL); @@ -2249,12 +2263,16 @@ static int winWrite( static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ winFile *pFile = (winFile*)id; /* File handle object */ int rc = SQLITE_OK; /* Return code for this function */ + DWORD lastErrno; assert( pFile ); OSTRACE(("TRUNCATE %d %lld\n", pFile->h, nByte)); SimulateIOError(return SQLITE_IOERR_TRUNCATE); + rc = winUnmap(id); + if( rc!=SQLITE_OK ) return rc; + /* If the user has configured a chunk-size for this file, truncate the ** file so that it consists of an integer number of chunks (i.e. the ** actual file size after the operation may be larger than the requested @@ -2267,11 +2285,20 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ if( seekWinFile(pFile, nByte) ){ rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, - "winTruncate1", pFile->zPath); - }else if( 0==osSetEndOfFile(pFile->h) ){ - pFile->lastErrno = osGetLastError(); + "winTruncate1", pFile->zPath); + }else if( 0==osSetEndOfFile(pFile->h) && + ((lastErrno = osGetLastError())!=ERROR_USER_MAPPED_FILE) ){ + pFile->lastErrno = lastErrno; rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, - "winTruncate2", pFile->zPath); + "winTruncate2", pFile->zPath); + }else{ + /* If the file was just truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( pFile->pMapRegion && nBytemmapSize ){ + pFile->mmapSize = nByte; + } } OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); @@ -3451,6 +3478,154 @@ shmpage_out: # define winShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ +/* +** Arguments x and y are both integers. Argument y must be a power of 2. +** Round x up to the nearest integer multiple of y. For example: +** +** ROUNDUP(0, 8) -> 0 +** ROUNDUP(13, 8) -> 16 +** ROUNDUP(32, 8) -> 32 +*/ +#define ROUNDUP(x,y) (((x)+y-1)&~(y-1)) + +/* +** Cleans up the mapped region of the specified file, if any. +*/ +static int winUnmap( + sqlite3_file *id +){ + winFile *pFile = (winFile*)id; + assert( pFile!=0 ); + if( pFile->pMapRegion ){ + if( !osUnmapViewOfFile(pFile->pMapRegion) ){ + pFile->lastErrno = osGetLastError(); + return winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, + "winUnmap1", pFile->zPath); + } + pFile->pMapRegion = 0; + pFile->mmapSize = 0; + } + if( pFile->hMap!=NULL ){ + if( !osCloseHandle(pFile->hMap) ){ + pFile->lastErrno = osGetLastError(); + return winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, + "winUnmap2", pFile->zPath); + } + pFile->hMap = NULL; + } + return SQLITE_OK; +} + +/* +** Map, remap or unmap part of the database file. +*/ +static int winMremap( + sqlite3_file *id, /* Main database file */ + int flags, /* Mask of SQLITE_MREMAP_XXX flags */ + sqlite3_int64 iOff, /* Offset to start mapping at */ + sqlite3_int64 nOld, /* Size of old mapping, or zero */ + sqlite3_int64 nNew, /* Size of new mapping, or zero */ + void **ppMap /* IN/OUT: Old/new mappings */ +){ + winFile *pFile = (winFile*)id; /* The underlying database file */ + int rc = SQLITE_OK; /* Return code */ + HANDLE hMap = NULL; /* New mapping handle */ + void *pNew = 0; /* New mapping */ + i64 nNewRnd; /* nNew rounded up */ + i64 nOldRnd; /* nOld rounded up */ + + assert( pFile!=0 ); + assert( iOff==0 ); + assert( nOld>=0 ); + assert( nOld==0 || pFile->pMapRegion==(*ppMap) ); + assert( nNew>=0 ); + assert( ppMap ); + assert( pFile->hMap==NULL || pFile->pMapRegion==(*ppMap) ); + assert( pFile->pMapRegion==0 || pFile->pMapRegion==(*ppMap) ); + /* assert( pFile->mmapSize==nOld ); */ + + assert( winSysInfo.dwPageSize>0 ); + nNewRnd = ROUNDUP(nNew, winSysInfo.dwPageSize*1); + assert( nNewRnd>=0 ); + nOldRnd = ROUNDUP(nOld, winSysInfo.dwPageSize*1); + assert( nOldRnd>=0 ); + + if( nNewRnd==nOldRnd ){ + pFile->mmapSize = nNew; + return SQLITE_OK; + } + + /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested + ** mapping (nNew bytes) may be greater than the size of the database file. + ** If this is the case, extend the file on disk using ftruncate(). */ + assert( nNewRnd>0 || (flags & SQLITE_MREMAP_EXTEND)==0 ); + if( flags & SQLITE_MREMAP_EXTEND ){ + sqlite3_int64 oldSz; + rc = winFileSize(id, &oldSz); + if( rc==SQLITE_OK && nNewRnd>oldSz ){ + rc = winTruncate(id, nNewRnd); + } + if( rc!=SQLITE_OK ) return rc; + } + + /* If we get this far, unmap any old mapping. */ + rc = winUnmap(id); + if( rc!=SQLITE_OK ) return rc; + + /* And, if required, create a new mapping. */ + if( nNewRnd>0 ){ + i64 offset = ((iOff / winSysInfo.dwAllocationGranularity) * + (winSysInfo.dwAllocationGranularity)); + DWORD protect = PAGE_READONLY; + DWORD flags = FILE_MAP_READ; + if( (pFile->ctrlFlags & WINFILE_RDONLY)==0 ){ + protect = PAGE_READWRITE; + flags |= FILE_MAP_WRITE; + } +#if SQLITE_OS_WINRT + hMap = osCreateFileMappingFromApp(pFile->h, NULL, protect, nNewRnd, NULL); +#elif defined(SQLITE_WIN32_HAS_WIDE) + hMap = osCreateFileMappingW(pFile->h, NULL, protect, + (DWORD)((nNewRnd>>32) & 0xffffffff), + (DWORD)(nNewRnd & 0xffffffff), NULL); +#elif defined(SQLITE_WIN32_HAS_ANSI) + hMap = osCreateFileMappingA(pFile->h, NULL, protect, + (DWORD)((nNewRnd>>32) & 0xffffffff), + (DWORD)(nNewRnd & 0xffffffff), NULL); +#endif + if( hMap==NULL ){ + pFile->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, + "winMremap1", pFile->zPath); + return rc; + } + assert( (nNewRnd % winSysInfo.dwPageSize)==0 ); +#if SQLITE_OS_WINRT + pNew = osMapViewOfFileFromApp(hMap, flags, offset, nNewRnd); +#else + assert( sizeof(SIZE_T)==sizeof(sqlite3_int64) || nNewRnd<=0xffffffff ); + pNew = osMapViewOfFile(hMap, flags, + (DWORD)((offset>>32) & 0xffffffff), + (DWORD)(offset & 0xffffffff), + (SIZE_T)nNewRnd); +#endif + if( pNew==NULL ){ + osCloseHandle(hMap); + hMap = NULL; + pFile->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, + "winMremap2", pFile->zPath); + } + } + + pFile->hMap = hMap; + pFile->pMapRegion = pNew; + pFile->mmapSize = nNew; + + *ppMap = pNew; + return rc; +} + /* ** Here ends the implementation of all sqlite3_file methods. ** @@ -3462,7 +3637,7 @@ shmpage_out: ** sqlite3_file for win32. */ static const sqlite3_io_methods winIoMethod = { - 2, /* iVersion */ + 3, /* iVersion */ winClose, /* xClose */ winRead, /* xRead */ winWrite, /* xWrite */ @@ -3478,7 +3653,8 @@ static const sqlite3_io_methods winIoMethod = { winShmMap, /* xShmMap */ winShmLock, /* xShmLock */ winShmBarrier, /* xShmBarrier */ - winShmUnmap /* xShmUnmap */ + winShmUnmap, /* xShmUnmap */ + winMremap, /* xMremap */ }; /**************************************************************************** @@ -3654,9 +3830,7 @@ static int winOpen( int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); int isDelete = (flags & SQLITE_OPEN_DELETEONCLOSE); int isCreate = (flags & SQLITE_OPEN_CREATE); -#ifndef NDEBUG int isReadonly = (flags & SQLITE_OPEN_READONLY); -#endif int isReadWrite = (flags & SQLITE_OPEN_READWRITE); #ifndef NDEBUG @@ -3867,11 +4041,17 @@ static int winOpen( pFile->pMethod = &winIoMethod; pFile->pVfs = pVfs; pFile->h = h; + if( isReadonly ){ + pFile->ctrlFlags |= WINFILE_RDONLY; + } if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pFile->ctrlFlags |= WINFILE_PSOW; } pFile->lastErrno = NO_ERROR; pFile->zPath = zName; + pFile->hMap = NULL; + pFile->pMapRegion = 0; + pFile->mmapSize = 0; OpenCounter(+1); return rc; @@ -4500,7 +4680,6 @@ int sqlite3_os_init(void){ ** correctly. See ticket [bb3a86e890c8e96ab] */ assert( ArraySize(aSyscall)==74 ); -#ifndef SQLITE_OMIT_WAL /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); #if SQLITE_OS_WINRT @@ -4508,8 +4687,8 @@ int sqlite3_os_init(void){ #else osGetSystemInfo(&winSysInfo); #endif - assert(winSysInfo.dwAllocationGranularity > 0); -#endif + assert( winSysInfo.dwAllocationGranularity>0 ); + assert( winSysInfo.dwPageSize>0 ); sqlite3_vfs_register(&winVfs, 1); return SQLITE_OK; diff --git a/src/test1.c b/src/test1.c index 500d60e0fe..fb340f92d0 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5844,6 +5844,7 @@ static int test_test_control( return TCL_OK; } +#if SQLITE_OS_UNIX #include #include @@ -5866,6 +5867,7 @@ static int test_getrusage( Tcl_SetObjResult(interp, Tcl_NewStringObj(buf, -1)); return TCL_OK; } +#endif #if SQLITE_OS_WIN /* @@ -6256,7 +6258,9 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "print_explain_query_plan", test_print_eqp, 0 }, #endif { "sqlite3_test_control", test_test_control }, +#if SQLITE_OS_UNIX { "getrusage", test_getrusage }, +#endif }; static int bitmask_size = sizeof(Bitmask)*8; int i; From 46fce002d9ed09dc1db71678e73eb1336de96f5b Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 23 Mar 2013 10:09:46 +0000 Subject: [PATCH 020/140] Do not unmap the mapped file region in winTruncate as there may be outstanding pointers to it. FossilOrigin-Name: f57a9c91e993f76ce8b923e06e721414980e8e61 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 3 --- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index b81d7d1140..a39a1c5070 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Preliminary\schanges\sto\ssupport\sthe\sxMremap\sVFS\smethod\son\sWindows. -D 2013-03-23T09:56:39.403 +C Do\snot\sunmap\sthe\smapped\sfile\sregion\sin\swinTruncate\sas\sthere\smay\sbe\soutstanding\spointers\sto\sit. +D 2013-03-23T10:09:46.701 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf -F src/os_win.c 46d2bbad7a65d2867ec0e9afb4aa28478402e9dc +F src/os_win.c 8f7de0046d5107be715aa91d7deebbf91d577644 F src/pager.c 12b8ff12519fe529a92884c4cdb99afecb1bea3c F src/pager.h bbc9170281c9d5d603b2175fdc8ea908e47269a7 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f7295872a36539d10edaf0c633f935f25cf73657 -R 33ae184619389f0a2e2f45e36bcba41c +P 75a85a1c6ac59f2d021c28fa2161afb1118deea4 +R 2ac6e41fa14b2fcb69f496caa13f7e10 U mistachkin -Z 7a5b6171912da74b28162981ae14cd6f +Z 3096704ce1da72ac81b0db8561977d4e diff --git a/manifest.uuid b/manifest.uuid index ade491dce4..18b01de7dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -75a85a1c6ac59f2d021c28fa2161afb1118deea4 \ No newline at end of file +f57a9c91e993f76ce8b923e06e721414980e8e61 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index fd0c8aeb89..8d58ab0d44 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2270,9 +2270,6 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ OSTRACE(("TRUNCATE %d %lld\n", pFile->h, nByte)); SimulateIOError(return SQLITE_IOERR_TRUNCATE); - rc = winUnmap(id); - if( rc!=SQLITE_OK ) return rc; - /* If the user has configured a chunk-size for this file, truncate the ** file so that it consists of an integer number of chunks (i.e. the ** actual file size after the operation may be larger than the requested From 2753388e8ae49d71e7b3edc71793950aa002c07d Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 23 Mar 2013 12:15:27 +0000 Subject: [PATCH 021/140] In the winMremap VFS function, unmap the region prior to attempting to truncate the file. FossilOrigin-Name: 8870c4cc6cda07154936644f7df25bd81d6ba38a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a39a1c5070..ada3b7db89 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sunmap\sthe\smapped\sfile\sregion\sin\swinTruncate\sas\sthere\smay\sbe\soutstanding\spointers\sto\sit. -D 2013-03-23T10:09:46.701 +C In\sthe\swinMremap\sVFS\sfunction,\sunmap\sthe\sregion\sprior\sto\sattempting\sto\struncate\sthe\sfile. +D 2013-03-23T12:15:27.020 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf -F src/os_win.c 8f7de0046d5107be715aa91d7deebbf91d577644 +F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac F src/pager.c 12b8ff12519fe529a92884c4cdb99afecb1bea3c F src/pager.h bbc9170281c9d5d603b2175fdc8ea908e47269a7 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 75a85a1c6ac59f2d021c28fa2161afb1118deea4 -R 2ac6e41fa14b2fcb69f496caa13f7e10 +P f57a9c91e993f76ce8b923e06e721414980e8e61 +R a68c203c916fb468751938d0762a7812 U mistachkin -Z 3096704ce1da72ac81b0db8561977d4e +Z c2b92e0c0a8c400cb287f5f47ed82e78 diff --git a/manifest.uuid b/manifest.uuid index 18b01de7dd..0e0a52b9c3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f57a9c91e993f76ce8b923e06e721414980e8e61 \ No newline at end of file +8870c4cc6cda07154936644f7df25bd81d6ba38a \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 8d58ab0d44..8242a33849 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3560,7 +3560,10 @@ static int winMremap( sqlite3_int64 oldSz; rc = winFileSize(id, &oldSz); if( rc==SQLITE_OK && nNewRnd>oldSz ){ - rc = winTruncate(id, nNewRnd); + rc = winUnmap(id); /* Cannot truncate the file with an open mapping. */ + if( rc==SQLITE_OK ){ + rc = winTruncate(id, nNewRnd); + } } if( rc!=SQLITE_OK ) return rc; } From 9d56c6df9aef1e9cb59246b010c55407ea1ab73b Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 23 Mar 2013 14:20:42 +0000 Subject: [PATCH 022/140] Fix a case where a checkpoint operation could write to an invalid part of a memory mapped region. FossilOrigin-Name: 8dbe89d05ce91428c69003f0da79d883fa23e2b5 --- manifest | 18 +++++++-------- manifest.uuid | 2 +- src/pager.c | 16 ++++++------- test/mmap1.test | 57 +++++++++++++++++++++++++++++++++++++++++++---- test/speed1p.test | 3 --- 5 files changed, 71 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index ada3b7db89..cf3732c267 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\swinMremap\sVFS\sfunction,\sunmap\sthe\sregion\sprior\sto\sattempting\sto\struncate\sthe\sfile. -D 2013-03-23T12:15:27.020 +C Fix\sa\scase\swhere\sa\scheckpoint\soperation\scould\swrite\sto\san\sinvalid\spart\sof\sa\smemory\smapped\sregion. +D 2013-03-23T14:20:42.975 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac -F src/pager.c 12b8ff12519fe529a92884c4cdb99afecb1bea3c +F src/pager.c cffcfe753445ef3a3d3830efd5f807fc171e7262 F src/pager.h bbc9170281c9d5d603b2175fdc8ea908e47269a7 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -630,7 +630,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 46ff4038b6d300bb9b39d3f89ea1aef3dff9b88f +F test/mmap1.test e3391ef6c069e5f5733b8339652ce37fd97e3487 F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 @@ -730,7 +730,7 @@ F test/softheap1.test c16709a16ad79fa43b32929b2e623d1d117ccf53 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb -F test/speed1p.test 2577211a9b054ffc973780f5b8708f02a0c7422d +F test/speed1p.test c4a469f29f135f4d76c55b1f2a52f36e209466cc F test/speed2.test 53177056baf6556dcbdcf032bbdfc41c1aa74ded F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f57a9c91e993f76ce8b923e06e721414980e8e61 -R a68c203c916fb468751938d0762a7812 -U mistachkin -Z c2b92e0c0a8c400cb287f5f47ed82e78 +P 8870c4cc6cda07154936644f7df25bd81d6ba38a +R 0b90a49ac5025c5ac7d6b4eb49fe1878 +U dan +Z c9ca2a38d0bbfc316a52f1d1e135baf9 diff --git a/manifest.uuid b/manifest.uuid index 0e0a52b9c3..bd17b56cca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8870c4cc6cda07154936644f7df25bd81d6ba38a \ No newline at end of file +8dbe89d05ce91428c69003f0da79d883fa23e2b5 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 660514d508..09cc4a2375 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4232,7 +4232,7 @@ static int syncJournal(Pager *pPager, int newHdr){ */ int sqlite3PagerSetFilesize(Pager *pPager, i64 szReq){ int rc; - sqlite3_int64 sz; + i64 sz; /* Size of file on disk in bytes */ assert( pPager->eState==PAGER_OPEN ); assert( pPager->nMmapOut==0 ); @@ -4244,13 +4244,13 @@ int sqlite3PagerSetFilesize(Pager *pPager, i64 szReq){ } } - if( rc==SQLITE_OK - && pPager->nMapLimit>0 - && pPager->nMapValidnMapValidnMapLimit - ){ - pPager->dbFileSize = (szReq / pPager->pageSize); - rc = pagerMap(pPager, 1); + + if( rc==SQLITE_OK ){ + i64 szMap = (szReq > pPager->nMapLimit) ? pPager->nMapLimit : szReq; + if( pPager->nMapValid!=pPager->nMap || szMap!=pPager->nMap ){ + pPager->dbFileSize = (szReq / pPager->pageSize); + rc = pagerMap(pPager, 1); + } } return rc; diff --git a/test/mmap1.test b/test/mmap1.test index 343ad230b1..41714e29d8 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -23,13 +23,17 @@ proc nRead {db} { return $stats(read) } -foreach {t mmap_size nRead} { - 1 { PRAGMA mmap_size = -65536 } 4 - 2 { PRAGMA mmap_size = -50 } 156 - 3 { PRAGMA mmap_size = 0 } 344 +foreach {t mmap_size nRead c2init} { + 1.1 { PRAGMA mmap_size = -65536 } 4 {} + 1.2 { PRAGMA mmap_size = -50 } 156 {} + 1.3 { PRAGMA mmap_size = 0 } 344 {} + 1.4 { PRAGMA mmap_size = -65536 } 4 {PRAGMA mmap_size = -65536} + 1.5 { PRAGMA mmap_size = -50 } 156 {PRAGMA mmap_size = -65536} + 1.6 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = -65536} } { do_multiclient_test tn { sql1 $mmap_size + sql2 $c2init code2 { set ::rcnt 0 @@ -79,6 +83,51 @@ foreach {t mmap_size nRead} { } } +set ::rcnt 0 +proc rblob {n} { + set ::rcnt [expr (($::rcnt << 3) + $::rcnt + 456) & 0xFFFFFFFF] + set str [format %.8x [expr $::rcnt ^ 0xbdf20da3]] + string range [string repeat $str [expr $n/4]] 1 $n +} + +reset_db +db func rblob rblob + +do_execsql_test 2.1 { + PRAGMA auto_vacuum = 1; + PRAGMA mmap_size = -65536; + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b, UNIQUE(a, b)); + INSERT INTO t1 VALUES(rblob(500), rblob(500)); + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 2 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 4 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 8 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 32 + PRAGMA wal_checkpoint; +} {wal 0 103 103} + +do_execsql_test 2.2 { + PRAGMA auto_vacuum; + SELECT count(*) FROM t1; +} {1 32} + +do_test 2.3 { + sqlite3 db2 test.db + db2 func rblob rblob + db2 eval { + DELETE FROM t1 WHERE (rowid%4); + PRAGMA wal_checkpoint; + } + db2 eval { + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 + SELECT count(*) FROM t1; + } +} {16} + +do_execsql_test 2.4 { + PRAGMA wal_checkpoint; +} {0 24 24} finish_test diff --git a/test/speed1p.test b/test/speed1p.test index 33be553b00..915f165354 100644 --- a/test/speed1p.test +++ b/test/speed1p.test @@ -65,9 +65,6 @@ proc number_name {n} { # do_test speed1p-1.0 { execsql { - PRAGMA mmap_size=65536; - PRAGMA journal_mode=wal; - PRAGMA page_size=1024; PRAGMA cache_size=500; PRAGMA locking_mode=EXCLUSIVE; From e5586a1a79f3d8e61ee6d263ec680c012ffbabfb Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 23 Mar 2013 17:29:06 +0000 Subject: [PATCH 023/140] Improve a comment in wal.c. No code changes. FossilOrigin-Name: 60b9f5e4dd2af54975ba78437239f0bebd472fd2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 13 +++++++++---- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index cf3732c267..6660b0b860 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scase\swhere\sa\scheckpoint\soperation\scould\swrite\sto\san\sinvalid\spart\sof\sa\smemory\smapped\sregion. -D 2013-03-23T14:20:42.975 +C Improve\sa\scomment\sin\swal.c.\sNo\scode\schanges. +D 2013-03-23T17:29:06.036 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c c61ca318681c0e7267da8be3abfca8469652a7e9 F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 -F src/wal.c 7fec703bb0770d9faad5190f49897a8bc8eb239f +F src/wal.c 923d90992fc3b069571cf3fbd635c54c3e839e55 F src/wal.h d99ce512ac60f9147a0640e9e6fb67dd1057b781 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b F src/where.c bdbbfa7ef4ea04c8d9b09585b45d4717a72f980a @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 8870c4cc6cda07154936644f7df25bd81d6ba38a -R 0b90a49ac5025c5ac7d6b4eb49fe1878 +P 8dbe89d05ce91428c69003f0da79d883fa23e2b5 +R b063a8588944a5f842812de15381707f U dan -Z c9ca2a38d0bbfc316a52f1d1e135baf9 +Z 350cbe536c1fd5aa73b476fafae6db5c diff --git a/manifest.uuid b/manifest.uuid index bd17b56cca..7aea0f37ae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8dbe89d05ce91428c69003f0da79d883fa23e2b5 \ No newline at end of file +60b9f5e4dd2af54975ba78437239f0bebd472fd2 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 8757003d78..051b77f515 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1725,12 +1725,17 @@ static int walCheckpoint( rc = sqlite3OsSync(pWal->pWalFd, sync_flags); } - /* If the database file may grow as a result of this checkpoint, hint - ** about the eventual size of the db file to the VFS layer. - */ + /* If the database file is currently smaller than mxPage pages in size, + ** the call below issues an SQLITE_FCNTL_SIZE_HINT to the OS layer to + ** inform it that it is likely to grow to that size. + ** + ** Additionally, if the pager is using mmap(), then the call to + ** SetFilesize() guarantees that the mapping is not larger than mxPage + ** pages. This makes the sqlite3OsTruncate() call below safe - no pages + ** that are part of the mapped region will be truncated away. */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); - rc = sqlite3PagerSetFilesize(pWal->pPager, nReq); + rc = sqlite3PagerSetFilesize(pWal->pPager, nReq); } /* Iterate through the contents of the WAL, copying data to the db file. */ From f23da96636f9de1a81c939bd7a0cc25ee1d37f49 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 23 Mar 2013 21:00:41 +0000 Subject: [PATCH 024/140] Replace the sqlite3_io_methods.xMremap interface with sqlite3_io_methods.xFetch and xUnfetch. FossilOrigin-Name: 1431be95579160fb70408d43e17fc23c7b69ab4a --- manifest | 28 +++--- manifest.uuid | 2 +- src/backup.c | 2 +- src/os.c | 15 ++-- src/os.h | 3 +- src/os_unix.c | 211 +++++++++++++++++++++++++++------------------ src/pager.c | 221 ++++++++++++------------------------------------ src/pager.h | 3 - src/sqlite.h.in | 30 ++----- src/wal.c | 22 ++--- src/wal.h | 3 +- 11 files changed, 222 insertions(+), 318 deletions(-) diff --git a/manifest b/manifest index 6660b0b860..8d71dc88ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sa\scomment\sin\swal.c.\sNo\scode\schanges. -D 2013-03-23T17:29:06.036 +C Replace\sthe\ssqlite3_io_methods.xMremap\sinterface\swith\ssqlite3_io_methods.xFetch\sand\sxUnfetch. +D 2013-03-23T21:00:41.457 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -118,7 +118,7 @@ F src/alter.c f8db986c03eb0bfb221523fc9bbb9d0b70de3168 F src/analyze.c d5f895810e8ff9737c9ec7b76abc3dcff5860335 F src/attach.c ea5247f240e2c08afd608e9beb380814b86655e1 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 -F src/backup.c 6256400ab8be4a15a2512277d8b214dbd44ff8ce +F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 26675fe8e431dc555e6f2d0e11e651d172234aa1 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c f5ea8d3d658887b5cae0369ef10a427d7469a768 @@ -157,13 +157,13 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c 87ea1cd1259c5840848e34007d72e772a2ab7528 -F src/os.h 8d92f87f5fe14b060a853ca704b8ef6d3daee79b +F src/os.c a7ec2ddc43fb2684bfd8cb30db15ea6cce3298e9 +F src/os.h 782980cd0d840b4240a6e29ba9f8ef0ca0d750ca F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 55d110879332831b734fd510cfbc5700e96a83cf +F src/os_unix.c d8cdf331ad08650fc8397eee79b55548070077b0 F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac -F src/pager.c cffcfe753445ef3a3d3830efd5f807fc171e7262 -F src/pager.h bbc9170281c9d5d603b2175fdc8ea908e47269a7 +F src/pager.c 2e63de7185c2a573d26e5087869a9e48326076f9 +F src/pager.h faf4bed79e80cb6527209dfe5cfb08aa5764b941 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 @@ -176,7 +176,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in d63c7fb5832287af7e8b903c4a4c30c90414876f +F src/sqlite.h.in f41949e1e778043dd9fc23b33b96e7a6e20de4d6 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796 @@ -249,8 +249,8 @@ F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c c61ca318681c0e7267da8be3abfca8469652a7e9 F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 -F src/wal.c 923d90992fc3b069571cf3fbd635c54c3e839e55 -F src/wal.h d99ce512ac60f9147a0640e9e6fb67dd1057b781 +F src/wal.c e84eff498c57ec2d79ca4496a3f4a638af378fb3 +F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b F src/where.c bdbbfa7ef4ea04c8d9b09585b45d4717a72f980a F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 8dbe89d05ce91428c69003f0da79d883fa23e2b5 -R b063a8588944a5f842812de15381707f +P 60b9f5e4dd2af54975ba78437239f0bebd472fd2 +R eee4d88d6400cf65980c5b201532d2c9 U dan -Z 350cbe536c1fd5aa73b476fafae6db5c +Z eabd20cdc7aabbc65591f96e4f6dff04 diff --git a/manifest.uuid b/manifest.uuid index 7aea0f37ae..f378aa2e8b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -60b9f5e4dd2af54975ba78437239f0bebd472fd2 \ No newline at end of file +1431be95579160fb70408d43e17fc23c7b69ab4a \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 8939bc67bc..71a8a1a3e7 100644 --- a/src/backup.c +++ b/src/backup.c @@ -520,7 +520,7 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); if( rc==SQLITE_OK ){ u8 *zData = sqlite3PagerGetData(pSrcPg); - rc = sqlite3PagerWriteData(pDestPager, zData, pgszSrc, iOff); + rc = sqlite3OsWrite(pFile, zData, pgszSrc, iOff); } sqlite3PagerUnref(pSrcPg); } diff --git a/src/os.c b/src/os.c index 38757dcdb1..08e0d42172 100644 --- a/src/os.c +++ b/src/os.c @@ -140,15 +140,12 @@ int sqlite3OsShmMap( DO_OS_MALLOC_TEST(id); return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } -int sqlite3OsMremap( - sqlite3_file *id, /* Database file handle */ - int flags, /* SQLITE_MREMAP_XXX flags */ - i64 iOff, /* Offset at which mapping(s) start */ - i64 nOld, /* Size of old mapping */ - i64 nNew, /* Size of requested mapping */ - void **pp /* IN/OUT: Pointer to mapped region */ -){ - return id->pMethods->xMremap(id, flags, iOff, nOld, nNew, pp); + +int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + return id->pMethods->xFetch(id, iOff, iAmt, pp); +} +int sqlite3OsUnfetch(sqlite3_file *id, void *p){ + return id->pMethods->xUnfetch(id, p); } /* diff --git a/src/os.h b/src/os.h index f08b92dcf0..a7dacfb6ea 100644 --- a/src/os.h +++ b/src/os.h @@ -259,7 +259,8 @@ int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); -int sqlite3OsMremap(sqlite3_file *id, int, i64, i64, i64, void **); +int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); +int sqlite3OsUnfetch(sqlite3_file *, void *); /* diff --git a/src/os_unix.c b/src/os_unix.c index 2f03c39670..334719200a 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -249,9 +249,14 @@ struct unixFile { unsigned char transCntrChng; /* True if the transaction counter changed */ unsigned char dbUpdate; /* True if any part of database file changed */ unsigned char inNormalWrite; /* True if in a normal write operation */ - sqlite3_int64 mmapSize; /* Size of xMremap() */ - void *pMapRegion; /* Area memory mapped */ + #endif + sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ + sqlite3_int64 mmapOrigsize; /* Actual size of mapping at pMapRegion */ + sqlite3_int64 mmapLimit; /* Configured FCNTL_MMAP_SIZE value */ + void *pMapRegion; /* Memory mapped region */ + int nFetchOut; /* Number of outstanding xFetch refs */ + #ifdef SQLITE_TEST /* In test mode, increase the size of this structure a bit so that ** it is larger than the struct CrashFile defined in test6.c. @@ -1805,6 +1810,9 @@ static int unixUnlock(sqlite3_file *id, int eFileLock){ return posixUnlock(id, eFileLock, 0); } +static int unixMapfile(unixFile *pFd, i64 nByte); +static void unixUnmapfile(unixFile *pFd); + /* ** This function performs the parts of the "close file" operation ** common to all locking schemes. It closes the directory and file @@ -1817,6 +1825,7 @@ static int unixUnlock(sqlite3_file *id, int eFileLock){ */ static int closeUnixFile(sqlite3_file *id){ unixFile *pFile = (unixFile*)id; + unixUnmapfile(pFile); if( pFile->h>=0 ){ robust_close(pFile, pFile->h, __LINE__); pFile->h = -1; @@ -3074,7 +3083,6 @@ static int unixRead( unixFile *pFile = (unixFile *)id; int got; assert( id ); - assert( offset>=pFile->mmapSize ); /* Never read from the mmapped region */ /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ @@ -3085,6 +3093,21 @@ static int unixRead( ); #endif + /* Deal with as much of this write request as possible by transfering + ** data to the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } + got = seekAndRead(pFile, offset, pBuf, amt); if( got==amt ){ return SQLITE_OK; @@ -3157,7 +3180,6 @@ static int unixWrite( int wrote = 0; assert( id ); assert( amt>0 ); - assert( offset>=pFile->mmapSize ); /* Never write into the mmapped region */ /* If this is a database file (not a journal, master-journal or temp ** file), the bytes in the locking range should never be read or written. */ @@ -3190,6 +3212,21 @@ static int unixWrite( } #endif + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + return SQLITE_OK; + }else{ + int nCopy = pFile->mmapSize - offset; + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } + while( amt>0 && (wrote = seekAndWrite(pFile, offset, pBuf, amt))>0 ){ amt -= wrote; offset += wrote; @@ -3470,6 +3507,7 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ if( pFile->inNormalWrite && nByte==0 ){ pFile->transCntrChng = 1; } +#endif /* If the file was just truncated to a size smaller than the currently ** mapped region, reduce the effective mapping size as well. SQLite will @@ -3478,7 +3516,6 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ if( nBytemmapSize ){ pFile->mmapSize = nByte; } -#endif return SQLITE_OK; } @@ -3568,6 +3605,19 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ } } + if( pFile->mmapLimit>0 ){ + int rc; + if( pFile->szChunk<=0 ){ + if( robust_ftruncate(pFile->h, nByte) ){ + pFile->lastErrno = errno; + return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); + } + } + + rc = unixMapfile(pFile, nByte); + return rc; + } + return SQLITE_OK; } @@ -3635,8 +3685,8 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } - case SQLITE_FCNTL_GETFD: { - *(int*)pArg = pFile->h; + case SQLITE_FCNTL_MMAP_SIZE: { + pFile->mmapLimit = *(i64*)pArg; return SQLITE_OK; } #ifdef SQLITE_DEBUG @@ -4451,91 +4501,86 @@ static int unixShmUnmap( */ #define ROUNDUP(x,y) (((x)+y-1)&~(y-1)) -/* -** Map, remap or unmap part of the database file. -*/ -static int unixMremap( - sqlite3_file *fd, /* Main database file */ - int flags, /* Mask of SQLITE_MREMAP_XXX flags */ - sqlite3_int64 iOff, /* Offset to start mapping at */ - sqlite3_int64 nOld, /* Size of old mapping, or zero */ - sqlite3_int64 nNew, /* Size of new mapping, or zero */ - void **ppMap /* IN/OUT: Old/new mappings */ -){ - unixFile *p = (unixFile *)fd; /* The underlying database file */ - int rc = SQLITE_OK; /* Return code */ - void *pNew = 0; /* New mapping */ - i64 nNewRnd; /* nNew rounded up */ - i64 nOldRnd; /* nOld rounded up */ +static void unixUnmapfile(unixFile *pFd){ + assert( pFd->nFetchOut==0 ); + if( pFd->pMapRegion ){ + munmap(pFd->pMapRegion, pFd->mmapOrigsize); + pFd->pMapRegion = 0; + pFd->mmapSize = 0; + pFd->mmapOrigsize = 0; + } +} - assert( iOff==0 ); - /* assert( p->mmapSize==nOld ); */ - assert( p->pMapRegion==0 || p->pMapRegion==(*ppMap) ); +static int unixMapfile(unixFile *pFd, i64 nByte){ + i64 nMap = nByte; + int rc; - /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested - ** mapping (nNew bytes) may be greater than the size of the database file. - ** If this is the case, extend the file on disk using ftruncate(). */ - assert( nNew>0 || (flags & SQLITE_MREMAP_EXTEND)==0 ); - if( flags & SQLITE_MREMAP_EXTEND ){ + assert( nMap>=0 || pFd->nFetchOut==0 ); + if( pFd->nFetchOut>0 ) return SQLITE_OK; + + if( nMap<0 ){ struct stat statbuf; /* Low-level file information */ - rc = osFstat(p->h, &statbuf); - if( rc==SQLITE_OK && nNew>statbuf.st_size ){ - rc = robust_ftruncate(p->h, nNew); + rc = osFstat(pFd->h, &statbuf); + if( rc!=SQLITE_OK ){ + return SQLITE_IOERR_FSTAT; } - if( rc!=SQLITE_OK ) return rc; + nMap = statbuf.st_size; + } + if( nMap>pFd->mmapLimit ){ + nMap = pFd->mmapLimit; } - /* According to some sources, the effect of changing the size of the - ** underlying file on mapped regions that correspond to the added or - ** removed pages is undefined. However, there is reason to believe that - ** on modern platforms like Linux or OSX, things just work. For example, - ** it is possible to create a mapping larger than the file on disk and - ** extend the file on disk later on. - ** - ** Exploit this on Linux and OSX to reduce the number of munmap()/mmap() - ** calls required if the file size is changing. In this case all mappings - ** are rounded up to the nearest 4MB. And if a new mapping is requested - ** that has the same rounded size as an old mapping, the old mapping can - ** be reused as is. */ -#if defined(__APPLE__) || defined(__linux__) - nNewRnd = ROUNDUP(nNew, 4096*1024); - nOldRnd = ROUNDUP(nOld, 4096*1024); -#else - nNewRnd = ROUNDUP(nNew, 4096*1); - nOldRnd = ROUNDUP(nOld, 4096*1); -#endif + if( nMap!=pFd->mmapSize ){ + void *pNew; + unixUnmapfile(pFd); - /* On OSX or Linux, reuse the old mapping if it is the right size. */ -#if defined(__APPLE__) || defined(__linux__) - if( nNewRnd==nOldRnd ){ - VVA_ONLY( p->mmapSize = nNew; ) - return SQLITE_OK; - } -#endif + if( nMap>0 ){ + void *pNew; + int flags = PROT_READ; + if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; + pNew = mmap(0, ROUNDUP(nMap, 4096), flags, MAP_SHARED, pFd->h, 0); + if( pNew==MAP_FAILED ){ + return SQLITE_IOERR_MREMAP; + } - /* If we get this far, unmap any old mapping. */ - if( nOldRnd!=0 ){ - void *pOld = *ppMap; - munmap(pOld, nOldRnd); - VVA_ONLY( p->mmapSize = 0; p->pMapRegion = 0; ); - } - - /* And, if required, use mmap() to create a new mapping. */ - if( nNewRnd>0 ){ - int flags = PROT_READ; - if( (p->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - pNew = mmap(0, nNewRnd, flags, MAP_SHARED, p->h, iOff); - if( pNew==MAP_FAILED ){ - pNew = 0; - VVA_ONLY( p->mmapSize = 0; p->pMapRegion = 0; ) - rc = SQLITE_IOERR_MREMAP; - }else{ - VVA_ONLY( p->mmapSize = nNew; p->pMapRegion = pNew; ) + pFd->pMapRegion = pNew; + pFd->mmapOrigsize = pFd->mmapSize = nMap; } } - *ppMap = pNew; - return rc; + return SQLITE_OK; +} + +static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + *pp = 0; + + if( pFd->mmapLimit>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = unixMapfile(pFd, -1); + if( rc!=SQLITE_OK ) return rc; + } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } + return SQLITE_OK; +} + +static int unixUnfetch(sqlite3_file *fd, void *p){ + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + + assert( (p==0)==(pFd->nFetchOut==0) ); + + if( p ){ + pFd->nFetchOut--; + }else{ + unixUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); + return SQLITE_OK; } /* @@ -4597,7 +4642,8 @@ static const sqlite3_io_methods METHOD = { \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ unixShmUnmap, /* xShmUnmap */ \ - unixMremap, /* xMremap */ \ + unixFetch, /* xFetch */ \ + unixUnfetch, /* xUnfetch */ \ }; \ static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ @@ -4865,7 +4911,6 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; - VVA_ONLY( pNew->mmapSize = 0; ) if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; diff --git a/src/pager.c b/src/pager.c index 09cc4a2375..7e00e1431a 100644 --- a/src/pager.c +++ b/src/pager.c @@ -656,14 +656,10 @@ struct Pager { int nSavepoint; /* Number of elements in aSavepoint[] */ char dbFileVers[16]; /* Changes whenever database file changes */ - void *pMap; /* Memory mapped prefix of database file */ - i64 nMap; /* Size of mapping at pMap in bytes */ - i64 nMapValid; /* Bytes at pMap known to be valid */ - i64 nMapLimit; /* Maximum permitted mapping size */ + u8 bUseFetch; /* True to use xFetch() */ int nMapCfgLimit; /* Configured limit value */ int nMmapOut; /* Number of mmap pages currently outstanding */ PgHdr *pFree; /* List of free mmap page headers (pDirty) */ - int bMapResize; /* Check if the mapping should be resized */ /* ** End of the routinely-changing class members ***************************************************************************/ @@ -2087,24 +2083,6 @@ static void pagerReportSize(Pager *pPager){ # define pagerReportSize(X) /* No-op if we do not support a codec */ #endif -/* -** Write nBuf bytes of data from buffer pBuf to offset iOff of the -** database file. If this part of the database file is memory mapped, -** use memcpy() to do so. Otherwise, call sqlite3OsWrite(). -** -** Return SQLITE_OK if successful, or an SQLite error code if an error -** occurs. -*/ -int sqlite3PagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff){ - int rc = SQLITE_OK; - if( pPager->nMapValid>=(iOff+nBuf) ){ - memcpy(&((u8 *)(pPager->pMap))[iOff], pBuf, nBuf); - }else{ - rc = sqlite3OsWrite(pPager->fd, pBuf, nBuf, iOff); - } - return rc; -} - /* ** Read a single page from either the journal file (if isMainJrnl==1) or ** from the sub-journal (if isMainJrnl==0) and playback that page. @@ -2279,7 +2257,7 @@ static int pager_playback_one_page( i64 ofst = (pgno-1)*(i64)pPager->pageSize; testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); assert( !pagerUseWal(pPager) ); - rc = sqlite3PagerWriteData(pPager, aData, pPager->pageSize, ofst); + rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } @@ -2534,9 +2512,6 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ if( rc==SQLITE_OK && currentSize!=newSize ){ if( currentSize>newSize ){ rc = sqlite3OsTruncate(pPager->fd, newSize); - if( newSizenMapValid ){ - pPager->nMapValid = newSize; - } }else if( (currentSize+szPage)<=newSize ){ char *pTmp = pPager->pTmpSpace; memset(pTmp, 0, szPage); @@ -2884,13 +2859,9 @@ static int readDbPage(PgHdr *pPg, u32 iFrame){ rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData); }else{ i64 iOffset = (pgno-1)*(i64)pPager->pageSize; - if( pPager->pMap && pPager->nMapValid>=iOffset+pPager->pageSize ){ - memcpy(pPg->pData, &((u8 *)(pPager->pMap))[iOffset], pPager->pageSize); - }else{ - rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); - if( rc==SQLITE_IOERR_SHORT_READ ){ - rc = SQLITE_OK; - } + rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); + if( rc==SQLITE_IOERR_SHORT_READ ){ + rc = SQLITE_OK; } } @@ -3120,6 +3091,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); if( rc!=SQLITE_OK || changed ){ pager_reset(pPager); + if( pPager->bUseFetch ) sqlite3OsUnfetch(pPager->fd, 0); } return rc; @@ -3382,22 +3354,24 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ } /* -** Set Pager.nMapLimit, the maximum permitted mapping size, based on the -** current values of Pager.nMapCfgLimit and Pager.pageSize. -** -** If this connection should not use mmap at all, set nMapLimit to zero. +** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of nMapCfgLimit. */ static void pagerFixMaplimit(Pager *pPager){ - if( isOpen(pPager->fd)==0 - || pPager->fd->pMethods->iVersion<3 - || pPager->fd->pMethods->xMremap==0 - || pPager->tempFile - ){ - pPager->nMapLimit = 0; - }else if( pPager->nMapCfgLimit<0 ){ - pPager->nMapLimit = (i64)pPager->nMapCfgLimit * -1024; - }else{ - pPager->nMapLimit = (i64)pPager->nMapCfgLimit * pPager->pageSize; + sqlite3_file *fd = pPager->fd; + if( isOpen(fd) ){ + pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->nMapCfgLimit!=0; + if( pPager->bUseFetch ){ + void *p; + i64 nMapLimit; + if( pPager->nMapCfgLimit<0 ){ + nMapLimit = (i64)pPager->nMapCfgLimit * -1024; + }else{ + nMapLimit = (i64)pPager->nMapCfgLimit * pPager->pageSize; + } + + p = (void *)&nMapLimit; + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_SIZE, p); + } } } @@ -3870,67 +3844,22 @@ static int pagerSyncHotJournal(Pager *pPager){ return rc; } -/* -** Unmap any memory mapping of the database file. -*/ -static int pagerUnmap(Pager *pPager){ - assert( pPager->nMmapOut==0 ); - if( pPager->pMap ){ - sqlite3OsMremap(pPager->fd, 0, 0, pPager->nMap, 0, &pPager->pMap); - pPager->nMap = 0; - pPager->nMapValid = 0; - } - return SQLITE_OK; -} - -/* -** Create, or recreate, the memory mapping of the database file. -*/ -static int pagerMap(Pager *pPager, int bExtend){ - int rc = SQLITE_OK; /* Return code */ - Pgno nPg; /* Size of mapping to request in pages */ - i64 sz; /* Size of mapping to request in bytes */ - - assert( isOpen(pPager->fd) && pPager->tempFile==0 ); - assert( pPager->pMap==0 || pPager->nMap>0 ); - /* assert( pPager->eState>=1 ); */ - assert( pPager->nMmapOut==0 ); - assert( pPager->nMapLimit>0 ); - - /* Figure out how large a mapping to request. Set variable sz to this - ** value in bytes. */ - nPg = (pPager->eState==1) ? pPager->dbSize : pPager->dbFileSize; - sz = (i64)nPg * pPager->pageSize; - if( sz>pPager->nMapLimit ) sz = pPager->nMapLimit; - - if( sz!=pPager->nMapValid ){ - int flags = (bExtend ? SQLITE_MREMAP_EXTEND : 0); - rc = sqlite3OsMremap(pPager->fd, flags, 0, pPager->nMap, sz, &pPager->pMap); - if( rc==SQLITE_OK ){ - assert( pPager->pMap!=0 ); - pPager->nMap = sz; - }else{ - assert( pPager->pMap==0 ); - pPager->nMap = 0; - } - pPager->nMapValid = pPager->nMap; - } - pPager->bMapResize = 0; - - return rc; -} - /* ** Obtain a reference to a memory mapped page object for page number pgno. -** The caller must ensure that page pgno lies within the currently mapped -** region. If successful, set *ppPage to point to the new page reference +** The new object will use the pointer pData, obtained from xFetch(). +** If successful, set *ppPage to point to the new page reference ** and return SQLITE_OK. Otherwise, return an SQLite error code and set ** *ppPage to zero. ** ** Page references obtained by calling this function should be released ** by calling pagerReleaseMapPage(). */ -static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ +static int pagerAcquireMapPage( + Pager *pPager, /* Pager object */ + Pgno pgno, /* Page number */ + void *pData, /* xFetch()'d data for this page */ + PgHdr **ppPage /* OUT: Acquired page object */ +){ PgHdr *p; /* Memory mapped page to return */ if( pPager->pFree ){ @@ -3955,8 +3884,8 @@ static int pagerAcquireMapPage(Pager *pPager, Pgno pgno, PgHdr **ppPage){ assert( p->pPager==pPager ); assert( p->nRef==1 ); - p->pData = &((u8 *)pPager->pMap)[(i64)(pgno-1) * pPager->pageSize]; p->pgno = pgno; + p->pData = pData; pPager->nMmapOut++; return SQLITE_OK; @@ -3971,6 +3900,9 @@ static void pagerReleaseMapPage(PgHdr *pPg){ pPager->nMmapOut--; pPg->pDirty = pPager->pFree; pPager->pFree = pPg; + + assert( pPager->fd->pMethods->iVersion>=3 ); + sqlite3OsUnfetch(pPager->fd, pPg->pData); } /* @@ -4006,7 +3938,6 @@ int sqlite3PagerClose(Pager *pPager){ assert( assert_pager_state(pPager) ); disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); - pagerUnmap(pPager); pagerFreeMapHdrs(pPager); /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; @@ -4216,46 +4147,6 @@ static int syncJournal(Pager *pPager, int newHdr){ return SQLITE_OK; } -/* -** This is called by the wal.c module at the start of a checkpoint. If the -** checkpoint runs to completion, it will set the database file size to -** szReq bytes. This function performs two tasks: -** -** * If the file is currently less than szReq bytes in size, an -** xFileControl(SQLITE_FNCTL_SIZE_HINT) is issued to inform the OS -** layer of the expected file size, and -** -** * If mmap is being used, then the mapping is extended to szReq -** bytes in size. -** -** SQLITE_OK is returned if successful, or an error code if an error occurs. -*/ -int sqlite3PagerSetFilesize(Pager *pPager, i64 szReq){ - int rc; - i64 sz; /* Size of file on disk in bytes */ - - assert( pPager->eState==PAGER_OPEN ); - assert( pPager->nMmapOut==0 ); - - rc = sqlite3OsFileSize(pPager->fd, &sz); - if( rc==SQLITE_OK ){ - if( sz>szReq ){ - sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &sz); - } - } - - - if( rc==SQLITE_OK ){ - i64 szMap = (szReq > pPager->nMapLimit) ? pPager->nMapLimit : szReq; - if( pPager->nMapValid!=pPager->nMap || szMap!=pPager->nMap ){ - pPager->dbFileSize = (szReq / pPager->pageSize); - rc = pagerMap(pPager, 1); - } - } - - return rc; -} - /* ** The argument is the first in a linked list of dirty pages connected ** by the PgHdr.pDirty pointer. This function writes each one of the @@ -4315,11 +4206,6 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ sqlite3_int64 szFile = pPager->pageSize * (sqlite3_int64)pPager->dbSize; sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &szFile); pPager->dbHintSize = pPager->dbSize; - - if( pPager->nMmapOut==0 && pPager->nMapLimit>0 ){ - pPager->dbFileSize = pPager->dbSize; - rc = pagerMap(pPager, 1); - } } while( rc==SQLITE_OK && pList ){ @@ -4344,7 +4230,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM, pData); /* Write out the page data. */ - rc = sqlite3PagerWriteData(pPager, pData, pPager->pageSize, offset); + rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); /* If page 1 was just written, update Pager.dbFileVers to match ** the value now stored in the database file. If writing this @@ -5164,7 +5050,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( !pPager->tempFile && ( pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0 - || pPager->pMap + || pPager->bUseFetch /* TODO: Currently required for xUnfetch(0) only. */ )){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous @@ -5188,13 +5074,9 @@ int sqlite3PagerSharedLock(Pager *pPager){ rc = pagerPagecount(pPager, &nPage); if( rc ) goto failed; - if( nPage>0 || pPager->pMap ){ + if( nPage>0 ){ IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); - if( pPager->pMap ){ - memcpy(&dbFileVers, &((u8 *)(pPager->pMap))[24], sizeof(dbFileVers)); - }else{ - rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); - } + rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); if( rc!=SQLITE_OK ){ goto failed; } @@ -5211,9 +5093,9 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** In this case there may exist a Pager.pMap mapping that appears ** to be the right size but is not actually valid. Avoid this ** possibility by unmapping the db here. */ - pagerUnmap(pPager); - }else if( pPager->pMap ){ - pPager->bMapResize = 1; + if( pPager->bUseFetch ){ + sqlite3OsUnfetch(pPager->fd, 0); + } } } @@ -5325,7 +5207,7 @@ int sqlite3PagerAcquire( ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY ** flag was specified by the caller. And so long as the db is not a ** temporary or in-memory database. */ - const int bMmapOk = (pPager->nMapLimit>0 && pgno!=1 + const int bMmapOk = (pgno!=1 && pPager->bUseFetch && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY)) ); @@ -5349,15 +5231,20 @@ int sqlite3PagerAcquire( } if( iFrame==0 && bMmapOk ){ - if( pPager->pMap==0 || (pPager->bMapResize && pPager->nMmapOut==0) ){ - rc = pagerMap(pPager, 0); - } - if( rc==SQLITE_OK && pPager->nMap>=((i64)pgno * pPager->pageSize) ){ + void *pData = 0; + + rc = sqlite3OsFetch(pPager->fd, + (i64)(pgno-1) * pPager->pageSize, pPager->pageSize, &pData + ); + + if( rc==SQLITE_OK && pData ){ if( pPager->eState>PAGER_READER ){ (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); } if( pPg==0 ){ - rc = pagerAcquireMapPage(pPager, pgno, &pPg); + rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); + }else{ + sqlite3OsUnfetch(pPager->fd, pData); } if( pPg ){ assert( rc==SQLITE_OK ); @@ -7117,7 +7004,7 @@ static int pagerOpenWal(Pager *pPager){ ** (e.g. due to malloc() failure), return an error code. */ if( rc==SQLITE_OK ){ - rc = sqlite3WalOpen(pPager->pVfs, pPager, + rc = sqlite3WalOpen(pPager->pVfs, pPager->fd, pPager->zWal, pPager->exclusiveMode, pPager->journalSizeLimit, &pPager->pWal ); @@ -7155,8 +7042,6 @@ int sqlite3PagerOpenWal( assert( pbOpen==0 || *pbOpen==0 ); assert( pbOpen!=0 || (!pPager->tempFile && !pPager->pWal) ); - pagerUnmap(pPager); - if( !pPager->tempFile && !pPager->pWal ){ if( !sqlite3PagerWalSupported(pPager) ) return SQLITE_CANTOPEN; diff --git a/src/pager.h b/src/pager.h index 970b5035bf..066d1f11d7 100644 --- a/src/pager.h +++ b/src/pager.h @@ -176,9 +176,6 @@ int sqlite3SectorSize(sqlite3_file *); void sqlite3PagerTruncateImage(Pager*,Pgno); int sqlite3PagerSetFilesize(Pager *, i64); -/* Write data to the database file */ -int sqlite3PagerWriteData(Pager *pPager, const void *pBuf, int nBuf, i64 iOff); - #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); #endif diff --git a/src/sqlite.h.in b/src/sqlite.h.in index c0a2803787..666a2e084d 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -707,24 +707,6 @@ struct sqlite3_file { ** fails to zero-fill short reads might seem to work. However, ** failure to zero-fill short reads will eventually lead to ** database corruption. -** -** Assuming parameter nNew is non-zero, the xMremap method should attempt -** to memory map a region nNew bytes in size starting at offset iOffset -** of the file. If successful, it should set *ppMap to point to the -** mapping and return SQLITE_OK. If the file is opened for read-write -** access, then the mapping should also be read-write. -** -** If nOld is non-zero, then the initial value of *ppMap points to a -** mapping returned by a previous call to xMremap. The existing mapping -** is nOld bytes in size and starts at offset iOffset of the file. In -** this case the xMremap method is expected to unmap the existing mapping -** and overwrite *ppMap with the pointer to the new mapping. If nOld is -** zero, then the initial value of *ppMap is undefined. -** -** If nNew is zero, then no new mapping should be created. Any old -** mapping must still be unmapped if nOld is non-zero. If the nOld -** parameter is non-zero, then the existing mapping is always unmapped - -** even if an error occurs. */ typedef struct sqlite3_io_methods sqlite3_io_methods; struct sqlite3_io_methods { @@ -747,14 +729,12 @@ struct sqlite3_io_methods { void (*xShmBarrier)(sqlite3_file*); int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ - int (*xMremap)(sqlite3_file *fd, int flags, - sqlite3_int64 iOff, sqlite3_int64 nOld, sqlite3_int64 nNew, void **ppMap); + int (*xFetch)(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); + int (*xUnfetch)(sqlite3_file*, void *p); /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; -#define SQLITE_MREMAP_EXTEND 0x0001 /* xMremap call may extend file */ - /* ** CAPI3REF: Standard File Control Opcodes ** @@ -906,6 +886,10 @@ struct sqlite3_io_methods { ** written into memory obtained from [sqlite3_malloc()]. The caller should ** invoke [sqlite3_free()] on the result to avoid a memory leak. ** +**
  • [[SQLITE_FCNTL_MMAP_SIZE]] +** The argument is assumed to point to a value of type sqlite3_int64. An +** advisory maximum amount of this file to memory map in bytes. +** ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -924,7 +908,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_PRAGMA 14 #define SQLITE_FCNTL_BUSYHANDLER 15 #define SQLITE_FCNTL_TEMPFILENAME 16 -#define SQLITE_FCNTL_GETFD 17 +#define SQLITE_FCNTL_MMAP_SIZE 18 /* ** CAPI3REF: Mutex Handle diff --git a/src/wal.c b/src/wal.c index 051b77f515..4602eb2c34 100644 --- a/src/wal.c +++ b/src/wal.c @@ -412,7 +412,6 @@ struct Wal { sqlite3_vfs *pVfs; /* The VFS used to create pDbFd */ sqlite3_file *pDbFd; /* File handle for the database file */ sqlite3_file *pWalFd; /* File handle for WAL file */ - Pager *pPager; /* Pager object */ u32 iCallback; /* Value to pass to log callback (or 0) */ i64 mxWalSize; /* Truncate WAL to this size upon reset */ int nWiData; /* Size of array apWiData */ @@ -1252,7 +1251,6 @@ static void walIndexClose(Wal *pWal, int isDelete){ */ int sqlite3WalOpen( sqlite3_vfs *pVfs, /* vfs module to open wal and wal-index */ - Pager *pPager, /* Pager object handle */ sqlite3_file *pDbFd, /* The open database file */ const char *zWalName, /* Name of the WAL file */ int bNoShm, /* True to run in heap-memory mode */ @@ -1293,7 +1291,6 @@ int sqlite3WalOpen( pRet->zWalName = zWalName; pRet->syncHeader = 1; pRet->padToSectorBoundary = 1; - pRet->pPager = pPager; pRet->exclusiveMode = (bNoShm ? WAL_HEAPMEMORY_MODE: WAL_NORMAL_MODE); /* Open file handle on the write-ahead log file. */ @@ -1725,19 +1722,18 @@ static int walCheckpoint( rc = sqlite3OsSync(pWal->pWalFd, sync_flags); } - /* If the database file is currently smaller than mxPage pages in size, - ** the call below issues an SQLITE_FCNTL_SIZE_HINT to the OS layer to - ** inform it that it is likely to grow to that size. - ** - ** Additionally, if the pager is using mmap(), then the call to - ** SetFilesize() guarantees that the mapping is not larger than mxPage - ** pages. This makes the sqlite3OsTruncate() call below safe - no pages - ** that are part of the mapped region will be truncated away. */ + /* If the database may grow as a result of this checkpoint, hint + ** about the eventual size of the db file to the VFS layer. + */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); - rc = sqlite3PagerSetFilesize(pWal->pPager, nReq); + rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); + if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); + } } + /* Iterate through the contents of the WAL, copying data to the db file. */ while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ i64 iOffset; @@ -1749,7 +1745,7 @@ static int walCheckpoint( if( rc!=SQLITE_OK ) break; iOffset = (iDbpage-1)*(i64)szPage; testcase( IS_BIG_INT(iOffset) ); - rc = sqlite3PagerWriteData(pWal->pPager, zBuf, szPage, iOffset); + rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); if( rc!=SQLITE_OK ) break; } diff --git a/src/wal.h b/src/wal.h index 96d4ff9adf..ff7624af63 100644 --- a/src/wal.h +++ b/src/wal.h @@ -53,8 +53,7 @@ typedef struct Wal Wal; /* Open and close a connection to a write-ahead log. */ -int sqlite3WalOpen( - sqlite3_vfs*, Pager *, sqlite3_file*, const char *, int, i64, Wal**); +int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**); int sqlite3WalClose(Wal *pWal, int sync_flags, int, u8 *); /* Set the limiting size of a WAL file. */ From a1afc7425a42b774b307416b3256d28c1d4a185f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 13:50:49 +0000 Subject: [PATCH 025/140] Fix a case in the pager where an xFetch() reference was being leaked following an OOM error. FossilOrigin-Name: 5885ba6ce768658ec25b60747430d147b315b55c --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os_unix.c | 2 +- src/pager.c | 1 + src/pager.h | 1 - 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 8d71dc88ab..5e9f5ad099 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Replace\sthe\ssqlite3_io_methods.xMremap\sinterface\swith\ssqlite3_io_methods.xFetch\sand\sxUnfetch. -D 2013-03-23T21:00:41.457 +C Fix\sa\scase\sin\sthe\spager\swhere\san\sxFetch()\sreference\swas\sbeing\sleaked\sfollowing\san\sOOM\serror. +D 2013-03-25T13:50:49.946 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,10 +160,10 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c a7ec2ddc43fb2684bfd8cb30db15ea6cce3298e9 F src/os.h 782980cd0d840b4240a6e29ba9f8ef0ca0d750ca F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c d8cdf331ad08650fc8397eee79b55548070077b0 +F src/os_unix.c 1e61abcb664f3430ce13e5c5f32b2b2e204a6f23 F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac -F src/pager.c 2e63de7185c2a573d26e5087869a9e48326076f9 -F src/pager.h faf4bed79e80cb6527209dfe5cfb08aa5764b941 +F src/pager.c 3da35f3287791354f170824fd847e75ba9f2a0dc +F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 60b9f5e4dd2af54975ba78437239f0bebd472fd2 -R eee4d88d6400cf65980c5b201532d2c9 +P 1431be95579160fb70408d43e17fc23c7b69ab4a +R 2702651dc0367254a02a7905a85901b0 U dan -Z eabd20cdc7aabbc65591f96e4f6dff04 +Z 4827c382c08dac8144315d5de3ed2c37 diff --git a/manifest.uuid b/manifest.uuid index f378aa2e8b..37accb35b1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1431be95579160fb70408d43e17fc23c7b69ab4a \ No newline at end of file +5885ba6ce768658ec25b60747430d147b315b55c \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 334719200a..d865810570 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -1807,6 +1807,7 @@ end_unlock: ** the requested locking level, this routine is a no-op. */ static int unixUnlock(sqlite3_file *id, int eFileLock){ + assert( eFileLock==SHARED_LOCK || ((unixFile *)id)->nFetchOut==0 ); return posixUnlock(id, eFileLock, 0); } @@ -4531,7 +4532,6 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ } if( nMap!=pFd->mmapSize ){ - void *pNew; unixUnmapfile(pFd); if( nMap>0 ){ diff --git a/src/pager.c b/src/pager.c index 7e00e1431a..d6e6b5b440 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3870,6 +3870,7 @@ static int pagerAcquireMapPage( }else{ *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); if( p==0 ){ + sqlite3OsUnfetch(pPager->fd, pData); return SQLITE_NOMEM; } p->pExtra = (void *)&p[1]; diff --git a/src/pager.h b/src/pager.h index 066d1f11d7..81ab30c115 100644 --- a/src/pager.h +++ b/src/pager.h @@ -174,7 +174,6 @@ int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); -int sqlite3PagerSetFilesize(Pager *, i64); #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); From 11ac84635b840afb91178063bd3fa30fff584963 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 14:31:39 +0000 Subject: [PATCH 026/140] Do not return SQLITE_IOERR when the user attempts to open a small file that is not a database with mmap enabled. Instead return SQLITE_NOTADB. FossilOrigin-Name: bbcaab3e80d0ff776d8567094b137d1483b3377b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 5e9f5ad099..01d74c7724 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scase\sin\sthe\spager\swhere\san\sxFetch()\sreference\swas\sbeing\sleaked\sfollowing\san\sOOM\serror. -D 2013-03-25T13:50:49.946 +C Do\snot\sreturn\sSQLITE_IOERR\swhen\sthe\suser\sattempts\sto\sopen\sa\ssmall\sfile\sthat\sis\snot\sa\sdatabase\swith\smmap\senabled.\sInstead\sreturn\sSQLITE_NOTADB. +D 2013-03-25T14:31:39.689 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h 782980cd0d840b4240a6e29ba9f8ef0ca0d750ca F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 1e61abcb664f3430ce13e5c5f32b2b2e204a6f23 F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac -F src/pager.c 3da35f3287791354f170824fd847e75ba9f2a0dc +F src/pager.c 2dd59f366b519d01d08b44f615058e7cfacd0bd7 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 1431be95579160fb70408d43e17fc23c7b69ab4a -R 2702651dc0367254a02a7905a85901b0 +P 5885ba6ce768658ec25b60747430d147b315b55c +R c243df2223b3431445e14c1469df7b20 U dan -Z 4827c382c08dac8144315d5de3ed2c37 +Z 6b851eca8762089c75296a1160d8987e diff --git a/manifest.uuid b/manifest.uuid index 37accb35b1..b4cdb9d207 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5885ba6ce768658ec25b60747430d147b315b55c \ No newline at end of file +bbcaab3e80d0ff776d8567094b137d1483b3377b \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index d6e6b5b440..927126c393 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5051,7 +5051,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( !pPager->tempFile && ( pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0 - || pPager->bUseFetch /* TODO: Currently required for xUnfetch(0) only. */ + || pPager->bUseFetch )){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous @@ -5078,7 +5078,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( nPage>0 ){ IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); - if( rc!=SQLITE_OK ){ + if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ goto failed; } }else{ From aef49d7141d9bbbc68a2c8ce7bd17cdafb06aa6b Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 16:28:54 +0000 Subject: [PATCH 027/140] Remove unnecessary code to round the size of a memory mapping to 4KB from os_unix.c. Rename SQLITE_IOERR_MREMAP to SQLITE_IOERR_MMAP. Fix other small issues in os_unix.c. FossilOrigin-Name: dce35c01a5fe66d2970075b1e3f0376026485e4c --- manifest | 14 ++++++------- manifest.uuid | 2 +- src/os_unix.c | 54 +++++++++++++++++++++++++++++++++++++++---------- src/sqlite.h.in | 2 +- 4 files changed, 52 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 01d74c7724..919b150ea9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sreturn\sSQLITE_IOERR\swhen\sthe\suser\sattempts\sto\sopen\sa\ssmall\sfile\sthat\sis\snot\sa\sdatabase\swith\smmap\senabled.\sInstead\sreturn\sSQLITE_NOTADB. -D 2013-03-25T14:31:39.689 +C Remove\sunnecessary\scode\sto\sround\sthe\ssize\sof\sa\smemory\smapping\sto\s4KB\sfrom\sos_unix.c.\sRename\sSQLITE_IOERR_MREMAP\sto\sSQLITE_IOERR_MMAP.\sFix\sother\ssmall\sissues\sin\sos_unix.c. +D 2013-03-25T16:28:54.437 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c a7ec2ddc43fb2684bfd8cb30db15ea6cce3298e9 F src/os.h 782980cd0d840b4240a6e29ba9f8ef0ca0d750ca F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 1e61abcb664f3430ce13e5c5f32b2b2e204a6f23 +F src/os_unix.c 82efc58c818c69e191f5560935037af365f99ff5 F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac F src/pager.c 2dd59f366b519d01d08b44f615058e7cfacd0bd7 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -176,7 +176,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in f41949e1e778043dd9fc23b33b96e7a6e20de4d6 +F src/sqlite.h.in a0866dafaa412d1e89abbc78d2b44b12cf95d1d1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5885ba6ce768658ec25b60747430d147b315b55c -R c243df2223b3431445e14c1469df7b20 +P bbcaab3e80d0ff776d8567094b137d1483b3377b +R fd25f18074b1cc787a38214a338c9127 U dan -Z 6b851eca8762089c75296a1160d8987e +Z ae05d407cd866087491e3302af269873 diff --git a/manifest.uuid b/manifest.uuid index b4cdb9d207..23ee7e3b08 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bbcaab3e80d0ff776d8567094b137d1483b3377b \ No newline at end of file +dce35c01a5fe66d2970075b1e3f0376026485e4c \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index d865810570..a97aba87f1 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4493,15 +4493,8 @@ static int unixShmUnmap( #endif /* #ifndef SQLITE_OMIT_WAL */ /* -** Arguments x and y are both integers. Argument y must be a power of 2. -** Round x up to the nearest integer multiple of y. For example: -** -** ROUNDUP(0, 8) -> 0 -** ROUNDUP(13, 8) -> 16 -** ROUNDUP(32, 8) -> 32 +** If it is currently memory mapped, unmap file pFd. */ -#define ROUNDUP(x,y) (((x)+y-1)&~(y-1)) - static void unixUnmapfile(unixFile *pFd){ assert( pFd->nFetchOut==0 ); if( pFd->pMapRegion ){ @@ -4512,6 +4505,22 @@ static void unixUnmapfile(unixFile *pFd){ } } +/* +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_SIZE, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. +*/ static int unixMapfile(unixFile *pFd, i64 nByte){ i64 nMap = nByte; int rc; @@ -4538,19 +4547,32 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ void *pNew; int flags = PROT_READ; if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - pNew = mmap(0, ROUNDUP(nMap, 4096), flags, MAP_SHARED, pFd->h, 0); + pNew = mmap(0, nMap, flags, MAP_SHARED, pFd->h, 0); if( pNew==MAP_FAILED ){ - return SQLITE_IOERR_MREMAP; + return SQLITE_IOERR_MMAP; } pFd->pMapRegion = pNew; - pFd->mmapOrigsize = pFd->mmapSize = nMap; + pFd->mmapSize = nMap; + pFd->mmapOrigsize = nMap; } } return SQLITE_OK; } +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling unixUnfetch(). +*/ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ unixFile *pFd = (unixFile *)fd; /* The underlying database file */ *pp = 0; @@ -4568,9 +4590,19 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ return SQLITE_OK; } +/* +** If the second argument is non-NULL, then this function releases a +** reference obtained by an earlier call to unixFetch(). Or, if the second +** argument is NULL, then this function is being called to inform the VFS +** layer that, according to POSIX, any existing mapping may now be invalid +** and should be unmapped. +*/ static int unixUnfetch(sqlite3_file *fd, void *p){ unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ assert( (p==0)==(pFd->nFetchOut==0) ); if( p ){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 666a2e084d..7bbd3fa000 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -470,7 +470,7 @@ int sqlite3_exec( #define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) #define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) #define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) -#define SQLITE_IOERR_MREMAP (SQLITE_IOERR | (24<<8)) +#define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) From df737fe6f51522c478235ff6ddc478243f718bf2 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 17:00:24 +0000 Subject: [PATCH 028/140] Change the signature of the xUnfetch method to "int (*xUnfetch)(sqlite3_file*, sqlite3_int64 iOfst, void *p)". FossilOrigin-Name: 115b830509e8f0aa9d5965c1e9cd4f2ed9d01938 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/os.c | 4 ++-- src/os.h | 2 +- src/os_unix.c | 18 ++++++++++++------ src/pager.c | 10 +++++----- src/sqlite.h.in | 2 +- 7 files changed, 32 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index 919b150ea9..143bb5bf58 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sunnecessary\scode\sto\sround\sthe\ssize\sof\sa\smemory\smapping\sto\s4KB\sfrom\sos_unix.c.\sRename\sSQLITE_IOERR_MREMAP\sto\sSQLITE_IOERR_MMAP.\sFix\sother\ssmall\sissues\sin\sos_unix.c. -D 2013-03-25T16:28:54.437 +C Change\sthe\ssignature\sof\sthe\sxUnfetch\smethod\sto\s"int\s(*xUnfetch)(sqlite3_file*,\ssqlite3_int64\siOfst,\svoid\s*p)". +D 2013-03-25T17:00:24.194 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,12 +157,12 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c a7ec2ddc43fb2684bfd8cb30db15ea6cce3298e9 -F src/os.h 782980cd0d840b4240a6e29ba9f8ef0ca0d750ca +F src/os.c 17c6e6e4a17135ec1c7c468f8d0b97ec6f8a9264 +F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 82efc58c818c69e191f5560935037af365f99ff5 +F src/os_unix.c 664fc178a9478246f6c84e46292e77fb5bee4431 F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac -F src/pager.c 2dd59f366b519d01d08b44f615058e7cfacd0bd7 +F src/pager.c d0cdf47664e487f735b4bea485ee70637ed34a55 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -176,7 +176,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c e1c6f6abdf9f359f4e735cb8ae11d2f359bf52a9 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in a0866dafaa412d1e89abbc78d2b44b12cf95d1d1 +F src/sqlite.h.in b04790da19184e883da788d07688f16230d2cb6b F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P bbcaab3e80d0ff776d8567094b137d1483b3377b -R fd25f18074b1cc787a38214a338c9127 +P dce35c01a5fe66d2970075b1e3f0376026485e4c +R 12377e21e3793332cede5cbfb008d2ac U dan -Z ae05d407cd866087491e3302af269873 +Z 1c478466922e4f7b4b376664d80146c7 diff --git a/manifest.uuid b/manifest.uuid index 23ee7e3b08..838ea0b60b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dce35c01a5fe66d2970075b1e3f0376026485e4c \ No newline at end of file +115b830509e8f0aa9d5965c1e9cd4f2ed9d01938 \ No newline at end of file diff --git a/src/os.c b/src/os.c index 08e0d42172..f2d85bba4d 100644 --- a/src/os.c +++ b/src/os.c @@ -144,8 +144,8 @@ int sqlite3OsShmMap( int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ return id->pMethods->xFetch(id, iOff, iAmt, pp); } -int sqlite3OsUnfetch(sqlite3_file *id, void *p){ - return id->pMethods->xUnfetch(id, p); +int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ + return id->pMethods->xUnfetch(id, iOff, p); } /* diff --git a/src/os.h b/src/os.h index a7dacfb6ea..82e07ee1cb 100644 --- a/src/os.h +++ b/src/os.h @@ -260,7 +260,7 @@ int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmUnmap(sqlite3_file *id, int); int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); -int sqlite3OsUnfetch(sqlite3_file *, void *); +int sqlite3OsUnfetch(sqlite3_file *, i64, void *); /* diff --git a/src/os_unix.c b/src/os_unix.c index a97aba87f1..00c0088f8a 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4591,13 +4591,16 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ } /* -** If the second argument is non-NULL, then this function releases a -** reference obtained by an earlier call to unixFetch(). Or, if the second -** argument is NULL, then this function is being called to inform the VFS -** layer that, according to POSIX, any existing mapping may now be invalid -** and should be unmapped. +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to unixFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the unixFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. */ -static int unixUnfetch(sqlite3_file *fd, void *p){ +static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ unixFile *pFd = (unixFile *)fd; /* The underlying database file */ /* If p==0 (unmap the entire file) then there must be no outstanding @@ -4605,6 +4608,9 @@ static int unixUnfetch(sqlite3_file *fd, void *p){ ** then there must be at least one outstanding. */ assert( (p==0)==(pFd->nFetchOut==0) ); + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + if( p ){ pFd->nFetchOut--; }else{ diff --git a/src/pager.c b/src/pager.c index 927126c393..06a7eefee1 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3091,7 +3091,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); if( rc!=SQLITE_OK || changed ){ pager_reset(pPager); - if( pPager->bUseFetch ) sqlite3OsUnfetch(pPager->fd, 0); + if( pPager->bUseFetch ) sqlite3OsUnfetch(pPager->fd, 0, 0); } return rc; @@ -3870,7 +3870,7 @@ static int pagerAcquireMapPage( }else{ *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); if( p==0 ){ - sqlite3OsUnfetch(pPager->fd, pData); + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1) * pPager->pageSize, pData); return SQLITE_NOMEM; } p->pExtra = (void *)&p[1]; @@ -3903,7 +3903,7 @@ static void pagerReleaseMapPage(PgHdr *pPg){ pPager->pFree = pPg; assert( pPager->fd->pMethods->iVersion>=3 ); - sqlite3OsUnfetch(pPager->fd, pPg->pData); + sqlite3OsUnfetch(pPager->fd, (i64)(pPg->pgno-1)*pPager->pageSize, pPg->pData); } /* @@ -5095,7 +5095,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** to be the right size but is not actually valid. Avoid this ** possibility by unmapping the db here. */ if( pPager->bUseFetch ){ - sqlite3OsUnfetch(pPager->fd, 0); + sqlite3OsUnfetch(pPager->fd, 0, 0); } } } @@ -5245,7 +5245,7 @@ int sqlite3PagerAcquire( if( pPg==0 ){ rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); }else{ - sqlite3OsUnfetch(pPager->fd, pData); + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1)*pPager->pageSize, pData); } if( pPg ){ assert( rc==SQLITE_OK ); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 7bbd3fa000..b68287b0e7 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -730,7 +730,7 @@ struct sqlite3_io_methods { int (*xShmUnmap)(sqlite3_file*, int deleteFlag); /* Methods above are valid for version 2 */ int (*xFetch)(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); - int (*xUnfetch)(sqlite3_file*, void *p); + int (*xUnfetch)(sqlite3_file*, sqlite3_int64 iOfst, void *p); /* Methods above are valid for version 3 */ /* Additional methods may be added in future releases */ }; From a64d5a15b7d85ee49c5632ee37f6c0bd3242d258 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 18:25:49 +0000 Subject: [PATCH 029/140] Simulate OOM errors in the sqlite3OsFetch() function. Run malloc.test as part of the "mmap" permutation. FossilOrigin-Name: 77443ef2cd0b29b7822eea544ab8c6c4a93cb67b --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os.c | 1 + test/permutations.test | 2 +- test/speed1p.test | 1 + 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 143bb5bf58..e0bcdeed1d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\ssignature\sof\sthe\sxUnfetch\smethod\sto\s"int\s(*xUnfetch)(sqlite3_file*,\ssqlite3_int64\siOfst,\svoid\s*p)". -D 2013-03-25T17:00:24.194 +C Simulate\sOOM\serrors\sin\sthe\ssqlite3OsFetch()\sfunction.\sRun\smalloc.test\sas\spart\sof\sthe\s"mmap"\spermutation. +D 2013-03-25T18:25:49.003 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c 17c6e6e4a17135ec1c7c468f8d0b97ec6f8a9264 +F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 664fc178a9478246f6c84e46292e77fb5bee4431 @@ -657,7 +657,7 @@ F test/pageropt.test 290cd59782b1890f02bb33795571facfc5ccac43 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test cf5d475f024cfd3fbb0255b8e43bf0c6887c6bb4 +F test/permutations.test 5b5397f33b9d129b734b9c57a5091ad494e025c5 F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -730,7 +730,7 @@ F test/softheap1.test c16709a16ad79fa43b32929b2e623d1d117ccf53 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb -F test/speed1p.test c4a469f29f135f4d76c55b1f2a52f36e209466cc +F test/speed1p.test 9912e8f915f1de2c82fc019a63920bf9dcfc69f7 F test/speed2.test 53177056baf6556dcbdcf032bbdfc41c1aa74ded F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P dce35c01a5fe66d2970075b1e3f0376026485e4c -R 12377e21e3793332cede5cbfb008d2ac +P 115b830509e8f0aa9d5965c1e9cd4f2ed9d01938 +R 81bdcfaea4ca0021abe5572b7c1cd51d U dan -Z 1c478466922e4f7b4b376664d80146c7 +Z 1c0203030ba88df7bcf075001cefd19c diff --git a/manifest.uuid b/manifest.uuid index 838ea0b60b..2ac770c1e3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -115b830509e8f0aa9d5965c1e9cd4f2ed9d01938 \ No newline at end of file +77443ef2cd0b29b7822eea544ab8c6c4a93cb67b \ No newline at end of file diff --git a/src/os.c b/src/os.c index f2d85bba4d..2ba857e2aa 100644 --- a/src/os.c +++ b/src/os.c @@ -142,6 +142,7 @@ int sqlite3OsShmMap( } int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + DO_OS_MALLOC_TEST(id); return id->pMethods->xFetch(id, iOff, iAmt, pp); } int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ diff --git a/test/permutations.test b/test/permutations.test index 6e6b5c90f7..9ff3a801c2 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -143,7 +143,7 @@ test_suite "mmap" -prefix "mm-" -description { } -presql { pragma mmap_size = -65536; } -files [ - test_set $allquicktests -exclude *malloc* *ioerr* *fault* + test_set $allquicktests -exclude *malloc* *ioerr* *fault* -include malloc.test ] test_suite "valgrind" -prefix "" -description { diff --git a/test/speed1p.test b/test/speed1p.test index 915f165354..df5abedde3 100644 --- a/test/speed1p.test +++ b/test/speed1p.test @@ -65,6 +65,7 @@ proc number_name {n} { # do_test speed1p-1.0 { execsql { + PRAGMA mmap_size=1000000; PRAGMA page_size=1024; PRAGMA cache_size=500; PRAGMA locking_mode=EXCLUSIVE; From 893c0ffc290bb59bcd8f505bec45087211579525 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 19:05:07 +0000 Subject: [PATCH 030/140] Add a test that simulates an error in mmap(). FossilOrigin-Name: 6ec7367d8e98425f00eeb8215ca8964313c1d0b7 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/os_unix.c | 7 +++++-- src/test_syscall.c | 20 +++++++++++++++++++- test/malloc_common.tcl | 2 +- test/sysfault.test | 31 +++++++++++++++++++++++++++++++ 6 files changed, 66 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index e0bcdeed1d..215ae805d4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simulate\sOOM\serrors\sin\sthe\ssqlite3OsFetch()\sfunction.\sRun\smalloc.test\sas\spart\sof\sthe\s"mmap"\spermutation. -D 2013-03-25T18:25:49.003 +C Add\sa\stest\sthat\ssimulates\san\serror\sin\smmap(). +D 2013-03-25T19:05:07.846 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9a804abbd3cae82d196e4d33aba13239e32522a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 664fc178a9478246f6c84e46292e77fb5bee4431 +F src/os_unix.c 6815e3dd9a1507f2e36ba5a8dcd36b67ac60f3b0 F src/os_win.c 386f8c034b177b672f7819ddc5d80be6c8d593ac F src/pager.c d0cdf47664e487f735b4bea485ee70637ed34a55 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -226,7 +226,7 @@ F src/test_spellfix.c 56dfa6d583ac34f61af0834d7b58d674e7e18e13 F src/test_sqllog.c 8acb843ddb9928dea8962e31bb09f421a72ffccb F src/test_stat.c d1569c7a4839f13e80187e2c26b2ab4da2d03935 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd -F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae +F src/test_syscall.c fe018f95049d4f24e036d1d649516b60689b4e57 F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c e286f2173563f2a1747c24bcda6b9d030bf4f4e4 F src/test_vfs.c fb16b2d9938cf0c1afc5a423b55b952fcc024275 @@ -611,7 +611,7 @@ F test/mallocH.test 79b65aed612c9b3ed2dcdaa727c85895fd1bfbdb F test/mallocI.test a88c2b9627c8506bf4703d8397420043a786cdb6 F test/mallocJ.test b5d1839da331d96223e5f458856f8ffe1366f62e F test/mallocK.test d79968641d1b70d88f6c01bdb9a7eb4a55582cc9 -F test/malloc_common.tcl 2930895b0962823ec679853e67e58dd6d8198b3c +F test/malloc_common.tcl 9a98856549bfb3fab205edbc1317216edc52e70d F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f F test/memdb.test 708a028d6d373e5b3842e4bdc8ba80998c9a4da6 @@ -747,7 +747,7 @@ F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test bea9bf329bff733c791310244617c2a76974e64a -F test/sysfault.test c79441d88d23696fbec7b147dba98d42a04f523f +F test/sysfault.test 3258d6c8213b44f2669994d798a5eb5327b40146 F test/table.test a59d985ca366e39b17b175f387f9d5db5a18d4e2 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 @@ -1039,7 +1039,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 115b830509e8f0aa9d5965c1e9cd4f2ed9d01938 -R 81bdcfaea4ca0021abe5572b7c1cd51d +P 77443ef2cd0b29b7822eea544ab8c6c4a93cb67b +R 602dbedb509e1b6f815316de036c2541 U dan -Z 1c0203030ba88df7bcf075001cefd19c +Z 3ee6172e1f5d6ed5486bb4fa69d324f2 diff --git a/manifest.uuid b/manifest.uuid index 2ac770c1e3..50a8aea1fc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -77443ef2cd0b29b7822eea544ab8c6c4a93cb67b \ No newline at end of file +6ec7367d8e98425f00eeb8215ca8964313c1d0b7 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 00c0088f8a..8175bd5c9a 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -444,6 +444,9 @@ static struct unix_syscall { { "fchown", (sqlite3_syscall_ptr)posixFchown, 0 }, #define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) + { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, +#define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[21].pCurrent) + }; /* End of the overrideable system calls */ /* @@ -4547,7 +4550,7 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ void *pNew; int flags = PROT_READ; if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - pNew = mmap(0, nMap, flags, MAP_SHARED, pFd->h, 0); + pNew = osMmap(0, nMap, flags, MAP_SHARED, pFd->h, 0); if( pNew==MAP_FAILED ){ return SQLITE_IOERR_MMAP; } @@ -7186,7 +7189,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==21 ); + assert( ArraySize(aSyscall)==22 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ diff --git a/src/test_syscall.c b/src/test_syscall.c index d484f22db4..0b053fc599 100644 --- a/src/test_syscall.c +++ b/src/test_syscall.c @@ -23,7 +23,7 @@ ** ** open close access getcwd stat fstat ** ftruncate fcntl read pread pread64 write -** pwrite pwrite64 fchmod fallocate +** pwrite pwrite64 fchmod fallocate mmap ** ** test_syscall uninstall ** Uninstall all wrapper functions. @@ -81,6 +81,7 @@ /* From test1.c */ extern const char *sqlite3TestErrorName(int); +#include #include #include @@ -106,6 +107,7 @@ static int ts_pwrite(int fd, const void *aBuf, size_t nBuf, off_t off); static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, off_t off); static int ts_fchmod(int fd, mode_t mode); static int ts_fallocate(int fd, off_t off, off_t len); +static void *ts_mmap(void *, size_t, int, int, int, off_t); struct TestSyscallArray { @@ -131,6 +133,7 @@ struct TestSyscallArray { /* 13 */ { "pwrite64", (sqlite3_syscall_ptr)ts_pwrite64, 0, 0, 0 }, /* 14 */ { "fchmod", (sqlite3_syscall_ptr)ts_fchmod, 0, 0, 0 }, /* 15 */ { "fallocate", (sqlite3_syscall_ptr)ts_fallocate, 0, 0, 0 }, + /* 16 */ { "mmap", (sqlite3_syscall_ptr)ts_mmap, 0, 0, 0 }, { 0, 0, 0, 0, 0 } }; @@ -152,6 +155,7 @@ struct TestSyscallArray { aSyscall[13].xOrig) #define orig_fchmod ((int(*)(int,mode_t))aSyscall[14].xOrig) #define orig_fallocate ((int(*)(int,off_t,off_t))aSyscall[15].xOrig) +#define orig_mmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[16].xOrig) /* ** This function is called exactly once from within each invocation of a @@ -377,6 +381,20 @@ static int ts_fallocate(int fd, off_t off, off_t len){ return orig_fallocate(fd, off, len); } +static void *ts_mmap( + void *pAddr, + size_t nByte, + int prot, + int flags, + int fd, + off_t iOff +){ + if( tsIsFailErrno("mmap") ){ + return MAP_FAILED; + } + return orig_mmap(pAddr, nByte, prot, flags, fd, iOff); +} + static int test_syscall_install( void * clientData, Tcl_Interp *interp, diff --git a/test/malloc_common.tcl b/test/malloc_common.tcl index 5937b958f8..2ac619b1df 100644 --- a/test/malloc_common.tcl +++ b/test/malloc_common.tcl @@ -264,7 +264,7 @@ proc faultsim_test_result_int {args} { set t [list $testrc $testresult] set r $args if { ($testnfail==0 && $t != [lindex $r 0]) || [lsearch $r $t]<0 } { - error "nfail=$testnfail rc=$testrc result=$testresult" + error "nfail=$testnfail rc=$testrc result=$testresult list=$r" } } diff --git a/test/sysfault.test b/test/sysfault.test index 07d525ca9e..bf7df5df5f 100644 --- a/test/sysfault.test +++ b/test/sysfault.test @@ -243,5 +243,36 @@ do_faultsim_test 3 -faults vfsfault-* -prep { faultsim_test_result {0 20000} } +#------------------------------------------------------------------------- +# Test errors in mmap(). +# +proc vfsfault_install {} { + test_syscall reset + test_syscall install {mmap} +} + +faultsim_delete_and_reopen +execsql { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); +} +faultsim_save_and_close + +do_faultsim_test 4 -faults vfsfault-* -prep { + faultsim_restore_and_reopen + file_control_chunksize_test db main 8192 + execsql { + PRAGMA mmap_size = 1000000; + } +} -body { + test_syscall errno mmap EACCES + + execsql { + SELECT * FROM t1; + } +} -test { + faultsim_test_result {0 {1 2}} {1 {disk I/O error}} +} + finish_test From b7e3a326fe84ff10e9b34a9c11c32e441ac0fc5b Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Mar 2013 20:30:13 +0000 Subject: [PATCH 031/140] Use mremap() on Linux. FossilOrigin-Name: 431aecc8600c29c203546e48d256510510238887 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 31 +++++++++++++++++++------------ 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 920c0ece2f..02d611b4ed 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\srecent\strunk\schanges\sinto\sthe\sexperimental-mmap\sbranch. -D 2013-03-25T19:57:26.572 +C Use\smremap()\son\sLinux. +D 2013-03-25T20:30:13.313 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 54b82f341b3c81630aa8c73efd2effe037b69270 +F src/os_unix.c a4d7076a6edd4e6a8594191880e540a31b2c8598 F src/os_win.c e4f17ddf79f2a9373e33ed70565e765d65324589 F src/pager.c 4e1b55e7a50db0466b06f8ec114799b432c18131 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 6ec7367d8e98425f00eeb8215ca8964313c1d0b7 5062db672c00c3365d51cd6b39815078f5b6b525 -R 868df99cd28788ba5be1e70b11892bee -U drh -Z 7385c5f656a5c7af6e1d7e1f0ea19ddb +P a607d63f0b6a3d3785e9385187d3e6b92e14fc70 +R 18e19b2d7d709a7b2b83b278bfc05f98 +U dan +Z f03c297d3465d1cb9dc069838022accc diff --git a/manifest.uuid b/manifest.uuid index 0965ca8390..e4f7651d39 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a607d63f0b6a3d3785e9385187d3e6b92e14fc70 \ No newline at end of file +431aecc8600c29c203546e48d256510510238887 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 9d2e2c130b..33be793423 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4544,21 +4544,28 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ } if( nMap!=pFd->mmapSize ){ - unixUnmapfile(pFd); + void *pNew = 0; - if( nMap>0 ){ - void *pNew; - int flags = PROT_READ; - if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - pNew = osMmap(0, nMap, flags, MAP_SHARED, pFd->h, 0); - if( pNew==MAP_FAILED ){ - return SQLITE_IOERR_MMAP; +#if defined(__linux__) && defined(_GNU_SOURCE) + if( pFd->pMapRegion && nMap>0 ){ + pNew = mremap(pFd->pMapRegion, pFd->mmapOrigsize, nMap, MREMAP_MAYMOVE); + }else +#endif + { + unixUnmapfile(pFd); + if( nMap>0 ){ + int flags = PROT_READ; + if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; + pNew = osMmap(0, nMap, flags, MAP_SHARED, pFd->h, 0); } - - pFd->pMapRegion = pNew; - pFd->mmapSize = nMap; - pFd->mmapOrigsize = nMap; } + + if( pNew==MAP_FAILED ){ + return SQLITE_IOERR_MMAP; + } + pFd->pMapRegion = pNew; + pFd->mmapSize = nMap; + pFd->mmapOrigsize = nMap; } return SQLITE_OK; From d1ab8065c1039db43eb413702cafb3baa500d69a Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Mar 2013 20:50:25 +0000 Subject: [PATCH 032/140] Add munmap and mremap to the set of os interfaces that can be overloaded in os_unix.c. FossilOrigin-Name: 8776047bd776bbf266eb9c3b56683badb84ae73e --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os_unix.c | 20 +++++++++++++++----- test/syscall.test | 2 +- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 02d611b4ed..b0a5aaeb5f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\smremap()\son\sLinux. -D 2013-03-25T20:30:13.313 +C Add\smunmap\sand\smremap\sto\sthe\sset\sof\sos\sinterfaces\sthat\scan\sbe\soverloaded\nin\sos_unix.c. +D 2013-03-25T20:50:25.204 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c a4d7076a6edd4e6a8594191880e540a31b2c8598 +F src/os_unix.c 6b3f972a8b515ce151d1d075080b10d52adbfbd6 F src/os_win.c e4f17ddf79f2a9373e33ed70565e765d65324589 F src/pager.c 4e1b55e7a50db0466b06f8ec114799b432c18131 F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c @@ -747,7 +747,7 @@ F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 -F test/syscall.test cf59bafccb143f8df21c043b3201d5afa9fe2474 +F test/syscall.test a653783d985108c4912cc64d341ffbbb55ad2806 F test/sysfault.test 3258d6c8213b44f2669994d798a5eb5327b40146 F test/table.test a59d985ca366e39b17b175f387f9d5db5a18d4e2 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a607d63f0b6a3d3785e9385187d3e6b92e14fc70 -R 18e19b2d7d709a7b2b83b278bfc05f98 -U dan -Z f03c297d3465d1cb9dc069838022accc +P 431aecc8600c29c203546e48d256510510238887 +R 420b4ff666854b5074068c14722ea276 +U drh +Z feb03effee9a1e5215de46a6338d0c52 diff --git a/manifest.uuid b/manifest.uuid index e4f7651d39..5b13e95f75 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -431aecc8600c29c203546e48d256510510238887 \ No newline at end of file +8776047bd776bbf266eb9c3b56683badb84ae73e \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 33be793423..625f413406 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -447,6 +447,16 @@ static struct unix_syscall { { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, #define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[21].pCurrent) + { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, +#define osMunmap ((void*(*)(void*,size_t))aSyscall[22].pCurrent) + +#if defined(__linux__) && defined(_GNU_SOURCE) + { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, +#else + { "mremap", (sqlite3_syscall_ptr)0, 0 }, +#endif +#define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent) + }; /* End of the overrideable system calls */ /* @@ -4005,7 +4015,7 @@ static void unixShmPurge(unixFile *pFd){ sqlite3_mutex_free(p->mutex); for(i=0; inRegion; i++){ if( p->h>=0 ){ - munmap(p->apRegion[i], p->szRegion); + osMunmap(p->apRegion[i], p->szRegion); }else{ sqlite3_free(p->apRegion[i]); } @@ -4278,7 +4288,7 @@ static int unixShmMap( while(pShmNode->nRegion<=iRegion){ void *pMem; if( pShmNode->h>=0 ){ - pMem = mmap(0, szRegion, + pMem = osMmap(0, szRegion, pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, MAP_SHARED, pShmNode->h, szRegion*(i64)pShmNode->nRegion ); @@ -4501,7 +4511,7 @@ static int unixShmUnmap( static void unixUnmapfile(unixFile *pFd){ assert( pFd->nFetchOut==0 ); if( pFd->pMapRegion ){ - munmap(pFd->pMapRegion, pFd->mmapOrigsize); + osMunmap(pFd->pMapRegion, pFd->mmapOrigsize); pFd->pMapRegion = 0; pFd->mmapSize = 0; pFd->mmapOrigsize = 0; @@ -4548,7 +4558,7 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ #if defined(__linux__) && defined(_GNU_SOURCE) if( pFd->pMapRegion && nMap>0 ){ - pNew = mremap(pFd->pMapRegion, pFd->mmapOrigsize, nMap, MREMAP_MAYMOVE); + pNew = osMremap(pFd->pMapRegion, pFd->mmapOrigsize, nMap, MREMAP_MAYMOVE); }else #endif { @@ -7196,7 +7206,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==22 ); + assert( ArraySize(aSyscall)==24 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ diff --git a/test/syscall.test b/test/syscall.test index b9a3b196e3..5bf1225867 100644 --- a/test/syscall.test +++ b/test/syscall.test @@ -60,7 +60,7 @@ foreach s { open close access getcwd stat fstat ftruncate fcntl read pread write pwrite fchmod fallocate pread64 pwrite64 unlink openDirectory mkdir rmdir - statvfs fchown umask mmap + statvfs fchown umask mmap munmap mremap } { if {[test_syscall exists $s]} {lappend syscall_list $s} } From 0d0614bdc6e59c1cb52bc79fdf8dafbbc78f57f9 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Mar 2013 23:09:28 +0000 Subject: [PATCH 033/140] Memory-mapped I/O is now on by default. The "PRAGMA mmap_limit(N)" can be used to issue a hint to the VFS to limit mmap space to N bytes. The VFS is free to ignore that hint if desired. However, if "PRAGMA mmap_limit(0)" is used, xFetch is never called. FossilOrigin-Name: 1b37c4effdd03aa2ea938a71b4f22ed27391689b --- manifest | 42 +++++++++++++++++++++--------------------- manifest.uuid | 2 +- src/btree.c | 4 ++-- src/btree.h | 2 +- src/os_unix.c | 15 ++++++++------- src/pager.c | 22 +++++++--------------- src/pager.h | 2 +- src/pragma.c | 25 +++++++++++-------------- src/sqlite.h.in | 8 ++++---- src/sqliteInt.h | 1 - src/sqliteLimit.h | 7 +++++++ test/dbstatus2.test | 2 +- test/incrblob.test | 8 ++++---- test/mmap1.test | 21 ++++++++++----------- test/permutations.test | 6 +++--- test/speed1p.test | 1 - test/sysfault.test | 3 +-- test/wal5.test | 5 ++--- 18 files changed, 84 insertions(+), 92 deletions(-) diff --git a/manifest b/manifest index b0a5aaeb5f..65993928dd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smunmap\sand\smremap\sto\sthe\sset\sof\sos\sinterfaces\sthat\scan\sbe\soverloaded\nin\sos_unix.c. -D 2013-03-25T20:50:25.204 +C Memory-mapped\sI/O\sis\snow\son\sby\sdefault.\s\sThe\s"PRAGMA\smmap_limit(N)"\scan\sbe\nused\sto\sissue\sa\shint\sto\sthe\sVFS\sto\slimit\smmap\sspace\sto\sN\sbytes.\s\sThe\sVFS\nis\sfree\sto\signore\sthat\shint\sif\sdesired.\s\sHowever,\sif\s"PRAGMA\smmap_limit(0)"\nis\sused,\sxFetch\sis\snever\scalled. +D 2013-03-25T23:09:28.646 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,8 +121,8 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c ef238f2d88c6072cc0d6cff33b72c2a9044a15e4 -F src/btree.h d3259057a38494c4c4358e377032158c762e3d8b +F src/btree.c 771f0391556317de19e02c9533b256e8299dd59b +F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc @@ -160,15 +160,15 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 6b3f972a8b515ce151d1d075080b10d52adbfbd6 +F src/os_unix.c 57306f1d2a2d783dae4365446b9f9a5da8958559 F src/os_win.c e4f17ddf79f2a9373e33ed70565e765d65324589 -F src/pager.c 4e1b55e7a50db0466b06f8ec114799b432c18131 -F src/pager.h 241d72dc0905df042da165f086d03505cb0bb50c +F src/pager.c c532d51ccc74de8551a28eb932f3b1a1034017d3 +F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c 86c8088ac6a12d3f3be5f7394542651f03fa9a38 +F src/pragma.c 8875d32d54530300e6aa31a9d5426b2088a69b12 F src/prepare.c 310eaff1ee5f3c700b3545afb095cfe9346efc3a F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -176,11 +176,11 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in 457eb0ba9fbc331281dca53182c90035b087527f +F src/sqlite.h.in 921ba42ec3531fe8cf021492ffd781b161383f1b F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 -F src/sqliteInt.h 2c3d830ae78b046ebf939c905c023610e43c2796 -F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d +F src/sqliteInt.h 0f8f05ee4db4ba9120b38f7a3992b325698f6e8a +F src/sqliteLimit.h 83d0cd0ce3050a80930406d3c7aecb48c1ccdfc5 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 @@ -369,7 +369,7 @@ F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/date.test f3228180c87bbe5d39c9397bf001c0095c3821b9 F test/dbstatus.test 207e5b63fcb7b9c3bb8e1fdf38ebd4654ad0e54b -F test/dbstatus2.test 10418e62b3db5dca070f0c3eef3ea13946f339c2 +F test/dbstatus2.test f329941d5f4a8bc0ba6ec5735897ef0cf34e2f5f F test/default.test 6faf23ccb300114924353007795aa9a8ec0aa9dc F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa @@ -526,7 +526,7 @@ F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test 64f3cc1acde1b9161ccdd8e5bde3daefdb5b2617 F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 -F test/incrblob.test e81846d214f3637622620fbde7cd526781cfe328 +F test/incrblob.test 2100cb8964e4a106e5ca9bf80e2c5c3e8be33f77 F test/incrblob2.test edc3a96e557bd61fb39acc8d2edd43371fbbaa19 F test/incrblob3.test aedbb35ea1b6450c33b98f2b6ed98e5020be8dc7 F test/incrblob4.test 09be37d3dd996a31ea6993bba7837ece549414a8 @@ -630,7 +630,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test e3391ef6c069e5f5733b8339652ce37fd97e3487 +F test/mmap1.test 9fb71daaca72f66fcc7a3b3a37fc3db99988b57a F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 @@ -658,7 +658,7 @@ F test/pageropt.test 290cd59782b1890f02bb33795571facfc5ccac43 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test 5b5397f33b9d129b734b9c57a5091ad494e025c5 +F test/permutations.test d298a74abae42b72b3640dc5505d19d8a3eddc9e F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -731,7 +731,7 @@ F test/softheap1.test c16709a16ad79fa43b32929b2e623d1d117ccf53 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb -F test/speed1p.test 9912e8f915f1de2c82fc019a63920bf9dcfc69f7 +F test/speed1p.test c4a469f29f135f4d76c55b1f2a52f36e209466cc F test/speed2.test 53177056baf6556dcbdcf032bbdfc41c1aa74ded F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 @@ -748,7 +748,7 @@ F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test a653783d985108c4912cc64d341ffbbb55ad2806 -F test/sysfault.test 3258d6c8213b44f2669994d798a5eb5327b40146 +F test/sysfault.test 503f72712b2b21cb80dc9899e53c2e39484d0313 F test/table.test a59d985ca366e39b17b175f387f9d5db5a18d4e2 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 @@ -955,7 +955,7 @@ F test/wal.test 5baa9a0e682c0394012aa9a06a3d06cf92655fbf F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c -F test/wal5.test 091bf7500e97087f1a87b346c8707e93f42d4f80 +F test/wal5.test f4d0aee6a2cf09e326ed2459011d396b4fdf661a F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test b3ee739fe8f7586aaebdc2367f477ebcf3e3b034 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 431aecc8600c29c203546e48d256510510238887 -R 420b4ff666854b5074068c14722ea276 +P 8776047bd776bbf266eb9c3b56683badb84ae73e +R 1fba4ef23570acddc66df7d627b1cb5f U drh -Z feb03effee9a1e5215de46a6338d0c52 +Z c62aae7ee656f659e8aebd3681920a9a diff --git a/manifest.uuid b/manifest.uuid index 5b13e95f75..8e0647f651 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8776047bd776bbf266eb9c3b56683badb84ae73e \ No newline at end of file +1b37c4effdd03aa2ea938a71b4f22ed27391689b \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 59970ded7f..04cc20d5c5 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2134,11 +2134,11 @@ int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ ** Change the limit on the amount of the database file that may be ** memory mapped. */ -int sqlite3BtreeSetMmapSize(Btree *p, int nMap){ +int sqlite3BtreeSetMmapLimit(Btree *p, sqlite3_int64 mxMmap){ BtShared *pBt = p->pBt; assert( sqlite3_mutex_held(p->db->mutex) ); sqlite3BtreeEnter(p); - sqlite3PagerSetMmapsize(pBt->pPager, nMap); + sqlite3PagerSetMmapLimit(pBt->pPager, mxMmap); sqlite3BtreeLeave(p); return SQLITE_OK; } diff --git a/src/btree.h b/src/btree.h index f2cca2fc71..23f6ad7068 100644 --- a/src/btree.h +++ b/src/btree.h @@ -63,7 +63,7 @@ int sqlite3BtreeOpen( int sqlite3BtreeClose(Btree*); int sqlite3BtreeSetCacheSize(Btree*,int); -int sqlite3BtreeSetMmapSize(Btree*, int); +int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); int sqlite3BtreeSetSafetyLevel(Btree*,int,int,int); int sqlite3BtreeSyncDisabled(Btree*); int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); diff --git a/src/os_unix.c b/src/os_unix.c index 625f413406..8a8516bcda 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -225,6 +225,11 @@ struct unixFile { const char *zPath; /* Name of the file */ unixShm *pShm; /* Shared memory segment information */ int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ + int nFetchOut; /* Number of outstanding xFetch refs */ + sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ + sqlite3_int64 mmapOrigsize; /* Actual size of mapping at pMapRegion */ + sqlite3_int64 mmapLimit; /* Configured FCNTL_MMAP_LIMIT value */ + void *pMapRegion; /* Memory mapped region */ #ifdef __QNXNTO__ int sectorSize; /* Device sector size */ int deviceCharacteristics; /* Precomputed device characteristics */ @@ -251,11 +256,6 @@ struct unixFile { unsigned char inNormalWrite; /* True if in a normal write operation */ #endif - sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ - sqlite3_int64 mmapOrigsize; /* Actual size of mapping at pMapRegion */ - sqlite3_int64 mmapLimit; /* Configured FCNTL_MMAP_SIZE value */ - void *pMapRegion; /* Memory mapped region */ - int nFetchOut; /* Number of outstanding xFetch refs */ #ifdef SQLITE_TEST /* In test mode, increase the size of this structure a bit so that @@ -3699,7 +3699,7 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } - case SQLITE_FCNTL_MMAP_SIZE: { + case SQLITE_FCNTL_MMAP_LIMIT: { pFile->mmapLimit = *(i64*)pArg; return SQLITE_OK; } @@ -4528,7 +4528,7 @@ static void unixUnmapfile(unixFile *pFd){ ** the mapping to create. Otherwise, if nByte is less than zero, then the ** requested size is the size of the file on disk. The actual size of the ** created mapping is either the requested size or the value configured -** using SQLITE_FCNTL_MMAP_SIZE, whichever is smaller. +** using SQLITE_FCNTL_MMAP_LIMIT, whichever is smaller. ** ** SQLITE_OK is returned if no error occurs (even if the mapping is not ** recreated as a result of outstanding references) or an SQLite error @@ -4969,6 +4969,7 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; + pNew->mmapLimit = SQLITE_DEFAULT_MMAP_LIMIT; if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; diff --git a/src/pager.c b/src/pager.c index b7534deb09..4cce844237 100644 --- a/src/pager.c +++ b/src/pager.c @@ -657,8 +657,8 @@ struct Pager { char dbFileVers[16]; /* Changes whenever database file changes */ u8 bUseFetch; /* True to use xFetch() */ - int nMapCfgLimit; /* Configured limit value */ int nMmapOut; /* Number of mmap pages currently outstanding */ + sqlite3_int64 mxMmap; /* Desired maximum mmap size */ PgHdr *pFree; /* List of free mmap page headers (pDirty) */ /* ** End of the routinely-changing class members @@ -3354,23 +3354,15 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ } /* -** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of nMapCfgLimit. +** Invoke SQLITE_FCNTL_MMAP_LIMIT based on the current value of mxMmap. */ static void pagerFixMaplimit(Pager *pPager){ sqlite3_file *fd = pPager->fd; if( isOpen(fd) ){ - pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->nMapCfgLimit!=0; + pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->mxMmap>0; if( pPager->bUseFetch ){ - void *p; - i64 nMapLimit; - if( pPager->nMapCfgLimit<0 ){ - nMapLimit = (i64)pPager->nMapCfgLimit * -1024; - }else{ - nMapLimit = (i64)pPager->nMapCfgLimit * pPager->pageSize; - } - - p = (void *)&nMapLimit; - sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_SIZE, p); + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT, + (void*)&pPager->mxMmap); } } } @@ -3378,8 +3370,8 @@ static void pagerFixMaplimit(Pager *pPager){ /* ** Change the maximum size of any memory mapping made of the database file. */ -void sqlite3PagerSetMmapsize(Pager *pPager, int nMap){ - pPager->nMapCfgLimit = nMap; +void sqlite3PagerSetMmapLimit(Pager *pPager, sqlite3_int64 mxMmap){ + pPager->mxMmap = mxMmap; pagerFixMaplimit(pPager); } diff --git a/src/pager.h b/src/pager.h index 81ab30c115..6f659136e2 100644 --- a/src/pager.h +++ b/src/pager.h @@ -108,7 +108,7 @@ void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); int sqlite3PagerSetPagesize(Pager*, u32*, int); int sqlite3PagerMaxPageCount(Pager*, int); void sqlite3PagerSetCachesize(Pager*, int); -void sqlite3PagerSetMmapsize(Pager *, int); +void sqlite3PagerSetMmapLimit(Pager *, sqlite3_int64); void sqlite3PagerShrink(Pager*); void sqlite3PagerSetSafetyLevel(Pager*,int,int,int); int sqlite3PagerLockingMode(Pager *, int); diff --git a/src/pragma.c b/src/pragma.c index 9b31797d3f..cd1cfbe213 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -745,26 +745,23 @@ void sqlite3Pragma( }else /* - ** PRAGMA [database.]mmap_size - ** PRAGMA [database.]mmap_size=N + ** PRAGMA [database.]mmap_limit(N) ** - ** Used to set or query the mapping size limit. The mapping size limit is + ** Used to set mapping size limit. The mapping size limit is ** used to limit the aggregate size of all memory mapped regions of the ** database file. If this parameter is set to zero, then memory mapping - ** is not used at all. If it is set to a positive value, then it is - ** interpreted as a maximum size in pages. If set to less than zero, then - ** the absolute value is interpreted as a size limit in KB. + ** is not used at all. The parameter N is measured in bytes. ** - ** The default value is zero (do not use memory mapped IO). + ** This value is advisory. The underlying VFS is free to memory map + ** as little or as much as it wants. Except, if N is set to 0 then the + ** upper layers will never invoke the xFetch interfaces to the VFS. */ - if( sqlite3StrICmp(zLeft,"mmap_size")==0 ){ + if( sqlite3StrICmp(zLeft,"mmap_limit")==0 ){ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - if( !zRight ){ - returnSingleInt(pParse, "mmap_size", pDb->pSchema->mmap_size); - }else{ - int size = sqlite3Atoi(zRight); - pDb->pSchema->mmap_size = size; - sqlite3BtreeSetMmapSize(pDb->pBt, pDb->pSchema->mmap_size); + if( zRight ){ + sqlite3_int64 size; + sqlite3Atoi64(zRight, &size, 1000, SQLITE_UTF8); + sqlite3BtreeSetMmapLimit(pDb->pBt, size); } }else diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 72f8d32939..acb49bf5f9 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -886,9 +886,9 @@ struct sqlite3_io_methods { ** written into memory obtained from [sqlite3_malloc()]. The caller should ** invoke [sqlite3_free()] on the result to avoid a memory leak. ** -**
  • [[SQLITE_FCNTL_MMAP_SIZE]] -** The argument is assumed to point to a value of type sqlite3_int64. An -** advisory maximum amount of this file to memory map in bytes. +**
  • [[SQLITE_FCNTL_MMAP_LIMIT]] +** The argument is assumed to pointer to a value of type sqlite3_int64 that +** is an advisory maximum number of bytes in the file to memory map. ** ** */ @@ -908,7 +908,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_PRAGMA 14 #define SQLITE_FCNTL_BUSYHANDLER 15 #define SQLITE_FCNTL_TEMPFILENAME 16 -#define SQLITE_FCNTL_MMAP_SIZE 18 +#define SQLITE_FCNTL_MMAP_LIMIT 18 /* ** CAPI3REF: Mutex Handle diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 59483bc87b..129c4c5ea5 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -746,7 +746,6 @@ struct Schema { u8 enc; /* Text encoding used by this database */ u16 flags; /* Flags associated with this schema */ int cache_size; /* Number of pages to use in the cache */ - int mmap_size; /* Number of pages to memory map */ }; /* diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index c7aee53ceb..2013139456 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -206,3 +206,10 @@ #ifndef SQLITE_MAX_TRIGGER_DEPTH # define SQLITE_MAX_TRIGGER_DEPTH 1000 #endif + +/* +** Default maximum size of memory used by xFetch in the VFS. +*/ +#ifndef SQLITE_DEFAULT_MMAP_LIMIT +# define SQLITE_DEFAULT_MMAP_LIMIT (256*1024*1024) +#endif diff --git a/test/dbstatus2.test b/test/dbstatus2.test index 2541a1a823..c2d9164fa2 100644 --- a/test/dbstatus2.test +++ b/test/dbstatus2.test @@ -40,7 +40,7 @@ proc db_write {db {reset 0}} { do_test 1.1 { db close sqlite3 db test.db - execsql { PRAGMA mmap_size = 0 } + execsql { PRAGMA mmap_limit = 0 } expr {[file size test.db] / 1024} } 6 diff --git a/test/incrblob.test b/test/incrblob.test index 4277e5c4c1..fa564de0cf 100644 --- a/test/incrblob.test +++ b/test/incrblob.test @@ -123,7 +123,7 @@ foreach AutoVacuumMode [list 0 1] { forcedelete test.db test.db-journal sqlite3 db test.db - execsql "PRAGMA mmap_size = 0" + execsql "PRAGMA mmap_limit = 0" execsql "PRAGMA auto_vacuum = $AutoVacuumMode" do_test incrblob-2.$AutoVacuumMode.1 { @@ -150,7 +150,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db - execsql "PRAGMA mmap_size = 0" + execsql "PRAGMA mmap_limit = 0" # Read the last 20 bytes of the blob via a blob handle. set ::blob [db incrblob blobs v 1] @@ -173,7 +173,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db - execsql "PRAGMA mmap_size = 0" + execsql "PRAGMA mmap_limit = 0" # Write the second-to-last 20 bytes of the blob via a blob handle. # @@ -203,7 +203,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db - execsql { PRAGMA mmap_size = 0 } + execsql { PRAGMA mmap_limit = 0 } execsql { SELECT i FROM blobs } } {45} diff --git a/test/mmap1.test b/test/mmap1.test index 41714e29d8..d97ada7e57 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -23,16 +23,16 @@ proc nRead {db} { return $stats(read) } -foreach {t mmap_size nRead c2init} { - 1.1 { PRAGMA mmap_size = -65536 } 4 {} - 1.2 { PRAGMA mmap_size = -50 } 156 {} - 1.3 { PRAGMA mmap_size = 0 } 344 {} - 1.4 { PRAGMA mmap_size = -65536 } 4 {PRAGMA mmap_size = -65536} - 1.5 { PRAGMA mmap_size = -50 } 156 {PRAGMA mmap_size = -65536} - 1.6 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = -65536} +foreach {t mmap_limit nRead c2init} { + 1.1 { PRAGMA mmap_limit = 70000000 } 4 {} + 1.2 { PRAGMA mmap_limit = 51200 } 156 {} + 1.3 { PRAGMA mmap_limit = 0 } 344 {} + 1.4 { PRAGMA mmap_limit = 70000000 } 4 {PRAGMA mmap_limit = 70000000 } + 1.5 { PRAGMA mmap_limit = 51200 } 156 {PRAGMA mmap_limit = 70000000 } + 1.6 { PRAGMA mmap_limit = 0 } 344 {PRAGMA mmap_limit = 70000000 } } { do_multiclient_test tn { - sql1 $mmap_size + sql1 $mmap_limit sql2 $c2init code2 { @@ -78,7 +78,7 @@ foreach {t mmap_size nRead c2init} { } {64 ok 149} # Check that the number of pages read by connection 1 indicates that the - # "PRAGMA mmap_size" command worked. + # "PRAGMA mmap_limit" command worked. do_test $t.$tn.5 { nRead db } $nRead } } @@ -95,7 +95,7 @@ db func rblob rblob do_execsql_test 2.1 { PRAGMA auto_vacuum = 1; - PRAGMA mmap_size = -65536; + PRAGMA mmap_limit = 70000000; PRAGMA journal_mode = wal; CREATE TABLE t1(a, b, UNIQUE(a, b)); INSERT INTO t1 VALUES(rblob(500), rblob(500)); @@ -130,4 +130,3 @@ do_execsql_test 2.4 { } {0 24 24} finish_test - diff --git a/test/permutations.test b/test/permutations.test index 9ff3a801c2..2e529fa880 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -138,10 +138,10 @@ test_suite "veryquick" -prefix "" -description { test_set $allquicktests -exclude *malloc* *ioerr* *fault* ] -test_suite "mmap" -prefix "mm-" -description { - Similar to veryquick. Except with memory mapping enabled. +test_suite "no-mmap" -prefix "nomm-" -description { + Similar to veryquick. Except with memory mapping disabled. } -presql { - pragma mmap_size = -65536; + pragma mmap_size = 0; } -files [ test_set $allquicktests -exclude *malloc* *ioerr* *fault* -include malloc.test ] diff --git a/test/speed1p.test b/test/speed1p.test index df5abedde3..915f165354 100644 --- a/test/speed1p.test +++ b/test/speed1p.test @@ -65,7 +65,6 @@ proc number_name {n} { # do_test speed1p-1.0 { execsql { - PRAGMA mmap_size=1000000; PRAGMA page_size=1024; PRAGMA cache_size=500; PRAGMA locking_mode=EXCLUSIVE; diff --git a/test/sysfault.test b/test/sysfault.test index bf7df5df5f..4ecaa1fabc 100644 --- a/test/sysfault.test +++ b/test/sysfault.test @@ -262,7 +262,7 @@ do_faultsim_test 4 -faults vfsfault-* -prep { faultsim_restore_and_reopen file_control_chunksize_test db main 8192 execsql { - PRAGMA mmap_size = 1000000; + PRAGMA mmap_limit = 1000000; } } -body { test_syscall errno mmap EACCES @@ -275,4 +275,3 @@ do_faultsim_test 4 -faults vfsfault-* -prep { } finish_test - diff --git a/test/wal5.test b/test/wal5.test index d8bf541a4b..ebd1dbfc31 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -241,8 +241,8 @@ foreach {testprefix do_wal_checkpoint} { # [sql2] commmand above. So normally, the db is 1 page in size here. # However, in mmap() mode, the db is pre-allocated to 2 pages at the # start of the checkpoint, even though page 2 cannot be written. - set nDb 1 - if {[permutation]=="mmap"} {set nDb 2} + set nDb 2 + if {[permutation]=="no-mmap"} {set nDb 1} do_test 2.3.$tn.8 { file_page_counts } [list $nDb 4 2 4] } @@ -351,4 +351,3 @@ foreach {testprefix do_wal_checkpoint} { finish_test - From 41f89cc6763b71bcad37964745c1ffdf2a391452 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 26 Mar 2013 01:07:50 +0000 Subject: [PATCH 034/140] Previous check-in accidently left mmap turned off by default. This checkin fixes that. Unfortunately, shared.test is now segfaulting. All other veryquick tests appear to work, however. FossilOrigin-Name: a850c7319c20b5757983443df05cf2aa4250053b --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/pager.c | 1 + test/exclusive2.test | 2 +- test/func.test | 2 +- test/pageropt.test | 2 +- test/permutations.test | 4 ++-- test/wal.test | 4 ++-- 8 files changed, 20 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 65993928dd..f9fdab7968 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Memory-mapped\sI/O\sis\snow\son\sby\sdefault.\s\sThe\s"PRAGMA\smmap_limit(N)"\scan\sbe\nused\sto\sissue\sa\shint\sto\sthe\sVFS\sto\slimit\smmap\sspace\sto\sN\sbytes.\s\sThe\sVFS\nis\sfree\sto\signore\sthat\shint\sif\sdesired.\s\sHowever,\sif\s"PRAGMA\smmap_limit(0)"\nis\sused,\sxFetch\sis\snever\scalled. -D 2013-03-25T23:09:28.646 +C Previous\scheck-in\saccidently\sleft\smmap\sturned\soff\sby\sdefault.\sThis\scheckin\nfixes\sthat.\s\sUnfortunately,\sshared.test\sis\snow\ssegfaulting.\s\sAll\sother\nveryquick\stests\sappear\sto\swork,\showever. +D 2013-03-26T01:07:50.891 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 57306f1d2a2d783dae4365446b9f9a5da8958559 F src/os_win.c e4f17ddf79f2a9373e33ed70565e765d65324589 -F src/pager.c c532d51ccc74de8551a28eb932f3b1a1034017d3 +F src/pager.c 8ac98fd95106e759870e92ff316b188e78f469cf F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -403,7 +403,7 @@ F test/eqp.test 46aa946dd55c90635327898275d3e533d23a9845 F test/errmsg.test 050717f1c6a5685de9c79f5f9f6b83d7c592f73a F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test a1b324cb21834a490cd052d409d34789cfef57cb -F test/exclusive2.test 881193eccec225cfed9d7f744b65e57f26adee25 +F test/exclusive2.test 354bdabe299a2546c898dff42f79079ff1590d88 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30 F test/expr.test 67c9fd6f8f829e239dc8b0f4a08a73c08b09196d @@ -508,7 +508,7 @@ F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7 F test/fts4unicode.test 25ccad45896f8e50f6a694cff738a35f798cdb40 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d -F test/func.test b0fc34fdc36897769651975a2b0a606312753643 +F test/func.test a4f24707e6af0bcd4cb5b856af4462db40b047b1 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a F test/fuzz-oss1.test 4912e528ec9cf2f42134456933659d371c9e0d74 @@ -654,11 +654,11 @@ F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f F test/pagerfault.test 452f2cc23e3bfcfa935f4442aec1da4fe1dc0442 F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 -F test/pageropt.test 290cd59782b1890f02bb33795571facfc5ccac43 +F test/pageropt.test 32cb5a6ed7ccddf8e8c842cb44240bd9340223ce F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test d298a74abae42b72b3640dc5505d19d8a3eddc9e +F test/permutations.test b6cb45ce4d3193b831231025c9d30c77317dd240 F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -951,7 +951,7 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 -F test/wal.test 5baa9a0e682c0394012aa9a06a3d06cf92655fbf +F test/wal.test bbd98ac09d8fbcaa80ec233757056eb1bbb7cd95 F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 8776047bd776bbf266eb9c3b56683badb84ae73e -R 1fba4ef23570acddc66df7d627b1cb5f +P 1b37c4effdd03aa2ea938a71b4f22ed27391689b +R 66cf00b9dd2b30a0d14b8b5c617a6bbb U drh -Z c62aae7ee656f659e8aebd3681920a9a +Z c802a777b217fa0e42ce25209bcf7fef diff --git a/manifest.uuid b/manifest.uuid index 8e0647f651..dc7e369204 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1b37c4effdd03aa2ea938a71b4f22ed27391689b \ No newline at end of file +a850c7319c20b5757983443df05cf2aa4250053b \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 4cce844237..65969bcc3e 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4749,6 +4749,7 @@ int sqlite3PagerOpen( /* pPager->pBusyHandlerArg = 0; */ pPager->xReiniter = xReinit; /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ + pPager->mxMmap = SQLITE_DEFAULT_MMAP_LIMIT; *ppPager = pPager; return SQLITE_OK; diff --git a/test/exclusive2.test b/test/exclusive2.test index 54203e3d8f..d09097671d 100644 --- a/test/exclusive2.test +++ b/test/exclusive2.test @@ -28,7 +28,7 @@ ifcapable {!pager_pragmas} { # Tests in this file verify that locking_mode=exclusive causes SQLite to # use cached pages even if the database is changed on disk. This doesn't # work with mmap. -if {[permutation]=="mmap"} { +if {[permutation]!="nommap"} { finish_test return } diff --git a/test/func.test b/test/func.test index 4ab7688461..032becc94f 100644 --- a/test/func.test +++ b/test/func.test @@ -1273,7 +1273,7 @@ do_test func-29.3 { sqlite3_db_status db CACHE_MISS 1 db eval {SELECT typeof(+x) FROM t29 ORDER BY id} } {integer null real blob text} -if {[permutation] != "mmap"} { +if {[permutation] == "nommap"} { do_test func-29.4 { set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] if {$x>100} {set x many} diff --git a/test/pageropt.test b/test/pageropt.test index de39bbaf05..10d3d4917e 100644 --- a/test/pageropt.test +++ b/test/pageropt.test @@ -92,7 +92,7 @@ do_test pageropt-1.5 { pagercount_sql { SELECT hex(x) FROM t1 } -} [list [expr {[permutation]=="mmap" ? 1 : 6}] 0 0 $blobcontent] +} [list [expr {[permutation]=="nommap" ? 6 : 1}] 0 0 $blobcontent] do_test pageropt-1.6 { pagercount_sql { SELECT hex(x) FROM t1 diff --git a/test/permutations.test b/test/permutations.test index 2e529fa880..3aba9d9b38 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -138,10 +138,10 @@ test_suite "veryquick" -prefix "" -description { test_set $allquicktests -exclude *malloc* *ioerr* *fault* ] -test_suite "no-mmap" -prefix "nomm-" -description { +test_suite "nommap" -prefix "nomm-" -description { Similar to veryquick. Except with memory mapping disabled. } -presql { - pragma mmap_size = 0; + pragma mmap_limit = 0; } -files [ test_set $allquicktests -exclude *malloc* *ioerr* *fault* -include malloc.test ] diff --git a/test/wal.test b/test/wal.test index ca7edef952..0539192d25 100644 --- a/test/wal.test +++ b/test/wal.test @@ -727,8 +727,8 @@ do_test wal-11.9 { list [expr [file size test.db]/1024] [log_deleted test.db-wal] } {37 1} sqlite3_wal db test.db -set nWal 37 -if {[permutation]=="mmap"} {set nWal 39} +set nWal 39 +if {[permutation]=="nommap"} {set nWal 37} do_test wal-11.10 { execsql { PRAGMA cache_size = 10; From 0aed84d19f8e1fcf90340e17cff6b774b0379255 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 26 Mar 2013 14:16:20 +0000 Subject: [PATCH 035/140] In btree.c, save the positions of any open cursors before moving any pages around to auto-vacuum the database on commit. FossilOrigin-Name: 30c0a69363931a72d1c34e5be71646932398d172 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 6 ++++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index f9fdab7968..7b9df5476e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Previous\scheck-in\saccidently\sleft\smmap\sturned\soff\sby\sdefault.\sThis\scheckin\nfixes\sthat.\s\sUnfortunately,\sshared.test\sis\snow\ssegfaulting.\s\sAll\sother\nveryquick\stests\sappear\sto\swork,\showever. -D 2013-03-26T01:07:50.891 +C In\sbtree.c,\ssave\sthe\spositions\sof\sany\sopen\scursors\sbefore\smoving\sany\spages\saround\sto\sauto-vacuum\sthe\sdatabase\son\scommit. +D 2013-03-26T14:16:20.450 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 771f0391556317de19e02c9533b256e8299dd59b +F src/btree.c 19b587b32569d4b92651b5eaa9de2e59c6f8f704 F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 1b37c4effdd03aa2ea938a71b4f22ed27391689b -R 66cf00b9dd2b30a0d14b8b5c617a6bbb -U drh -Z c802a777b217fa0e42ce25209bcf7fef +P a850c7319c20b5757983443df05cf2aa4250053b +R 211ffabed726a7468ad9f8d3eeabe8a8 +U dan +Z 3ec7da7f1db0278991c8541b22037fbf diff --git a/manifest.uuid b/manifest.uuid index dc7e369204..12eaba3359 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a850c7319c20b5757983443df05cf2aa4250053b \ No newline at end of file +30c0a69363931a72d1c34e5be71646932398d172 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 04cc20d5c5..7d2f2d3840 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3185,7 +3185,9 @@ static int autoVacuumCommit(BtShared *pBt){ nFree = get4byte(&pBt->pPage1->aData[36]); nFin = finalDbSize(pBt, nOrig, nFree); if( nFin>nOrig ) return SQLITE_CORRUPT_BKPT; - + if( nFinnFin && rc==SQLITE_OK; iFree--){ rc = incrVacuumStep(pBt, nFin, iFree, 1); } @@ -3202,7 +3204,7 @@ static int autoVacuumCommit(BtShared *pBt){ } } - assert( nRef==sqlite3PagerRefcount(pPager) ); + assert( nRef>=sqlite3PagerRefcount(pPager) ); return rc; } From c86e51358b3e6cd21d2b324116933b9fb7943956 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 26 Mar 2013 14:36:11 +0000 Subject: [PATCH 036/140] Change the name of the Pager.pFree field to Pager.pMmapFreelist. FossilOrigin-Name: 611bd824c24a60d298f28705de323fa2e813a308 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 16 ++++++++-------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 7b9df5476e..a2e312e3a6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sbtree.c,\ssave\sthe\spositions\sof\sany\sopen\scursors\sbefore\smoving\sany\spages\saround\sto\sauto-vacuum\sthe\sdatabase\son\scommit. -D 2013-03-26T14:16:20.450 +C Change\sthe\sname\sof\sthe\sPager.pFree\sfield\sto\sPager.pMmapFreelist. +D 2013-03-26T14:36:11.815 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 57306f1d2a2d783dae4365446b9f9a5da8958559 F src/os_win.c e4f17ddf79f2a9373e33ed70565e765d65324589 -F src/pager.c 8ac98fd95106e759870e92ff316b188e78f469cf +F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a850c7319c20b5757983443df05cf2aa4250053b -R 211ffabed726a7468ad9f8d3eeabe8a8 -U dan -Z 3ec7da7f1db0278991c8541b22037fbf +P 30c0a69363931a72d1c34e5be71646932398d172 +R 11ebc3388f5e3939639a27582f5015a5 +U drh +Z 2c5cf3d5be983a0c21f765acf3ca05b3 diff --git a/manifest.uuid b/manifest.uuid index 12eaba3359..cd71ef1d3b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -30c0a69363931a72d1c34e5be71646932398d172 \ No newline at end of file +611bd824c24a60d298f28705de323fa2e813a308 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 65969bcc3e..550c6c7564 100644 --- a/src/pager.c +++ b/src/pager.c @@ -659,7 +659,7 @@ struct Pager { u8 bUseFetch; /* True to use xFetch() */ int nMmapOut; /* Number of mmap pages currently outstanding */ sqlite3_int64 mxMmap; /* Desired maximum mmap size */ - PgHdr *pFree; /* List of free mmap page headers (pDirty) */ + PgHdr *pMmapFreelist; /* List of free mmap page headers (pDirty) */ /* ** End of the routinely-changing class members ***************************************************************************/ @@ -3854,9 +3854,9 @@ static int pagerAcquireMapPage( ){ PgHdr *p; /* Memory mapped page to return */ - if( pPager->pFree ){ - *ppPage = p = pPager->pFree; - pPager->pFree = p->pDirty; + if( pPager->pMmapFreelist ){ + *ppPage = p = pPager->pMmapFreelist; + pPager->pMmapFreelist = p->pDirty; p->pDirty = 0; memset(p->pExtra, 0, pPager->nExtra); }else{ @@ -3891,20 +3891,20 @@ static int pagerAcquireMapPage( static void pagerReleaseMapPage(PgHdr *pPg){ Pager *pPager = pPg->pPager; pPager->nMmapOut--; - pPg->pDirty = pPager->pFree; - pPager->pFree = pPg; + pPg->pDirty = pPager->pMmapFreelist; + pPager->pMmapFreelist = pPg; assert( pPager->fd->pMethods->iVersion>=3 ); sqlite3OsUnfetch(pPager->fd, (i64)(pPg->pgno-1)*pPager->pageSize, pPg->pData); } /* -** Free all PgHdr objects stored in the Pager.pFree list. +** Free all PgHdr objects stored in the Pager.pMmapFreelist list. */ static void pagerFreeMapHdrs(Pager *pPager){ PgHdr *p; PgHdr *pNext; - for(p=pPager->pFree; p; p=pNext){ + for(p=pPager->pMmapFreelist; p; p=pNext){ pNext = p->pDirty; sqlite3_free(p); } From 6c5696381e5720eaa9bd6c780bd75f5669f04c96 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 26 Mar 2013 18:48:11 +0000 Subject: [PATCH 037/140] Fix a comment in os_unix.c. No code changes. FossilOrigin-Name: 72813b8ec924b91583c679668f7c4561dff82a02 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index a2e312e3a6..715e199440 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sname\sof\sthe\sPager.pFree\sfield\sto\sPager.pMmapFreelist. -D 2013-03-26T14:36:11.815 +C Fix\sa\scomment\sin\sos_unix.c.\s\sNo\scode\schanges. +D 2013-03-26T18:48:11.374 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 57306f1d2a2d783dae4365446b9f9a5da8958559 +F src/os_unix.c d6981218583748080374ed98a03f6a87e2bdc9e0 F src/os_win.c e4f17ddf79f2a9373e33ed70565e765d65324589 F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 30c0a69363931a72d1c34e5be71646932398d172 -R 11ebc3388f5e3939639a27582f5015a5 +P 611bd824c24a60d298f28705de323fa2e813a308 +R 8f591838a5bc479ac7493be19f46439f U drh -Z 2c5cf3d5be983a0c21f765acf3ca05b3 +Z be06b34348d93c4a77e7034db0a8802f diff --git a/manifest.uuid b/manifest.uuid index cd71ef1d3b..855425c986 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -611bd824c24a60d298f28705de323fa2e813a308 \ No newline at end of file +72813b8ec924b91583c679668f7c4561dff82a02 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 8a8516bcda..f91f4c4e7a 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3107,8 +3107,8 @@ static int unixRead( ); #endif - /* Deal with as much of this write request as possible by transfering - ** data to the memory mapping using memcpy(). */ + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); From fde3b1c71596165d04dc3e4a46ae70f173a1adb3 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 29 Mar 2013 19:52:04 +0000 Subject: [PATCH 038/140] For the clean target, tidy up a couple more generated (or copied) source files. FossilOrigin-Name: 5687e5ee7bafa00d2b353c3eda1e5dfb219cb185 --- Makefile.in | 2 ++ Makefile.msc | 1 + Makefile.vxworks | 2 ++ main.mk | 2 ++ manifest | 23 ++++++++++------------- manifest.uuid | 2 +- 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/Makefile.in b/Makefile.in index 1ecaf2bbd5..edd29ae011 100644 --- a/Makefile.in +++ b/Makefile.in @@ -946,6 +946,8 @@ clean: rm -f testfixture$(TEXE) test.db rm -f sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def rm -f sqlite3.c + rm -f sqlite3rc.h + rm -f shell.c sqlite3ext.h rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c rm -f sqlite-*-output.vsix diff --git a/Makefile.msc b/Makefile.msc index df6ed4d46d..70be76afa8 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1250,6 +1250,7 @@ clean: del /Q sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def del /Q sqlite3.c del /Q sqlite3rc.h + del /Q shell.c sqlite3ext.h del /Q sqlite3_analyzer.exe sqlite3_analyzer.exp sqlite3_analyzer.c del /Q sqlite-*-output.vsix diff --git a/Makefile.vxworks b/Makefile.vxworks index d04028833c..c9058da3e5 100644 --- a/Makefile.vxworks +++ b/Makefile.vxworks @@ -662,4 +662,6 @@ clean: rm -rf tsrc target_source rm -f testloadext.dll libtestloadext.so rm -f sqlite3.c fts?amal.c tclsqlite3.c + rm -f sqlite3rc.h + rm -f shell.c sqlite3ext.h rm -f $(SHPREFIX)sqlite3.$(SO) diff --git a/main.mk b/main.mk index d644079f59..1294121c0a 100644 --- a/main.mk +++ b/main.mk @@ -622,5 +622,7 @@ clean: rm -f testfixture testfixture.exe rm -f threadtest3 threadtest3.exe rm -f sqlite3.c fts?amal.c tclsqlite3.c + rm -f sqlite3rc.h + rm -f shell.c sqlite3ext.h rm -f sqlite3_analyzer sqlite3_analyzer.exe sqlite3_analyzer.c rm -f sqlite-*-output.vsix diff --git a/manifest b/manifest index 5cf11cc2fb..453b33c22e 100644 --- a/manifest +++ b/manifest @@ -1,10 +1,10 @@ -C Version\s3.7.16.1 -D 2013-03-29T13:44:34.265 +C For\sthe\sclean\starget,\stidy\sup\sa\scouple\smore\sgenerated\s(or\scopied)\ssource\sfiles. +D 2013-03-29T19:52:04.965 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 +F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 0c1abc21c8deefc88e8a32ad6a07e5f96e158761 -F Makefile.vxworks b18ad88e9a8c6a001f5cf4a389116a4f1a7ab45f +F Makefile.msc 9ab787d472c9cb8bad29c2c4aeba15b06fba7bc4 +F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 05604ccde96fe1b37f922eddbdcb5654308261db F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -103,7 +103,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk c5437f64aad34844d18efe7b2d2adaa36efca49c +F main.mk b25a7a0b657409b2dcf9b09d39562703d5df702d F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -1041,10 +1041,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 39534b4baa53b9360696a9d5ee8cc2108af34592 -R e8e4e98970281a95c0de62c6e5f4d562 -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.7.16.1 * -U drh -Z 17d1076ce5eb9d83e75734278cd22a73 +P 527231bc67285f01fb18d4451b28f61da3c4e39d +R c463228a20804b5f57c5669f0d0e45b8 +U mistachkin +Z bfb2a9ca80a29b00e5f7b4db495d38d4 diff --git a/manifest.uuid b/manifest.uuid index 68abab9466..6960809f06 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -527231bc67285f01fb18d4451b28f61da3c4e39d \ No newline at end of file +5687e5ee7bafa00d2b353c3eda1e5dfb219cb185 \ No newline at end of file From 71be5c5c876747be8a8352170519271a20b7ca9f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 1 Apr 2013 14:29:33 +0000 Subject: [PATCH 039/140] Fix a problem in btree.c that could cause a crash following an OOM. FossilOrigin-Name: b724aa5889f2e09cf0cbef133a672e6486755104 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 11 ++++++----- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 715e199440..8a6c9f5c8d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scomment\sin\sos_unix.c.\s\sNo\scode\schanges. -D 2013-03-26T18:48:11.374 +C Fix\sa\sproblem\sin\sbtree.c\sthat\scould\scause\sa\scrash\sfollowing\san\sOOM. +D 2013-04-01T14:29:33.960 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 19b587b32569d4b92651b5eaa9de2e59c6f8f704 +F src/btree.c 84c9a4a20b827a6f7dbcbcda59607dbee57af446 F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 611bd824c24a60d298f28705de323fa2e813a308 -R 8f591838a5bc479ac7493be19f46439f -U drh -Z be06b34348d93c4a77e7034db0a8802f +P 72813b8ec924b91583c679668f7c4561dff82a02 +R ff1f51b04ee42accb0ddfde388849db6 +U dan +Z fbb073bc333f3d3b9c496b3226200b28 diff --git a/manifest.uuid b/manifest.uuid index 855425c986..069c9bb2ee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -72813b8ec924b91583c679668f7c4561dff82a02 \ No newline at end of file +b724aa5889f2e09cf0cbef133a672e6486755104 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 7d2f2d3840..03764a9267 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2596,12 +2596,13 @@ static int btreeSwapOutMmap(BtShared *pBt){ if( pPg && pPg->pDbPage->flags & PGHDR_MMAP ){ MemPage *pNew = 0; rc = getAndInitPage(pBt, pPg->pgno, &pNew, 0); - if( rc==SQLITE_OK && pCsr->iPage==0 ){ - pCsr->info.pCell = pNew->aData + (pCsr->info.pCell - pPg->aData); + if( rc==SQLITE_OK ){ + if( pCsr->iPage==0 ){ + pCsr->info.pCell = pNew->aData + (pCsr->info.pCell - pPg->aData); + } + pCsr->apPage[0] = pNew; + releasePage(pPg); } - pCsr->apPage[0] = pNew; - releasePage(pPg); - if( rc!=SQLITE_OK ) return rc; } } } From 0943f0bdc870249bf1d2e8692de9fd2f296190a3 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 1 Apr 2013 14:35:01 +0000 Subject: [PATCH 040/140] Fix a typo in a log message in wal.c. FossilOrigin-Name: 83fc48d16572443373e5de62adbd4cd53c2f7786 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 8a6c9f5c8d..2da91083ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sbtree.c\sthat\scould\scause\sa\scrash\sfollowing\san\sOOM. -D 2013-04-01T14:29:33.960 +C Fix\sa\stypo\sin\sa\slog\smessage\sin\swal.c. +D 2013-04-01T14:35:01.078 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c 4fad64071ae120c25f39dcac572d716b9cadeb7f F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 -F src/wal.c e84eff498c57ec2d79ca4496a3f4a638af378fb3 +F src/wal.c 04029f2691e937d7f5dd638580679a560fec93bf F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b F src/where.c 9a16c0b84bbeb054d11fda96e9e037ae310bd54e @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 72813b8ec924b91583c679668f7c4561dff82a02 -R ff1f51b04ee42accb0ddfde388849db6 +P b724aa5889f2e09cf0cbef133a672e6486755104 +R 96d4d8288650bd963603cf34d02b8e9c U dan -Z fbb073bc333f3d3b9c496b3226200b28 +Z 48fdc69d5f63c210da250932a430f955 diff --git a/manifest.uuid b/manifest.uuid index 069c9bb2ee..9b3ffd9c61 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b724aa5889f2e09cf0cbef133a672e6486755104 \ No newline at end of file +83fc48d16572443373e5de62adbd4cd53c2f7786 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 4602eb2c34..abbd7032f5 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1208,7 +1208,7 @@ finished: */ if( pWal->hdr.nPage ){ sqlite3_log(SQLITE_OK, "Recovered %d frames from WAL file %s", - pWal->hdr.nPage, pWal->zWalName + pWal->hdr.mxFrame, pWal->zWalName ); } } From f55a4cf80c73937610e8b39d553f2c8aea02af8f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 1 Apr 2013 16:56:41 +0000 Subject: [PATCH 041/140] Ensure that a checkpoint does not use an out-of-date mapping. FossilOrigin-Name: a1040f0397d57855500926494c978623286ddc77 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 2da91083ab..334e902f99 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sa\slog\smessage\sin\swal.c. -D 2013-04-01T14:35:01.078 +C Ensure\sthat\sa\scheckpoint\sdoes\snot\suse\san\sout-of-date\smapping. +D 2013-04-01T16:56:41.264 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -249,7 +249,7 @@ F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c 4fad64071ae120c25f39dcac572d716b9cadeb7f F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 -F src/wal.c 04029f2691e937d7f5dd638580679a560fec93bf +F src/wal.c 94b5fed2df988fb12f5bf17256e2840e56957a85 F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b F src/where.c 9a16c0b84bbeb054d11fda96e9e037ae310bd54e @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b724aa5889f2e09cf0cbef133a672e6486755104 -R 96d4d8288650bd963603cf34d02b8e9c +P 83fc48d16572443373e5de62adbd4cd53c2f7786 +R 8c156c43b98e65e90e42898654b4787f U dan -Z 48fdc69d5f63c210da250932a430f955 +Z 8dac7ace59503631e98c649abc9ec1b4 diff --git a/manifest.uuid b/manifest.uuid index 9b3ffd9c61..b6b3710dd3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83fc48d16572443373e5de62adbd4cd53c2f7786 \ No newline at end of file +a1040f0397d57855500926494c978623286ddc77 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index abbd7032f5..881f0a4c78 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2956,6 +2956,9 @@ int sqlite3WalCheckpoint( /* Read the wal-index header. */ if( rc==SQLITE_OK ){ rc = walIndexReadHdr(pWal, &isChanged); + if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ + sqlite3OsUnfetch(pWal->pDbFd, 0, 0); + } } /* Copy data from the log to the database file. */ From 5175b324b84ecab7132382bbb8de698988fcd4b9 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Apr 2013 17:22:51 +0000 Subject: [PATCH 042/140] Add xFetch and xUnfetch methods to the os_win.c VFS. FossilOrigin-Name: a1653a257d6af6e8b10c819e68b12f6c2f485811 --- manifest | 14 +-- manifest.uuid | 2 +- src/os_win.c | 288 ++++++++++++++++++++++++++++++-------------------- 3 files changed, 182 insertions(+), 122 deletions(-) diff --git a/manifest b/manifest index 334e902f99..e6eca96ae6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sa\scheckpoint\sdoes\snot\suse\san\sout-of-date\smapping. -D 2013-04-01T16:56:41.264 +C Add\sxFetch\sand\sxUnfetch\smethods\sto\sthe\sos_win.c\sVFS. +D 2013-04-01T17:22:51.830 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d6981218583748080374ed98a03f6a87e2bdc9e0 -F src/os_win.c e4f17ddf79f2a9373e33ed70565e765d65324589 +F src/os_win.c f705e7ce230f86104dedcc2987a21d564b236659 F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 83fc48d16572443373e5de62adbd4cd53c2f7786 -R 8c156c43b98e65e90e42898654b4787f -U dan -Z 8dac7ace59503631e98c649abc9ec1b4 +P a1040f0397d57855500926494c978623286ddc77 +R 52b13f77704d4760e77e0cfd9ab57bff +U drh +Z 878a27609130450cef13cc90a8feab48 diff --git a/manifest.uuid b/manifest.uuid index b6b3710dd3..c685b6feac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a1040f0397d57855500926494c978623286ddc77 \ No newline at end of file +a1653a257d6af6e8b10c819e68b12f6c2f485811 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index a836664565..b1f5748638 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -150,9 +150,12 @@ struct winFile { winceLock local; /* Locks obtained by this instance of winFile */ winceLock *shared; /* Global shared lock memory for the file */ #endif - HANDLE hMap; /* Handle for accessing memory mapping */ - void *pMapRegion; /* Area memory mapped */ - sqlite3_int64 mmapSize; /* Size of xMremap() */ + int nFetchOut; /* Number of outstanding xFetch references */ + HANDLE hMap; /* Handle for accessing memory mapping */ + void *pMapRegion; /* Area memory mapped */ + sqlite3_int64 mmapSize; /* Usable size of mapped region */ + sqlite3_int64 mmapOrigsize; /* Actual size of mapped region */ + sqlite3_int64 mmapLimit; /* Configured FCNTL_MMAP_LIMIT value */ }; /* @@ -2066,7 +2069,7 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ } /* Forward references to VFS methods */ -static int winUnmap(sqlite3_file *); +static int winUnmapfile(winFile*); /* ** Close a file. @@ -2090,7 +2093,7 @@ static int winClose(sqlite3_file *id){ OSTRACE(("CLOSE %d\n", pFile->h)); assert( pFile->h!=NULL && pFile->h!=INVALID_HANDLE_VALUE ); - rc = winUnmap(id); + rc = winUnmapfile(pFile); if( rc!=SQLITE_OK ) return rc; do{ @@ -2142,10 +2145,24 @@ static int winRead( assert( id!=0 ); assert( amt>0 ); - assert( offset>=pFile->mmapSize ); /* Never read from the mmapped region */ SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); + /* Deal with as much of this read request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], amt); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(pBuf, &((u8 *)(pFile->pMapRegion))[offset], nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } + #if SQLITE_OS_WINCE if( seekWinFile(pFile, offset) ){ return SQLITE_FULL; @@ -2189,13 +2206,27 @@ static int winWrite( int nRetry = 0; /* Number of retries */ assert( amt>0 ); - assert( offset>=pFile->mmapSize ); /* Never write into the mmapped region */ assert( pFile ); SimulateIOError(return SQLITE_IOERR_WRITE); SimulateDiskfullError(return SQLITE_FULL); OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); + /* Deal with as much of this write request as possible by transfering + ** data from the memory mapping using memcpy(). */ + if( offsetmmapSize ){ + if( offset+amt <= pFile->mmapSize ){ + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, amt); + return SQLITE_OK; + }else{ + int nCopy = (int)(pFile->mmapSize - offset); + memcpy(&((u8 *)(pFile->pMapRegion))[offset], pBuf, nCopy); + pBuf = &((u8 *)pBuf)[nCopy]; + amt -= nCopy; + offset += nCopy; + } + } + #if SQLITE_OS_WINCE rc = seekWinFile(pFile, offset); if( rc==0 ){ @@ -2288,14 +2319,14 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ pFile->lastErrno = lastErrno; rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, "winTruncate2", pFile->zPath); - }else{ - /* If the file was just truncated to a size smaller than the currently - ** mapped region, reduce the effective mapping size as well. SQLite will - ** use read() and write() to access data beyond this point from now on. - */ - if( pFile->pMapRegion && nBytemmapSize ){ - pFile->mmapSize = nByte; - } + } + + /* If the file was truncated to a size smaller than the currently + ** mapped region, reduce the effective mapping size as well. SQLite will + ** use read() and write() to access data beyond this point from now on. + */ + if( pFile->pMapRegion && nBytemmapSize ){ + pFile->mmapSize = nByte; } OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); @@ -2805,6 +2836,10 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } + case SQLITE_FCNTL_MMAP_LIMIT: { + pFile->mmapLimit = *(i64*)pArg; + return SQLITE_OK; + } } return SQLITE_NOTFOUND; } @@ -3475,37 +3510,25 @@ shmpage_out: # define winShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ -/* -** Arguments x and y are both integers. Argument y must be a power of 2. -** Round x up to the nearest integer multiple of y. For example: -** -** ROUNDUP(0, 8) -> 0 -** ROUNDUP(13, 8) -> 16 -** ROUNDUP(32, 8) -> 32 -*/ -#define ROUNDUP(x,y) (((x)+y-1)&~(y-1)) - /* ** Cleans up the mapped region of the specified file, if any. */ -static int winUnmap( - sqlite3_file *id -){ - winFile *pFile = (winFile*)id; +static int winUnmapfile(winFile *pFile){ assert( pFile!=0 ); if( pFile->pMapRegion ){ if( !osUnmapViewOfFile(pFile->pMapRegion) ){ pFile->lastErrno = osGetLastError(); - return winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, "winUnmap1", pFile->zPath); } pFile->pMapRegion = 0; pFile->mmapSize = 0; + pFile->mmapOrigsize = 0; } if( pFile->hMap!=NULL ){ if( !osCloseHandle(pFile->hMap) ){ pFile->lastErrno = osGetLastError(); - return winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, + return winLogError(SQLITE_IOERR_MMAP, pFile->lastErrno, "winUnmap2", pFile->zPath); } pFile->hMap = NULL; @@ -3514,116 +3537,150 @@ static int winUnmap( } /* -** Map, remap or unmap part of the database file. +** Memory map or remap the file opened by file-descriptor pFd (if the file +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still +** outstanding xFetch() references to it, this function is a no-op. +** +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the +** requested size is the size of the file on disk. The actual size of the +** created mapping is either the requested size or the value configured +** using SQLITE_FCNTL_MMAP_LIMIT, whichever is smaller. +** +** SQLITE_OK is returned if no error occurs (even if the mapping is not +** recreated as a result of outstanding references) or an SQLite error +** code otherwise. */ -static int winMremap( - sqlite3_file *id, /* Main database file */ - int flags, /* Mask of SQLITE_MREMAP_XXX flags */ - sqlite3_int64 iOff, /* Offset to start mapping at */ - sqlite3_int64 nOld, /* Size of old mapping, or zero */ - sqlite3_int64 nNew, /* Size of new mapping, or zero */ - void **ppMap /* IN/OUT: Old/new mappings */ -){ - winFile *pFile = (winFile*)id; /* The underlying database file */ - int rc = SQLITE_OK; /* Return code */ +static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ + sqlite3_int64 nMap = nByte; + int rc; HANDLE hMap = NULL; /* New mapping handle */ - void *pNew = 0; /* New mapping */ - i64 nNewRnd; /* nNew rounded up */ - i64 nOldRnd; /* nOld rounded up */ - assert( pFile!=0 ); - assert( iOff==0 ); - assert( nOld>=0 ); - assert( nOld==0 || pFile->pMapRegion==(*ppMap) ); - assert( nNew>=0 ); - assert( ppMap ); - assert( pFile->hMap==NULL || pFile->pMapRegion==(*ppMap) ); - assert( pFile->pMapRegion==0 || pFile->pMapRegion==(*ppMap) ); - /* assert( pFile->mmapSize==nOld ); */ + assert( nMap>=0 || pFd->nFetchOut==0 ); + if( pFd->nFetchOut>0 ) return SQLITE_OK; - assert( winSysInfo.dwPageSize>0 ); - nNewRnd = ROUNDUP(nNew, winSysInfo.dwPageSize*1); - assert( nNewRnd>=0 ); - nOldRnd = ROUNDUP(nOld, winSysInfo.dwPageSize*1); - assert( nOldRnd>=0 ); - - if( nNewRnd==nOldRnd ){ - pFile->mmapSize = nNew; - return SQLITE_OK; - } - - /* If the SQLITE_MREMAP_EXTEND flag is set, then the size of the requested - ** mapping (nNew bytes) may be greater than the size of the database file. - ** If this is the case, extend the file on disk using ftruncate(). */ - assert( nNewRnd>0 || (flags & SQLITE_MREMAP_EXTEND)==0 ); - if( flags & SQLITE_MREMAP_EXTEND ){ - sqlite3_int64 oldSz; - rc = winFileSize(id, &oldSz); - if( rc==SQLITE_OK && nNewRnd>oldSz ){ - rc = winUnmap(id); /* Cannot truncate the file with an open mapping. */ - if( rc==SQLITE_OK ){ - rc = winTruncate(id, nNewRnd); - } + if( nMap<0 ){ + rc = winFileSize((sqlite3_file*)pFd, &nMap); + if( rc ){ + return SQLITE_IOERR_FSTAT; } - if( rc!=SQLITE_OK ) return rc; } - - /* If we get this far, unmap any old mapping. */ - rc = winUnmap(id); - if( rc!=SQLITE_OK ) return rc; - - /* And, if required, create a new mapping. */ - if( nNewRnd>0 ){ - i64 offset = ((iOff / winSysInfo.dwAllocationGranularity) * - (winSysInfo.dwAllocationGranularity)); + if( nMap>pFd->mmapLimit ){ + nMap = pFd->mmapLimit; + } + + if( nMap==0 && pFd->mmapSize>0 ){ + winUnmapfile(pFd); + } + if( nMap!=pFd->mmapSize ){ + void *pNew = 0; DWORD protect = PAGE_READONLY; DWORD flags = FILE_MAP_READ; - if( (pFile->ctrlFlags & WINFILE_RDONLY)==0 ){ + + winUnmapfile(pFd); + if( (pFd->ctrlFlags & WINFILE_RDONLY)==0 ){ protect = PAGE_READWRITE; flags |= FILE_MAP_WRITE; } #if SQLITE_OS_WINRT - hMap = osCreateFileMappingFromApp(pFile->h, NULL, protect, nNewRnd, NULL); + hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); #elif defined(SQLITE_WIN32_HAS_WIDE) - hMap = osCreateFileMappingW(pFile->h, NULL, protect, - (DWORD)((nNewRnd>>32) & 0xffffffff), - (DWORD)(nNewRnd & 0xffffffff), NULL); + hMap = osCreateFileMappingW(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); #elif defined(SQLITE_WIN32_HAS_ANSI) - hMap = osCreateFileMappingA(pFile->h, NULL, protect, - (DWORD)((nNewRnd>>32) & 0xffffffff), - (DWORD)(nNewRnd & 0xffffffff), NULL); + hMap = osCreateFileMappingA(pFd->h, NULL, protect, + (DWORD)((nMap>>32) & 0xffffffff), + (DWORD)(nMap & 0xffffffff), NULL); #endif if( hMap==NULL ){ - pFile->lastErrno = osGetLastError(); - rc = winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, - "winMremap1", pFile->zPath); - return rc; + pFd->lastErrno = osGetLastError(); + rc = winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile", pFd->zPath); + /* Log the error, but continue normal operation using xRead/xWrite */ + return SQLITE_OK; } assert( (nNewRnd % winSysInfo.dwPageSize)==0 ); #if SQLITE_OS_WINRT - pNew = osMapViewOfFileFromApp(hMap, flags, offset, nNewRnd); + pNew = osMapViewOfFileFromApp(hMap, flags, 0, nMap); #else - assert( sizeof(SIZE_T)==sizeof(sqlite3_int64) || nNewRnd<=0xffffffff ); - pNew = osMapViewOfFile(hMap, flags, - (DWORD)((offset>>32) & 0xffffffff), - (DWORD)(offset & 0xffffffff), - (SIZE_T)nNewRnd); + pNew = osMapViewOfFile(hMap, flags, 0, 0, (SIZE_T)nMap); #endif if( pNew==NULL ){ osCloseHandle(hMap); hMap = NULL; - pFile->lastErrno = osGetLastError(); - rc = winLogError(SQLITE_IOERR_MREMAP, pFile->lastErrno, - "winMremap2", pFile->zPath); + pFd->lastErrno = osGetLastError(); + winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, + "winMapfile", pFd->zPath); + return SQLITE_OK; } + pFd->pMapRegion = pNew; + pFd->mmapSize = nMap; + pFd->mmapOrigsize = nMap; } - pFile->hMap = hMap; - pFile->pMapRegion = pNew; - pFile->mmapSize = nNew; + return SQLITE_OK; +} - *ppMap = pNew; - return rc; +/* +** If possible, return a pointer to a mapping of file fd starting at offset +** iOff. The mapping must be valid for at least nAmt bytes. +** +** If such a pointer can be obtained, store it in *pp and return SQLITE_OK. +** Or, if one cannot but no error occurs, set *pp to 0 and return SQLITE_OK. +** Finally, if an error does occur, return an SQLite error code. The final +** value of *pp is undefined in this case. +** +** If this function does return a pointer, the caller must eventually +** release the reference by calling unixUnfetch(). +*/ +static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ + winFile *pFd = (winFile*)fd; /* The underlying database file */ + *pp = 0; + + if( pFd->mmapLimit>0 ){ + if( pFd->pMapRegion==0 ){ + int rc = winMapfile(pFd, -1); + if( rc!=SQLITE_OK ) return rc; + } + if( pFd->mmapSize >= iOff+nAmt ){ + *pp = &((u8 *)pFd->pMapRegion)[iOff]; + pFd->nFetchOut++; + } + } + return SQLITE_OK; +} + +/* +** If the third argument is non-NULL, then this function releases a +** reference obtained by an earlier call to unixFetch(). The second +** argument passed to this function must be the same as the corresponding +** argument that was passed to the unixFetch() invocation. +** +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping +** may now be invalid and should be unmapped. +*/ +static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ + winFile *pFd = (winFile*)fd; /* The underlying database file */ + + /* If p==0 (unmap the entire file) then there must be no outstanding + ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), + ** then there must be at least one outstanding. */ + assert( (p==0)==(pFd->nFetchOut==0) ); + + /* If p!=0, it must match the iOff value. */ + assert( p==0 || p==&((u8 *)pFd->pMapRegion)[iOff] ); + + if( p ){ + pFd->nFetchOut--; + }else{ + winUnmapfile(pFd); + } + + assert( pFd->nFetchOut>=0 ); + return SQLITE_OK; } /* @@ -3654,7 +3711,8 @@ static const sqlite3_io_methods winIoMethod = { winShmLock, /* xShmLock */ winShmBarrier, /* xShmBarrier */ winShmUnmap, /* xShmUnmap */ - winMremap, /* xMremap */ + winFetch, /* xFetch */ + winUnfetch /* xUnfetch */ }; /**************************************************************************** @@ -4052,6 +4110,8 @@ static int winOpen( pFile->hMap = NULL; pFile->pMapRegion = 0; pFile->mmapSize = 0; + pFile->mmapOrigsize = 0; + pFile->mmapLimit = SQLITE_DEFAULT_MMAP_LIMIT; OpenCounter(+1); return rc; From f9d18e472bd853c2849de8aee6b8498f20e254f4 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Apr 2013 17:56:28 +0000 Subject: [PATCH 043/140] Bug fix in the winMapfile() subroutine: Be sure to record the map object handle in the sqlite3_file object. FossilOrigin-Name: ee4d188e207efa24a26776fa4f025c6ac39cce73 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 17 ++++++++--------- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index e6eca96ae6..e46bdd10e1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sxFetch\sand\sxUnfetch\smethods\sto\sthe\sos_win.c\sVFS. -D 2013-04-01T17:22:51.830 +C Bug\sfix\sin\sthe\swinMapfile()\ssubroutine:\s\sBe\ssure\sto\srecord\sthe\smap\sobject\nhandle\sin\sthe\ssqlite3_file\sobject. +D 2013-04-01T17:56:28.844 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d6981218583748080374ed98a03f6a87e2bdc9e0 -F src/os_win.c f705e7ce230f86104dedcc2987a21d564b236659 +F src/os_win.c 390653f5c9b216bd445366f3c2b4863e52d51c24 F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a1040f0397d57855500926494c978623286ddc77 -R 52b13f77704d4760e77e0cfd9ab57bff +P a1653a257d6af6e8b10c819e68b12f6c2f485811 +R 88910ffb14df2e25c8fd9ccc5fce98d7 U drh -Z 878a27609130450cef13cc90a8feab48 +Z d2117afc31dea61942683c2e81660a00 diff --git a/manifest.uuid b/manifest.uuid index c685b6feac..2f1539dbd2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a1653a257d6af6e8b10c819e68b12f6c2f485811 \ No newline at end of file +ee4d188e207efa24a26776fa4f025c6ac39cce73 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index b1f5748638..5ce5a4762a 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3555,7 +3555,6 @@ static int winUnmapfile(winFile *pFile){ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ sqlite3_int64 nMap = nByte; int rc; - HANDLE hMap = NULL; /* New mapping handle */ assert( nMap>=0 || pFd->nFetchOut==0 ); if( pFd->nFetchOut>0 ) return SQLITE_OK; @@ -3584,17 +3583,17 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ flags |= FILE_MAP_WRITE; } #if SQLITE_OS_WINRT - hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); + pFd->hMap = osCreateFileMappingFromApp(pFd->h, NULL, protect, nMap, NULL); #elif defined(SQLITE_WIN32_HAS_WIDE) - hMap = osCreateFileMappingW(pFd->h, NULL, protect, + pFd->hMap = osCreateFileMappingW(pFd->h, NULL, protect, (DWORD)((nMap>>32) & 0xffffffff), (DWORD)(nMap & 0xffffffff), NULL); #elif defined(SQLITE_WIN32_HAS_ANSI) - hMap = osCreateFileMappingA(pFd->h, NULL, protect, + pFd->hMap = osCreateFileMappingA(pFd->h, NULL, protect, (DWORD)((nMap>>32) & 0xffffffff), (DWORD)(nMap & 0xffffffff), NULL); #endif - if( hMap==NULL ){ + if( pFd->hMap==NULL ){ pFd->lastErrno = osGetLastError(); rc = winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, "winMapfile", pFd->zPath); @@ -3603,13 +3602,13 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ } assert( (nNewRnd % winSysInfo.dwPageSize)==0 ); #if SQLITE_OS_WINRT - pNew = osMapViewOfFileFromApp(hMap, flags, 0, nMap); + pNew = osMapViewOfFileFromApp(pFd->hMap, flags, 0, nMap); #else - pNew = osMapViewOfFile(hMap, flags, 0, 0, (SIZE_T)nMap); + pNew = osMapViewOfFile(pFd->hMap, flags, 0, 0, (SIZE_T)nMap); #endif if( pNew==NULL ){ - osCloseHandle(hMap); - hMap = NULL; + osCloseHandle(pFd->hMap); + pFd->hMap = NULL; pFd->lastErrno = osGetLastError(); winLogError(SQLITE_IOERR_MMAP, pFd->lastErrno, "winMapfile", pFd->zPath); From e6ecd6630dd7fadec974ffd94ea678212e959717 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 1 Apr 2013 17:56:59 +0000 Subject: [PATCH 044/140] Attempt to emulate mremap() on non-Linux systems by allocating a second mapping immediately following the first in virtual memory. FossilOrigin-Name: 4d67433db8fb4754ae6b192945e479f3d7bad579 --- manifest | 14 +++--- manifest.uuid | 2 +- src/os_unix.c | 137 +++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 122 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index e6eca96ae6..52c1269c2e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sxFetch\sand\sxUnfetch\smethods\sto\sthe\sos_win.c\sVFS. -D 2013-04-01T17:22:51.830 +C Attempt\sto\semulate\smremap()\son\snon-Linux\ssystems\sby\sallocating\sa\ssecond\smapping\simmediately\sfollowing\sthe\sfirst\sin\svirtual\smemory. +D 2013-04-01T17:56:59.408 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c d6981218583748080374ed98a03f6a87e2bdc9e0 +F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c F src/os_win.c f705e7ce230f86104dedcc2987a21d564b236659 F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a1040f0397d57855500926494c978623286ddc77 -R 52b13f77704d4760e77e0cfd9ab57bff -U drh -Z 878a27609130450cef13cc90a8feab48 +P a1653a257d6af6e8b10c819e68b12f6c2f485811 +R 144bfd1ec5d6fde06365c0d3ea4b879a +U dan +Z 4e8a79b712821b5d3d9730a626894d79 diff --git a/manifest.uuid b/manifest.uuid index c685b6feac..694ae46ba8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a1653a257d6af6e8b10c819e68b12f6c2f485811 \ No newline at end of file +4d67433db8fb4754ae6b192945e479f3d7bad579 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index f91f4c4e7a..e92c7cc4a4 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -313,6 +313,17 @@ struct unixFile { #define threadid 0 #endif +/* +** HAVE_MREMAP defaults to true on Linux and false everywhere else. +*/ +#if !defined(HAVE_MREMAP) +# if defined(__linux__) && defined(_GNU_SOURCE) +# define HAVE_MREMAP 1 +# else +# define HAVE_MREMAP 0 +# endif +#endif + /* ** Different Unix systems declare open() in different ways. Same use ** open(const char*,int,mode_t). Others use open(const char*,int,...). @@ -450,7 +461,7 @@ static struct unix_syscall { { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, #define osMunmap ((void*(*)(void*,size_t))aSyscall[22].pCurrent) -#if defined(__linux__) && defined(_GNU_SOURCE) +#if HAVE_MREMAP { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, #else { "mremap", (sqlite3_syscall_ptr)0, 0 }, @@ -1124,7 +1135,6 @@ static int unixLogErrorAtLine( zErr = strerror(iErrno); #endif - assert( errcode!=SQLITE_OK ); if( zPath==0 ) zPath = ""; sqlite3_log(errcode, "os_unix.c:%d: (%d) %s(%s) - %s", @@ -3619,7 +3629,7 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ } } - if( pFile->mmapLimit>0 ){ + if( pFile->mmapLimit>0 && nByte>pFile->mmapSize ){ int rc; if( pFile->szChunk<=0 ){ if( robust_ftruncate(pFile->h, nByte) ){ @@ -4518,6 +4528,104 @@ static void unixUnmapfile(unixFile *pFd){ } } +/* +** Return the system page size. +*/ +static int unixGetPagesize(void){ +#if HAVE_MREMAP + return 512; +#elif _BSD_SOURCE + return getpagesize(); +#else + return (int)sysconf(_SC_PAGESIZE); +#endif +} + +/* +** Attempt to set the size of the memory mapping maintained by file +** descriptor pFd to nNew bytes. Any existing mapping is discarded. +** +** If successful, this function sets the following variables: +** +** unixFile.pMapRegion +** unixFile.mmapSize +** unixFile.mmapOrigsize +** +** If unsuccessful, an error message is logged via sqlite3_log() and +** the three variables above are zeroed. In this case SQLite should +** continue accessing the database using the xRead() and xWrite() +** methods. +*/ +static void unixRemapfile( + unixFile *pFd, /* File descriptor object */ + i64 nNew /* Required mapping size */ +){ + int h = pFd->h; /* File descriptor open on db file */ + u8 *pOrig = (u8 *)pFd->pMapRegion; /* Pointer to current file mapping */ + i64 nOrig = pFd->mmapOrigsize; /* Size of pOrig region in bytes */ + u8 *pNew = 0; /* Location of new mapping */ + int flags = PROT_READ; /* Flags to pass to mmap() */ + + assert( pFd->nFetchOut==0 ); + assert( nNew>pFd->mmapSize ); + assert( nNew<=pFd->mmapLimit ); + assert( nNew>0 ); + assert( pFd->mmapOrigsize>=pFd->mmapSize ); + + if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; + + if( pOrig ){ + const int szSyspage = unixGetPagesize(); + i64 nReuse = (pFd->mmapSize & ~(szSyspage-1)); + u8 *pReq = &pOrig[nReuse]; + + /* Unmap any pages of the existing mapping that cannot be reused. */ + if( nReuse!=nOrig ){ + osMunmap(pReq, nOrig-nReuse); + } + +#if HAVE_MREMAP + pNew = osMremap(pOrig, nReuse, nNew, MREMAP_MAYMOVE); +#else + pNew = osMmap(pReq, nNew-nReuse, flags, MAP_SHARED, h, nReuse); + if( pNew!=MAP_FAILED ){ + if( pNew!=pReq ){ + osMunmap(pNew, nNew - nReuse); + pNew = MAP_FAILED; + }else{ + pNew = pOrig; + } + } +#endif + + /* The attempt to extend the existing mapping failed. Free the existing + ** mapping and set pNew to NULL so that the code below will create a + ** new mapping from scratch. */ + if( pNew==MAP_FAILED ){ + pNew = 0; + osMunmap(pOrig, nReuse); + } + } + + /* If pNew is still NULL, try to create an entirely new mapping. */ + if( pNew==0 ){ + pNew = osMmap(0, nNew, flags, MAP_SHARED, h, 0); + if( pNew==MAP_FAILED ){ + pNew = 0; + nNew = 0; + unixLogError(SQLITE_OK, "mmap", pFd->zPath); + + /* If the mmap() above failed, assume that all subsequent mmap() calls + ** will probably fail too. Fall back to using xRead/xWrite exclusively + ** in this case. */ + pFd->mmapLimit = 0; + } + } + + pFd->pMapRegion = (void *)pNew; + pFd->mmapSize = pFd->mmapOrigsize = nNew; +} + /* ** Memory map or remap the file opened by file-descriptor pFd (if the file ** is already mapped, the existing mapping is replaced by the new). Or, if @@ -4554,28 +4662,11 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ } if( nMap!=pFd->mmapSize ){ - void *pNew = 0; - -#if defined(__linux__) && defined(_GNU_SOURCE) - if( pFd->pMapRegion && nMap>0 ){ - pNew = osMremap(pFd->pMapRegion, pFd->mmapOrigsize, nMap, MREMAP_MAYMOVE); - }else -#endif - { + if( nMap>0 ){ + unixRemapfile(pFd, nMap); + }else{ unixUnmapfile(pFd); - if( nMap>0 ){ - int flags = PROT_READ; - if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; - pNew = osMmap(0, nMap, flags, MAP_SHARED, pFd->h, 0); - } } - - if( pNew==MAP_FAILED ){ - return SQLITE_IOERR_MMAP; - } - pFd->pMapRegion = pNew; - pFd->mmapSize = nMap; - pFd->mmapOrigsize = nMap; } return SQLITE_OK; From db56bcbe42c7d847eda67533717316e4ff6212ae Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Apr 2013 18:15:50 +0000 Subject: [PATCH 045/140] In windows, round down the size of the memory mapped region to the next smaller multiple of the page size. FossilOrigin-Name: 0e44ed1badde6d766f18d1373d968e5b286b15bd --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ed4a17e30f..d43ccc6be3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\saccidental\sfork. -D 2013-04-01T17:58:10.824 +C In\swindows,\sround\sdown\sthe\ssize\sof\sthe\smemory\smapped\sregion\sto\sthe\snext\nsmaller\smultiple\sof\sthe\spage\ssize. +D 2013-04-01T18:15:50.853 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c -F src/os_win.c 390653f5c9b216bd445366f3c2b4863e52d51c24 +F src/os_win.c 4cb9b2621f4869e17c8caa627903dc114b1dfb37 F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 4d67433db8fb4754ae6b192945e479f3d7bad579 ee4d188e207efa24a26776fa4f025c6ac39cce73 -R 5abed6f6d2f091dbf577966219204ad2 -U dan -Z 994fbbedca554dbff615409d58ee1a84 +P 5f4437c0e35c8aa740515aac3e81e0146716d2bb +R bde1c152af7414f4a8d87d63f33cdc11 +U drh +Z 9887751c05c44b393c15dab1850774d3 diff --git a/manifest.uuid b/manifest.uuid index 5c2360cd0a..91d9770506 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5f4437c0e35c8aa740515aac3e81e0146716d2bb \ No newline at end of file +0e44ed1badde6d766f18d1373d968e5b286b15bd \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 5ce5a4762a..6965ee90bb 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3568,6 +3568,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ if( nMap>pFd->mmapLimit ){ nMap = pFd->mmapLimit; } + nMap &= ~(sqlite3_int64)(winSysInfo.dwPageSize - 1); if( nMap==0 && pFd->mmapSize>0 ){ winUnmapfile(pFd); From a539c8a4642c91fed157ee48ddf60f92985c5ac9 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Apr 2013 18:25:48 +0000 Subject: [PATCH 046/140] Add a comment to xUnfetch on windows to say that the call to winUnmapfile() might be unnecessary. FossilOrigin-Name: e3c63c31a21b27806592d066a013a111b280c61f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index d43ccc6be3..9d43650636 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\swindows,\sround\sdown\sthe\ssize\sof\sthe\smemory\smapped\sregion\sto\sthe\snext\nsmaller\smultiple\sof\sthe\spage\ssize. -D 2013-04-01T18:15:50.853 +C Add\sa\scomment\sto\sxUnfetch\son\swindows\sto\ssay\sthat\sthe\scall\sto\swinUnmapfile()\nmight\sbe\sunnecessary. +D 2013-04-01T18:25:48.030 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c -F src/os_win.c 4cb9b2621f4869e17c8caa627903dc114b1dfb37 +F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5f4437c0e35c8aa740515aac3e81e0146716d2bb -R bde1c152af7414f4a8d87d63f33cdc11 +P 0e44ed1badde6d766f18d1373d968e5b286b15bd +R 8c3ade0fe410bf8f08819b27491eb958 U drh -Z 9887751c05c44b393c15dab1850774d3 +Z 1d97c5c3c6a55788c16fe769f3e976ae diff --git a/manifest.uuid b/manifest.uuid index 91d9770506..224e9238f1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0e44ed1badde6d766f18d1373d968e5b286b15bd \ No newline at end of file +e3c63c31a21b27806592d066a013a111b280c61f \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 6965ee90bb..ee7a71018e 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3676,6 +3676,10 @@ static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ if( p ){ pFd->nFetchOut--; }else{ + /* FIXME: If Windows truly always prevents truncating or deleting a + ** file while a mapping is held, then the following winUnmapfile() call + ** is unnecessary can can be omitted - potentially improving + ** performance. */ winUnmapfile(pFd); } From a1f42c7c32b7326568305f64cf5c592910928a3e Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Apr 2013 22:38:06 +0000 Subject: [PATCH 047/140] Add the SQLITE_CONFIG_MMAP_LIMIT configuration option for overriding the SQLITE_DEFAULT_MMAP_LIMIT compile-time setting. Enhance "PRAGMA mmap_limit" so that without a specific database name, it sets the limit on all database files and changes the default for any future databases that might be added using ATTACH. FossilOrigin-Name: 78141d0a16dd1d56b575fccd149de7fa789cb06c --- manifest | 26 +++++++++++++------------- manifest.uuid | 2 +- src/btree.c | 1 + src/global.c | 1 + src/main.c | 8 ++++++++ src/os_unix.c | 1 + src/pager.c | 2 +- src/pragma.c | 13 +++++++++++-- src/sqlite.h.in | 14 ++++++++++++++ src/sqliteInt.h | 2 ++ 10 files changed, 53 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 9d43650636..995cb45fa0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\scomment\sto\sxUnfetch\son\swindows\sto\ssay\sthat\sthe\scall\sto\swinUnmapfile()\nmight\sbe\sunnecessary. -D 2013-04-01T18:25:48.030 +C Add\sthe\sSQLITE_CONFIG_MMAP_LIMIT\sconfiguration\soption\sfor\soverriding\sthe\nSQLITE_DEFAULT_MMAP_LIMIT\scompile-time\ssetting.\s\sEnhance\s"PRAGMA\smmap_limit"\nso\sthat\swithout\sa\sspecific\sdatabase\sname,\sit\ssets\sthe\slimit\son\sall\sdatabase\nfiles\sand\schanges\sthe\sdefault\sfor\sany\sfuture\sdatabases\sthat\smight\sbe\sadded\nusing\sATTACH. +D 2013-04-01T22:38:06.929 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 84c9a4a20b827a6f7dbcbcda59607dbee57af446 +F src/btree.c bb679fddb7269dd7cfb950f845b1b18f19df4f4a F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -134,7 +134,7 @@ F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c e16942bd5c8a868ac53287886464a5ed0e72b179 F src/func.c 48987c025d69399f59a1c2a553cea5da41bf105d -F src/global.c e59ecd2c553ad0d4bfbc84ca71231336f8993a7a +F src/global.c d2494a1cea8f66a2cab8258449df07f8f0ae6330 F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4 F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -143,7 +143,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 1422eba4aa2b1fb5f7b3aef574752272477d21e2 -F src/main.c 379160ec3680e3009aa4978eac47027c3ef27ac5 +F src/main.c 54a841854734b6731c4d026834788cac6a19f3d1 F src/malloc.c fe085aa851b666b7c375c1ff957643dc20a04bf6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa @@ -160,15 +160,15 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c +F src/os_unix.c 9912ab0cfcd7fa2b058544d861a9fead3adac8da F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 -F src/pager.c 2b9980e87296812a6ce51121a3335550ae25e3ec +F src/pager.c 072512d05dacd61b1f147736825bd32fdc890ba4 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c 8875d32d54530300e6aa31a9d5426b2088a69b12 +F src/pragma.c d0231f412dea33c35c1309b45b535cf0c10c63c8 F src/prepare.c 310eaff1ee5f3c700b3545afb095cfe9346efc3a F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -176,10 +176,10 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca -F src/sqlite.h.in 921ba42ec3531fe8cf021492ffd781b161383f1b +F src/sqlite.h.in 42c69ea00dfcce84394d6ce0d8dac7bb3d2ecff2 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 -F src/sqliteInt.h 0f8f05ee4db4ba9120b38f7a3992b325698f6e8a +F src/sqliteInt.h 1664dc5ad6f8d4dab871416628aa3271044d66c0 F src/sqliteLimit.h 83d0cd0ce3050a80930406d3c7aecb48c1ccdfc5 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 0e44ed1badde6d766f18d1373d968e5b286b15bd -R 8c3ade0fe410bf8f08819b27491eb958 +P e3c63c31a21b27806592d066a013a111b280c61f +R 6d1c950d2134f5ef56d487146725ba7a U drh -Z 1d97c5c3c6a55788c16fe769f3e976ae +Z 72c1547cadf3e5127066e34139489015 diff --git a/manifest.uuid b/manifest.uuid index 224e9238f1..838dd7eeb0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e3c63c31a21b27806592d066a013a111b280c61f \ No newline at end of file +78141d0a16dd1d56b575fccd149de7fa789cb06c \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 03764a9267..9672e0e000 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1864,6 +1864,7 @@ int sqlite3BtreeOpen( rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, EXTRA_SIZE, flags, vfsFlags, pageReinit); if( rc==SQLITE_OK ){ + sqlite3PagerSetMmapLimit(pBt->pPager, db->mxMmap); rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); } if( rc!=SQLITE_OK ){ diff --git a/src/global.c b/src/global.c index f5da7e7f1f..aec3958c1f 100644 --- a/src/global.c +++ b/src/global.c @@ -156,6 +156,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { (void*)0, /* pHeap */ 0, /* nHeap */ 0, 0, /* mnHeap, mxHeap */ + SQLITE_DEFAULT_MMAP_LIMIT, /* mxMmap */ (void*)0, /* pScratch */ 0, /* szScratch */ 0, /* nScratch */ diff --git a/src/main.c b/src/main.c index 2fcb236e83..288d4535f3 100644 --- a/src/main.c +++ b/src/main.c @@ -496,6 +496,13 @@ int sqlite3_config(int op, ...){ } #endif + case SQLITE_CONFIG_MMAP_LIMIT: { + sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); + if( mxMmap<0 ) mxMmap = SQLITE_DEFAULT_MMAP_LIMIT; + sqlite3GlobalConfig.mxMmap = mxMmap; + break; + } + default: { rc = SQLITE_ERROR; break; @@ -2316,6 +2323,7 @@ static int openDatabase( memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); db->autoCommit = 1; db->nextAutovac = -1; + db->mxMmap = sqlite3GlobalConfig.mxMmap; db->nextPagesize = 0; db->flags |= SQLITE_ShortColNames | SQLITE_AutoIndex | SQLITE_EnableTrigger #if SQLITE_DEFAULT_FILE_FORMAT<4 diff --git a/src/os_unix.c b/src/os_unix.c index e92c7cc4a4..c10841abf9 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3711,6 +3711,7 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } case SQLITE_FCNTL_MMAP_LIMIT: { pFile->mmapLimit = *(i64*)pArg; +printf("MMAP-LIMIT(%s) -> %lld\n", pFile->zPath, pFile->mmapLimit); return SQLITE_OK; } #ifdef SQLITE_DEBUG diff --git a/src/pager.c b/src/pager.c index 550c6c7564..68b2e2d667 100644 --- a/src/pager.c +++ b/src/pager.c @@ -4749,7 +4749,7 @@ int sqlite3PagerOpen( /* pPager->pBusyHandlerArg = 0; */ pPager->xReiniter = xReinit; /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ - pPager->mxMmap = SQLITE_DEFAULT_MMAP_LIMIT; + /* pPager->mxMmap = SQLITE_DEFAULT_MMAP_LIMIT // will be set by btree.c */ *ppPager = pPager; return SQLITE_OK; diff --git a/src/pragma.c b/src/pragma.c index cd1cfbe213..7a515cba2e 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -750,7 +750,9 @@ void sqlite3Pragma( ** Used to set mapping size limit. The mapping size limit is ** used to limit the aggregate size of all memory mapped regions of the ** database file. If this parameter is set to zero, then memory mapping - ** is not used at all. The parameter N is measured in bytes. + ** is not used at all. If N is negative, then the default memory map + ** limit determined by sqlite3_config(SQLITE_CONFIG_MMAP_LIMIT) is set. + ** The parameter N is measured in bytes. ** ** This value is advisory. The underlying VFS is free to memory map ** as little or as much as it wants. Except, if N is set to 0 then the @@ -760,8 +762,15 @@ void sqlite3Pragma( assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( zRight ){ sqlite3_int64 size; + int ii; sqlite3Atoi64(zRight, &size, 1000, SQLITE_UTF8); - sqlite3BtreeSetMmapLimit(pDb->pBt, size); + if( size<0 ) size = sqlite3GlobalConfig.mxMmap; + if( pId2->n==0 ) db->mxMmap = size; + for(ii=db->nDb-1; ii>=0; ii--){ + if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ + sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, size); + } + } } }else diff --git a/src/sqlite.h.in b/src/sqlite.h.in index acb49bf5f9..de1316606b 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1637,6 +1637,19 @@ struct sqlite3_mem_methods { ** points to has just been executed. Or, if the fourth parameter is 2, then ** the connection being passed as the second parameter is being closed. The ** third parameter is passed NULL In this case. +** +** [[SQLITE_CONFIG_MMAP_LIMIT]] +**
    SQLITE_CONFIG_MMAP_LIMIT +**
    The sole argument should be a 64-bit integer (an sqlite3_int64) that +** is the default maximum number of bytes of process address space that +** should be used for accessing each database file using memory mapping. +** The default setting can be overridden by each database connection using +** either the [PRAGMA mmap_limit] command or the "mmaplimit" query parameter +** on the [URI filename] when opening the databaes file or by using the +** [SQLITE_FCNTL_MMAP_LIMIT] file control. The value set here overrides the +** compile-time default that is set using the [SQLITE_DEFAULT_MMAP_LIMIT] +** compile-time option. If the argument to this option is negative, then +** the memory map limit is set to the compile-time default. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -1660,6 +1673,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ #define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_LIMIT 22 /* sqlite3_int64 */ /* ** CAPI3REF: Database Connection Configuration Options diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 129c4c5ea5..d894051790 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -834,6 +834,7 @@ struct sqlite3 { int nDb; /* Number of backends currently in use */ int flags; /* Miscellaneous flags. See below */ i64 lastRowid; /* ROWID of most recent insert (see above) */ + i64 mxMmap; /* Default mmap_limit setting */ unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ @@ -2505,6 +2506,7 @@ struct Sqlite3Config { void *pHeap; /* Heap storage space */ int nHeap; /* Size of pHeap[] */ int mnReq, mxReq; /* Min and max heap requests sizes */ + sqlite3_int64 mxMmap; /* Maximum mmap() space per open file */ void *pScratch; /* Scratch memory */ int szScratch; /* Size of each scratch buffer */ int nScratch; /* Number of scratch buffers */ From 3861f546e3f3752cdce48fb75e37da7394aba882 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Apr 2013 22:42:48 +0000 Subject: [PATCH 048/140] Remove a debugging printf() accidently left in the previous check-in. FossilOrigin-Name: 8198cdd8ac5dcc1c677fffa869ac965186b96abf --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 995cb45fa0..b2e4dbad76 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_CONFIG_MMAP_LIMIT\sconfiguration\soption\sfor\soverriding\sthe\nSQLITE_DEFAULT_MMAP_LIMIT\scompile-time\ssetting.\s\sEnhance\s"PRAGMA\smmap_limit"\nso\sthat\swithout\sa\sspecific\sdatabase\sname,\sit\ssets\sthe\slimit\son\sall\sdatabase\nfiles\sand\schanges\sthe\sdefault\sfor\sany\sfuture\sdatabases\sthat\smight\sbe\sadded\nusing\sATTACH. -D 2013-04-01T22:38:06.929 +C Remove\sa\sdebugging\sprintf()\saccidently\sleft\sin\sthe\sprevious\scheck-in. +D 2013-04-01T22:42:48.981 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 9912ab0cfcd7fa2b058544d861a9fead3adac8da +F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 F src/pager.c 072512d05dacd61b1f147736825bd32fdc890ba4 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e3c63c31a21b27806592d066a013a111b280c61f -R 6d1c950d2134f5ef56d487146725ba7a +P 78141d0a16dd1d56b575fccd149de7fa789cb06c +R fed26fbe7b97d6a5db36e7e764991fa9 U drh -Z 72c1547cadf3e5127066e34139489015 +Z c0440f45f7713ec5c0ea751d4cf10d7e diff --git a/manifest.uuid b/manifest.uuid index 838dd7eeb0..ee803ab26e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -78141d0a16dd1d56b575fccd149de7fa789cb06c \ No newline at end of file +8198cdd8ac5dcc1c677fffa869ac965186b96abf \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index c10841abf9..e92c7cc4a4 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3711,7 +3711,6 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } case SQLITE_FCNTL_MMAP_LIMIT: { pFile->mmapLimit = *(i64*)pArg; -printf("MMAP-LIMIT(%s) -> %lld\n", pFile->zPath, pFile->mmapLimit); return SQLITE_OK; } #ifdef SQLITE_DEBUG From 98d2038fc2325c968620afe515e3a99a8a08f098 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 00:15:23 +0000 Subject: [PATCH 049/140] Always send the SQLITE_FCNTL_MMAP_LIMIT pragma to the VFS, even if the limit is zero and even if the VFS does not support xFetch(). FossilOrigin-Name: 01ffdabbad30f1c157f2b33b1e85ee4d6c4632dd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 6 ++---- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index b2e4dbad76..3d3492ec7c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\sdebugging\sprintf()\saccidently\sleft\sin\sthe\sprevious\scheck-in. -D 2013-04-01T22:42:48.981 +C Always\ssend\sthe\sSQLITE_FCNTL_MMAP_LIMIT\spragma\sto\sthe\sVFS,\seven\sif\sthe\slimit\nis\szero\sand\seven\sif\sthe\sVFS\sdoes\snot\ssupport\sxFetch(). +D 2013-04-02T00:15:23.044 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 -F src/pager.c 072512d05dacd61b1f147736825bd32fdc890ba4 +F src/pager.c 95e7a64bfdd244603d595b455f4e9ef6cf124b54 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 78141d0a16dd1d56b575fccd149de7fa789cb06c -R fed26fbe7b97d6a5db36e7e764991fa9 +P 8198cdd8ac5dcc1c677fffa869ac965186b96abf +R a603a97de401ee9321fd272228a03ad0 U drh -Z c0440f45f7713ec5c0ea751d4cf10d7e +Z 4afc43008f65fbdcf2b0b7902c3ce20b diff --git a/manifest.uuid b/manifest.uuid index ee803ab26e..4e3c99ee29 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8198cdd8ac5dcc1c677fffa869ac965186b96abf \ No newline at end of file +01ffdabbad30f1c157f2b33b1e85ee4d6c4632dd \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 68b2e2d667..e391652f48 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3360,10 +3360,8 @@ static void pagerFixMaplimit(Pager *pPager){ sqlite3_file *fd = pPager->fd; if( isOpen(fd) ){ pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->mxMmap>0; - if( pPager->bUseFetch ){ - sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT, - (void*)&pPager->mxMmap); - } + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT, + (void*)&pPager->mxMmap); } } From 092d993cb79d11675246e04b1e15d2c48a825f2a Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 00:41:18 +0000 Subject: [PATCH 050/140] Remove an unreachable condition from pager.c. FossilOrigin-Name: 3628e86bf131cb205f08a4b299d84007cd6b25c3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 5 ++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 3d3492ec7c..767726d357 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Always\ssend\sthe\sSQLITE_FCNTL_MMAP_LIMIT\spragma\sto\sthe\sVFS,\seven\sif\sthe\slimit\nis\szero\sand\seven\sif\sthe\sVFS\sdoes\snot\ssupport\sxFetch(). -D 2013-04-02T00:15:23.044 +C Remove\san\sunreachable\scondition\sfrom\spager.c. +D 2013-04-02T00:41:18.590 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 -F src/pager.c 95e7a64bfdd244603d595b455f4e9ef6cf124b54 +F src/pager.c 37ac3175cb46bf4eba5c23005c49700b33ed2dbe F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 8198cdd8ac5dcc1c677fffa869ac965186b96abf -R a603a97de401ee9321fd272228a03ad0 +P 01ffdabbad30f1c157f2b33b1e85ee4d6c4632dd +R 683cfbd9fb419480e60db9265a2009fc U drh -Z 4afc43008f65fbdcf2b0b7902c3ce20b +Z c53b9215e567f2251d489b56e620483a diff --git a/manifest.uuid b/manifest.uuid index 4e3c99ee29..cd77251101 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -01ffdabbad30f1c157f2b33b1e85ee4d6c4632dd \ No newline at end of file +3628e86bf131cb205f08a4b299d84007cd6b25c3 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index e391652f48..75d12cdc1e 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2936,15 +2936,14 @@ static int pagerUndoCallback(void *pCtx, Pgno iPg){ Pager *pPager = (Pager *)pCtx; PgHdr *pPg; + assert( pagerUseWal(pPager) ); pPg = sqlite3PagerLookup(pPager, iPg); if( pPg ){ if( sqlite3PcachePageRefcount(pPg)==1 ){ sqlite3PcacheDrop(pPg); }else{ u32 iFrame = 0; - if( pagerUseWal(pPager) ){ - rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); - } + rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); if( rc==SQLITE_OK ){ rc = readDbPage(pPg, iFrame); } From 6d81449fc6cabb4ee186cf72547eb1e949e011ac Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 01:45:10 +0000 Subject: [PATCH 051/140] Reorder two conditions to make coverage testing easier. FossilOrigin-Name: 793ba4e996426522eeaa86589a9e974fa1fc1522 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 767726d357..37da7c88aa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunreachable\scondition\sfrom\spager.c. -D 2013-04-02T00:41:18.590 +C Reorder\stwo\sconditions\sto\smake\scoverage\stesting\seasier. +D 2013-04-02T01:45:10.945 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 -F src/pager.c 37ac3175cb46bf4eba5c23005c49700b33ed2dbe +F src/pager.c 46fa7d8b29e7fb8baa6111690d900c9111a5a397 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 01ffdabbad30f1c157f2b33b1e85ee4d6c4632dd -R 683cfbd9fb419480e60db9265a2009fc +P 3628e86bf131cb205f08a4b299d84007cd6b25c3 +R 5022c5197e31d50bddc957f6984f11f1 U drh -Z c53b9215e567f2251d489b56e620483a +Z 379e6b5efd0a997c87c1f9a5494b7aa5 diff --git a/manifest.uuid b/manifest.uuid index cd77251101..8133886ec9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3628e86bf131cb205f08a4b299d84007cd6b25c3 \ No newline at end of file +793ba4e996426522eeaa86589a9e974fa1fc1522 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 75d12cdc1e..cb2b39a992 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5128,7 +5128,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** nothing to rollback, so this routine is a no-op. */ static void pagerUnlockIfUnused(Pager *pPager){ - if( (sqlite3PcacheRefCount(pPager->pPCache)==0) && pPager->nMmapOut==0 ){ + if( pPager->nMmapOut==0 && (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ pagerUnlockAndRollback(pPager); } } From a3a2b3bdc0cfad4e96a547ff953f240ca9845061 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 10:29:13 +0000 Subject: [PATCH 052/140] Proposed template preprocessor magic for activating mmap only on platforms where we know it works. FossilOrigin-Name: d96272f031102514b02bd839aac9e37618872200 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqliteLimit.h | 21 +++++++++++++++++++-- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 37da7c88aa..edce1feec8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reorder\stwo\sconditions\sto\smake\scoverage\stesting\seasier. -D 2013-04-02T01:45:10.945 +C Proposed\stemplate\spreprocessor\smagic\sfor\sactivating\smmap\sonly\son\splatforms\nwhere\swe\sknow\sit\sworks. +D 2013-04-02T10:29:13.935 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -180,7 +180,7 @@ F src/sqlite.h.in 42c69ea00dfcce84394d6ce0d8dac7bb3d2ecff2 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 1664dc5ad6f8d4dab871416628aa3271044d66c0 -F src/sqliteLimit.h 83d0cd0ce3050a80930406d3c7aecb48c1ccdfc5 +F src/sqliteLimit.h 4d8d04c43bf2886d94f1ab7992fee5081e09a7f5 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 @@ -1040,7 +1040,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3628e86bf131cb205f08a4b299d84007cd6b25c3 -R 5022c5197e31d50bddc957f6984f11f1 +P 793ba4e996426522eeaa86589a9e974fa1fc1522 +R 2b5ee6ceec98b53d533d4ea410a1bfc7 U drh -Z 379e6b5efd0a997c87c1f9a5494b7aa5 +Z ead310385cbe70188969099b54288940 diff --git a/manifest.uuid b/manifest.uuid index 8133886ec9..6c51e2ad5a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -793ba4e996426522eeaa86589a9e974fa1fc1522 \ No newline at end of file +d96272f031102514b02bd839aac9e37618872200 \ No newline at end of file diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index 2013139456..6adf3e8726 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -210,6 +210,23 @@ /* ** Default maximum size of memory used by xFetch in the VFS. */ -#ifndef SQLITE_DEFAULT_MMAP_LIMIT -# define SQLITE_DEFAULT_MMAP_LIMIT (256*1024*1024) +#ifdef __APPLE__ +# include +# if defined(TARGET_OS_IPHONE) +# define SQLITE_DEFAULT_MMAP_LIMIT 0 +# endif +#endif +#ifndef SQLITE_DEFAULT_MMAP_LIMIT +# if defined(__linux__) \ + || defined(_WIN32) \ + || (defined(__APPLE__) && defined(__MACH__) && !defined(TARGET_OS_IPHONE)) \ + || defined(__sun) \ + || defined(__DragonFly__) \ + || defined(__FreeBSD__) \ + || defined(__NetBSD__) \ + || defined(__OpenBSD__) +# define SQLITE_DEFAULT_MMAP_LIMIT (256*1024*1024) +# else +# define SQLITE_DEFAULT_MMAP_LIMIT 0 +# endif #endif From 4ff7bc45c6c6a7156e433ff50395dfe5e2885c38 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Apr 2013 12:04:09 +0000 Subject: [PATCH 053/140] Add test cases for errors in mmap() or mremap() is os_unix.c. FossilOrigin-Name: 3098a3c1e7305033904a496ef534cb312a876fab --- manifest | 19 +++++------ manifest.uuid | 2 +- src/os_unix.c | 26 ++++++++------- src/test_syscall.c | 15 ++++++++- test/mmap2.test | 79 ++++++++++++++++++++++++++++++++++++++++++++++ test/speed1p.test | 3 +- 6 files changed, 120 insertions(+), 24 deletions(-) create mode 100644 test/mmap2.test diff --git a/manifest b/manifest index edce1feec8..645d467c46 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Proposed\stemplate\spreprocessor\smagic\sfor\sactivating\smmap\sonly\son\splatforms\nwhere\swe\sknow\sit\sworks. -D 2013-04-02T10:29:13.935 +C Add\stest\scases\sfor\serrors\sin\smmap()\sor\smremap()\sis\sos_unix.c. +D 2013-04-02T12:04:09.729 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c d136eca0cff2f8a5ac2f45867d5e5153d60e377c +F src/os_unix.c 42684a59b06b912b3ab80a8ad9f4295c0572a1f4 F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 F src/pager.c 46fa7d8b29e7fb8baa6111690d900c9111a5a397 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -226,7 +226,7 @@ F src/test_spellfix.c 56dfa6d583ac34f61af0834d7b58d674e7e18e13 F src/test_sqllog.c bc50e5afeb7fb50e77b4594e42302df9d05446aa F src/test_stat.c d1569c7a4839f13e80187e2c26b2ab4da2d03935 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd -F src/test_syscall.c fe018f95049d4f24e036d1d649516b60689b4e57 +F src/test_syscall.c 7e8293e4e6971b0f44c7f7f37b1315a8cc9f6018 F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c e286f2173563f2a1747c24bcda6b9d030bf4f4e4 F src/test_vfs.c fb16b2d9938cf0c1afc5a423b55b952fcc024275 @@ -631,6 +631,7 @@ F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 F test/mmap1.test 9fb71daaca72f66fcc7a3b3a37fc3db99988b57a +F test/mmap2.test bb0c97cdac0db06a5acdc6836eef9866815481b1 F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 @@ -731,7 +732,7 @@ F test/softheap1.test c16709a16ad79fa43b32929b2e623d1d117ccf53 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb -F test/speed1p.test c4a469f29f135f4d76c55b1f2a52f36e209466cc +F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 F test/speed2.test 53177056baf6556dcbdcf032bbdfc41c1aa74ded F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 @@ -1040,7 +1041,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 793ba4e996426522eeaa86589a9e974fa1fc1522 -R 2b5ee6ceec98b53d533d4ea410a1bfc7 -U drh -Z ead310385cbe70188969099b54288940 +P d96272f031102514b02bd839aac9e37618872200 +R 323dfbce5d7140c5ce46f22a5112dcc8 +U dan +Z 3d1807f4fa3a8cf06a6141b170b215ce diff --git a/manifest.uuid b/manifest.uuid index 6c51e2ad5a..dfa80de2b5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d96272f031102514b02bd839aac9e37618872200 \ No newline at end of file +3098a3c1e7305033904a496ef534cb312a876fab \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index e92c7cc4a4..07c919c983 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4560,6 +4560,7 @@ static void unixRemapfile( unixFile *pFd, /* File descriptor object */ i64 nNew /* Required mapping size */ ){ + const char *zErr = "mmap"; int h = pFd->h; /* File descriptor open on db file */ u8 *pOrig = (u8 *)pFd->pMapRegion; /* Pointer to current file mapping */ i64 nOrig = pFd->mmapOrigsize; /* Size of pOrig region in bytes */ @@ -4571,6 +4572,7 @@ static void unixRemapfile( assert( nNew<=pFd->mmapLimit ); assert( nNew>0 ); assert( pFd->mmapOrigsize>=pFd->mmapSize ); + assert( MAP_FAILED!=0 ); if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; @@ -4586,12 +4588,13 @@ static void unixRemapfile( #if HAVE_MREMAP pNew = osMremap(pOrig, nReuse, nNew, MREMAP_MAYMOVE); + zErr = "mremap"; #else pNew = osMmap(pReq, nNew-nReuse, flags, MAP_SHARED, h, nReuse); if( pNew!=MAP_FAILED ){ if( pNew!=pReq ){ osMunmap(pNew, nNew - nReuse); - pNew = MAP_FAILED; + pNew = 0; }else{ pNew = pOrig; } @@ -4602,7 +4605,6 @@ static void unixRemapfile( ** mapping and set pNew to NULL so that the code below will create a ** new mapping from scratch. */ if( pNew==MAP_FAILED ){ - pNew = 0; osMunmap(pOrig, nReuse); } } @@ -4610,18 +4612,18 @@ static void unixRemapfile( /* If pNew is still NULL, try to create an entirely new mapping. */ if( pNew==0 ){ pNew = osMmap(0, nNew, flags, MAP_SHARED, h, 0); - if( pNew==MAP_FAILED ){ - pNew = 0; - nNew = 0; - unixLogError(SQLITE_OK, "mmap", pFd->zPath); - - /* If the mmap() above failed, assume that all subsequent mmap() calls - ** will probably fail too. Fall back to using xRead/xWrite exclusively - ** in this case. */ - pFd->mmapLimit = 0; - } } + if( pNew==MAP_FAILED ){ + pNew = 0; + nNew = 0; + unixLogError(SQLITE_OK, zErr, pFd->zPath); + + /* If the mmap() above failed, assume that all subsequent mmap() calls + ** will probably fail too. Fall back to using xRead/xWrite exclusively + ** in this case. */ + pFd->mmapLimit = 0; + } pFd->pMapRegion = (void *)pNew; pFd->mmapSize = pFd->mmapOrigsize = nNew; } diff --git a/src/test_syscall.c b/src/test_syscall.c index 0b053fc599..3d117fb2ab 100644 --- a/src/test_syscall.c +++ b/src/test_syscall.c @@ -108,7 +108,7 @@ static int ts_pwrite64(int fd, const void *aBuf, size_t nBuf, off_t off); static int ts_fchmod(int fd, mode_t mode); static int ts_fallocate(int fd, off_t off, off_t len); static void *ts_mmap(void *, size_t, int, int, int, off_t); - +static void *ts_mremap(void*, size_t, size_t, int, ...); struct TestSyscallArray { const char *zName; @@ -134,6 +134,7 @@ struct TestSyscallArray { /* 14 */ { "fchmod", (sqlite3_syscall_ptr)ts_fchmod, 0, 0, 0 }, /* 15 */ { "fallocate", (sqlite3_syscall_ptr)ts_fallocate, 0, 0, 0 }, /* 16 */ { "mmap", (sqlite3_syscall_ptr)ts_mmap, 0, 0, 0 }, + /* 17 */ { "mremap", (sqlite3_syscall_ptr)ts_mremap, 0, 0, 0 }, { 0, 0, 0, 0, 0 } }; @@ -156,6 +157,7 @@ struct TestSyscallArray { #define orig_fchmod ((int(*)(int,mode_t))aSyscall[14].xOrig) #define orig_fallocate ((int(*)(int,off_t,off_t))aSyscall[15].xOrig) #define orig_mmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[16].xOrig) +#define orig_mremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[17].xOrig) /* ** This function is called exactly once from within each invocation of a @@ -395,6 +397,17 @@ static void *ts_mmap( return orig_mmap(pAddr, nByte, prot, flags, fd, iOff); } +static void *ts_mremap(void *a, size_t b, size_t c, int d, ...){ + va_list ap; + void *pArg; + if( tsIsFailErrno("mremap") ){ + return MAP_FAILED; + } + va_start(ap, d); + pArg = va_arg(ap, void *); + return orig_mremap(a, b, c, d, pArg); +} + static int test_syscall_install( void * clientData, Tcl_Interp *interp, diff --git a/test/mmap2.test b/test/mmap2.test new file mode 100644 index 0000000000..d43ea97adf --- /dev/null +++ b/test/mmap2.test @@ -0,0 +1,79 @@ +# 2013 March 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file tests the effect of the mmap() or mremap() system calls +# returning an error on the library. +# +# If either mmap() or mremap() fails, SQLite should log an error +# message, then continue accessing the database using read() and +# write() exclusively. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix mmap2 + +if {[test_syscall defaultvfs] != "unix"} { + finish_test + return +} + +db close +sqlite3_shutdown +test_sqlite3_log xLog +proc xLog {error_code msg} { + if {[string match os_unix.c* $msg]} { + lappend ::log $msg + } +} + +foreach syscall {mmap mremap} { + test_syscall uninstall + if {[catch {test_syscall install $syscall}]} continue + + for {set i 1} {$i < 20} {incr i} { + reset_db + + test_syscall fault $i 1 + test_syscall errno $syscall ENOMEM + set ::log "" + + do_execsql_test 1.$syscall.$i.1 { + CREATE TABLE t1(a, b, UNIQUE(a, b)); + INSERT INTO t1 VALUES(randomblob(1000), randomblob(1000)); + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + } + + set nFail [test_syscall fault 0 0] + + do_execsql_test 1.$syscall.$i.2 { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } {64 ok} + + do_test 1.$syscall.$i.3 { + expr {$nFail==0 || $nFail==1} + } {1} + + do_test 1.$syscall.$i.4.nFail=$nFail { + regexp ".*${syscall}.*" $::log + } [expr $nFail>0] + } +} + +test_syscall uninstall +finish_test + diff --git a/test/speed1p.test b/test/speed1p.test index 915f165354..6bf7b10e1f 100644 --- a/test/speed1p.test +++ b/test/speed1p.test @@ -24,6 +24,8 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl speed_trial_init speed1 +sqlite3_memdebug_vfs_oom_test 0 + # Set a uniform random seed expr srand(0) @@ -78,7 +80,6 @@ do_test speed1p-1.0 { } } {i2a i2b t1 t2} - # 50000 INSERTs on an unindexed table # set list {} From 936c6d7e9e640e1269545ed9e878186a9b2b83f2 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 13:56:53 +0000 Subject: [PATCH 054/140] If the SQLITE_TRACE_SIZE_LIMIT compile-time parameter is set to a positive integer then limit the expansion of strings and blobs in trace output to approximately that many bytes. FossilOrigin-Name: e5b710849dd66673ba0e0d935b103cb29abfcc4b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbetrace.c | 40 ++++++++++++++++++++++++++++++++-------- 3 files changed, 40 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 453b33c22e..dce40a05fc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\sthe\sclean\starget,\stidy\sup\sa\scouple\smore\sgenerated\s(or\scopied)\ssource\sfiles. -D 2013-03-29T19:52:04.965 +C If\sthe\sSQLITE_TRACE_SIZE_LIMIT\scompile-time\sparameter\sis\sset\sto\sa\spositive\ninteger\sthen\slimit\sthe\sexpansion\sof\sstrings\sand\sblobs\sin\strace\soutput\sto\napproximately\sthat\smany\sbytes. +D 2013-04-02T13:56:53.795 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -247,7 +247,7 @@ F src/vdbeaux.c ecb43014bcd3019e5aa2b5561e5c3a447f007a08 F src/vdbeblob.c 32f2a4899d67f69634ea4dd93e3f651936d732cb F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c 4fad64071ae120c25f39dcac572d716b9cadeb7f -F src/vdbetrace.c 8bd5da325fc90f28464335e4cc4ad1407fe30835 +F src/vdbetrace.c a22263ab47f6ba4fcd176515cec1e732866b25f0 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6 @@ -1041,7 +1041,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 527231bc67285f01fb18d4451b28f61da3c4e39d -R c463228a20804b5f57c5669f0d0e45b8 -U mistachkin -Z bfb2a9ca80a29b00e5f7b4db495d38d4 +P 5687e5ee7bafa00d2b353c3eda1e5dfb219cb185 +R b688b887865b3d8981922c40426941a0 +U drh +Z 66f86621cb15eccea274cb5b5e71ec26 diff --git a/manifest.uuid b/manifest.uuid index 6960809f06..ec266fb546 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5687e5ee7bafa00d2b353c3eda1e5dfb219cb185 \ No newline at end of file +e5b710849dd66673ba0e0d935b103cb29abfcc4b \ No newline at end of file diff --git a/src/vdbetrace.c b/src/vdbetrace.c index 35825c8736..91554d6717 100644 --- a/src/vdbetrace.c +++ b/src/vdbetrace.c @@ -53,6 +53,11 @@ static int findNextHostParameter(const char *zSql, int *pnToken){ ** then the returned string holds a copy of zRawSql with "-- " prepended ** to each line of text. ** +** If the SQLITE_TRACE_SIZE_LIMIT macro is defined to an integer, then +** then long strings and blobs are truncated to that many bytes. This +** can be used to prevent unreasonably large trace strings when dealing +** with large (multi-megabyte) strings and blobs. +** ** The calling function is responsible for making sure the memory returned ** is eventually freed. ** @@ -123,30 +128,49 @@ char *sqlite3VdbeExpandSql( }else if( pVar->flags & MEM_Real ){ sqlite3XPrintf(&out, "%!.15g", pVar->r); }else if( pVar->flags & MEM_Str ){ + int n; /* Number of bytes of the string text to include in output */ #ifndef SQLITE_OMIT_UTF16 u8 enc = ENC(db); + Mem utf8; if( enc!=SQLITE_UTF8 ){ - Mem utf8; memset(&utf8, 0, sizeof(utf8)); utf8.db = db; sqlite3VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE_STATIC); sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8); - sqlite3XPrintf(&out, "'%.*q'", utf8.n, utf8.z); - sqlite3VdbeMemRelease(&utf8); - }else -#endif - { - sqlite3XPrintf(&out, "'%.*q'", pVar->n, pVar->z); + pVar = &utf8; } +#endif + n = pVar->n; +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( n>SQLITE_TRACE_SIZE_LIMIT ){ + n = SQLITE_TRACE_SIZE_LIMIT; + while( nn && (pVar->z[n]&0xc0)==0x80 ){ n++; } + } +#endif + sqlite3XPrintf(&out, "'%.*q'", n, pVar->z); +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nn ) sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-n); +#endif +#ifndef SQLITE_OMIT_UTF16 + if( enc!=SQLITE_UTF8 ) sqlite3VdbeMemRelease(&utf8); +#endif }else if( pVar->flags & MEM_Zero ){ sqlite3XPrintf(&out, "zeroblob(%d)", pVar->u.nZero); }else{ + int n; /* Number of bytes of the blob to include in output */ assert( pVar->flags & MEM_Blob ); sqlite3StrAccumAppend(&out, "x'", 2); - for(i=0; in; i++){ + n = pVar->n; +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( n>SQLITE_TRACE_SIZE_LIMIT ) n = SQLITE_TRACE_SIZE_LIMIT; +#endif + for(i=0; iz[i]&0xff); } sqlite3StrAccumAppend(&out, "'", 1); +#ifdef SQLITE_TRACE_SIZE_LIMIT + if( nn ) sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-n); +#endif } } } From 07fa864d15ae88e9fb999b37128b7698cbea037a Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 14:37:40 +0000 Subject: [PATCH 055/140] Fix a faulty assert() in the os_win.c VFS. FossilOrigin-Name: fd6ee54969322203c02ce9bd8744e095faf6a69c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 645d467c46..7730e2979b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\scases\sfor\serrors\sin\smmap()\sor\smremap()\sis\sos_unix.c. -D 2013-04-02T12:04:09.729 +C Fix\sa\sfaulty\sassert()\sin\sthe\sos_win.c\sVFS. +D 2013-04-02T14:37:40.848 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 42684a59b06b912b3ab80a8ad9f4295c0572a1f4 -F src/os_win.c a2f41c9dfb9b065285ba94d4a40a31ca2e2b6ea3 +F src/os_win.c 6822b2cda33f025ce7fe240ec050b9bcba213903 F src/pager.c 46fa7d8b29e7fb8baa6111690d900c9111a5a397 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1041,7 +1041,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d96272f031102514b02bd839aac9e37618872200 -R 323dfbce5d7140c5ce46f22a5112dcc8 -U dan -Z 3d1807f4fa3a8cf06a6141b170b215ce +P 3098a3c1e7305033904a496ef534cb312a876fab +R 3eb86ed3ddaa0095d1a3da7f2b0ccdb4 +U drh +Z 7afe4013f9081817d8523bb6f04fa909 diff --git a/manifest.uuid b/manifest.uuid index dfa80de2b5..c2883df6c5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3098a3c1e7305033904a496ef534cb312a876fab \ No newline at end of file +fd6ee54969322203c02ce9bd8744e095faf6a69c \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index ee7a71018e..d66faba4cb 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3601,7 +3601,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ /* Log the error, but continue normal operation using xRead/xWrite */ return SQLITE_OK; } - assert( (nNewRnd % winSysInfo.dwPageSize)==0 ); + assert( (nMap % winSysInfo.dwPageSize)==0 ); #if SQLITE_OS_WINRT pNew = osMapViewOfFileFromApp(pFd->hMap, flags, 0, nMap); #else From f6653871b218c18667a5eadd6b449c7d5bdb768d Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Apr 2013 15:37:30 +0000 Subject: [PATCH 056/140] Add new test file btreefault.test. FossilOrigin-Name: 7fe908afeba16f64edc16824c67b396138581a8f --- manifest | 13 ++++++------ manifest.uuid | 2 +- test/btreefault.test | 50 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 7 deletions(-) create mode 100644 test/btreefault.test diff --git a/manifest b/manifest index 7730e2979b..4e4e7e16f5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sfaulty\sassert()\sin\sthe\sos_win.c\sVFS. -D 2013-04-02T14:37:40.848 +C Add\snew\stest\sfile\sbtreefault.test. +D 2013-04-02T15:37:30.257 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -314,6 +314,7 @@ F test/boundary3.tcl 8901d6a503d0bf64251dd81cc74e5ad3add4b119 F test/boundary3.test 56ef82096b4329aca2be74fa1e2b0f762ea0eb45 F test/boundary4.tcl 0bb4b1a94f4fc5ae59b79b9a2b7a140c405e2983 F test/boundary4.test 89e02fa66397b8a325d5eb102b5806f961f8ec4b +F test/btreefault.test 06899a377f31a8c1a3048ec69831522d4e5c6045 F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de F test/capi2.test e8b18cc61090b6e5e388f54d6b125d711d1b265a @@ -1041,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3098a3c1e7305033904a496ef534cb312a876fab -R 3eb86ed3ddaa0095d1a3da7f2b0ccdb4 -U drh -Z 7afe4013f9081817d8523bb6f04fa909 +P fd6ee54969322203c02ce9bd8744e095faf6a69c +R afe8ecd9cb367696b6be602590bc8be6 +U dan +Z 42e6019db12bcc782c7a96e67c5fd3c7 diff --git a/manifest.uuid b/manifest.uuid index c2883df6c5..005403f1ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fd6ee54969322203c02ce9bd8744e095faf6a69c \ No newline at end of file +7fe908afeba16f64edc16824c67b396138581a8f \ No newline at end of file diff --git a/test/btreefault.test b/test/btreefault.test new file mode 100644 index 0000000000..45d326da7f --- /dev/null +++ b/test/btreefault.test @@ -0,0 +1,50 @@ +# 2013 April 02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains fault injection tests designed to test the btree.c +# module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set testprefix btreefault + +do_test 1-pre1 { + execsql { + PRAGMA auto_vacuum = incremental; + PRAGMA journal_mode = DELETE; + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES(randomblob(1000), randomblob(100)); + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + INSERT INTO t1 SELECT randomblob(1000), randomblob(1000) FROM t1; + DELETE FROM t1 WHERE rowid%2; + } + faultsim_save_and_close +} {} + +do_faultsim_test 1 -prep { + faultsim_restore_and_reopen + set ::STMT [sqlite3_prepare db "SELECT * FROM t1 ORDER BY a" -1 DUMMY] + sqlite3_step $::STMT + sqlite3_step $::STMT +} -body { + execsql { PRAGMA incremental_vacuum = 10 } +} -test { + sqlite3_finalize $::STMT + faultsim_test_result {0 {}} + faultsim_integrity_check +} + +finish_test + From a2c16c084a8a6d6654ac9a8be7bfe5d181638900 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 18:25:04 +0000 Subject: [PATCH 057/140] Fix the build for Mac. FossilOrigin-Name: d36ce0bbad58445c983597ee93438fe07b659cdc --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqliteLimit.h | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 4e4e7e16f5..0d221db3d2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\stest\sfile\sbtreefault.test. -D 2013-04-02T15:37:30.257 +C Fix\sthe\sbuild\sfor\sMac. +D 2013-04-02T18:25:04.196 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -180,7 +180,7 @@ F src/sqlite.h.in 42c69ea00dfcce84394d6ce0d8dac7bb3d2ecff2 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 1664dc5ad6f8d4dab871416628aa3271044d66c0 -F src/sqliteLimit.h 4d8d04c43bf2886d94f1ab7992fee5081e09a7f5 +F src/sqliteLimit.h edc1e61e6b745dc26ea16542838c100cef02df0e F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P fd6ee54969322203c02ce9bd8744e095faf6a69c -R afe8ecd9cb367696b6be602590bc8be6 -U dan -Z 42e6019db12bcc782c7a96e67c5fd3c7 +P 7fe908afeba16f64edc16824c67b396138581a8f +R 1e903cf5fbaea8405561162b82cab3c1 +U drh +Z 5ee32a75ebc407142eddeaec11f8a36d diff --git a/manifest.uuid b/manifest.uuid index 005403f1ad..d710366705 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7fe908afeba16f64edc16824c67b396138581a8f \ No newline at end of file +d36ce0bbad58445c983597ee93438fe07b659cdc \ No newline at end of file diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index 6adf3e8726..826d19dd9c 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -211,7 +211,7 @@ ** Default maximum size of memory used by xFetch in the VFS. */ #ifdef __APPLE__ -# include +# include # if defined(TARGET_OS_IPHONE) # define SQLITE_DEFAULT_MMAP_LIMIT 0 # endif From d409792838233ae25045e223278a2d195d47d347 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Apr 2013 18:31:29 +0000 Subject: [PATCH 058/140] Change an assert in pager.c to acknowledge that it is possible for sqlite3PagerRollback() to return SQLITE_CORRUPT. FossilOrigin-Name: d641d3d20d80fdb86d91de31fcf2eb5ca3c5c715 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pager.c | 2 +- test/wal.test | 20 ++++++++++++++++++++ 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 0d221db3d2..a0457959ef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sbuild\sfor\sMac. -D 2013-04-02T18:25:04.196 +C Change\san\sassert\sin\spager.c\sto\sacknowledge\sthat\sit\sis\spossible\sfor\ssqlite3PagerRollback()\sto\sreturn\sSQLITE_CORRUPT. +D 2013-04-02T18:31:29.703 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 42684a59b06b912b3ab80a8ad9f4295c0572a1f4 F src/os_win.c 6822b2cda33f025ce7fe240ec050b9bcba213903 -F src/pager.c 46fa7d8b29e7fb8baa6111690d900c9111a5a397 +F src/pager.c e26184d451207542d56bd09b84339252a194791e F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -953,7 +953,7 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 -F test/wal.test bbd98ac09d8fbcaa80ec233757056eb1bbb7cd95 +F test/wal.test 62454b2cf00b311e9e65f898aad0fef536ed561a F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 7fe908afeba16f64edc16824c67b396138581a8f -R 1e903cf5fbaea8405561162b82cab3c1 -U drh -Z 5ee32a75ebc407142eddeaec11f8a36d +P d36ce0bbad58445c983597ee93438fe07b659cdc +R 338634a1c64bdacb6f3ed638ff14c520 +U dan +Z 2347325502a78b990959c6fe11156215 diff --git a/manifest.uuid b/manifest.uuid index d710366705..b9069347bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d36ce0bbad58445c983597ee93438fe07b659cdc \ No newline at end of file +d641d3d20d80fdb86d91de31fcf2eb5ca3c5c715 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index cb2b39a992..f626a21d89 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6261,7 +6261,7 @@ int sqlite3PagerRollback(Pager *pPager){ } assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); - assert( rc==SQLITE_OK || rc==SQLITE_FULL + assert( rc==SQLITE_OK || rc==SQLITE_FULL || rc==SQLITE_CORRUPT || rc==SQLITE_NOMEM || (rc&0xFF)==SQLITE_IOERR ); /* If an error occurs during a ROLLBACK, we can no longer trust the pager diff --git a/test/wal.test b/test/wal.test index 0539192d25..f833b6ab1a 100644 --- a/test/wal.test +++ b/test/wal.test @@ -1569,6 +1569,26 @@ ifcapable autovacuum { } [wal_file_size 1 1024] } +reset_db +do_test 25 { + sqlite3 db test.db + + execsql { + CREATE TABLE t1(x, y); + PRAGMA journal_mode = WAL; + INSERT INTO t1 VALUES(1, 2); + } + + execsql { + BEGIN; + CREATE TABLE t2(a, b); + } + + hexio_write test.db-shm [expr 16*1024] [string repeat 0055 8192] + catchsql ROLLBACK +} {0 {}} + + db close sqlite3_shutdown test_sqlite3_log From db082408c6d68e5c8b6958806c5364bda3849a16 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Apr 2013 18:33:55 +0000 Subject: [PATCH 059/140] Add test cases to mmap1.test and pagerfault.test. FossilOrigin-Name: 3050136be962427cc4dcced6077ef29b2a941405 --- manifest | 14 +++++------ manifest.uuid | 2 +- test/mmap1.test | 25 +++++++++++++++++++ test/pagerfault.test | 57 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a0457959ef..856014e0b4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\san\sassert\sin\spager.c\sto\sacknowledge\sthat\sit\sis\spossible\sfor\ssqlite3PagerRollback()\sto\sreturn\sSQLITE_CORRUPT. -D 2013-04-02T18:31:29.703 +C Add\stest\scases\sto\smmap1.test\sand\spagerfault.test. +D 2013-04-02T18:33:55.243 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -631,7 +631,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 9fb71daaca72f66fcc7a3b3a37fc3db99988b57a +F test/mmap1.test 245eb4a6280d4b31523eb79486863afadac4f3c6 F test/mmap2.test bb0c97cdac0db06a5acdc6836eef9866815481b1 F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -653,7 +653,7 @@ F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 31c04bec797dda1bde337810b52efa08d1f1f08e F test/pager2.test 745b911dde3d1f24ae0870bd433dfa83d7c658c1 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f -F test/pagerfault.test 452f2cc23e3bfcfa935f4442aec1da4fe1dc0442 +F test/pagerfault.test fc2e37b2da626826dd54bb13720b4d721719b660 F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 F test/pageropt.test 32cb5a6ed7ccddf8e8c842cb44240bd9340223ce @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d36ce0bbad58445c983597ee93438fe07b659cdc -R 338634a1c64bdacb6f3ed638ff14c520 +P d641d3d20d80fdb86d91de31fcf2eb5ca3c5c715 +R 93a80e505d81032f015b51889ab16396 U dan -Z 2347325502a78b990959c6fe11156215 +Z 374a5f4f4b29eb3200a93246d589254e diff --git a/manifest.uuid b/manifest.uuid index b9069347bf..66226dcc40 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d641d3d20d80fdb86d91de31fcf2eb5ca3c5c715 \ No newline at end of file +3050136be962427cc4dcced6077ef29b2a941405 \ No newline at end of file diff --git a/test/mmap1.test b/test/mmap1.test index d97ada7e57..e63e09481c 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -129,4 +129,29 @@ do_execsql_test 2.4 { PRAGMA wal_checkpoint; } {0 24 24} + +reset_db +db func rblob rblob +do_execsql_test 3.1 { + PRAGMA auto_vacuum = 1; + + CREATE TABLE t1(a, b, UNIQUE(a, b)); + INSERT INTO t1 VALUES(rblob(500), rblob(500)); + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 2 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 4 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 8 + + CREATE TABLE t2(a, b, UNIQUE(a, b)); + INSERT INTO t2 SELECT * FROM t1; +} {} + +do_test 3.2 { + set nRow 0 + db eval {SELECT * FROM t2 ORDER BY a, b} { + if {$nRow==4} { db eval { DELETE FROM t1 } } + incr nRow + } + set nRow +} {8} + finish_test diff --git a/test/pagerfault.test b/test/pagerfault.test index e04e97e4fc..9ea1aa6328 100644 --- a/test/pagerfault.test +++ b/test/pagerfault.test @@ -1246,5 +1246,62 @@ do_faultsim_test pagerfault-27 -faults ioerr-persistent -prep { faultsim_integrity_check } +#------------------------------------------------------------------------- +# +do_test pagerfault-28-pre { + faultsim_delete_and_reopen + db func a_string a_string + execsql { + PRAGMA page_size = 512; + + PRAGMA journal_mode = wal; + PRAGMA wal_autocheckpoint = 0; + PRAGMA cache_size = 100000; + + BEGIN; + CREATE TABLE t2(a UNIQUE, b UNIQUE); + INSERT INTO t2 VALUES( a_string(800), a_string(800) ); + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + COMMIT; + CREATE TABLE t1(a PRIMARY KEY, b); + } + expr {[file size test.db-shm] >= 96*1024} +} {1} +faultsim_save_and_close + +do_faultsim_test pagerfault-28 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { PRAGMA mmap_limit=0 } + + sqlite3 db2 test.db + db2 eval { SELECT count(*) FROM t2 } + + db func a_string a_string + execsql { + BEGIN; + INSERT INTO t1 VALUES(a_string(2000), a_string(2000)); + INSERT INTO t1 VALUES(a_string(2000), a_string(2000)); + } + set ::STMT [sqlite3_prepare db "SELECT * FROM t1 ORDER BY a" -1 DUMMY] + sqlite3_step $::STMT +} -body { + execsql { ROLLBACK } +} -test { + db2 close + sqlite3_finalize $::STMT + catchsql { ROLLBACK } + faultsim_integrity_check +} + finish_test From a9d79ae95d54d4138356ac5801ff98b555bea939 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 2 Apr 2013 20:13:04 +0000 Subject: [PATCH 060/140] Add assert to the Windows VFS pertaining to the size of the SIZE_T type. FossilOrigin-Name: 4eeea58354e21797ca1f4dc4ea785d0330381f0b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 856014e0b4..1ca22b55cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\scases\sto\smmap1.test\sand\spagerfault.test. -D 2013-04-02T18:33:55.243 +C Add\sassert\sto\sthe\sWindows\sVFS\spertaining\sto\sthe\ssize\sof\sthe\sSIZE_T\stype. +D 2013-04-02T20:13:04.021 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 42684a59b06b912b3ab80a8ad9f4295c0572a1f4 -F src/os_win.c 6822b2cda33f025ce7fe240ec050b9bcba213903 +F src/os_win.c 6a7a1017fb6a0783a3aa7c3f5eadd9b1600a9ddf F src/pager.c e26184d451207542d56bd09b84339252a194791e F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d641d3d20d80fdb86d91de31fcf2eb5ca3c5c715 -R 93a80e505d81032f015b51889ab16396 -U dan -Z 374a5f4f4b29eb3200a93246d589254e +P 3050136be962427cc4dcced6077ef29b2a941405 +R 69136e8e9adf1c4824020b7a88c6a9a7 +U mistachkin +Z 0311bae3bb3a8c1122f90eda3126446b diff --git a/manifest.uuid b/manifest.uuid index 66226dcc40..341d71bfa9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3050136be962427cc4dcced6077ef29b2a941405 \ No newline at end of file +4eeea58354e21797ca1f4dc4ea785d0330381f0b \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index d66faba4cb..a978613e2e 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3605,6 +3605,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ #if SQLITE_OS_WINRT pNew = osMapViewOfFileFromApp(pFd->hMap, flags, 0, nMap); #else + assert( sizeof(SIZE_T)==sizeof(sqlite3_int64) || nMap<=0xffffffff ); pNew = osMapViewOfFile(pFd->hMap, flags, 0, 0, (SIZE_T)nMap); #endif if( pNew==NULL ){ From 81d1765ad837a99c2ddd9cb2001aba3213779395 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Apr 2013 20:19:22 +0000 Subject: [PATCH 061/140] Fix a bug in mmap2.test that was leaving an invalid sqlite3_log() callback installed, causing a crash. FossilOrigin-Name: c0cdaa07355f51bd217881b5b91ec2a609fa56f4 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/mmap2.test | 4 ++++ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 1ca22b55cf..aa9f6ee2cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sassert\sto\sthe\sWindows\sVFS\spertaining\sto\sthe\ssize\sof\sthe\sSIZE_T\stype. -D 2013-04-02T20:13:04.021 +C Fix\sa\sbug\sin\smmap2.test\sthat\swas\sleaving\san\sinvalid\ssqlite3_log()\scallback\sinstalled,\scausing\sa\scrash. +D 2013-04-02T20:19:22.780 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -632,7 +632,7 @@ F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 F test/mmap1.test 245eb4a6280d4b31523eb79486863afadac4f3c6 -F test/mmap2.test bb0c97cdac0db06a5acdc6836eef9866815481b1 +F test/mmap2.test c0cbb978eda8d06d755ba8d9e59ec06ebf60c5cb F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3050136be962427cc4dcced6077ef29b2a941405 -R 69136e8e9adf1c4824020b7a88c6a9a7 -U mistachkin -Z 0311bae3bb3a8c1122f90eda3126446b +P 4eeea58354e21797ca1f4dc4ea785d0330381f0b +R 6d15151e9d49c450ba09b87b5b0a03e1 +U dan +Z 90fc4c0a95395512a6e1551c81a3562f diff --git a/manifest.uuid b/manifest.uuid index 341d71bfa9..6ec6c81697 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4eeea58354e21797ca1f4dc4ea785d0330381f0b \ No newline at end of file +c0cdaa07355f51bd217881b5b91ec2a609fa56f4 \ No newline at end of file diff --git a/test/mmap2.test b/test/mmap2.test index d43ea97adf..fb36507efd 100644 --- a/test/mmap2.test +++ b/test/mmap2.test @@ -74,6 +74,10 @@ foreach syscall {mmap mremap} { } } +db close test_syscall uninstall +sqlite3_shutdown +test_sqlite3_log +sqlite3_initialize finish_test From c3d53189fca4ec61a2f697ecc23bfb16328360b0 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Apr 2013 20:29:33 +0000 Subject: [PATCH 062/140] Adjustments to mmap1.test so that it works better on windows. It still gets some answers wrong, but it no longer crashes. FossilOrigin-Name: ee8d3ceeec40d84adb8798f084b6f1215ab56d92 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/mmap1.test | 19 +++++++++++-------- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index aa9f6ee2cd..abae49480a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sin\smmap2.test\sthat\swas\sleaving\san\sinvalid\ssqlite3_log()\scallback\sinstalled,\scausing\sa\scrash. -D 2013-04-02T20:19:22.780 +C Adjustments\sto\smmap1.test\sso\sthat\sit\sworks\sbetter\son\swindows.\s\sIt\sstill\sgets\nsome\sanswers\swrong,\sbut\sit\sno\slonger\scrashes. +D 2013-04-02T20:29:33.219 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -631,7 +631,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 245eb4a6280d4b31523eb79486863afadac4f3c6 +F test/mmap1.test 95f8f90b7f4e1ef3499ea5cc69a9c3e7b1d36227 F test/mmap2.test c0cbb978eda8d06d755ba8d9e59ec06ebf60c5cb F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 4eeea58354e21797ca1f4dc4ea785d0330381f0b -R 6d15151e9d49c450ba09b87b5b0a03e1 -U dan -Z 90fc4c0a95395512a6e1551c81a3562f +P c0cdaa07355f51bd217881b5b91ec2a609fa56f4 +R 2d6cafbf0f715d3130e0ad2821f95af0 +U drh +Z eb8137777309edfff1693e4a3a7e15de diff --git a/manifest.uuid b/manifest.uuid index 6ec6c81697..6a1646888a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c0cdaa07355f51bd217881b5b91ec2a609fa56f4 \ No newline at end of file +ee8d3ceeec40d84adb8798f084b6f1215ab56d92 \ No newline at end of file diff --git a/test/mmap1.test b/test/mmap1.test index e63e09481c..f3e7268105 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -20,18 +20,20 @@ proc nRead {db} { db_enter $db array set stats [btree_pager_stats $bt] db_leave $db + # puts [array get stats] return $stats(read) } foreach {t mmap_limit nRead c2init} { - 1.1 { PRAGMA mmap_limit = 70000000 } 4 {} - 1.2 { PRAGMA mmap_limit = 51200 } 156 {} + 1.1 { PRAGMA mmap_limit = 67108864 } 4 {} + 1.2 { PRAGMA mmap_limit = 53248 } 150 {} 1.3 { PRAGMA mmap_limit = 0 } 344 {} - 1.4 { PRAGMA mmap_limit = 70000000 } 4 {PRAGMA mmap_limit = 70000000 } - 1.5 { PRAGMA mmap_limit = 51200 } 156 {PRAGMA mmap_limit = 70000000 } - 1.6 { PRAGMA mmap_limit = 0 } 344 {PRAGMA mmap_limit = 70000000 } + 1.4 { PRAGMA mmap_limit = 67108864 } 4 {PRAGMA mmap_limit = 67108864 } + 1.5 { PRAGMA mmap_limit = 53248 } 150 {PRAGMA mmap_limit = 67108864 } + 1.6 { PRAGMA mmap_limit = 0 } 344 {PRAGMA mmap_limit = 67108864 } } { do_multiclient_test tn { + sql1 {PRAGMA page_size=1024} sql1 $mmap_limit sql2 $c2init @@ -45,7 +47,8 @@ foreach {t mmap_limit nRead c2init} { db2 func rblob rblob } - sql2 { + sql2 { + PRAGMA page_size=1024; PRAGMA auto_vacuum = 1; CREATE TABLE t1(a, b, UNIQUE(a, b)); INSERT INTO t1 VALUES(rblob(500), rblob(500)); @@ -95,7 +98,7 @@ db func rblob rblob do_execsql_test 2.1 { PRAGMA auto_vacuum = 1; - PRAGMA mmap_limit = 70000000; + PRAGMA mmap_limit = 67108864; PRAGMA journal_mode = wal; CREATE TABLE t1(a, b, UNIQUE(a, b)); INSERT INTO t1 VALUES(rblob(500), rblob(500)); @@ -129,7 +132,7 @@ do_execsql_test 2.4 { PRAGMA wal_checkpoint; } {0 24 24} - +db2 close reset_db db func rblob rblob do_execsql_test 3.1 { From 48ccef80595731d2c2b6d759455837db3113f647 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Apr 2013 20:55:01 +0000 Subject: [PATCH 063/140] Fix a resource leak in os_unix.c. FossilOrigin-Name: b29cda03fe4e8d8f5b5acbbea2d69f284a2bdf23 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 6 ++---- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index abae49480a..041087f917 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjustments\sto\smmap1.test\sso\sthat\sit\sworks\sbetter\son\swindows.\s\sIt\sstill\sgets\nsome\sanswers\swrong,\sbut\sit\sno\slonger\scrashes. -D 2013-04-02T20:29:33.219 +C Fix\sa\sresource\sleak\sin\sos_unix.c. +D 2013-04-02T20:55:01.697 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 42684a59b06b912b3ab80a8ad9f4295c0572a1f4 +F src/os_unix.c e8e76c95425e60a4d0c749c1ed5d6e00109782f5 F src/os_win.c 6a7a1017fb6a0783a3aa7c3f5eadd9b1600a9ddf F src/pager.c e26184d451207542d56bd09b84339252a194791e F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c0cdaa07355f51bd217881b5b91ec2a609fa56f4 -R 2d6cafbf0f715d3130e0ad2821f95af0 -U drh -Z eb8137777309edfff1693e4a3a7e15de +P ee8d3ceeec40d84adb8798f084b6f1215ab56d92 +R 40861d78d26ec4eddf163e4d116b8b4f +U dan +Z 36ae50dfb6860cc1a436ea46d4e8b8f3 diff --git a/manifest.uuid b/manifest.uuid index 6a1646888a..cb24cad099 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ee8d3ceeec40d84adb8798f084b6f1215ab56d92 \ No newline at end of file +b29cda03fe4e8d8f5b5acbbea2d69f284a2bdf23 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 07c919c983..581afdbc91 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4601,10 +4601,8 @@ static void unixRemapfile( } #endif - /* The attempt to extend the existing mapping failed. Free the existing - ** mapping and set pNew to NULL so that the code below will create a - ** new mapping from scratch. */ - if( pNew==MAP_FAILED ){ + /* The attempt to extend the existing mapping failed. Free it. */ + if( pNew==MAP_FAILED || pNew==0 ){ osMunmap(pOrig, nReuse); } } From 85830a70c2d07e345564211e13e7f5e6182f4135 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 00:42:01 +0000 Subject: [PATCH 064/140] Fix the unix driver to check defined(_BSD_SOURCE) rather than just the plain _BSD_SOURCE macro. This fixes the build for OpenBSD. FossilOrigin-Name: 1dd42ef4144ee08fb4ee1676d934a56a0e34bac2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 041087f917..f907c58839 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sresource\sleak\sin\sos_unix.c. -D 2013-04-02T20:55:01.697 +C Fix\sthe\sunix\sdriver\sto\scheck\sdefined(_BSD_SOURCE)\srather\sthan\sjust\sthe\splain\n_BSD_SOURCE\smacro.\s\sThis\sfixes\sthe\sbuild\sfor\sOpenBSD. +D 2013-04-03T00:42:01.544 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,7 +160,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c e8e76c95425e60a4d0c749c1ed5d6e00109782f5 +F src/os_unix.c f7d8c6ef5f45111f62a77984053c17e767f9679e F src/os_win.c 6a7a1017fb6a0783a3aa7c3f5eadd9b1600a9ddf F src/pager.c e26184d451207542d56bd09b84339252a194791e F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ee8d3ceeec40d84adb8798f084b6f1215ab56d92 -R 40861d78d26ec4eddf163e4d116b8b4f -U dan -Z 36ae50dfb6860cc1a436ea46d4e8b8f3 +P b29cda03fe4e8d8f5b5acbbea2d69f284a2bdf23 +R 12d119a8bfdd7260b7f76958004bb415 +U drh +Z 8724856398770e48d9cea9c9c7fa6257 diff --git a/manifest.uuid b/manifest.uuid index cb24cad099..4593a953a2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b29cda03fe4e8d8f5b5acbbea2d69f284a2bdf23 \ No newline at end of file +1dd42ef4144ee08fb4ee1676d934a56a0e34bac2 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 581afdbc91..6ca99fa532 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4534,7 +4534,7 @@ static void unixUnmapfile(unixFile *pFd){ static int unixGetPagesize(void){ #if HAVE_MREMAP return 512; -#elif _BSD_SOURCE +#elif defined(_BSD_SOURCE) return getpagesize(); #else return (int)sysconf(_SC_PAGESIZE); From 7d9f3943ba060edd2389b4b2c27db0d02524f11d Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 01:26:54 +0000 Subject: [PATCH 065/140] Add the -mmap option to the command-line shell, for setting the default mmap_limit. FossilOrigin-Name: fc30d06c94c8212abb0477fb4cec4520d05bea34 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 53 +++++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 52 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index f907c58839..0b74d42186 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sunix\sdriver\sto\scheck\sdefined(_BSD_SOURCE)\srather\sthan\sjust\sthe\splain\n_BSD_SOURCE\smacro.\s\sThis\sfixes\sthe\sbuild\sfor\sOpenBSD. -D 2013-04-03T00:42:01.544 +C Add\sthe\s-mmap\soption\sto\sthe\scommand-line\sshell,\sfor\ssetting\sthe\sdefault\nmmap_limit. +D 2013-04-03T01:26:54.789 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -175,7 +175,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 -F src/shell.c 7c41bfcd9e5bf9d96b9215f79b03a5b2b44a3bca +F src/shell.c 319b7791cee6c763b60fde1b590bfaf62613cf37 F src/sqlite.h.in 42c69ea00dfcce84394d6ce0d8dac7bb3d2ecff2 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b29cda03fe4e8d8f5b5acbbea2d69f284a2bdf23 -R 12d119a8bfdd7260b7f76958004bb415 +P 1dd42ef4144ee08fb4ee1676d934a56a0e34bac2 +R 414201a51e65d2b5d08b22317803af41 U drh -Z 8724856398770e48d9cea9c9c7fa6257 +Z ee0a75b1373c6c61317486123f3b4064 diff --git a/manifest.uuid b/manifest.uuid index 4593a953a2..ac22576f6d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1dd42ef4144ee08fb4ee1676d934a56a0e34bac2 \ No newline at end of file +fc30d06c94c8212abb0477fb4cec4520d05bea34 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index c6d7fa3d53..6a0e047475 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1552,6 +1552,43 @@ static int booleanValue(char *zArg){ return 0; } +/* +** Interpret zArg as an integer value, possibly with suffixes. +*/ +static sqlite3_int64 integerValue(const char *zArg){ + sqlite3_int64 v = 0; + static const struct { char *zSuffix; int iMult; } aMult[] = { + { "KiB", 1024 }, + { "MiB", 1024*1024 }, + { "GiB", 1024*1024*1024 }, + { "KB", 1000 }, + { "MB", 1000000 }, + { "GB", 1000000000 }, + { "K", 1000 }, + { "M", 1000000 }, + { "G", 1000000000 }, + }; + int i; + int isNeg = 0; + if( zArg[0]=='-' ){ + isNeg = 1; + zArg++; + }else if( zArg[0]=='+' ){ + zArg++; + } + while( isdigit(zArg[0]) ){ + v = v*10 + zArg[0] - '0'; + zArg++; + } + for(i=0; i0x7fff0000 ) szHeap = 0x7fff0000; sqlite3_config(SQLITE_CONFIG_HEAP, malloc((int)szHeap), (int)szHeap, 64); #endif @@ -3026,6 +3059,8 @@ int main(int argc, char **argv){ extern int sqlite3_multiple_initialize(const char*,int); sqlite3_multiplex_initialize(0, 1); #endif + }else if( strcmp(z,"-mmap")==0 ){ + sqlite3_config(SQLITE_CONFIG_MMAP_LIMIT, integerValue(cmdline_option_value(argc,argv,++i))); }else if( strcmp(z,"-vfs")==0 ){ sqlite3_vfs *pVfs = sqlite3_vfs_find(cmdline_option_value(argc,argv,++i)); if( pVfs ){ @@ -3111,6 +3146,8 @@ int main(int argc, char **argv){ stdin_is_interactive = 0; }else if( strcmp(z,"-heap")==0 ){ i++; + }else if( strcmp(z,"-mmap")==0 ){ + i++; }else if( strcmp(z,"-vfs")==0 ){ i++; #ifdef SQLITE_ENABLE_VFSTRACE From 7e36962e6c152768483b9282149ff975359d7ed4 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 03:53:15 +0000 Subject: [PATCH 066/140] Make use of xFetch, if it is available when doing a VACUUM or a backup. FossilOrigin-Name: 83bc37af07857960c11275891f853a358dcbbf05 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/backup.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 0b74d42186..cbde44af45 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s-mmap\soption\sto\sthe\scommand-line\sshell,\sfor\ssetting\sthe\sdefault\nmmap_limit. -D 2013-04-03T01:26:54.789 +C Make\suse\sof\sxFetch,\sif\sit\sis\savailable\swhen\sdoing\sa\sVACUUM\sor\sa\sbackup. +D 2013-04-03T03:53:15.647 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -118,7 +118,7 @@ F src/alter.c f8db986c03eb0bfb221523fc9bbb9d0b70de3168 F src/analyze.c d5f895810e8ff9737c9ec7b76abc3dcff5860335 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 -F src/backup.c b2cac9f7993f3f9588827b824b1501d0c820fa68 +F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c bb679fddb7269dd7cfb950f845b1b18f19df4f4a @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 1dd42ef4144ee08fb4ee1676d934a56a0e34bac2 -R 414201a51e65d2b5d08b22317803af41 +P fc30d06c94c8212abb0477fb4cec4520d05bea34 +R 8f21f6233365f9239c040421dd942d16 U drh -Z ee0a75b1373c6c61317486123f3b4064 +Z fab31801d3b721c34a1407901bd055e3 diff --git a/manifest.uuid b/manifest.uuid index ac22576f6d..847cb431dc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fc30d06c94c8212abb0477fb4cec4520d05bea34 \ No newline at end of file +83bc37af07857960c11275891f853a358dcbbf05 \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 71a8a1a3e7..252f61cfca 100644 --- a/src/backup.c +++ b/src/backup.c @@ -397,7 +397,8 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ const Pgno iSrcPg = p->iNext; /* Source page number */ if( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ){ DbPage *pSrcPg; /* Source page object */ - rc = sqlite3PagerGet(pSrcPager, iSrcPg, &pSrcPg); + rc = sqlite3PagerAcquire(pSrcPager, iSrcPg, &pSrcPg, + PAGER_ACQUIRE_READONLY); if( rc==SQLITE_OK ){ rc = backupOnePage(p, iSrcPg, sqlite3PagerGetData(pSrcPg), 0); sqlite3PagerUnref(pSrcPg); From 2b8246e3e23cb98b6d7339de019d446aa3bc3bb8 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 10:50:02 +0000 Subject: [PATCH 067/140] Initialize the mmap_limit of temporary files to the configured mmap_limit. FossilOrigin-Name: 24bab7596bb7385981a5d331df5eeb05353547f7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 2 +- src/os_win.c | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index cbde44af45..78b0bffaa7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\suse\sof\sxFetch,\sif\sit\sis\savailable\swhen\sdoing\sa\sVACUUM\sor\sa\sbackup. -D 2013-04-03T03:53:15.647 +C Initialize\sthe\smmap_limit\sof\stemporary\sfiles\sto\sthe\sconfigured\smmap_limit. +D 2013-04-03T10:50:02.049 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -160,8 +160,8 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c f7d8c6ef5f45111f62a77984053c17e767f9679e -F src/os_win.c 6a7a1017fb6a0783a3aa7c3f5eadd9b1600a9ddf +F src/os_unix.c 626ddccfd5b1be0c91ca5a9d147c8c3e824d7013 +F src/os_win.c 6718b053fe8295ede770b7a1e48ba4ef73310e9c F src/pager.c e26184d451207542d56bd09b84339252a194791e F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P fc30d06c94c8212abb0477fb4cec4520d05bea34 -R 8f21f6233365f9239c040421dd942d16 +P 83bc37af07857960c11275891f853a358dcbbf05 +R 32aa00c0c8e64a488f9b37d8aa36924f U drh -Z fab31801d3b721c34a1407901bd055e3 +Z db8bee74489339af40a69aa3740a7e67 diff --git a/manifest.uuid b/manifest.uuid index 847cb431dc..56b76309a3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83bc37af07857960c11275891f853a358dcbbf05 \ No newline at end of file +24bab7596bb7385981a5d331df5eeb05353547f7 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 6ca99fa532..39a31cb188 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5060,7 +5060,7 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; - pNew->mmapLimit = SQLITE_DEFAULT_MMAP_LIMIT; + pNew->mmapLimit = sqlite3GlobalConfig.mxMmap; if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; diff --git a/src/os_win.c b/src/os_win.c index a978613e2e..74aede5704 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -4116,7 +4116,7 @@ static int winOpen( pFile->pMapRegion = 0; pFile->mmapSize = 0; pFile->mmapOrigsize = 0; - pFile->mmapLimit = SQLITE_DEFAULT_MMAP_LIMIT; + pFile->mmapLimit = sqlite3GlobalConfig.mxMmap; OpenCounter(+1); return rc; From 227a1c482cb2e725712f5a5f75fa0fd977a586ef Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 3 Apr 2013 11:17:39 +0000 Subject: [PATCH 068/140] In btree.c, save the positions of other cursors open on the same table when writing via an incremental blob handle. Otherwise, they may be left holding an out-of-date xFetch page reference. FossilOrigin-Name: 3f09fba18f7e61e21381ffea13c31b968efd7d77 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 9 +++++++++ test/mmap1.test | 51 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 78b0bffaa7..a260417387 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Initialize\sthe\smmap_limit\sof\stemporary\sfiles\sto\sthe\sconfigured\smmap_limit. -D 2013-04-03T10:50:02.049 +C In\sbtree.c,\ssave\sthe\spositions\sof\sother\scursors\sopen\son\sthe\ssame\stable\swhen\swriting\svia\san\sincremental\sblob\shandle.\sOtherwise,\sthey\smay\sbe\sleft\sholding\san\sout-of-date\sxFetch\spage\sreference. +D 2013-04-03T11:17:39.067 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c bb679fddb7269dd7cfb950f845b1b18f19df4f4a +F src/btree.c b38cb3a1758d46e76bec080994fed7313622687d F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -631,7 +631,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 95f8f90b7f4e1ef3499ea5cc69a9c3e7b1d36227 +F test/mmap1.test 9f4cee0877549c80b13acd688c775254b2d61ec7 F test/mmap2.test c0cbb978eda8d06d755ba8d9e59ec06ebf60c5cb F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 83bc37af07857960c11275891f853a358dcbbf05 -R 32aa00c0c8e64a488f9b37d8aa36924f -U drh -Z db8bee74489339af40a69aa3740a7e67 +P 24bab7596bb7385981a5d331df5eeb05353547f7 +R 5d2a80f93297e2bae93eb7d43d59c6c0 +U dan +Z a5b2fa9ebb6e4e86d546b82b43bf4939 diff --git a/manifest.uuid b/manifest.uuid index 56b76309a3..d1325fe2f5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -24bab7596bb7385981a5d331df5eeb05353547f7 \ No newline at end of file +3f09fba18f7e61e21381ffea13c31b968efd7d77 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9672e0e000..a1877ebfe8 100644 --- a/src/btree.c +++ b/src/btree.c @@ -8400,6 +8400,15 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ return SQLITE_ABORT; } + /* Save the positions of all other cursors open on this table. This is + ** required in case any of them are holding references to an xFetch + ** version of the b-tree page modified by the accessPayload call below. + */ + rc = saveAllCursors(pCsr->pBt, pCsr->pgnoRoot, pCsr); + if( rc!=SQLITE_OK ){ + return SQLITE_OK; + } + /* Check some assumptions: ** (a) the cursor is open for writing, ** (b) there is a read/write transaction open, diff --git a/test/mmap1.test b/test/mmap1.test index f3e7268105..802b921048 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -157,4 +157,55 @@ do_test 3.2 { set nRow } {8} +#------------------------------------------------------------------------- +# Ensure that existing cursors using xFetch() pages see changes made +# to rows using the incrblob API. +# +reset_db +set aaa [string repeat a 400] +set bbb [string repeat b 400] +set ccc [string repeat c 400] +set ddd [string repeat d 400] +set eee [string repeat e 400] + +do_execsql_test 4.1 { + PRAGMA page_size = 1024; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES($aaa); + INSERT INTO t1 VALUES($bbb); + INSERT INTO t1 VALUES($ccc); + INSERT INTO t1 VALUES($ddd); + SELECT * FROM t1; + BEGIN; +} [list $aaa $bbb $ccc $ddd] + +do_test 4.2 { + set ::STMT [sqlite3_prepare db "SELECT * FROM t1 ORDER BY rowid" -1 dummy] + sqlite3_step $::STMT + sqlite3_column_text $::STMT 0 +} $aaa + +do_test 4.3 { + foreach r {2 3 4} { + set fd [db incrblob t1 x $r] + puts -nonewline $fd $eee + close $fd + } + + set res [list] + while {"SQLITE_ROW" == [sqlite3_step $::STMT]} { + lappend res [sqlite3_column_text $::STMT 0] + } + set res +} [list $eee $eee $eee] + +do_test 4.4 { + sqlite3_finalize $::STMT +} SQLITE_OK + +do_execsql_test 4.5 { COMMIT } + finish_test + + + From f7679ad19eaaf249bd803010eab2da14f21c61e9 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 3 Apr 2013 11:38:36 +0000 Subject: [PATCH 069/140] When moving a page to make way for the root page of a new table or index in an auto-vacuum database, save the positions of any cursors that may be holding xFetch references to the page being moved. FossilOrigin-Name: 9d9b1da54a555e8fb6037d63d1952458c12956d2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 8 ++++++++ test/mmap1.test | 41 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index a260417387..cf7c875169 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sbtree.c,\ssave\sthe\spositions\sof\sother\scursors\sopen\son\sthe\ssame\stable\swhen\swriting\svia\san\sincremental\sblob\shandle.\sOtherwise,\sthey\smay\sbe\sleft\sholding\san\sout-of-date\sxFetch\spage\sreference. -D 2013-04-03T11:17:39.067 +C When\smoving\sa\spage\sto\smake\sway\sfor\sthe\sroot\spage\sof\sa\snew\stable\sor\sindex\sin\san\sauto-vacuum\sdatabase,\ssave\sthe\spositions\sof\sany\scursors\sthat\smay\sbe\sholding\sxFetch\sreferences\sto\sthe\spage\sbeing\smoved. +D 2013-04-03T11:38:36.388 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c b38cb3a1758d46e76bec080994fed7313622687d +F src/btree.c 4f4b23941b38f68db425f68960f26031204d9807 F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -631,7 +631,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 9f4cee0877549c80b13acd688c775254b2d61ec7 +F test/mmap1.test 5a662d506c367b082b24cb9186acb11efa6cbe3d F test/mmap2.test c0cbb978eda8d06d755ba8d9e59ec06ebf60c5cb F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 24bab7596bb7385981a5d331df5eeb05353547f7 -R 5d2a80f93297e2bae93eb7d43d59c6c0 +P 3f09fba18f7e61e21381ffea13c31b968efd7d77 +R 7ec6dcb52e20c580b46b605bb884042f U dan -Z a5b2fa9ebb6e4e86d546b82b43bf4939 +Z c27841a9d742e32f5c92b396fb592137 diff --git a/manifest.uuid b/manifest.uuid index d1325fe2f5..ab30a2773c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3f09fba18f7e61e21381ffea13c31b968efd7d77 \ No newline at end of file +9d9b1da54a555e8fb6037d63d1952458c12956d2 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index a1877ebfe8..dd37112c39 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7269,6 +7269,14 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ u8 eType = 0; Pgno iPtrPage = 0; + /* Save the positions of any open cursors. This is required in + ** case they are holding a reference to an xFetch reference + ** corresponding to page pgnoRoot. */ + rc = saveAllCursors(pBt, 0, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + releasePage(pPageMove); /* Move the page currently at pgnoRoot to pgnoMove. */ diff --git a/test/mmap1.test b/test/mmap1.test index 802b921048..d9babe3d86 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -205,7 +205,46 @@ do_test 4.4 { do_execsql_test 4.5 { COMMIT } +#------------------------------------------------------------------------- +# Ensure that existing cursors holding xFetch() references are not +# confused if those pages are moved to make way for the root page of a +# new table or index. +# +reset_db +do_execsql_test 5.1 { + PRAGMA auto_vacuum = 2; + PRAGMA page_size = 1024; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES($aaa); + INSERT INTO t1 VALUES($bbb); + INSERT INTO t1 VALUES($ccc); + INSERT INTO t1 VALUES($ddd); + + PRAGMA auto_vacuum; + SELECT * FROM t1; +} [list 2 $aaa $bbb $ccc $ddd] + +do_test 5.2 { + set ::STMT [sqlite3_prepare db "SELECT * FROM t1 ORDER BY rowid" -1 dummy] + sqlite3_step $::STMT + sqlite3_column_text $::STMT 0 +} $aaa + +do_execsql_test 5.3 { + CREATE TABLE t2(x); + INSERT INTO t2 VALUES('tricked you!'); + INSERT INTO t2 VALUES('tricked you!'); +} + +do_test 5.4 { + sqlite3_step $::STMT + sqlite3_column_text $::STMT 0 +} $bbb + +do_test 5.5 { + sqlite3_finalize $::STMT +} SQLITE_OK + finish_test - From 6c96946475fd5b61168ba6b7e43b33e46fee2a9f Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 3 Apr 2013 11:52:16 +0000 Subject: [PATCH 070/140] Remove the restriction on using xFetch to load the root pages of tables and indexes. It appears to have been based on a misconception. FossilOrigin-Name: 5b082efead488a2fccc18171e640e0aa5252d1d0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 33 +-------------------------------- 3 files changed, 8 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index cf7c875169..5aaf2a1f4f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\smoving\sa\spage\sto\smake\sway\sfor\sthe\sroot\spage\sof\sa\snew\stable\sor\sindex\sin\san\sauto-vacuum\sdatabase,\ssave\sthe\spositions\sof\sany\scursors\sthat\smay\sbe\sholding\sxFetch\sreferences\sto\sthe\spage\sbeing\smoved. -D 2013-04-03T11:38:36.388 +C Remove\sthe\srestriction\son\susing\sxFetch\sto\sload\sthe\sroot\spages\sof\stables\sand\sindexes.\sIt\sappears\sto\shave\sbeen\sbased\son\sa\smisconception. +D 2013-04-03T11:52:16.489 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 4f4b23941b38f68db425f68960f26031204d9807 +F src/btree.c fd492d65eee0eb35f4000ddf478215c99dca6838 F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3f09fba18f7e61e21381ffea13c31b968efd7d77 -R 7ec6dcb52e20c580b46b605bb884042f +P 9d9b1da54a555e8fb6037d63d1952458c12956d2 +R a9b149f10f100efb054372d5cd9076ff U dan -Z c27841a9d742e32f5c92b396fb592137 +Z cc4101c000adda873dbba809b24aeeeb diff --git a/manifest.uuid b/manifest.uuid index ab30a2773c..a21ce49234 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9d9b1da54a555e8fb6037d63d1952458c12956d2 \ No newline at end of file +5b082efead488a2fccc18171e640e0aa5252d1d0 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index dd37112c39..42c4ee3cc0 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2583,34 +2583,6 @@ int sqlite3BtreeNewDb(Btree *p){ return rc; } -/* -** Ensure that any root page references held by open cursors are not -** mmap pages. -*/ -static int btreeSwapOutMmap(BtShared *pBt){ - int rc = SQLITE_OK; /* Return code */ - BtCursor *pCsr; /* Used to iterate through all open cursors */ - - for(pCsr=pBt->pCursor; pCsr && rc==SQLITE_OK; pCsr=pCsr->pNext){ - if( pCsr->iPage>=0 ){ - MemPage *pPg = pCsr->apPage[0]; - if( pPg && pPg->pDbPage->flags & PGHDR_MMAP ){ - MemPage *pNew = 0; - rc = getAndInitPage(pBt, pPg->pgno, &pNew, 0); - if( rc==SQLITE_OK ){ - if( pCsr->iPage==0 ){ - pCsr->info.pCell = pNew->aData + (pCsr->info.pCell - pPg->aData); - } - pCsr->apPage[0] = pNew; - releasePage(pPg); - } - } - } - } - - return rc; -} - /* ** Attempt to start a new transaction. A write-transaction ** is started if the second argument is nonzero, otherwise a read- @@ -2717,9 +2689,6 @@ int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ rc = SQLITE_READONLY; }else{ rc = sqlite3PagerBegin(pBt->pPager,wrflag>1,sqlite3TempInMemory(p->db)); - if( rc==SQLITE_OK ){ - rc = btreeSwapOutMmap(pBt); - } if( rc==SQLITE_OK ){ rc = newDatabase(pBt); } @@ -4419,7 +4388,7 @@ static int moveToRoot(BtCursor *pCur){ pCur->eState = CURSOR_INVALID; return SQLITE_OK; }else{ - rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0], 0); + rc = getAndInitPage(pBt, pCur->pgnoRoot, &pCur->apPage[0], pCur->wrFlag==0); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; From 34f7490311f77d0da8c90260d083f19550fd6c61 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 13:09:18 +0000 Subject: [PATCH 071/140] Change the mmap_limit pragma to report the new limit, or to report the existing limit if called with no arguments. Report the default mmap_limit as part of PRAGMA compile_options. Set the default mmmap_limit to 0 for all systems other than linux, mac, windows, and solaris. FossilOrigin-Name: 2d9f1327fe79e40435ce1e2594d7cd9a5aea0ef2 --- manifest | 28 ++++++++++++++-------------- manifest.uuid | 2 +- src/ctime.c | 3 +++ src/os_unix.c | 4 +++- src/os_win.c | 4 +++- src/pager.c | 5 +++-- src/pragma.c | 14 +++++++++----- src/sqlite.h.in | 4 +++- src/sqliteLimit.h | 10 +++------- test/mmap1.test | 2 +- 10 files changed, 43 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 5aaf2a1f4f..a97a1a1a46 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\srestriction\son\susing\sxFetch\sto\sload\sthe\sroot\spages\sof\stables\sand\sindexes.\sIt\sappears\sto\shave\sbeen\sbased\son\sa\smisconception. -D 2013-04-03T11:52:16.489 +C Change\sthe\smmap_limit\spragma\sto\sreport\sthe\snew\slimit,\sor\sto\sreport\sthe\nexisting\slimit\sif\scalled\swith\sno\sarguments.\s\sReport\sthe\sdefault\smmap_limit\nas\spart\sof\sPRAGMA\scompile_options.\s\sSet\sthe\sdefault\smmmap_limit\sto\s0\sfor\nall\ssystems\sother\sthan\slinux,\smac,\swindows,\sand\ssolaris. +D 2013-04-03T13:09:18.208 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -127,7 +127,7 @@ F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c 2a5f251fcd7393808df77ccfc817e7058df08c4c +F src/ctime.c 16658a257bc6a3ca8d8961f574cf61a57e4d6faf F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 @@ -160,15 +160,15 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 626ddccfd5b1be0c91ca5a9d147c8c3e824d7013 -F src/os_win.c 6718b053fe8295ede770b7a1e48ba4ef73310e9c -F src/pager.c e26184d451207542d56bd09b84339252a194791e +F src/os_unix.c f0ecce40d92469d5cc737ae883e776eb3e5c0af5 +F src/os_win.c 1d0ccc1880b626ac2b6868284f6338c90687e8d8 +F src/pager.c 2e68df46d4086027cb6b527d47a6dedbf1a6b7ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c d0231f412dea33c35c1309b45b535cf0c10c63c8 +F src/pragma.c 682e97f3e3b77fd6c9b569eabfbf4a14c987aca3 F src/prepare.c 310eaff1ee5f3c700b3545afb095cfe9346efc3a F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -176,11 +176,11 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c 319b7791cee6c763b60fde1b590bfaf62613cf37 -F src/sqlite.h.in 42c69ea00dfcce84394d6ce0d8dac7bb3d2ecff2 +F src/sqlite.h.in faeb6b3470193e599d79289f77f984b8a78136e1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 1664dc5ad6f8d4dab871416628aa3271044d66c0 -F src/sqliteLimit.h edc1e61e6b745dc26ea16542838c100cef02df0e +F src/sqliteLimit.h 299a15148feccc4b85db3b9f3c652ee5b60267b4 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 @@ -631,7 +631,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 5a662d506c367b082b24cb9186acb11efa6cbe3d +F test/mmap1.test df5105f08e6000e57b4de7e748f8c2ae3fed75da F test/mmap2.test c0cbb978eda8d06d755ba8d9e59ec06ebf60c5cb F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9d9b1da54a555e8fb6037d63d1952458c12956d2 -R a9b149f10f100efb054372d5cd9076ff -U dan -Z cc4101c000adda873dbba809b24aeeeb +P 5b082efead488a2fccc18171e640e0aa5252d1d0 +R 779b4d1891b69fa3c237308161c44940 +U drh +Z c380833e5cb3768ffb9f1e28bd39c0ad diff --git a/manifest.uuid b/manifest.uuid index a21ce49234..a842cd26d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5b082efead488a2fccc18171e640e0aa5252d1d0 \ No newline at end of file +2d9f1327fe79e40435ce1e2594d7cd9a5aea0ef2 \ No newline at end of file diff --git a/src/ctime.c b/src/ctime.c index c42454ca71..ee6d4cbe2e 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -57,6 +57,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_DEFAULT_LOCKING_MODE "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), #endif +#ifdef SQLITE_DEFAULT_MMAP_LIMIT + "DEFAULT_MMAP_LIMIT=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_LIMIT), +#endif #ifdef SQLITE_DISABLE_DIRSYNC "DISABLE_DIRSYNC", #endif diff --git a/src/os_unix.c b/src/os_unix.c index 39a31cb188..75f9236013 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3710,7 +3710,9 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ return SQLITE_OK; } case SQLITE_FCNTL_MMAP_LIMIT: { - pFile->mmapLimit = *(i64*)pArg; + i64 newLimit = *(i64*)pArg; + *(i64*)pArg = pFile->mmapLimit; + if( newLimit>=0 ) pFile->mmapLimit = newLimit; return SQLITE_OK; } #ifdef SQLITE_DEBUG diff --git a/src/os_win.c b/src/os_win.c index 74aede5704..dc69b47ae2 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2837,7 +2837,9 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ return SQLITE_OK; } case SQLITE_FCNTL_MMAP_LIMIT: { - pFile->mmapLimit = *(i64*)pArg; + i64 newLimit = *(i64*)pArg; + *(i64*) = pFile->mmapLimit; + if( newLimit>=0 ) pFile->mmapLimit = newLimit; return SQLITE_OK; } } diff --git a/src/pager.c b/src/pager.c index f626a21d89..ae2f10069f 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3358,9 +3358,10 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ static void pagerFixMaplimit(Pager *pPager){ sqlite3_file *fd = pPager->fd; if( isOpen(fd) ){ + sqlite3_int64 mx; pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->mxMmap>0; - sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT, - (void*)&pPager->mxMmap); + mx = pPager->mxMmap; + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT, &mx); } } diff --git a/src/pragma.c b/src/pragma.c index 7a515cba2e..3527c0974c 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -759,19 +759,23 @@ void sqlite3Pragma( ** upper layers will never invoke the xFetch interfaces to the VFS. */ if( sqlite3StrICmp(zLeft,"mmap_limit")==0 ){ + sqlite3_int64 mx; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( zRight ){ - sqlite3_int64 size; int ii; - sqlite3Atoi64(zRight, &size, 1000, SQLITE_UTF8); - if( size<0 ) size = sqlite3GlobalConfig.mxMmap; - if( pId2->n==0 ) db->mxMmap = size; + sqlite3Atoi64(zRight, &mx, 1000, SQLITE_UTF8); + if( mx<0 ) mx = sqlite3GlobalConfig.mxMmap; + if( pId2->n==0 ) db->mxMmap = mx; for(ii=db->nDb-1; ii>=0; ii--){ if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ - sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, size); + sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, mx); } } } + mx = -1; + if( sqlite3_file_control(db,zDb,SQLITE_FCNTL_MMAP_LIMIT,&mx)==SQLITE_OK ){ + returnSingleInt(pParse, "mmap_limit", mx); + } }else /* diff --git a/src/sqlite.h.in b/src/sqlite.h.in index de1316606b..a4fb1a8a9a 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -888,7 +888,9 @@ struct sqlite3_io_methods { ** **
  • [[SQLITE_FCNTL_MMAP_LIMIT]] ** The argument is assumed to pointer to a value of type sqlite3_int64 that -** is an advisory maximum number of bytes in the file to memory map. +** is an advisory maximum number of bytes in the file to memory map. The +** pointer is overwritten with the old value. The limit is not changed if +** the original value pointed to is negative. ** ** */ diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index 826d19dd9c..2b02cc5662 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -219,13 +219,9 @@ #ifndef SQLITE_DEFAULT_MMAP_LIMIT # if defined(__linux__) \ || defined(_WIN32) \ - || (defined(__APPLE__) && defined(__MACH__) && !defined(TARGET_OS_IPHONE)) \ - || defined(__sun) \ - || defined(__DragonFly__) \ - || defined(__FreeBSD__) \ - || defined(__NetBSD__) \ - || defined(__OpenBSD__) -# define SQLITE_DEFAULT_MMAP_LIMIT (256*1024*1024) + || (defined(__APPLE__) && defined(__MACH__)) \ + || defined(__sun) +# define SQLITE_DEFAULT_MMAP_LIMIT 268435456 /* = 256*1024*1024 */ # else # define SQLITE_DEFAULT_MMAP_LIMIT 0 # endif diff --git a/test/mmap1.test b/test/mmap1.test index d9babe3d86..805fb76525 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -108,7 +108,7 @@ do_execsql_test 2.1 { INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 32 PRAGMA wal_checkpoint; -} {wal 0 103 103} +} {67108864 wal 0 103 103} do_execsql_test 2.2 { PRAGMA auto_vacuum; From 716d6d2da760f51b7c22e3091bfa2a35fccdd005 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 13:38:46 +0000 Subject: [PATCH 072/140] Fix a bug in the detection of iOS when setting the default mmap_limit. FossilOrigin-Name: ac1432b3a8e968be4cbc138b4a35f34187c0c36f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqliteLimit.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a97a1a1a46..d40b31d70b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\smmap_limit\spragma\sto\sreport\sthe\snew\slimit,\sor\sto\sreport\sthe\nexisting\slimit\sif\scalled\swith\sno\sarguments.\s\sReport\sthe\sdefault\smmap_limit\nas\spart\sof\sPRAGMA\scompile_options.\s\sSet\sthe\sdefault\smmmap_limit\sto\s0\sfor\nall\ssystems\sother\sthan\slinux,\smac,\swindows,\sand\ssolaris. -D 2013-04-03T13:09:18.208 +C Fix\sa\sbug\sin\sthe\sdetection\sof\siOS\swhen\ssetting\sthe\sdefault\smmap_limit. +D 2013-04-03T13:38:46.133 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -180,7 +180,7 @@ F src/sqlite.h.in faeb6b3470193e599d79289f77f984b8a78136e1 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 1664dc5ad6f8d4dab871416628aa3271044d66c0 -F src/sqliteLimit.h 299a15148feccc4b85db3b9f3c652ee5b60267b4 +F src/sqliteLimit.h 1097d2c541147d303e66515040067c6bfcf8cf21 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5b082efead488a2fccc18171e640e0aa5252d1d0 -R 779b4d1891b69fa3c237308161c44940 +P 2d9f1327fe79e40435ce1e2594d7cd9a5aea0ef2 +R e2d5c255f4ba7e0fb00c083dac088003 U drh -Z c380833e5cb3768ffb9f1e28bd39c0ad +Z 519f2edad4b962613f9e09a344e900e8 diff --git a/manifest.uuid b/manifest.uuid index a842cd26d5..7c7984449a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2d9f1327fe79e40435ce1e2594d7cd9a5aea0ef2 \ No newline at end of file +ac1432b3a8e968be4cbc138b4a35f34187c0c36f \ No newline at end of file diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index 2b02cc5662..d3a3b4ac95 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -212,7 +212,7 @@ */ #ifdef __APPLE__ # include -# if defined(TARGET_OS_IPHONE) +# if TARGET_OS_IPHONE # define SQLITE_DEFAULT_MMAP_LIMIT 0 # endif #endif From 370c9f498dbb223c05197e532668a6ba5595f78e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 20:04:04 +0000 Subject: [PATCH 073/140] Remove an unreachable branch in the direct blob I/O logic of btree.c. FossilOrigin-Name: f97d7274f48e3bb98ed17eae11abb55064134874 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 10 ++++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index d40b31d70b..e986a3818e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sin\sthe\sdetection\sof\siOS\swhen\ssetting\sthe\sdefault\smmap_limit. -D 2013-04-03T13:38:46.133 +C Remove\san\sunreachable\sbranch\sin\sthe\sdirect\sblob\sI/O\slogic\sof\sbtree.c. +D 2013-04-03T20:04:04.697 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c fd492d65eee0eb35f4000ddf478215c99dca6838 +F src/btree.c 4e3d07b3781a41e6ba9ea13ca97d7baab0a252a3 F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 2d9f1327fe79e40435ce1e2594d7cd9a5aea0ef2 -R e2d5c255f4ba7e0fb00c083dac088003 +P ac1432b3a8e968be4cbc138b4a35f34187c0c36f +R d830cc1d5cf92f6d3df5e32e65cb6d76 U drh -Z 519f2edad4b962613f9e09a344e900e8 +Z d56f9eb968ec676dc0a522c6c207a8e6 diff --git a/manifest.uuid b/manifest.uuid index 7c7984449a..d557c3a92c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac1432b3a8e968be4cbc138b4a35f34187c0c36f \ No newline at end of file +f97d7274f48e3bb98ed17eae11abb55064134874 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 42c4ee3cc0..c24481bdf7 100644 --- a/src/btree.c +++ b/src/btree.c @@ -8380,11 +8380,13 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ /* Save the positions of all other cursors open on this table. This is ** required in case any of them are holding references to an xFetch ** version of the b-tree page modified by the accessPayload call below. + ** + ** Note that pCsr must be open on a BTREE_INTKEY table and saveCursorPosition() + ** and hence saveAllCursors() cannot fail on a BTREE_INTKEY table, hence + ** saveAllCursors can only return SQLITE_OK. */ - rc = saveAllCursors(pCsr->pBt, pCsr->pgnoRoot, pCsr); - if( rc!=SQLITE_OK ){ - return SQLITE_OK; - } + VVA_ONLY(rc =) saveAllCursors(pCsr->pBt, pCsr->pgnoRoot, pCsr); + assert( rc==SQLITE_OK ); /* Check some assumptions: ** (a) the cursor is open for writing, From b0a8ea046a5083b6f0e862e5e5ddcb50c6957334 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Apr 2013 21:23:28 +0000 Subject: [PATCH 074/140] Release resources prior to bailing out of the btreeCreateTable() routine following an OOM error. FossilOrigin-Name: b2a72be9bab77f050bef75477a278a6294d3e854 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e986a3818e..1dc8e386cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunreachable\sbranch\sin\sthe\sdirect\sblob\sI/O\slogic\sof\sbtree.c. -D 2013-04-03T20:04:04.697 +C Release\sresources\sprior\sto\sbailing\sout\sof\sthe\sbtreeCreateTable()\sroutine\nfollowing\san\sOOM\serror. +D 2013-04-03T21:23:28.794 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in df3e48659d80e1b7765785d8d66c86b320f72cc7 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -121,7 +121,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 4e3d07b3781a41e6ba9ea13ca97d7baab0a252a3 +F src/btree.c 58c4d68415ca7b13a501b682a9c0ad1e287bcacb F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 @@ -1042,7 +1042,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ac1432b3a8e968be4cbc138b4a35f34187c0c36f -R d830cc1d5cf92f6d3df5e32e65cb6d76 +P f97d7274f48e3bb98ed17eae11abb55064134874 +R 133fe42d87ba7d3deff6d35af20e1f64 U drh -Z d56f9eb968ec676dc0a522c6c207a8e6 +Z 8d00d860d2c818ad1f98631f4c1a926a diff --git a/manifest.uuid b/manifest.uuid index d557c3a92c..a6f66ce7d4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f97d7274f48e3bb98ed17eae11abb55064134874 \ No newline at end of file +b2a72be9bab77f050bef75477a278a6294d3e854 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index c24481bdf7..2fb5becb57 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7242,12 +7242,11 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ ** case they are holding a reference to an xFetch reference ** corresponding to page pgnoRoot. */ rc = saveAllCursors(pBt, 0, 0); + releasePage(pPageMove); if( rc!=SQLITE_OK ){ return rc; } - releasePage(pPageMove); - /* Move the page currently at pgnoRoot to pgnoMove. */ rc = btreeGetPage(pBt, pgnoRoot, &pRoot, 0, 0); if( rc!=SQLITE_OK ){ From 6109e50f87cbf45c2f0dd9d5939e0471ca377e4d Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 4 Apr 2013 00:51:14 +0000 Subject: [PATCH 075/140] Increase the version number to 3.7.17. FossilOrigin-Name: 5c8c5a2a12f3f5e29c91c4d16e8b4afd4307354e --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/VERSION b/VERSION index 9efca4c4be..9a4c4371ee 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.7.16.1 +3.7.17 diff --git a/configure b/configure index ddd6364831..90452c95a2 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.7.16.1. +# Generated by GNU Autoconf 2.62 for sqlite 3.7.17. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.7.16.1' -PACKAGE_STRING='sqlite 3.7.16.1' +PACKAGE_VERSION='3.7.17' +PACKAGE_STRING='sqlite 3.7.17' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1484,7 +1484,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.7.16.1 to adapt to many kinds of systems. +\`configure' configures sqlite 3.7.17 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1549,7 +1549,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.7.16.1:";; + short | recursive ) echo "Configuration of sqlite 3.7.17:";; esac cat <<\_ACEOF @@ -1665,7 +1665,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.7.16.1 +sqlite configure 3.7.17 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1679,7 +1679,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.7.16.1, which was +It was created by sqlite $as_me 3.7.17, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -14032,7 +14032,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.7.16.1, which was +This file was extended by sqlite $as_me 3.7.17, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14085,7 +14085,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.7.16.1 +sqlite config.status 3.7.17 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/manifest b/manifest index ebd18074c8..b53e120abe 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Try\sto\suse\smmap()\sto\sspeed\saccess\sto\sthe\sdatabase\sfile\son\swindows,\slinux,\nand\smac. -D 2013-04-04T00:40:17.148 +C Increase\sthe\sversion\snumber\sto\s3.7.17. +D 2013-04-04T00:51:14.877 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc 9ab787d472c9cb8bad29c2c4aeba15b06fba7bc4 F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 -F VERSION 05604ccde96fe1b37f922eddbdcb5654308261db +F VERSION 05c7bd63b96f31cfdef5c766ed91307ac121f5aa F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 17dc593f791f874d2c23a0f9360850ded0286531 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -15,7 +15,7 @@ F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 0921066a13130082764ab4ab6456f7b5bebe56de F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure 2f9994fd2e2005753809a2246c7a0816c80b2d2f x +F configure 8bb8bd13d3c918c4c1c73480930e81f955ac298a x F configure.ac 81c43d151d0b0e406be056394cc9ff4cb3fd0444 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 @@ -1044,7 +1044,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e5b710849dd66673ba0e0d935b103cb29abfcc4b b2a72be9bab77f050bef75477a278a6294d3e854 -R 60af9b4d9e73e7757090b3abb77cbb41 +P fff2be60779571c8fb89158db976ec3755e9a223 +R 31e6af012b822f077ea81ef26342c1b3 U drh -Z d05e425b384f57133414b300483bca27 +Z 4751dcf9197532f06483d5731e95cbb0 diff --git a/manifest.uuid b/manifest.uuid index 4f646cb565..c01513bb51 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fff2be60779571c8fb89158db976ec3755e9a223 \ No newline at end of file +5c8c5a2a12f3f5e29c91c4d16e8b4afd4307354e \ No newline at end of file From 67f7c780f305ea6717448f68d2a4b64c03567337 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 4 Apr 2013 01:54:10 +0000 Subject: [PATCH 076/140] Updates to the file-control documentation. No changes to code. FossilOrigin-Name: 51fab480d0487107709e6c9e00bc0512b82ef1aa --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 21 +++++++++++++-------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index b53e120abe..30005c082d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sversion\snumber\sto\s3.7.17. -D 2013-04-04T00:51:14.877 +C Updates\sto\sthe\sfile-control\sdocumentation.\s\sNo\schanges\sto\scode. +D 2013-04-04T01:54:10.804 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,7 +176,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c 319b7791cee6c763b60fde1b590bfaf62613cf37 -F src/sqlite.h.in faeb6b3470193e599d79289f77f984b8a78136e1 +F src/sqlite.h.in 5fdc866a06ac82fd3c6ccb9f59a100151a36568c F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 1664dc5ad6f8d4dab871416628aa3271044d66c0 @@ -1044,7 +1044,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P fff2be60779571c8fb89158db976ec3755e9a223 -R 31e6af012b822f077ea81ef26342c1b3 +P 5c8c5a2a12f3f5e29c91c4d16e8b4afd4307354e +R 88aa21f5b7ea257c6d372937f19d04ff U drh -Z 4751dcf9197532f06483d5731e95cbb0 +Z 1441bae4ec4fe09d305292864aa8123c diff --git a/manifest.uuid b/manifest.uuid index c01513bb51..556e99572e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5c8c5a2a12f3f5e29c91c4d16e8b4afd4307354e \ No newline at end of file +51fab480d0487107709e6c9e00bc0512b82ef1aa \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index a4fb1a8a9a..74cecbcf22 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -868,7 +868,8 @@ struct sqlite3_io_methods { ** it is able to override built-in [PRAGMA] statements. ** **
  • [[SQLITE_FCNTL_BUSYHANDLER]] -** ^This file-control may be invoked by SQLite on the database file handle +** ^The [SQLITE_FCNTL_BUSYHANDLER] +** file-control may be invoked by SQLite on the database file handle ** shortly after it is opened in order to provide a custom VFS with access ** to the connections busy-handler callback. The argument is of type (void **) ** - an array of two (void *) values. The first (void *) actually points @@ -879,7 +880,8 @@ struct sqlite3_io_methods { ** current operation. ** **
  • [[SQLITE_FCNTL_TEMPFILENAME]] -** ^Application can invoke this file-control to have SQLite generate a +** ^Application can invoke the [SQLITE_FCNTL_TEMPFILENAME] file-control +** to have SQLite generate a ** temporary filename using the same algorithm that is followed to generate ** temporary filenames for TEMP tables and other internal uses. The ** argument should be a char** which will be filled with the filename @@ -887,10 +889,14 @@ struct sqlite3_io_methods { ** invoke [sqlite3_free()] on the result to avoid a memory leak. ** **
  • [[SQLITE_FCNTL_MMAP_LIMIT]] -** The argument is assumed to pointer to a value of type sqlite3_int64 that +** The [SQLITE_FCNTL_MMAP_LIMIT] file control is used to query or set the +** maximum number of bytes that will be used for memory-mapped I/O. +** The argument is a pointer to a value of type sqlite3_int64 that ** is an advisory maximum number of bytes in the file to memory map. The ** pointer is overwritten with the old value. The limit is not changed if -** the original value pointed to is negative. +** the originally pointed to is negative, and so the current limit can be +** queried by passing in a pointer to a negative number. This file-control +** is used internally to implement [PRAGMA mmap_limit]. ** ** */ @@ -1646,11 +1652,10 @@ struct sqlite3_mem_methods { ** is the default maximum number of bytes of process address space that ** should be used for accessing each database file using memory mapping. ** The default setting can be overridden by each database connection using -** either the [PRAGMA mmap_limit] command or the "mmaplimit" query parameter -** on the [URI filename] when opening the databaes file or by using the +** either the [PRAGMA mmap_limit] command, or by using the ** [SQLITE_FCNTL_MMAP_LIMIT] file control. The value set here overrides the -** compile-time default that is set using the [SQLITE_DEFAULT_MMAP_LIMIT] -** compile-time option. If the argument to this option is negative, then +** compile-time default that is set using [SQLITE_DEFAULT_MMAP_LIMIT]. +** If the argument to this option is negative, then ** the memory map limit is set to the compile-time default. ** */ From 75f2d540acf87280e506c6ca8bf21430d2b124fe Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 4 Apr 2013 17:28:11 +0000 Subject: [PATCH 077/140] Disable memory-mapped I/O for the win32lock.test module FossilOrigin-Name: 2b3d9805a28e51f646113e22c2dff2272342306e --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/win32lock.test | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 30005c082d..e8f2288842 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\sthe\sfile-control\sdocumentation.\s\sNo\schanges\sto\scode. -D 2013-04-04T01:54:10.804 +C Disable\smemory-mapped\sI/O\sfor\sthe\swin32lock.test\smodule +D 2013-04-04T17:28:11.286 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -997,7 +997,7 @@ F test/whereD.test 3f3ee93825c94804f1fc91eef2de0d365981759a F test/whereE.test 7bd34945797efef15819368479bacc34215e4e1d F test/whereF.test a0e296643cabe5278379bc1a0aa158cf3c54a1c9 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 -F test/win32lock.test b2a539e85ae6b2d78475e016a9636b4451dc7fb9 +F test/win32lock.test 21810396eee150680cc398970839e6abdcb807b9 F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F test/zerodamage.test e7f77fded01dfcdf92ac2c5400f1e35d7a21463c F tool/build-all-msvc.bat 74fb6e5cca66ebdb6c9bbafb2f8b802f08146d38 x @@ -1044,7 +1044,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5c8c5a2a12f3f5e29c91c4d16e8b4afd4307354e -R 88aa21f5b7ea257c6d372937f19d04ff +P 51fab480d0487107709e6c9e00bc0512b82ef1aa +R c0f8ddd1641a176c1bcbffa647af40c5 U drh -Z 1441bae4ec4fe09d305292864aa8123c +Z f901f35bfd9928cf6d746ecf21982f94 diff --git a/manifest.uuid b/manifest.uuid index 556e99572e..d4cf081120 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -51fab480d0487107709e6c9e00bc0512b82ef1aa \ No newline at end of file +2b3d9805a28e51f646113e22c2dff2272342306e \ No newline at end of file diff --git a/test/win32lock.test b/test/win32lock.test index d014be4d2c..50c9f1d5f4 100644 --- a/test/win32lock.test +++ b/test/win32lock.test @@ -27,6 +27,7 @@ proc xLog {error_code msg} { lappend ::log $msg } sqlite3 db test.db +db eval {PRAGMA mmap_limit=0} do_test win32lock-1.1 { db eval { From ced9813b12a9320e0a16ddf7926a323d672ce36b Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 4 Apr 2013 19:32:53 +0000 Subject: [PATCH 078/140] Add further multi-client tests to mmap1.test. FossilOrigin-Name: a107f75d93b8acd3403d8473a75137378041a833 --- manifest | 14 ++++---- manifest.uuid | 2 +- test/mmap1.test | 89 ++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 85 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index e8f2288842..16f4057920 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\smemory-mapped\sI/O\sfor\sthe\swin32lock.test\smodule -D 2013-04-04T17:28:11.286 +C Add\sfurther\smulti-client\stests\sto\smmap1.test. +D 2013-04-04T19:32:53.279 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -631,7 +631,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test df5105f08e6000e57b4de7e748f8c2ae3fed75da +F test/mmap1.test 146a755bb382677b6f9993c61ddc7732b157d730 F test/mmap2.test 62dbb5d718e66d654d232116c5a2d96e26a071a5 F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -1044,7 +1044,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 51fab480d0487107709e6c9e00bc0512b82ef1aa -R c0f8ddd1641a176c1bcbffa647af40c5 -U drh -Z f901f35bfd9928cf6d746ecf21982f94 +P 2b3d9805a28e51f646113e22c2dff2272342306e +R 4c631120a8e96e37be49975ee7d63ac7 +U dan +Z d6d074a538192756f7d91b52bba8b7ce diff --git a/manifest.uuid b/manifest.uuid index d4cf081120..d68eca1adf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2b3d9805a28e51f646113e22c2dff2272342306e \ No newline at end of file +a107f75d93b8acd3403d8473a75137378041a833 \ No newline at end of file diff --git a/test/mmap1.test b/test/mmap1.test index 805fb76525..417a0756b1 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -24,10 +24,22 @@ proc nRead {db} { return $stats(read) } +proc register_rblob_code {dbname seed} { + return [subst -nocommands { + set ::rcnt $seed + proc rblob {n} { + set ::rcnt [expr (([set ::rcnt] << 3) + [set ::rcnt] + 456) & 0xFFFFFFFF] + set str [format %.8x [expr [set ::rcnt] ^ 0xbdf20da3]] + string range [string repeat [set str] [expr [set n]/4]] 1 [set n] + } + $dbname func rblob rblob + }] +} + foreach {t mmap_limit nRead c2init} { - 1.1 { PRAGMA mmap_limit = 67108864 } 4 {} - 1.2 { PRAGMA mmap_limit = 53248 } 150 {} - 1.3 { PRAGMA mmap_limit = 0 } 344 {} + 1.1 { PRAGMA mmap_limit = 67108864 } 4 {PRAGMA mmap_limit = 0} + 1.2 { PRAGMA mmap_limit = 53248 } 150 {PRAGMA mmap_limit = 0} + 1.3 { PRAGMA mmap_limit = 0 } 344 {PRAGMA mmap_limit = 0} 1.4 { PRAGMA mmap_limit = 67108864 } 4 {PRAGMA mmap_limit = 67108864 } 1.5 { PRAGMA mmap_limit = 53248 } 150 {PRAGMA mmap_limit = 67108864 } 1.6 { PRAGMA mmap_limit = 0 } 344 {PRAGMA mmap_limit = 67108864 } @@ -37,15 +49,7 @@ foreach {t mmap_limit nRead c2init} { sql1 $mmap_limit sql2 $c2init - code2 { - set ::rcnt 0 - proc rblob {n} { - set ::rcnt [expr (($::rcnt << 3) + $::rcnt + 456) & 0xFFFFFFFF] - set str [format %.8x [expr $::rcnt ^ 0xbdf20da3]] - string range [string repeat $str [expr $n/4]] 1 $n - } - db2 func rblob rblob - } + code2 [register_rblob_code db2 0] sql2 { PRAGMA page_size=1024; @@ -245,6 +249,67 @@ do_test 5.5 { sqlite3_finalize $::STMT } SQLITE_OK +#------------------------------------------------------------------------- +# Test various mmap_limit settings. +# +foreach {tn1 mmap1 mmap2} { + 1 6144 167773 + 2 18432 140399 + 3 43008 401302 + 4 92160 253899 + 5 190464 2 + 6 387072 752431 + 7 780288 291143 + 8 1566720 594306 + 9 3139584 829137 + 10 6285312 793963 + 11 12576768 1015590 +} { + do_multiclient_test tn { + sql1 { + CREATE TABLE t1(a PRIMARY KEY); + CREATE TABLE t2(x); + INSERT INTO t2 VALUES(''); + } + + code1 [register_rblob_code db 0] + code2 [register_rblob_code db2 444] + + sql1 "PRAGMA mmap_limit = $mmap1" + sql2 "PRAGMA mmap_limit = $mmap2" + + do_test $tn1.$tn { + for {set i 1} {$i <= 100} {incr i} { + if {$i % 2} { + set c1 sql1 + set c2 sql2 + } else { + set c1 sql2 + set c2 sql1 + } + + $c1 { + INSERT INTO t1 VALUES( rblob(5000) ); + UPDATE t2 SET x = (SELECT md5sum(a) FROM t1); + } + + set res [$c2 { + SELECT count(*) FROM t1; + SELECT x == (SELECT md5sum(a) FROM t1) FROM t2; + PRAGMA integrity_check; + }] + if {$res != [list $i 1 ok]} { + do_test $tn1.$tn.$i { + set ::res + } [list $i 1 ok] + } + } + set res 1 + } {1} + } +} + + finish_test From d7a558a9fbf6feb84ed990c67e41fe45dafd4a18 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 5 Apr 2013 20:40:43 +0000 Subject: [PATCH 079/140] Improve the code coverage of "permutations.test coverage-pager". FossilOrigin-Name: b095e2cdb61ca8487255687f58fb1024d40f3986 --- manifest | 20 ++-- manifest.uuid | 2 +- src/test_vfs.c | 12 ++- test/pager1.test | 211 ++++++++++++++++++++++++++++++++++++++++++- test/pagerfault.test | 183 ++++++++++++++++++++++++++++++++++++- test/wal.test | 20 ---- test/walfault.test | 40 +++++++- 7 files changed, 451 insertions(+), 37 deletions(-) diff --git a/manifest b/manifest index 16f4057920..2b89f591ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfurther\smulti-client\stests\sto\smmap1.test. -D 2013-04-04T19:32:53.279 +C Improve\sthe\scode\scoverage\sof\s"permutations.test\scoverage-pager". +D 2013-04-05T20:40:43.227 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd F src/test_syscall.c 7e8293e4e6971b0f44c7f7f37b1315a8cc9f6018 F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c e286f2173563f2a1747c24bcda6b9d030bf4f4e4 -F src/test_vfs.c fb16b2d9938cf0c1afc5a423b55b952fcc024275 +F src/test_vfs.c 8e6087a8b3dcc260282074b0efba14b76311120c F src/test_vfstrace.c 34b544e80ba7fb77be15395a609c669df2e660a2 F src/test_wholenumber.c 3d2b9ed1505c40ad5c5ca2ad16ae7a289d6cc251 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 @@ -651,10 +651,10 @@ F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 -F test/pager1.test 31c04bec797dda1bde337810b52efa08d1f1f08e +F test/pager1.test 622bfaea80604a96010f35256a4fc22fce584e95 F test/pager2.test 745b911dde3d1f24ae0870bd433dfa83d7c658c1 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f -F test/pagerfault.test fc2e37b2da626826dd54bb13720b4d721719b660 +F test/pagerfault.test 636c01bdbdb2bb2baf68532d6c4ef4c633022743 F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 F test/pageropt.test 32cb5a6ed7ccddf8e8c842cb44240bd9340223ce @@ -955,7 +955,7 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 -F test/wal.test 62454b2cf00b311e9e65f898aad0fef536ed561a +F test/wal.test bbd98ac09d8fbcaa80ec233757056eb1bbb7cd95 F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c @@ -971,7 +971,7 @@ F test/walcksum.test f5447800a157c9e2234fbb8e80243f0813941bde F test/walcrash.test 4457436593be8c136f9148487c7dccd5e9013af2 F test/walcrash2.test 019d60b89d96c1937adb2b30b850ac7e86e5a142 F test/walcrash3.test 595e44c6197f0d0aa509fc135be2fd0209d11a2c -F test/walfault.test 97394d8de82a99f7abf1c12ed229640607fd0ad2 +F test/walfault.test 54ad6e849c727f4da463964b9eb8c8e8e155cf82 F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483 F test/walmode.test 4022fe03ae6e830583672caa101f046438a0473c F test/walnoshm.test 84ca10c544632a756467336b7c3b864d493ee496 @@ -1044,7 +1044,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 2b3d9805a28e51f646113e22c2dff2272342306e -R 4c631120a8e96e37be49975ee7d63ac7 +P a107f75d93b8acd3403d8473a75137378041a833 +R 6d11c98ab18b87c4b8798ffe987d9d38 U dan -Z d6d074a538192756f7d91b52bba8b7ce +Z 50aa2590d101ae076870fa94f81e976a diff --git a/manifest.uuid b/manifest.uuid index d68eca1adf..8dbe787aae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a107f75d93b8acd3403d8473a75137378041a833 \ No newline at end of file +b095e2cdb61ca8487255687f58fb1024d40f3986 \ No newline at end of file diff --git a/src/test_vfs.c b/src/test_vfs.c index 933de99515..fcd577439c 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -125,8 +125,9 @@ struct Testvfs { #define TESTVFS_ACCESS_MASK 0x00004000 #define TESTVFS_FULLPATHNAME_MASK 0x00008000 #define TESTVFS_READ_MASK 0x00010000 +#define TESTVFS_UNLOCK_MASK 0x00020000 -#define TESTVFS_ALL_MASK 0x0001FFFF +#define TESTVFS_ALL_MASK 0x0003FFFF #define TESTVFS_MAX_PAGES 1024 @@ -467,8 +468,12 @@ static int tvfsLock(sqlite3_file *pFile, int eLock){ ** Unlock an tvfs-file. */ static int tvfsUnlock(sqlite3_file *pFile, int eLock){ - TestvfsFd *p = tvfsGetFd(pFile); - return sqlite3OsUnlock(p->pReal, eLock); + TestvfsFd *pFd = tvfsGetFd(pFile); + Testvfs *p = (Testvfs *)pFd->pVfs->pAppData; + if( p->mask&TESTVFS_WRITE_MASK && tvfsInjectIoerr(p) ){ + return SQLITE_IOERR_UNLOCK; + } + return sqlite3OsUnlock(pFd->pReal, eLock); } /* @@ -1101,6 +1106,7 @@ static int testvfs_obj_cmd( { "xClose", TESTVFS_CLOSE_MASK }, { "xAccess", TESTVFS_ACCESS_MASK }, { "xFullPathname", TESTVFS_FULLPATHNAME_MASK }, + { "xUnlock", TESTVFS_UNLOCK_MASK }, }; Tcl_Obj **apElem = 0; int nElem = 0; diff --git a/test/pager1.test b/test/pager1.test index ae47b3d932..22c84d55d0 100644 --- a/test/pager1.test +++ b/test/pager1.test @@ -15,6 +15,7 @@ source $testdir/tester.tcl source $testdir/lock_common.tcl source $testdir/malloc_common.tcl source $testdir/wal_common.tcl +set testprefix pager1 # Do not use a codec for tests in this file, as the database file is # manipulated directly using tcl scripts (using the [hexio_write] command). @@ -1382,6 +1383,7 @@ db2 close # testvfs tv -default 1 foreach sectorsize { + 16 32 64 128 256 512 1024 2048 4096 8192 16384 32768 65536 131072 262144 } { @@ -1404,7 +1406,7 @@ foreach sectorsize { COMMIT; } file size test.db-journal - } [expr $sectorsize > 65536 ? 65536 : $sectorsize] + } [expr $sectorsize > 65536 ? 65536 : ($sectorsize<32 ? 512 : $sectorsize)] do_test pager1-10.$sectorsize.2 { execsql { @@ -2523,4 +2525,211 @@ if {$::tcl_platform(platform)=="unix"} { } {one two} } +#------------------------------------------------------------------------- +# Test that appending pages to the database file then moving those pages +# to the free-list before the transaction is committed does not cause +# an error. +# +foreach {tn pragma strsize} { + 1 { PRAGMA mmap_limit = 0 } 2400 + 2 { } 2400 + 3 { PRAGMA mmap_limit = 0 } 4400 + 4 { } 4400 +} { + reset_db + db func a_string a_string + db eval $pragma + do_execsql_test 34.$tn.1 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + } + breakpoint + do_execsql_test 34.$tn.2 { + BEGIN; + INSERT INTO t1 VALUES(2, a_string($strsize)); + DELETE FROM t1 WHERE oid=2; + COMMIT; + PRAGMA integrity_check; + } {ok} +} + +#------------------------------------------------------------------------- +# +reset_db +do_test 35 { + sqlite3 db test.db + + execsql { + CREATE TABLE t1(x, y); + PRAGMA journal_mode = WAL; + INSERT INTO t1 VALUES(1, 2); + } + + execsql { + BEGIN; + CREATE TABLE t2(a, b); + } + + hexio_write test.db-shm [expr 16*1024] [string repeat 0055 8192] + catchsql ROLLBACK +} {0 {}} + +do_multiclient_test tn { + sql1 { + PRAGMA auto_vacuum = 0; + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(1, 2); + } + + do_test 36.$tn.1 { + sql2 { PRAGMA max_page_count = 2 } + list [catch { sql2 { CREATE TABLE t2(x) } } msg] $msg + } {1 {database or disk is full}} + + sql1 { PRAGMA checkpoint_fullfsync = 1 } + sql1 { CREATE TABLE t2(x) } + + do_test 36.$tn.2 { + sql2 { INSERT INTO t2 VALUES('xyz') } + list [catch { sql2 { CREATE TABLE t3(x) } } msg] $msg + } {1 {database or disk is full}} +} + +forcedelete test1 test2 +foreach {tn uri} { + 1 {file:?mode=memory&cache=shared} + 2 {file:one?mode=memory&cache=shared} + 3 {file:test1?cache=shared} + 4 {file:test2?another=parameter&yet=anotherone} +} { + do_test 37.$tn { + catch { db close } + sqlite3_shutdown + sqlite3_config_uri 1 + sqlite3 db $uri + db eval { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(1); + SELECT * FROM t1; + } + } {1} + db close + sqlite3_shutdown + sqlite3_config_uri 0 +} + +do_test 38.1 { + catch { db close } + forcedelete test.db + set fd [open test.db w] + puts $fd "hello world" + close $fd + sqlite3 db test.db + catchsql { CREATE TABLE t1(x) } +} {1 {file is encrypted or is not a database}} +do_test 38.2 { + catch { db close } + forcedelete test.db +} {} + +do_test 39.1 { + sqlite3 db test.db + execsql { + PRAGMA auto_vacuum = 1; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES('xxx'); + INSERT INTO t1 VALUES('two'); + INSERT INTO t1 VALUES(randomblob(400)); + INSERT INTO t1 VALUES(randomblob(400)); + INSERT INTO t1 VALUES(randomblob(400)); + INSERT INTO t1 VALUES(randomblob(400)); + BEGIN; + UPDATE t1 SET x = 'one' WHERE rowid=1; + } + set ::stmt [sqlite3_prepare db "SELECT * FROM t1 ORDER BY rowid" -1 dummy] + sqlite3_step $::stmt + sqlite3_column_text $::stmt 0 +} {one} +do_test 39.2 { + execsql { CREATE TABLE t2(x) } + sqlite3_step $::stmt + sqlite3_column_text $::stmt 0 +} {two} +do_test 39.3 { + sqlite3_finalize $::stmt + execsql COMMIT +} {} + +do_execsql_test 39.4 { + PRAGMA auto_vacuum = 2; + CREATE TABLE t3(x); + CREATE TABLE t4(x); + + DROP TABLE t2; + DROP TABLE t3; + DROP TABLE t4; +} +do_test 39.5 { + db close + sqlite3 db test.db + execsql { + PRAGMA cache_size = 1; + PRAGMA incremental_vacuum; + PRAGMA integrity_check; + } +} {ok} + +do_test 40.1 { + reset_db + execsql { + PRAGMA auto_vacuum = 1; + CREATE TABLE t1(x PRIMARY KEY); + INSERT INTO t1 VALUES(randomblob(1200)); + PRAGMA page_count; + } +} {6} +do_test 40.2 { + execsql { + INSERT INTO t1 VALUES(randomblob(1200)); + INSERT INTO t1 VALUES(randomblob(1200)); + INSERT INTO t1 VALUES(randomblob(1200)); + } +} {} +do_test 40.3 { + db close + sqlite3 db test.db + execsql { + PRAGMA cache_size = 1; + CREATE TABLE t2(x); + PRAGMA integrity_check; + } +} {ok} + +do_test 41.1 { + reset_db + execsql { + CREATE TABLE t1(x PRIMARY KEY); + INSERT INTO t1 VALUES(randomblob(200)); + INSERT INTO t1 SELECT randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200) FROM t1; + } +} {} +do_test 41.2 { + testvfs tv -default 1 + tv sectorsize 16384; + tv devchar [list] + db close + sqlite3 db test.db + execsql { + PRAGMA cache_size = 1; + DELETE FROM t1 WHERE rowid%4; + PRAGMA integrity_check; + } +} {ok} + + finish_test diff --git a/test/pagerfault.test b/test/pagerfault.test index 9ea1aa6328..440909d1a1 100644 --- a/test/pagerfault.test +++ b/test/pagerfault.test @@ -1246,6 +1246,7 @@ do_faultsim_test pagerfault-27 -faults ioerr-persistent -prep { faultsim_integrity_check } + #------------------------------------------------------------------------- # do_test pagerfault-28-pre { @@ -1279,7 +1280,7 @@ do_test pagerfault-28-pre { } {1} faultsim_save_and_close -do_faultsim_test pagerfault-28 -faults oom* -prep { +do_faultsim_test pagerfault-28a -faults oom* -prep { faultsim_restore_and_reopen execsql { PRAGMA mmap_limit=0 } @@ -1303,5 +1304,185 @@ do_faultsim_test pagerfault-28 -faults oom* -prep { faultsim_integrity_check } +faultsim_restore_and_reopen +sqlite3 db2 test.db +db2 eval {SELECT count(*) FROM t2} +db close + +do_faultsim_test pagerfault-28b -faults oom* -prep { + sqlite3 db test.db +} -body { + execsql { SELECT count(*) FROM t2 } +} -test { + faultsim_test_result {0 2048} + db close +} + +db2 close + +#------------------------------------------------------------------------- +# Try this: +# +# 1) Put the pager in ERROR state (error during rollback) +# +# 2) Next time the connection is used inject errors into all xWrite() and +# xUnlock() calls. This causes the hot-journal rollback to fail and +# the pager to declare its locking state UNKNOWN. +# +# 3) Same again. +# +# 4a) Stop injecting errors. Allow the rollback to succeed. Check that +# the database is Ok. Or, +# +# 4b) Close and reopen the db. Check that the db is Ok. +# +proc custom_injectinstall {} { + testvfs custom -default true + custom filter {xWrite xUnlock} +} +proc custom_injectuninstall {} { + catch {db close} + catch {db2 close} + custom delete +} +proc custom_injectstart {iFail} { + custom ioerr $iFail 1 +} +proc custom_injectstop {} { + custom ioerr +} +set ::FAULTSIM(custom) [list \ + -injectinstall custom_injectinstall \ + -injectstart custom_injectstart \ + -injectstop custom_injectstop \ + -injecterrlist {{1 {disk I/O error}}} \ + -injectuninstall custom_injectuninstall \ +] + +do_test pagerfault-29-pre { + faultsim_delete_and_reopen + db func a_string a_string + execsql { + PRAGMA page_size = 1024; + PRAGMA cache_size = 5; + + BEGIN; + CREATE TABLE t2(a UNIQUE, b UNIQUE); + INSERT INTO t2 VALUES( a_string(800), a_string(800) ); + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + INSERT INTO t2 SELECT a_string(800), a_string(800) FROM t2; + COMMIT; + } + expr {[file size test.db] >= 50*1024} +} {1} +faultsim_save_and_close +foreach {tn tt} { + 29 { catchsql ROLLBACK } + 30 { db close ; sqlite3 db test.db } +} { + do_faultsim_test pagerfault-$tn -faults custom -prep { + faultsim_restore_and_reopen + db func a_string a_string + execsql { + PRAGMA cache_size = 5; + BEGIN; + UPDATE t2 SET a = a_string(799); + } + } -body { + catchsql ROLLBACK + catchsql ROLLBACK + catchsql ROLLBACK + } -test { + eval $::tt + if {"ok" != [db one {PRAGMA integrity_check}]} { + error "integrity check failed" + } + } +} + +do_test pagerfault-31-pre { + sqlite3_shutdown + sqlite3_config_uri 1 +} {SQLITE_OK} +do_faultsim_test pagerfault-31 -faults oom* -body { + sqlite3 db {file:one?mode=memory&cache=shared} + db eval { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(1); + SELECT * FROM t1; + } +} -test { + faultsim_test_result {0 1} {1 {}} + catch { db close } +} +sqlite3_shutdown +sqlite3_config_uri 0 + +do_test pagerfault-32-pre { + reset_db + execsql { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES('one'); + } +} {} +faultsim_save_and_close + +do_faultsim_test pagerfault-32 -prep { + faultsim_restore_and_reopen + db eval { SELECT * FROM t1; } +} -body { + execsql { SELECT * FROM t1; } +} -test { + faultsim_test_result {0 one} +} +sqlite3_shutdown +sqlite3_config_uri 0 + +do_faultsim_test pagerfault-33a -prep { + sqlite3 db :memory: + execsql { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + } +} -body { + execsql { VACUUM } +} -test { + faultsim_test_result {0 {}} +} +do_faultsim_test pagerfault-33b -prep { + sqlite3 db "" + execsql { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + } +} -body { + execsql { VACUUM } +} -test { + faultsim_test_result {0 {}} +} + +do_test pagerfault-34-pre { + reset_db + execsql { + CREATE TABLE t1(x PRIMARY KEY); + } +} {} +faultsim_save_and_close +do_faultsim_test pagerfault-34 -prep { + faultsim_restore_and_reopen + execsql { + BEGIN; + INSERT INTO t1 VALUES( randomblob(4000) ); + DELETE FROM t1; + } +} -body { + execsql COMMIT +} -test { + faultsim_test_result {0 {}} +} + + finish_test diff --git a/test/wal.test b/test/wal.test index f833b6ab1a..0539192d25 100644 --- a/test/wal.test +++ b/test/wal.test @@ -1569,26 +1569,6 @@ ifcapable autovacuum { } [wal_file_size 1 1024] } -reset_db -do_test 25 { - sqlite3 db test.db - - execsql { - CREATE TABLE t1(x, y); - PRAGMA journal_mode = WAL; - INSERT INTO t1 VALUES(1, 2); - } - - execsql { - BEGIN; - CREATE TABLE t2(a, b); - } - - hexio_write test.db-shm [expr 16*1024] [string repeat 0055 8192] - catchsql ROLLBACK -} {0 {}} - - db close sqlite3_shutdown test_sqlite3_log diff --git a/test/walfault.test b/test/walfault.test index 6f9aeddea2..4a9d98afda 100644 --- a/test/walfault.test +++ b/test/walfault.test @@ -548,6 +548,44 @@ do_faultsim_test walfault-14 -prep { set nRow [db eval {SELECT count(*) FROM abc}] if {!(($nRow==2 && $testrc) || $nRow==3)} { error "Bad db content" } } -finish_test + +#------------------------------------------------------------------------- +# Test fault-handling when switching out of exclusive-locking mode. +# +do_test walfault-14-pre { + faultsim_delete_and_reopen + execsql { + PRAGMA auto_vacuum = 0; + PRAGMA journal_mode = WAL; + BEGIN; + CREATE TABLE abc(a PRIMARY KEY); + INSERT INTO abc VALUES(randomblob(1500)); + INSERT INTO abc VALUES(randomblob(1500)); + COMMIT; + } + faultsim_save_and_close +} {} +do_faultsim_test walfault-14 -prep { + faultsim_restore_and_reopen + breakpoint + execsql { + SELECT count(*) FROM abc; + PRAGMA locking_mode = exclusive; + BEGIN; + INSERT INTO abc VALUES(randomblob(1500)); + COMMIT; + } +} -body { + db eval { + PRAGMA locking_mode = normal; + BEGIN; + INSERT INTO abc VALUES(randomblob(1500)); + COMMIT; + } +} -test { + faultsim_integrity_check + set nRow [db eval {SELECT count(*) FROM abc}] + if {$nRow!=3 && $nRow!=4} { error "Bad db content" } +} finish_test From 27338e6ed7eb0436896329dc6ca29959038e27a2 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 6 Apr 2013 00:19:37 +0000 Subject: [PATCH 080/140] Add a prototype TH3-script-style test harness that starts multiple processes operating on the same database file at the same time. FossilOrigin-Name: c318fafe686120d7fb8e487eb3bb4942d497665c --- manifest | 16 +- manifest.uuid | 2 +- mptest/mptest.c | 1031 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 1042 insertions(+), 7 deletions(-) create mode 100644 mptest/mptest.c diff --git a/manifest b/manifest index 2b89f591ab..0c4454579c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\scode\scoverage\sof\s"permutations.test\scoverage-pager". -D 2013-04-05T20:40:43.227 +C Add\sa\sprototype\sTH3-script-style\stest\sharness\sthat\sstarts\smultiple\sprocesses\noperating\son\sthe\ssame\sdatabase\sfile\sat\sthe\ssame\stime. +D 2013-04-06T00:19:37.887 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,6 +110,7 @@ F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 +F mptest/mptest.c 04110c35f415218e54e030b2e37215ad4adf01f3 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc @@ -1044,7 +1045,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a107f75d93b8acd3403d8473a75137378041a833 -R 6d11c98ab18b87c4b8798ffe987d9d38 -U dan -Z 50aa2590d101ae076870fa94f81e976a +P b095e2cdb61ca8487255687f58fb1024d40f3986 +R c143d7671ed86699bfdf0a0519618d31 +T *branch * mptest +T *sym-mptest * +T -sym-trunk * +U drh +Z f0aabffa9579562b9fff6a9c66b5a272 diff --git a/manifest.uuid b/manifest.uuid index 8dbe787aae..fcf249224b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b095e2cdb61ca8487255687f58fb1024d40f3986 \ No newline at end of file +c318fafe686120d7fb8e487eb3bb4942d497665c \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c new file mode 100644 index 0000000000..01660af690 --- /dev/null +++ b/mptest/mptest.c @@ -0,0 +1,1031 @@ +/* +** 2013-04-05 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a program used for testing SQLite, and specifically for testing +** the ability of independent processes to access the same SQLite database +** concurrently. +** +** Compile this program as follows: +** +** gcc -g -c -Wall sqlite3.c $(OPTS) +** gcc -g -o mptest mptest.c sqlite3.o $(LIBS) +** +** Recommended options: +** +** -DHAVE_USLEEP +** -DSQLITE_NO_SYNC +** -DSQLITE_THREADSAFE=0 +** -DSQLITE_OMIT_LOAD_EXTENSION +** +** Run like this: +** +** ./mptest $database $script +** +** where $database is the database to use for testing and $script is a +** test script. +*/ +#include "sqlite3.h" +#include +#include +#include +#include +#include +#include + +/* Global data +*/ +static struct Global { + char *argv0; /* Name of the executable */ + const char *zVfs; /* Name of VFS to use. Often NULL meaning "default" */ + char *zDbFile; /* Name of the database */ + sqlite3 *db; /* Open connection to database */ + char *zErrLog; /* Filename for error log */ + FILE *pErrLog; /* Where to write errors */ + char *zLog; /* Name of output log file */ + FILE *pLog; /* Where to write log messages */ + char zName[12]; /* Symbolic name of this process */ + int taskId; /* Task ID. 0 means supervisor. */ + int iTrace; /* Tracing level */ + int bSqlTrace; /* True to trace SQL commands */ + int nError; /* Number of errors */ +} g; + +/* +** Print a message adding zPrefix[] to the beginning of every line. +*/ +static void printWithPrefix(FILE *pOut, const char *zPrefix, const char *zMsg){ + while( zMsg && zMsg[0] ){ + int i; + for(i=0; zMsg[i] && zMsg[i]!='\n' && zMsg[i]!='\r'; i++){} + fprintf(pOut, "%s%.*s\n", zPrefix, i, zMsg); + zMsg += i; + while( zMsg[0]=='\n' || zMsg[0]=='\r' ) zMsg++; + } +} + +/* +** Compare two pointers to strings, where the pointers might be NULL. +*/ +static int safe_strcmp(const char *a, const char *b){ + if( a==b ) return 0; + if( a==0 ) return -1; + if( b==0 ) return 1; + return strcmp(a,b); +} + +/* +** Return TRUE if string z[] matches glob pattern zGlob[]. +** Return FALSE if the pattern does not match. +** +** Globbing rules: +** +** '*' Matches any sequence of zero or more characters. +** +** '?' Matches exactly one character. +** +** [...] Matches one character from the enclosed list of +** characters. +** +** [^...] Matches one character not in the enclosed list. +** +** '#' Matches any sequence of one or more digits with an +** optional + or - sign in front +*/ +int strglob(const char *zGlob, const char *z){ + int c, c2; + int invert; + int seen; + + while( (c = (*(zGlob++)))!=0 ){ + if( c=='*' ){ + while( (c=(*(zGlob++))) == '*' || c=='?' ){ + if( c=='?' && (*(z++))==0 ) return 0; + } + if( c==0 ){ + return 1; + }else if( c=='[' ){ + while( *z && strglob(zGlob-1,z) ){ + z++; + } + return (*z)!=0; + } + while( (c2 = (*(z++)))!=0 ){ + while( c2!=c ){ + c2 = *(z++); + if( c2==0 ) return 0; + } + if( strglob(zGlob,z) ) return 1; + } + return 0; + }else if( c=='?' ){ + if( (*(z++))==0 ) return 0; + }else if( c=='[' ){ + int prior_c = 0; + seen = 0; + invert = 0; + c = *(z++); + if( c==0 ) return 0; + c2 = *(zGlob++); + if( c2=='^' ){ + invert = 1; + c2 = *(zGlob++); + } + if( c2==']' ){ + if( c==']' ) seen = 1; + c2 = *(zGlob++); + } + while( c2 && c2!=']' ){ + if( c2=='-' && zGlob[0]!=']' && zGlob[0]!=0 && prior_c>0 ){ + c2 = *(zGlob++); + if( c>=prior_c && c<=c2 ) seen = 1; + prior_c = 0; + }else{ + if( c==c2 ){ + seen = 1; + } + prior_c = c2; + } + c2 = *(zGlob++); + } + if( c2==0 || (seen ^ invert)==0 ) return 0; + }else if( c=='#' ){ + if( (z[0]=='-' || z[0]=='+') && isdigit(z[1]) ) z++; + if( !isdigit(z[0]) ) return 0; + z++; + while( isdigit(z[0]) ){ z++; } + }else{ + if( c!=(*(z++)) ) return 0; + } + } + return *z==0; +} + +/* +** Close output stream pOut if it is not stdout or stderr +*/ +static void maybeClose(FILE *pOut){ + if( pOut!=stdout && pOut!=stderr ) fclose(pOut); +} + +/* +** Print an error message +*/ +static void errorMessage(const char *zFormat, ...){ + va_list ap; + char *zMsg; + char zPrefix[30]; + va_start(ap, zFormat); + zMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + sqlite3_snprintf(sizeof(zPrefix), zPrefix, "%s:ERROR: ", g.zName); + if( g.pLog ){ + printWithPrefix(g.pLog, zPrefix, zMsg); + fflush(g.pLog); + } + if( g.pErrLog && safe_strcmp(g.zErrLog,g.zLog) ){ + printWithPrefix(g.pErrLog, zPrefix, zMsg); + fflush(g.pErrLog); + } + sqlite3_free(zMsg); + g.nError++; +} + +/* Forward declaration */ +static int trySql(const char*, ...); + +/* +** Print an error message and then quit. +*/ +static void fatalError(const char *zFormat, ...){ + va_list ap; + char *zMsg; + char zPrefix[30]; + va_start(ap, zFormat); + zMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + sqlite3_snprintf(sizeof(zPrefix), zPrefix, "%s:FATAL: ", g.zName); + if( g.pLog ){ + printWithPrefix(g.pLog, zPrefix, zMsg); + fflush(g.pLog); + maybeClose(g.pLog); + } + if( g.pErrLog && safe_strcmp(g.zErrLog,g.zLog) ){ + printWithPrefix(g.pErrLog, zPrefix, zMsg); + fflush(g.pErrLog); + maybeClose(g.pErrLog); + } + sqlite3_free(zMsg); + if( g.db ){ + int nTry = 0; + while( trySql("CREATE TABLE halt(x);")==SQLITE_BUSY && (nTry++)<100 ){ + sqlite3_sleep(10); + } + } + sqlite3_close(g.db); + exit(1); +} + + +/* +** Print a log message +*/ +static void logMessage(const char *zFormat, ...){ + va_list ap; + char *zMsg; + char zPrefix[30]; + va_start(ap, zFormat); + zMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + sqlite3_snprintf(sizeof(zPrefix), zPrefix, "%s: ", g.zName); + if( g.pLog ){ + printWithPrefix(g.pLog, zPrefix, zMsg); + fflush(g.pLog); + } + sqlite3_free(zMsg); +} + +/* +** Return the length of a string omitting trailing whitespace +*/ +static int clipLength(const char *z){ + int n = (int)strlen(z); + while( n>0 && isspace(z[n-1]) ){ n--; } + return n; +} + +/* +** SQL Trace callback +*/ +static void sqlTraceCallback(void *NotUsed1, const char *zSql){ + logMessage("[%.*s]", clipLength(zSql), zSql); +} + +/* +** Prepare an SQL statement. Issue a fatal error if unable. +*/ +static sqlite3_stmt *prepareSql(const char *zFormat, ...){ + va_list ap; + char *zSql; + int rc; + sqlite3_stmt *pStmt = 0; + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + rc = sqlite3_prepare_v2(g.db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + sqlite3_finalize(pStmt); + fatalError("%s\n%s\n", sqlite3_errmsg(g.db), zSql); + } + sqlite3_free(zSql); + return pStmt; +} + +/* +** Run arbitrary SQL. Issue a fatal error on failure. +*/ +static void runSql(const char *zFormat, ...){ + va_list ap; + char *zSql; + int rc; + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + rc = sqlite3_exec(g.db, zSql, 0, 0, 0); + if( rc!=SQLITE_OK ){ + fatalError("%s\n%s\n", sqlite3_errmsg(g.db), zSql); + } + sqlite3_free(zSql); +} + +/* +** Try to run arbitrary SQL. Return success code. +*/ +static int trySql(const char *zFormat, ...){ + va_list ap; + char *zSql; + int rc; + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + rc = sqlite3_exec(g.db, zSql, 0, 0, 0); + sqlite3_free(zSql); + return rc; +} + +/* Structure for holding an arbitrary length string +*/ +typedef struct String String; +struct String { + char *z; /* the string */ + int n; /* Slots of z[] used */ + int nAlloc; /* Slots of z[] allocated */ +}; + +/* Free a string */ +static void stringFree(String *p){ + if( p->z ) sqlite3_free(p->z); + memset(p, 0, sizeof(*p)); +} + +/* Append n bytes of text to a string. If n<0 append the entire string. */ +static void stringAppend(String *p, const char *z, int n){ + if( n<0 ) n = (int)strlen(z); + if( p->n+n>=p->nAlloc ){ + int nAlloc = p->nAlloc*2 + n + 100; + char *z = sqlite3_realloc(p->z, nAlloc); + if( z==0 ) fatalError("out of memory"); + p->z = z; + p->nAlloc = nAlloc; + } + memcpy(p->z+p->n, z, n); + p->n += n; + p->z[p->n] = 0; +} + +/* Reset a string to an empty string */ +static void stringReset(String *p){ + if( p->z==0 ) stringAppend(p, " ", 1); + p->n = 0; + p->z[0] = 0; +} + +/* Append a new token onto the end of the string */ +static void stringAppendTerm(String *p, const char *z){ + int i; + if( p->n ) stringAppend(p, " ", 1); + if( z==0 ){ + stringAppend(p, "nil", 3); + return; + } + for(i=0; z[i] && !isspace(z[i]); i++){} + if( i>0 && z[i]==0 ){ + stringAppend(p, z, i); + return; + } + stringAppend(p, "'", 1); + while( z[0] ){ + for(i=0; z[i] && z[i]!='\''; i++){} + if( z[i] ){ + stringAppend(p, z, i+1); + stringAppend(p, "'", 1); + z += i+1; + }else{ + stringAppend(p, z, i); + break; + } + } + stringAppend(p, "'", 1); +} + +/* +** Callback function for evalSql() +*/ +static int evalCallback(void *pCData, int argc, char **argv, char **azCol){ + String *p = (String*)pCData; + int i; + for(i=0; i0 ){ + pStmt = prepareSql( + "SELECT 1 FROM task WHERE client=%d AND endtime IS NULL", + iClient); + }else{ + pStmt = prepareSql( + "SELECT 1 FROM task WHERE client=%d AND endtime IS NULL", + iClient); + } + while( ((rc = sqlite3_step(pStmt))==SQLITE_BUSY || rc==SQLITE_ROW) + && iTimeout>0 + ){ + sqlite3_reset(pStmt); + sqlite3_sleep(50); + iTimeout -= 50; + } + sqlite3_finalize(pStmt); + if( rc!=SQLITE_DONE ){ + if( zErrPrefix==0 ) zErrPrefix = ""; + if( iClient>0 ){ + errorMessage("%stimeout waiting for client %d", zErrPrefix, iClient); + }else{ + errorMessage("%stimeout waiting for all clients", zErrPrefix); + } + } +} + +/* Maximum number of arguments to a --command */ +#define MX_ARG 5 + +/* +** Run a script. +*/ +static void runScript( + int iClient, /* The client number, or 0 for the master */ + int taskId, /* The task ID for clients. 0 for master */ + char *zScript, /* Text of the script */ + char *zFilename /* File from which script was read. */ +){ + int lineno = 1; + int prevLine = 1; + int ii = 0; + int iBegin = 0; + int n, c, j; + int rc; + int len; + int nArg; + String sResult; + char zCmd[30]; + char zError[1000]; + char azArg[MX_ARG][100]; + unsigned char isRunning[100]; + + memset(isRunning, 0, sizeof(isRunning)); + memset(&sResult, 0, sizeof(sResult)); + stringReset(&sResult); + while( (c = zScript[ii])!=0 ){ + prevLine = lineno; + len = tokenLength(zScript+ii, &lineno); + if( isspace(c) || (c=='/' && zScript[ii+1]=='*') ){ + ii += len; + continue; + } + if( c!='-' || zScript[ii+1]!='-' || !isalpha(zScript[ii+2]) ){ + ii += len; + continue; + } + + /* Run any prior SQL before processing the new --command */ + if( ii>iBegin ){ + char *zSql = sqlite3_mprintf("%.*s", ii-iBegin, zScript+iBegin); + evalSql(&sResult, zSql); + sqlite3_free(zSql); + iBegin = ii + len; + } + + /* Parse the --command */ + if( g.iTrace>=2 ) logMessage("%.*s", len, zScript+ii); + n = extractToken(zScript+ii+2, len-2, zCmd, sizeof(zCmd)); + for(nArg=0; n=len-2 ) break; + n += extractToken(zScript+ii+2+n, len-2-n, + azArg[nArg], sizeof(azArg[nArg])); + } + for(j=nArg; j0 then exit without shutting down + ** SQLite. (In other words, simulate a crash.) + */ + if( strcmp(zCmd, "exit")==0 ){ + int rc = atoi(azArg[0]); + finishScript(taskId); + if( rc==0 ) sqlite3_close(g.db); + exit(rc); + }else + + /* + ** --result + ** + ** Reset accumulated results back to an empty string + */ + if( strcmp(zCmd, "reset")==0 ){ + stringReset(&sResult); + }else + + /* + ** --match ANSWER... + ** + ** Check to see if output matches ANSWER. Report an error if not. + */ + if( strcmp(zCmd, "match")==0 ){ + int jj; + char *zAns = zScript+ii; + for(jj=7; jj=sizeof(isRunning) ){ + errorMessage("line %d of %s: bad client number: %d", + prevLine, zFilename, iNewClient); + goto start_error; + } + if( isRunning[iNewClient] ){ + errorMessage("line %d of %s: client already running: %d", + prevLine, zFilename, iNewClient); + goto start_error; + } + zSys = sqlite3_mprintf( + "%s \"%s\" --client %d --trace %d %s&", + g.argv0, g.zDbFile, iNewClient, g.iTrace, + g.bSqlTrace ? "--sqltrace " : ""); + + system(zSys); + sqlite3_free(zSys); + isRunning[iNewClient] = 1; + start_error: {/* no-op */} + }else + + /* + ** --wait CLIENT TIMEOUT + ** + ** Wait until all tasks complete for the given client. If CLIENT is + ** "all" then wait for all clients to complete. Wait no longer than + ** TIMEOUT milliseconds (default 10,000) + */ + if( strcmp(zCmd, "wait")==0 ){ + int iTimeout = nArg>=2 ? atoi(azArg[1]) : 10000; + sqlite3_snprintf(sizeof(zError),zError,"line %d of %s\n", + prevLine, zFilename); + waitForClient(atoi(azArg[0]), iTimeout, zError); + }else + + /* + ** --task CLIENT + ** + ** --end + ** + ** Assign work to a client. + */ + if( strcmp(zCmd, "task")==0 ){ + int iTarget = atoi(azArg[0]); + int iEnd; + char *zTask; + sqlite3_stmt *pStmt; + iEnd = findEnd(zScript+ii+len, &lineno); + if( iTarget<0 || iTarget>=sizeof(isRunning) + || !isRunning[iTarget] ){ + errorMessage("line %d of %s: client %d is not running", + prevLine, zFilename, iTarget); + goto task_error; + } + zTask = sqlite3_mprintf("%.*s", iEnd, zScript+ii+len); + pStmt = prepareSql("INSERT INTO task(client,script)" + " VALUES(%d,'%q')", iTarget, zTask); + sqlite3_free(zTask); + while( (rc = sqlite3_step(pStmt))==SQLITE_BUSY ){ + sqlite3_reset(pStmt); + sqlite3_sleep(10); + } + sqlite3_finalize(pStmt); + task_error: + iEnd += tokenLength(zScript+ii+len+iEnd, &lineno); + len += iEnd; + iBegin = ii+len; + }else + + /* error */{ + errorMessage("line %d of %s: unknown command --%s", + prevLine, zFilename, zCmd); + } + ii += len; + } + if( iBegin= nArg ) break; + z = azArg[i]; + if( z[0]!='-' ) continue; + z++; + if( z[0]=='-' ){ + if( z[1]==0 ) break; + z++; + } + if( strcmp(z,zOption)==0 ){ + if( hasArg && i==nArg-1 ){ + fatalError("command-line option \"--%s\" requires an argument", z); + } + if( hasArg ){ + zReturn = azArg[i+1]; + }else{ + zReturn = azArg[i]; + } + j = i+1+(hasArg!=0); + while( j0 ){ + if( n>0 ) unrecognizedArguments(argv[0], n, argv+2); + if( g.iTrace ) logMessage("start-client"); + while(1){ + char zTaskName[50]; + rc = startScript(iClient, &zScript, &taskId); + if( rc==SQLITE_DONE ) break; + if( g.iTrace ) logMessage("begin task %d", taskId); + sqlite3_snprintf(sizeof(zTaskName), zTaskName, "client%02d-task-%d", + iClient, taskId); + runScript(iClient, taskId, zScript, zTaskName); + if( g.iTrace ) logMessage("end task %d", taskId); + finishScript(taskId); + sqlite3_sleep(10); + } + if( g.iTrace ) logMessage("end-client"); + }else{ + sqlite3_stmt *pStmt; + if( n==0 ){ + fatalError("missing script filename"); + } + if( n>1 ) unrecognizedArguments(argv[0], n, argv+2); + runSql( + "CREATE TABLE task(\n" + " id INTEGER PRIMARY KEY,\n" + " client INTEGER,\n" + " starttime DATE,\n" + " endtime DATE,\n" + " script TEXT\n" + ");" + "CREATE TABLE clienterror(cnt);\n" + "INSERT INTO clienterror VALUES(0);\n" + ); + zScript = readFile(argv[2]); + if( g.iTrace ) logMessage("begin script [%s]\n", argv[2]); + runScript(0, 0, zScript, argv[2]); + sqlite3_free(zScript); + if( g.iTrace ) logMessage("end script [%s]\n", argv[2]); + waitForClient(0, 2000, "during shutdown...\n"); + while( trySql("CREATE TABLE halt(x);")==SQLITE_BUSY ){ + sqlite3_sleep(10); + } + sqlite3_sleep(100); + pStmt = prepareSql("SELECT cnt FROM clienterror"); + while( (rc = sqlite3_step(pStmt))==SQLITE_BUSY ){ + sqlite3_sleep(10); + } + if( rc==SQLITE_ROW ){ + g.nError += sqlite3_column_int(pStmt, 0); + } + sqlite3_finalize(pStmt); + } + sqlite3_close(g.db); + maybeClose(g.pLog); + maybeClose(g.pErrLog); + if( iClient==0 ){ + if( g.nError ){ + printf("ERRORS: %d\n", g.nError); + }else if( g.iTrace ){ + printf("All OK\n"); + } + } + return g.nError>0; +} From abd6d84a11781229fb3f09a03c45516f1081a4dc Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 6 Apr 2013 11:03:09 +0000 Subject: [PATCH 081/140] Add tests to "permutations.test coverage-pager" to cover uncovered branches. FossilOrigin-Name: 07a0e4e9820a85bf1690214fef3577866c763f92 --- manifest | 16 ++++---- manifest.uuid | 2 +- test/pager1.test | 91 ++++++++++++++++++++++++++++++++++++++++++-- test/pager2.test | 19 ++++++++- test/pagerfault.test | 61 +++++++++++++++++++++++++++++ 5 files changed, 175 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 2b89f591ab..7e72d182cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\scode\scoverage\sof\s"permutations.test\scoverage-pager". -D 2013-04-05T20:40:43.227 +C Add\stests\sto\s"permutations.test\scoverage-pager"\sto\scover\suncovered\sbranches. +D 2013-04-06T11:03:09.144 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -651,10 +651,10 @@ F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 -F test/pager1.test 622bfaea80604a96010f35256a4fc22fce584e95 -F test/pager2.test 745b911dde3d1f24ae0870bd433dfa83d7c658c1 +F test/pager1.test 134144862f6a98f862dc2edda1b10ff499141bfa +F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f -F test/pagerfault.test 636c01bdbdb2bb2baf68532d6c4ef4c633022743 +F test/pagerfault.test bba948be0564254b823ec6ca1fd95fbbfb0839a3 F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 F test/pageropt.test 32cb5a6ed7ccddf8e8c842cb44240bd9340223ce @@ -1044,7 +1044,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a107f75d93b8acd3403d8473a75137378041a833 -R 6d11c98ab18b87c4b8798ffe987d9d38 +P b095e2cdb61ca8487255687f58fb1024d40f3986 +R bbe90127c799120f0deb57d78c345328 U dan -Z 50aa2590d101ae076870fa94f81e976a +Z 0a43fd582a2aa921b5f65df032585318 diff --git a/manifest.uuid b/manifest.uuid index 8dbe787aae..10faadaead 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b095e2cdb61ca8487255687f58fb1024d40f3986 \ No newline at end of file +07a0e4e9820a85bf1690214fef3577866c763f92 \ No newline at end of file diff --git a/test/pager1.test b/test/pager1.test index 22c84d55d0..ab0ec67526 100644 --- a/test/pager1.test +++ b/test/pager1.test @@ -2245,7 +2245,6 @@ do_test pager1-25-1 { } db close } {} -breakpoint do_test pager1-25-2 { faultsim_delete_and_reopen execsql { @@ -2543,7 +2542,6 @@ foreach {tn pragma strsize} { CREATE TABLE t1(a, b); INSERT INTO t1 VALUES(1, 2); } - breakpoint do_execsql_test 34.$tn.2 { BEGIN; INSERT INTO t1 VALUES(2, a_string($strsize)); @@ -2605,14 +2603,21 @@ foreach {tn uri} { do_test 37.$tn { catch { db close } sqlite3_shutdown - sqlite3_config_uri 1 - sqlite3 db $uri + sqlite3_config_uri 1 + sqlite3 db $uri + db eval { CREATE TABLE t1(x); INSERT INTO t1 VALUES(1); SELECT * FROM t1; } } {1} + + do_execsql_test 37.$tn.2 { + VACUUM; + SELECT * FROM t1; + } {1} + db close sqlite3_shutdown sqlite3_config_uri 0 @@ -2730,6 +2735,84 @@ do_test 41.2 { PRAGMA integrity_check; } } {ok} +db close +tv delete +set pending_prev [sqlite3_test_control_pending_byte 0x1000000] +do_test 42.1 { + reset_db + execsql { + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(randomblob(200), randomblob(200)); + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + } + db close + sqlite3_test_control_pending_byte 0x0010000 + sqlite3 db test.db + db eval { PRAGMA mmap_limit = 0 } + catchsql { SELECT sum(length(y)) FROM t1 } +} {1 {database disk image is malformed}} +do_test 42.2 { + reset_db + execsql { + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(randomblob(200), randomblob(200)); + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + } + db close + + testvfs tv -default 1 + tv sectorsize 16384; + tv devchar [list] + sqlite3 db test.db -vfs tv + execsql { UPDATE t1 SET x = randomblob(200) } +} {} +db close +tv delete +sqlite3_test_control_pending_byte $pending_prev + +do_test 43.1 { + reset_db + execsql { + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(1, 2); + CREATE TABLE t2(x, y); + INSERT INTO t2 VALUES(1, 2); + CREATE TABLE t3(x, y); + INSERT INTO t3 VALUES(1, 2); + } + db close + sqlite3 db test.db + + db eval { PRAGMA mmap_limit = 0 } + db eval { SELECT * FROM t1 } + sqlite3_db_status db CACHE_MISS 0 +} {0 2 0} + +do_test 43.2 { + db eval { SELECT * FROM t2 } + sqlite3_db_status db CACHE_MISS 1 +} {0 3 0} + +do_test 43.3 { + db eval { SELECT * FROM t3 } + sqlite3_db_status db CACHE_MISS 0 +} {0 1 0} finish_test + diff --git a/test/pager2.test b/test/pager2.test index fa5f7b9611..0e2b33b833 100644 --- a/test/pager2.test +++ b/test/pager2.test @@ -118,7 +118,6 @@ tv delete #------------------------------------------------------------------------- -# # pager2-2.1: Test a ROLLBACK with journal_mode=off. # pager2-2.2: Test shrinking the database (auto-vacuum) with # journal_mode=off @@ -148,4 +147,22 @@ do_test pager2-2.2 { file size test.db } {3072} +#------------------------------------------------------------------------- +# Test that shared in-memory databases seem to work. +# +db close +do_test pager2-3.1 { + forcedelete test.db + sqlite3_shutdown + sqlite3_config_uri 1 + + sqlite3 db1 {file:test.db?mode=memory&cache=shared} + sqlite3 db2 {file:test.db?mode=memory&cache=shared} + sqlite3 db3 test.db + + db1 eval { CREATE TABLE t1(a, b) } + db2 eval { INSERT INTO t1 VALUES(1, 2) } + list [catch { db3 eval { INSERT INTO t1 VALUES(3, 4) } } msg] $msg +} {1 {no such table: t1}} + finish_test diff --git a/test/pagerfault.test b/test/pagerfault.test index 440909d1a1..9b005c7a03 100644 --- a/test/pagerfault.test +++ b/test/pagerfault.test @@ -1483,6 +1483,67 @@ do_faultsim_test pagerfault-34 -prep { faultsim_test_result {0 {}} } +do_test pagerfault-35-pre { + faultsim_delete_and_reopen + execsql { + CREATE TABLE t1(x PRIMARY KEY, y); + INSERT INTO t1 VALUES(randomblob(200), randomblob(200)); + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + } + faultsim_save_and_close +} {} +testvfs tv -default 1 +tv sectorsize 8192; +tv devchar [list] +do_faultsim_test pagerfault-35 -prep { + faultsim_restore_and_reopen +} -body { + execsql { UPDATE t1 SET x=randomblob(200) } +} -test { + faultsim_test_result {0 {}} +} +catch {db close} +tv delete + +sqlite3_shutdown +sqlite3_config_uri 1 +do_test pagerfault-36-pre { + faultsim_delete_and_reopen + execsql { + CREATE TABLE t1(x PRIMARY KEY, y); + INSERT INTO t1 VALUES(randomblob(200), randomblob(200)); + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + INSERT INTO t1 SELECT randomblob(200), randomblob(200) FROM t1; + } + faultsim_save_and_close +} {} +do_faultsim_test pagerfault-36 -prep { + faultsim_restore + sqlite3 db file:test.db?cache=shared + sqlite3 db2 file:test.db?cache=shared + db2 eval { + BEGIN; + SELECT count(*) FROM sqlite_master; + } + db eval { + PRAGMA cache_size = 1; + BEGIN; + UPDATE t1 SET x = randomblob(200); + } +} -body { + execsql ROLLBACK db +} -test { + catch { db eval {UPDATE t1 SET x = randomblob(200)} } + faultsim_test_result {0 {}} + catch { db close } + catch { db2 close } +} + +sqlite3_shutdown +sqlite3_config_uri 0 finish_test From 3f5bc38037eda131bd0505cd8961e4955d5ce0cf Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 6 Apr 2013 13:09:11 +0000 Subject: [PATCH 082/140] Many improvements to the mptest program. Added a simple test script. FossilOrigin-Name: 07b0401a9b61b1664fc6dcddac3b5969fc0f481a --- manifest | 17 ++-- manifest.uuid | 2 +- mptest/config01.test | 19 ++++ mptest/mptest.c | 194 ++++++++++++++++++++++++--------------- mptest/multiwrite01.test | 141 ++++++++++++++++++++++++++++ 5 files changed, 289 insertions(+), 84 deletions(-) create mode 100644 mptest/config01.test create mode 100644 mptest/multiwrite01.test diff --git a/manifest b/manifest index 0c4454579c..53c2472e9b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sprototype\sTH3-script-style\stest\sharness\sthat\sstarts\smultiple\sprocesses\noperating\son\sthe\ssame\sdatabase\sfile\sat\sthe\ssame\stime. -D 2013-04-06T00:19:37.887 +C Many\simprovements\sto\sthe\smptest\sprogram.\s\sAdded\sa\ssimple\stest\sscript. +D 2013-04-06T13:09:11.141 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,7 +110,9 @@ F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 -F mptest/mptest.c 04110c35f415218e54e030b2e37215ad4adf01f3 +F mptest/config01.test 058a9bc2b0db710d36003ab06dc1618566f27b52 +F mptest/mptest.c d73b294c354719f33086d8d3ebc273553b26073c +F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc @@ -1045,10 +1047,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b095e2cdb61ca8487255687f58fb1024d40f3986 -R c143d7671ed86699bfdf0a0519618d31 -T *branch * mptest -T *sym-mptest * -T -sym-trunk * +P c318fafe686120d7fb8e487eb3bb4942d497665c +R e5c361902695f1e216fb1ed024ee2204 U drh -Z f0aabffa9579562b9fff6a9c66b5a272 +Z 1a794dfe7c097a8f7d97d3bcdba2a731 diff --git a/manifest.uuid b/manifest.uuid index fcf249224b..a5a84885b3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c318fafe686120d7fb8e487eb3bb4942d497665c \ No newline at end of file +07b0401a9b61b1664fc6dcddac3b5969fc0f481a \ No newline at end of file diff --git a/mptest/config01.test b/mptest/config01.test new file mode 100644 index 0000000000..be6b570678 --- /dev/null +++ b/mptest/config01.test @@ -0,0 +1,19 @@ +/* +** Configure five tasks in different ways, then run tests. +*/ +PRAGMA page_size=8192; +--task 1 + PRAGMA journal_mode=PERSIST; + PRAGMA mmap_limit=0; +--end +--task 2 + PRAGMA journal_mode=TRUNCATE; + PRAGMA mmap_limit=28672; +--end +--task 3 + PRAGMA journal_mode=MEMORY; +--end +--task 4 + PRAGMA journal_mode=OFF; +--end +--source multiwrite01.test diff --git a/mptest/mptest.c b/mptest/mptest.c index 01660af690..e07e974de3 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -57,8 +57,13 @@ static struct Global { int iTrace; /* Tracing level */ int bSqlTrace; /* True to trace SQL commands */ int nError; /* Number of errors */ + int nTest; /* Number of --match operators */ + int iTimeout; /* Milliseconds until a busy timeout */ } g; +/* Default timeout */ +#define DEFAULT_TIMEOUT 10000 + /* ** Print a message adding zPrefix[] to the beginning of every line. */ @@ -226,7 +231,9 @@ static void fatalError(const char *zFormat, ...){ sqlite3_free(zMsg); if( g.db ){ int nTry = 0; - while( trySql("CREATE TABLE halt(x);")==SQLITE_BUSY && (nTry++)<100 ){ + g.iTimeout = 0; + while( trySql("UPDATE client SET wantHalt=1;")==SQLITE_BUSY + && (nTry++)<100 ){ sqlite3_sleep(10); } } @@ -262,6 +269,18 @@ static int clipLength(const char *z){ return n; } +/* +** Busy handler with a g.iTimeout-millisecond timeout +*/ +static int busyHandler(void *pCD, int count){ + if( count*10>g.iTimeout ){ + if( g.iTimeout>0 ) errorMessage("timeout after %dms", g.iTimeout); + return 0; + } + sqlite3_sleep(10); + return 1; +} + /* ** SQL Trace callback */ @@ -408,6 +427,7 @@ static int evalSql(String *p, const char *zFormat, ...){ va_start(ap, zFormat); zSql = sqlite3_vmprintf(zFormat, ap); va_end(ap); + assert( g.iTimeout>0 ); rc = sqlite3_exec(g.db, zSql, evalCallback, p, &zErrMsg); sqlite3_free(zSql); if( rc ){ @@ -435,21 +455,34 @@ static int startScript( sqlite3_stmt *pStmt = 0; int taskId; int rc; + int totalTime = 0; *pzScript = 0; + g.iTimeout = 0; while(1){ - if( trySql("SELECT * FROM halt")==SQLITE_OK ) return SQLITE_DONE; rc = trySql("BEGIN IMMEDIATE"); if( rc==SQLITE_BUSY ){ sqlite3_sleep(10); + totalTime += 10; continue; } if( rc!=SQLITE_OK ){ fatalError("%s\nBEGIN IMMEDIATE", sqlite3_errmsg(g.db)); } - if( g.nError ){ - runSql("UPDATE clienterror SET cnt=cnt+%d", g.nError); + if( g.nError || g.nTest ){ + runSql("UPDATE counters SET nError=nError+%d, nTest=nTest+%d", + g.nError, g.nTest); g.nError = 0; + g.nTest = 0; + } + pStmt = prepareSql("SELECT 1 FROM client WHERE id=%d AND wantHalt",iClient); + rc = sqlite3_step(pStmt); + sqlite3_finalize(pStmt); + if( rc==SQLITE_ROW ){ + runSql("DELETE FROM client WHERE id=%d", iClient); + runSql("COMMIT"); + g.iTimeout = DEFAULT_TIMEOUT; + return SQLITE_DONE; } pStmt = prepareSql( "SELECT script, id FROM task" @@ -466,37 +499,58 @@ static int startScript( " SET starttime=strftime('%%Y-%%m-%%d %%H:%%M:%%f','now')" " WHERE id=%d;", taskId); runSql("COMMIT;"); + g.iTimeout = DEFAULT_TIMEOUT; return SQLITE_OK; } sqlite3_finalize(pStmt); if( rc==SQLITE_DONE ){ + if( totalTime>30000 ){ + errorMessage("Waited over 30 seconds with no work. Giving up."); + runSql("DELETE FROM client WHERE id=%d; COMMIT;", iClient); + sqlite3_close(g.db); + exit(1); + } runSql("COMMIT;"); sqlite3_sleep(100); + totalTime += 100; continue; } fatalError("%s", sqlite3_errmsg(g.db)); } + g.iTimeout = DEFAULT_TIMEOUT; } /* -** Mark a script as having finished. +** Mark a script as having finished. Remove the CLIENT table entry +** if bShutdown is true. */ -static int finishScript(int taskId){ - sqlite3_stmt *pStmt; - int rc; - pStmt = prepareSql( - "UPDATE task" - " SET endtime=strftime('%%Y-%%m-%%d %%H:%%M:%%f','now')" - " WHERE id=%d;", taskId); - do{ - rc = sqlite3_step(pStmt); - sqlite3_reset(pStmt); - }while( rc==SQLITE_BUSY ); - sqlite3_finalize(pStmt); - if( rc!=SQLITE_DONE ){ - fatalError("%s\n", sqlite3_errmsg(g.db)); +static int finishScript(int iClient, int taskId, int bShutdown){ + runSql("UPDATE task" + " SET endtime=strftime('%%Y-%%m-%%d %%H:%%M:%%f','now')" + " WHERE id=%d;", taskId); + if( bShutdown ){ + runSql("DELETE FROM client WHERE id=%d", iClient); + } + return SQLITE_OK; +} + +/* +** Start up a client process for iClient, if it is not already +** running. If the client is already running, then this routine +** is a no-op. +*/ +static void startClient(int iClient){ + runSql("INSERT OR IGNORE INTO client VALUES(%d,0)", iClient); + if( sqlite3_changes(g.db) ){ + char *zSys; + zSys = sqlite3_mprintf( + "%s \"%s\" --client %d --trace %d %s&", + g.argv0, g.zDbFile, iClient, g.iTrace, + g.bSqlTrace ? "--sqltrace " : ""); + + system(zSys); + sqlite3_free(zSys); } - return rc; } /* @@ -597,13 +651,18 @@ static void waitForClient(int iClient, int iTimeout, char *zErrPrefix){ int rc; if( iClient>0 ){ pStmt = prepareSql( - "SELECT 1 FROM task WHERE client=%d AND endtime IS NULL", + "SELECT 1 FROM task" + " WHERE client=%d" + " AND client IN (SELECT id FROM client)" + " AND endtime IS NULL", iClient); }else{ pStmt = prepareSql( - "SELECT 1 FROM task WHERE client=%d AND endtime IS NULL", - iClient); + "SELECT 1 FROM task" + " WHERE client IN (SELECT id FROM client)" + " AND endtime IS NULL"); } + g.iTimeout = 0; while( ((rc = sqlite3_step(pStmt))==SQLITE_BUSY || rc==SQLITE_ROW) && iTimeout>0 ){ @@ -612,6 +671,7 @@ static void waitForClient(int iClient, int iTimeout, char *zErrPrefix){ iTimeout -= 50; } sqlite3_finalize(pStmt); + g.iTimeout = DEFAULT_TIMEOUT; if( rc!=SQLITE_DONE ){ if( zErrPrefix==0 ) zErrPrefix = ""; if( iClient>0 ){ @@ -639,16 +699,13 @@ static void runScript( int ii = 0; int iBegin = 0; int n, c, j; - int rc; int len; int nArg; String sResult; char zCmd[30]; char zError[1000]; char azArg[MX_ARG][100]; - unsigned char isRunning[100]; - memset(isRunning, 0, sizeof(isRunning)); memset(&sResult, 0, sizeof(sResult)); stringReset(&sResult); while( (c = zScript[ii])!=0 ){ @@ -699,7 +756,7 @@ static void runScript( */ if( strcmp(zCmd, "exit")==0 ){ int rc = atoi(azArg[0]); - finishScript(taskId); + finishScript(iClient, taskId, 1); if( rc==0 ) sqlite3_close(g.db); exit(rc); }else @@ -727,6 +784,7 @@ static void runScript( errorMessage("line %d of %s:\nExpected [%.*s]\n Got [%s]", prevLine, zFilename, len-jj-1, zAns, sResult.z); } + g.nTest++; stringReset(&sResult); }else @@ -736,7 +794,7 @@ static void runScript( ** Run a subscript from a separate file. */ if( strcmp(zCmd, "source")==0 ){ - char *zNewFile = azArg[1]; + char *zNewFile = azArg[0]; char *zNewScript = readFile(zNewFile); if( g.iTrace ) logMessage("begin script [%s]\n", zNewFile); runScript(0, 0, zNewScript, zNewFile); @@ -762,26 +820,9 @@ static void runScript( */ if( strcmp(zCmd, "start")==0 ){ int iNewClient = atoi(azArg[0]); - char *zSys; - if( iNewClient<1 || iNewClient>=sizeof(isRunning) ){ - errorMessage("line %d of %s: bad client number: %d", - prevLine, zFilename, iNewClient); - goto start_error; + if( iNewClient>0 ){ + startClient(iNewClient); } - if( isRunning[iNewClient] ){ - errorMessage("line %d of %s: client already running: %d", - prevLine, zFilename, iNewClient); - goto start_error; - } - zSys = sqlite3_mprintf( - "%s \"%s\" --client %d --trace %d %s&", - g.argv0, g.zDbFile, iNewClient, g.iTrace, - g.bSqlTrace ? "--sqltrace " : ""); - - system(zSys); - sqlite3_free(zSys); - isRunning[iNewClient] = 1; - start_error: {/* no-op */} }else /* @@ -803,30 +844,24 @@ static void runScript( ** ** --end ** - ** Assign work to a client. + ** Assign work to a client. Start the client if it is not running + ** already. */ if( strcmp(zCmd, "task")==0 ){ int iTarget = atoi(azArg[0]); int iEnd; char *zTask; - sqlite3_stmt *pStmt; iEnd = findEnd(zScript+ii+len, &lineno); - if( iTarget<0 || iTarget>=sizeof(isRunning) - || !isRunning[iTarget] ){ - errorMessage("line %d of %s: client %d is not running", + if( iTarget<0 ){ + errorMessage("line %d of %s: bad client number: %d", prevLine, zFilename, iTarget); - goto task_error; + }else{ + zTask = sqlite3_mprintf("%.*s", iEnd, zScript+ii+len); + startClient(iTarget); + runSql("INSERT INTO task(client,script)" + " VALUES(%d,'%q')", iTarget, zTask); + sqlite3_free(zTask); } - zTask = sqlite3_mprintf("%.*s", iEnd, zScript+ii+len); - pStmt = prepareSql("INSERT INTO task(client,script)" - " VALUES(%d,'%q')", iTarget, zTask); - sqlite3_free(zTask); - while( (rc = sqlite3_step(pStmt))==SQLITE_BUSY ){ - sqlite3_reset(pStmt); - sqlite3_sleep(10); - } - sqlite3_finalize(pStmt); - task_error: iEnd += tokenLength(zScript+ii+len+iEnd, &lineno); len += iEnd; iBegin = ii+len; @@ -964,6 +999,8 @@ int main(int argc, char **argv){ } rc = sqlite3_open_v2(g.zDbFile, &g.db, openFlags, g.zVfs); if( rc ) fatalError("cannot open [%s]", g.zDbFile); + sqlite3_busy_handler(g.db, busyHandler, 0); + g.iTimeout = DEFAULT_TIMEOUT; if( g.bSqlTrace ) sqlite3_trace(g.db, sqlTraceCallback, 0); if( iClient>0 ){ if( n>0 ) unrecognizedArguments(argv[0], n, argv+2); @@ -977,12 +1014,13 @@ int main(int argc, char **argv){ iClient, taskId); runScript(iClient, taskId, zScript, zTaskName); if( g.iTrace ) logMessage("end task %d", taskId); - finishScript(taskId); + finishScript(iClient, taskId, 0); sqlite3_sleep(10); } if( g.iTrace ) logMessage("end-client"); }else{ sqlite3_stmt *pStmt; + int iTimeout; if( n==0 ){ fatalError("missing script filename"); } @@ -995,8 +1033,11 @@ int main(int argc, char **argv){ " endtime DATE,\n" " script TEXT\n" ");" - "CREATE TABLE clienterror(cnt);\n" - "INSERT INTO clienterror VALUES(0);\n" + "CREATE INDEX task_i1 ON task(client, starttime);\n" + "CREATE INDEX task_i2 ON task(client, endtime);\n" + "CREATE TABLE counters(nError,nTest);\n" + "INSERT INTO counters VALUES(0,0);\n" + "CREATE TABLE client(id INTEGER PRIMARY KEY, wantHalt);\n" ); zScript = readFile(argv[2]); if( g.iTrace ) logMessage("begin script [%s]\n", argv[2]); @@ -1004,16 +1045,25 @@ int main(int argc, char **argv){ sqlite3_free(zScript); if( g.iTrace ) logMessage("end script [%s]\n", argv[2]); waitForClient(0, 2000, "during shutdown...\n"); - while( trySql("CREATE TABLE halt(x);")==SQLITE_BUSY ){ + trySql("UPDATE client SET wantHalt=1"); + sqlite3_sleep(10); + g.iTimeout = 0; + iTimeout = 1000; + while( ((rc = trySql("SELECT 1 FROM client"))==SQLITE_BUSY + || rc==SQLITE_ROW) && iTimeout>0 ){ sqlite3_sleep(10); + iTimeout -= 10; } sqlite3_sleep(100); - pStmt = prepareSql("SELECT cnt FROM clienterror"); - while( (rc = sqlite3_step(pStmt))==SQLITE_BUSY ){ + pStmt = prepareSql("SELECT nError, nTest FROM counters"); + iTimeout = 1000; + while( (rc = sqlite3_step(pStmt))==SQLITE_BUSY && iTimeout>0 ){ sqlite3_sleep(10); + iTimeout -= 10; } if( rc==SQLITE_ROW ){ g.nError += sqlite3_column_int(pStmt, 0); + g.nTest += sqlite3_column_int(pStmt, 1); } sqlite3_finalize(pStmt); } @@ -1021,11 +1071,7 @@ int main(int argc, char **argv){ maybeClose(g.pLog); maybeClose(g.pErrLog); if( iClient==0 ){ - if( g.nError ){ - printf("ERRORS: %d\n", g.nError); - }else if( g.iTrace ){ - printf("All OK\n"); - } + printf("Summary: %d errors in %d tests\n", g.nError, g.nTest); } return g.nError>0; } diff --git a/mptest/multiwrite01.test b/mptest/multiwrite01.test new file mode 100644 index 0000000000..dc71884047 --- /dev/null +++ b/mptest/multiwrite01.test @@ -0,0 +1,141 @@ +/* +** This script sets up five different tasks all writing and updating +** the database at the same time, but each in its own table. +*/ +--task 1 + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + --sleep 1 + INSERT INTO t1 VALUES(1, randomblob(2000)); + INSERT INTO t1 VALUES(2, randomblob(1000)); + --sleep 1 + INSERT INTO t1 SELECT a+2, randomblob(1500) FROM t1; + INSERT INTO t1 SELECT a+4, randomblob(1500) FROM t1; + INSERT INTO t1 SELECT a+8, randomblob(1500) FROM t1; + --sleep 1 + INSERT INTO t1 SELECT a+16, randomblob(1500) FROM t1; + --sleep 1 + INSERT INTO t1 SELECT a+32, randomblob(1500) FROM t1; + SELECT count(*) FROM t1; + --match 64 + SELECT avg(length(b)) FROM t1; + --match 1500.0 + --sleep 2 + UPDATE t1 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t1; + --match 247 +--end + + +--task 2 + DROP TABLE IF EXISTS t2; + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + --sleep 1 + INSERT INTO t2 VALUES(1, randomblob(2000)); + INSERT INTO t2 VALUES(2, randomblob(1000)); + --sleep 1 + INSERT INTO t2 SELECT a+2, randomblob(1500) FROM t2; + INSERT INTO t2 SELECT a+4, randomblob(1500) FROM t2; + INSERT INTO t2 SELECT a+8, randomblob(1500) FROM t2; + --sleep 1 + INSERT INTO t2 SELECT a+16, randomblob(1500) FROM t2; + --sleep 1 + INSERT INTO t2 SELECT a+32, randomblob(1500) FROM t2; + SELECT count(*) FROM t2; + --match 64 + SELECT avg(length(b)) FROM t2; + --match 1500.0 + --sleep 2 + UPDATE t2 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t2; + --match 247 +--end + +--task 3 + DROP TABLE IF EXISTS t3; + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + --sleep 1 + INSERT INTO t3 VALUES(1, randomblob(2000)); + INSERT INTO t3 VALUES(2, randomblob(1000)); + --sleep 1 + INSERT INTO t3 SELECT a+2, randomblob(1500) FROM t3; + INSERT INTO t3 SELECT a+4, randomblob(1500) FROM t3; + INSERT INTO t3 SELECT a+8, randomblob(1500) FROM t3; + --sleep 1 + INSERT INTO t3 SELECT a+16, randomblob(1500) FROM t3; + --sleep 1 + INSERT INTO t3 SELECT a+32, randomblob(1500) FROM t3; + SELECT count(*) FROM t3; + --match 64 + SELECT avg(length(b)) FROM t3; + --match 1500.0 + --sleep 2 + UPDATE t3 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t3; + --match 247 +--end + +--task 4 + DROP TABLE IF EXISTS t4; + CREATE TABLE t4(a INTEGER PRIMARY KEY, b); + --sleep 1 + INSERT INTO t4 VALUES(1, randomblob(2000)); + INSERT INTO t4 VALUES(2, randomblob(1000)); + --sleep 1 + INSERT INTO t4 SELECT a+2, randomblob(1500) FROM t4; + INSERT INTO t4 SELECT a+4, randomblob(1500) FROM t4; + INSERT INTO t4 SELECT a+8, randomblob(1500) FROM t4; + --sleep 1 + INSERT INTO t4 SELECT a+16, randomblob(1500) FROM t4; + --sleep 1 + INSERT INTO t4 SELECT a+32, randomblob(1500) FROM t4; + SELECT count(*) FROM t4; + --match 64 + SELECT avg(length(b)) FROM t4; + --match 1500.0 + --sleep 2 + UPDATE t4 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t4; + --match 247 +--end + +--task 5 + DROP TABLE IF EXISTS t5; + CREATE TABLE t5(a INTEGER PRIMARY KEY, b); + --sleep 1 + INSERT INTO t5 VALUES(1, randomblob(2000)); + INSERT INTO t5 VALUES(2, randomblob(1000)); + --sleep 1 + INSERT INTO t5 SELECT a+2, randomblob(1500) FROM t5; + INSERT INTO t5 SELECT a+4, randomblob(1500) FROM t5; + INSERT INTO t5 SELECT a+8, randomblob(1500) FROM t5; + --sleep 1 + INSERT INTO t5 SELECT a+16, randomblob(1500) FROM t5; + --sleep 1 + INSERT INTO t5 SELECT a+32, randomblob(1500) FROM t5; + SELECT count(*) FROM t5; + --match 64 + SELECT avg(length(b)) FROM t5; + --match 1500.0 + --sleep 2 + UPDATE t5 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t5; + --match 247 +--end + +--wait all +SELECT count(*), total(length(b)) FROM t1; +--match 64 247 +SELECT count(*), total(length(b)) FROM t2; +--match 64 247 +SELECT count(*), total(length(b)) FROM t3; +--match 64 247 +SELECT count(*), total(length(b)) FROM t4; +--match 64 247 +SELECT count(*), total(length(b)) FROM t5; +--match 64 247 +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +DROP TABLE t4; +DROP TABLE t5; From f012ae07593dfe0006e579200f27e4f53f420d5e Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 6 Apr 2013 14:04:22 +0000 Subject: [PATCH 083/140] Get the mptest program running on windows. FossilOrigin-Name: 3966b33284355325afda627ed6f4e26f967438f8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 24 +++++++++++++++++++++++- 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 53c2472e9b..df3387a104 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Many\simprovements\sto\sthe\smptest\sprogram.\s\sAdded\sa\ssimple\stest\sscript. -D 2013-04-06T13:09:11.141 +C Get\sthe\smptest\sprogram\srunning\son\swindows. +D 2013-04-06T14:04:22.154 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -111,7 +111,7 @@ F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 058a9bc2b0db710d36003ab06dc1618566f27b52 -F mptest/mptest.c d73b294c354719f33086d8d3ebc273553b26073c +F mptest/mptest.c 7dcc7afacc87be958c181f36a8494ef145ebd131 F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1047,7 +1047,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c318fafe686120d7fb8e487eb3bb4942d497665c -R e5c361902695f1e216fb1ed024ee2204 +P 07b0401a9b61b1664fc6dcddac3b5969fc0f481a +R 61768559c8a66440452ca26012ff3963 U drh -Z 1a794dfe7c097a8f7d97d3bcdba2a731 +Z 2d9f9bfb3324d80756ef1485b1717ef9 diff --git a/manifest.uuid b/manifest.uuid index a5a84885b3..6a4ccb2841 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07b0401a9b61b1664fc6dcddac3b5969fc0f481a \ No newline at end of file +3966b33284355325afda627ed6f4e26f967438f8 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index e07e974de3..ae77770380 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -22,6 +22,7 @@ ** Recommended options: ** ** -DHAVE_USLEEP +** -DSQLITE_MAX_SCHEMA_RETRY=100 ** -DSQLITE_NO_SYNC ** -DSQLITE_THREADSAFE=0 ** -DSQLITE_OMIT_LOAD_EXTENSION @@ -542,14 +543,35 @@ static int finishScript(int iClient, int taskId, int bShutdown){ static void startClient(int iClient){ runSql("INSERT OR IGNORE INTO client VALUES(%d,0)", iClient); if( sqlite3_changes(g.db) ){ +#if defined(__unix__) char *zSys; zSys = sqlite3_mprintf( "%s \"%s\" --client %d --trace %d %s&", g.argv0, g.zDbFile, iClient, g.iTrace, g.bSqlTrace ? "--sqltrace " : ""); - system(zSys); sqlite3_free(zSys); +#endif +#if defined(_WIN32) + char *argv[10]; + char zClient[20]; + char zTrace[20]; + argv[0] = g.argv0; + argv[1] = g.zDbFile; + argv[2] = "--client"; + sqlite3_snprintf(sizeof(zClient),zClient,"%d",iClient); + argv[3] = zClient; + argv[4] = "--trace"; + sqlite3_snprintf(sizeof(zTrace),zTrace,"%d",g.iTrace); + argv[5] = zTrace; + if( g.bSqlTrace ){ + argv[6] = "--sqltrace"; + argv[7] = 0; + }else{ + argv[6] = 0; + } + _spawnv(_P_NOWAIT, g.argv0, argv); +#endif } } From 896be7d93ac2f92443e40e9015235f224616a4f5 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 6 Apr 2013 14:16:41 +0000 Subject: [PATCH 084/140] Get the build of mptest working on mac. FossilOrigin-Name: a0bae686c9d7f08afac00f66067f5147be6231e0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index df3387a104..5709059ad5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sthe\smptest\sprogram\srunning\son\swindows. -D 2013-04-06T14:04:22.154 +C Get\sthe\sbuild\sof\smptest\sworking\son\smac. +D 2013-04-06T14:16:41.777 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -111,7 +111,7 @@ F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 058a9bc2b0db710d36003ab06dc1618566f27b52 -F mptest/mptest.c 7dcc7afacc87be958c181f36a8494ef145ebd131 +F mptest/mptest.c 5d48c89704556ccf01602627fb26b9ec920ae0a6 F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1047,7 +1047,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 07b0401a9b61b1664fc6dcddac3b5969fc0f481a -R 61768559c8a66440452ca26012ff3963 +P 3966b33284355325afda627ed6f4e26f967438f8 +R 8058bb54d4c385606834ce76ac0c79c1 U drh -Z 2d9f9bfb3324d80756ef1485b1717ef9 +Z 71f79ad69c84c0cc8a1fd074e63ccdd7 diff --git a/manifest.uuid b/manifest.uuid index 6a4ccb2841..b1f322192d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3966b33284355325afda627ed6f4e26f967438f8 \ No newline at end of file +a0bae686c9d7f08afac00f66067f5147be6231e0 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index ae77770380..517bb2e302 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -543,7 +543,7 @@ static int finishScript(int iClient, int taskId, int bShutdown){ static void startClient(int iClient){ runSql("INSERT OR IGNORE INTO client VALUES(%d,0)", iClient); if( sqlite3_changes(g.db) ){ -#if defined(__unix__) +#if !defined(_WIN32) char *zSys; zSys = sqlite3_mprintf( "%s \"%s\" --client %d --trace %d %s&", From 1790bb36c57b7ba86477a6b2df097b80df279202 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 6 Apr 2013 14:30:29 +0000 Subject: [PATCH 085/140] Add SQLITE_CONFIG_LOG error logging to the mptest program. FossilOrigin-Name: 716c25bd125c8934499c1fb59a60d7e844c0691c --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 9 +++++++++ 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 5709059ad5..e1cc87dabd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sthe\sbuild\sof\smptest\sworking\son\smac. -D 2013-04-06T14:16:41.777 +C Add\sSQLITE_CONFIG_LOG\serror\slogging\sto\sthe\smptest\sprogram. +D 2013-04-06T14:30:29.439 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -111,7 +111,7 @@ F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 058a9bc2b0db710d36003ab06dc1618566f27b52 -F mptest/mptest.c 5d48c89704556ccf01602627fb26b9ec920ae0a6 +F mptest/mptest.c 4f1a7ad2b9d089fb1bfbbac8cd9f6fe840575f10 F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1047,7 +1047,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3966b33284355325afda627ed6f4e26f967438f8 -R 8058bb54d4c385606834ce76ac0c79c1 +P a0bae686c9d7f08afac00f66067f5147be6231e0 +R 861169f81b752fa53a7328f523d2af55 U drh -Z 71f79ad69c84c0cc8a1fd074e63ccdd7 +Z 4d9e999545d1ee7e59c6e7a0f5ed5b2c diff --git a/manifest.uuid b/manifest.uuid index b1f322192d..f6ccd0c84f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a0bae686c9d7f08afac00f66067f5147be6231e0 \ No newline at end of file +716c25bd125c8934499c1fb59a60d7e844c0691c \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 517bb2e302..768a73ed72 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -289,6 +289,14 @@ static void sqlTraceCallback(void *NotUsed1, const char *zSql){ logMessage("[%.*s]", clipLength(zSql), zSql); } +/* +** SQL error log callback +*/ +static void sqlErrorCallback(void *pArg, int iErrCode, const char *zMsg){ + if( (iErrCode&0xff)==SQLITE_SCHEMA && g.iTrace<3 ) return; + errorMessage("(errcode=%d) %s", iErrCode, zMsg); +} + /* ** Prepare an SQL statement. Issue a fatal error if unable. */ @@ -1010,6 +1018,7 @@ int main(int argc, char **argv){ }else{ g.pLog = stdout; } + sqlite3_config(SQLITE_CONFIG_LOG, sqlErrorCallback, 0); if( zClient ){ iClient = atoi(zClient); if( iClient<1 ) fatalError("illegal client number: %d\n", iClient); From 60625313b04bd9dc09c8f770095b9b02b24a407c Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 6 Apr 2013 18:06:51 +0000 Subject: [PATCH 086/140] Increase the default SQLITE_MAX_SCHEMA_RETRY to 50. Make sure that macro covers every case where a prepared statement might need to be reprepared due to a schema change. The sqlite3_exec() interface now uses sqlite3_prepare_v2(). FossilOrigin-Name: c1d7304c80c4a6244c8a9f6fad1eebd0f339c724 --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/legacy.c | 12 ++++-------- src/prepare.c | 1 - src/vdbeInt.h | 8 ++++++++ src/vdbeapi.c | 8 -------- src/vdbeblob.c | 2 +- 7 files changed, 25 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index 7e72d182cd..5497db1fb1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sto\s"permutations.test\scoverage-pager"\sto\scover\suncovered\sbranches. -D 2013-04-06T11:03:09.144 +C Increase\sthe\sdefault\sSQLITE_MAX_SCHEMA_RETRY\sto\s50.\s\sMake\ssure\sthat\smacro\ncovers\severy\scase\swhere\sa\sprepared\sstatement\smight\sneed\sto\sbe\sreprepared\sdue\nto\sa\sschema\schange.\s\sThe\ssqlite3_exec()\sinterface\snow\suses\s\nsqlite3_prepare_v2(). +D 2013-04-06T18:06:51.485 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -140,7 +140,7 @@ F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 F src/insert.c f7cb141e8ce257cb6b15c497f09e4e23d6055599 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d -F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f +F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 1422eba4aa2b1fb5f7b3aef574752272477d21e2 F src/main.c 54a841854734b6731c4d026834788cac6a19f3d1 @@ -169,7 +169,7 @@ F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 F src/pragma.c 682e97f3e3b77fd6c9b569eabfbf4a14c987aca3 -F src/prepare.c 310eaff1ee5f3c700b3545afb095cfe9346efc3a +F src/prepare.c 743e484233c51109666d402f470523553b41797c F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 @@ -241,10 +241,10 @@ F src/util.c f566b5138099a2df8533b190d0dcc74b7dfbe0c9 F src/vacuum.c 2727bdd08847fcb6b2d2da6d14f018910e8645d3 F src/vdbe.c 1d56617d816098a41bdbe73536a67dcdefad52bd F src/vdbe.h b52887278cb173e66188da84dfab216bea61119d -F src/vdbeInt.h 3bb8531d6768379f086e1576b31d52331f6eb982 -F src/vdbeapi.c 56f9ad6329d093f54b5dd321a2b4a643f89a6cb6 +F src/vdbeInt.h c1e830268b75f04a2901dd895b51a637a26c7045 +F src/vdbeapi.c 085cf9bf169b859a6c8fa43791702bac805cb7aa F src/vdbeaux.c ecb43014bcd3019e5aa2b5561e5c3a447f007a08 -F src/vdbeblob.c 32f2a4899d67f69634ea4dd93e3f651936d732cb +F src/vdbeblob.c 5dc79627775bd9a9b494dd956e26297946417d69 F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c 4fad64071ae120c25f39dcac572d716b9cadeb7f F src/vdbetrace.c a22263ab47f6ba4fcd176515cec1e732866b25f0 @@ -1044,7 +1044,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b095e2cdb61ca8487255687f58fb1024d40f3986 -R bbe90127c799120f0deb57d78c345328 -U dan -Z 0a43fd582a2aa921b5f65df032585318 +P 07a0e4e9820a85bf1690214fef3577866c763f92 +R 2fd4d54ffecba80bddc0b0bfdd8ef6ea +U drh +Z d5343d886a406d77dc84001af2f5f17d diff --git a/manifest.uuid b/manifest.uuid index 10faadaead..85d44ed73b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07a0e4e9820a85bf1690214fef3577866c763f92 \ No newline at end of file +c1d7304c80c4a6244c8a9f6fad1eebd0f339c724 \ No newline at end of file diff --git a/src/legacy.c b/src/legacy.c index ebab2de37d..94649ae705 100644 --- a/src/legacy.c +++ b/src/legacy.c @@ -38,7 +38,6 @@ int sqlite3_exec( const char *zLeftover; /* Tail of unprocessed SQL */ sqlite3_stmt *pStmt = 0; /* The current SQL statement */ char **azCols = 0; /* Names of result columns */ - int nRetry = 0; /* Number of retry attempts */ int callbackIsInit; /* True if callback data is initialized */ if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; @@ -46,12 +45,12 @@ int sqlite3_exec( sqlite3_mutex_enter(db->mutex); sqlite3Error(db, SQLITE_OK, 0); - while( (rc==SQLITE_OK || (rc==SQLITE_SCHEMA && (++nRetry)<2)) && zSql[0] ){ + while( rc==SQLITE_OK && zSql[0] ){ int nCol; char **azVals = 0; pStmt = 0; - rc = sqlite3_prepare(db, zSql, -1, &pStmt, &zLeftover); + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover); assert( rc==SQLITE_OK || pStmt==0 ); if( rc!=SQLITE_OK ){ continue; @@ -108,11 +107,8 @@ int sqlite3_exec( if( rc!=SQLITE_ROW ){ rc = sqlite3VdbeFinalize((Vdbe *)pStmt); pStmt = 0; - if( rc!=SQLITE_SCHEMA ){ - nRetry = 0; - zSql = zLeftover; - while( sqlite3Isspace(zSql[0]) ) zSql++; - } + zSql = zLeftover; + while( sqlite3Isspace(zSql[0]) ) zSql++; break; } } diff --git a/src/prepare.c b/src/prepare.c index 26d6c2614f..d78d83cbd8 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -654,7 +654,6 @@ static int sqlite3Prepare( } #endif - assert( db->init.busy==0 || saveSqlFlag==0 ); if( db->init.busy==0 ){ Vdbe *pVdbe = pParse->pVdbe; sqlite3VdbeSetSql(pVdbe, zSql, (int)(pParse->zTail-zSql), saveSqlFlag); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 2372b83849..3a5b4028bb 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -18,6 +18,14 @@ #ifndef _VDBEINT_H_ #define _VDBEINT_H_ +/* +** The maximum number of times that a statement will try to reparse +** itself before giving up and returning SQLITE_SCHEMA. +*/ +#ifndef SQLITE_MAX_SCHEMA_RETRY +# define SQLITE_MAX_SCHEMA_RETRY 50 +#endif + /* ** SQL is translated into a sequence of instructions to be ** executed by a virtual machine. Each instruction is an instance diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 2387438356..7c861e2d47 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -453,14 +453,6 @@ end_of_step: return (rc&db->errMask); } -/* -** The maximum number of times that a statement will try to reparse -** itself before giving up and returning SQLITE_SCHEMA. -*/ -#ifndef SQLITE_MAX_SCHEMA_RETRY -# define SQLITE_MAX_SCHEMA_RETRY 5 -#endif - /* ** This is the top-level implementation of sqlite3_step(). Call ** sqlite3Step() to do most of the work. If a schema error occurs, diff --git a/src/vdbeblob.c b/src/vdbeblob.c index ae77a47ba3..2e8fd8ee74 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -313,7 +313,7 @@ int sqlite3_blob_open( } sqlite3_bind_int64(pBlob->pStmt, 1, iRow); rc = blobSeekToRow(pBlob, iRow, &zErr); - } while( (++nAttempt)<5 && rc==SQLITE_SCHEMA ); + } while( (++nAttempt)mallocFailed==0 ){ From e348fc7fe8433fe5f8dcb36317cf0e3661bb7f19 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 6 Apr 2013 18:35:07 +0000 Subject: [PATCH 087/140] Fix the --source command so that its argument is relative to the script. Verify that the SQLite header and library match. Print SQLite version and compile-time configuration information on startup. FossilOrigin-Name: 15cb0db7583c3a24cbea0c72576047a93fba0801 --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 33 +++++++++++++++++++++++++++++---- 3 files changed, 36 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index bfff919823..36209600bd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sthe\smptest\sbranch.\s\sThe\smptest\sneeds\sthe\nSQLITE_MAX_SCHEMA_RETRY\sfix\sin\sparticular. -D 2013-04-06T18:10:27.166 +C Fix\sthe\s--source\scommand\sso\sthat\sits\sargument\sis\srelative\sto\sthe\sscript.\nVerify\sthat\sthe\sSQLite\sheader\sand\slibrary\smatch.\s\sPrint\sSQLite\sversion\nand\scompile-time\sconfiguration\sinformation\son\sstartup. +D 2013-04-06T18:35:07.098 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -111,7 +111,7 @@ F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 058a9bc2b0db710d36003ab06dc1618566f27b52 -F mptest/mptest.c 4f1a7ad2b9d089fb1bfbbac8cd9f6fe840575f10 +F mptest/mptest.c 6e23dade7b4cde4801b7a6df29f52a492d82d450 F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1047,7 +1047,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 716c25bd125c8934499c1fb59a60d7e844c0691c c1d7304c80c4a6244c8a9f6fad1eebd0f339c724 -R 865d94cb53d70dd89436e58ffa63ebac +P fb7a7773965697855c12193408cb75300decc23c +R 22a5a5addda1bdaa7f5e87ce6cbfc935 U drh -Z 83b40ff56b301adcba03b758b3b36603 +Z 9527a4645b09957de5c779d28c3eb994 diff --git a/manifest.uuid b/manifest.uuid index 284652c9b5..aa637d58c4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb7a7773965697855c12193408cb75300decc23c \ No newline at end of file +15cb0db7583c3a24cbea0c72576047a93fba0801 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 768a73ed72..d2fdb64396 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -22,7 +22,6 @@ ** Recommended options: ** ** -DHAVE_USLEEP -** -DSQLITE_MAX_SCHEMA_RETRY=100 ** -DSQLITE_NO_SYNC ** -DSQLITE_THREADSAFE=0 ** -DSQLITE_OMIT_LOAD_EXTENSION @@ -824,11 +823,21 @@ static void runScript( ** Run a subscript from a separate file. */ if( strcmp(zCmd, "source")==0 ){ - char *zNewFile = azArg[0]; - char *zNewScript = readFile(zNewFile); + char *zNewFile, *zNewScript; + char *zToDel = 0; + zNewFile = azArg[0]; + if( zNewFile[0]!='/' ){ + int k; + for(k=(int)strlen(zFilename)-1; k>=0 && zFilename[k]!='/'; k--){} + if( k>0 ){ + zNewFile = zToDel = sqlite3_mprintf("%.*s/%s", k,zFilename,zNewFile); + } + } + zNewScript = readFile(zNewFile); if( g.iTrace ) logMessage("begin script [%s]\n", zNewFile); runScript(0, 0, zNewScript, zNewFile); sqlite3_free(zNewScript); + sqlite3_free(zToDel); if( g.iTrace ) logMessage("end script [%s]\n", zNewFile); }else @@ -987,17 +996,26 @@ static void unrecognizedArguments( int main(int argc, char **argv){ const char *zClient; int iClient; - int n; + int n, i; int openFlags = SQLITE_OPEN_READWRITE; int rc; char *zScript; int taskId; const char *zTrace; + const char *zCOption; g.argv0 = argv[0]; g.iTrace = 1; if( argc<2 ) usage(argv[0]); g.zDbFile = argv[1]; + if( strglob("*.test", g.zDbFile) ) usage(argv[0]); + if( strcmp(sqlite3_sourceid(), SQLITE_SOURCE_ID)!=0 ){ + fprintf(stderr, "SQLite library and header mismatch\n" + "Library: %s\n" + "Header: %s\n", + sqlite3_sourceid(), SQLITE_SOURCE_ID); + exit(1); + } n = argc-2; sqlite3_snprintf(sizeof(g.zName), g.zName, "mptest"); g.zVfs = findOption(argv+2, &n, "vfs", 1); @@ -1024,6 +1042,13 @@ int main(int argc, char **argv){ if( iClient<1 ) fatalError("illegal client number: %d\n", iClient); sqlite3_snprintf(sizeof(g.zName), g.zName, "client%02d", iClient); }else{ + if( g.iTrace>0 ){ + printf("With SQLite " SQLITE_VERSION " " SQLITE_SOURCE_ID "\n" ); + for(i=0; (zCOption = sqlite3_compileoption_get(i))!=0; i++){ + printf("-DSQLITE_%s\n", zCOption); + } + fflush(stdout); + } iClient = 0; unlink(g.zDbFile); openFlags |= SQLITE_OPEN_CREATE; From fbfe388d314f44de6891ede3eaaaa926b0c7769e Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 8 Apr 2013 10:38:57 +0000 Subject: [PATCH 088/140] Fix an incorrect type cast in an assignment in vdbe.c. FossilOrigin-Name: 320b5295da57023399c7b9eed82eae40cf619803 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 5497db1fb1..20d73d0f91 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sdefault\sSQLITE_MAX_SCHEMA_RETRY\sto\s50.\s\sMake\ssure\sthat\smacro\ncovers\severy\scase\swhere\sa\sprepared\sstatement\smight\sneed\sto\sbe\sreprepared\sdue\nto\sa\sschema\schange.\s\sThe\ssqlite3_exec()\sinterface\snow\suses\s\nsqlite3_prepare_v2(). -D 2013-04-06T18:06:51.485 +C Fix\san\sincorrect\stype\scast\sin\san\sassignment\sin\svdbe.c. +D 2013-04-08T10:38:57.721 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -239,7 +239,7 @@ F src/update.c a2a5631d618cbe240fc83725fa9e95c56ae0084c F src/utf.c 8d819e2e5104a430fc2005f018db14347c95a38f F src/util.c f566b5138099a2df8533b190d0dcc74b7dfbe0c9 F src/vacuum.c 2727bdd08847fcb6b2d2da6d14f018910e8645d3 -F src/vdbe.c 1d56617d816098a41bdbe73536a67dcdefad52bd +F src/vdbe.c 5f0047130f80c7fd0bc41bc51a653b5542c4fbd5 F src/vdbe.h b52887278cb173e66188da84dfab216bea61119d F src/vdbeInt.h c1e830268b75f04a2901dd895b51a637a26c7045 F src/vdbeapi.c 085cf9bf169b859a6c8fa43791702bac805cb7aa @@ -1044,7 +1044,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 07a0e4e9820a85bf1690214fef3577866c763f92 -R 2fd4d54ffecba80bddc0b0bfdd8ef6ea -U drh -Z d5343d886a406d77dc84001af2f5f17d +P c1d7304c80c4a6244c8a9f6fad1eebd0f339c724 +R 9455b151c7a6a645888f7422c4787f65 +U dan +Z 3be6f6ee1162b72435a62041d8441b1f diff --git a/manifest.uuid b/manifest.uuid index 85d44ed73b..8700cad9be 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c1d7304c80c4a6244c8a9f6fad1eebd0f339c724 \ No newline at end of file +320b5295da57023399c7b9eed82eae40cf619803 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 688621d951..f343e13d4e 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3515,7 +3515,7 @@ case OP_SeekGt: { /* jump, in3 */ ** r.flags = 0; ** } */ - r.flags = (u16)(UNPACKED_INCRKEY * (1 & (oc - OP_SeekLt))); + r.flags = (u8)(UNPACKED_INCRKEY * (1 & (oc - OP_SeekLt))); assert( oc!=OP_SeekGt || r.flags==UNPACKED_INCRKEY ); assert( oc!=OP_SeekLe || r.flags==UNPACKED_INCRKEY ); assert( oc!=OP_SeekGe || r.flags==0 ); From 7dfe8e2bde9b7dc655fa8b112efaecbc3acf0dfd Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Apr 2013 13:13:43 +0000 Subject: [PATCH 089/140] Add --if, --else, --endif processing to mptest.c. FossilOrigin-Name: 51265acae3088a51ac0ce6ab8731e6e15a48d4ae --- manifest | 12 ++++---- manifest.uuid | 2 +- mptest/mptest.c | 73 +++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 74 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 36209600bd..3e8ee81047 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s--source\scommand\sso\sthat\sits\sargument\sis\srelative\sto\sthe\sscript.\nVerify\sthat\sthe\sSQLite\sheader\sand\slibrary\smatch.\s\sPrint\sSQLite\sversion\nand\scompile-time\sconfiguration\sinformation\son\sstartup. -D 2013-04-06T18:35:07.098 +C Add\s--if,\s--else,\s--endif\sprocessing\sto\smptest.c. +D 2013-04-08T13:13:43.458 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -111,7 +111,7 @@ F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 058a9bc2b0db710d36003ab06dc1618566f27b52 -F mptest/mptest.c 6e23dade7b4cde4801b7a6df29f52a492d82d450 +F mptest/mptest.c b4030a7ef4a2bfd2913062d637ae3f55fb9443c2 F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1047,7 +1047,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P fb7a7773965697855c12193408cb75300decc23c -R 22a5a5addda1bdaa7f5e87ce6cbfc935 +P 15cb0db7583c3a24cbea0c72576047a93fba0801 +R ebe6490efbbe63db101a13263cb552f3 U drh -Z 9527a4645b09957de5c779d28c3eb994 +Z bd1b6a8c139e5f3d36fca35a646773b9 diff --git a/manifest.uuid b/manifest.uuid index aa637d58c4..b0847befc9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -15cb0db7583c3a24cbea0c72576047a93fba0801 \ No newline at end of file +51265acae3088a51ac0ce6ab8731e6e15a48d4ae \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index d2fdb64396..9a19ef2cd6 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -662,7 +662,7 @@ static int extractToken(const char *zIn, int nIn, char *zOut, int nOut){ } /* -** Find the number of characters up to the next "--end" token. +** Find the number of characters up to the start of the next "--end" token. */ static int findEnd(const char *z, int *pnLine){ int n = 0; @@ -672,6 +672,30 @@ static int findEnd(const char *z, int *pnLine){ return n; } +/* +** Find the number of characters up to the first character past the +** of the next "--endif" or "--else" token. Nested --if commands are +** also skipped. +*/ +static int findEndif(const char *z, int stopAtElse, int *pnLine){ + int n = 0; + while( z[n] ){ + int len = tokenLength(z+n, pnLine); + if( (strncmp(z+n,"--endif",7)==0 && isspace(z[n+7])) + || (stopAtElse && strncmp(z+n,"--else",6)==0 && isspace(z[n+6])) + ){ + return n+len; + } + if( strncmp(z+n,"--if",4)==0 && isspace(z[n+4]) ){ + int skip = findEndif(z+n+len, 0, pnLine); + n += skip + len; + }else{ + n += len; + } + } + return n; +} + /* ** Wait for a client process to complete all its tasks */ @@ -712,7 +736,7 @@ static void waitForClient(int iClient, int iTimeout, char *zErrPrefix){ } /* Maximum number of arguments to a --command */ -#define MX_ARG 5 +#define MX_ARG 2 /* ** Run a script. @@ -791,7 +815,7 @@ static void runScript( }else /* - ** --result + ** --reset ** ** Reset accumulated results back to an empty string */ @@ -852,12 +876,49 @@ static void runScript( logMessage("%.*s", len-jj, zScript+ii+jj); }else + /* + ** --if EXPR + ** + ** Skip forward to the next matching --endif or --else if EXPR is false. + */ + if( strcmp(zCmd, "if")==0 ){ + int jj, rc; + sqlite3_stmt *pStmt; + for(jj=4; jj0 ){ startClient(iNewClient); @@ -871,7 +932,7 @@ static void runScript( ** "all" then wait for all clients to complete. Wait no longer than ** TIMEOUT milliseconds (default 10,000) */ - if( strcmp(zCmd, "wait")==0 ){ + if( strcmp(zCmd, "wait")==0 && iClient==0 ){ int iTimeout = nArg>=2 ? atoi(azArg[1]) : 10000; sqlite3_snprintf(sizeof(zError),zError,"line %d of %s\n", prevLine, zFilename); @@ -886,7 +947,7 @@ static void runScript( ** Assign work to a client. Start the client if it is not running ** already. */ - if( strcmp(zCmd, "task")==0 ){ + if( strcmp(zCmd, "task")==0 && iClient==0 ){ int iTarget = atoi(azArg[0]); int iEnd; char *zTask; From 1bf44c7acba2a3bdfb2d2843402cce6a9b1733f0 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Apr 2013 13:48:29 +0000 Subject: [PATCH 090/140] Add the vfsname() and eval() SQL functions to mptest.c. Enhancements to the test/config01.test script. FossilOrigin-Name: 91397a147ce4f67a7ea1182f06a7dda3a96ec465 --- manifest | 14 +++++------ manifest.uuid | 2 +- mptest/config01.test | 24 +++++++++++++++++++ mptest/mptest.c | 55 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3e8ee81047..6766f10a07 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\s--if,\s--else,\s--endif\sprocessing\sto\smptest.c. -D 2013-04-08T13:13:43.458 +C Add\sthe\svfsname()\sand\seval()\sSQL\sfunctions\sto\smptest.c.\s\nEnhancements\sto\sthe\stest/config01.test\sscript. +D 2013-04-08T13:48:29.192 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,8 +110,8 @@ F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 -F mptest/config01.test 058a9bc2b0db710d36003ab06dc1618566f27b52 -F mptest/mptest.c b4030a7ef4a2bfd2913062d637ae3f55fb9443c2 +F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da +F mptest/mptest.c 6614ad2110ce44a78b4745036880275ef939bc6e F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1047,7 +1047,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 15cb0db7583c3a24cbea0c72576047a93fba0801 -R ebe6490efbbe63db101a13263cb552f3 +P 51265acae3088a51ac0ce6ab8731e6e15a48d4ae +R 70ae859a3e60d8d3ce14cc3f23701768 U drh -Z bd1b6a8c139e5f3d36fca35a646773b9 +Z 17c730545a4fcc3406090d39f090bdb3 diff --git a/manifest.uuid b/manifest.uuid index b0847befc9..e40c60c705 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -51265acae3088a51ac0ce6ab8731e6e15a48d4ae \ No newline at end of file +91397a147ce4f67a7ea1182f06a7dda3a96ec465 \ No newline at end of file diff --git a/mptest/config01.test b/mptest/config01.test index be6b570678..75ba2f8fc2 100644 --- a/mptest/config01.test +++ b/mptest/config01.test @@ -1,6 +1,7 @@ /* ** Configure five tasks in different ways, then run tests. */ +--if vfsname() GLOB 'unix' PRAGMA page_size=8192; --task 1 PRAGMA journal_mode=PERSIST; @@ -17,3 +18,26 @@ PRAGMA page_size=8192; PRAGMA journal_mode=OFF; --end --source multiwrite01.test +--wait all +PRAGMA page_size=16384; +VACUUM; +CREATE TABLE pgsz(taskid, sz INTEGER); +--task 1 + INSERT INTO pgsz VALUES(1, eval('PRAGMA page_size')); +--end +--task 2 + INSERT INTO pgsz VALUES(2, eval('PRAGMA page_size')); +--end +--task 3 + INSERT INTO pgsz VALUES(3, eval('PRAGMA page_size')); +--end +--task 4 + INSERT INTO pgsz VALUES(4, eval('PRAGMA page_size')); +--end +--task 5 + INSERT INTO pgsz VALUES(5, eval('PRAGMA page_size')); +--end +--source multiwrite01.test +--wait all +SELECT sz FROM pgsz; +--match 16384 16384 16384 16384 16384 diff --git a/mptest/mptest.c b/mptest/mptest.c index 9a19ef2cd6..8fc13d069a 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -269,6 +269,22 @@ static int clipLength(const char *z){ return n; } +/* +** Auxiliary SQL function to return the name of the VFS +*/ +static void vfsNameFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + char *zVfs = 0; + sqlite3_file_control(db, "main", SQLITE_FCNTL_VFSNAME, &zVfs); + if( zVfs ){ + sqlite3_result_text(context, zVfs, -1, sqlite3_free); + } +} + /* ** Busy handler with a g.iTimeout-millisecond timeout */ @@ -450,6 +466,32 @@ static int evalSql(String *p, const char *zFormat, ...){ return rc; } +/* +** Auxiliary SQL function to recursively evaluate SQL. +*/ +static void evalFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + String res; + char *zErrMsg = 0; + int rc; + memset(&res, 0, sizeof(res)); + rc = sqlite3_exec(db, zSql, evalCallback, &res, &zErrMsg); + if( zErrMsg ){ + sqlite3_result_error(context, zErrMsg, -1); + sqlite3_free(zErrMsg); + }else if( rc ){ + sqlite3_result_error_code(context, rc); + }else{ + sqlite3_result_text(context, res.z, -1, SQLITE_TRANSIENT); + } + stringFree(&res); +} + /* ** Look up the next task for client iClient in the database. ** Return the task script and the task number and mark that @@ -841,6 +883,15 @@ static void runScript( stringReset(&sResult); }else + /* + ** --output + ** + ** Output the result of the previous SQL. + */ + if( strcmp(zCmd, "output")==0 ){ + logMessage("%s", sResult.z); + }else + /* ** --source FILENAME ** @@ -1117,6 +1168,10 @@ int main(int argc, char **argv){ rc = sqlite3_open_v2(g.zDbFile, &g.db, openFlags, g.zVfs); if( rc ) fatalError("cannot open [%s]", g.zDbFile); sqlite3_busy_handler(g.db, busyHandler, 0); + sqlite3_create_function(g.db, "vfsname", 0, SQLITE_UTF8, 0, + vfsNameFunc, 0, 0); + sqlite3_create_function(g.db, "eval", 1, SQLITE_UTF8, 0, + evalFunc, 0, 0); g.iTimeout = DEFAULT_TIMEOUT; if( g.bSqlTrace ) sqlite3_trace(g.db, sqlTraceCallback, 0); if( iClient>0 ){ From 841810cef6e8f7f15b8bfe789e0a77ebbe82459f Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Apr 2013 13:59:11 +0000 Subject: [PATCH 091/140] Fix harmless compiler warnings in mptest.c. FossilOrigin-Name: 59bdbb10ed67bf79e0991e2ad58a28321dedb959 --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 12 +++++++++++- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 6766f10a07..1a6f824d31 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\svfsname()\sand\seval()\sSQL\sfunctions\sto\smptest.c.\s\nEnhancements\sto\sthe\stest/config01.test\sscript. -D 2013-04-08T13:48:29.192 +C Fix\sharmless\scompiler\swarnings\sin\smptest.c. +D 2013-04-08T13:59:11.021 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -111,7 +111,7 @@ F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da -F mptest/mptest.c 6614ad2110ce44a78b4745036880275ef939bc6e +F mptest/mptest.c fb46946df7d6f2e8be8c028b21a580a977326f24 F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1047,7 +1047,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 51265acae3088a51ac0ce6ab8731e6e15a48d4ae -R 70ae859a3e60d8d3ce14cc3f23701768 +P 91397a147ce4f67a7ea1182f06a7dda3a96ec465 +R ede2c5555feb77dd0a5488dbe1f8d502 U drh -Z 17c730545a4fcc3406090d39f090bdb3 +Z 8e0fdcf5a1bb9a273c7156627676f7a5 diff --git a/manifest.uuid b/manifest.uuid index e40c60c705..a4c5a4a477 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -91397a147ce4f67a7ea1182f06a7dda3a96ec465 \ No newline at end of file +59bdbb10ed67bf79e0991e2ad58a28321dedb959 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 8fc13d069a..30449bddfb 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -41,6 +41,9 @@ #include #include +/* Mark a parameter as unused to suppress compiler warnings */ +#define UNUSED_PARAMETER(x) (void)x + /* Global data */ static struct Global { @@ -279,6 +282,8 @@ static void vfsNameFunc( ){ sqlite3 *db = sqlite3_context_db_handle(context); char *zVfs = 0; + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); sqlite3_file_control(db, "main", SQLITE_FCNTL_VFSNAME, &zVfs); if( zVfs ){ sqlite3_result_text(context, zVfs, -1, sqlite3_free); @@ -289,6 +294,7 @@ static void vfsNameFunc( ** Busy handler with a g.iTimeout-millisecond timeout */ static int busyHandler(void *pCD, int count){ + UNUSED_PARAMETER(pCD); if( count*10>g.iTimeout ){ if( g.iTimeout>0 ) errorMessage("timeout after %dms", g.iTimeout); return 0; @@ -301,6 +307,7 @@ static int busyHandler(void *pCD, int count){ ** SQL Trace callback */ static void sqlTraceCallback(void *NotUsed1, const char *zSql){ + UNUSED_PARAMETER(NotUsed1); logMessage("[%.*s]", clipLength(zSql), zSql); } @@ -308,6 +315,7 @@ static void sqlTraceCallback(void *NotUsed1, const char *zSql){ ** SQL error log callback */ static void sqlErrorCallback(void *pArg, int iErrCode, const char *zMsg){ + UNUSED_PARAMETER(pArg); if( (iErrCode&0xff)==SQLITE_SCHEMA && g.iTrace<3 ) return; errorMessage("(errcode=%d) %s", iErrCode, zMsg); } @@ -435,6 +443,7 @@ static void stringAppendTerm(String *p, const char *z){ static int evalCallback(void *pCData, int argc, char **argv, char **azCol){ String *p = (String*)pCData; int i; + UNUSED_PARAMETER(azCol); for(i=0; i Date: Mon, 8 Apr 2013 14:28:33 +0000 Subject: [PATCH 092/140] Add the "mptester" target to the makefiles. Make mptester run with synchronous off by default. FossilOrigin-Name: 1397830bfead3699a7e6b1858ae67ae15259b89e --- Makefile.in | 6 ++++++ Makefile.msc | 5 +++++ main.mk | 5 +++++ manifest | 18 +++++++++--------- manifest.uuid | 2 +- mptest/mptest.c | 21 ++++++++++++++++----- 6 files changed, 42 insertions(+), 15 deletions(-) diff --git a/Makefile.in b/Makefile.in index edd29ae011..0b702def79 100644 --- a/Makefile.in +++ b/Makefile.in @@ -502,6 +502,11 @@ sqlite3$(TEXE): $(TOP)/src/shell.c libsqlite3.la sqlite3.h -o $@ $(TOP)/src/shell.c libsqlite3.la \ $(LIBREADLINE) $(TLIBS) -rpath "$(libdir)" +mptester$(EXE): sqlite3.c $(TOP)/mptest/mptest.c + $(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \ + $(TLIBS) -rpath "$(libdir)" + + # This target creates a directory named "tsrc" and fills it with # copies of all of the C source code and header files needed to # build on the target system. Some of the C source code and header @@ -950,6 +955,7 @@ clean: rm -f shell.c sqlite3ext.h rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c rm -f sqlite-*-output.vsix + rm -f mptest mptest.exe distclean: clean rm -f config.log config.status libtool Makefile sqlite3.pc diff --git a/Makefile.msc b/Makefile.msc index 70be76afa8..8c91e86bc8 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -813,6 +813,10 @@ sqlite3.exe: $(TOP)\src\shell.c libsqlite3.lib $(LIBRESOBJS) sqlite3.h $(TOP)\src\shell.c \ /link $(LTLINKOPTS) $(LTLIBPATHS) libsqlite3.lib $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) +mptester.exe: $(TOP)\mptest\mptest.c libsqlite3.lib $(LIBRESOBJS) sqlite3.h + $(LTLINK) $(TOP)\mptest\mptest.c \ + /link $(LTLINKOPTS) $(LTLIBPATHS) libsqlite3.lib $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS) + # This target creates a directory named "tsrc" and fills it with # copies of all of the C source code and header files needed to # build on the target system. Some of the C source code and header @@ -1253,6 +1257,7 @@ clean: del /Q shell.c sqlite3ext.h del /Q sqlite3_analyzer.exe sqlite3_analyzer.exp sqlite3_analyzer.c del /Q sqlite-*-output.vsix + del /Q mptester.exe # Dynamic link library section. # diff --git a/main.mk b/main.mk index 1294121c0a..7ab2bca2ba 100644 --- a/main.mk +++ b/main.mk @@ -365,6 +365,10 @@ sqlite3$(EXE): $(TOP)/src/shell.c libsqlite3.a sqlite3.h $(TOP)/src/shell.c \ libsqlite3.a $(LIBREADLINE) $(TLIBS) $(THREADLIB) +mptester$(EXE): sqlite3.c $(TOP)/mptest/mptest.c + $(TCCX) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \ + $(TLIBS) $(THREADLIB) + sqlite3.o: sqlite3.c $(TCCX) -c sqlite3.c @@ -626,3 +630,4 @@ clean: rm -f shell.c sqlite3ext.h rm -f sqlite3_analyzer sqlite3_analyzer.exe sqlite3_analyzer.c rm -f sqlite-*-output.vsix + rm -f mptest mptest.exe diff --git a/manifest b/manifest index 1a6f824d31..7acc35ecdc 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Fix\sharmless\scompiler\swarnings\sin\smptest.c. -D 2013-04-08T13:59:11.021 +C Add\sthe\s"mptester"\starget\sto\sthe\smakefiles.\s\sMake\smptester\srun\swith\nsynchronous\soff\sby\sdefault. +D 2013-04-08T14:28:33.142 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in aafa71d66bab7e87fb2f348152340645f79f0244 +F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 9ab787d472c9cb8bad29c2c4aeba15b06fba7bc4 +F Makefile.msc a244e5b5ba2493625db95b4d46b79201ea868bfe F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 05c7bd63b96f31cfdef5c766ed91307ac121f5aa @@ -103,7 +103,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk b25a7a0b657409b2dcf9b09d39562703d5df702d +F main.mk 1c26f3cc57d7c357da5218ae2928b7ca5acfecde F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -111,7 +111,7 @@ F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da -F mptest/mptest.c fb46946df7d6f2e8be8c028b21a580a977326f24 +F mptest/mptest.c 739950807a48af97fb3e432256fef96540d0e936 F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1047,7 +1047,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 91397a147ce4f67a7ea1182f06a7dda3a96ec465 -R ede2c5555feb77dd0a5488dbe1f8d502 +P 59bdbb10ed67bf79e0991e2ad58a28321dedb959 +R 1f9c112eec8d81ff124d0eb10d39eea3 U drh -Z 8e0fdcf5a1bb9a273c7156627676f7a5 +Z 523cf56f3e6158426b3e246d7dff8e8c diff --git a/manifest.uuid b/manifest.uuid index a4c5a4a477..dbce6398ba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -59bdbb10ed67bf79e0991e2ad58a28321dedb959 \ No newline at end of file +1397830bfead3699a7e6b1858ae67ae15259b89e \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 30449bddfb..48bc382846 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -35,7 +35,11 @@ */ #include "sqlite3.h" #include -#include +#if defined(_WIN32) +# include +#else +# include +#endif #include #include #include @@ -62,6 +66,7 @@ static struct Global { int nError; /* Number of errors */ int nTest; /* Number of --match operators */ int iTimeout; /* Milliseconds until a busy timeout */ + int bSync; /* Call fsync() */ } g; /* Default timeout */ @@ -604,11 +609,15 @@ static void startClient(int iClient){ if( sqlite3_changes(g.db) ){ #if !defined(_WIN32) char *zSys; + int rc; zSys = sqlite3_mprintf( - "%s \"%s\" --client %d --trace %d %s&", + "%s \"%s\" --client %d --trace %d %s%s&", g.argv0, g.zDbFile, iClient, g.iTrace, - g.bSqlTrace ? "--sqltrace " : ""); - system(zSys); + g.bSqlTrace ? "--sqltrace " : "", + g.bSync ? "--sync " : "" + ); + rc = system(zSys); + if( rc ) errorMessage("system() fails with error code %d", rc); sqlite3_free(zSys); #endif #if defined(_WIN32) @@ -922,8 +931,8 @@ static void runScript( if( g.iTrace ) logMessage("begin script [%s]\n", zNewFile); runScript(0, 0, zNewScript, zNewFile); sqlite3_free(zNewScript); - sqlite3_free(zToDel); if( g.iTrace ) logMessage("end script [%s]\n", zNewFile); + sqlite3_free(zToDel); }else /* @@ -1148,6 +1157,7 @@ int main(int argc, char **argv){ if( zTrace ) g.iTrace = atoi(zTrace); if( findOption(argv+2, &n, "quiet", 0)!=0 ) g.iTrace = 0; g.bSqlTrace = findOption(argv+2, &n, "sqltrace", 0)!=0; + g.bSync = findOption(argv+2, &n, "sync", 0)!=0; if( g.zErrLog ){ g.pErrLog = fopen(g.zErrLog, "a"); }else{ @@ -1184,6 +1194,7 @@ int main(int argc, char **argv){ evalFunc, 0, 0); g.iTimeout = DEFAULT_TIMEOUT; if( g.bSqlTrace ) sqlite3_trace(g.db, sqlTraceCallback, 0); + if( !g.bSync ) trySql("PRAGMA synchronous=OFF"); if( iClient>0 ){ if( n>0 ) unrecognizedArguments(argv[0], n, argv+2); if( g.iTrace ) logMessage("start-client"); From bcb8a868cec66d9edd20e99f7bdb5d2959e2a60d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 8 Apr 2013 15:30:41 +0000 Subject: [PATCH 093/140] Handle the case in os_unix.c where SQLITE_FCNTL_MMAP_LIMIT requests that the mmap limit be set to a value smaller than the current mapping. FossilOrigin-Name: 360473493ec1a7094a2b1c5436f3b70914a6dfdd --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 5 ++++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index c64575a45a..9ff25841fa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\smulti-process\stester\sinto\strunk. -D 2013-04-08T14:41:34.434 +C Handle\sthe\scase\sin\sos_unix.c\swhere\sSQLITE_FCNTL_MMAP_LIMIT\srequests\sthat\sthe\smmap\slimit\sbe\sset\sto\sa\svalue\ssmaller\sthan\sthe\scurrent\smapping. +D 2013-04-08T15:30:41.180 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -164,7 +164,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c f0ecce40d92469d5cc737ae883e776eb3e5c0af5 +F src/os_unix.c 6d584190db2ed442e5c849b7f73dcfbbac7c032e F src/os_win.c 3265df8c762e0b8caf8d02b3352fa8c22e91ae6b F src/pager.c 2e68df46d4086027cb6b527d47a6dedbf1a6b7ec F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1048,7 +1048,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 320b5295da57023399c7b9eed82eae40cf619803 1397830bfead3699a7e6b1858ae67ae15259b89e -R 518213540ee221b372b2870b99865f42 -U drh -Z 23b5a3829f6928c651f9b0e92776d749 +P ea53c45d79772986d2ad289f4f56f333541747e7 +R 1216631425c14cd22746e029dd206b1f +U dan +Z a5b14b8642cd44330c439704b21c4b8f diff --git a/manifest.uuid b/manifest.uuid index 2449f90305..644f2b2eed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ea53c45d79772986d2ad289f4f56f333541747e7 \ No newline at end of file +360473493ec1a7094a2b1c5436f3b70914a6dfdd \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 75f9236013..1d4c8ad348 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3712,7 +3712,10 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ case SQLITE_FCNTL_MMAP_LIMIT: { i64 newLimit = *(i64*)pArg; *(i64*)pArg = pFile->mmapLimit; - if( newLimit>=0 ) pFile->mmapLimit = newLimit; + if( newLimit>=0 ){ + pFile->mmapLimit = newLimit; + if( newLimitmmapSize ) pFile->mmapSize = newLimit; + } return SQLITE_OK; } #ifdef SQLITE_DEBUG From e5ebd22d992384f8e7fbf94775f695ea6cbe4554 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Apr 2013 15:36:51 +0000 Subject: [PATCH 094/140] Change mptester to avoid reporting SQLITE_BUSY errors if the busy handler is turned off and the tracing level is set low. FossilOrigin-Name: d77407ae6a9e09dea124dd59b14fdbbc36ed3d4e --- manifest | 14 +++++++------- manifest.uuid | 2 +- mptest/mptest.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 9ff25841fa..96187630f4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Handle\sthe\scase\sin\sos_unix.c\swhere\sSQLITE_FCNTL_MMAP_LIMIT\srequests\sthat\sthe\smmap\slimit\sbe\sset\sto\sa\svalue\ssmaller\sthan\sthe\scurrent\smapping. -D 2013-04-08T15:30:41.180 +C Change\smptester\sto\savoid\sreporting\sSQLITE_BUSY\serrors\sif\sthe\sbusy\shandler\nis\sturned\soff\sand\sthe\stracing\slevel\sis\sset\slow. +D 2013-04-08T15:36:51.699 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -112,7 +112,7 @@ F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da F mptest/config02.test 381ec0acbb1e5aab28ca9aa91e17ae9a02bbd2bb -F mptest/mptest.c 739950807a48af97fb3e432256fef96540d0e936 +F mptest/mptest.c 7048030a410707f77845c0fda9c4a223c7069e0b F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1048,7 +1048,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ea53c45d79772986d2ad289f4f56f333541747e7 -R 1216631425c14cd22746e029dd206b1f -U dan -Z a5b14b8642cd44330c439704b21c4b8f +P 360473493ec1a7094a2b1c5436f3b70914a6dfdd +R 138dfe64c0fb20cc8b3e6aefe188c7ba +U drh +Z bbdf55a7e5fdae8e0e462019e357a9d2 diff --git a/manifest.uuid b/manifest.uuid index 644f2b2eed..3944faa67f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -360473493ec1a7094a2b1c5436f3b70914a6dfdd \ No newline at end of file +d77407ae6a9e09dea124dd59b14fdbbc36ed3d4e \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 48bc382846..ca056f76e6 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -322,6 +322,7 @@ static void sqlTraceCallback(void *NotUsed1, const char *zSql){ static void sqlErrorCallback(void *pArg, int iErrCode, const char *zMsg){ UNUSED_PARAMETER(pArg); if( (iErrCode&0xff)==SQLITE_SCHEMA && g.iTrace<3 ) return; + if( g.iTimeout==0 && (iErrCode&0xff)==SQLITE_BUSY && g.iTrace<3 ) return; errorMessage("(errcode=%d) %s", iErrCode, zMsg); } From 023dd903cc2cd8ad87fba3e3b4e8ad6ae5ba19f1 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Apr 2013 17:57:26 +0000 Subject: [PATCH 095/140] Adding tests to the mptester scripts. FossilOrigin-Name: e121adceb1e267e032dcbadb74eaa692a12a65bb --- manifest | 14 +- manifest.uuid | 2 +- mptest/config02.test | 8 ++ mptest/multiwrite01.test | 273 ++++++++++++++++++++++++++++++++++++++- 4 files changed, 284 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 96187630f4..b55b1213e9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\smptester\sto\savoid\sreporting\sSQLITE_BUSY\serrors\sif\sthe\sbusy\shandler\nis\sturned\soff\sand\sthe\stracing\slevel\sis\sset\slow. -D 2013-04-08T15:36:51.699 +C Adding\stests\sto\sthe\smptester\sscripts. +D 2013-04-08T17:57:26.382 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -111,9 +111,9 @@ F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da -F mptest/config02.test 381ec0acbb1e5aab28ca9aa91e17ae9a02bbd2bb +F mptest/config02.test baaee6bba0122f45c895a88c7ac8b62ff77c6b70 F mptest/mptest.c 7048030a410707f77845c0fda9c4a223c7069e0b -F mptest/multiwrite01.test aef0af17f1ce1beacd158e403a45a21008d7a70c +F mptest/multiwrite01.test 16f9bb13640dfe1be1e544d49b16aa9e7503ae2d F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc @@ -1048,7 +1048,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 360473493ec1a7094a2b1c5436f3b70914a6dfdd -R 138dfe64c0fb20cc8b3e6aefe188c7ba +P d77407ae6a9e09dea124dd59b14fdbbc36ed3d4e +R 8e0a5a89df2079b6986569045ce512eb U drh -Z bbdf55a7e5fdae8e0e462019e357a9d2 +Z a3e7b3daf8fd9801ae226170daba9d61 diff --git a/manifest.uuid b/manifest.uuid index 3944faa67f..35124abccb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d77407ae6a9e09dea124dd59b14fdbbc36ed3d4e \ No newline at end of file +e121adceb1e267e032dcbadb74eaa692a12a65bb \ No newline at end of file diff --git a/mptest/config02.test b/mptest/config02.test index 70c3a6cac2..303a5b4d56 100644 --- a/mptest/config02.test +++ b/mptest/config02.test @@ -103,3 +103,11 @@ DELETE FROM pgsz; --wait all SELECT sz FROM pgsz; --match 16384 16384 16384 16384 16384 +PRAGMA auto_vacuum=FULL; +VACUUM; +--source multiwrite01.test +--wait all +PRAGMA auto_vacuum=FULL; +PRAGMA page_size=512; +VACUUM; +--source multiwrite01.test diff --git a/mptest/multiwrite01.test b/mptest/multiwrite01.test index dc71884047..a3cfb0dbb4 100644 --- a/mptest/multiwrite01.test +++ b/mptest/multiwrite01.test @@ -24,6 +24,13 @@ UPDATE t1 SET b='x'||a||'y'; SELECT total(length(b)) FROM t1; --match 247 + SELECT a FROM t1 WHERE b='x17y'; + --match 17 + CREATE INDEX t1b ON t1(b); + SELECT a FROM t1 WHERE b='x17y'; + --match 17 + SELECT a FROM t1 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5; + --match 29 28 27 26 25 --end @@ -49,6 +56,13 @@ UPDATE t2 SET b='x'||a||'y'; SELECT total(length(b)) FROM t2; --match 247 + SELECT a FROM t2 WHERE b='x17y'; + --match 17 + CREATE INDEX t2b ON t2(b); + SELECT a FROM t2 WHERE b='x17y'; + --match 17 + SELECT a FROM t2 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5; + --match 29 28 27 26 25 --end --task 3 @@ -73,6 +87,13 @@ UPDATE t3 SET b='x'||a||'y'; SELECT total(length(b)) FROM t3; --match 247 + SELECT a FROM t3 WHERE b='x17y'; + --match 17 + CREATE INDEX t3b ON t3(b); + SELECT a FROM t3 WHERE b='x17y'; + --match 17 + SELECT a FROM t3 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5; + --match 29 28 27 26 25 --end --task 4 @@ -97,6 +118,13 @@ UPDATE t4 SET b='x'||a||'y'; SELECT total(length(b)) FROM t4; --match 247 + SELECT a FROM t4 WHERE b='x17y'; + --match 17 + CREATE INDEX t4b ON t4(b); + SELECT a FROM t4 WHERE b='x17y'; + --match 17 + SELECT a FROM t4 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5; + --match 29 28 27 26 25 --end --task 5 @@ -121,6 +149,13 @@ UPDATE t5 SET b='x'||a||'y'; SELECT total(length(b)) FROM t5; --match 247 + SELECT a FROM t5 WHERE b='x17y'; + --match 17 + CREATE INDEX t5b ON t5(b); + SELECT a FROM t5 WHERE b='x17y'; + --match 17 + SELECT a FROM t5 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5; + --match 29 28 27 26 25 --end --wait all @@ -134,8 +169,236 @@ SELECT count(*), total(length(b)) FROM t4; --match 64 247 SELECT count(*), total(length(b)) FROM t5; --match 64 247 -DROP TABLE t1; -DROP TABLE t2; -DROP TABLE t3; -DROP TABLE t4; -DROP TABLE t5; + +--task 1 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 5 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 3 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 2 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 4 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--wait all + +--task 5 + DROP INDEX t5b; + --sleep 5 + PRAGMA integrity_check; + --match ok + CREATE INDEX t5b ON t5(b DESC); +--end +--task 3 + DROP INDEX t3b; + --sleep 5 + PRAGMA integrity_check; + --match ok + CREATE INDEX t3b ON t3(b DESC); +--end +--task 1 + DROP INDEX t1b; + --sleep 5 + PRAGMA integrity_check; + --match ok + CREATE INDEX t1b ON t1(b DESC); +--end +--task 2 + DROP INDEX t2b; + --sleep 5 + PRAGMA integrity_check; + --match ok + CREATE INDEX t2b ON t2(b DESC); +--end +--task 4 + DROP INDEX t4b; + --sleep 5 + PRAGMA integrity_check; + --match ok + CREATE INDEX t4b ON t4(b DESC); +--end +--wait all + +--task 1 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 5 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 3 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 2 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 4 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--wait all + +VACUUM; +PRAGMA integrity_check; +--match ok + +--task 1 + UPDATE t1 SET b=randomblob(20000); + --sleep 5 + UPDATE t1 SET b='x'||a||'y'; + SELECT a FROM t1 WHERE b='x63y'; + --match 63 +--end +--task 2 + UPDATE t2 SET b=randomblob(20000); + --sleep 5 + UPDATE t2 SET b='x'||a||'y'; + SELECT a FROM t2 WHERE b='x63y'; + --match 63 +--end +--task 3 + UPDATE t3 SET b=randomblob(20000); + --sleep 5 + UPDATE t3 SET b='x'||a||'y'; + SELECT a FROM t3 WHERE b='x63y'; + --match 63 +--end +--task 4 + UPDATE t4 SET b=randomblob(20000); + --sleep 5 + UPDATE t4 SET b='x'||a||'y'; + SELECT a FROM t4 WHERE b='x63y'; + --match 63 +--end +--task 5 + UPDATE t5 SET b=randomblob(20000); + --sleep 5 + UPDATE t5 SET b='x'||a||'y'; + SELECT a FROM t5 WHERE b='x63y'; + --match 63 +--end +--wait all + +--task 1 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 5 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 3 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 2 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end +--task 4 + SELECT t1.a FROM t1, t2 + WHERE t2.b GLOB 'x3?y' AND t1.b=('x'||(t2.a+3)||'y') + ORDER BY t1.a LIMIT 4 + --match 33 34 35 36 + SELECT t3.a FROM t3, t4 + WHERE t4.b GLOB 'x4?y' AND t3.b=('x'||(t4.a+5)||'y') + ORDER BY t3.a LIMIT 7 + --match 45 46 47 48 49 50 51 +--end From 6adab7a1a870a1af8dfeb18b82957e8e4385b2fa Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Apr 2013 18:58:00 +0000 Subject: [PATCH 096/140] Still more tests using mptester. Added test cases that leave rollback journals that must be recovered by other threads. FossilOrigin-Name: 437b674f00610e2b925dc00ca27b5022e3858994 --- manifest | 18 ++++--- manifest.uuid | 2 +- mptest/config02.test | 7 +++ mptest/crash01.test | 102 +++++++++++++++++++++++++++++++++++++++ mptest/crash02.subtest | 57 ++++++++++++++++++++++ mptest/mptest.c | 20 ++++++-- mptest/multiwrite01.test | 1 + 7 files changed, 193 insertions(+), 14 deletions(-) create mode 100644 mptest/crash01.test create mode 100644 mptest/crash02.subtest diff --git a/manifest b/manifest index b55b1213e9..94bee9a115 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adding\stests\sto\sthe\smptester\sscripts. -D 2013-04-08T17:57:26.382 +C Still\smore\stests\susing\smptester.\s\sAdded\stest\scases\sthat\sleave\srollback\njournals\sthat\smust\sbe\srecovered\sby\sother\sthreads. +D 2013-04-08T18:58:00.322 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -111,9 +111,11 @@ F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da -F mptest/config02.test baaee6bba0122f45c895a88c7ac8b62ff77c6b70 -F mptest/mptest.c 7048030a410707f77845c0fda9c4a223c7069e0b -F mptest/multiwrite01.test 16f9bb13640dfe1be1e544d49b16aa9e7503ae2d +F mptest/config02.test 962913ed2b537d60de4126db7fe54716865cdd22 +F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 +F mptest/crash02.subtest c4647feafa04109e1dc60121a895b6cba8d63ceb +F mptest/mptest.c 76c8cb03f9cc91811907d8f9ccf2a5253bbb229d +F mptest/multiwrite01.test d6cb707062268e65396bd71eb53e9d5edb5519c0 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc @@ -1048,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d77407ae6a9e09dea124dd59b14fdbbc36ed3d4e -R 8e0a5a89df2079b6986569045ce512eb +P e121adceb1e267e032dcbadb74eaa692a12a65bb +R 82dd3ae164ec063cf354b1e10f2e05ba U drh -Z a3e7b3daf8fd9801ae226170daba9d61 +Z f4809a26a66812142b3835cff8ebb4b8 diff --git a/manifest.uuid b/manifest.uuid index 35124abccb..c637c165c7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e121adceb1e267e032dcbadb74eaa692a12a65bb \ No newline at end of file +437b674f00610e2b925dc00ca27b5022e3858994 \ No newline at end of file diff --git a/mptest/config02.test b/mptest/config02.test index 303a5b4d56..e25d94081e 100644 --- a/mptest/config02.test +++ b/mptest/config02.test @@ -15,6 +15,7 @@ PRAGMA page_size=512; PRAGMA mmap_limit=65536; --end --source multiwrite01.test +--source crash02.subtest PRAGMA page_size=1024; VACUUM; CREATE TABLE pgsz(taskid, sz INTEGER); @@ -34,6 +35,7 @@ CREATE TABLE pgsz(taskid, sz INTEGER); INSERT INTO pgsz VALUES(5, eval('PRAGMA page_size')); --end --source multiwrite01.test +--source crash02.subtest --wait all SELECT sz FROM pgsz; --match 1024 1024 1024 1024 1024 @@ -56,6 +58,7 @@ DELETE FROM pgsz; INSERT INTO pgsz VALUES(5, eval('PRAGMA page_size')); --end --source multiwrite01.test +--source crash02.subtest --wait all SELECT sz FROM pgsz; --match 2048 2048 2048 2048 2048 @@ -78,6 +81,7 @@ DELETE FROM pgsz; INSERT INTO pgsz VALUES(5, eval('PRAGMA page_size')); --end --source multiwrite01.test +--source crash02.subtest --wait all SELECT sz FROM pgsz; --match 8192 8192 8192 8192 8192 @@ -100,14 +104,17 @@ DELETE FROM pgsz; INSERT INTO pgsz VALUES(5, eval('PRAGMA page_size')); --end --source multiwrite01.test +--source crash02.subtest --wait all SELECT sz FROM pgsz; --match 16384 16384 16384 16384 16384 PRAGMA auto_vacuum=FULL; VACUUM; --source multiwrite01.test +--source crash02.subtest --wait all PRAGMA auto_vacuum=FULL; PRAGMA page_size=512; VACUUM; --source multiwrite01.test +--source crash02.subtest diff --git a/mptest/crash01.test b/mptest/crash01.test new file mode 100644 index 0000000000..d094d72f29 --- /dev/null +++ b/mptest/crash01.test @@ -0,0 +1,102 @@ +/* Test cases involving incomplete transactions that must be rolled back. +*/ +--task 1 + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + --sleep 1 + INSERT INTO t1 VALUES(1, randomblob(2000)); + INSERT INTO t1 VALUES(2, randomblob(1000)); + --sleep 1 + INSERT INTO t1 SELECT a+2, randomblob(1500) FROM t1; + INSERT INTO t1 SELECT a+4, randomblob(1500) FROM t1; + INSERT INTO t1 SELECT a+8, randomblob(1500) FROM t1; + --sleep 1 + INSERT INTO t1 SELECT a+16, randomblob(1500) FROM t1; + --sleep 1 + INSERT INTO t1 SELECT a+32, randomblob(1500) FROM t1; + SELECT count(*) FROM t1; + --match 64 + SELECT avg(length(b)) FROM t1; + --match 1500.0 + --sleep 2 + UPDATE t1 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t1; + --match 247 + SELECT a FROM t1 WHERE b='x17y'; + --match 17 + CREATE INDEX t1b ON t1(b); + SELECT a FROM t1 WHERE b='x17y'; + --match 17 + SELECT a FROM t1 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5; + --match 29 28 27 26 25 +--end +--wait 1 +--task 2 + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + INSERT INTO t2 SELECT a, b FROM t1; + UPDATE t1 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t2; + --match 247 + SELECT a FROM t2 WHERE b='x17y'; + --match 17 + CREATE INDEX t2b ON t2(b); + SELECT a FROM t2 WHERE b='x17y'; + --match 17 + SELECT a FROM t2 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5; + --match 29 28 27 26 25 +--end +--task 3 + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + INSERT INTO t3 SELECT a, b FROM t1; + UPDATE t1 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t3; + --match 247 + SELECT a FROM t3 WHERE b='x17y'; + --match 17 + CREATE INDEX t3b ON t3(b); + SELECT a FROM t3 WHERE b='x17y'; + --match 17 + SELECT a FROM t3 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5; + --match 29 28 27 26 25 +--end +--task 4 + CREATE TABLE t4(a INTEGER PRIMARY KEY, b); + INSERT INTO t4 SELECT a, b FROM t1; + UPDATE t1 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t4; + --match 247 + SELECT a FROM t4 WHERE b='x17y'; + --match 17 + CREATE INDEX t4b ON t4(b); + SELECT a FROM t4 WHERE b='x17y'; + --match 17 + SELECT a FROM t4 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5; + --match 29 28 27 26 25 +--end +--task 5 + CREATE TABLE t5(a INTEGER PRIMARY KEY, b); + INSERT INTO t5 SELECT a, b FROM t1; + UPDATE t1 SET b='x'||a||'y'; + SELECT total(length(b)) FROM t5; + --match 247 + SELECT a FROM t5 WHERE b='x17y'; + --match 17 + CREATE INDEX t5b ON t5(b); + SELECT a FROM t5 WHERE b='x17y'; + --match 17 + SELECT a FROM t5 WHERE b GLOB 'x2?y' ORDER BY b DESC LIMIT 5; + --match 29 28 27 26 25 +--end + +--wait all +/* After the database file has been set up, run the crash2 subscript +** multiple times. */ +--source crash02.subtest +--source crash02.subtest +--source crash02.subtest +--source crash02.subtest +--source crash02.subtest +--source crash02.subtest +--source crash02.subtest +--source crash02.subtest +--source crash02.subtest diff --git a/mptest/crash02.subtest b/mptest/crash02.subtest new file mode 100644 index 0000000000..053a9faf93 --- /dev/null +++ b/mptest/crash02.subtest @@ -0,0 +1,57 @@ +/* +** This script is called from crash01.test and config02.test and perhaps other +** script. After the database file has been set up, make a big rollback +** journal in client 1, then crash client 1. +** Then in the other clients, do an integrity check. +*/ +--task 1 + --sleep 5 + --finish + PRAGMA cache_size=10; + BEGIN; + UPDATE t1 SET b=randomblob(20000); + UPDATE t2 SET b=randomblob(20000); + UPDATE t3 SET b=randomblob(20000); + UPDATE t4 SET b=randomblob(20000); + UPDATE t5 SET b=randomblob(20000); + UPDATE t1 SET b=NULL; + UPDATE t2 SET b=NULL; + UPDATE t3 SET b=NULL; + UPDATE t4 SET b=NULL; + UPDATE t5 SET b=NULL; + --print Task one crashing an incomplete transaction + --exit 1 +--end +--task 2 + SELECT count(*) FROM t1; + --match 64 + --sleep 100 + PRAGMA integrity_check; + --output + --match ok +--end +--task 3 + SELECT count(*) FROM t1; + --match 64 + --sleep 100 + PRAGMA integrity_check; + --output + --match ok +--end +--task 4 + SELECT count(*) FROM t1; + --match 64 + --sleep 100 + PRAGMA integrity_check; + --output + --match ok +--end +--task 5 + SELECT count(*) FROM t1; + --match 64 + --sleep 100 + PRAGMA integrity_check; + --output + --match ok +--end +--wait all diff --git a/mptest/mptest.c b/mptest/mptest.c index ca056f76e6..cf8eb14e1e 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -526,14 +526,14 @@ static int startScript( *pzScript = 0; g.iTimeout = 0; while(1){ - rc = trySql("BEGIN IMMEDIATE"); + rc = trySql("BEGIN EXCLUSIVE"); if( rc==SQLITE_BUSY ){ sqlite3_sleep(10); totalTime += 10; continue; } if( rc!=SQLITE_OK ){ - fatalError("%s\nBEGIN IMMEDIATE", sqlite3_errmsg(g.db)); + fatalError("in startScript: %s", sqlite3_errmsg(g.db)); } if( g.nError || g.nTest ){ runSql("UPDATE counters SET nError=nError+%d, nTest=nTest+%d", @@ -869,13 +869,23 @@ static void runScript( ** Exit this process. If N>0 then exit without shutting down ** SQLite. (In other words, simulate a crash.) */ - if( strcmp(zCmd, "exit")==0 ){ + if( strcmp(zCmd, "exit")==0 && iClient>0 ){ int rc = atoi(azArg[0]); finishScript(iClient, taskId, 1); if( rc==0 ) sqlite3_close(g.db); exit(rc); }else + /* + ** --finish + ** + ** Mark the current task as having finished, even if it is not. + ** This can be used in conjunction with --exit to simulate a crash. + */ + if( strcmp(zCmd, "finish")==0 && iClient>0 ){ + finishScript(iClient, taskId, 1); + }else + /* ** --reset ** @@ -1203,11 +1213,11 @@ int main(int argc, char **argv){ char zTaskName[50]; rc = startScript(iClient, &zScript, &taskId); if( rc==SQLITE_DONE ) break; - if( g.iTrace ) logMessage("begin task %d", taskId); sqlite3_snprintf(sizeof(zTaskName), zTaskName, "client%02d-task-%d", iClient, taskId); + if( g.iTrace ) logMessage("begin %s", zTaskName); runScript(iClient, taskId, zScript, zTaskName); - if( g.iTrace ) logMessage("end task %d", taskId); + if( g.iTrace ) logMessage("end %s", zTaskName); finishScript(iClient, taskId, 0); sqlite3_sleep(10); } diff --git a/mptest/multiwrite01.test b/mptest/multiwrite01.test index a3cfb0dbb4..dc7d62b8f9 100644 --- a/mptest/multiwrite01.test +++ b/mptest/multiwrite01.test @@ -402,3 +402,4 @@ PRAGMA integrity_check; ORDER BY t3.a LIMIT 7 --match 45 46 47 48 49 50 51 --end +--wait all From f90e50f60bd01af442a9b68eb5d808ef6faa4ca7 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Apr 2013 19:13:48 +0000 Subject: [PATCH 097/140] In mptester, handle startScript no-change commits using a retry rather than grabbing an exclusive transaction. FossilOrigin-Name: ae8bb24bfb014bdcb8aaa8159ddae0cd376afe99 --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 13 ++++++++----- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 94bee9a115..535fa7ba3a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Still\smore\stests\susing\smptester.\s\sAdded\stest\scases\sthat\sleave\srollback\njournals\sthat\smust\sbe\srecovered\sby\sother\sthreads. -D 2013-04-08T18:58:00.322 +C In\smptester,\shandle\sstartScript\sno-change\scommits\susing\sa\sretry\srather\nthan\sgrabbing\san\sexclusive\stransaction. +D 2013-04-08T19:13:48.566 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da F mptest/config02.test 962913ed2b537d60de4126db7fe54716865cdd22 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest c4647feafa04109e1dc60121a895b6cba8d63ceb -F mptest/mptest.c 76c8cb03f9cc91811907d8f9ccf2a5253bbb229d +F mptest/mptest.c e31bb60d3de960aa0f7d08af7ae59c6dfecc552b F mptest/multiwrite01.test d6cb707062268e65396bd71eb53e9d5edb5519c0 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e121adceb1e267e032dcbadb74eaa692a12a65bb -R 82dd3ae164ec063cf354b1e10f2e05ba +P 437b674f00610e2b925dc00ca27b5022e3858994 +R 609e885f5484356f11d8798a6d9ad296 U drh -Z f4809a26a66812142b3835cff8ebb4b8 +Z f55865a5fe0b6f2c658b1f5dce1b30e8 diff --git a/manifest.uuid b/manifest.uuid index c637c165c7..b7c4fb39fc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -437b674f00610e2b925dc00ca27b5022e3858994 \ No newline at end of file +ae8bb24bfb014bdcb8aaa8159ddae0cd376afe99 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index cf8eb14e1e..f14a640e8c 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -526,7 +526,7 @@ static int startScript( *pzScript = 0; g.iTimeout = 0; while(1){ - rc = trySql("BEGIN EXCLUSIVE"); + rc = trySql("BEGIN IMMEDIATE"); if( rc==SQLITE_BUSY ){ sqlite3_sleep(10); totalTime += 10; @@ -546,8 +546,8 @@ static int startScript( sqlite3_finalize(pStmt); if( rc==SQLITE_ROW ){ runSql("DELETE FROM client WHERE id=%d", iClient); - runSql("COMMIT"); g.iTimeout = DEFAULT_TIMEOUT; + runSql("COMMIT TRANSACTION;"); return SQLITE_DONE; } pStmt = prepareSql( @@ -557,15 +557,15 @@ static int startScript( rc = sqlite3_step(pStmt); if( rc==SQLITE_ROW ){ int n = sqlite3_column_bytes(pStmt, 0); - *pzScript = sqlite3_malloc( n+ 1 ); + *pzScript = sqlite3_malloc(n+1); strcpy(*pzScript, (const char*)sqlite3_column_text(pStmt, 0)); *pTaskId = taskId = sqlite3_column_int(pStmt, 1); sqlite3_finalize(pStmt); runSql("UPDATE task" " SET starttime=strftime('%%Y-%%m-%%d %%H:%%M:%%f','now')" " WHERE id=%d;", taskId); - runSql("COMMIT;"); g.iTimeout = DEFAULT_TIMEOUT; + runSql("COMMIT TRANSACTION;"); return SQLITE_OK; } sqlite3_finalize(pStmt); @@ -576,7 +576,10 @@ static int startScript( sqlite3_close(g.db); exit(1); } - runSql("COMMIT;"); + while( trySql("COMMIT")==SQLITE_BUSY ){ + sqlite3_sleep(10); + totalTime += 10; + } sqlite3_sleep(100); totalTime += 100; continue; From 188d488409c9d931bb8b59eb86956ec6130017e7 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Apr 2013 20:47:49 +0000 Subject: [PATCH 098/140] Disable the use of memory-mapped I/O if the SQLITE_DISABLE_MMAP macro is defined. Automatically define this macro for OpenBSD and QNX. Other systems are likely to be added to the disabled list over time. FossilOrigin-Name: 8a4314a398c6c6efb25e6993c826b7e60d9bb1f9 --- manifest | 34 +++++++++++++++++----------------- manifest.uuid | 2 +- src/ctime.c | 3 +++ src/os.c | 12 ++++++++++++ src/pager.c | 20 ++++++++++++++++---- src/pragma.c | 3 +++ src/sqliteInt.h | 8 ++++++++ src/sqliteLimit.h | 4 ++++ src/test_config.c | 6 ++++++ test/mmap1.test | 6 ++++-- test/mmap2.test | 4 ++++ test/pageropt.test | 7 ++++++- test/wal.test | 1 + test/wal5.test | 1 + 14 files changed, 86 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 535fa7ba3a..fdbb616c1d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\smptester,\shandle\sstartScript\sno-change\scommits\susing\sa\sretry\srather\nthan\sgrabbing\san\sexclusive\stransaction. -D 2013-04-08T19:13:48.566 +C Disable\sthe\suse\sof\smemory-mapped\sI/O\sif\sthe\sSQLITE_DISABLE_MMAP\smacro\sis\ndefined.\s\sAutomatically\sdefine\sthis\smacro\sfor\sOpenBSD\sand\sQNX.\s\sOther\nsystems\sare\slikely\sto\sbe\sadded\sto\sthe\sdisabled\slist\sover\stime. +D 2013-04-08T20:47:49.018 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -133,7 +133,7 @@ F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c 16658a257bc6a3ca8d8961f574cf61a57e4d6faf +F src/ctime.c 61a53c7e0db5e7ba60cf23b73deaf6779aa715d3 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 @@ -163,18 +163,18 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c 809d0707cec693e1b9b376ab229271ad74c3d35d +F src/os.c ca679b293a6233327e418fd1dde2cd5db3e90932 F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 6d584190db2ed442e5c849b7f73dcfbbac7c032e F src/os_win.c 3265df8c762e0b8caf8d02b3352fa8c22e91ae6b -F src/pager.c 2e68df46d4086027cb6b527d47a6dedbf1a6b7ec +F src/pager.c eb6d272f4dc1ade4b8f664599cd54a9c95cb2729 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c 682e97f3e3b77fd6c9b569eabfbf4a14c987aca3 +F src/pragma.c 8bbc8c8697d13f76394e592dcecafce3491ed8ba F src/prepare.c 743e484233c51109666d402f470523553b41797c F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -185,8 +185,8 @@ F src/shell.c 319b7791cee6c763b60fde1b590bfaf62613cf37 F src/sqlite.h.in 5fdc866a06ac82fd3c6ccb9f59a100151a36568c F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 -F src/sqliteInt.h 1664dc5ad6f8d4dab871416628aa3271044d66c0 -F src/sqliteLimit.h 1097d2c541147d303e66515040067c6bfcf8cf21 +F src/sqliteInt.h 1c517a89cfdba28084093607ec79af70359c7f9b +F src/sqliteLimit.h 3ea5c07b370b8033c30f66515ccf4e875c252429 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 @@ -203,7 +203,7 @@ F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de F src/test_btree.c 5b89601dcb42a33ba8b820a6b763cc9cb48bac16 -F src/test_config.c ab6605a41f2020912d5e44b8dd4b9fb21bd75ac0 +F src/test_config.c b6f5d680f0cf03d0ae9d6165bd2e447d9b356e4a F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c 8f786bfd0ad48030cf2a06fb1f050e9c60a150d7 @@ -637,8 +637,8 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test 146a755bb382677b6f9993c61ddc7732b157d730 -F test/mmap2.test 62dbb5d718e66d654d232116c5a2d96e26a071a5 +F test/mmap1.test f064a9025136cd688c982ccdae81c7b4fac95724 +F test/mmap2.test a5ba639f90b5fc487400a49e158e14e465943e98 F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 @@ -663,7 +663,7 @@ F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f F test/pagerfault.test bba948be0564254b823ec6ca1fd95fbbfb0839a3 F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 -F test/pageropt.test 32cb5a6ed7ccddf8e8c842cb44240bd9340223ce +F test/pageropt.test d6b695e6c12290c406150fc51eec32878d48df4c F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 @@ -961,11 +961,11 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 -F test/wal.test bbd98ac09d8fbcaa80ec233757056eb1bbb7cd95 +F test/wal.test b934b757c4c3776b1e93140a9febc9ae068f9d90 F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c -F test/wal5.test f4d0aee6a2cf09e326ed2459011d396b4fdf661a +F test/wal5.test 13c980d4d8d2d6b558361f19bed4ab31d41a2c7c F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test b3ee739fe8f7586aaebdc2367f477ebcf3e3b034 @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 437b674f00610e2b925dc00ca27b5022e3858994 -R 609e885f5484356f11d8798a6d9ad296 +P ae8bb24bfb014bdcb8aaa8159ddae0cd376afe99 +R 7e782b1e3841573f5a2d86903a4acfcf U drh -Z f55865a5fe0b6f2c658b1f5dce1b30e8 +Z a7d6d975495e51a139482ed48c14db14 diff --git a/manifest.uuid b/manifest.uuid index b7c4fb39fc..387ecfb3dc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ae8bb24bfb014bdcb8aaa8159ddae0cd376afe99 \ No newline at end of file +8a4314a398c6c6efb25e6993c826b7e60d9bb1f9 \ No newline at end of file diff --git a/src/ctime.c b/src/ctime.c index ee6d4cbe2e..450a84045d 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -66,6 +66,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_DISABLE_LFS "DISABLE_LFS", #endif +#ifdef SQLITE_DISABLE_MMAP + "DISABLE_MMAP", +#endif #ifdef SQLITE_ENABLE_ATOMIC_WRITE "ENABLE_ATOMIC_WRITE", #endif diff --git a/src/os.c b/src/os.c index 2ba857e2aa..90cbe5492d 100644 --- a/src/os.c +++ b/src/os.c @@ -141,6 +141,8 @@ int sqlite3OsShmMap( return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } +#if !defined(SQLITE_DISABLE_MMAP) +/* The real implementation of xFetch and xUnfetch */ int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ DO_OS_MALLOC_TEST(id); return id->pMethods->xFetch(id, iOff, iAmt, pp); @@ -148,6 +150,16 @@ int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ return id->pMethods->xUnfetch(id, iOff, p); } +#else +/* No-op stubs to use when memory-mapped I/O is disabled */ +int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ + *pp = 0; + return SQLITE_OK; +} +int sqlite3OsUnfetch(sqlite3_file *id, i64 iOff, void *p){ + return SQLITE_OK; +} +#endif /* ** The next group of routines are convenience wrappers around the diff --git a/src/pager.c b/src/pager.c index ae2f10069f..e3ac49cd47 100644 --- a/src/pager.c +++ b/src/pager.c @@ -770,6 +770,16 @@ static const unsigned char aJournalMagic[] = { # define MEMDB pPager->memDb #endif +/* +** The macro USEFETCH is true if we are allowed to use the xFetch and xUnfetch +** interfaces to access the database using memory-mapped I/O. +*/ +#ifdef SQLITE_DISABLE_MMAP +# define USEFETCH(x) 0 +#else +# define USEFETCH(x) ((x)->bUseFetch) +#endif + /* ** The maximum legal page number is (2^31 - 1). */ @@ -3090,7 +3100,7 @@ static int pagerBeginReadTransaction(Pager *pPager){ rc = sqlite3WalBeginReadTransaction(pPager->pWal, &changed); if( rc!=SQLITE_OK || changed ){ pager_reset(pPager); - if( pPager->bUseFetch ) sqlite3OsUnfetch(pPager->fd, 0, 0); + if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); } return rc; @@ -3356,6 +3366,7 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ ** Invoke SQLITE_FCNTL_MMAP_LIMIT based on the current value of mxMmap. */ static void pagerFixMaplimit(Pager *pPager){ +#if !defined(SQLITE_DISABLE_MMAP) sqlite3_file *fd = pPager->fd; if( isOpen(fd) ){ sqlite3_int64 mx; @@ -3363,6 +3374,7 @@ static void pagerFixMaplimit(Pager *pPager){ mx = pPager->mxMmap; sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT, &mx); } +#endif } /* @@ -5042,7 +5054,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ if( !pPager->tempFile && ( pPager->pBackup || sqlite3PcachePagecount(pPager->pPCache)>0 - || pPager->bUseFetch + || USEFETCH(pPager) )){ /* The shared-lock has just been acquired on the database file ** and there are already pages in the cache (from a previous @@ -5085,7 +5097,7 @@ int sqlite3PagerSharedLock(Pager *pPager){ ** In this case there may exist a Pager.pMap mapping that appears ** to be the right size but is not actually valid. Avoid this ** possibility by unmapping the db here. */ - if( pPager->bUseFetch ){ + if( USEFETCH(pPager) ){ sqlite3OsUnfetch(pPager->fd, 0, 0); } } @@ -5199,7 +5211,7 @@ int sqlite3PagerAcquire( ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY ** flag was specified by the caller. And so long as the db is not a ** temporary or in-memory database. */ - const int bMmapOk = (pgno!=1 && pPager->bUseFetch + const int bMmapOk = (pgno!=1 && USEFETCH(pPager) && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY)) ); diff --git a/src/pragma.c b/src/pragma.c index 3527c0974c..340e62d0dd 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -774,6 +774,9 @@ void sqlite3Pragma( } mx = -1; if( sqlite3_file_control(db,zDb,SQLITE_FCNTL_MMAP_LIMIT,&mx)==SQLITE_OK ){ +#if defined(SQLITE_DISABLE_MMAP) + mx = 0; +#endif returnSingleInt(pParse, "mmap_limit", mx); } }else diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d894051790..a528e37e44 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -539,6 +539,14 @@ extern const int sqlite3one; # define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&7)==0) #endif +/* +** Disable MMAP on platforms where it is not supported +*/ +#if defined(__OpenBSD__) || defined(__QNXNTO__) +# undef SQLITE_DISABLE_MMAP +# define SQLITE_DISABLE_MMAP 1 +#endif + /* ** An instance of the following structure is used to store the busy-handler diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index d3a3b4ac95..d8f748e6b2 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -210,6 +210,10 @@ /* ** Default maximum size of memory used by xFetch in the VFS. */ +#ifdef SQLITE_DISABLE_MMAP +# undef SQLITE_DEFAULT_MMAP_LIMIT +# define SQLITE_DEFAULT_MMAP_LIMIT 0 +#endif #ifdef __APPLE__ # include # if TARGET_OS_IPHONE diff --git a/src/test_config.c b/src/test_config.c index f3c2f61162..39608dbb64 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -87,6 +87,12 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "lfs", "1", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_DISABLE_MMAP + Tcl_SetVar2(interp, "sqlite_options", "mmap", "0", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "mmap", "1", TCL_GLOBAL_ONLY); +#endif + #if 1 /* def SQLITE_MEMDEBUG */ Tcl_SetVar2(interp, "sqlite_options", "memdebug", "1", TCL_GLOBAL_ONLY); #else diff --git a/test/mmap1.test b/test/mmap1.test index 417a0756b1..9618109933 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -12,6 +12,10 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +ifcapable !mmap { + finish_test + return +} source $testdir/lock_common.tcl set testprefix mmap1 @@ -311,5 +315,3 @@ foreach {tn1 mmap1 mmap2} { finish_test - - diff --git a/test/mmap2.test b/test/mmap2.test index 33830a6943..2c80a0eb7f 100644 --- a/test/mmap2.test +++ b/test/mmap2.test @@ -25,6 +25,10 @@ if {$::tcl_platform(platform)!="unix" || [test_syscall defaultvfs] != "unix"} { finish_test return } +ifcapable !mmap { + finish_test + return +} db close sqlite3_shutdown diff --git a/test/pageropt.test b/test/pageropt.test index 10d3d4917e..a734e39a49 100644 --- a/test/pageropt.test +++ b/test/pageropt.test @@ -87,12 +87,17 @@ do_test pageropt-1.4 { # But if the other thread modifies the database, then the cache # must refill. # +ifcapable mmap { + set x [expr {[permutation]=="nommap" ? 6 : 1}] +} else { + set x 6 +} do_test pageropt-1.5 { db2 eval {CREATE TABLE t2(y)} pagercount_sql { SELECT hex(x) FROM t1 } -} [list [expr {[permutation]=="nommap" ? 6 : 1}] 0 0 $blobcontent] +} [list $x 0 0 $blobcontent] do_test pageropt-1.6 { pagercount_sql { SELECT hex(x) FROM t1 diff --git a/test/wal.test b/test/wal.test index 0539192d25..b997155540 100644 --- a/test/wal.test +++ b/test/wal.test @@ -729,6 +729,7 @@ do_test wal-11.9 { sqlite3_wal db test.db set nWal 39 if {[permutation]=="nommap"} {set nWal 37} +ifcapable !mmap {set nWal 37} do_test wal-11.10 { execsql { PRAGMA cache_size = 10; diff --git a/test/wal5.test b/test/wal5.test index ebd1dbfc31..4a6309886b 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -243,6 +243,7 @@ foreach {testprefix do_wal_checkpoint} { # start of the checkpoint, even though page 2 cannot be written. set nDb 2 if {[permutation]=="no-mmap"} {set nDb 1} + ifcapable !mmap {set nDb 1} do_test 2.3.$tn.8 { file_page_counts } [list $nDb 4 2 4] } From 6e0b6d52da6a6245c97a40997a95f8cb14fed502 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 9 Apr 2013 16:19:20 +0000 Subject: [PATCH 099/140] Add extra #ifndef statements in os_unix.c and os_win.c to make sure the memory mapped I/O really is disabled when SQLITE_DISABLE_MMAP is set. FossilOrigin-Name: c1e2523c9051782569291fff998140f7e0b70b6d --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 12 ++++++++++++ src/os_win.c | 12 ++++++++++++ 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index fdbb616c1d..7f8d4dc923 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sthe\suse\sof\smemory-mapped\sI/O\sif\sthe\sSQLITE_DISABLE_MMAP\smacro\sis\ndefined.\s\sAutomatically\sdefine\sthis\smacro\sfor\sOpenBSD\sand\sQNX.\s\sOther\nsystems\sare\slikely\sto\sbe\sadded\sto\sthe\sdisabled\slist\sover\stime. -D 2013-04-08T20:47:49.018 +C Add\sextra\s#ifndef\sstatements\sin\sos_unix.c\sand\sos_win.c\sto\smake\ssure\sthe\nmemory\smapped\sI/O\sreally\sis\sdisabled\swhen\sSQLITE_DISABLE_MMAP\sis\sset. +D 2013-04-09T16:19:20.624 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,8 +166,8 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c ca679b293a6233327e418fd1dde2cd5db3e90932 F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 6d584190db2ed442e5c849b7f73dcfbbac7c032e -F src/os_win.c 3265df8c762e0b8caf8d02b3352fa8c22e91ae6b +F src/os_unix.c a6e9caadd8956a39153d1bd920e459211a281388 +F src/os_win.c 3aec291c40048653c00a153fee96d3a442ab85ee F src/pager.c eb6d272f4dc1ade4b8f664599cd54a9c95cb2729 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ae8bb24bfb014bdcb8aaa8159ddae0cd376afe99 -R 7e782b1e3841573f5a2d86903a4acfcf +P 8a4314a398c6c6efb25e6993c826b7e60d9bb1f9 +R 815c6131bff16243fe333d79ef346ae3 U drh -Z a7d6d975495e51a139482ed48c14db14 +Z 73d68bfce668b2e7f104132473fd4e42 diff --git a/manifest.uuid b/manifest.uuid index 387ecfb3dc..991d418bcd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8a4314a398c6c6efb25e6993c826b7e60d9bb1f9 \ No newline at end of file +c1e2523c9051782569291fff998140f7e0b70b6d \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 1d4c8ad348..a65f894eb8 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3117,6 +3117,7 @@ static int unixRead( ); #endif +#if !defined(SQLITE_DISABLE_MMAP) /* Deal with as much of this read request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ @@ -3131,6 +3132,7 @@ static int unixRead( offset += nCopy; } } +#endif got = seekAndRead(pFile, offset, pBuf, amt); if( got==amt ){ @@ -3236,6 +3238,7 @@ static int unixWrite( } #endif +#if !defined(SQLITE_DISABLE_MMAP) /* Deal with as much of this write request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ @@ -3250,6 +3253,7 @@ static int unixWrite( offset += nCopy; } } +#endif while( amt>0 && (wrote = seekAndWrite(pFile, offset, pBuf, amt))>0 ){ amt -= wrote; @@ -4525,12 +4529,14 @@ static int unixShmUnmap( */ static void unixUnmapfile(unixFile *pFd){ assert( pFd->nFetchOut==0 ); +#ifndef SQLITE_DISABLE_MMAP if( pFd->pMapRegion ){ osMunmap(pFd->pMapRegion, pFd->mmapOrigsize); pFd->pMapRegion = 0; pFd->mmapSize = 0; pFd->mmapOrigsize = 0; } +#endif } /* @@ -4546,6 +4552,7 @@ static int unixGetPagesize(void){ #endif } +#ifndef SQLITE_DISABLE_MMAP /* ** Attempt to set the size of the memory mapping maintained by file ** descriptor pFd to nNew bytes. Any existing mapping is discarded. @@ -4630,6 +4637,7 @@ static void unixRemapfile( pFd->pMapRegion = (void *)pNew; pFd->mmapSize = pFd->mmapOrigsize = nNew; } +#endif /* ** Memory map or remap the file opened by file-descriptor pFd (if the file @@ -4651,6 +4659,7 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ i64 nMap = nByte; int rc; +#ifndef SQLITE_DISABLE_MMAP assert( nMap>=0 || pFd->nFetchOut==0 ); if( pFd->nFetchOut>0 ) return SQLITE_OK; @@ -4673,6 +4682,7 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ unixUnmapfile(pFd); } } +#endif return SQLITE_OK; } @@ -4693,6 +4703,7 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ unixFile *pFd = (unixFile *)fd; /* The underlying database file */ *pp = 0; +#ifndef SQLITE_DISABLE_MMAP if( pFd->mmapLimit>0 ){ if( pFd->pMapRegion==0 ){ int rc = unixMapfile(pFd, -1); @@ -4703,6 +4714,7 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ pFd->nFetchOut++; } } +#endif return SQLITE_OK; } diff --git a/src/os_win.c b/src/os_win.c index a7966171b5..220ebac268 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2148,6 +2148,7 @@ static int winRead( SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); +#if !defined(SQLITE_DISABLE_MMAP) /* Deal with as much of this read request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ @@ -2162,6 +2163,7 @@ static int winRead( offset += nCopy; } } +#endif #if SQLITE_OS_WINCE if( seekWinFile(pFile, offset) ){ @@ -2212,6 +2214,7 @@ static int winWrite( OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); +#if !defined(SQLITE_DISABLE_MMAP) /* Deal with as much of this write request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ @@ -2226,6 +2229,7 @@ static int winWrite( offset += nCopy; } } +#endif #if SQLITE_OS_WINCE rc = seekWinFile(pFile, offset); @@ -2321,6 +2325,7 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ "winTruncate2", pFile->zPath); } +#if !defined(SQLITE_DISABLE_MMAP) /* If the file was truncated to a size smaller than the currently ** mapped region, reduce the effective mapping size as well. SQLite will ** use read() and write() to access data beyond this point from now on. @@ -2328,6 +2333,7 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ if( pFile->pMapRegion && nBytemmapSize ){ pFile->mmapSize = nByte; } +#endif OSTRACE(("TRUNCATE %d %lld %s\n", pFile->h, nByte, rc ? "failed" : "ok")); return rc; @@ -3517,6 +3523,7 @@ shmpage_out: */ static int winUnmapfile(winFile *pFile){ assert( pFile!=0 ); +#if !defined(SQLITE_DISABLE_MMAP) if( pFile->pMapRegion ){ if( !osUnmapViewOfFile(pFile->pMapRegion) ){ pFile->lastErrno = osGetLastError(); @@ -3535,9 +3542,11 @@ static int winUnmapfile(winFile *pFile){ } pFile->hMap = NULL; } +#endif return SQLITE_OK; } +#if !defined(SQLITE_DISABLE_MMAP) /* ** Memory map or remap the file opened by file-descriptor pFd (if the file ** is already mapped, the existing mapping is replaced by the new). Or, if @@ -3625,6 +3634,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ return SQLITE_OK; } +#endif /* !defined(SQLITE_DISABLE_MMAP) */ /* ** If possible, return a pointer to a mapping of file fd starting at offset @@ -3642,6 +3652,7 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ winFile *pFd = (winFile*)fd; /* The underlying database file */ *pp = 0; +#if !defined(SQLITE_DISABLE_MMAP) if( pFd->mmapLimit>0 ){ if( pFd->pMapRegion==0 ){ int rc = winMapfile(pFd, -1); @@ -3652,6 +3663,7 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ pFd->nFetchOut++; } } +#endif return SQLITE_OK; } From ab755ac81a0c4fd17c8de2f777b757ca50070b58 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 9 Apr 2013 18:36:36 +0000 Subject: [PATCH 100/140] Change the rollback journal so that it invokes sqlite3_log() make a record of a recovery, just as the WAL journal does. FossilOrigin-Name: 7cd3f6cd3a39ed1c4bbf9e3508824150632c5bd9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- mptest/mptest.c | 6 +++++- src/pager.c | 9 ++++++++- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 7f8d4dc923..982e5065c0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\s#ifndef\sstatements\sin\sos_unix.c\sand\sos_win.c\sto\smake\ssure\sthe\nmemory\smapped\sI/O\sreally\sis\sdisabled\swhen\sSQLITE_DISABLE_MMAP\sis\sset. -D 2013-04-09T16:19:20.624 +C Change\sthe\srollback\sjournal\sso\sthat\sit\sinvokes\ssqlite3_log()\smake\sa\srecord\nof\sa\srecovery,\sjust\sas\sthe\sWAL\sjournal\sdoes. +D 2013-04-09T18:36:36.048 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da F mptest/config02.test 962913ed2b537d60de4126db7fe54716865cdd22 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest c4647feafa04109e1dc60121a895b6cba8d63ceb -F mptest/mptest.c e31bb60d3de960aa0f7d08af7ae59c6dfecc552b +F mptest/mptest.c c9fc81464dcd2de18ceb5317c5e1b075cb3b9fce F mptest/multiwrite01.test d6cb707062268e65396bd71eb53e9d5edb5519c0 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -168,7 +168,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c a6e9caadd8956a39153d1bd920e459211a281388 F src/os_win.c 3aec291c40048653c00a153fee96d3a442ab85ee -F src/pager.c eb6d272f4dc1ade4b8f664599cd54a9c95cb2729 +F src/pager.c cc60816e677528e4c2e23bbf56ee5c81f4cdfe58 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 8a4314a398c6c6efb25e6993c826b7e60d9bb1f9 -R 815c6131bff16243fe333d79ef346ae3 +P c1e2523c9051782569291fff998140f7e0b70b6d +R 0d15a7f4fe4e9fd889905fd047909cd0 U drh -Z 73d68bfce668b2e7f104132473fd4e42 +Z 5a68f7be27490e18f34eef73cb3a90fe diff --git a/manifest.uuid b/manifest.uuid index 991d418bcd..62d2bcb7d9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c1e2523c9051782569291fff998140f7e0b70b6d \ No newline at end of file +7cd3f6cd3a39ed1c4bbf9e3508824150632c5bd9 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index f14a640e8c..7a8bb08c8b 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -323,7 +323,11 @@ static void sqlErrorCallback(void *pArg, int iErrCode, const char *zMsg){ UNUSED_PARAMETER(pArg); if( (iErrCode&0xff)==SQLITE_SCHEMA && g.iTrace<3 ) return; if( g.iTimeout==0 && (iErrCode&0xff)==SQLITE_BUSY && g.iTrace<3 ) return; - errorMessage("(errcode=%d) %s", iErrCode, zMsg); + if( iErrCode==SQLITE_OK ){ + logMessage("(info) %s", zMsg); + }else{ + errorMessage("(errcode=%d) %s", iErrCode, zMsg); + } } /* diff --git a/src/pager.c b/src/pager.c index e3ac49cd47..d35d91ac12 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2658,6 +2658,7 @@ static int pager_playback(Pager *pPager, int isHot){ int res = 1; /* Value returned by sqlite3OsAccess() */ char *zMaster = 0; /* Name of master journal file if any */ int needPagerReset; /* True to reset page prior to first page rollback */ + int nPlayback = 0; /* Total number of pages restored from journal */ /* Figure out how many records are in the journal. Abort early if ** the journal is empty. @@ -2758,7 +2759,9 @@ static int pager_playback(Pager *pPager, int isHot){ needPagerReset = 0; } rc = pager_playback_one_page(pPager,&pPager->journalOff,0,1,0); - if( rc!=SQLITE_OK ){ + if( rc==SQLITE_OK ){ + nPlayback++; + }else{ if( rc==SQLITE_DONE ){ pPager->journalOff = szJ; break; @@ -2828,6 +2831,10 @@ end_playback: rc = pager_delmaster(pPager, zMaster); testcase( rc!=SQLITE_OK ); } + if( isHot && nPlayback ){ + sqlite3_log(SQLITE_OK, "Recovered %d pages from %s", + nPlayback, pPager->zJournal); + } /* The Pager.sectorSize variable may have been updated while rolling ** back a journal created by a process with a different sector size From 7de96f7fc1e825d6d69dc0475ef412a1f6a82758 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 9 Apr 2013 20:04:09 +0000 Subject: [PATCH 101/140] Limit integrity_check output to 10 lines in mptester scripts. FossilOrigin-Name: 67ee0dc0f4de5dc2ae92fb98932f1fdda3e76bdf --- manifest | 14 +++++++------- manifest.uuid | 2 +- mptest/crash02.subtest | 8 ++++---- mptest/multiwrite01.test | 12 ++++++------ 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 982e5065c0..3bcf945534 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\srollback\sjournal\sso\sthat\sit\sinvokes\ssqlite3_log()\smake\sa\srecord\nof\sa\srecovery,\sjust\sas\sthe\sWAL\sjournal\sdoes. -D 2013-04-09T18:36:36.048 +C Limit\sintegrity_check\soutput\sto\s10\slines\sin\smptester\sscripts. +D 2013-04-09T20:04:09.979 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -113,9 +113,9 @@ F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da F mptest/config02.test 962913ed2b537d60de4126db7fe54716865cdd22 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 -F mptest/crash02.subtest c4647feafa04109e1dc60121a895b6cba8d63ceb +F mptest/crash02.subtest 86641730725895da7baf6443bc26aa1c8e781711 F mptest/mptest.c c9fc81464dcd2de18ceb5317c5e1b075cb3b9fce -F mptest/multiwrite01.test d6cb707062268e65396bd71eb53e9d5edb5519c0 +F mptest/multiwrite01.test 8fa273e97284edaa855f0b0e99d78216e204be15 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c1e2523c9051782569291fff998140f7e0b70b6d -R 0d15a7f4fe4e9fd889905fd047909cd0 +P 7cd3f6cd3a39ed1c4bbf9e3508824150632c5bd9 +R c57dbdff2c71d3b29909dfda1d0fd94f U drh -Z 5a68f7be27490e18f34eef73cb3a90fe +Z 71d9517d85d0a37449eba7c0abca70d4 diff --git a/manifest.uuid b/manifest.uuid index 62d2bcb7d9..0d698f714e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7cd3f6cd3a39ed1c4bbf9e3508824150632c5bd9 \ No newline at end of file +67ee0dc0f4de5dc2ae92fb98932f1fdda3e76bdf \ No newline at end of file diff --git a/mptest/crash02.subtest b/mptest/crash02.subtest index 053a9faf93..320f788a70 100644 --- a/mptest/crash02.subtest +++ b/mptest/crash02.subtest @@ -26,7 +26,7 @@ SELECT count(*) FROM t1; --match 64 --sleep 100 - PRAGMA integrity_check; + PRAGMA integrity_check(10); --output --match ok --end @@ -34,7 +34,7 @@ SELECT count(*) FROM t1; --match 64 --sleep 100 - PRAGMA integrity_check; + PRAGMA integrity_check(10); --output --match ok --end @@ -42,7 +42,7 @@ SELECT count(*) FROM t1; --match 64 --sleep 100 - PRAGMA integrity_check; + PRAGMA integrity_check(10); --output --match ok --end @@ -50,7 +50,7 @@ SELECT count(*) FROM t1; --match 64 --sleep 100 - PRAGMA integrity_check; + PRAGMA integrity_check(10); --output --match ok --end diff --git a/mptest/multiwrite01.test b/mptest/multiwrite01.test index dc7d62b8f9..7fb35a25a7 100644 --- a/mptest/multiwrite01.test +++ b/mptest/multiwrite01.test @@ -225,35 +225,35 @@ SELECT count(*), total(length(b)) FROM t5; --task 5 DROP INDEX t5b; --sleep 5 - PRAGMA integrity_check; + PRAGMA integrity_check(10); --match ok CREATE INDEX t5b ON t5(b DESC); --end --task 3 DROP INDEX t3b; --sleep 5 - PRAGMA integrity_check; + PRAGMA integrity_check(10); --match ok CREATE INDEX t3b ON t3(b DESC); --end --task 1 DROP INDEX t1b; --sleep 5 - PRAGMA integrity_check; + PRAGMA integrity_check(10); --match ok CREATE INDEX t1b ON t1(b DESC); --end --task 2 DROP INDEX t2b; --sleep 5 - PRAGMA integrity_check; + PRAGMA integrity_check(10); --match ok CREATE INDEX t2b ON t2(b DESC); --end --task 4 DROP INDEX t4b; --sleep 5 - PRAGMA integrity_check; + PRAGMA integrity_check(10); --match ok CREATE INDEX t4b ON t4(b DESC); --end @@ -312,7 +312,7 @@ SELECT count(*), total(length(b)) FROM t5; --wait all VACUUM; -PRAGMA integrity_check; +PRAGMA integrity_check(10); --match ok --task 1 From 65eb64765c6b573848028d7a68077e8582bd9677 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 10 Apr 2013 02:56:24 +0000 Subject: [PATCH 102/140] Correct test numbering on several rtree tests. FossilOrigin-Name: 0adf8a3a4f2ca370568b35c9e8ce0e18f18701a2 --- ext/rtree/rtree1.test | 1 + ext/rtree/rtree5.test | 2 +- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/ext/rtree/rtree1.test b/ext/rtree/rtree1.test index e3c7d68e82..275b13264f 100644 --- a/ext/rtree/rtree1.test +++ b/ext/rtree/rtree1.test @@ -17,6 +17,7 @@ if {![info exists testdir]} { } source [file join [file dirname [info script]] rtree_util.tcl] source $testdir/tester.tcl +set testprefix rtree1 # Test plan: # diff --git a/ext/rtree/rtree5.test b/ext/rtree/rtree5.test index 8990772356..8ff90b0cb7 100644 --- a/ext/rtree/rtree5.test +++ b/ext/rtree/rtree5.test @@ -61,7 +61,7 @@ do_test rtree5-1.9 { do_test rtree5-1.10 { execsql { SELECT (1<<31)-5, (1<<31)-1, -1*(1<<31), -1*(1<<31)+5 } } {2147483643 2147483647 -2147483648 -2147483643} -do_test rtree5-1.10 { +do_test rtree5-1.11 { execsql { INSERT INTO t1 VALUES(2, (1<<31)-5, (1<<31)-1, -1*(1<<31), -1*(1<<31)+5) } diff --git a/manifest b/manifest index 3bcf945534..6ddc91d8f5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Limit\sintegrity_check\soutput\sto\s10\slines\sin\smptester\sscripts. -D 2013-04-09T20:04:09.979 +C Correct\stest\snumbering\son\sseveral\srtree\stests. +D 2013-04-10T02:56:24.509 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -85,11 +85,11 @@ F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 757abea591d4ff67c0ff4e8f9776aeda86b18c14 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e -F ext/rtree/rtree1.test e474a2b5eff231496dbd073fe67e5fbaf7f444c9 +F ext/rtree/rtree1.test cf679265ecafff494a768ac9c2f43a70915a6290 F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba F ext/rtree/rtree3.test a494da55c30ee0bc9b01a91c80c81b387b22d2dc F ext/rtree/rtree4.test c8fe384f60ebd49540a5fecc990041bf452eb6e0 -F ext/rtree/rtree5.test 9a229678a00f40e6aedb40cb3a07ec5444af892c +F ext/rtree/rtree5.test 6a510494f12454bf57ef28f45bc7764ea279431e F ext/rtree/rtree6.test 3ff9113b4a872fa935309e3511cd9b7cdb4d2472 F ext/rtree/rtree7.test 1fa710b9e6bf997a0c1a537b81be7bb6fded1971 F ext/rtree/rtree8.test 9772e16da71e17e02bdebf0a5188590f289ab37d @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 7cd3f6cd3a39ed1c4bbf9e3508824150632c5bd9 -R c57dbdff2c71d3b29909dfda1d0fd94f -U drh -Z 71d9517d85d0a37449eba7c0abca70d4 +P 67ee0dc0f4de5dc2ae92fb98932f1fdda3e76bdf +R 1390797dd2d0e0c300030d714045b447 +U mistachkin +Z 8596d7b58bb179eaa78f2319025347e8 diff --git a/manifest.uuid b/manifest.uuid index 0d698f714e..ed3822bd21 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -67ee0dc0f4de5dc2ae92fb98932f1fdda3e76bdf \ No newline at end of file +0adf8a3a4f2ca370568b35c9e8ce0e18f18701a2 \ No newline at end of file From 21c9b6bdc9dc939c113628142be2482227384f8c Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 10 Apr 2013 03:06:43 +0000 Subject: [PATCH 103/140] Fix typo in name of mptest executable files in clean targets. FossilOrigin-Name: 4c7d9e1ed8722e0c75b5fdbeffb9030ccef0bd64 --- Makefile.in | 2 +- main.mk | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Makefile.in b/Makefile.in index 0b702def79..486333157e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -955,7 +955,7 @@ clean: rm -f shell.c sqlite3ext.h rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c rm -f sqlite-*-output.vsix - rm -f mptest mptest.exe + rm -f mptester mptester.exe distclean: clean rm -f config.log config.status libtool Makefile sqlite3.pc diff --git a/main.mk b/main.mk index 7ab2bca2ba..2ec009a11d 100644 --- a/main.mk +++ b/main.mk @@ -630,4 +630,4 @@ clean: rm -f shell.c sqlite3ext.h rm -f sqlite3_analyzer sqlite3_analyzer.exe sqlite3_analyzer.c rm -f sqlite-*-output.vsix - rm -f mptest mptest.exe + rm -f mptester mptester.exe diff --git a/manifest b/manifest index 6ddc91d8f5..19b2e38f48 100644 --- a/manifest +++ b/manifest @@ -1,7 +1,7 @@ -C Correct\stest\snumbering\son\sseveral\srtree\stests. -D 2013-04-10T02:56:24.509 +C Fix\stypo\sin\sname\sof\smptest\sexecutable\sfiles\sin\sclean\stargets. +D 2013-04-10T03:06:43.090 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 6a6a8c548822302d3a9a651f0296bbc3dad55e83 +F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc a244e5b5ba2493625db95b4d46b79201ea868bfe F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 @@ -103,7 +103,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk 1c26f3cc57d7c357da5218ae2928b7ca5acfecde +F main.mk dcb05737414488383369b93e33cf1a1484c4b6e6 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 67ee0dc0f4de5dc2ae92fb98932f1fdda3e76bdf -R 1390797dd2d0e0c300030d714045b447 +P 0adf8a3a4f2ca370568b35c9e8ce0e18f18701a2 +R 87c959156377d2c80b90d9a959edeb9c U mistachkin -Z 8596d7b58bb179eaa78f2319025347e8 +Z f06e039212bcf2d4f3c2b172131c9afb diff --git a/manifest.uuid b/manifest.uuid index ed3822bd21..f78573fb87 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0adf8a3a4f2ca370568b35c9e8ce0e18f18701a2 \ No newline at end of file +4c7d9e1ed8722e0c75b5fdbeffb9030ccef0bd64 \ No newline at end of file From 4c5298f352c92a723a37d4ccab701f6c28e9f680 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 10 Apr 2013 12:01:21 +0000 Subject: [PATCH 104/140] Use symbolic names for tasks in mptester. FossilOrigin-Name: f0d95afc73f8dbce8943dceb4a14b7de650c8823 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- mptest/crash02.subtest | 14 +++++--------- mptest/mptest.c | 37 +++++++++++++++++++++++++++---------- mptest/multiwrite01.test | 10 +++++----- 5 files changed, 47 insertions(+), 34 deletions(-) diff --git a/manifest b/manifest index 19b2e38f48..a97a348927 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypo\sin\sname\sof\smptest\sexecutable\sfiles\sin\sclean\stargets. -D 2013-04-10T03:06:43.090 +C Use\ssymbolic\snames\sfor\stasks\sin\smptester. +D 2013-04-10T12:01:21.074 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -113,9 +113,9 @@ F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da F mptest/config02.test 962913ed2b537d60de4126db7fe54716865cdd22 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 -F mptest/crash02.subtest 86641730725895da7baf6443bc26aa1c8e781711 -F mptest/mptest.c c9fc81464dcd2de18ceb5317c5e1b075cb3b9fce -F mptest/multiwrite01.test 8fa273e97284edaa855f0b0e99d78216e204be15 +F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 +F mptest/mptest.c d5228be2db65634783b302423704b3928284adb5 +F mptest/multiwrite01.test 81fbc17657964889b60750bd7bbb1deffe8f4d42 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 0adf8a3a4f2ca370568b35c9e8ce0e18f18701a2 -R 87c959156377d2c80b90d9a959edeb9c -U mistachkin -Z f06e039212bcf2d4f3c2b172131c9afb +P 4c7d9e1ed8722e0c75b5fdbeffb9030ccef0bd64 +R f44444aacae5f7c52822e030959d44b9 +U drh +Z f26dfaa28d7c4e704618af941a7b95f6 diff --git a/manifest.uuid b/manifest.uuid index f78573fb87..e0711c6d56 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4c7d9e1ed8722e0c75b5fdbeffb9030ccef0bd64 \ No newline at end of file +f0d95afc73f8dbce8943dceb4a14b7de650c8823 \ No newline at end of file diff --git a/mptest/crash02.subtest b/mptest/crash02.subtest index 320f788a70..86f64ddbe7 100644 --- a/mptest/crash02.subtest +++ b/mptest/crash02.subtest @@ -4,7 +4,7 @@ ** journal in client 1, then crash client 1. ** Then in the other clients, do an integrity check. */ ---task 1 +--task 1 leave-hot-journal --sleep 5 --finish PRAGMA cache_size=10; @@ -22,36 +22,32 @@ --print Task one crashing an incomplete transaction --exit 1 --end ---task 2 +--task 2 integrity_check-2 SELECT count(*) FROM t1; --match 64 --sleep 100 PRAGMA integrity_check(10); - --output --match ok --end ---task 3 +--task 3 integrity_check-3 SELECT count(*) FROM t1; --match 64 --sleep 100 PRAGMA integrity_check(10); - --output --match ok --end ---task 4 +--task 4 integrity_check-4 SELECT count(*) FROM t1; --match 64 --sleep 100 PRAGMA integrity_check(10); - --output --match ok --end ---task 5 +--task 5 integrity_check-5 SELECT count(*) FROM t1; --match 64 --sleep 100 PRAGMA integrity_check(10); - --output --match ok --end --wait all diff --git a/mptest/mptest.c b/mptest/mptest.c index 7a8bb08c8b..6de32b0a05 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -520,7 +520,8 @@ static void evalFunc( static int startScript( int iClient, /* The client number */ char **pzScript, /* Write task script here */ - int *pTaskId /* Write task number here */ + int *pTaskId, /* Write task number here */ + char **pzTaskName /* Name of the task */ ){ sqlite3_stmt *pStmt = 0; int taskId; @@ -555,7 +556,7 @@ static int startScript( return SQLITE_DONE; } pStmt = prepareSql( - "SELECT script, id FROM task" + "SELECT script, id, name FROM task" " WHERE client=%d AND starttime IS NULL" " ORDER BY id LIMIT 1", iClient); rc = sqlite3_step(pStmt); @@ -564,6 +565,7 @@ static int startScript( *pzScript = sqlite3_malloc(n+1); strcpy(*pzScript, (const char*)sqlite3_column_text(pStmt, 0)); *pTaskId = taskId = sqlite3_column_int(pStmt, 1); + *pzTaskName = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 2)); sqlite3_finalize(pStmt); runSql("UPDATE task" " SET starttime=strftime('%%Y-%%m-%%d %%H:%%M:%%f','now')" @@ -804,6 +806,14 @@ static void waitForClient(int iClient, int iTimeout, char *zErrPrefix){ } } +/* Return a pointer to the tail of a filename +*/ +static char *filenameTail(char *z){ + int i, j; + for(i=j=0; z[i]; i++) if( z[i]=='/' ) j = i+1; + return z+j; +} + /* Maximum number of arguments to a --command */ #define MX_ARG 2 @@ -1039,16 +1049,23 @@ static void runScript( int iTarget = atoi(azArg[0]); int iEnd; char *zTask; + char *zTName; iEnd = findEnd(zScript+ii+len, &lineno); if( iTarget<0 ){ errorMessage("line %d of %s: bad client number: %d", prevLine, zFilename, iTarget); }else{ zTask = sqlite3_mprintf("%.*s", iEnd, zScript+ii+len); + if( nArg>1 ){ + zTName = sqlite3_mprintf("%s", azArg[1]); + }else{ + zTName = sqlite3_mprintf("%s:%d", filenameTail(zFilename), prevLine); + } startClient(iTarget); - runSql("INSERT INTO task(client,script)" - " VALUES(%d,'%q')", iTarget, zTask); + runSql("INSERT INTO task(client,script,name)" + " VALUES(%d,'%q',%Q)", iTarget, zTask, zTName); sqlite3_free(zTask); + sqlite3_free(zTName); } iEnd += tokenLength(zScript+ii+len+iEnd, &lineno); len += iEnd; @@ -1217,15 +1234,14 @@ int main(int argc, char **argv){ if( n>0 ) unrecognizedArguments(argv[0], n, argv+2); if( g.iTrace ) logMessage("start-client"); while(1){ - char zTaskName[50]; - rc = startScript(iClient, &zScript, &taskId); + char *zTaskName = 0; + rc = startScript(iClient, &zScript, &taskId, &zTaskName); if( rc==SQLITE_DONE ) break; - sqlite3_snprintf(sizeof(zTaskName), zTaskName, "client%02d-task-%d", - iClient, taskId); - if( g.iTrace ) logMessage("begin %s", zTaskName); + if( g.iTrace ) logMessage("begin %s (%d)", zTaskName, taskId); runScript(iClient, taskId, zScript, zTaskName); - if( g.iTrace ) logMessage("end %s", zTaskName); + if( g.iTrace ) logMessage("end %s (%d)", zTaskName, taskId); finishScript(iClient, taskId, 0); + sqlite3_free(zTaskName); sqlite3_sleep(10); } if( g.iTrace ) logMessage("end-client"); @@ -1239,6 +1255,7 @@ int main(int argc, char **argv){ runSql( "CREATE TABLE task(\n" " id INTEGER PRIMARY KEY,\n" + " name TEXT,\n" " client INTEGER,\n" " starttime DATE,\n" " endtime DATE,\n" diff --git a/mptest/multiwrite01.test b/mptest/multiwrite01.test index 7fb35a25a7..9b3ccb8473 100644 --- a/mptest/multiwrite01.test +++ b/mptest/multiwrite01.test @@ -2,7 +2,7 @@ ** This script sets up five different tasks all writing and updating ** the database at the same time, but each in its own table. */ ---task 1 +--task 1 build-t1 DROP TABLE IF EXISTS t1; CREATE TABLE t1(a INTEGER PRIMARY KEY, b); --sleep 1 @@ -34,7 +34,7 @@ --end ---task 2 +--task 2 build-t2 DROP TABLE IF EXISTS t2; CREATE TABLE t2(a INTEGER PRIMARY KEY, b); --sleep 1 @@ -65,7 +65,7 @@ --match 29 28 27 26 25 --end ---task 3 +--task 3 build-t3 DROP TABLE IF EXISTS t3; CREATE TABLE t3(a INTEGER PRIMARY KEY, b); --sleep 1 @@ -96,7 +96,7 @@ --match 29 28 27 26 25 --end ---task 4 +--task 4 build-t4 DROP TABLE IF EXISTS t4; CREATE TABLE t4(a INTEGER PRIMARY KEY, b); --sleep 1 @@ -127,7 +127,7 @@ --match 29 28 27 26 25 --end ---task 5 +--task 5 build-t5 DROP TABLE IF EXISTS t5; CREATE TABLE t5(a INTEGER PRIMARY KEY, b); --sleep 1 From 56282a5bc75945ef9953f0cab3faff8b86af69f0 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 10 Apr 2013 16:13:38 +0000 Subject: [PATCH 105/140] Add the sqlite3_strglob() interface. FossilOrigin-Name: 41d6ff32a6dd1311dc3aabe5156335a64a886919 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/func.c | 7 +++++++ src/sqlite.h.in | 15 +++++++++++++++ 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a97a348927..5efbde65d6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\ssymbolic\snames\sfor\stasks\sin\smptester. -D 2013-04-10T12:01:21.074 +C Add\sthe\ssqlite3_strglob()\sinterface. +D 2013-04-10T16:13:38.085 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -139,7 +139,7 @@ F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c e16942bd5c8a868ac53287886464a5ed0e72b179 -F src/func.c 48987c025d69399f59a1c2a553cea5da41bf105d +F src/func.c d3fdcff9274bc161152e67ed3f626841c247f4b9 F src/global.c d2494a1cea8f66a2cab8258449df07f8f0ae6330 F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4 F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 @@ -182,7 +182,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c 319b7791cee6c763b60fde1b590bfaf62613cf37 -F src/sqlite.h.in 5fdc866a06ac82fd3c6ccb9f59a100151a36568c +F src/sqlite.h.in 90b4c427a6d67b15a469cfd907405de516e36e10 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 1c517a89cfdba28084093607ec79af70359c7f9b @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 4c7d9e1ed8722e0c75b5fdbeffb9030ccef0bd64 -R f44444aacae5f7c52822e030959d44b9 +P f0d95afc73f8dbce8943dceb4a14b7de650c8823 +R 98c7ddae1a722b08cd7989e28c30adec U drh -Z f26dfaa28d7c4e704618af941a7b95f6 +Z 0b357dfedb1b0f83bb0de889b4353615 diff --git a/manifest.uuid b/manifest.uuid index e0711c6d56..2be4ede130 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f0d95afc73f8dbce8943dceb4a14b7de650c8823 \ No newline at end of file +41d6ff32a6dd1311dc3aabe5156335a64a886919 \ No newline at end of file diff --git a/src/func.c b/src/func.c index 0fce95904f..e40fdad55a 100644 --- a/src/func.c +++ b/src/func.c @@ -693,6 +693,13 @@ static int patternCompare( return *zString==0; } +/* +** The sqlite3_strglob() interface. +*/ +int sqlite3_strglob(const char *zGlobPattern, const char *zString){ + return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, 0)==0; +} + /* ** Count the number of times that the LIKE operator (or GLOB which is ** just a variation of LIKE) gets called. This is used for testing diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 74cecbcf22..8638e5453e 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6837,6 +6837,21 @@ int sqlite3_unlock_notify( int sqlite3_stricmp(const char *, const char *); int sqlite3_strnicmp(const char *, const char *, int); +/* +** CAPI3REF: String Globbing +* +** ^The [sqlite3_strglob(P,X)] interface returns zero if string X matches +** the glob pattern P, and it returns non-zero if string X does not match +** the glob pattern P. ^The definition of glob pattern matching used in +** [sqlite3_strglob(P,X) is the same as for the "X GLOB P" operator in the +** SQL dialect used by SQLite. ^The sqlite3_strglob(P,X) function is case +** sensitive. +** +** Note that this routine returns zero on a match and non-zero if the strings +** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. +*/ +int sqlite3_strglob(const char *zGlob, const char *zStr); + /* ** CAPI3REF: Error Logging Interface ** From d040e764427bfa02bed83e5312a95d2877b182ff Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 10 Apr 2013 23:48:37 +0000 Subject: [PATCH 106/140] Add new primary error codes SQLITE_NOTICE and SQLITE_WARNING for use with sqlite3_log(). Add new extended error codes SQLITE_NOTICE_RECOVER_WAL and SQLITE_NOTICE_RECOVER_ROLLBACK to use with sqlite3_log() messages that occur on each recovery. FossilOrigin-Name: be7d2c5482c41baf000d7fb5dccc31b974e91064 --- manifest | 28 +++++----- manifest.uuid | 2 +- src/pager.c | 2 +- src/sqlite.h.in | 4 ++ src/test1.c | 133 +++++++++++++++++++++++++----------------------- src/test2.c | 54 +++++--------------- src/test3.c | 40 +++------------ src/test4.c | 31 ++--------- src/test7.c | 31 ++--------- src/wal.c | 3 +- test/wal.test | 2 +- 11 files changed, 119 insertions(+), 211 deletions(-) diff --git a/manifest b/manifest index 5efbde65d6..1059d70fbc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssqlite3_strglob()\sinterface. -D 2013-04-10T16:13:38.085 +C Add\snew\sprimary\serror\scodes\sSQLITE_NOTICE\sand\sSQLITE_WARNING\sfor\suse\swith\nsqlite3_log().\s\sAdd\snew\sextended\serror\scodes\sSQLITE_NOTICE_RECOVER_WAL\sand\nSQLITE_NOTICE_RECOVER_ROLLBACK\sto\suse\swith\ssqlite3_log()\smessages\sthat\soccur\non\seach\srecovery. +D 2013-04-10T23:48:37.567 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,7 +168,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c a6e9caadd8956a39153d1bd920e459211a281388 F src/os_win.c 3aec291c40048653c00a153fee96d3a442ab85ee -F src/pager.c cc60816e677528e4c2e23bbf56ee5c81f4cdfe58 +F src/pager.c 28f45e60d9a173368872d6e688e7a848c3926344 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -182,7 +182,7 @@ F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c 319b7791cee6c763b60fde1b590bfaf62613cf37 -F src/sqlite.h.in 90b4c427a6d67b15a469cfd907405de516e36e10 +F src/sqlite.h.in f606903594d314d1b63e677affb1a03e3c9a49c7 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 1c517a89cfdba28084093607ec79af70359c7f9b @@ -190,13 +190,13 @@ F src/sqliteLimit.h 3ea5c07b370b8033c30f66515ccf4e875c252429 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 -F src/test1.c 26e66b839f42c2eed6833f9023e0098f0d863f35 -F src/test2.c 4178056dd1e7d70f954ad8a1e3edb71a2a784daf -F src/test3.c 3c3c2407fa6ec7a19e24ae23f7cb439d0275a60d -F src/test4.c bf9fa9bece01de08e6f5e02314e4af5c13590dfa +F src/test1.c 6784fdacb35c33ba564ef749b62c4718fe515484 +F src/test2.c 29e7154112f7448d64204e8d31179cf497ecf425 +F src/test3.c 96aed72a8e1d542fed127e3e8350ae357712fa82 +F src/test4.c cea2c55110241e4674e66d476d29c914627999f5 F src/test5.c a6d1ac55ac054d0b2b8f37b5e655b6c92645a013 F src/test6.c a437f76f9874d2563352a7e6cd0d43217663c220 -F src/test7.c 2e0781754905c8adc3268d8f0967e7633af58843 +F src/test7.c f4b894b7931f8cf9f5cbf37cfa0727703f526a40 F src/test8.c 58ea1d9698f3947e4662107ef98f429e84ae20e0 F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad @@ -255,7 +255,7 @@ F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c 4fad64071ae120c25f39dcac572d716b9cadeb7f F src/vdbetrace.c a22263ab47f6ba4fcd176515cec1e732866b25f0 F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 -F src/wal.c 94b5fed2df988fb12f5bf17256e2840e56957a85 +F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b F src/where.c 4ad2329c439a30ddb915a780f6f80bdffafe3a64 @@ -961,7 +961,7 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 -F test/wal.test b934b757c4c3776b1e93140a9febc9ae068f9d90 +F test/wal.test 20fb8d0f8e65e50eb675994cc3a4f0e692fd4faf F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f0d95afc73f8dbce8943dceb4a14b7de650c8823 -R 98c7ddae1a722b08cd7989e28c30adec +P 41d6ff32a6dd1311dc3aabe5156335a64a886919 +R d4b63b2237f99e77bcb422f5c3bb48b3 U drh -Z 0b357dfedb1b0f83bb0de889b4353615 +Z 6b5c5fede4cd1a096bfdb2b852b5aff1 diff --git a/manifest.uuid b/manifest.uuid index 2be4ede130..088566df4f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -41d6ff32a6dd1311dc3aabe5156335a64a886919 \ No newline at end of file +be7d2c5482c41baf000d7fb5dccc31b974e91064 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index d35d91ac12..6c3376a836 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2832,7 +2832,7 @@ end_playback: testcase( rc!=SQLITE_OK ); } if( isHot && nPlayback ){ - sqlite3_log(SQLITE_OK, "Recovered %d pages from %s", + sqlite3_log(SQLITE_NOTICE_RECOVER_ROLLBACK, "recovered %d pages from %s", nPlayback, pPager->zJournal); } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 8638e5453e..349e36ea92 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -420,6 +420,8 @@ int sqlite3_exec( #define SQLITE_FORMAT 24 /* Auxiliary database format error */ #define SQLITE_RANGE 25 /* 2nd parameter to sqlite3_bind out of range */ #define SQLITE_NOTADB 26 /* File opened that is not a database file */ +#define SQLITE_NOTICE 27 /* Notifications from sqlite3_log() */ +#define SQLITE_WARNING 28 /* Warnings from sqlite3_log() */ #define SQLITE_ROW 100 /* sqlite3_step() has another row ready */ #define SQLITE_DONE 101 /* sqlite3_step() has finished executing */ /* end-of-error-codes */ @@ -490,6 +492,8 @@ int sqlite3_exec( #define SQLITE_CONSTRAINT_TRIGGER (SQLITE_CONSTRAINT | (7<<8)) #define SQLITE_CONSTRAINT_UNIQUE (SQLITE_CONSTRAINT | (8<<8)) #define SQLITE_CONSTRAINT_VTAB (SQLITE_CONSTRAINT | (9<<8)) +#define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) +#define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) /* ** CAPI3REF: Flags For File Open Operations diff --git a/src/test1.c b/src/test1.c index fb340f92d0..63d014cc17 100644 --- a/src/test1.c +++ b/src/test1.c @@ -116,71 +116,76 @@ int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb){ const char *sqlite3TestErrorName(int rc){ const char *zName = 0; - switch( rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; - case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; - case SQLITE_CONSTRAINT_FOREIGNKEY: - zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; - case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; - case SQLITE_CONSTRAINT_PRIMARYKEY: - zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; - case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; - case SQLITE_CONSTRAINT_COMMITHOOK: - zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; - case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; - case SQLITE_CONSTRAINT_FUNCTION: zName = "SQLITE_CONSTRAINT_FUNCTION";break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; - case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; - case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; - case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; - case SQLITE_ROW: zName = "SQLITE_ROW"; break; - case SQLITE_DONE: zName = "SQLITE_DONE"; break; - case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; - case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; - case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; - case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; - case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; - case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; - case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; - case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; - case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; - case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; - case SQLITE_IOERR_BLOCKED: zName = "SQLITE_IOERR_BLOCKED"; break; - case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; - case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; - case SQLITE_IOERR_CHECKRESERVEDLOCK: - zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; - case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; - case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; - case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; - case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; - case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; - default: zName = "SQLITE_Unknown"; break; + int i; + for(i=0; i<2 && zName==0; i++, rc &= 0xff){ + switch( rc ){ + case SQLITE_OK: zName = "SQLITE_OK"; break; + case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; + case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; + case SQLITE_PERM: zName = "SQLITE_PERM"; break; + case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; + case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; + case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; + case SQLITE_LOCKED_SHAREDCACHE: zName = "SQLITE_LOCKED_SHAREDCACHE";break; + case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; + case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; + case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; + case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; + case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; + case SQLITE_NOTFOUND: zName = "SQLITE_NOTFOUND"; break; + case SQLITE_FULL: zName = "SQLITE_FULL"; break; + case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; + case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; + case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; + case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; + case SQLITE_TOOBIG: zName = "SQLITE_TOOBIG"; break; + case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; + case SQLITE_CONSTRAINT_UNIQUE: zName = "SQLITE_CONSTRAINT_UNIQUE"; break; + case SQLITE_CONSTRAINT_TRIGGER: zName = "SQLITE_CONSTRAINT_TRIGGER";break; + case SQLITE_CONSTRAINT_FOREIGNKEY: + zName = "SQLITE_CONSTRAINT_FOREIGNKEY"; break; + case SQLITE_CONSTRAINT_CHECK: zName = "SQLITE_CONSTRAINT_CHECK"; break; + case SQLITE_CONSTRAINT_PRIMARYKEY: + zName = "SQLITE_CONSTRAINT_PRIMARYKEY"; break; + case SQLITE_CONSTRAINT_NOTNULL: zName = "SQLITE_CONSTRAINT_NOTNULL";break; + case SQLITE_CONSTRAINT_COMMITHOOK: + zName = "SQLITE_CONSTRAINT_COMMITHOOK"; break; + case SQLITE_CONSTRAINT_VTAB: zName = "SQLITE_CONSTRAINT_VTAB"; break; + case SQLITE_CONSTRAINT_FUNCTION: zName = "SQLITE_CONSTRAINT_FUNCTION";break; + case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; + case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; + case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; + case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; + case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; + case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; + case SQLITE_NOTADB: zName = "SQLITE_NOTADB"; break; + case SQLITE_ROW: zName = "SQLITE_ROW"; break; + case SQLITE_NOTICE: zName = "SQLITE_NOTICE"; break; + case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; + case SQLITE_DONE: zName = "SQLITE_DONE"; break; + case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; + case SQLITE_IOERR_SHORT_READ: zName = "SQLITE_IOERR_SHORT_READ"; break; + case SQLITE_IOERR_WRITE: zName = "SQLITE_IOERR_WRITE"; break; + case SQLITE_IOERR_FSYNC: zName = "SQLITE_IOERR_FSYNC"; break; + case SQLITE_IOERR_DIR_FSYNC: zName = "SQLITE_IOERR_DIR_FSYNC"; break; + case SQLITE_IOERR_TRUNCATE: zName = "SQLITE_IOERR_TRUNCATE"; break; + case SQLITE_IOERR_FSTAT: zName = "SQLITE_IOERR_FSTAT"; break; + case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; + case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; + case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; + case SQLITE_IOERR_BLOCKED: zName = "SQLITE_IOERR_BLOCKED"; break; + case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; + case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; + case SQLITE_IOERR_CHECKRESERVEDLOCK: + zName = "SQLITE_IOERR_CHECKRESERVEDLOCK"; break; + case SQLITE_IOERR_LOCK: zName = "SQLITE_IOERR_LOCK"; break; + case SQLITE_CORRUPT_VTAB: zName = "SQLITE_CORRUPT_VTAB"; break; + case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; + case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; + case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; + } } + if( zName==0 ) zName = "SQLITE_Unknown"; return zName; } #define t1ErrorName sqlite3TestErrorName diff --git a/src/test2.c b/src/test2.c index 8acdf6f4ef..3faec12619 100644 --- a/src/test2.c +++ b/src/test2.c @@ -19,35 +19,7 @@ #include #include -/* -** Interpret an SQLite error number -*/ -static char *errorName(int rc){ - char *zName; - switch( rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - default: zName = "SQLITE_Unknown"; break; - } - return zName; -} +extern const char *sqlite3TestErrorName(int rc); /* ** Page size and reserved size used for testing. @@ -87,7 +59,7 @@ static int pager_open( SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MAIN_DB, pager_test_reiniter); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } sqlite3PagerSetCachesize(pPager, nPage); @@ -119,7 +91,7 @@ static int pager_close( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerClose(pPager); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -146,7 +118,7 @@ static int pager_rollback( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerRollback(pPager); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -173,12 +145,12 @@ static int pager_commit( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerCommitPhaseOne(pPager, 0, 0); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } rc = sqlite3PagerCommitPhaseTwo(pPager); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -205,7 +177,7 @@ static int pager_stmt_begin( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerOpenSavepoint(pPager, 1); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -233,7 +205,7 @@ static int pager_stmt_rollback( rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, 0); sqlite3PagerSavepoint(pPager, SAVEPOINT_RELEASE, 0); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -260,7 +232,7 @@ static int pager_stmt_commit( pPager = sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_RELEASE, 0); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -353,7 +325,7 @@ static int page_get( rc = sqlite3PagerGet(pPager, pgno, &pPage); } if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf),zBuf,"%p",pPage); @@ -507,7 +479,7 @@ static int page_write( pPage = (DbPage *)sqlite3TestTextToPtr(argv[1]); rc = sqlite3PagerWrite(pPage); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } pData = sqlite3PagerGetData(pPage); @@ -556,7 +528,7 @@ static int fake_big_file( (SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE|SQLITE_OPEN_MAIN_DB), 0 ); if( rc ){ - Tcl_AppendResult(interp, "open failed: ", errorName(rc), 0); + Tcl_AppendResult(interp, "open failed: ", sqlite3TestErrorName(rc), 0); sqlite3_free(zFile); return TCL_ERROR; } @@ -566,7 +538,7 @@ static int fake_big_file( sqlite3OsCloseFree(fd); sqlite3_free(zFile); if( rc ){ - Tcl_AppendResult(interp, "write failed: ", errorName(rc), 0); + Tcl_AppendResult(interp, "write failed: ", sqlite3TestErrorName(rc), 0); return TCL_ERROR; } return TCL_OK; diff --git a/src/test3.c b/src/test3.c index e460c42e46..ecc3a543be 100644 --- a/src/test3.c +++ b/src/test3.c @@ -19,31 +19,7 @@ #include #include -/* -** Interpret an SQLite error number -*/ -static char *errorName(int rc){ - char *zName; - switch( rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - default: zName = "SQLITE_Unknown"; break; - } - return zName; -} +extern const char *sqlite3TestErrorName(int rc); /* ** A bogus sqlite3 connection structure for use in the btree @@ -89,7 +65,7 @@ static int btree_open( SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_MAIN_DB); sqlite3_free(zFilename); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } sqlite3BtreeSetCacheSize(pBt, nCache); @@ -119,7 +95,7 @@ static int btree_close( pBt = sqlite3TestTextToPtr(argv[1]); rc = sqlite3BtreeClose(pBt); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } nRefSqlite3--; @@ -156,7 +132,7 @@ static int btree_begin_transaction( rc = sqlite3BtreeBeginTrans(pBt, 1); sqlite3BtreeLeave(pBt); if( rc!=SQLITE_OK ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } return TCL_OK; @@ -250,7 +226,7 @@ static int btree_cursor( sqlite3BtreeLeave(pBt); if( rc ){ ckfree((char *)pCur); - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf), zBuf,"%p", pCur); @@ -285,7 +261,7 @@ static int btree_close_cursor( sqlite3BtreeLeave(pBt); ckfree((char *)pCur); if( rc ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } return SQLITE_OK; @@ -319,7 +295,7 @@ static int btree_next( rc = sqlite3BtreeNext(pCur, &res); sqlite3BtreeLeave(pCur->pBtree); if( rc ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf),zBuf,"%d",res); @@ -354,7 +330,7 @@ static int btree_first( rc = sqlite3BtreeFirst(pCur, &res); sqlite3BtreeLeave(pCur->pBtree); if( rc ){ - Tcl_AppendResult(interp, errorName(rc), 0); + Tcl_AppendResult(interp, sqlite3TestErrorName(rc), 0); return TCL_ERROR; } sqlite3_snprintf(sizeof(zBuf),zBuf,"%d",res); diff --git a/src/test4.c b/src/test4.c index 5b4c5a1c17..83c56d3d7e 100644 --- a/src/test4.c +++ b/src/test4.c @@ -20,6 +20,8 @@ #include #include +extern const char *sqlite3TestErrorName(int rc); + /* ** Each thread is controlled by an instance of the following ** structure. @@ -372,34 +374,7 @@ static int tcl_thread_result( return TCL_ERROR; } thread_wait(&threadset[i]); - switch( threadset[i].rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; - case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; - case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; - case SQLITE_ROW: zName = "SQLITE_ROW"; break; - case SQLITE_DONE: zName = "SQLITE_DONE"; break; - default: zName = "SQLITE_Unknown"; break; - } + zName = sqlite3TestErrorName(threadset[i].rc); Tcl_AppendResult(interp, zName, 0); return TCL_OK; } diff --git a/src/test7.c b/src/test7.c index 852cd1db5e..707d2c89af 100644 --- a/src/test7.c +++ b/src/test7.c @@ -376,6 +376,8 @@ static int tcl_client_colname( return TCL_OK; } +extern const char *sqlite3TestErrorName(int rc); + /* ** Usage: client_result ID ** @@ -403,34 +405,7 @@ static int tcl_client_result( return TCL_ERROR; } client_wait(&threadset[i]); - switch( threadset[i].rc ){ - case SQLITE_OK: zName = "SQLITE_OK"; break; - case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; - case SQLITE_PERM: zName = "SQLITE_PERM"; break; - case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; - case SQLITE_BUSY: zName = "SQLITE_BUSY"; break; - case SQLITE_LOCKED: zName = "SQLITE_LOCKED"; break; - case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; - case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; - case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; - case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; - case SQLITE_CORRUPT: zName = "SQLITE_CORRUPT"; break; - case SQLITE_FULL: zName = "SQLITE_FULL"; break; - case SQLITE_CANTOPEN: zName = "SQLITE_CANTOPEN"; break; - case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; - case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; - case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; - case SQLITE_CONSTRAINT: zName = "SQLITE_CONSTRAINT"; break; - case SQLITE_MISMATCH: zName = "SQLITE_MISMATCH"; break; - case SQLITE_MISUSE: zName = "SQLITE_MISUSE"; break; - case SQLITE_NOLFS: zName = "SQLITE_NOLFS"; break; - case SQLITE_AUTH: zName = "SQLITE_AUTH"; break; - case SQLITE_FORMAT: zName = "SQLITE_FORMAT"; break; - case SQLITE_RANGE: zName = "SQLITE_RANGE"; break; - case SQLITE_ROW: zName = "SQLITE_ROW"; break; - case SQLITE_DONE: zName = "SQLITE_DONE"; break; - default: zName = "SQLITE_Unknown"; break; - } + zName = sqlite3TestErrorName(threadset[i].rc); Tcl_AppendResult(interp, zName, 0); return TCL_OK; } diff --git a/src/wal.c b/src/wal.c index 881f0a4c78..e642ea2105 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1207,7 +1207,8 @@ finished: ** checkpointing the log file. */ if( pWal->hdr.nPage ){ - sqlite3_log(SQLITE_OK, "Recovered %d frames from WAL file %s", + sqlite3_log(SQLITE_NOTICE_RECOVER_WAL, + "recovered %d frames from WAL file %s", pWal->hdr.mxFrame, pWal->zWalName ); } diff --git a/test/wal.test b/test/wal.test index b997155540..66000d0951 100644 --- a/test/wal.test +++ b/test/wal.test @@ -1515,7 +1515,7 @@ do_test wal-23.3 { set nPage [expr 2+$AUTOVACUUM] do_test wal-23.4 { set ::log -} [list SQLITE_OK "Recovered $nPage frames from WAL file $walfile"] +} [list SQLITE_NOTICE "recovered $nPage frames from WAL file $walfile"] ifcapable autovacuum { From 08d418904a124936ff274240795a471cc6c1af77 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 11 Apr 2013 00:09:44 +0000 Subject: [PATCH 107/140] Enhance multi-process tester integration with the Win32 API. FossilOrigin-Name: 0fdc743583c67a3a017b9ad812c62a5104b9aee7 --- manifest | 14 ++++++------- manifest.uuid | 2 +- mptest/mptest.c | 54 +++++++++++++++++++++++++++---------------------- 3 files changed, 38 insertions(+), 32 deletions(-) diff --git a/manifest b/manifest index 1059d70fbc..6d3e751585 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\sprimary\serror\scodes\sSQLITE_NOTICE\sand\sSQLITE_WARNING\sfor\suse\swith\nsqlite3_log().\s\sAdd\snew\sextended\serror\scodes\sSQLITE_NOTICE_RECOVER_WAL\sand\nSQLITE_NOTICE_RECOVER_ROLLBACK\sto\suse\swith\ssqlite3_log()\smessages\sthat\soccur\non\seach\srecovery. -D 2013-04-10T23:48:37.567 +C Enhance\smulti-process\stester\sintegration\swith\sthe\sWin32\sAPI. +D 2013-04-11T00:09:44.820 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da F mptest/config02.test 962913ed2b537d60de4126db7fe54716865cdd22 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 -F mptest/mptest.c d5228be2db65634783b302423704b3928284adb5 +F mptest/mptest.c 8b6d7878addccbb9a1b28d9dbf2b0b516a3279f1 F mptest/multiwrite01.test 81fbc17657964889b60750bd7bbb1deffe8f4d42 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 41d6ff32a6dd1311dc3aabe5156335a64a886919 -R d4b63b2237f99e77bcb422f5c3bb48b3 -U drh -Z 6b5c5fede4cd1a096bfdb2b852b5aff1 +P be7d2c5482c41baf000d7fb5dccc31b974e91064 +R 255ee63faf4906e3f01e118bd1c7af53 +U mistachkin +Z 56dc3349791505b3e20ff09949e0bcd7 diff --git a/manifest.uuid b/manifest.uuid index 088566df4f..557333d3d0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -be7d2c5482c41baf000d7fb5dccc31b974e91064 \ No newline at end of file +0fdc743583c67a3a017b9ad812c62a5104b9aee7 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 6de32b0a05..0a8a8cedbd 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -36,7 +36,8 @@ #include "sqlite3.h" #include #if defined(_WIN32) -# include +# define WIN32_LEAN_AND_MEAN +# include #else # include #endif @@ -45,6 +46,13 @@ #include #include +/* The suffix to append to the child command lines, if any */ +#if defined(_WIN32) +# define CMDLINE_SUFFIX "" +#else +# define CMDLINE_SUFFIX "&" +#endif + /* Mark a parameter as unused to suppress compiler warnings */ #define UNUSED_PARAMETER(x) (void)x @@ -617,39 +625,37 @@ static int finishScript(int iClient, int taskId, int bShutdown){ static void startClient(int iClient){ runSql("INSERT OR IGNORE INTO client VALUES(%d,0)", iClient); if( sqlite3_changes(g.db) ){ -#if !defined(_WIN32) char *zSys; int rc; zSys = sqlite3_mprintf( - "%s \"%s\" --client %d --trace %d %s%s&", + "%s \"%s\" --client %d --trace %d %s%s%s", g.argv0, g.zDbFile, iClient, g.iTrace, g.bSqlTrace ? "--sqltrace " : "", - g.bSync ? "--sync " : "" + g.bSync ? "--sync " : "", + CMDLINE_SUFFIX ); +#if !defined(_WIN32) rc = system(zSys); if( rc ) errorMessage("system() fails with error code %d", rc); - sqlite3_free(zSys); -#endif -#if defined(_WIN32) - char *argv[10]; - char zClient[20]; - char zTrace[20]; - argv[0] = g.argv0; - argv[1] = g.zDbFile; - argv[2] = "--client"; - sqlite3_snprintf(sizeof(zClient),zClient,"%d",iClient); - argv[3] = zClient; - argv[4] = "--trace"; - sqlite3_snprintf(sizeof(zTrace),zTrace,"%d",g.iTrace); - argv[5] = zTrace; - if( g.bSqlTrace ){ - argv[6] = "--sqltrace"; - argv[7] = 0; - }else{ - argv[6] = 0; +#else + { + STARTUPINFOA startupInfo; + PROCESS_INFORMATION processInfo; + memset(&startupInfo, 0, sizeof(startupInfo)); + startupInfo.cb = sizeof(startupInfo); + memset(&processInfo, 0, sizeof(processInfo)); + rc = CreateProcessA(NULL, zSys, NULL, NULL, FALSE, 0, NULL, NULL, + &startupInfo, &processInfo); + if( rc ){ + CloseHandle(processInfo.hThread); + CloseHandle(processInfo.hProcess); + }else{ + errorMessage("CreateProcessA() fails with error code %lu", + GetLastError()); + } } - _spawnv(_P_NOWAIT, g.argv0, (const char*const*)argv); #endif + sqlite3_free(zSys); } } From fbc7e8845d6c3a0b140306d46c3454ca6ac45288 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 11 Apr 2013 01:16:15 +0000 Subject: [PATCH 108/140] Have the UNIX VFS issue warnings via sqlite3_log() if a database file is renamed or unlinked or linked to more than one name while the file is open. FossilOrigin-Name: e238dcf9189c029fbdcf89339e21d9cdd8fbf2c5 --- manifest | 14 ++++++------- manifest.uuid | 2 +- src/os_unix.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 60 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 6d3e751585..672a059405 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\smulti-process\stester\sintegration\swith\sthe\sWin32\sAPI. -D 2013-04-11T00:09:44.820 +C Have\sthe\sUNIX\sVFS\sissue\swarnings\svia\ssqlite3_log()\sif\sa\sdatabase\sfile\sis\nrenamed\sor\sunlinked\sor\slinked\sto\smore\sthan\sone\sname\swhile\sthe\sfile\sis\sopen. +D 2013-04-11T01:16:15.249 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c ca679b293a6233327e418fd1dde2cd5db3e90932 F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c a6e9caadd8956a39153d1bd920e459211a281388 +F src/os_unix.c 5707fcb125f043e2d3376ea862e8ec83633c5e0e F src/os_win.c 3aec291c40048653c00a153fee96d3a442ab85ee F src/pager.c 28f45e60d9a173368872d6e688e7a848c3926344 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P be7d2c5482c41baf000d7fb5dccc31b974e91064 -R 255ee63faf4906e3f01e118bd1c7af53 -U mistachkin -Z 56dc3349791505b3e20ff09949e0bcd7 +P 0fdc743583c67a3a017b9ad812c62a5104b9aee7 +R 6358d94b9e9fcf447041dc89384ceff9 +U drh +Z 7f7d7e8aee1747123050111170b5d1bc diff --git a/manifest.uuid b/manifest.uuid index 557333d3d0..784964e39b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0fdc743583c67a3a017b9ad812c62a5104b9aee7 \ No newline at end of file +e238dcf9189c029fbdcf89339e21d9cdd8fbf2c5 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index a65f894eb8..778575f7b1 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -280,6 +280,7 @@ struct unixFile { #define UNIXFILE_DELETE 0x20 /* Delete on close */ #define UNIXFILE_URI 0x40 /* Filename might have query parameters */ #define UNIXFILE_NOLOCK 0x80 /* Do no file locking */ +#define UNIXFILE_WARNED 0x0100 /* verifyDbFile() warnings have been issued */ /* ** Include code that is common to all os_*.c files @@ -799,7 +800,6 @@ static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { } - /****************************************************************************** ****************** Begin Unique File ID Utility Used By VxWorks *************** ** @@ -1300,6 +1300,50 @@ static int findInodeInfo( } +/* +** Check a unixFile that is a database. Verify the following: +** +** (1) There is exactly one hard link on the file +** (2) The file is not a symbolic link +** (3) The file has not been renamed or unlinked +** +** Issue sqlite3_log(SQLITE_WARNING,...) messages if anything is not right. +*/ +static void verifyDbFile(unixFile *pFile){ + struct stat buf; + int rc; + if( pFile->ctrlFlags & UNIXFILE_WARNED ){ + /* One or more of the following warnings have already been issued. Do not + ** repeat them so as not to clutter the error log */ + return; + } + rc = osFstat(pFile->h, &buf); + if( rc!=0 ){ + sqlite3_log(SQLITE_WARNING, "cannot fstat db file %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } + if( buf.st_nlink==0 && (pFile->ctrlFlags & UNIXFILE_DELETE)==0 ){ + sqlite3_log(SQLITE_WARNING, "file unlinked while open: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } + if( buf.st_nlink>1 ){ + sqlite3_log(SQLITE_WARNING, "multiple links to file: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } + if( pFile->pInode!=0 + && ((rc = osStat(pFile->zPath, &buf))!=0 + || buf.st_ino!=pFile->pInode->fileId.ino) + ){ + sqlite3_log(SQLITE_WARNING, "file renamed while open: %s", pFile->zPath); + pFile->ctrlFlags |= UNIXFILE_WARNED; + return; + } +} + + /* ** This routine checks if there is a RESERVED lock held on the specified ** file by this or any other process. If such a lock is held, set *pResOut @@ -1876,6 +1920,7 @@ static int closeUnixFile(sqlite3_file *id){ static int unixClose(sqlite3_file *id){ int rc = SQLITE_OK; unixFile *pFile = (unixFile *)id; + verifyDbFile(pFile); unixUnlock(id, NO_LOCK); unixEnterMutex(); @@ -4539,6 +4584,7 @@ static void unixUnmapfile(unixFile *pFd){ #endif } +#ifndef SQLITE_DISABLE_MMAP /* ** Return the system page size. */ @@ -4551,6 +4597,7 @@ static int unixGetPagesize(void){ return (int)sysconf(_SC_PAGESIZE); #endif } +#endif /* SQLITE_DISABLE_MMAP */ #ifndef SQLITE_DISABLE_MMAP /* @@ -4656,10 +4703,10 @@ static void unixRemapfile( ** code otherwise. */ static int unixMapfile(unixFile *pFd, i64 nByte){ +#ifndef SQLITE_DISABLE_MMAP i64 nMap = nByte; int rc; -#ifndef SQLITE_DISABLE_MMAP assert( nMap>=0 || pFd->nFetchOut==0 ); if( pFd->nFetchOut>0 ) return SQLITE_OK; @@ -4700,7 +4747,9 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ ** release the reference by calling unixUnfetch(). */ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ +#ifndef SQLITE_DISABLE_MMAP unixFile *pFd = (unixFile *)fd; /* The underlying database file */ +#endif *pp = 0; #ifndef SQLITE_DISABLE_MMAP @@ -5222,6 +5271,7 @@ static int fillInUnixFile( }else{ pNew->pMethod = pLockingStyle; OpenCounter(+1); + verifyDbFile(pNew); } return rc; } From e3be8c86277b72d8ce281d347792c543f5084544 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 11 Apr 2013 11:53:45 +0000 Subject: [PATCH 109/140] Show the process-id on log messages from mptester. FossilOrigin-Name: 6748a83dc5c02db37ecd963e678c5c69db142cac --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 13 +++++++++---- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 672a059405..8a66fe58aa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\sthe\sUNIX\sVFS\sissue\swarnings\svia\ssqlite3_log()\sif\sa\sdatabase\sfile\sis\nrenamed\sor\sunlinked\sor\slinked\sto\smore\sthan\sone\sname\swhile\sthe\sfile\sis\sopen. -D 2013-04-11T01:16:15.249 +C Show\sthe\sprocess-id\son\slog\smessages\sfrom\smptester. +D 2013-04-11T11:53:45.288 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da F mptest/config02.test 962913ed2b537d60de4126db7fe54716865cdd22 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 -F mptest/mptest.c 8b6d7878addccbb9a1b28d9dbf2b0b516a3279f1 +F mptest/mptest.c 1a8ca3d21e285d38f3ee06d708e634dda3f05bed F mptest/multiwrite01.test 81fbc17657964889b60750bd7bbb1deffe8f4d42 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 0fdc743583c67a3a017b9ad812c62a5104b9aee7 -R 6358d94b9e9fcf447041dc89384ceff9 +P e238dcf9189c029fbdcf89339e21d9cdd8fbf2c5 +R 961d2f18ca03b287ca1083092d6feb69 U drh -Z 7f7d7e8aee1747123050111170b5d1bc +Z d718ebcef0d40f58f76fbe3a2824e294 diff --git a/manifest.uuid b/manifest.uuid index 784964e39b..68dd000d58 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e238dcf9189c029fbdcf89339e21d9cdd8fbf2c5 \ No newline at end of file +6748a83dc5c02db37ecd963e678c5c69db142cac \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 0a8a8cedbd..210ac6fa7b 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -38,8 +38,11 @@ #if defined(_WIN32) # define WIN32_LEAN_AND_MEAN # include +# include +# define GETPID _getpid #else # include +# define GETPID getpid #endif #include #include @@ -67,7 +70,7 @@ static struct Global { FILE *pErrLog; /* Where to write errors */ char *zLog; /* Name of output log file */ FILE *pLog; /* Where to write log messages */ - char zName[12]; /* Symbolic name of this process */ + char zName[32]; /* Symbolic name of this process */ int taskId; /* Task ID. 0 means supervisor. */ int iTrace; /* Tracing level */ int bSqlTrace; /* True to trace SQL commands */ @@ -331,7 +334,7 @@ static void sqlErrorCallback(void *pArg, int iErrCode, const char *zMsg){ UNUSED_PARAMETER(pArg); if( (iErrCode&0xff)==SQLITE_SCHEMA && g.iTrace<3 ) return; if( g.iTimeout==0 && (iErrCode&0xff)==SQLITE_BUSY && g.iTrace<3 ) return; - if( iErrCode==SQLITE_OK ){ + if( (iErrCode&0xff)==SQLITE_NOTICE ){ logMessage("(info) %s", zMsg); }else{ errorMessage("(errcode=%d) %s", iErrCode, zMsg); @@ -1189,7 +1192,7 @@ int main(int argc, char **argv){ exit(1); } n = argc-2; - sqlite3_snprintf(sizeof(g.zName), g.zName, "mptest"); + sqlite3_snprintf(sizeof(g.zName), g.zName, "%05d.mptest", GETPID()); g.zVfs = findOption(argv+2, &n, "vfs", 1); zClient = findOption(argv+2, &n, "client", 1); g.zErrLog = findOption(argv+2, &n, "errlog", 1); @@ -1209,11 +1212,13 @@ int main(int argc, char **argv){ }else{ g.pLog = stdout; } + sqlite3_config(SQLITE_CONFIG_LOG, sqlErrorCallback, 0); if( zClient ){ iClient = atoi(zClient); if( iClient<1 ) fatalError("illegal client number: %d\n", iClient); - sqlite3_snprintf(sizeof(g.zName), g.zName, "client%02d", iClient); + sqlite3_snprintf(sizeof(g.zName), g.zName, "%05d.client%02d", + GETPID(), iClient); }else{ if( g.iTrace>0 ){ printf("With SQLite " SQLITE_VERSION " " SQLITE_SOURCE_ID "\n" ); From ef8e986bde954308f2bcb951dfccd23c04809fc4 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 11 Apr 2013 13:26:18 +0000 Subject: [PATCH 110/140] Fix pragma code generation so that it always outputs an OP_Trace opcode so that pragmas are shown in sqlite3_trace() output. FossilOrigin-Name: 663f04bd48bc6f302230a22bd22b82bde2353943 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pragma.c | 7 ++++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 8a66fe58aa..4fe8619c70 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Show\sthe\sprocess-id\son\slog\smessages\sfrom\smptester. -D 2013-04-11T11:53:45.288 +C Fix\spragma\scode\sgeneration\sso\sthat\sit\salways\soutputs\san\sOP_Trace\sopcode\sso\nthat\spragmas\sare\sshown\sin\ssqlite3_trace()\soutput. +D 2013-04-11T13:26:18.388 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -174,7 +174,7 @@ F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c 8bbc8c8697d13f76394e592dcecafce3491ed8ba +F src/pragma.c 216d6c24520f7b2e267a2356e5d3269d28690f49 F src/prepare.c 743e484233c51109666d402f470523553b41797c F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e238dcf9189c029fbdcf89339e21d9cdd8fbf2c5 -R 961d2f18ca03b287ca1083092d6feb69 +P 6748a83dc5c02db37ecd963e678c5c69db142cac +R d345752e155752ef7baeb6380027dfce U drh -Z d718ebcef0d40f58f76fbe3a2824e294 +Z c8c7d1ee07a5c8a52f14de0a5a9d5c85 diff --git a/manifest.uuid b/manifest.uuid index 68dd000d58..234c88c0fe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6748a83dc5c02db37ecd963e678c5c69db142cac \ No newline at end of file +663f04bd48bc6f302230a22bd22b82bde2353943 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 340e62d0dd..f4fc73d297 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -319,7 +319,7 @@ void sqlite3Pragma( int rc; /* return value form SQLITE_FCNTL_PRAGMA */ sqlite3 *db = pParse->db; /* The database connection */ Db *pDb; /* The specific database being pragmaed */ - Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(db); /* Prepared statement */ + Vdbe *v = sqlite3GetVdbe(pParse); /* Prepared statement */ if( v==0 ) return; sqlite3VdbeRunOnlyOnce(v); @@ -402,11 +402,12 @@ void sqlite3Pragma( static const VdbeOpList getCacheSize[] = { { OP_Transaction, 0, 0, 0}, /* 0 */ { OP_ReadCookie, 0, 1, BTREE_DEFAULT_CACHE_SIZE}, /* 1 */ - { OP_IfPos, 1, 7, 0}, + { OP_IfPos, 1, 8, 0}, { OP_Integer, 0, 2, 0}, { OP_Subtract, 1, 2, 1}, - { OP_IfPos, 1, 7, 0}, + { OP_IfPos, 1, 8, 0}, { OP_Integer, 0, 1, 0}, /* 6 */ + { OP_Noop, 0, 0, 0}, { OP_ResultRow, 1, 1, 0}, }; int addr; From e82c6c39f41724006ce063c313fe22e78e8b1da4 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 11 Apr 2013 18:28:55 +0000 Subject: [PATCH 111/140] Fix the xCheckReservedLock() method on the windows VFS so that it cannot return a false positive when two or more processes use it at the same time on the same file. Ticket [7ff3120e4fa54abb55] FossilOrigin-Name: dd3510bb20ade173c81c9874c05466011c8a249d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 4fe8619c70..8623e2f65f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\spragma\scode\sgeneration\sso\sthat\sit\salways\soutputs\san\sOP_Trace\sopcode\sso\nthat\spragmas\sare\sshown\sin\ssqlite3_trace()\soutput. -D 2013-04-11T13:26:18.388 +C Fix\sthe\sxCheckReservedLock()\smethod\son\sthe\swindows\sVFS\sso\sthat\sit\scannot\nreturn\sa\sfalse\spositive\swhen\stwo\sor\smore\sprocesses\suse\sit\sat\sthe\ssame\stime\non\sthe\ssame\sfile.\s\sTicket\s[7ff3120e4fa54abb55] +D 2013-04-11T18:28:55.139 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F src/os.c ca679b293a6233327e418fd1dde2cd5db3e90932 F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 5707fcb125f043e2d3376ea862e8ec83633c5e0e -F src/os_win.c 3aec291c40048653c00a153fee96d3a442ab85ee +F src/os_win.c 426563476ac4b0939c1bb2a59c6330e6864db5b0 F src/pager.c 28f45e60d9a173368872d6e688e7a848c3926344 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 6748a83dc5c02db37ecd963e678c5c69db142cac -R d345752e155752ef7baeb6380027dfce +P 663f04bd48bc6f302230a22bd22b82bde2353943 +R bd2e6ca88b98f09bd2a36a3fcc06c9f0 U drh -Z c8c7d1ee07a5c8a52f14de0a5a9d5c85 +Z 2e31b5ec3b1e364139cb9a99bd138f76 diff --git a/manifest.uuid b/manifest.uuid index 234c88c0fe..8f27dd8da9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -663f04bd48bc6f302230a22bd22b82bde2353943 \ No newline at end of file +dd3510bb20ade173c81c9874c05466011c8a249d \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 220ebac268..034f719fbd 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2702,7 +2702,7 @@ static int winCheckReservedLock(sqlite3_file *id, int *pResOut){ rc = 1; OSTRACE(("TEST WR-LOCK %d %d (local)\n", pFile->h, rc)); }else{ - rc = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, RESERVED_BYTE, 0, 1, 0); + rc = winLockFile(&pFile->h, SQLITE_LOCKFILEEX_FLAGS,RESERVED_BYTE, 0, 1, 0); if( rc ){ winUnlockFile(&pFile->h, RESERVED_BYTE, 0, 1, 0); } From fdd72c9a2fc9b4f1d3eb256830dc055dcdbad4e0 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 11 Apr 2013 21:13:10 +0000 Subject: [PATCH 112/140] For the multi-process tester on Win32, make use of the GetCurrentProcessId API. FossilOrigin-Name: f1b524b9d9ea3db96d54ac55c39f15e6879085bd --- manifest | 14 +++++++------- manifest.uuid | 2 +- mptest/mptest.c | 5 ++--- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 8623e2f65f..347b207be7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sxCheckReservedLock()\smethod\son\sthe\swindows\sVFS\sso\sthat\sit\scannot\nreturn\sa\sfalse\spositive\swhen\stwo\sor\smore\sprocesses\suse\sit\sat\sthe\ssame\stime\non\sthe\ssame\sfile.\s\sTicket\s[7ff3120e4fa54abb55] -D 2013-04-11T18:28:55.139 +C For\sthe\smulti-process\stester\son\sWin32,\smake\suse\sof\sthe\sGetCurrentProcessId\sAPI. +D 2013-04-11T21:13:10.779 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da F mptest/config02.test 962913ed2b537d60de4126db7fe54716865cdd22 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 -F mptest/mptest.c 1a8ca3d21e285d38f3ee06d708e634dda3f05bed +F mptest/mptest.c 53066ffe4ae6f0c4de451cc77985b88d0fee2aec F mptest/multiwrite01.test 81fbc17657964889b60750bd7bbb1deffe8f4d42 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 663f04bd48bc6f302230a22bd22b82bde2353943 -R bd2e6ca88b98f09bd2a36a3fcc06c9f0 -U drh -Z 2e31b5ec3b1e364139cb9a99bd138f76 +P dd3510bb20ade173c81c9874c05466011c8a249d +R 69eb3039bde7025dc2da94e69ac2fbdc +U mistachkin +Z 5b9c93ebe8aafa80e045145422e5fe06 diff --git a/manifest.uuid b/manifest.uuid index 8f27dd8da9..0a9b5274e1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dd3510bb20ade173c81c9874c05466011c8a249d \ No newline at end of file +f1b524b9d9ea3db96d54ac55c39f15e6879085bd \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 210ac6fa7b..33f4a3fa71 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -38,11 +38,8 @@ #if defined(_WIN32) # define WIN32_LEAN_AND_MEAN # include -# include -# define GETPID _getpid #else # include -# define GETPID getpid #endif #include #include @@ -52,8 +49,10 @@ /* The suffix to append to the child command lines, if any */ #if defined(_WIN32) # define CMDLINE_SUFFIX "" +# define GETPID (int)GetCurrentProcessId #else # define CMDLINE_SUFFIX "&" +# define GETPID getpid #endif /* Mark a parameter as unused to suppress compiler warnings */ From 739ee7ffac34ef1b93c4560f34dc272d3864f45f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 12 Apr 2013 01:04:36 +0000 Subject: [PATCH 113/140] In mptester: improve the way that child processes are dispatched. Pass the --vfs option through to children. Log the command used to start child processes when the tracing level is high enough. FossilOrigin-Name: 55718ae3462b2b6e0774d49e1c4c74143bc9e3a5 --- manifest | 14 +++++++------- manifest.uuid | 2 +- mptest/mptest.c | 22 +++++++++++++--------- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 347b207be7..1fbc9f7b6e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\sthe\smulti-process\stester\son\sWin32,\smake\suse\sof\sthe\sGetCurrentProcessId\sAPI. -D 2013-04-11T21:13:10.779 +C In\smptester:\simprove\sthe\sway\sthat\schild\sprocesses\sare\sdispatched.\s\sPass\nthe\s--vfs\soption\sthrough\sto\schildren.\s\sLog\sthe\scommand\sused\sto\sstart\nchild\sprocesses\swhen\sthe\stracing\slevel\sis\shigh\senough. +D 2013-04-12T01:04:36.558 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da F mptest/config02.test 962913ed2b537d60de4126db7fe54716865cdd22 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 -F mptest/mptest.c 53066ffe4ae6f0c4de451cc77985b88d0fee2aec +F mptest/mptest.c 2a263e88f18762ac5d7fb56749d2d2ea8e22cb93 F mptest/multiwrite01.test 81fbc17657964889b60750bd7bbb1deffe8f4d42 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P dd3510bb20ade173c81c9874c05466011c8a249d -R 69eb3039bde7025dc2da94e69ac2fbdc -U mistachkin -Z 5b9c93ebe8aafa80e045145422e5fe06 +P f1b524b9d9ea3db96d54ac55c39f15e6879085bd +R 42c876901a232f69407f221f508add46 +U drh +Z f52bc6984c23b912a4265566462deba4 diff --git a/manifest.uuid b/manifest.uuid index 0a9b5274e1..6692706ce6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f1b524b9d9ea3db96d54ac55c39f15e6879085bd \ No newline at end of file +55718ae3462b2b6e0774d49e1c4c74143bc9e3a5 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 33f4a3fa71..e47be5ae6d 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -48,10 +48,8 @@ /* The suffix to append to the child command lines, if any */ #if defined(_WIN32) -# define CMDLINE_SUFFIX "" # define GETPID (int)GetCurrentProcessId #else -# define CMDLINE_SUFFIX "&" # define GETPID getpid #endif @@ -629,14 +627,20 @@ static void startClient(int iClient){ if( sqlite3_changes(g.db) ){ char *zSys; int rc; - zSys = sqlite3_mprintf( - "%s \"%s\" --client %d --trace %d %s%s%s", - g.argv0, g.zDbFile, iClient, g.iTrace, - g.bSqlTrace ? "--sqltrace " : "", - g.bSync ? "--sync " : "", - CMDLINE_SUFFIX - ); + zSys = sqlite3_mprintf("%s \"%s\" --client %d --trace %d", + g.argv0, g.zDbFile, iClient, g.iTrace); + if( g.bSqlTrace ){ + zSys = sqlite3_mprintf("%z --sqltrace", zSys); + } + if( g.bSync ){ + zSys = sqlite3_mprintf("%z --sync", zSys); + } + if( g.zVfs ){ + zSys = sqlite3_mprintf("%z --vfs \"%s\"", zSys, g.zVfs); + } + if( g.iTrace>=2 ) logMessage("system('%q')", zSys); #if !defined(_WIN32) + zSys = sqlite3_mprintf("%z &", zSys); rc = system(zSys); if( rc ) errorMessage("system() fails with error code %d", rc); #else From a8f60381a93d037d39d5dc340f08e3cecb70ec7e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 12 Apr 2013 16:47:27 +0000 Subject: [PATCH 114/140] Change the fts4aux virtual table module so that fts4aux tables created in the temp database may report on fts3/fts4 tables in any attached database. FossilOrigin-Name: 546a187f1361dad340ed8f6c28dd89e6c19f9c33 --- ext/fts3/fts3_aux.c | 29 +++++++++++++++------ manifest | 16 ++++++------ manifest.uuid | 2 +- test/fts3aux1.test | 61 ++++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 88 insertions(+), 20 deletions(-) diff --git a/ext/fts3/fts3_aux.c b/ext/fts3/fts3_aux.c index a2bff2e1d3..9b582fcf8b 100644 --- a/ext/fts3/fts3_aux.c +++ b/ext/fts3/fts3_aux.c @@ -70,17 +70,26 @@ static int fts3auxConnectMethod( UNUSED_PARAMETER(pUnused); - /* The user should specify a single argument - the name of an fts3 table. */ - if( argc!=4 ){ - *pzErr = sqlite3_mprintf( - "wrong number of arguments to fts4aux constructor" - ); - return SQLITE_ERROR; - } + /* The user should invoke this in one of two forms: + ** + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table); + ** CREATE VIRTUAL TABLE xxx USING fts4aux(fts4-table-db, fts4-table); + */ + if( argc!=4 && argc!=5 ) goto bad_args; zDb = argv[1]; nDb = (int)strlen(zDb); - zFts3 = argv[3]; + if( argc==5 ){ + if( nDb==4 && 0==sqlite3_strnicmp("temp", zDb, 4) ){ + zDb = argv[3]; + nDb = (int)strlen(zDb); + zFts3 = argv[4]; + }else{ + goto bad_args; + } + }else{ + zFts3 = argv[3]; + } nFts3 = (int)strlen(zFts3); rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA); @@ -103,6 +112,10 @@ static int fts3auxConnectMethod( *ppVtab = (sqlite3_vtab *)p; return SQLITE_OK; + + bad_args: + *pzErr = sqlite3_mprintf("invalid arguments to fts4aux constructor"); + return SQLITE_ERROR; } /* diff --git a/manifest b/manifest index 1fbc9f7b6e..e1fd998e5e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\smptester:\simprove\sthe\sway\sthat\schild\sprocesses\sare\sdispatched.\s\sPass\nthe\s--vfs\soption\sthrough\sto\schildren.\s\sLog\sthe\scommand\sused\sto\sstart\nchild\sprocesses\swhen\sthe\stracing\slevel\sis\shigh\senough. -D 2013-04-12T01:04:36.558 +C Change\sthe\sfts4aux\svirtual\stable\smodule\sso\sthat\sfts4aux\stables\screated\sin\sthe\stemp\sdatabase\smay\sreport\son\sfts3/fts4\stables\sin\sany\sattached\sdatabase. +D 2013-04-12T16:47:27.407 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -58,7 +58,7 @@ F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d F ext/fts3/fts3.c 0eaedfc6d2eb22563ef1d044dcfed93b70ec79f2 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 1e58825246b56259267382d2f9902774c049460a -F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e +F ext/fts3/fts3_aux.c b02632f6dd0e375ce97870206d914ea6d8df5ccd F ext/fts3/fts3_expr.c 6cb4410f87676ae633bd7923bbc78526cb839c4d F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf @@ -478,7 +478,7 @@ F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test e7b80272efcced57d1d087a9da5c690dd7c21fd9 F test/fts3atoken.test fb398ab50aa232489e2a17f9b29d7ad3a3885f36 F test/fts3auto.test 74315a7377403a57ba82a652a33704197fe1e4be -F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 +F test/fts3aux1.test 575cc4b46320d96be10b878c954a0a08573b53b2 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 F test/fts3comp1.test a0f5b16a2df44dd0b15751787130af2183167c0c @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f1b524b9d9ea3db96d54ac55c39f15e6879085bd -R 42c876901a232f69407f221f508add46 -U drh -Z f52bc6984c23b912a4265566462deba4 +P 55718ae3462b2b6e0774d49e1c4c74143bc9e3a5 +R dc56d874edc4dcec111c7be4534352ac +U dan +Z fb78d8a2b290ae6166ab0175cd5b9a72 diff --git a/manifest.uuid b/manifest.uuid index 6692706ce6..e651e4517b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -55718ae3462b2b6e0774d49e1c4c74143bc9e3a5 \ No newline at end of file +546a187f1361dad340ed8f6c28dd89e6c19f9c33 \ No newline at end of file diff --git a/test/fts3aux1.test b/test/fts3aux1.test index adda586353..1688d6b231 100644 --- a/test/fts3aux1.test +++ b/test/fts3aux1.test @@ -354,10 +354,10 @@ do_execsql_test 3.1.1 { do_catchsql_test 3.1.2 { CREATE VIRTUAL TABLE terms2 USING fts4aux; -} {1 {wrong number of arguments to fts4aux constructor}} +} {1 {invalid arguments to fts4aux constructor}} do_catchsql_test 3.1.3 { CREATE VIRTUAL TABLE terms2 USING fts4aux(t2, t2); -} {1 {wrong number of arguments to fts4aux constructor}} +} {1 {invalid arguments to fts4aux constructor}} do_execsql_test 3.2.1 { CREATE VIRTUAL TABLE terms3 USING fts4aux(does_not_exist) @@ -444,7 +444,6 @@ do_plansql_test 4.5 { # odd name (one that requires quoting for use in SQL statements). And that # the argument to the fts4aux constructor is properly dequoted before use. # -# do_execsql_test 5.1 { CREATE VIRTUAL TABLE "abc '!' def" USING fts4(x, y); INSERT INTO "abc '!' def" VALUES('XX', 'YY'); @@ -458,5 +457,61 @@ do_execsql_test 5.2 { SELECT * FROM "%%^^%%"; } {xx * 1 1 xx 0 1 1 yy * 1 1 yy 1 1 1} +#------------------------------------------------------------------------- +# Test that we can create an fts4aux table in the temp database. +# +forcedelete test.db2 +do_execsql_test 6.1 { + CREATE VIRTUAL TABLE ft1 USING fts4(x, y); + INSERT INTO ft1 VALUES('a b', 'c d'); + INSERT INTO ft1 VALUES('e e', 'c d'); + INSERT INTO ft1 VALUES('a a', 'b b'); + CREATE VIRTUAL TABLE temp.aux1 USING fts4aux(main, ft1); + SELECT * FROM aux1; +} { + a * 2 3 a 0 2 3 + b * 2 3 b 0 1 1 b 1 1 2 + c * 2 2 c 1 2 2 + d * 2 2 d 1 2 2 + e * 1 2 e 0 1 2 +} + +do_execsql_test 6.2 { + ATTACH 'test.db2' AS att; + CREATE VIRTUAL TABLE att.ft1 USING fts4(x, y); + INSERT INTO att.ft1 VALUES('v w', 'x y'); + INSERT INTO att.ft1 VALUES('z z', 'x y'); + INSERT INTO att.ft1 VALUES('v v', 'w w'); + CREATE VIRTUAL TABLE temp.aux2 USING fts4aux(att, ft1); + SELECT * FROM aux2; +} { + v * 2 3 v 0 2 3 + w * 2 3 w 0 1 1 w 1 1 2 + x * 2 2 x 1 2 2 + y * 2 2 y 1 2 2 + z * 1 2 z 0 1 2 +} + +foreach {tn q res1 res2} { + 1 { SELECT * FROM %%% WHERE term = 'a' } {a * 2 3 a 0 2 3} {} + 2 { SELECT * FROM %%% WHERE term = 'x' } {} {x * 2 2 x 1 2 2} + + 3 { SELECT * FROM %%% WHERE term >= 'y' } + {} {y * 2 2 y 1 2 2 z * 1 2 z 0 1 2} + + 4 { SELECT * FROM %%% WHERE term <= 'c' } + {a * 2 3 a 0 2 3 b * 2 3 b 0 1 1 b 1 1 2 c * 2 2 c 1 2 2} {} +} { + set sql1 [string map {%%% aux1} $q] + set sql2 [string map {%%% aux2} $q] + + do_execsql_test 7.$tn.1 $sql1 $res1 + do_execsql_test 7.$tn.2 $sql2 $res2 +} + +do_test 8.1 { + catchsql { CREATE VIRTUAL TABLE att.aux3 USING fts4aux(main, ft1) } +} {1 {invalid arguments to fts4aux constructor}} finish_test + From 4b2ac35e5bf760bad76366e44c3ed6eb4448ed41 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 12 Apr 2013 16:53:19 +0000 Subject: [PATCH 115/140] Add a test case for detaching a database referenced by an fts4aux table created in the temp schema. FossilOrigin-Name: 6d6f0592abe237256dbdf30ab0ba0bb0c365dd95 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/fts3aux1.test | 5 +++++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index e1fd998e5e..c4d5a473e1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sfts4aux\svirtual\stable\smodule\sso\sthat\sfts4aux\stables\screated\sin\sthe\stemp\sdatabase\smay\sreport\son\sfts3/fts4\stables\sin\sany\sattached\sdatabase. -D 2013-04-12T16:47:27.407 +C Add\sa\stest\scase\sfor\sdetaching\sa\sdatabase\sreferenced\sby\san\sfts4aux\stable\screated\sin\sthe\stemp\sschema. +D 2013-04-12T16:53:19.544 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -478,7 +478,7 @@ F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test e7b80272efcced57d1d087a9da5c690dd7c21fd9 F test/fts3atoken.test fb398ab50aa232489e2a17f9b29d7ad3a3885f36 F test/fts3auto.test 74315a7377403a57ba82a652a33704197fe1e4be -F test/fts3aux1.test 575cc4b46320d96be10b878c954a0a08573b53b2 +F test/fts3aux1.test 03cec2dea379931c219dd4406817485caa69d1d8 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 F test/fts3comp1.test a0f5b16a2df44dd0b15751787130af2183167c0c @@ -1050,7 +1050,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 55718ae3462b2b6e0774d49e1c4c74143bc9e3a5 -R dc56d874edc4dcec111c7be4534352ac +P 546a187f1361dad340ed8f6c28dd89e6c19f9c33 +R 05383c39b4618f3bdf18eec38f2fc283 U dan -Z fb78d8a2b290ae6166ab0175cd5b9a72 +Z db66ae2fc0950c0cd2f83a29d6b2c3f8 diff --git a/manifest.uuid b/manifest.uuid index e651e4517b..d1d8020d4c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -546a187f1361dad340ed8f6c28dd89e6c19f9c33 \ No newline at end of file +6d6f0592abe237256dbdf30ab0ba0bb0c365dd95 \ No newline at end of file diff --git a/test/fts3aux1.test b/test/fts3aux1.test index 1688d6b231..ef17949fd9 100644 --- a/test/fts3aux1.test +++ b/test/fts3aux1.test @@ -513,5 +513,10 @@ do_test 8.1 { catchsql { CREATE VIRTUAL TABLE att.aux3 USING fts4aux(main, ft1) } } {1 {invalid arguments to fts4aux constructor}} +do_test 8.2 { + execsql {DETACH att} + catchsql { SELECT * FROM aux2 } +} {1 {SQL logic error or missing database}} + finish_test From a3a5bd9b62324bcfafaf2e6d4d98fbc57e1a1d94 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 13 Apr 2013 19:59:58 +0000 Subject: [PATCH 116/140] Only consider AS names from the result set as candidates for resolving identifiers in the WHERE clause if there are no other matches. In the ORDER BY clause, AS names take priority over any column names. Candidate fix for ticket [2500cdb9be]. FossilOrigin-Name: ad53924dcadffb95c6497c46c228c67e8f5370e4 --- manifest | 22 +++++++++++++--------- manifest.uuid | 2 +- src/resolve.c | 18 +++++++++++------- src/sqliteInt.h | 2 ++ test/resolver01.test | 39 +++++++++++++++++++++++++++++++++++++++ test/tkt2822.test | 5 ++++- 6 files changed, 70 insertions(+), 18 deletions(-) create mode 100644 test/resolver01.test diff --git a/manifest b/manifest index c4d5a473e1..3a4b72859e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\scase\sfor\sdetaching\sa\sdatabase\sreferenced\sby\san\sfts4aux\stable\screated\sin\sthe\stemp\sschema. -D 2013-04-12T16:53:19.544 +C Only\sconsider\sAS\snames\sfrom\sthe\sresult\sset\sas\scandidates\sfor\sresolving\nidentifiers\sin\sthe\sWHERE\sclause\sif\sthere\sare\sno\sother\smatches.\s\sIn\sthe\nORDER\sBY\sclause,\sAS\snames\stake\spriority\sover\sany\scolumn\snames.\nCandidate\sfix\sfor\sticket\s[2500cdb9be]. +D 2013-04-13T19:59:58.676 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -178,14 +178,14 @@ F src/pragma.c 216d6c24520f7b2e267a2356e5d3269d28690f49 F src/prepare.c 743e484233c51109666d402f470523553b41797c F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 -F src/resolve.c 9079da7d59aed2bb14ec8315bc7f720dd85b5b65 +F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c 319b7791cee6c763b60fde1b590bfaf62613cf37 F src/sqlite.h.in f606903594d314d1b63e677affb1a03e3c9a49c7 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 -F src/sqliteInt.h 1c517a89cfdba28084093607ec79af70359c7f9b +F src/sqliteInt.h 7580edf7e7e13af8dff6152666e4ea32535abf8c F src/sqliteLimit.h 3ea5c07b370b8033c30f66515ccf4e875c252429 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -685,6 +685,7 @@ F test/regexp1.test 5cbb6e7122ca51260d71079cf9245b63b8f64e1a F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a F test/releasetest.tcl 06d289d8255794073a58d2850742f627924545ce +F test/resolver01.test d1b487fc567bdb70b5dd09ccaaa877ddc61a233e F test/rollback.test a1b4784b864331eae8b2a98c189efa2a8b11ff07 F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test f777404492adb0e00868fd706a3721328fd3af48 @@ -854,7 +855,7 @@ F test/tkt2686.test 6ee01c9b9e9c48f6d3a1fdd553b1cc4258f903d6 F test/tkt2767.test 569000d842678f9cf2db7e0d1b27cbc9011381b0 F test/tkt2817.test f31839e01f4243cff7399ef654d3af3558cb8d8d F test/tkt2820.test 39940276b3436d125deb7d8ebeee053e4cf13213 -F test/tkt2822.test f391776423a7c0d0949edfce375708bfb0f3141e +F test/tkt2822.test c3589494fba643e039bcf0bfde7554ff6028406d F test/tkt2832.test a9b0b74a02dca166a04d9e37739c414b10929caa F test/tkt2854.test e432965db29e27e16f539b2ba7f502eb2ccc49af F test/tkt2920.test a8737380e4ae6424e00c0273dc12775704efbebf @@ -1050,7 +1051,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 546a187f1361dad340ed8f6c28dd89e6c19f9c33 -R 05383c39b4618f3bdf18eec38f2fc283 -U dan -Z db66ae2fc0950c0cd2f83a29d6b2c3f8 +P 6d6f0592abe237256dbdf30ab0ba0bb0c365dd95 +R b0858137a1ee08b4782056449cba9377 +T *branch * tkt-2500cdb9be +T *sym-tkt-2500cdb9be * +T -sym-trunk * +U drh +Z 489735ec39077e65a3d08cafc2217189 diff --git a/manifest.uuid b/manifest.uuid index d1d8020d4c..ac4feed793 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6d6f0592abe237256dbdf30ab0ba0bb0c365dd95 \ No newline at end of file +ad53924dcadffb95c6497c46c228c67e8f5370e4 \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index 0380138678..9b350caf80 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -388,7 +388,10 @@ static int lookupName( ** Note that the expression in the result set should have already been ** resolved by the time the WHERE clause is resolved. */ - if( cnt==0 && (pEList = pNC->pEList)!=0 && zTab==0 ){ + if( (pEList = pNC->pEList)!=0 + && zTab==0 + && ((pNC->ncFlags & NC_AsMaybe)==0 || cnt==0) + ){ for(j=0; jnExpr; j++){ char *zAs = pEList->a[j].zName; if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){ @@ -479,7 +482,9 @@ static int lookupName( lookupname_end: if( cnt==1 ){ assert( pNC!=0 ); - sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); + if( pExpr->op!=TK_AS ){ + sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); + } /* Increment the nRef value on all name contexts from TopNC up to ** the point where the name matched. */ for(;;){ @@ -1154,11 +1159,10 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** re-evaluated for each reference to it. */ sNC.pEList = p->pEList; - if( sqlite3ResolveExprNames(&sNC, p->pWhere) || - sqlite3ResolveExprNames(&sNC, p->pHaving) - ){ - return WRC_Abort; - } + if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; + sNC.ncFlags |= NC_AsMaybe; + if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; + sNC.ncFlags &= ~NC_AsMaybe; /* The ORDER BY and GROUP BY clauses may not refer to terms in ** outer queries diff --git a/src/sqliteInt.h b/src/sqliteInt.h index a528e37e44..264db39d14 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2079,6 +2079,8 @@ struct NameContext { #define NC_HasAgg 0x02 /* One or more aggregate functions seen */ #define NC_IsCheck 0x04 /* True if resolving names in a CHECK constraint */ #define NC_InAggFunc 0x08 /* True if analyzing arguments to an agg func */ +#define NC_AsMaybe 0x10 /* Resolve to AS terms of the result set only + ** if no other resolution is available */ /* ** An instance of the following structure contains all information diff --git a/test/resolver01.test b/test/resolver01.test new file mode 100644 index 0000000000..3ca6acec47 --- /dev/null +++ b/test/resolver01.test @@ -0,0 +1,39 @@ +# 2013-04-13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file tests features of the name resolver (the component that +# figures out what identifiers in the SQL statement refer to) that +# were fixed by ticket [2500cdb9be] +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_test resolver01-1.1 { + catchsql { + CREATE TABLE t1(x, y); INSERT INTO t1 VALUES(11,22); + CREATE TABLE t2(y, z); INSERT INTO t2 VALUES(33,44); + SELECT 1 AS y FROM t1, t2 ORDER BY y; + } +} {0 1} +do_test resolver01-1.2 { + catchsql { + SELECT 2 AS y FROM t1, t2 ORDER BY y COLLATE nocase; + } +} {0 2} +do_test resolver01-1.3 { + catchsql { + SELECT 3 AS y FROM t1, t2 ORDER BY +y; + } +} {0 3} + + +finish_test diff --git a/test/tkt2822.test b/test/tkt2822.test index d0b16338c6..d3512d3038 100644 --- a/test/tkt2822.test +++ b/test/tkt2822.test @@ -208,12 +208,15 @@ do_test tkt2822-5.4 { # In "ORDER BY +b" the term is now an expression rather than # a label. It therefore matches by rule (3) instead of rule (2). +# +# 2013-04-13: This is busted. Changed to conform to PostgreSQL and +# MySQL and Oracle behavior. # do_test tkt2822-5.5 { execsql { SELECT a AS b FROM t3 ORDER BY +b; } -} {9 1} +} {1 9} # Tests for rule 2 in compound queries # From a1710cc2ea2ed88207b8b4f7ae52c7e95e96b2c4 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 15 Apr 2013 13:10:30 +0000 Subject: [PATCH 117/140] Fix a formatting typo in a comment. No changes to code. FossilOrigin-Name: 3412424990c93d2978e819e6099811f1cdde316d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2c5683ef6b..5428c2214c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Only\sconsider\sAS\snames\sfrom\sthe\sresult\sset\sas\scandidates\sfor\sresolving\nidentifiers\sin\sthe\sWHERE\sclause\sif\sthere\sare\sno\sother\smatches.\nIn\sthe\sORDER\sBY\sclause,\sAS\snames\stake\spriority\sover\sany\scolumn\snames.\nFix\sfor\sticket\s[2500cdb9be]. -D 2013-04-14T23:51:27.821 +C Fix\sa\sformatting\stypo\sin\sa\scomment.\s\sNo\schanges\sto\scode. +D 2013-04-15T13:10:30.554 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -182,7 +182,7 @@ F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c 319b7791cee6c763b60fde1b590bfaf62613cf37 -F src/sqlite.h.in f606903594d314d1b63e677affb1a03e3c9a49c7 +F src/sqlite.h.in 696ecfc764a32b47daaaf0e10438ff2aaa0739e0 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 7580edf7e7e13af8dff6152666e4ea32535abf8c @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 6d6f0592abe237256dbdf30ab0ba0bb0c365dd95 ad53924dcadffb95c6497c46c228c67e8f5370e4 -R b0858137a1ee08b4782056449cba9377 +P d0d5af79a005de30194863c57c5018ee75e03581 +R 18db1cd249206022b16580a94625cafa U drh -Z c608eb98c484fd069203789a0c999b98 +Z 4a78dc5e25d8c90b818007b69b504f95 diff --git a/manifest.uuid b/manifest.uuid index 1cbd0dc9a2..a5a10fb745 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d0d5af79a005de30194863c57c5018ee75e03581 \ No newline at end of file +3412424990c93d2978e819e6099811f1cdde316d \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 349e36ea92..a2e85053d6 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6847,7 +6847,7 @@ int sqlite3_strnicmp(const char *, const char *, int); ** ^The [sqlite3_strglob(P,X)] interface returns zero if string X matches ** the glob pattern P, and it returns non-zero if string X does not match ** the glob pattern P. ^The definition of glob pattern matching used in -** [sqlite3_strglob(P,X) is the same as for the "X GLOB P" operator in the +** [sqlite3_strglob(P,X)] is the same as for the "X GLOB P" operator in the ** SQL dialect used by SQLite. ^The sqlite3_strglob(P,X) function is case ** sensitive. ** From 9b4c59fa1b591e4474ca458a4af239dd9c1d540d Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 15 Apr 2013 17:03:42 +0000 Subject: [PATCH 118/140] Refactoring the mmap interface. The controlling pragma is now "mmap_size" instead of "mmap_limit". Also change SQLITE_CONFIG_MMAP_LIMIT and SQLITE_FCNTL_MMAP_LIMIT to SQLITE_CONFIG_MMAP_SIZE and SQLITE_FCNTL_MMAP_SIZE, respecctively. The default mmap_size is now always 0, meaning that memory mapped I/O is off by default. There is a new compile-time option SQLITE_MAX_MMAP_SIZE that determines a hard upper bound on the mmap_size. Setting SQLITE_MAX_MMAP_SIZE to zero disables the memory-mapped I/O logic and causes it to be omitted from the build. An extra argument is added to SQLITE_CONFIG_MMAP_SIZE that can optionally lower the SQLITE_MAX_MMAP_SIZE at start-time. The SQLITE_MAX_MMAP_SIZE is zero for platforms where we know that it does not work, meaning that it cannot be turned on by mistake on those platforms. FossilOrigin-Name: ea1404a10abd7f68e1f8e0708c8a3199d1f79665 --- manifest | 68 +++++++++++++++++++++--------------------- manifest.uuid | 2 +- mptest/config01.test | 7 +++-- mptest/config02.test | 11 ++++--- src/btree.c | 6 ++-- src/ctime.c | 10 +++---- src/global.c | 3 +- src/main.c | 12 ++++++-- src/os.c | 2 +- src/os_unix.c | 57 ++++++++++++++++++----------------- src/os_win.c | 51 ++++++++++++++++--------------- src/pager.c | 26 ++++++++-------- src/pragma.c | 26 ++++++++-------- src/shell.c | 3 +- src/sqlite.h.in | 36 +++++++++++----------- src/sqliteInt.h | 44 +++++++++++++++++++++++---- src/sqliteLimit.h | 24 --------------- src/test_config.c | 6 ++-- test/dbstatus2.test | 2 +- test/exclusive2.test | 2 +- test/func.test | 2 +- test/incrblob.test | 8 ++--- test/mmap1.test | 26 ++++++++-------- test/pager1.test | 8 ++--- test/pagerfault.test | 2 +- test/pageropt.test | 2 +- test/permutations.test | 4 +-- test/sysfault.test | 2 +- test/wal.test | 2 +- test/wal5.test | 2 +- test/win32lock.test | 2 +- 31 files changed, 245 insertions(+), 213 deletions(-) diff --git a/manifest b/manifest index 5428c2214c..215a7b5881 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sformatting\stypo\sin\sa\scomment.\s\sNo\schanges\sto\scode. -D 2013-04-15T13:10:30.554 +C Refactoring\sthe\smmap\sinterface.\s\sThe\scontrolling\spragma\sis\snow\s"mmap_size"\ninstead\sof\s"mmap_limit".\s\sAlso\schange\sSQLITE_CONFIG_MMAP_LIMIT\sand\nSQLITE_FCNTL_MMAP_LIMIT\sto\sSQLITE_CONFIG_MMAP_SIZE\sand\nSQLITE_FCNTL_MMAP_SIZE,\srespecctively.\s\s\nThe\sdefault\smmap_size\sis\snow\salways\s0,\smeaning\sthat\nmemory\smapped\sI/O\sis\soff\sby\sdefault.\s\sThere\sis\sa\snew\scompile-time\soption\nSQLITE_MAX_MMAP_SIZE\sthat\sdetermines\sa\shard\supper\sbound\son\sthe\smmap_size.\nSetting\sSQLITE_MAX_MMAP_SIZE\sto\szero\sdisables\sthe\smemory-mapped\sI/O\slogic\nand\scauses\sit\sto\sbe\somitted\sfrom\sthe\sbuild.\s\sAn\sextra\sargument\sis\sadded\nto\sSQLITE_CONFIG_MMAP_SIZE\sthat\scan\soptionally\slower\sthe\sSQLITE_MAX_MMAP_SIZE\nat\sstart-time.\sThe\sSQLITE_MAX_MMAP_SIZE\sis\szero\sfor\splatforms\swhere\swe\s\nknow\sthat\sit\sdoes\snot\swork,\smeaning\sthat\sit\scannot\sbe\sturned\son\sby\smistake\non\sthose\splatforms. +D 2013-04-15T17:03:42.119 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -110,8 +110,8 @@ F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac F mkopcodec.awk f6fccee29e68493bfd90a2e0466ede5fa94dd2fc F mkopcodeh.awk 29b84656502eee5f444c3147f331ee686956ab0e F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 -F mptest/config01.test 3f4ddeb152a4f83872f0fa7fcb48d9fd609893da -F mptest/config02.test 962913ed2b537d60de4126db7fe54716865cdd22 +F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 +F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 F mptest/mptest.c 2a263e88f18762ac5d7fb56749d2d2ea8e22cb93 @@ -127,20 +127,20 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c b266767351ae2d847716c56fcb2a1fea7c761c03 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 5f2c4fe72f663bba837c5d01a732799c1d1c93f8 +F src/btree.c 480a6d255cc4f066029daf23dd54acf152cd0e13 F src/btree.h d9490cd37aaeb530a41b07f06e1262950b1be916 F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c 61a53c7e0db5e7ba60cf23b73deaf6779aa715d3 +F src/ctime.c 726e88b7282e7f348f805437db3222f97a136d35 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c e16942bd5c8a868ac53287886464a5ed0e72b179 F src/func.c d3fdcff9274bc161152e67ed3f626841c247f4b9 -F src/global.c d2494a1cea8f66a2cab8258449df07f8f0ae6330 +F src/global.c 5caf4deab621abb45b4c607aad1bd21c20aac759 F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4 F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -149,7 +149,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 1422eba4aa2b1fb5f7b3aef574752272477d21e2 -F src/main.c 54a841854734b6731c4d026834788cac6a19f3d1 +F src/main.c de47c3e6b68f512b9e64ba2533ee4eecc9d501c1 F src/malloc.c fe085aa851b666b7c375c1ff957643dc20a04bf6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa @@ -163,30 +163,30 @@ F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c ca679b293a6233327e418fd1dde2cd5db3e90932 +F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 5707fcb125f043e2d3376ea862e8ec83633c5e0e -F src/os_win.c 426563476ac4b0939c1bb2a59c6330e6864db5b0 -F src/pager.c 28f45e60d9a173368872d6e688e7a848c3926344 +F src/os_unix.c 0a561eae5965c9371300b0419027f5ae9f847af2 +F src/os_win.c 873bbc5dff8eb9f2bf8d9287d29df082eacb8391 +F src/pager.c a55adacb1842b83354198c408e7adde95ecd1189 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 9fd22671c270b35131ef480bbc00392b8b5f8ab9 -F src/pragma.c 216d6c24520f7b2e267a2356e5d3269d28690f49 +F src/pragma.c 3eacf001cbf4becbd494f8d82d08fdf1648cf8cb F src/prepare.c 743e484233c51109666d402f470523553b41797c F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 -F src/shell.c 319b7791cee6c763b60fde1b590bfaf62613cf37 -F src/sqlite.h.in 696ecfc764a32b47daaaf0e10438ff2aaa0739e0 +F src/shell.c 06bca04044b312ccadc43bf72385e7440cf60d1b +F src/sqlite.h.in d897fd15b9c74d4e78e4c42319484ceb1649c5d7 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 -F src/sqliteInt.h 7580edf7e7e13af8dff6152666e4ea32535abf8c -F src/sqliteLimit.h 3ea5c07b370b8033c30f66515ccf4e875c252429 +F src/sqliteInt.h 9933ad95b2c1e017f16428e9c0a929867598c2ae +F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 9a716c737590d2f129d71c8fc7065e5aba0e7222 @@ -203,7 +203,7 @@ F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de F src/test_btree.c 5b89601dcb42a33ba8b820a6b763cc9cb48bac16 -F src/test_config.c b6f5d680f0cf03d0ae9d6165bd2e447d9b356e4a +F src/test_config.c 49c99b714c1dab68aa6b37d53681a4278982be8d F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c 8f786bfd0ad48030cf2a06fb1f050e9c60a150d7 @@ -376,7 +376,7 @@ F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/date.test f3228180c87bbe5d39c9397bf001c0095c3821b9 F test/dbstatus.test 207e5b63fcb7b9c3bb8e1fdf38ebd4654ad0e54b -F test/dbstatus2.test f329941d5f4a8bc0ba6ec5735897ef0cf34e2f5f +F test/dbstatus2.test 10418e62b3db5dca070f0c3eef3ea13946f339c2 F test/default.test 6faf23ccb300114924353007795aa9a8ec0aa9dc F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa @@ -410,7 +410,7 @@ F test/eqp.test 46aa946dd55c90635327898275d3e533d23a9845 F test/errmsg.test 050717f1c6a5685de9c79f5f9f6b83d7c592f73a F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test a1b324cb21834a490cd052d409d34789cfef57cb -F test/exclusive2.test 354bdabe299a2546c898dff42f79079ff1590d88 +F test/exclusive2.test 881193eccec225cfed9d7f744b65e57f26adee25 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30 F test/expr.test 67c9fd6f8f829e239dc8b0f4a08a73c08b09196d @@ -515,7 +515,7 @@ F test/fts4merge2.test 5faa558d1b672f82b847d2a337465fa745e46891 F test/fts4merge3.test aab02a09f50fe6baaddc2e159c3eabc116d45fc7 F test/fts4unicode.test 25ccad45896f8e50f6a694cff738a35f798cdb40 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d -F test/func.test a4f24707e6af0bcd4cb5b856af4462db40b047b1 +F test/func.test b0fc34fdc36897769651975a2b0a606312753643 F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a F test/fuzz-oss1.test 4912e528ec9cf2f42134456933659d371c9e0d74 @@ -533,7 +533,7 @@ F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test 64f3cc1acde1b9161ccdd8e5bde3daefdb5b2617 F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 -F test/incrblob.test 2100cb8964e4a106e5ca9bf80e2c5c3e8be33f77 +F test/incrblob.test e81846d214f3637622620fbde7cd526781cfe328 F test/incrblob2.test edc3a96e557bd61fb39acc8d2edd43371fbbaa19 F test/incrblob3.test aedbb35ea1b6450c33b98f2b6ed98e5020be8dc7 F test/incrblob4.test 09be37d3dd996a31ea6993bba7837ece549414a8 @@ -637,7 +637,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test dd82ec9250b89178b96cd28b2aca70639d21e5b3 F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 -F test/mmap1.test f064a9025136cd688c982ccdae81c7b4fac95724 +F test/mmap1.test 0b5802cf64acaa509ea889b3c708196cc6eb9d31 F test/mmap2.test a5ba639f90b5fc487400a49e158e14e465943e98 F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a @@ -657,17 +657,17 @@ F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 -F test/pager1.test 134144862f6a98f862dc2edda1b10ff499141bfa +F test/pager1.test 30e63afd425fea12285e9ec5fa1fd000808031f1 F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f -F test/pagerfault.test bba948be0564254b823ec6ca1fd95fbbfb0839a3 +F test/pagerfault.test 8483e65d33d5636e5b7656204bb274d826e728d9 F test/pagerfault2.test 1f79ea40d1133b2683a2f811b00f2399f7ec2401 F test/pagerfault3.test f16e2efcb5fc9996d1356f7cbc44c998318ae1d7 -F test/pageropt.test d6b695e6c12290c406150fc51eec32878d48df4c +F test/pageropt.test 6b8f6a123a5572c195ad4ae40f2987007923bbd6 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/permutations.test b6cb45ce4d3193b831231025c9d30c77317dd240 +F test/permutations.test d2918e9ee6d0008292e5a70e90eed23b0440ca88 F test/pragma.test 60d29cd3d8098a2c20bf4c072810f99e3bf2757a F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -758,7 +758,7 @@ F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test a653783d985108c4912cc64d341ffbbb55ad2806 -F test/sysfault.test 503f72712b2b21cb80dc9899e53c2e39484d0313 +F test/sysfault.test fa776e60bf46bdd3ae69f0b73e46ee3977a58ae6 F test/table.test a59d985ca366e39b17b175f387f9d5db5a18d4e2 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 @@ -962,11 +962,11 @@ F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 82f463886e18d7f8395a4b6167c91815efe54839 -F test/wal.test 20fb8d0f8e65e50eb675994cc3a4f0e692fd4faf +F test/wal.test 0b4837cd5e9283c116d30810a6859bed7425a95e F test/wal2.test d4b470f13c87f6d8268b004380afa04c3c67cb90 F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c -F test/wal5.test 13c980d4d8d2d6b558361f19bed4ab31d41a2c7c +F test/wal5.test 8f888b50f66b78821e61ed0e233ded5de378224b F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test b3ee739fe8f7586aaebdc2367f477ebcf3e3b034 @@ -1004,7 +1004,7 @@ F test/whereD.test 3f3ee93825c94804f1fc91eef2de0d365981759a F test/whereE.test 7bd34945797efef15819368479bacc34215e4e1d F test/whereF.test a0e296643cabe5278379bc1a0aa158cf3c54a1c9 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 -F test/win32lock.test 21810396eee150680cc398970839e6abdcb807b9 +F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F test/zerodamage.test e7f77fded01dfcdf92ac2c5400f1e35d7a21463c F tool/build-all-msvc.bat 74fb6e5cca66ebdb6c9bbafb2f8b802f08146d38 x @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d0d5af79a005de30194863c57c5018ee75e03581 -R 18db1cd249206022b16580a94625cafa +P 3412424990c93d2978e819e6099811f1cdde316d +R aa0c6ea54adf9732b395ebf6e1c98243 U drh -Z 4a78dc5e25d8c90b818007b69b504f95 +Z 71eb761a9109889dcfada6d3231b23c2 diff --git a/manifest.uuid b/manifest.uuid index a5a10fb745..65cc51dd9d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3412424990c93d2978e819e6099811f1cdde316d \ No newline at end of file +ea1404a10abd7f68e1f8e0708c8a3199d1f79665 \ No newline at end of file diff --git a/mptest/config01.test b/mptest/config01.test index 75ba2f8fc2..683ee91143 100644 --- a/mptest/config01.test +++ b/mptest/config01.test @@ -5,11 +5,11 @@ PRAGMA page_size=8192; --task 1 PRAGMA journal_mode=PERSIST; - PRAGMA mmap_limit=0; + PRAGMA mmap_size=0; --end --task 2 PRAGMA journal_mode=TRUNCATE; - PRAGMA mmap_limit=28672; + PRAGMA mmap_size=28672; --end --task 3 PRAGMA journal_mode=MEMORY; @@ -17,6 +17,9 @@ PRAGMA page_size=8192; --task 4 PRAGMA journal_mode=OFF; --end +--task 4 + PRAGMA mmap_size(268435456); +--end --source multiwrite01.test --wait all PRAGMA page_size=16384; diff --git a/mptest/config02.test b/mptest/config02.test index e25d94081e..7d4b27898b 100644 --- a/mptest/config02.test +++ b/mptest/config02.test @@ -3,16 +3,19 @@ */ PRAGMA page_size=512; --task 1 - PRAGMA mmap_limit=0; + PRAGMA mmap_size=0; --end --task 2 - PRAGMA mmap_limit=28672; + PRAGMA mmap_size=28672; --end --task 3 - PRAGMA mmap_limit=8192; + PRAGMA mmap_size=8192; --end --task 4 - PRAGMA mmap_limit=65536; + PRAGMA mmap_size=65536; +--end +--task 5 + PRAGMA mmap_size=268435456; --end --source multiwrite01.test --source crash02.subtest diff --git a/src/btree.c b/src/btree.c index c13f0194f6..616d403681 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1876,7 +1876,7 @@ int sqlite3BtreeOpen( rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, EXTRA_SIZE, flags, vfsFlags, pageReinit); if( rc==SQLITE_OK ){ - sqlite3PagerSetMmapLimit(pBt->pPager, db->mxMmap); + sqlite3PagerSetMmapLimit(pBt->pPager, db->szMmap); rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); } if( rc!=SQLITE_OK ){ @@ -2147,11 +2147,11 @@ int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ ** Change the limit on the amount of the database file that may be ** memory mapped. */ -int sqlite3BtreeSetMmapLimit(Btree *p, sqlite3_int64 mxMmap){ +int sqlite3BtreeSetMmapLimit(Btree *p, sqlite3_int64 szMmap){ BtShared *pBt = p->pBt; assert( sqlite3_mutex_held(p->db->mutex) ); sqlite3BtreeEnter(p); - sqlite3PagerSetMmapLimit(pBt->pPager, mxMmap); + sqlite3PagerSetMmapLimit(pBt->pPager, szMmap); sqlite3BtreeLeave(p); return SQLITE_OK; } diff --git a/src/ctime.c b/src/ctime.c index 450a84045d..d60de40dd4 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -57,8 +57,8 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_DEFAULT_LOCKING_MODE "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), #endif -#ifdef SQLITE_DEFAULT_MMAP_LIMIT - "DEFAULT_MMAP_LIMIT=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_LIMIT), +#ifdef SQLITE_DEFAULT_MMAP_SIZE + "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE), #endif #ifdef SQLITE_DISABLE_DIRSYNC "DISABLE_DIRSYNC", @@ -66,9 +66,6 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_DISABLE_LFS "DISABLE_LFS", #endif -#ifdef SQLITE_DISABLE_MMAP - "DISABLE_MMAP", -#endif #ifdef SQLITE_ENABLE_ATOMIC_WRITE "ENABLE_ATOMIC_WRITE", #endif @@ -153,6 +150,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_LOCK_TRACE "LOCK_TRACE", #endif +#ifdef SQLITE_MAX_MMAP_SIZE + "MAX_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_MMAP_SIZE), +#endif #ifdef SQLITE_MAX_SCHEMA_RETRY "MAX_SCHEMA_RETRY=" CTIMEOPT_VAL(SQLITE_MAX_SCHEMA_RETRY), #endif diff --git a/src/global.c b/src/global.c index aec3958c1f..7b02cf2130 100644 --- a/src/global.c +++ b/src/global.c @@ -156,7 +156,8 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { (void*)0, /* pHeap */ 0, /* nHeap */ 0, 0, /* mnHeap, mxHeap */ - SQLITE_DEFAULT_MMAP_LIMIT, /* mxMmap */ + SQLITE_DEFAULT_MMAP_SIZE, /* szMmap */ + SQLITE_MAX_MMAP_SIZE, /* mxMmap */ (void*)0, /* pScratch */ 0, /* szScratch */ 0, /* nScratch */ diff --git a/src/main.c b/src/main.c index 288d4535f3..062fe40756 100644 --- a/src/main.c +++ b/src/main.c @@ -496,10 +496,16 @@ int sqlite3_config(int op, ...){ } #endif - case SQLITE_CONFIG_MMAP_LIMIT: { + case SQLITE_CONFIG_MMAP_SIZE: { + sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); - if( mxMmap<0 ) mxMmap = SQLITE_DEFAULT_MMAP_LIMIT; + if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ + mxMmap = SQLITE_MAX_MMAP_SIZE; + } sqlite3GlobalConfig.mxMmap = mxMmap; + if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; + if( szMmap>mxMmap) szMmap = mxMmap; + sqlite3GlobalConfig.szMmap = szMmap; break; } @@ -2323,7 +2329,7 @@ static int openDatabase( memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); db->autoCommit = 1; db->nextAutovac = -1; - db->mxMmap = sqlite3GlobalConfig.mxMmap; + db->szMmap = sqlite3GlobalConfig.szMmap; db->nextPagesize = 0; db->flags |= SQLITE_ShortColNames | SQLITE_AutoIndex | SQLITE_EnableTrigger #if SQLITE_DEFAULT_FILE_FORMAT<4 diff --git a/src/os.c b/src/os.c index 90cbe5492d..be2ea4cfc0 100644 --- a/src/os.c +++ b/src/os.c @@ -141,7 +141,7 @@ int sqlite3OsShmMap( return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } -#if !defined(SQLITE_DISABLE_MMAP) +#if SQLITE_MAX_MMAP_SIZE>0 /* The real implementation of xFetch and xUnfetch */ int sqlite3OsFetch(sqlite3_file *id, i64 iOff, int iAmt, void **pp){ DO_OS_MALLOC_TEST(id); diff --git a/src/os_unix.c b/src/os_unix.c index 778575f7b1..8dfbe61814 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -227,8 +227,8 @@ struct unixFile { int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ int nFetchOut; /* Number of outstanding xFetch refs */ sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ - sqlite3_int64 mmapOrigsize; /* Actual size of mapping at pMapRegion */ - sqlite3_int64 mmapLimit; /* Configured FCNTL_MMAP_LIMIT value */ + sqlite3_int64 mmapSizeActual; /* Actual size of mapping at pMapRegion */ + sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ void *pMapRegion; /* Memory mapped region */ #ifdef __QNXNTO__ int sectorSize; /* Device sector size */ @@ -3162,7 +3162,7 @@ static int unixRead( ); #endif -#if !defined(SQLITE_DISABLE_MMAP) +#if SQLITE_MAX_MMAP_SIZE>0 /* Deal with as much of this read request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ @@ -3283,7 +3283,7 @@ static int unixWrite( } #endif -#if !defined(SQLITE_DISABLE_MMAP) +#if SQLITE_MAX_MMAP_SIZE>0 /* Deal with as much of this write request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ @@ -3678,7 +3678,7 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ } } - if( pFile->mmapLimit>0 && nByte>pFile->mmapSize ){ + if( pFile->mmapSizeMax>0 && nByte>pFile->mmapSize ){ int rc; if( pFile->szChunk<=0 ){ if( robust_ftruncate(pFile->h, nByte) ){ @@ -3758,11 +3758,14 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } - case SQLITE_FCNTL_MMAP_LIMIT: { + case SQLITE_FCNTL_MMAP_SIZE: { i64 newLimit = *(i64*)pArg; - *(i64*)pArg = pFile->mmapLimit; + if( newLimit>sqlite3GlobalConfig.mxMmap ){ + newLimit = sqlite3GlobalConfig.mxMmap; + } + *(i64*)pArg = pFile->mmapSizeMax; if( newLimit>=0 ){ - pFile->mmapLimit = newLimit; + pFile->mmapSizeMax = newLimit; if( newLimitmmapSize ) pFile->mmapSize = newLimit; } return SQLITE_OK; @@ -4574,17 +4577,17 @@ static int unixShmUnmap( */ static void unixUnmapfile(unixFile *pFd){ assert( pFd->nFetchOut==0 ); -#ifndef SQLITE_DISABLE_MMAP +#if SQLITE_MAX_MMAP_SIZE>0 if( pFd->pMapRegion ){ - osMunmap(pFd->pMapRegion, pFd->mmapOrigsize); + osMunmap(pFd->pMapRegion, pFd->mmapSizeActual); pFd->pMapRegion = 0; pFd->mmapSize = 0; - pFd->mmapOrigsize = 0; + pFd->mmapSizeActual = 0; } #endif } -#ifndef SQLITE_DISABLE_MMAP +#if SQLITE_MAX_MMAP_SIZE>0 /* ** Return the system page size. */ @@ -4597,9 +4600,9 @@ static int unixGetPagesize(void){ return (int)sysconf(_SC_PAGESIZE); #endif } -#endif /* SQLITE_DISABLE_MMAP */ +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ -#ifndef SQLITE_DISABLE_MMAP +#if SQLITE_MAX_MMAP_SIZE>0 /* ** Attempt to set the size of the memory mapping maintained by file ** descriptor pFd to nNew bytes. Any existing mapping is discarded. @@ -4608,7 +4611,7 @@ static int unixGetPagesize(void){ ** ** unixFile.pMapRegion ** unixFile.mmapSize -** unixFile.mmapOrigsize +** unixFile.mmapSizeActual ** ** If unsuccessful, an error message is logged via sqlite3_log() and ** the three variables above are zeroed. In this case SQLite should @@ -4622,15 +4625,15 @@ static void unixRemapfile( const char *zErr = "mmap"; int h = pFd->h; /* File descriptor open on db file */ u8 *pOrig = (u8 *)pFd->pMapRegion; /* Pointer to current file mapping */ - i64 nOrig = pFd->mmapOrigsize; /* Size of pOrig region in bytes */ + i64 nOrig = pFd->mmapSizeActual; /* Size of pOrig region in bytes */ u8 *pNew = 0; /* Location of new mapping */ int flags = PROT_READ; /* Flags to pass to mmap() */ assert( pFd->nFetchOut==0 ); assert( nNew>pFd->mmapSize ); - assert( nNew<=pFd->mmapLimit ); + assert( nNew<=pFd->mmapSizeMax ); assert( nNew>0 ); - assert( pFd->mmapOrigsize>=pFd->mmapSize ); + assert( pFd->mmapSizeActual>=pFd->mmapSize ); assert( MAP_FAILED!=0 ); if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; @@ -4679,10 +4682,10 @@ static void unixRemapfile( /* If the mmap() above failed, assume that all subsequent mmap() calls ** will probably fail too. Fall back to using xRead/xWrite exclusively ** in this case. */ - pFd->mmapLimit = 0; + pFd->mmapSizeMax = 0; } pFd->pMapRegion = (void *)pNew; - pFd->mmapSize = pFd->mmapOrigsize = nNew; + pFd->mmapSize = pFd->mmapSizeActual = nNew; } #endif @@ -4703,7 +4706,7 @@ static void unixRemapfile( ** code otherwise. */ static int unixMapfile(unixFile *pFd, i64 nByte){ -#ifndef SQLITE_DISABLE_MMAP +#if SQLITE_MAX_MMAP_SIZE>0 i64 nMap = nByte; int rc; @@ -4718,8 +4721,8 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ } nMap = statbuf.st_size; } - if( nMap>pFd->mmapLimit ){ - nMap = pFd->mmapLimit; + if( nMap>pFd->mmapSizeMax ){ + nMap = pFd->mmapSizeMax; } if( nMap!=pFd->mmapSize ){ @@ -4747,13 +4750,13 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ ** release the reference by calling unixUnfetch(). */ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ -#ifndef SQLITE_DISABLE_MMAP +#if SQLITE_MAX_MMAP_SIZE>0 unixFile *pFd = (unixFile *)fd; /* The underlying database file */ #endif *pp = 0; -#ifndef SQLITE_DISABLE_MMAP - if( pFd->mmapLimit>0 ){ +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->mmapSizeMax>0 ){ if( pFd->pMapRegion==0 ){ int rc = unixMapfile(pFd, -1); if( rc!=SQLITE_OK ) return rc; @@ -5126,7 +5129,7 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; - pNew->mmapLimit = sqlite3GlobalConfig.mxMmap; + pNew->mmapSizeMax = sqlite3GlobalConfig.mxMmap; if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; diff --git a/src/os_win.c b/src/os_win.c index 034f719fbd..e56ccb50c6 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -150,12 +150,12 @@ struct winFile { winceLock local; /* Locks obtained by this instance of winFile */ winceLock *shared; /* Global shared lock memory for the file */ #endif - int nFetchOut; /* Number of outstanding xFetch references */ - HANDLE hMap; /* Handle for accessing memory mapping */ - void *pMapRegion; /* Area memory mapped */ - sqlite3_int64 mmapSize; /* Usable size of mapped region */ - sqlite3_int64 mmapOrigsize; /* Actual size of mapped region */ - sqlite3_int64 mmapLimit; /* Configured FCNTL_MMAP_LIMIT value */ + int nFetchOut; /* Number of outstanding xFetch references */ + HANDLE hMap; /* Handle for accessing memory mapping */ + void *pMapRegion; /* Area memory mapped */ + sqlite3_int64 mmapSize; /* Usable size of mapped region */ + sqlite3_int64 mmapSizeActual; /* Actual size of mapped region */ + sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ }; /* @@ -2148,7 +2148,7 @@ static int winRead( SimulateIOError(return SQLITE_IOERR_READ); OSTRACE(("READ %d lock=%d\n", pFile->h, pFile->locktype)); -#if !defined(SQLITE_DISABLE_MMAP) +#if SQLITE_MAX_MMAP_SIZE>0 /* Deal with as much of this read request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ @@ -2214,7 +2214,7 @@ static int winWrite( OSTRACE(("WRITE %d lock=%d\n", pFile->h, pFile->locktype)); -#if !defined(SQLITE_DISABLE_MMAP) +#if SQLITE_MAX_MMAP_SIZE>0 /* Deal with as much of this write request as possible by transfering ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ @@ -2325,7 +2325,7 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ "winTruncate2", pFile->zPath); } -#if !defined(SQLITE_DISABLE_MMAP) +#if SQLITE_MAX_MMAP_SIZE>0 /* If the file was truncated to a size smaller than the currently ** mapped region, reduce the effective mapping size as well. SQLite will ** use read() and write() to access data beyond this point from now on. @@ -2842,10 +2842,13 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } - case SQLITE_FCNTL_MMAP_LIMIT: { + case SQLITE_FCNTL_MMAP_SIZE: { i64 newLimit = *(i64*)pArg; - *(i64*)pArg = pFile->mmapLimit; - if( newLimit>=0 ) pFile->mmapLimit = newLimit; + if( newLimit>sqlite3GlobalConfig.mxMmap ){ + newLimit = sqlite3GlobalConfig.mxMmap; + } + *(i64*)pArg = pFile->mmapSizeMax; + if( newLimit>=0 ) pFile->mmapSizeMax = newLimit; return SQLITE_OK; } } @@ -3523,7 +3526,7 @@ shmpage_out: */ static int winUnmapfile(winFile *pFile){ assert( pFile!=0 ); -#if !defined(SQLITE_DISABLE_MMAP) +#if SQLITE_MAX_MMAP_SIZE>0 if( pFile->pMapRegion ){ if( !osUnmapViewOfFile(pFile->pMapRegion) ){ pFile->lastErrno = osGetLastError(); @@ -3532,7 +3535,7 @@ static int winUnmapfile(winFile *pFile){ } pFile->pMapRegion = 0; pFile->mmapSize = 0; - pFile->mmapOrigsize = 0; + pFile->mmapSizeActual = 0; } if( pFile->hMap!=NULL ){ if( !osCloseHandle(pFile->hMap) ){ @@ -3546,7 +3549,7 @@ static int winUnmapfile(winFile *pFile){ return SQLITE_OK; } -#if !defined(SQLITE_DISABLE_MMAP) +#if SQLITE_MAX_MMAP_SIZE>0 /* ** Memory map or remap the file opened by file-descriptor pFd (if the file ** is already mapped, the existing mapping is replaced by the new). Or, if @@ -3557,7 +3560,7 @@ static int winUnmapfile(winFile *pFile){ ** the mapping to create. Otherwise, if nByte is less than zero, then the ** requested size is the size of the file on disk. The actual size of the ** created mapping is either the requested size or the value configured -** using SQLITE_FCNTL_MMAP_LIMIT, whichever is smaller. +** using SQLITE_FCNTL_MMAP_SIZE, whichever is smaller. ** ** SQLITE_OK is returned if no error occurs (even if the mapping is not ** recreated as a result of outstanding references) or an SQLite error @@ -3576,8 +3579,8 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ return SQLITE_IOERR_FSTAT; } } - if( nMap>pFd->mmapLimit ){ - nMap = pFd->mmapLimit; + if( nMap>pFd->mmapSizeMax ){ + nMap = pFd->mmapSizeMax; } nMap &= ~(sqlite3_int64)(winSysInfo.dwPageSize - 1); @@ -3629,12 +3632,12 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ } pFd->pMapRegion = pNew; pFd->mmapSize = nMap; - pFd->mmapOrigsize = nMap; + pFd->mmapSizeActual = nMap; } return SQLITE_OK; } -#endif /* !defined(SQLITE_DISABLE_MMAP) */ +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ /* ** If possible, return a pointer to a mapping of file fd starting at offset @@ -3652,8 +3655,8 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ winFile *pFd = (winFile*)fd; /* The underlying database file */ *pp = 0; -#if !defined(SQLITE_DISABLE_MMAP) - if( pFd->mmapLimit>0 ){ +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFd->mmapSizeMax>0 ){ if( pFd->pMapRegion==0 ){ int rc = winMapfile(pFd, -1); if( rc!=SQLITE_OK ) return rc; @@ -4129,8 +4132,8 @@ static int winOpen( pFile->hMap = NULL; pFile->pMapRegion = 0; pFile->mmapSize = 0; - pFile->mmapOrigsize = 0; - pFile->mmapLimit = sqlite3GlobalConfig.mxMmap; + pFile->mmapSizeActual = 0; + pFile->mmapSizeMax = sqlite3GlobalConfig.mxMmap; OpenCounter(+1); return rc; diff --git a/src/pager.c b/src/pager.c index 6c3376a836..96032b2896 100644 --- a/src/pager.c +++ b/src/pager.c @@ -658,7 +658,7 @@ struct Pager { u8 bUseFetch; /* True to use xFetch() */ int nMmapOut; /* Number of mmap pages currently outstanding */ - sqlite3_int64 mxMmap; /* Desired maximum mmap size */ + sqlite3_int64 szMmap; /* Desired maximum mmap size */ PgHdr *pMmapFreelist; /* List of free mmap page headers (pDirty) */ /* ** End of the routinely-changing class members @@ -774,10 +774,10 @@ static const unsigned char aJournalMagic[] = { ** The macro USEFETCH is true if we are allowed to use the xFetch and xUnfetch ** interfaces to access the database using memory-mapped I/O. */ -#ifdef SQLITE_DISABLE_MMAP -# define USEFETCH(x) 0 -#else +#if SQLITE_MAX_MMAP_SIZE>0 # define USEFETCH(x) ((x)->bUseFetch) +#else +# define USEFETCH(x) 0 #endif /* @@ -3370,16 +3370,16 @@ void sqlite3PagerSetCachesize(Pager *pPager, int mxPage){ } /* -** Invoke SQLITE_FCNTL_MMAP_LIMIT based on the current value of mxMmap. +** Invoke SQLITE_FCNTL_MMAP_SIZE based on the current value of szMmap. */ static void pagerFixMaplimit(Pager *pPager){ -#if !defined(SQLITE_DISABLE_MMAP) +#if SQLITE_MAX_MMAP_SIZE>0 sqlite3_file *fd = pPager->fd; if( isOpen(fd) ){ - sqlite3_int64 mx; - pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->mxMmap>0; - mx = pPager->mxMmap; - sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_LIMIT, &mx); + sqlite3_int64 sz; + pPager->bUseFetch = (fd->pMethods->iVersion>=3) && pPager->szMmap>0; + sz = pPager->szMmap; + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_SIZE, &sz); } #endif } @@ -3387,8 +3387,8 @@ static void pagerFixMaplimit(Pager *pPager){ /* ** Change the maximum size of any memory mapping made of the database file. */ -void sqlite3PagerSetMmapLimit(Pager *pPager, sqlite3_int64 mxMmap){ - pPager->mxMmap = mxMmap; +void sqlite3PagerSetMmapLimit(Pager *pPager, sqlite3_int64 szMmap){ + pPager->szMmap = szMmap; pagerFixMaplimit(pPager); } @@ -4766,7 +4766,7 @@ int sqlite3PagerOpen( /* pPager->pBusyHandlerArg = 0; */ pPager->xReiniter = xReinit; /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ - /* pPager->mxMmap = SQLITE_DEFAULT_MMAP_LIMIT // will be set by btree.c */ + /* pPager->szMmap = SQLITE_DEFAULT_MMAP_SIZE // will be set by btree.c */ *ppPager = pPager; return SQLITE_OK; diff --git a/src/pragma.c b/src/pragma.c index f4fc73d297..65efbd8b21 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -746,39 +746,39 @@ void sqlite3Pragma( }else /* - ** PRAGMA [database.]mmap_limit(N) + ** PRAGMA [database.]mmap_size(N) ** ** Used to set mapping size limit. The mapping size limit is ** used to limit the aggregate size of all memory mapped regions of the ** database file. If this parameter is set to zero, then memory mapping ** is not used at all. If N is negative, then the default memory map - ** limit determined by sqlite3_config(SQLITE_CONFIG_MMAP_LIMIT) is set. + ** limit determined by sqlite3_config(SQLITE_CONFIG_MMAP_SIZE) is set. ** The parameter N is measured in bytes. ** ** This value is advisory. The underlying VFS is free to memory map ** as little or as much as it wants. Except, if N is set to 0 then the ** upper layers will never invoke the xFetch interfaces to the VFS. */ - if( sqlite3StrICmp(zLeft,"mmap_limit")==0 ){ - sqlite3_int64 mx; + if( sqlite3StrICmp(zLeft,"mmap_size")==0 ){ + sqlite3_int64 sz; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( zRight ){ int ii; - sqlite3Atoi64(zRight, &mx, 1000, SQLITE_UTF8); - if( mx<0 ) mx = sqlite3GlobalConfig.mxMmap; - if( pId2->n==0 ) db->mxMmap = mx; + sqlite3Atoi64(zRight, &sz, 1000, SQLITE_UTF8); + if( sz<0 ) sz = sqlite3GlobalConfig.szMmap; + if( pId2->n==0 ) db->szMmap = sz; for(ii=db->nDb-1; ii>=0; ii--){ if( db->aDb[ii].pBt && (ii==iDb || pId2->n==0) ){ - sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, mx); + sqlite3BtreeSetMmapLimit(db->aDb[ii].pBt, sz); } } } - mx = -1; - if( sqlite3_file_control(db,zDb,SQLITE_FCNTL_MMAP_LIMIT,&mx)==SQLITE_OK ){ -#if defined(SQLITE_DISABLE_MMAP) - mx = 0; + sz = -1; + if( sqlite3_file_control(db,zDb,SQLITE_FCNTL_MMAP_SIZE,&sz)==SQLITE_OK ){ +#if SQLITE_MAX_MMAP_SIZE==0 + sz = 0; #endif - returnSingleInt(pParse, "mmap_limit", mx); + returnSingleInt(pParse, "mmap_size", sz); } }else diff --git a/src/shell.c b/src/shell.c index 6a0e047475..7dde2203ec 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3060,7 +3060,8 @@ int main(int argc, char **argv){ sqlite3_multiplex_initialize(0, 1); #endif }else if( strcmp(z,"-mmap")==0 ){ - sqlite3_config(SQLITE_CONFIG_MMAP_LIMIT, integerValue(cmdline_option_value(argc,argv,++i))); + sqlite3_int64 sz = integerValue(cmdline_option_value(argc,argv,++i)); + sqlite3_config(SQLITE_CONFIG_MMAP_SIZE, sz, sz); }else if( strcmp(z,"-vfs")==0 ){ sqlite3_vfs *pVfs = sqlite3_vfs_find(cmdline_option_value(argc,argv,++i)); if( pVfs ){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index a2e85053d6..69f79a0377 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -892,15 +892,15 @@ struct sqlite3_io_methods { ** written into memory obtained from [sqlite3_malloc()]. The caller should ** invoke [sqlite3_free()] on the result to avoid a memory leak. ** -**
  • [[SQLITE_FCNTL_MMAP_LIMIT]] -** The [SQLITE_FCNTL_MMAP_LIMIT] file control is used to query or set the +**
  • [[SQLITE_FCNTL_MMAP_SIZE]] +** The [SQLITE_FCNTL_MMAP_SIZE] file control is used to query or set the ** maximum number of bytes that will be used for memory-mapped I/O. ** The argument is a pointer to a value of type sqlite3_int64 that ** is an advisory maximum number of bytes in the file to memory map. The ** pointer is overwritten with the old value. The limit is not changed if -** the originally pointed to is negative, and so the current limit can be -** queried by passing in a pointer to a negative number. This file-control -** is used internally to implement [PRAGMA mmap_limit]. +** the value originally pointed to is negative, and so the current limit +** can be queried by passing in a pointer to a negative number. This +** file-control is used internally to implement [PRAGMA mmap_size]. ** ** */ @@ -920,7 +920,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_PRAGMA 14 #define SQLITE_FCNTL_BUSYHANDLER 15 #define SQLITE_FCNTL_TEMPFILENAME 16 -#define SQLITE_FCNTL_MMAP_LIMIT 18 +#define SQLITE_FCNTL_MMAP_SIZE 18 /* ** CAPI3REF: Mutex Handle @@ -1650,17 +1650,19 @@ struct sqlite3_mem_methods { ** the connection being passed as the second parameter is being closed. The ** third parameter is passed NULL In this case. ** -** [[SQLITE_CONFIG_MMAP_LIMIT]] -**
    SQLITE_CONFIG_MMAP_LIMIT -**
    The sole argument should be a 64-bit integer (an sqlite3_int64) that -** is the default maximum number of bytes of process address space that -** should be used for accessing each database file using memory mapping. +** [[SQLITE_CONFIG_MMAP_SIZE]] +**
    SQLITE_CONFIG_MMAP_SIZE +**
    SQLITE_CONFIG_MMAP_SIZE takes two 64-bit integer (sqlite3_int64) values +** that are the default mmap size limit (the default setting for +** [PRAGMA mmap_size]) and the maximum allowed mmap size limit. ** The default setting can be overridden by each database connection using -** either the [PRAGMA mmap_limit] command, or by using the -** [SQLITE_FCNTL_MMAP_LIMIT] file control. The value set here overrides the -** compile-time default that is set using [SQLITE_DEFAULT_MMAP_LIMIT]. -** If the argument to this option is negative, then -** the memory map limit is set to the compile-time default. +** either the [PRAGMA mmap_size] command, or by using the +** [SQLITE_FCNTL_MMAP_SIZE] file control. The maximum allowed mmap size +** cannot be changed at run-time. Nor may the maximum allowed mmap size +** exceed the compile-time maximum mmap size set by the +** [SQLITE_MAX_MMAP_SIZE] compile-time option. +** If either argument to this option is negative, then that argument is +** changed to its compile-time default. ** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ @@ -1684,7 +1686,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ #define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ -#define SQLITE_CONFIG_MMAP_LIMIT 22 /* sqlite3_int64 */ +#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ /* ** CAPI3REF: Database Connection Configuration Options diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 264db39d14..92a5061406 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -540,13 +540,46 @@ extern const int sqlite3one; #endif /* -** Disable MMAP on platforms where it is not supported +** Disable MMAP on platforms where it is known to not work */ #if defined(__OpenBSD__) || defined(__QNXNTO__) -# undef SQLITE_DISABLE_MMAP -# define SQLITE_DISABLE_MMAP 1 +# undef SQLITE_MAX_MMAP_SIZE +# define SQLITE_MAX_MMAP_SIZE 0 #endif +/* +** Default maximum size of memory used by memory-mapped I/O in the VFS +*/ +#ifdef __APPLE__ +# include +# if TARGET_OS_IPHONE +# undef SQLITE_MAX_MMAP_SIZE +# define SQLITE_MAX_MMAP_SIZE 0 +# endif +#endif +#ifndef SQLITE_MAX_MMAP_SIZE +# if defined(__linux__) \ + || defined(_WIN32) \ + || (defined(__APPLE__) && defined(__MACH__)) \ + || defined(__sun) +# define SQLITE_MAX_MMAP_SIZE 2147483648 +# else +# define SQLITE_MAX_MMAP_SIZE 0 +# endif +#endif + +/* +** The default MMAP_SIZE is zero on all platforms. Or, even if a larger +** default MMAP_SIZE is specified at compile-time, make sure that it does +** not exceed the maximum mmap size. +*/ +#ifndef SQLITE_DEFAULT_MMAP_SIZE +# define SQLITE_DEFAULT_MMAP_SIZE 0 +#endif +#if SQLITE_DEFAULT_MMAP_SIZE>SQLITE_MAX_MMAP_SIZE +# undef SQLITE_DEFAULT_MMAP_SIZE +# define SQLITE_DEFAULT_MMAP_SIZE SQLITE_MAX_MMAP_SIZE +#endif /* ** An instance of the following structure is used to store the busy-handler @@ -842,7 +875,7 @@ struct sqlite3 { int nDb; /* Number of backends currently in use */ int flags; /* Miscellaneous flags. See below */ i64 lastRowid; /* ROWID of most recent insert (see above) */ - i64 mxMmap; /* Default mmap_limit setting */ + i64 szMmap; /* Default mmap_size setting */ unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ @@ -2516,7 +2549,8 @@ struct Sqlite3Config { void *pHeap; /* Heap storage space */ int nHeap; /* Size of pHeap[] */ int mnReq, mxReq; /* Min and max heap requests sizes */ - sqlite3_int64 mxMmap; /* Maximum mmap() space per open file */ + sqlite3_int64 szMmap; /* mmap() space per open file */ + sqlite3_int64 mxMmap; /* Maximum value for szMmap */ void *pScratch; /* Scratch memory */ int szScratch; /* Size of each scratch buffer */ int nScratch; /* Number of scratch buffers */ diff --git a/src/sqliteLimit.h b/src/sqliteLimit.h index d8f748e6b2..c7aee53ceb 100644 --- a/src/sqliteLimit.h +++ b/src/sqliteLimit.h @@ -206,27 +206,3 @@ #ifndef SQLITE_MAX_TRIGGER_DEPTH # define SQLITE_MAX_TRIGGER_DEPTH 1000 #endif - -/* -** Default maximum size of memory used by xFetch in the VFS. -*/ -#ifdef SQLITE_DISABLE_MMAP -# undef SQLITE_DEFAULT_MMAP_LIMIT -# define SQLITE_DEFAULT_MMAP_LIMIT 0 -#endif -#ifdef __APPLE__ -# include -# if TARGET_OS_IPHONE -# define SQLITE_DEFAULT_MMAP_LIMIT 0 -# endif -#endif -#ifndef SQLITE_DEFAULT_MMAP_LIMIT -# if defined(__linux__) \ - || defined(_WIN32) \ - || (defined(__APPLE__) && defined(__MACH__)) \ - || defined(__sun) -# define SQLITE_DEFAULT_MMAP_LIMIT 268435456 /* = 256*1024*1024 */ -# else -# define SQLITE_DEFAULT_MMAP_LIMIT 0 -# endif -#endif diff --git a/src/test_config.c b/src/test_config.c index 39608dbb64..4f61f4bb2b 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -87,10 +87,10 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "lfs", "1", TCL_GLOBAL_ONLY); #endif -#ifdef SQLITE_DISABLE_MMAP - Tcl_SetVar2(interp, "sqlite_options", "mmap", "0", TCL_GLOBAL_ONLY); -#else +#if SQLITE_MAX_MMAP_SIZE>0 Tcl_SetVar2(interp, "sqlite_options", "mmap", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "mmap", "0", TCL_GLOBAL_ONLY); #endif #if 1 /* def SQLITE_MEMDEBUG */ diff --git a/test/dbstatus2.test b/test/dbstatus2.test index c2d9164fa2..2541a1a823 100644 --- a/test/dbstatus2.test +++ b/test/dbstatus2.test @@ -40,7 +40,7 @@ proc db_write {db {reset 0}} { do_test 1.1 { db close sqlite3 db test.db - execsql { PRAGMA mmap_limit = 0 } + execsql { PRAGMA mmap_size = 0 } expr {[file size test.db] / 1024} } 6 diff --git a/test/exclusive2.test b/test/exclusive2.test index d09097671d..54203e3d8f 100644 --- a/test/exclusive2.test +++ b/test/exclusive2.test @@ -28,7 +28,7 @@ ifcapable {!pager_pragmas} { # Tests in this file verify that locking_mode=exclusive causes SQLite to # use cached pages even if the database is changed on disk. This doesn't # work with mmap. -if {[permutation]!="nommap"} { +if {[permutation]=="mmap"} { finish_test return } diff --git a/test/func.test b/test/func.test index 032becc94f..4ab7688461 100644 --- a/test/func.test +++ b/test/func.test @@ -1273,7 +1273,7 @@ do_test func-29.3 { sqlite3_db_status db CACHE_MISS 1 db eval {SELECT typeof(+x) FROM t29 ORDER BY id} } {integer null real blob text} -if {[permutation] == "nommap"} { +if {[permutation] != "mmap"} { do_test func-29.4 { set x [lindex [sqlite3_db_status db CACHE_MISS 1] 1] if {$x>100} {set x many} diff --git a/test/incrblob.test b/test/incrblob.test index fa564de0cf..4277e5c4c1 100644 --- a/test/incrblob.test +++ b/test/incrblob.test @@ -123,7 +123,7 @@ foreach AutoVacuumMode [list 0 1] { forcedelete test.db test.db-journal sqlite3 db test.db - execsql "PRAGMA mmap_limit = 0" + execsql "PRAGMA mmap_size = 0" execsql "PRAGMA auto_vacuum = $AutoVacuumMode" do_test incrblob-2.$AutoVacuumMode.1 { @@ -150,7 +150,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db - execsql "PRAGMA mmap_limit = 0" + execsql "PRAGMA mmap_size = 0" # Read the last 20 bytes of the blob via a blob handle. set ::blob [db incrblob blobs v 1] @@ -173,7 +173,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db - execsql "PRAGMA mmap_limit = 0" + execsql "PRAGMA mmap_size = 0" # Write the second-to-last 20 bytes of the blob via a blob handle. # @@ -203,7 +203,7 @@ foreach AutoVacuumMode [list 0 1] { # Open and close the db to make sure the page cache is empty. db close sqlite3 db test.db - execsql { PRAGMA mmap_limit = 0 } + execsql { PRAGMA mmap_size = 0 } execsql { SELECT i FROM blobs } } {45} diff --git a/test/mmap1.test b/test/mmap1.test index 9618109933..0d0e3fd7a4 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -40,17 +40,17 @@ proc register_rblob_code {dbname seed} { }] } -foreach {t mmap_limit nRead c2init} { - 1.1 { PRAGMA mmap_limit = 67108864 } 4 {PRAGMA mmap_limit = 0} - 1.2 { PRAGMA mmap_limit = 53248 } 150 {PRAGMA mmap_limit = 0} - 1.3 { PRAGMA mmap_limit = 0 } 344 {PRAGMA mmap_limit = 0} - 1.4 { PRAGMA mmap_limit = 67108864 } 4 {PRAGMA mmap_limit = 67108864 } - 1.5 { PRAGMA mmap_limit = 53248 } 150 {PRAGMA mmap_limit = 67108864 } - 1.6 { PRAGMA mmap_limit = 0 } 344 {PRAGMA mmap_limit = 67108864 } +foreach {t mmap_size nRead c2init} { + 1.1 { PRAGMA mmap_size = 67108864 } 4 {PRAGMA mmap_size = 0} + 1.2 { PRAGMA mmap_size = 53248 } 150 {PRAGMA mmap_size = 0} + 1.3 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = 0} + 1.4 { PRAGMA mmap_size = 67108864 } 4 {PRAGMA mmap_size = 67108864 } + 1.5 { PRAGMA mmap_size = 53248 } 150 {PRAGMA mmap_size = 67108864 } + 1.6 { PRAGMA mmap_size = 0 } 344 {PRAGMA mmap_size = 67108864 } } { do_multiclient_test tn { sql1 {PRAGMA page_size=1024} - sql1 $mmap_limit + sql1 $mmap_size sql2 $c2init code2 [register_rblob_code db2 0] @@ -89,7 +89,7 @@ foreach {t mmap_limit nRead c2init} { } {64 ok 149} # Check that the number of pages read by connection 1 indicates that the - # "PRAGMA mmap_limit" command worked. + # "PRAGMA mmap_size" command worked. do_test $t.$tn.5 { nRead db } $nRead } } @@ -106,7 +106,7 @@ db func rblob rblob do_execsql_test 2.1 { PRAGMA auto_vacuum = 1; - PRAGMA mmap_limit = 67108864; + PRAGMA mmap_size = 67108864; PRAGMA journal_mode = wal; CREATE TABLE t1(a, b, UNIQUE(a, b)); INSERT INTO t1 VALUES(rblob(500), rblob(500)); @@ -254,7 +254,7 @@ do_test 5.5 { } SQLITE_OK #------------------------------------------------------------------------- -# Test various mmap_limit settings. +# Test various mmap_size settings. # foreach {tn1 mmap1 mmap2} { 1 6144 167773 @@ -279,8 +279,8 @@ foreach {tn1 mmap1 mmap2} { code1 [register_rblob_code db 0] code2 [register_rblob_code db2 444] - sql1 "PRAGMA mmap_limit = $mmap1" - sql2 "PRAGMA mmap_limit = $mmap2" + sql1 "PRAGMA mmap_size = $mmap1" + sql2 "PRAGMA mmap_size = $mmap2" do_test $tn1.$tn { for {set i 1} {$i <= 100} {incr i} { diff --git a/test/pager1.test b/test/pager1.test index ab0ec67526..72e47805a4 100644 --- a/test/pager1.test +++ b/test/pager1.test @@ -2530,9 +2530,9 @@ if {$::tcl_platform(platform)=="unix"} { # an error. # foreach {tn pragma strsize} { - 1 { PRAGMA mmap_limit = 0 } 2400 + 1 { PRAGMA mmap_size = 0 } 2400 2 { } 2400 - 3 { PRAGMA mmap_limit = 0 } 4400 + 3 { PRAGMA mmap_size = 0 } 4400 4 { } 4400 } { reset_db @@ -2757,7 +2757,7 @@ do_test 42.1 { db close sqlite3_test_control_pending_byte 0x0010000 sqlite3 db test.db - db eval { PRAGMA mmap_limit = 0 } + db eval { PRAGMA mmap_size = 0 } catchsql { SELECT sum(length(y)) FROM t1 } } {1 {database disk image is malformed}} do_test 42.2 { @@ -2799,7 +2799,7 @@ do_test 43.1 { db close sqlite3 db test.db - db eval { PRAGMA mmap_limit = 0 } + db eval { PRAGMA mmap_size = 0 } db eval { SELECT * FROM t1 } sqlite3_db_status db CACHE_MISS 0 } {0 2 0} diff --git a/test/pagerfault.test b/test/pagerfault.test index 9b005c7a03..23754fa9de 100644 --- a/test/pagerfault.test +++ b/test/pagerfault.test @@ -1282,7 +1282,7 @@ faultsim_save_and_close do_faultsim_test pagerfault-28a -faults oom* -prep { faultsim_restore_and_reopen - execsql { PRAGMA mmap_limit=0 } + execsql { PRAGMA mmap_size=0 } sqlite3 db2 test.db db2 eval { SELECT count(*) FROM t2 } diff --git a/test/pageropt.test b/test/pageropt.test index a734e39a49..7191661ba5 100644 --- a/test/pageropt.test +++ b/test/pageropt.test @@ -88,7 +88,7 @@ do_test pageropt-1.4 { # must refill. # ifcapable mmap { - set x [expr {[permutation]=="nommap" ? 6 : 1}] + set x [expr {[permutation]=="mmap" ? 1 : 6}] } else { set x 6 } diff --git a/test/permutations.test b/test/permutations.test index 3aba9d9b38..9334fc49e6 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -138,10 +138,10 @@ test_suite "veryquick" -prefix "" -description { test_set $allquicktests -exclude *malloc* *ioerr* *fault* ] -test_suite "nommap" -prefix "nomm-" -description { +test_suite "mmap" -prefix "mm-" -description { Similar to veryquick. Except with memory mapping disabled. } -presql { - pragma mmap_limit = 0; + pragma mmap_size = 268435456; } -files [ test_set $allquicktests -exclude *malloc* *ioerr* *fault* -include malloc.test ] diff --git a/test/sysfault.test b/test/sysfault.test index 4ecaa1fabc..92fb534dd0 100644 --- a/test/sysfault.test +++ b/test/sysfault.test @@ -262,7 +262,7 @@ do_faultsim_test 4 -faults vfsfault-* -prep { faultsim_restore_and_reopen file_control_chunksize_test db main 8192 execsql { - PRAGMA mmap_limit = 1000000; + PRAGMA mmap_size = 1000000; } } -body { test_syscall errno mmap EACCES diff --git a/test/wal.test b/test/wal.test index 66000d0951..d9f2145b2e 100644 --- a/test/wal.test +++ b/test/wal.test @@ -728,7 +728,7 @@ do_test wal-11.9 { } {37 1} sqlite3_wal db test.db set nWal 39 -if {[permutation]=="nommap"} {set nWal 37} +if {[permutation]!="mmap"} {set nWal 37} ifcapable !mmap {set nWal 37} do_test wal-11.10 { execsql { diff --git a/test/wal5.test b/test/wal5.test index 4a6309886b..68750f1479 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -242,7 +242,7 @@ foreach {testprefix do_wal_checkpoint} { # However, in mmap() mode, the db is pre-allocated to 2 pages at the # start of the checkpoint, even though page 2 cannot be written. set nDb 2 - if {[permutation]=="no-mmap"} {set nDb 1} + if {[permutation]!="mmap"} {set nDb 1} ifcapable !mmap {set nDb 1} do_test 2.3.$tn.8 { file_page_counts } [list $nDb 4 2 4] } diff --git a/test/win32lock.test b/test/win32lock.test index 50c9f1d5f4..724172065f 100644 --- a/test/win32lock.test +++ b/test/win32lock.test @@ -27,7 +27,7 @@ proc xLog {error_code msg} { lappend ::log $msg } sqlite3 db test.db -db eval {PRAGMA mmap_limit=0} +db eval {PRAGMA mmap_size=0} do_test win32lock-1.1 { db eval { From 5824e05315453704c332f5cf64665932a059260f Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 15 Apr 2013 20:08:27 +0000 Subject: [PATCH 119/140] Expand scope of the SQLITE_MAX_MMAP_SIZE define for the Win32 VFS. FossilOrigin-Name: f4b8faab45e217f227bd7ce65d4741754731b8cd --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 18 +++++++++++++++--- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 215a7b5881..a560dd55b0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactoring\sthe\smmap\sinterface.\s\sThe\scontrolling\spragma\sis\snow\s"mmap_size"\ninstead\sof\s"mmap_limit".\s\sAlso\schange\sSQLITE_CONFIG_MMAP_LIMIT\sand\nSQLITE_FCNTL_MMAP_LIMIT\sto\sSQLITE_CONFIG_MMAP_SIZE\sand\nSQLITE_FCNTL_MMAP_SIZE,\srespecctively.\s\s\nThe\sdefault\smmap_size\sis\snow\salways\s0,\smeaning\sthat\nmemory\smapped\sI/O\sis\soff\sby\sdefault.\s\sThere\sis\sa\snew\scompile-time\soption\nSQLITE_MAX_MMAP_SIZE\sthat\sdetermines\sa\shard\supper\sbound\son\sthe\smmap_size.\nSetting\sSQLITE_MAX_MMAP_SIZE\sto\szero\sdisables\sthe\smemory-mapped\sI/O\slogic\nand\scauses\sit\sto\sbe\somitted\sfrom\sthe\sbuild.\s\sAn\sextra\sargument\sis\sadded\nto\sSQLITE_CONFIG_MMAP_SIZE\sthat\scan\soptionally\slower\sthe\sSQLITE_MAX_MMAP_SIZE\nat\sstart-time.\sThe\sSQLITE_MAX_MMAP_SIZE\sis\szero\sfor\splatforms\swhere\swe\s\nknow\sthat\sit\sdoes\snot\swork,\smeaning\sthat\sit\scannot\sbe\sturned\son\sby\smistake\non\sthose\splatforms. -D 2013-04-15T17:03:42.119 +C Expand\sscope\sof\sthe\sSQLITE_MAX_MMAP_SIZE\sdefine\sfor\sthe\sWin32\sVFS. +D 2013-04-15T20:08:27.655 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 0a561eae5965c9371300b0419027f5ae9f847af2 -F src/os_win.c 873bbc5dff8eb9f2bf8d9287d29df082eacb8391 +F src/os_win.c 673b3e3d1fa3040d8d95a7f1f5e0e553aed56cfb F src/pager.c a55adacb1842b83354198c408e7adde95ecd1189 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3412424990c93d2978e819e6099811f1cdde316d -R aa0c6ea54adf9732b395ebf6e1c98243 -U drh -Z 71eb761a9109889dcfada6d3231b23c2 +P ea1404a10abd7f68e1f8e0708c8a3199d1f79665 +R 4ce5175815a884ac178e2248b6e66ee1 +U mistachkin +Z 55620fd00464e5f458c15f94b0f9c9d2 diff --git a/manifest.uuid b/manifest.uuid index 65cc51dd9d..b2fbd3d393 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ea1404a10abd7f68e1f8e0708c8a3199d1f79665 \ No newline at end of file +f4b8faab45e217f227bd7ce65d4741754731b8cd \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index e56ccb50c6..12eefed8be 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -150,12 +150,14 @@ struct winFile { winceLock local; /* Locks obtained by this instance of winFile */ winceLock *shared; /* Global shared lock memory for the file */ #endif +#if SQLITE_MAX_MMAP_SIZE>0 int nFetchOut; /* Number of outstanding xFetch references */ HANDLE hMap; /* Handle for accessing memory mapping */ void *pMapRegion; /* Area memory mapped */ sqlite3_int64 mmapSize; /* Usable size of mapped region */ sqlite3_int64 mmapSizeActual; /* Actual size of mapped region */ sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ +#endif }; /* @@ -2068,8 +2070,10 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ #endif } +#if SQLITE_MAX_MMAP_SIZE>0 /* Forward references to VFS methods */ static int winUnmapfile(winFile*); +#endif /* ** Close a file. @@ -2093,8 +2097,10 @@ static int winClose(sqlite3_file *id){ OSTRACE(("CLOSE %d\n", pFile->h)); assert( pFile->h!=NULL && pFile->h!=INVALID_HANDLE_VALUE ); +#if SQLITE_MAX_MMAP_SIZE>0 rc = winUnmapfile(pFile); if( rc!=SQLITE_OK ) return rc; +#endif do{ rc = osCloseHandle(pFile->h); @@ -2842,6 +2848,7 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } +#if SQLITE_MAX_MMAP_SIZE>0 case SQLITE_FCNTL_MMAP_SIZE: { i64 newLimit = *(i64*)pArg; if( newLimit>sqlite3GlobalConfig.mxMmap ){ @@ -2851,6 +2858,7 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ if( newLimit>=0 ) pFile->mmapSizeMax = newLimit; return SQLITE_OK; } +#endif } return SQLITE_NOTFOUND; } @@ -3524,9 +3532,9 @@ shmpage_out: /* ** Cleans up the mapped region of the specified file, if any. */ +#if SQLITE_MAX_MMAP_SIZE>0 static int winUnmapfile(winFile *pFile){ assert( pFile!=0 ); -#if SQLITE_MAX_MMAP_SIZE>0 if( pFile->pMapRegion ){ if( !osUnmapViewOfFile(pFile->pMapRegion) ){ pFile->lastErrno = osGetLastError(); @@ -3545,11 +3553,9 @@ static int winUnmapfile(winFile *pFile){ } pFile->hMap = NULL; } -#endif return SQLITE_OK; } -#if SQLITE_MAX_MMAP_SIZE>0 /* ** Memory map or remap the file opened by file-descriptor pFd (if the file ** is already mapped, the existing mapping is replaced by the new). Or, if @@ -3652,7 +3658,9 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ ** release the reference by calling unixUnfetch(). */ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ +#if SQLITE_MAX_MMAP_SIZE>0 winFile *pFd = (winFile*)fd; /* The underlying database file */ +#endif *pp = 0; #if SQLITE_MAX_MMAP_SIZE>0 @@ -3681,6 +3689,7 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ ** may now be invalid and should be unmapped. */ static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ +#if SQLITE_MAX_MMAP_SIZE>0 winFile *pFd = (winFile*)fd; /* The underlying database file */ /* If p==0 (unmap the entire file) then there must be no outstanding @@ -3702,6 +3711,7 @@ static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ } assert( pFd->nFetchOut>=0 ); +#endif return SQLITE_OK; } @@ -4129,11 +4139,13 @@ static int winOpen( } pFile->lastErrno = NO_ERROR; pFile->zPath = zName; +#if SQLITE_MAX_MMAP_SIZE>0 pFile->hMap = NULL; pFile->pMapRegion = 0; pFile->mmapSize = 0; pFile->mmapSizeActual = 0; pFile->mmapSizeMax = sqlite3GlobalConfig.mxMmap; +#endif OpenCounter(+1); return rc; From 0f2340132ff70a7ffd5d36ae256c47e57ec789f1 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 16 Apr 2013 14:52:47 +0000 Subject: [PATCH 120/140] Disable memory-mapped I/O when the codec is enabled. FossilOrigin-Name: 340cca3079cd7aac2f51071435666e9e00ed8bd5 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 3 +++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a560dd55b0..e954f9c948 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Expand\sscope\sof\sthe\sSQLITE_MAX_MMAP_SIZE\sdefine\sfor\sthe\sWin32\sVFS. -D 2013-04-15T20:08:27.655 +C Disable\smemory-mapped\sI/O\swhen\sthe\scodec\sis\senabled. +D 2013-04-16T14:52:47.532 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,7 +168,7 @@ F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 0a561eae5965c9371300b0419027f5ae9f847af2 F src/os_win.c 673b3e3d1fa3040d8d95a7f1f5e0e553aed56cfb -F src/pager.c a55adacb1842b83354198c408e7adde95ecd1189 +F src/pager.c 6c3a8a5d665498b0344395a2c9f82d5abc4cc771 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 F src/parse.y 5d5e12772845805fdfeb889163516b84fbb9ae95 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ea1404a10abd7f68e1f8e0708c8a3199d1f79665 -R 4ce5175815a884ac178e2248b6e66ee1 -U mistachkin -Z 55620fd00464e5f458c15f94b0f9c9d2 +P f4b8faab45e217f227bd7ce65d4741754731b8cd +R 6717674ae7188e500403f1978f39e75b +U drh +Z 9185ac7186177dd4c0f45720166ae532 diff --git a/manifest.uuid b/manifest.uuid index b2fbd3d393..f78765dbdd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f4b8faab45e217f227bd7ce65d4741754731b8cd \ No newline at end of file +340cca3079cd7aac2f51071435666e9e00ed8bd5 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 96032b2896..1c8b5df651 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5220,6 +5220,9 @@ int sqlite3PagerAcquire( ** temporary or in-memory database. */ const int bMmapOk = (pgno!=1 && USEFETCH(pPager) && (pPager->eState==PAGER_READER || (flags & PAGER_ACQUIRE_READONLY)) +#ifdef SQLITE_HAS_CODEC + && pPager->xCodec==0 +#endif ); assert( pPager->eState>=PAGER_READER ); From 50287f5c0ca7730074da612f209541963e466e9e Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 16 Apr 2013 14:58:52 +0000 Subject: [PATCH 121/140] Remove the unused SQLITE_CURDIR macro. FossilOrigin-Name: 16f7ba4afaa69f6150b4defd4ddf5800f5d23084 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/ctime.c | 3 --- src/os.h | 8 -------- src/test_config.c | 2 +- 5 files changed, 10 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index e954f9c948..7f0f08bfbd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\smemory-mapped\sI/O\swhen\sthe\scodec\sis\senabled. -D 2013-04-16T14:52:47.532 +C Remove\sthe\sunused\sSQLITE_CURDIR\smacro. +D 2013-04-16T14:58:52.577 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -133,7 +133,7 @@ F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c 726e88b7282e7f348f805437db3222f97a136d35 +F src/ctime.c cb3cd5bf69bdf4ff3e1fc8ca3b1f0e405729dd80 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 @@ -164,7 +164,7 @@ F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be -F src/os.h ae08bcc5f6ec6b339f4a2adf3931bb88cc14c3e4 +F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 0a561eae5965c9371300b0419027f5ae9f847af2 F src/os_win.c 673b3e3d1fa3040d8d95a7f1f5e0e553aed56cfb @@ -203,7 +203,7 @@ F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de F src/test_btree.c 5b89601dcb42a33ba8b820a6b763cc9cb48bac16 -F src/test_config.c 49c99b714c1dab68aa6b37d53681a4278982be8d +F src/test_config.c 95bb33e9dcaa340a296c0bf0e0ba3d1a1c8004c0 F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c 8f786bfd0ad48030cf2a06fb1f050e9c60a150d7 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f4b8faab45e217f227bd7ce65d4741754731b8cd -R 6717674ae7188e500403f1978f39e75b +P 340cca3079cd7aac2f51071435666e9e00ed8bd5 +R 189962300d1abed770df02e09aa73052 U drh -Z 9185ac7186177dd4c0f45720166ae532 +Z 5f64b27d98cd9642b02d695bfb0e5a2b diff --git a/manifest.uuid b/manifest.uuid index f78765dbdd..367855f236 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -340cca3079cd7aac2f51071435666e9e00ed8bd5 \ No newline at end of file +16f7ba4afaa69f6150b4defd4ddf5800f5d23084 \ No newline at end of file diff --git a/src/ctime.c b/src/ctime.c index d60de40dd4..e9d023401f 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -48,9 +48,6 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_COVERAGE_TEST "COVERAGE_TEST", #endif -#ifdef SQLITE_CURDIR - "CURDIR", -#endif #ifdef SQLITE_DEBUG "DEBUG", #endif diff --git a/src/os.h b/src/os.h index 82e07ee1cb..070a2ddd17 100644 --- a/src/os.h +++ b/src/os.h @@ -99,14 +99,6 @@ # define SQLITE_OS_WINRT 0 #endif -/* -** When compiled for WinCE or WinRT, there is no concept of the current -** directory. - */ -#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT -# define SQLITE_CURDIR 1 -#endif - /* If the SET_FULLSYNC macro is not defined above, then make it ** a no-op */ diff --git a/src/test_config.c b/src/test_config.c index 4f61f4bb2b..534727a080 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -57,7 +57,7 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options","casesensitivelike","0",TCL_GLOBAL_ONLY); #endif -#ifdef SQLITE_CURDIR +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT Tcl_SetVar2(interp, "sqlite_options", "curdir", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "curdir", "0", TCL_GLOBAL_ONLY); From d16d0bc58b44ad97c6c12fd01600400f4776f653 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 16 Apr 2013 18:24:34 +0000 Subject: [PATCH 122/140] Omit sqlite3_compileoption_used() entries that are not explicitly stated on the compiler command-line, except keep SQLITE_THREADSAFE around so that there is always at least one compilation option. FossilOrigin-Name: 109d6767d2626741db1d15ed55e699ac3184a536 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/ctime.c | 20 +++++++++----------- src/sqliteInt.h | 13 ++++++++----- 4 files changed, 25 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 7f0f08bfbd..993bd65c98 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sunused\sSQLITE_CURDIR\smacro. -D 2013-04-16T14:58:52.577 +C Omit\ssqlite3_compileoption_used()\sentries\sthat\sare\snot\sexplicitly\sstated\non\sthe\scompiler\scommand-line,\sexcept\skeep\sSQLITE_THREADSAFE\saround\sso\sthat\nthere\sis\salways\sat\sleast\sone\scompilation\soption. +D 2013-04-16T18:24:34.635 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -133,7 +133,7 @@ F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c cb3cd5bf69bdf4ff3e1fc8ca3b1f0e405729dd80 +F src/ctime.c 387589abe92946f47dac6fa1d4479af977b4f11b F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 @@ -185,7 +185,7 @@ F src/shell.c 06bca04044b312ccadc43bf72385e7440cf60d1b F src/sqlite.h.in d897fd15b9c74d4e78e4c42319484ceb1649c5d7 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 -F src/sqliteInt.h 9933ad95b2c1e017f16428e9c0a929867598c2ae +F src/sqliteInt.h 2a83cfec9963372b636b0cabd4b200c1f1074a99 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 340cca3079cd7aac2f51071435666e9e00ed8bd5 -R 189962300d1abed770df02e09aa73052 +P 16f7ba4afaa69f6150b4defd4ddf5800f5d23084 +R 28c9578f8f913595ef0f108427330fb4 U drh -Z 5f64b27d98cd9642b02d695bfb0e5a2b +Z e5270d047eef465302b7108533238f0a diff --git a/manifest.uuid b/manifest.uuid index 367855f236..dd03c38f4f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -16f7ba4afaa69f6150b4defd4ddf5800f5d23084 \ No newline at end of file +109d6767d2626741db1d15ed55e699ac3184a536 \ No newline at end of file diff --git a/src/ctime.c b/src/ctime.c index e9d023401f..804bed704c 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -54,7 +54,7 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_DEFAULT_LOCKING_MODE "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), #endif -#ifdef SQLITE_DEFAULT_MMAP_SIZE +#if defined(SQLITE_DEFAULT_MMAP_SIZE) && !defined(SQLITE_DEFAULT_MMAP_SIZE_xc) "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE), #endif #ifdef SQLITE_DISABLE_DIRSYNC @@ -147,7 +147,7 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_LOCK_TRACE "LOCK_TRACE", #endif -#ifdef SQLITE_MAX_MMAP_SIZE +#if defined(SQLITE_MAX_MMAP_SIZE) && !defined(SQLITE_MAX_MMAP_SIZE_xc) "MAX_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_MMAP_SIZE), #endif #ifdef SQLITE_MAX_SCHEMA_RETRY @@ -207,11 +207,6 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_OMIT_CHECK "OMIT_CHECK", #endif -/* // redundant -** #ifdef SQLITE_OMIT_COMPILEOPTION_DIAGS -** "OMIT_COMPILEOPTION_DIAGS", -** #endif -*/ #ifdef SQLITE_OMIT_COMPLETE "OMIT_COMPLETE", #endif @@ -353,13 +348,13 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_TCL "TCL", #endif -#ifdef SQLITE_TEMP_STORE +#if defined(SQLITE_TEMP_STORE) && !defined(SQLITE_TEMP_STORE_xc) "TEMP_STORE=" CTIMEOPT_VAL(SQLITE_TEMP_STORE), #endif #ifdef SQLITE_TEST "TEST", #endif -#ifdef SQLITE_THREADSAFE +#if defined(SQLITE_THREADSAFE) "THREADSAFE=" CTIMEOPT_VAL(SQLITE_THREADSAFE), #endif #ifdef SQLITE_USE_ALLOCA @@ -385,8 +380,11 @@ int sqlite3_compileoption_used(const char *zOptName){ /* Since ArraySize(azCompileOpt) is normally in single digits, a ** linear search is adequate. No need for a binary search. */ for(i=0; iSQLITE_MAX_MMAP_SIZE # undef SQLITE_DEFAULT_MMAP_SIZE From 99b390827b9f4de28ea2095d8e26047c6041d532 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 17 Apr 2013 12:19:48 +0000 Subject: [PATCH 123/140] Update the command-line shell to give an exit code of 0 when the ".quit" command is run from the command-line. FossilOrigin-Name: 467e218120d63d0f34c447f7324bac1364ac0eac --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 993bd65c98..74612d5403 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\ssqlite3_compileoption_used()\sentries\sthat\sare\snot\sexplicitly\sstated\non\sthe\scompiler\scommand-line,\sexcept\skeep\sSQLITE_THREADSAFE\saround\sso\sthat\nthere\sis\salways\sat\sleast\sone\scompilation\soption. -D 2013-04-16T18:24:34.635 +C Update\sthe\scommand-line\sshell\sto\sgive\san\sexit\scode\sof\s0\swhen\sthe\s".quit"\ncommand\sis\srun\sfrom\sthe\scommand-line. +D 2013-04-17T12:19:48.061 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 -F src/shell.c 06bca04044b312ccadc43bf72385e7440cf60d1b +F src/shell.c a303eb4529ee78a68ee54e8ba6941f76ee5601d2 F src/sqlite.h.in d897fd15b9c74d4e78e4c42319484ceb1649c5d7 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 16f7ba4afaa69f6150b4defd4ddf5800f5d23084 -R 28c9578f8f913595ef0f108427330fb4 +P 109d6767d2626741db1d15ed55e699ac3184a536 +R 11f575757c220b40832c775a36cd7f16 U drh -Z e5270d047eef465302b7108533238f0a +Z 387afe95e75dc05ba67ce426c2ff6b48 diff --git a/manifest.uuid b/manifest.uuid index dd03c38f4f..fd6a44aeb7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -109d6767d2626741db1d15ed55e699ac3184a536 \ No newline at end of file +467e218120d63d0f34c447f7324bac1364ac0eac \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 7dde2203ec..1bb1b377b3 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3166,7 +3166,7 @@ int main(int argc, char **argv){ z = cmdline_option_value(argc,argv,++i); if( z[0]=='.' ){ rc = do_meta_command(z, &data); - if( rc && bail_on_error ) return rc; + if( rc && bail_on_error ) return rc==2 ? 0 : rc; }else{ open_db(&data); rc = shell_exec(data.db, z, shell_callback, &data, &zErrMsg); @@ -3190,6 +3190,7 @@ int main(int argc, char **argv){ */ if( zFirstCmd[0]=='.' ){ rc = do_meta_command(zFirstCmd, &data); + if( rc==2 ) rc = 0; }else{ open_db(&data); rc = shell_exec(data.db, zFirstCmd, shell_callback, &data, &zErrMsg); From 4670f6d5f60e8172d36d663b19353bfef06f0760 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 17 Apr 2013 14:04:52 +0000 Subject: [PATCH 124/140] Improvements to some of the comments that are used to render documentation concerning loadable extensions. No changes to code. FossilOrigin-Name: 76f4e31245fd1676a4520a2f7488bca6eb981e4a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 74612d5403..7d65c5f899 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\scommand-line\sshell\sto\sgive\san\sexit\scode\sof\s0\swhen\sthe\s".quit"\ncommand\sis\srun\sfrom\sthe\scommand-line. -D 2013-04-17T12:19:48.061 +C Improvements\sto\ssome\sof\sthe\scomments\sthat\sare\sused\sto\srender\sdocumentation\nconcerning\sloadable\sextensions.\s\sNo\schanges\sto\scode. +D 2013-04-17T14:04:52.335 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -182,7 +182,7 @@ F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c a303eb4529ee78a68ee54e8ba6941f76ee5601d2 -F src/sqlite.h.in d897fd15b9c74d4e78e4c42319484ceb1649c5d7 +F src/sqlite.h.in 31442751f3a63a538777188b6dfe70aebed338ac F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 2a83cfec9963372b636b0cabd4b200c1f1074a99 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 109d6767d2626741db1d15ed55e699ac3184a536 -R 11f575757c220b40832c775a36cd7f16 +P 467e218120d63d0f34c447f7324bac1364ac0eac +R 8ddf4f94c02f24d9b9c0c2b780ed2098 U drh -Z 387afe95e75dc05ba67ce426c2ff6b48 +Z d6edc377ec44c72133adc32403868eee diff --git a/manifest.uuid b/manifest.uuid index fd6a44aeb7..b12409c708 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -467e218120d63d0f34c447f7324bac1364ac0eac \ No newline at end of file +76f4e31245fd1676a4520a2f7488bca6eb981e4a \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 69f79a0377..f4f02d3eac 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -4218,7 +4218,7 @@ void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); ** the content before returning. ** ** The typedef is necessary to work around problems in certain -** C++ compilers. See ticket #2191. +** C++ compilers. */ typedef void (*sqlite3_destructor_type)(void*); #define SQLITE_STATIC ((sqlite3_destructor_type)0) @@ -5017,7 +5017,7 @@ int sqlite3_table_column_metadata( ** ^This interface loads an SQLite extension library from the named file. ** ** ^The sqlite3_load_extension() interface attempts to load an -** SQLite extension library contained in the file zFile. +** [SQLite extension] library contained in the file zFile. ** ** ^The entry point is zProc. ** ^zProc may be 0, in which case the name of the entry point @@ -5047,11 +5047,11 @@ int sqlite3_load_extension( ** CAPI3REF: Enable Or Disable Extension Loading ** ** ^So as not to open security holes in older applications that are -** unprepared to deal with extension loading, and as a means of disabling -** extension loading while evaluating user-entered SQL, the following API +** unprepared to deal with [extension loading], and as a means of disabling +** [extension loading] while evaluating user-entered SQL, the following API ** is provided to turn the [sqlite3_load_extension()] mechanism on and off. ** -** ^Extension loading is off by default. See ticket #1863. +** ^Extension loading is off by default. ** ^Call the sqlite3_enable_load_extension() routine with onoff==1 ** to turn extension loading on and call it with onoff==0 to turn ** it back off again. @@ -5063,7 +5063,7 @@ int sqlite3_enable_load_extension(sqlite3 *db, int onoff); ** ** ^This interface causes the xEntryPoint() function to be invoked for ** each new [database connection] that is created. The idea here is that -** xEntryPoint() is the entry point for a statically linked SQLite extension +** xEntryPoint() is the entry point for a statically linked [SQLite extension] ** that is to be automatically loaded into all new database connections. ** ** ^(Even though the function prototype shows that xEntryPoint() takes From 7a411f441bd10739266df96fb029b0c1068e2278 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 17 Apr 2013 17:33:17 +0000 Subject: [PATCH 125/140] In the command-line shell, allow a dot-command to occur after a multi-line c-style comment. FossilOrigin-Name: e2c94ab930f0e5a6dbe8cdd34ebb8cfeeedca56c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 7d65c5f899..3e8e1903cb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\ssome\sof\sthe\scomments\sthat\sare\sused\sto\srender\sdocumentation\nconcerning\sloadable\sextensions.\s\sNo\schanges\sto\scode. -D 2013-04-17T14:04:52.335 +C In\sthe\scommand-line\sshell,\sallow\sa\sdot-command\sto\soccur\safter\sa\nmulti-line\sc-style\scomment. +D 2013-04-17T17:33:17.457 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 -F src/shell.c a303eb4529ee78a68ee54e8ba6941f76ee5601d2 +F src/shell.c aca9d94653decd4496846dee0c7ba83eaf96a46d F src/sqlite.h.in 31442751f3a63a538777188b6dfe70aebed338ac F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 467e218120d63d0f34c447f7324bac1364ac0eac -R 8ddf4f94c02f24d9b9c0c2b780ed2098 +P 76f4e31245fd1676a4520a2f7488bca6eb981e4a +R 7329574a8fd763912288e6955296b961 U drh -Z d6edc377ec44c72133adc32403868eee +Z d7a8232879b542fa4edbd90644b25d07 diff --git a/manifest.uuid b/manifest.uuid index b12409c708..2d365214d6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -76f4e31245fd1676a4520a2f7488bca6eb981e4a \ No newline at end of file +e2c94ab930f0e5a6dbe8cdd34ebb8cfeeedca56c \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 1bb1b377b3..13a2f70cbc 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2784,6 +2784,10 @@ static int process_input(struct callback_data *p, FILE *in){ free(zSql); zSql = 0; nSql = 0; + }else if( zSql && _all_whitespace(zSql) ){ + free(zSql); + zSql = 0; + nSql = 0; } } if( zSql ){ From 87f9caa85ca7249d13d814993ac6e375d16b8535 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 17 Apr 2013 18:56:16 +0000 Subject: [PATCH 126/140] In the mptester, add --glob and --notglob and --testcase. Make --exit work on the main thread. Enable the load_extension() SQL function. FossilOrigin-Name: c273c171f511475045ef0aa68ecf8e22b8351996 --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 38 +++++++++++++++++++++++++++++++++++++- 3 files changed, 44 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3e8e1903cb..8c393daed2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\scommand-line\sshell,\sallow\sa\sdot-command\sto\soccur\safter\sa\nmulti-line\sc-style\scomment. -D 2013-04-17T17:33:17.457 +C In\sthe\smptester,\sadd\s--glob\sand\s--notglob\sand\s--testcase.\s\sMake\s--exit\swork\non\sthe\smain\sthread.\s\sEnable\sthe\sload_extension()\sSQL\sfunction. +D 2013-04-17T18:56:16.833 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 -F mptest/mptest.c 2a263e88f18762ac5d7fb56749d2d2ea8e22cb93 +F mptest/mptest.c 36c511bf0b562ccadede7cd31208dc1ec75b7ce2 F mptest/multiwrite01.test 81fbc17657964889b60750bd7bbb1deffe8f4d42 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 76f4e31245fd1676a4520a2f7488bca6eb981e4a -R 7329574a8fd763912288e6955296b961 +P e2c94ab930f0e5a6dbe8cdd34ebb8cfeeedca56c +R d244e7c9ac86257619e9ec497e6803a5 U drh -Z d7a8232879b542fa4edbd90644b25d07 +Z 0c9d37cd97d14173f04c5765259a70d1 diff --git a/manifest.uuid b/manifest.uuid index 2d365214d6..6821f27d9a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e2c94ab930f0e5a6dbe8cdd34ebb8cfeeedca56c \ No newline at end of file +c273c171f511475045ef0aa68ecf8e22b8351996 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index e47be5ae6d..8dc95be7bc 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -898,13 +898,24 @@ static void runScript( ** Exit this process. If N>0 then exit without shutting down ** SQLite. (In other words, simulate a crash.) */ - if( strcmp(zCmd, "exit")==0 && iClient>0 ){ + if( strcmp(zCmd, "exit")==0 ){ int rc = atoi(azArg[0]); finishScript(iClient, taskId, 1); if( rc==0 ) sqlite3_close(g.db); exit(rc); }else + /* + ** --testcase NAME + ** + ** Exit this process. If N>0 then exit without shutting down + ** SQLite. (In other words, simulate a crash.) + */ + if( strcmp(zCmd, "testcase")==0 ){ + if( g.iTrace==1 ) logMessage("%.*s", len - 1, zScript+ii); + stringReset(&sResult); + }else + /* ** --finish ** @@ -942,6 +953,30 @@ static void runScript( stringReset(&sResult); }else + /* + ** --glob ANSWER... + ** --notglob ANSWER.... + ** + ** Check to see if output does or does not match the glob pattern + ** ANSWER. + */ + if( strcmp(zCmd, "glob")==0 || strcmp(zCmd, "notglob")==0 ){ + int jj; + char *zAns = zScript+ii; + char *zCopy; + int isGlob = (zCmd[0]=='g'); + for(jj=9-3*isGlob; jj Date: Wed, 17 Apr 2013 19:42:17 +0000 Subject: [PATCH 127/140] Fix the --match command in mptester so that it checks the entire result, not just a prefix of the result. FossilOrigin-Name: 3e3ecad2b5d6c97032f2a9fb920c1b8a16ab8ef5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 8c393daed2..6d229bb880 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\smptester,\sadd\s--glob\sand\s--notglob\sand\s--testcase.\s\sMake\s--exit\swork\non\sthe\smain\sthread.\s\sEnable\sthe\sload_extension()\sSQL\sfunction. -D 2013-04-17T18:56:16.833 +C Fix\sthe\s--match\scommand\sin\smptester\sso\sthat\sit\schecks\sthe\sentire\sresult,\nnot\sjust\sa\sprefix\sof\sthe\sresult. +D 2013-04-17T19:42:17.479 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 -F mptest/mptest.c 36c511bf0b562ccadede7cd31208dc1ec75b7ce2 +F mptest/mptest.c 23365b9f65f220f1206caed7bcee651538a31559 F mptest/multiwrite01.test 81fbc17657964889b60750bd7bbb1deffe8f4d42 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e2c94ab930f0e5a6dbe8cdd34ebb8cfeeedca56c -R d244e7c9ac86257619e9ec497e6803a5 +P c273c171f511475045ef0aa68ecf8e22b8351996 +R c2176482a0665dc72d6ff4f34c638c84 U drh -Z 0c9d37cd97d14173f04c5765259a70d1 +Z 66ee4dc6efdb5b603ad21e244f0428fb diff --git a/manifest.uuid b/manifest.uuid index 6821f27d9a..60b8e3a6e5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c273c171f511475045ef0aa68ecf8e22b8351996 \ No newline at end of file +3e3ecad2b5d6c97032f2a9fb920c1b8a16ab8ef5 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 8dc95be7bc..cb6dd55d91 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -945,7 +945,7 @@ static void runScript( char *zAns = zScript+ii; for(jj=7; jj Date: Thu, 18 Apr 2013 02:55:54 +0000 Subject: [PATCH 128/140] Do not suppress the ORDER BY clause on a virtual table query if an IN constraint is used. Fix for ticket [f69b96e3076e]. Testing done on TH3 using cov1/where37.test. FossilOrigin-Name: 61b2a7be3b9c04bf45bffa93a7d3a480fc5c947a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 19 ++++++++----------- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 6d229bb880..d34134f033 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s--match\scommand\sin\smptester\sso\sthat\sit\schecks\sthe\sentire\sresult,\nnot\sjust\sa\sprefix\sof\sthe\sresult. -D 2013-04-17T19:42:17.479 +C Do\snot\ssuppress\sthe\sORDER\sBY\sclause\son\sa\svirtual\stable\squery\sif\san\sIN\nconstraint\sis\sused.\s\sFix\sfor\sticket\s[f69b96e3076e].\s\sTesting\sdone\son\nTH3\susing\scov1/where37.test. +D 2013-04-18T02:55:54.826 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -258,7 +258,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c 4ad2329c439a30ddb915a780f6f80bdffafe3a64 +F src/where.c e63f84e5583133c77573b78d3696dccd97bfd00d F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c273c171f511475045ef0aa68ecf8e22b8351996 -R c2176482a0665dc72d6ff4f34c638c84 +P 3e3ecad2b5d6c97032f2a9fb920c1b8a16ab8ef5 +R 6fada9489bb28405c28c5a46dce1bedb U drh -Z 66ee4dc6efdb5b603ad21e244f0428fb +Z da33d363d10fd9678c928e7a78f01d61 diff --git a/manifest.uuid b/manifest.uuid index 60b8e3a6e5..8101a7d5ae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3e3ecad2b5d6c97032f2a9fb920c1b8a16ab8ef5 \ No newline at end of file +61b2a7be3b9c04bf45bffa93a7d3a480fc5c947a \ No newline at end of file diff --git a/src/where.c b/src/where.c index bd8a7541fa..a0dc868422 100644 --- a/src/where.c +++ b/src/where.c @@ -2275,9 +2275,8 @@ static void bestVirtualIndex(WhereBestIdx *p){ struct sqlite3_index_constraint *pIdxCons; struct sqlite3_index_constraint_usage *pUsage; WhereTerm *pTerm; - int i, j, k; + int i, j; int nOrderBy; - int sortOrder; /* Sort order for IN clauses */ int bAllowIN; /* Allow IN optimizations */ double rCost; @@ -2376,7 +2375,6 @@ static void bestVirtualIndex(WhereBestIdx *p){ return; } - sortOrder = SQLITE_SO_ASC; pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; for(i=0; inConstraint; i++, pIdxCons++){ if( pUsage[i].argvIndex>0 ){ @@ -2391,12 +2389,12 @@ static void bestVirtualIndex(WhereBestIdx *p){ ** repeated in the output. */ break; } - for(k=0; knOrderBy; k++){ - if( pIdxInfo->aOrderBy[k].iColumn==pIdxCons->iColumn ){ - sortOrder = pIdxInfo->aOrderBy[k].desc; - break; - } - } + /* A virtual table that is constrained by an IN clause may not + ** consume the ORDER BY clause because (1) the order of IN terms + ** is not necessarily related to the order of output terms and + ** (2) Multiple outputs from a single IN value will not merge + ** together. */ + pIdxInfo->orderByConsumed = 0; } } } @@ -2426,8 +2424,7 @@ static void bestVirtualIndex(WhereBestIdx *p){ } p->cost.plan.u.pVtabIdx = pIdxInfo; if( pIdxInfo->orderByConsumed ){ - assert( sortOrder==0 || sortOrder==1 ); - p->cost.plan.wsFlags |= WHERE_ORDERED + sortOrder*WHERE_REVERSE; + p->cost.plan.wsFlags |= WHERE_ORDERED; p->cost.plan.nOBSat = nOrderBy; }else{ p->cost.plan.nOBSat = p->i ? p->aLevel[p->i-1].plan.nOBSat : 0; From a956af67e1a34d4a60fe8bddb9496969525a894a Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 18 Apr 2013 03:10:43 +0000 Subject: [PATCH 129/140] Fix a compiler warning in sqlite3_compileoption_used(). FossilOrigin-Name: e9f9d84b223b69d36688cd7a4c7c696bfda8a1ad --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/ctime.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d34134f033..a10eb45721 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\ssuppress\sthe\sORDER\sBY\sclause\son\sa\svirtual\stable\squery\sif\san\sIN\nconstraint\sis\sused.\s\sFix\sfor\sticket\s[f69b96e3076e].\s\sTesting\sdone\son\nTH3\susing\scov1/where37.test. -D 2013-04-18T02:55:54.826 +C Fix\sa\scompiler\swarning\sin\ssqlite3_compileoption_used(). +D 2013-04-18T03:10:43.014 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -133,7 +133,7 @@ F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c 083da8466fd7e481cb8bd5264398f537507f6176 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c 387589abe92946f47dac6fa1d4479af977b4f11b +F src/ctime.c 4262c227bc91cecc61ae37ed3a40f08069cfa267 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c aeabdabeeeaa0584127f291baa9617153d334778 F src/expr.c 48048fca951eedbc74aa32262154410d56c83812 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3e3ecad2b5d6c97032f2a9fb920c1b8a16ab8ef5 -R 6fada9489bb28405c28c5a46dce1bedb +P 61b2a7be3b9c04bf45bffa93a7d3a480fc5c947a +R 6659de1ffecbe8f368709b1c4cb6dc19 U drh -Z da33d363d10fd9678c928e7a78f01d61 +Z 0f74e436c879f4a5d6bdb52557a8b2af diff --git a/manifest.uuid b/manifest.uuid index 8101a7d5ae..26f13d75e5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -61b2a7be3b9c04bf45bffa93a7d3a480fc5c947a \ No newline at end of file +e9f9d84b223b69d36688cd7a4c7c696bfda8a1ad \ No newline at end of file diff --git a/src/ctime.c b/src/ctime.c index 804bed704c..60595ff88d 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -381,7 +381,7 @@ int sqlite3_compileoption_used(const char *zOptName){ ** linear search is adequate. No need for a binary search. */ for(i=0; i Date: Thu, 18 Apr 2013 15:11:03 +0000 Subject: [PATCH 130/140] Add the --breakpoint and --show-sql-errors commands to mptester. FossilOrigin-Name: d0898fd76a54512894418e53ba28703e250c9ed3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index a10eb45721..519938b65f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scompiler\swarning\sin\ssqlite3_compileoption_used(). -D 2013-04-18T03:10:43.014 +C Add\sthe\s--breakpoint\sand\s--show-sql-errors\scommands\sto\smptester. +D 2013-04-18T15:11:03.053 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 -F mptest/mptest.c 23365b9f65f220f1206caed7bcee651538a31559 +F mptest/mptest.c ce0d99750a9c01624f29b8942241592e94a7d0d1 F mptest/multiwrite01.test 81fbc17657964889b60750bd7bbb1deffe8f4d42 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 61b2a7be3b9c04bf45bffa93a7d3a480fc5c947a -R 6659de1ffecbe8f368709b1c4cb6dc19 +P e9f9d84b223b69d36688cd7a4c7c696bfda8a1ad +R fd7f4162984b752563533589ad89bff8 U drh -Z 0f74e436c879f4a5d6bdb52557a8b2af +Z e0f5854023cf43fccbc89e940a13a42c diff --git a/manifest.uuid b/manifest.uuid index 26f13d75e5..530384dc8c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e9f9d84b223b69d36688cd7a4c7c696bfda8a1ad \ No newline at end of file +d0898fd76a54512894418e53ba28703e250c9ed3 \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index cb6dd55d91..ac3d408204 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -71,6 +71,7 @@ static struct Global { int taskId; /* Task ID. 0 means supervisor. */ int iTrace; /* Tracing level */ int bSqlTrace; /* True to trace SQL commands */ + int bIgnoreSqlErrors; /* Ignore errors in SQL statements */ int nError; /* Number of errors */ int nTest; /* Number of --match operators */ int iTimeout; /* Milliseconds until a busy timeout */ @@ -329,6 +330,7 @@ static void sqlTraceCallback(void *NotUsed1, const char *zSql){ */ static void sqlErrorCallback(void *pArg, int iErrCode, const char *zMsg){ UNUSED_PARAMETER(pArg); + if( iErrCode==SQLITE_ERROR && g.bIgnoreSqlErrors ) return; if( (iErrCode&0xff)==SQLITE_SCHEMA && g.iTrace<3 ) return; if( g.iTimeout==0 && (iErrCode&0xff)==SQLITE_BUSY && g.iTrace<3 ) return; if( (iErrCode&0xff)==SQLITE_NOTICE ){ @@ -826,6 +828,30 @@ static char *filenameTail(char *z){ return z+j; } +/* +** Interpret zArg as a boolean value. Return either 0 or 1. +*/ +static int booleanValue(char *zArg){ + int i; + if( zArg==0 ) return 0; + for(i=0; zArg[i]>='0' && zArg[i]<='9'; i++){} + if( i>0 && zArg[i]==0 ) return atoi(zArg); + if( sqlite3_stricmp(zArg, "on")==0 || sqlite3_stricmp(zArg,"yes")==0 ){ + return 1; + } + if( sqlite3_stricmp(zArg, "off")==0 || sqlite3_stricmp(zArg,"no")==0 ){ + return 0; + } + errorMessage("unknown boolean: [%s]", zArg); + return 0; +} + + +/* This routine exists as a convenient place to set a debugger +** breakpoint. +*/ +static void test_breakpoint(void){ static volatile int cnt = 0; cnt++; } + /* Maximum number of arguments to a --command */ #define MX_ARG 2 @@ -1119,6 +1145,26 @@ static void runScript( iBegin = ii+len; }else + /* + ** --breakpoint + ** + ** This command calls "test_breakpoint()" which is a routine provided + ** as a convenient place to set a debugger breakpoint. + */ + if( strcmp(zCmd, "breakpoint")==0 ){ + test_breakpoint(); + }else + + /* + ** --show-sql-errors BOOLEAN + ** + ** Turn display of SQL errors on and off. + */ + if( strcmp(zCmd, "show-sql-errors")==0 ){ + g.bIgnoreSqlErrors = nArg>=1 ? !booleanValue(azArg[0]) : 1; + }else + + /* error */{ errorMessage("line %d of %s: unknown command --%s", prevLine, zFilename, zCmd); From 93c8c457e85e9fec3bd62278c62e77feaf722225 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 18 Apr 2013 20:33:41 +0000 Subject: [PATCH 131/140] Fix an incorrect comment (a copy/paste error) in the mptester. No code changes. FossilOrigin-Name: e94783f36ade43ef9d5b94db278622bcbce4179b --- manifest | 12 ++++++------ manifest.uuid | 2 +- mptest/mptest.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 519938b65f..e07c960b3c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s--breakpoint\sand\s--show-sql-errors\scommands\sto\smptester. -D 2013-04-18T15:11:03.053 +C Fix\san\sincorrect\scomment\s(a\scopy/paste\serror)\sin\sthe\smptester.\s\sNo\scode\nchanges. +D 2013-04-18T20:33:41.233 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test a5f31998ed48de8267d6620e8af107ec148e5f12 F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 -F mptest/mptest.c ce0d99750a9c01624f29b8942241592e94a7d0d1 +F mptest/mptest.c 499a74af4be293b7c1c7c3d40f332b67227dd739 F mptest/multiwrite01.test 81fbc17657964889b60750bd7bbb1deffe8f4d42 F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e9f9d84b223b69d36688cd7a4c7c696bfda8a1ad -R fd7f4162984b752563533589ad89bff8 +P d0898fd76a54512894418e53ba28703e250c9ed3 +R 489ed5534c8ff7ba356053afa67de654 U drh -Z e0f5854023cf43fccbc89e940a13a42c +Z da35d51e74cda792a97ee9cf150ece03 diff --git a/manifest.uuid b/manifest.uuid index 530384dc8c..3b6aeacff2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d0898fd76a54512894418e53ba28703e250c9ed3 \ No newline at end of file +e94783f36ade43ef9d5b94db278622bcbce4179b \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index ac3d408204..059ae102fa 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -934,8 +934,8 @@ static void runScript( /* ** --testcase NAME ** - ** Exit this process. If N>0 then exit without shutting down - ** SQLite. (In other words, simulate a crash.) + ** Begin a new test case. Announce in the log that the test case + ** has begun. */ if( strcmp(zCmd, "testcase")==0 ){ if( g.iTrace==1 ) logMessage("%.*s", len - 1, zScript+ii); From c288e442e3400e4672f81c1acf86bf2ed61480d1 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 18 Apr 2013 22:56:42 +0000 Subject: [PATCH 132/140] Experiments in making loadable extensions easier to program. The shared library filename tries various operating-system specific extensions if the bare library name does not work. And the entry point is derived from the filename is the legacy entry point name "sqlite3_extension_init" is not found. FossilOrigin-Name: 6cfa1d73d79b9f0be8157f0a37c264dd95b031c8 --- manifest | 17 +++++++----- manifest.uuid | 2 +- src/loadext.c | 74 +++++++++++++++++++++++++++++++++++++++++++------ src/sqlite.h.in | 15 ++++++++-- 4 files changed, 89 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index e07c960b3c..ed0d92dbad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sincorrect\scomment\s(a\scopy/paste\serror)\sin\sthe\smptester.\s\sNo\scode\nchanges. -D 2013-04-18T20:33:41.233 +C Experiments\sin\smaking\sloadable\sextensions\seasier\sto\sprogram.\s\sThe\nshared\slibrary\sfilename\stries\svarious\soperating-system\sspecific\sextensions\nif\sthe\sbare\slibrary\sname\sdoes\snot\swork.\s\sAnd\sthe\sentry\spoint\sis\sderived\nfrom\sthe\sfilename\sis\sthe\slegacy\sentry\spoint\sname\s"sqlite3_extension_init"\nis\snot\sfound. +D 2013-04-18T22:56:42.784 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -148,7 +148,7 @@ F src/insert.c f7cb141e8ce257cb6b15c497f09e4e23d6055599 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b -F src/loadext.c 1422eba4aa2b1fb5f7b3aef574752272477d21e2 +F src/loadext.c c48f7f3f170e502fe0cc20748e03c6e0b5a016c2 F src/main.c de47c3e6b68f512b9e64ba2533ee4eecc9d501c1 F src/malloc.c fe085aa851b666b7c375c1ff957643dc20a04bf6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -182,7 +182,7 @@ F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c aca9d94653decd4496846dee0c7ba83eaf96a46d -F src/sqlite.h.in 31442751f3a63a538777188b6dfe70aebed338ac +F src/sqlite.h.in a291a999d28ebc16cd1fe7fdd010dfe5266ef767 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 F src/sqliteInt.h 2a83cfec9963372b636b0cabd4b200c1f1074a99 @@ -1051,7 +1051,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d0898fd76a54512894418e53ba28703e250c9ed3 -R 489ed5534c8ff7ba356053afa67de654 +P e94783f36ade43ef9d5b94db278622bcbce4179b +R 5eb9b691174dd0ce00c815b019b68747 +T *branch * loadext +T *sym-loadext * +T -sym-trunk * U drh -Z da35d51e74cda792a97ee9cf150ece03 +Z 1611e9ed5f0542b10fe5f2f8888ad81f diff --git a/manifest.uuid b/manifest.uuid index 3b6aeacff2..fc3b50454d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e94783f36ade43ef9d5b94db278622bcbce4179b \ No newline at end of file +6cfa1d73d79b9f0be8157f0a37c264dd95b031c8 \ No newline at end of file diff --git a/src/loadext.c b/src/loadext.c index 76450f2e8b..cdcf6a93b8 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -415,8 +415,23 @@ static int sqlite3LoadExtension( void *handle; int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); char *zErrmsg = 0; + const char *zEntry; + char *zAltEntry = 0; void **aHandle; int nMsg = 300 + sqlite3Strlen30(zFile); + int ii; + + /* Shared library endings to try if zFile cannot be loaded as written */ + static const char *azEndings[] = { +#if SQLITE_OS_WIN + "dll" +#elif defined(__APPLE__) + "dylib" +#else + "so" +#endif + }; + if( pzErrMsg ) *pzErrMsg = 0; @@ -433,11 +448,17 @@ static int sqlite3LoadExtension( return SQLITE_ERROR; } - if( zProc==0 ){ - zProc = "sqlite3_extension_init"; - } + zEntry = zProc ? zProc : "sqlite3_extension_init"; handle = sqlite3OsDlOpen(pVfs, zFile); +#if SQLITE_OS_UNIX || SQLITE_OS_WIN + for(ii=0; ii sqlite3_example_init + ** C:/lib/mathfuncs.dll ==> sqlite3_mathfuncs_init + */ + if( xInit==0 && zProc==0 ){ + int iFile, iEntry, c; + int ncFile = sqlite3Strlen30(zFile); + zAltEntry = sqlite3_malloc(ncFile+30); + if( zAltEntry==0 ){ + sqlite3OsDlClose(pVfs, handle); + return SQLITE_NOMEM; + } + memcpy(zAltEntry, "sqlite3_", 8); + for(iFile=ncFile-1; iFile>=0 && zFile[iFile]!='/'; iFile--){} + iFile++; + if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; + for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ + if( sqlite3Isalpha(c) ){ + zAltEntry[iEntry++] = (char)sqlite3UpperToLower[(unsigned)c]; + } + } + memcpy(zAltEntry+iEntry, "_init", 6); + zEntry = zAltEntry; + xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) + sqlite3OsDlSym(pVfs, handle, zEntry); + } if( xInit==0 ){ if( pzErrMsg ){ - nMsg += sqlite3Strlen30(zProc); + nMsg += sqlite3Strlen30(zEntry); *pzErrMsg = zErrmsg = sqlite3_malloc(nMsg); if( zErrmsg ){ sqlite3_snprintf(nMsg, zErrmsg, - "no entry point [%s] in shared library [%s]", zProc,zFile); + "no entry point [%s] in shared library [%s]", zEntry, zFile); sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); } - sqlite3OsDlClose(pVfs, handle); } + sqlite3OsDlClose(pVfs, handle); + sqlite3_free(zAltEntry); return SQLITE_ERROR; - }else if( xInit(db, &zErrmsg, &sqlite3Apis) ){ + } + sqlite3_free(zAltEntry); + if( xInit(db, &zErrmsg, &sqlite3Apis) ){ if( pzErrMsg ){ *pzErrMsg = sqlite3_mprintf("error during initialization: %s", zErrmsg); } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index f4f02d3eac..3c6291eebe 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5017,11 +5017,20 @@ int sqlite3_table_column_metadata( ** ^This interface loads an SQLite extension library from the named file. ** ** ^The sqlite3_load_extension() interface attempts to load an -** [SQLite extension] library contained in the file zFile. +** [SQLite extension] library contained in the file zFile. If +** the file cannot be loaded directly, attempts are made to load +** with various operating-system specific extensions added. +** So for example, if "samplelib" cannot be loaded, then names like +** "samplelib.so" or "samplelib.dylib" or "samplelib.dll" might +** be tried also. ** ** ^The entry point is zProc. -** ^zProc may be 0, in which case the name of the entry point -** defaults to "sqlite3_extension_init". +** ^(zProc may be 0, in which case SQLite will try to come up with an +** entry point name on its own. It first tries "sqlite3_extension_init". +** If that does not work, it constructs a name "sqlite3_X_init" where the +** X is consists of the lower-case equivalent of all ASCII alphabetic +** characters in the filename from the last "/" to the first following +** "." and omitting any initial "lib".)^ ** ^The sqlite3_load_extension() interface returns ** [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. ** ^If an error occurs and pzErrMsg is not 0, then the From 08ef8d7311624b78af522275de4e0d62b799882c Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 19 Apr 2013 01:23:06 +0000 Subject: [PATCH 133/140] Make the SQLITE_EXTENSION_INIT1 and SQLITE_EXTENSION_INIT2 macros into no-ops if the extension is statically linked. FossilOrigin-Name: 86eb3eed4c70730dba0e319f243f8cb4fa8de06c --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/sqlite3ext.h | 13 +++++++++++-- src/test_autoext.c | 2 +- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index ed0d92dbad..d25b1c8d4d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experiments\sin\smaking\sloadable\sextensions\seasier\sto\sprogram.\s\sThe\nshared\slibrary\sfilename\stries\svarious\soperating-system\sspecific\sextensions\nif\sthe\sbare\slibrary\sname\sdoes\snot\swork.\s\sAnd\sthe\sentry\spoint\sis\sderived\nfrom\sthe\sfilename\sis\sthe\slegacy\sentry\spoint\sname\s"sqlite3_extension_init"\nis\snot\sfound. -D 2013-04-18T22:56:42.784 +C Make\sthe\sSQLITE_EXTENSION_INIT1\sand\sSQLITE_EXTENSION_INIT2\smacros\sinto\nno-ops\sif\sthe\sextension\sis\sstatically\slinked. +D 2013-04-19T01:23:06.788 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -184,7 +184,7 @@ F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c aca9d94653decd4496846dee0c7ba83eaf96a46d F src/sqlite.h.in a291a999d28ebc16cd1fe7fdd010dfe5266ef767 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 -F src/sqlite3ext.h 7183ab832e23db0f934494f16928da127a571d75 +F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 F src/sqliteInt.h 2a83cfec9963372b636b0cabd4b200c1f1074a99 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9 @@ -200,7 +200,7 @@ F src/test7.c f4b894b7931f8cf9f5cbf37cfa0727703f526a40 F src/test8.c 58ea1d9698f3947e4662107ef98f429e84ae20e0 F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad -F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e +F src/test_autoext.c 5c95b5d435eaa09d6c0e7d90371c5ca8cd567701 F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de F src/test_btree.c 5b89601dcb42a33ba8b820a6b763cc9cb48bac16 F src/test_config.c 95bb33e9dcaa340a296c0bf0e0ba3d1a1c8004c0 @@ -1051,10 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e94783f36ade43ef9d5b94db278622bcbce4179b -R 5eb9b691174dd0ce00c815b019b68747 -T *branch * loadext -T *sym-loadext * -T -sym-trunk * +P 6cfa1d73d79b9f0be8157f0a37c264dd95b031c8 +R 6c3fdd8d85d98822b9ca8decd52bbb5c U drh -Z 1611e9ed5f0542b10fe5f2f8888ad81f +Z 311c6fc82989346d7dcdfa6c2765eba0 diff --git a/manifest.uuid b/manifest.uuid index fc3b50454d..ee9ed3efc1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6cfa1d73d79b9f0be8157f0a37c264dd95b031c8 \ No newline at end of file +86eb3eed4c70730dba0e319f243f8cb4fa8de06c \ No newline at end of file diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index a465b8eccc..928bb3bad9 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -469,7 +469,16 @@ struct sqlite3_api_routines { #define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2 #endif /* SQLITE_CORE */ -#define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0; -#define SQLITE_EXTENSION_INIT2(v) sqlite3_api = v; +#ifndef SQLITE_CORE + /* This case when the file really is being compiled as a loadable + ** extension */ +# define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; +# define SQLITE_EXTENSION_INIT2(v) sqlite3_api=v; +#else + /* This case when the file is being statically linked into the + ** application */ +# define SQLITE_EXTENSION_INIT1 /*no-op*/ +# define SQLITE_EXTENSION_INIT2(v) (void)v; /* unused parameter */ +#endif #endif /* _SQLITE3EXT_H_ */ diff --git a/src/test_autoext.c b/src/test_autoext.c index 6b1e297ab7..b5013f3173 100644 --- a/src/test_autoext.c +++ b/src/test_autoext.c @@ -15,7 +15,7 @@ #include "sqlite3ext.h" #ifndef SQLITE_OMIT_LOAD_EXTENSION -static SQLITE_EXTENSION_INIT1 +SQLITE_EXTENSION_INIT1 /* ** The sqr() SQL function returns the square of its input value. From cca17c303baade3ce4e823ecfcd89f9f8051e051 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 19 Apr 2013 12:32:52 +0000 Subject: [PATCH 134/140] Add the "warning" command to the TCL test infrastructure. Fix problems with the loadext.test module. FossilOrigin-Name: 514adbbd8cf3e296f55e8f803bddaac8ad8b2c96 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/loadext.test | 16 +++++++++------- test/tester.tcl | 16 ++++++++++++++++ 4 files changed, 33 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 1dd6c2db3b..1110498763 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sextension\sloading\smechanism\sso\sthat\sthe\ssame\ssource\scode\ncan\sbe\sused\sfor\srun-time\sloadable\sextensions\sand\sstatically\slinked\nextensions,\sand\sto\smask\sarchitectural\sdifferences\sin\sshared\slibraries. -D 2013-04-19T02:47:05.486 +C Add\sthe\s"warning"\scommand\sto\sthe\sTCL\stest\sinfrastructure.\s\sFix\sproblems\nwith\sthe\sloadext.test\smodule. +D 2013-04-19T12:32:52.666 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -585,7 +585,7 @@ F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/like.test 0e5412f4dac4a849f613e1ef8b529d56a6e31d08 F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da F test/limit.test cc0ab63385239b63c72452b0e93700bf5e8f0b99 -F test/loadext.test 2b5e249c51c986a5aff1f0950cf7ba30976c8f22 +F test/loadext.test 92e6dfefd1229c3ef4aaabd87419efd8fa57a7a5 F test/loadext2.test 0bcaeb4d81cd5b6e883fdfea3c1bdbe1f173cbca F test/lock.test 87af515b0c4cf928576d0f89946d67d7c265dfb4 F test/lock2.test 5242d8ac4e2d59c403aebff606af449b455aceff @@ -765,7 +765,7 @@ F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptrigger.test 26670ed7a39cf2296a7f0a9e0a1d7bdb7abe936d -F test/tester.tcl 0d95028653ad6782e17400b7a32a046b87c67a48 +F test/tester.tcl 3b08771e6d601612fe62d13787db0e50aac4cf7b F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e94783f36ade43ef9d5b94db278622bcbce4179b 86eb3eed4c70730dba0e319f243f8cb4fa8de06c -R 6c3fdd8d85d98822b9ca8decd52bbb5c +P a85b6ecefa14fcfe6ebfceabf910f00c3721be7d +R 2c3aca07297ff95a6b74db849941a18d U drh -Z 0d2df06bcea7e6410ce186a367eefd17 +Z bfc695f8f3325c7943f46501f9086d37 diff --git a/manifest.uuid b/manifest.uuid index 02b97e09e3..d269cbb390 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a85b6ecefa14fcfe6ebfceabf910f00c3721be7d \ No newline at end of file +514adbbd8cf3e296f55e8f803bddaac8ad8b2c96 \ No newline at end of file diff --git a/test/loadext.test b/test/loadext.test index 72eff12f7a..0d5b841980 100644 --- a/test/loadext.test +++ b/test/loadext.test @@ -139,20 +139,22 @@ do_test loadext-2.1 { sqlite3_load_extension db "${testextension}xx" } msg] list $rc $msg -} [list 1 [format $dlerror_nosuchfile ${testextension}xx]] +} /[list 1 [format $dlerror_nosuchfile ${testextension}xx.*]]/ # Try to load an extension for which the file is not a shared object # do_test loadext-2.2 { - set fd [open "${testextension}xx" w] + set fd [open "./notasharedlib.so" w] + puts $fd blah + close $fd + set fd [open "./notasharedlib.dll" w] puts $fd blah close $fd set rc [catch { - sqlite3_load_extension db "${testextension}xx" + sqlite3_load_extension db "./notasharedlib" } msg] - set expected_error_pattern [format $dlerror_notadll ${testextension}xx] - list $rc [string match $expected_error_pattern $msg] -} [list 1 1] + list $rc $msg +} /[list 1 [format $dlerror_notadll ./notasharedlib.*]]/ # Try to load an extension for which the file is present but the # entry point is not. @@ -196,7 +198,7 @@ do_test loadext-3.2 { regsub {0x[1234567890abcdefABCDEF]*} $res XXX res } set res -} [list 1 [format $dlerror_nosymbol $testextension sqlite3_extension_init]] +} /[list 1 [format $dlerror_nosymbol $testextension sqlite3_.*_init]]/ do_test loadext-3.3 { catchsql { SELECT load_extension($::testextension,'testloadext_init') diff --git a/test/tester.tcl b/test/tester.tcl index d6468f7b14..761a36e355 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -462,6 +462,7 @@ if {0==[info exists ::SLAVE]} { set TC(count) 0 set TC(fail_list) [list] set TC(omit_list) [list] + set TC(warn_list) [list] proc set_test_counter {counter args} { if {[llength $args]} { @@ -496,6 +497,18 @@ proc fail_test {name} { } } +# Remember a warning message to be displayed at the conclusion of all testing +# +proc warning {msg {append 1}} { + puts "Warning: $msg" + set warnList [set_test_counter warn_list] + if {$append} { + lappend warnList $msg + } + set_test_counter warn_list $warnList +} + + # Increment the number of tests run # proc incr_ntest {} { @@ -784,6 +797,9 @@ proc finalize_testing {} { if {$nErr>0} { puts "Failures on these tests: [set_test_counter fail_list]" } + foreach warning [set_test_counter warn_list] { + puts "Warning: $warning" + } run_thread_tests 1 if {[llength $omitList]>0} { puts "Omitted test cases:" From 0c41d22920741a7904ac89de203c463c0b12ec7f Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 22 Apr 2013 02:39:10 +0000 Subject: [PATCH 135/140] Use transitivity to move constraints into the outer loops of a join whenever possible, thereby reducing the amount of work that needs to occur in inner loops. FossilOrigin-Name: 5f4907e1c6230e3dd904bd99e1c48c576c669f63 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 36 ++++++++++++++++++++++++++++++++---- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 1110498763..9cf2da3aad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"warning"\scommand\sto\sthe\sTCL\stest\sinfrastructure.\s\sFix\sproblems\nwith\sthe\sloadext.test\smodule. -D 2013-04-19T12:32:52.666 +C Use\stransitivity\sto\smove\sconstraints\sinto\sthe\souter\sloops\sof\sa\sjoin\swhenever\npossible,\sthereby\sreducing\sthe\samount\sof\swork\sthat\sneeds\sto\soccur\sin\ninner\sloops. +D 2013-04-22T02:39:10.341 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -258,7 +258,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c e63f84e5583133c77573b78d3696dccd97bfd00d +F src/where.c c7c74fe8e354b60dee0810ea8b954c5ccb9e0f3c F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a85b6ecefa14fcfe6ebfceabf910f00c3721be7d -R 2c3aca07297ff95a6b74db849941a18d +P 514adbbd8cf3e296f55e8f803bddaac8ad8b2c96 +R a81fa97db156009318d49bf4c6dae35e U drh -Z bfc695f8f3325c7943f46501f9086d37 +Z 1c3c5b72018db605301d161eada3f156 diff --git a/manifest.uuid b/manifest.uuid index d269cbb390..bd39d46376 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -514adbbd8cf3e296f55e8f803bddaac8ad8b2c96 \ No newline at end of file +5f4907e1c6230e3dd904bd99e1c48c576c669f63 \ No newline at end of file diff --git a/src/where.c b/src/where.c index a0dc868422..24b059e2d5 100644 --- a/src/where.c +++ b/src/where.c @@ -4161,6 +4161,7 @@ static Bitmask codeOneLoopStart( int addrCont; /* Jump here to continue with next cycle */ int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ int iReleaseReg = 0; /* Temp register to free before returning */ + Bitmask newNotReady; /* Return value */ pParse = pWInfo->pParse; v = pParse->pVdbe; @@ -4171,6 +4172,7 @@ static Bitmask codeOneLoopStart( bRev = (pLevel->plan.wsFlags & WHERE_REVERSE)!=0; omitTable = (pLevel->plan.wsFlags & WHERE_IDX_ONLY)!=0 && (wctrlFlags & WHERE_FORCE_TABLE)==0; + VdbeNoopComment((v, "Begin Join Loop %d", iLevel)); /* Create labels for the "break" and "continue" instructions ** for the current loop. Jump to addrBrk to break out of a loop. @@ -4821,7 +4823,7 @@ static Bitmask codeOneLoopStart( pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; } - notReady &= ~getMask(pWC->pMaskSet, iCur); + newNotReady = notReady & ~getMask(pWC->pMaskSet, iCur); /* Insert code to test every subexpression that can be completely ** computed using the current set of tables. @@ -4835,7 +4837,7 @@ static Bitmask codeOneLoopStart( testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & notReady)!=0 ){ + if( (pTerm->prereqAll & newNotReady)!=0 ){ testcase( pWInfo->untestedTerms==0 && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ); pWInfo->untestedTerms = 1; @@ -4850,6 +4852,32 @@ static Bitmask codeOneLoopStart( pTerm->wtFlags |= TERM_CODED; } + /* Insert code to test for implied constraints based on transitivity + ** of the "==" operator. + ** + ** Example: If the WHERE clause contains "t1.a=t2.b" and "t2.b=123" + ** and we are coding the t1 loop and the t2 loop has not yet coded, + ** then we cannot use the "t1.a=t2.b" constraint, but we can code + ** the implied "t1.a=123" constraint. + */ + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE; + WhereTerm *pAlt; + Expr sEq; + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( pTerm->eOperator!=(WO_EQUIV|WO_EQ) ) continue; + if( pTerm->leftCursor!=iCur ) continue; + pE = pTerm->pExpr; + assert( !ExprHasProperty(pE, EP_FromJoin) ); + assert( (pTerm->prereqRight & newNotReady)!=0 ); + pAlt = findTerm(pWC, iCur, pTerm->u.leftColumn, notReady, WO_EQ|WO_IN, 0); + if( pAlt==0 ) continue; + VdbeNoopComment((v, "begin transitive constraint")); + sEq = *pAlt->pExpr; + sEq.pLeft = pE->pLeft; + sqlite3ExprIfFalse(pParse, &sEq, addrCont, SQLITE_JUMPIFNULL); + } + /* For a LEFT OUTER JOIN, generate code that will record the fact that ** at least one row of the right table has matched the left table. */ @@ -4862,7 +4890,7 @@ static Bitmask codeOneLoopStart( testcase( pTerm->wtFlags & TERM_VIRTUAL ); /* IMP: R-30575-11662 */ testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & notReady)!=0 ){ + if( (pTerm->prereqAll & newNotReady)!=0 ){ assert( pWInfo->untestedTerms ); continue; } @@ -4873,7 +4901,7 @@ static Bitmask codeOneLoopStart( } sqlite3ReleaseTempReg(pParse, iReleaseReg); - return notReady; + return newNotReady; } #if defined(SQLITE_TEST) From b9830a14543c823f6690a70b5b43d8bed409edd9 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 22 Apr 2013 13:51:09 +0000 Subject: [PATCH 136/140] Enhanced documentation for the SQLITE_CONFIG_SQLLOG mechanism and the test_sqllog.c demonstration file. No changes to code. FossilOrigin-Name: 206caf21f6f6963aa38a9aa0f6ea11c7230bb726 --- manifest | 14 +++++------ manifest.uuid | 2 +- src/sqlite.h.in | 8 ++++--- src/test_sqllog.c | 59 +++++++++++++++++++++++++++++++++++++---------- 4 files changed, 60 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 9cf2da3aad..68a8c40938 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\stransitivity\sto\smove\sconstraints\sinto\sthe\souter\sloops\sof\sa\sjoin\swhenever\npossible,\sthereby\sreducing\sthe\samount\sof\swork\sthat\sneeds\sto\soccur\sin\ninner\sloops. -D 2013-04-22T02:39:10.341 +C Enhanced\sdocumentation\sfor\sthe\sSQLITE_CONFIG_SQLLOG\smechanism\sand\sthe\ntest_sqllog.c\sdemonstration\sfile.\s\sNo\schanges\sto\scode. +D 2013-04-22T13:51:09.759 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -182,7 +182,7 @@ F src/resolve.c 10a1b332e3eb36e5d561085e18c58a8578cd7d73 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 01540bcd3df3c8f1187158e77986028b1c667258 F src/shell.c aca9d94653decd4496846dee0c7ba83eaf96a46d -F src/sqlite.h.in a291a999d28ebc16cd1fe7fdd010dfe5266ef767 +F src/sqlite.h.in ec279b782bea05db63b8b29481f9642b406004af F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h d936f797812c28b81b26ed18345baf8db28a21a5 F src/sqliteInt.h 2a83cfec9963372b636b0cabd4b200c1f1074a99 @@ -229,7 +229,7 @@ F src/test_rtree.c aba603c949766c4193f1068b91c787f57274e0d9 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f F src/test_spellfix.c 56dfa6d583ac34f61af0834d7b58d674e7e18e13 -F src/test_sqllog.c bc50e5afeb7fb50e77b4594e42302df9d05446aa +F src/test_sqllog.c c1c1bbedbcaf82b93d83e4f9dd990e62476a680e F src/test_stat.c d1569c7a4839f13e80187e2c26b2ab4da2d03935 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd F src/test_syscall.c 7e8293e4e6971b0f44c7f7f37b1315a8cc9f6018 @@ -1051,7 +1051,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 514adbbd8cf3e296f55e8f803bddaac8ad8b2c96 -R a81fa97db156009318d49bf4c6dae35e +P 5f4907e1c6230e3dd904bd99e1c48c576c669f63 +R 3f851c92fc64d9ebdb6dc226b2109b9c U drh -Z 1c3c5b72018db605301d161eada3f156 +Z 928e878ceb923ac2280cddfe9d967aac diff --git a/manifest.uuid b/manifest.uuid index bd39d46376..77c2db74de 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5f4907e1c6230e3dd904bd99e1c48c576c669f63 \ No newline at end of file +206caf21f6f6963aa38a9aa0f6ea11c7230bb726 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 3c6291eebe..3265b3196b 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1633,12 +1633,12 @@ struct sqlite3_mem_methods { **
    SQLITE_CONFIG_PCACHE and SQLITE_CONFIG_GETPCACHE **
    These options are obsolete and should not be used by new code. ** They are retained for backwards compatibility but are now no-ops. -** +**
    ** ** [[SQLITE_CONFIG_SQLLOG]] **
    SQLITE_CONFIG_SQLLOG **
    This option is only available if sqlite is compiled with the -** SQLITE_ENABLE_SQLLOG pre-processor macro defined. The first argument should +** [SQLITE_ENABLE_SQLLOG] pre-processor macro defined. The first argument should ** be a pointer to a function of type void(*)(void*,sqlite3*,const char*, int). ** The second should be of type (void*). The callback is invoked by the library ** in three separate circumstances, identified by the value passed as the @@ -1648,7 +1648,9 @@ struct sqlite3_mem_methods { ** fourth parameter is 1, then the SQL statement that the third parameter ** points to has just been executed. Or, if the fourth parameter is 2, then ** the connection being passed as the second parameter is being closed. The -** third parameter is passed NULL In this case. +** third parameter is passed NULL In this case. An example of using this +** configuration option can be seen in the "test_sqllog.c" source file in +** the canonical SQLite source tree.
    ** ** [[SQLITE_CONFIG_MMAP_SIZE]] **
    SQLITE_CONFIG_MMAP_SIZE diff --git a/src/test_sqllog.c b/src/test_sqllog.c index 24c675b5e9..4aa68b7c42 100644 --- a/src/test_sqllog.c +++ b/src/test_sqllog.c @@ -13,22 +13,32 @@ ** OVERVIEW ** ** This file contains experimental code used to record data from live -** SQLite applications that may be useful for offline analysis. Specifically: +** SQLite applications that may be useful for offline analysis. +** Specifically, this module can be used to capture the following +** information: ** ** 1) The initial contents of all database files opened by the ** application, and ** ** 2) All SQL statements executed by the application. ** +** The captured information can then be used to run (for example) +** performance analysis looking for slow queries or to look for +** optimization opportunities in either the application or in SQLite +** itself. +** ** USAGE ** ** To use this module, SQLite must be compiled with the SQLITE_ENABLE_SQLLOG -** pre-processor symbol defined and this file linked into the application -** somehow. +** pre-processor symbol defined and this file linked into the application. +** One way to link this file into the application is to append the content +** of this file onto the end of the "sqlite3.c" amalgamation and then +** recompile the application as normal except with the addition of the +** -DSQLITE_ENABLE_SQLLOG option. ** ** At runtime, logging is enabled by setting environment variable ** SQLITE_SQLLOG_DIR to the name of a directory in which to store logged -** data. The directory must already exist. +** data. The logging directory must already exist. ** ** Usually, if the application opens the same database file more than once ** (either by attaching it or by using more than one database handle), only @@ -57,14 +67,16 @@ ** logging proceeds as expected. Errors are logged by calling sqlite3_log(). */ +#ifndef _SQLITE3_H_ #include "sqlite3.h" -#include "stdio.h" -#include "stdlib.h" -#include "string.h" -#include "assert.h" +#endif +#include +#include +#include +#include -#include "sys/types.h" -#include "unistd.h" +#include +#include static int getProcessId(void){ #if SQLITE_OS_WIN return (int)_getpid(); @@ -73,7 +85,7 @@ static int getProcessId(void){ #endif } - +/* Names of environment variables to be used */ #define ENVIRONMENT_VARIABLE1_NAME "SQLITE_SQLLOG_DIR" #define ENVIRONMENT_VARIABLE2_NAME "SQLITE_SQLLOG_REUSE_FILES" @@ -86,6 +98,9 @@ static int getProcessId(void){ */ #define MAX_CONNECTIONS 256 +/* There is one instance of this object for each SQLite database connection +** that is being logged. +*/ struct SLConn { int isErr; /* True if an error has occurred */ sqlite3 *db; /* Connection handle */ @@ -93,7 +108,9 @@ struct SLConn { FILE *fd; /* File descriptor for log file */ }; -struct SLGlobal { +/* This object is a singleton that keeps track of all data loggers. +*/ +static struct SLGlobal { /* Protected by MUTEX_STATIC_MASTER */ sqlite3_mutex *mutex; /* Recursive mutex */ int nConn; /* Size of aConn[] array */ @@ -388,6 +405,24 @@ static void testSqllogStmt(struct SLConn *p, const char *zSql){ /* ** The SQLITE_CONFIG_SQLLOG callback registered by sqlite3_init_sqllog(). +** +** The eType parameter has the following values: +** +** 0: Opening a new database connection. zSql is the name of the +** file being opened. db is a pointer to the newly created database +** connection. +** +** 1: An SQL statement has run to completion. zSql is the text of the +** SQL statement with all parameters expanded to their actual values. +** +** 2: Closing a database connection. zSql is NULL. The db pointer to +** the database connection being closed has already been shut down +** and cannot be used for any further SQL. +** +** The pCtx parameter is a copy of the pointer that was originally passed +** into the sqlite3_config(SQLITE_CONFIG_SQLLOG) statement. In this +** particular implementation, pCtx is always a pointer to the +** sqllogglobal global variable define above. */ static void testSqllog(void *pCtx, sqlite3 *db, const char *zSql, int eType){ struct SLConn *p = 0; From d7a959c69453e8b12976b2189c99b839751b5020 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 22 Apr 2013 15:30:37 +0000 Subject: [PATCH 137/140] Add virtual table module 'fts3tokenize' to fts3. fts3tokenize provides SQL access to FTS tokenizer implementations. FossilOrigin-Name: c5a4b21a15bd8a184380817ff41657acbccb866d --- Makefile.in | 5 + ext/fts3/fts3.c | 3 + ext/fts3/fts3_tokenize_vtab.c | 459 ++++++++++++++++++++++++++++++++++ main.mk | 5 + manifest | 23 +- manifest.uuid | 2 +- test/fts3tok1.test | 125 +++++++++ test/fts3tok_err.test | 49 ++++ tool/mksqlite3c.tcl | 1 + 9 files changed, 661 insertions(+), 11 deletions(-) create mode 100644 ext/fts3/fts3_tokenize_vtab.c create mode 100644 test/fts3tok1.test create mode 100644 test/fts3tok_err.test diff --git a/Makefile.in b/Makefile.in index 486333157e..5c636209d8 100644 --- a/Makefile.in +++ b/Makefile.in @@ -167,6 +167,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ expr.lo fault.lo fkey.lo \ fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \ fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \ + fts3_tokenize_vtab.lo \ fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \ func.lo global.lo hash.lo \ icu.lo insert.lo journal.lo legacy.lo loadext.lo \ @@ -316,6 +317,7 @@ SRC += \ $(TOP)/ext/fts3/fts3_tokenizer.h \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_tokenizer1.c \ + $(TOP)/ext/fts3/fts3_tokenize_vtab.c \ $(TOP)/ext/fts3/fts3_unicode.c \ $(TOP)/ext/fts3/fts3_unicode2.c \ $(TOP)/ext/fts3/fts3_write.c @@ -858,6 +860,9 @@ fts3_tokenizer.lo: $(TOP)/ext/fts3/fts3_tokenizer.c $(HDR) $(EXTHDR) fts3_tokenizer1.lo: $(TOP)/ext/fts3/fts3_tokenizer1.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer1.c +fts3_tokenizer_vtab.lo: $(TOP)/ext/fts3/fts3_tokenizer_vtab.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer_vtab.c + fts3_unicode.lo: $(TOP)/ext/fts3/fts3_unicode.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_unicode.c diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 6b3a7b1565..0c7e07740d 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3593,6 +3593,9 @@ int sqlite3Fts3Init(sqlite3 *db){ rc = sqlite3Fts3InitAux(db); if( rc!=SQLITE_OK ) return rc; + rc = sqlite3Fts3InitTok(db); + if( rc!=SQLITE_OK ) return rc; + sqlite3Fts3SimpleTokenizerModule(&pSimple); sqlite3Fts3PorterTokenizerModule(&pPorter); diff --git a/ext/fts3/fts3_tokenize_vtab.c b/ext/fts3/fts3_tokenize_vtab.c new file mode 100644 index 0000000000..f00fefe7a9 --- /dev/null +++ b/ext/fts3/fts3_tokenize_vtab.c @@ -0,0 +1,459 @@ +/* +** 2013 Apr 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code for the "fts3tokenize" virtual table module. +** An fts3tokenize virtual table is created as follows: +** +** CREATE VIRTUAL TABLE USING fts3tokenize( +** , , ... +** ); +** +** The table created has the following schema: +** +** CREATE TABLE (input, token, start, end, position) +** +** When queried, the query must include a WHERE clause of type: +** +** input = +** +** The virtual table module tokenizes this , using the FTS3 +** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE +** statement and returns one row for each token in the result. With +** fields set as follows: +** +** input: Always set to a copy of +** token: A token from the input. +** start: Byte offset of the token within the input . +** end: Byte offset of the byte immediately following the end of the +** token within the input string. +** pos: Token offset of token within input. +** +*/ +#include "fts3Int.h" +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +#include +#include + +typedef struct Fts3tokTable Fts3tokTable; +typedef struct Fts3tokCursor Fts3tokCursor; + +/* +** Virtual table structure. +*/ +struct Fts3tokTable { + sqlite3_vtab base; /* Base class used by SQLite core */ + const sqlite3_tokenizer_module *pMod; + sqlite3_tokenizer *pTok; +}; + +/* +** Virtual table cursor structure. +*/ +struct Fts3tokCursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + char *zInput; /* Input string */ + sqlite3_tokenizer_cursor *pCsr; /* Cursor to iterate through zInput */ + int iRowid; /* Current 'rowid' value */ + const char *zToken; /* Current 'token' value */ + int nToken; /* Size of zToken in bytes */ + int iStart; /* Current 'start' value */ + int iEnd; /* Current 'end' value */ + int iPos; /* Current 'pos' value */ +}; + +/* +** Query FTS for the tokenizer implementation named zName. +*/ +static int fts3tokQueryTokenizer( + sqlite3 *db, + const char *zName, + const sqlite3_tokenizer_module **pp +){ + int rc; + sqlite3_stmt *pStmt; + const char *zSql = "SELECT fts3_tokenizer(?)"; + + *pp = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + return rc; + } + + sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ + memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); + } + } + + return sqlite3_finalize(pStmt); +} + +/* +** The second argument, argv[], is an array of pointers to nul-terminated +** strings. This function makes a copy of the array and strings into a +** single block of memory. It then dequotes any of the strings that appear +** to be quoted. +** +** If successful, output parameter *pazDequote is set to point at the +** array of dequoted strings and SQLITE_OK is returned. The caller is +** responsible for eventually calling sqlite3_free() to free the array +** in this case. Or, if an error occurs, an SQLite error code is returned. +** The final value of *pazDequote is undefined in this case. +*/ +static int fts3tokDequoteArray( + int argc, /* Number of elements in argv[] */ + const char * const *argv, /* Input array */ + char ***pazDequote /* Output array */ +){ + int rc = SQLITE_OK; /* Return code */ + if( argc==0 ){ + *pazDequote = 0; + }else{ + int i; + int nByte; + char **azDequote; + + for(i=0; ixCreate((nDequote>1 ? nDequote-1 : 0), azArg, &pTok); + } + + if( rc==SQLITE_OK ){ + pTab = (Fts3tokTable *)sqlite3_malloc(sizeof(Fts3tokTable)); + if( pTab==0 ){ + rc = SQLITE_NOMEM; + } + } + + if( rc==SQLITE_OK ){ + memset(pTab, 0, sizeof(Fts3tokTable)); + pTab->pMod = pMod; + pTab->pTok = pTok; + *ppVtab = &pTab->base; + }else{ + if( pTok ){ + pMod->xDestroy(pTok); + } + } + + sqlite3_free(azDequote); + return rc; +} + +/* +** This function does the work for both the xDisconnect and xDestroy methods. +** These tables have no persistent representation of their own, so xDisconnect +** and xDestroy are identical operations. +*/ +static int fts3tokDisconnectMethod(sqlite3_vtab *pVtab){ + Fts3tokTable *pTab = (Fts3tokTable *)pVtab; + + pTab->pMod->xDestroy(pTab->pTok); + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** xBestIndex - Analyze a WHERE and ORDER BY clause. +*/ +static int fts3tokBestIndexMethod( + sqlite3_vtab *pVTab, + sqlite3_index_info *pInfo +){ + int i; + + for(i=0; inConstraint; i++){ + if( pInfo->aConstraint[i].usable + && pInfo->aConstraint[i].iColumn==0 + && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + pInfo->idxNum = 1; + pInfo->aConstraintUsage[i].argvIndex = 1; + pInfo->aConstraintUsage[i].omit = 1; + pInfo->estimatedCost = 1; + return SQLITE_OK; + } + } + + pInfo->idxNum = 0; + assert( pInfo->estimatedCost>1000000.0 ); + + return SQLITE_OK; +} + +/* +** xOpen - Open a cursor. +*/ +static int fts3tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ + Fts3tokCursor *pCsr; + + pCsr = (Fts3tokCursor *)sqlite3_malloc(sizeof(Fts3tokCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + } + memset(pCsr, 0, sizeof(Fts3tokCursor)); + + *ppCsr = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Reset the tokenizer cursor passed as the only argument. As if it had +** just been returned by fts3tokOpenMethod(). +*/ +static void fts3tokResetCursor(Fts3tokCursor *pCsr){ + if( pCsr->pCsr ){ + Fts3tokTable *pTab = (Fts3tokTable *)(pCsr->base.pVtab); + pTab->pMod->xClose(pCsr->pCsr); + pCsr->pCsr = 0; + } + sqlite3_free(pCsr->zInput); + pCsr->zInput = 0; + pCsr->zToken = 0; + pCsr->nToken = 0; + pCsr->iStart = 0; + pCsr->iEnd = 0; + pCsr->iPos = 0; + pCsr->iRowid = 0; +} + +/* +** xClose - Close a cursor. +*/ +static int fts3tokCloseMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + + fts3tokResetCursor(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** xNext - Advance the cursor to the next row, if any. +*/ +static int fts3tokNextMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + int rc; /* Return code */ + + pCsr->iRowid++; + rc = pTab->pMod->xNext(pCsr->pCsr, + &pCsr->zToken, &pCsr->nToken, + &pCsr->iStart, &pCsr->iEnd, &pCsr->iPos + ); + + if( rc!=SQLITE_OK ){ + fts3tokResetCursor(pCsr); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + } + + return rc; +} + +/* +** xFilter - Initialize a cursor to point at the start of its data. +*/ +static int fts3tokFilterMethod( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, /* Strategy index */ + const char *idxStr, /* Unused */ + int nVal, /* Number of elements in apVal */ + sqlite3_value **apVal /* Arguments for the indexing scheme */ +){ + int rc = SQLITE_ERROR; + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + + fts3tokResetCursor(pCsr); + if( idxNum==1 ){ + const char *zByte = (const char *)sqlite3_value_text(apVal[0]); + int nByte = sqlite3_value_bytes(apVal[0]); + pCsr->zInput = sqlite3_malloc(nByte+1); + if( pCsr->zInput==0 ){ + rc = SQLITE_NOMEM; + }else{ + memcpy(pCsr->zInput, zByte, nByte); + pCsr->zInput[nByte] = 0; + rc = pTab->pMod->xOpen(pTab->pTok, pCsr->zInput, nByte, &pCsr->pCsr); + if( rc==SQLITE_OK ){ + pCsr->pCsr->pTokenizer = pTab->pTok; + } + } + } + + if( rc!=SQLITE_OK ) return rc; + return fts3tokNextMethod(pCursor); +} + +/* +** xEof - Return true if the cursor is at EOF, or false otherwise. +*/ +static int fts3tokEofMethod(sqlite3_vtab_cursor *pCursor){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + return (pCsr->zToken==0); +} + +/* +** xColumn - Return a column value. +*/ +static int fts3tokColumnMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ + int iCol /* Index of column to read value from */ +){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + + /* CREATE TABLE x(input, token, start, end, position) */ + switch( iCol ){ + case 0: + sqlite3_result_text(pCtx, pCsr->zInput, -1, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_text(pCtx, pCsr->zToken, pCsr->nToken, SQLITE_TRANSIENT); + break; + case 2: + sqlite3_result_int(pCtx, pCsr->iStart); + break; + case 3: + sqlite3_result_int(pCtx, pCsr->iEnd); + break; + default: + assert( iCol==4 ); + sqlite3_result_int(pCtx, pCsr->iPos); + break; + } + return SQLITE_OK; +} + +/* +** xRowid - Return the current rowid for the cursor. +*/ +static int fts3tokRowidMethod( + sqlite3_vtab_cursor *pCursor, /* Cursor to retrieve value from */ + sqlite_int64 *pRowid /* OUT: Rowid value */ +){ + Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; + *pRowid = (sqlite3_int64)pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Register the fts3tok module with database connection db. Return SQLITE_OK +** if successful or an error code if sqlite3_create_module() fails. +*/ +int sqlite3Fts3InitTok(sqlite3 *db){ + static const sqlite3_module fts3tok_module = { + 0, /* iVersion */ + fts3tokConnectMethod, /* xCreate */ + fts3tokConnectMethod, /* xConnect */ + fts3tokBestIndexMethod, /* xBestIndex */ + fts3tokDisconnectMethod, /* xDisconnect */ + fts3tokDisconnectMethod, /* xDestroy */ + fts3tokOpenMethod, /* xOpen */ + fts3tokCloseMethod, /* xClose */ + fts3tokFilterMethod, /* xFilter */ + fts3tokNextMethod, /* xNext */ + fts3tokEofMethod, /* xEof */ + fts3tokColumnMethod, /* xColumn */ + fts3tokRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ + }; + int rc; /* Return code */ + + rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, 0); + return rc; +} + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ diff --git a/main.mk b/main.mk index 2ec009a11d..36a4da0294 100644 --- a/main.mk +++ b/main.mk @@ -55,6 +55,7 @@ LIBOBJ+= alter.o analyze.o attach.o auth.o \ callback.o complete.o ctime.o date.o delete.o expr.o fault.o fkey.o \ fts3.o fts3_aux.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \ fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o \ + fts3_tokenize_vtab.o \ fts3_unicode.o fts3_unicode2.o \ fts3_write.o func.o global.o hash.o \ icu.o insert.o journal.o legacy.o loadext.o \ @@ -197,6 +198,7 @@ SRC += \ $(TOP)/ext/fts3/fts3_tokenizer.h \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_tokenizer1.c \ + $(TOP)/ext/fts3/fts3_tokenize_vtab.c \ $(TOP)/ext/fts3/fts3_unicode.c \ $(TOP)/ext/fts3/fts3_unicode2.c \ $(TOP)/ext/fts3/fts3_write.c @@ -516,6 +518,9 @@ fts3_tokenizer.o: $(TOP)/ext/fts3/fts3_tokenizer.c $(HDR) $(EXTHDR) fts3_tokenizer1.o: $(TOP)/ext/fts3/fts3_tokenizer1.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer1.c +fts3_tokenize_vtab.o: $(TOP)/ext/fts3/fts3_tokenize_vtab.c $(HDR) $(EXTHDR) + $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenize_vtab.c + fts3_unicode.o: $(TOP)/ext/fts3/fts3_unicode.c $(HDR) $(EXTHDR) $(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_unicode.c diff --git a/manifest b/manifest index 68a8c40938..9d7bde3b76 100644 --- a/manifest +++ b/manifest @@ -1,7 +1,7 @@ -C Enhanced\sdocumentation\sfor\sthe\sSQLITE_CONFIG_SQLLOG\smechanism\sand\sthe\ntest_sqllog.c\sdemonstration\sfile.\s\sNo\schanges\sto\scode. -D 2013-04-22T13:51:09.759 +C Add\svirtual\stable\smodule\s'fts3tokenize'\sto\sfts3.\sfts3tokenize\sprovides\sSQL\saccess\sto\sFTS\stokenizer\simplementations. +D 2013-04-22T15:30:37.451 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 3dd3fcb87b70c78d99b2c8a03e44ec86d6ca9ce2 +F Makefile.in 87591ea5bf7d6ed521ad42d5bc69c124debe11a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc a244e5b5ba2493625db95b4d46b79201ea868bfe F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 @@ -55,7 +55,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 0eaedfc6d2eb22563ef1d044dcfed93b70ec79f2 +F ext/fts3/fts3.c 784aadfb4c2a217c3eb1feaecac924989f29728f F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 1e58825246b56259267382d2f9902774c049460a F ext/fts3/fts3_aux.c b02632f6dd0e375ce97870206d914ea6d8df5ccd @@ -67,6 +67,7 @@ F ext/fts3/fts3_porter.c a465b49fcb8249a755792f87516eff182efa42b3 F ext/fts3/fts3_snippet.c 5fcfcafff46a2a3a63b8e59fcb51987d01c74695 F ext/fts3/fts3_term.c a521f75132f9a495bdca1bdd45949b3191c52763 F ext/fts3/fts3_test.c f9a1a1702db1bfad3e2d0064746eeb808f125489 +F ext/fts3/fts3_tokenize_vtab.c 1019e24b7f12d234c902053ad8e70f43424d360a F ext/fts3/fts3_tokenizer.c bbdc731bc91338050675c6d1da9ab82147391e16 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 @@ -103,7 +104,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk dcb05737414488383369b93e33cf1a1484c4b6e6 +F main.mk e97fbfe4c69060560574e1b59c43f4f88a38a0ec F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -506,6 +507,8 @@ F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0 F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2 F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659 +F test/fts3tok1.test 05ff5c57098282bacba7192507702b17f4061295 +F test/fts3tok_err.test 41e5413139c2ef536ffadfcd1584ee50b1665ec0 F test/fts4aa.test 95f448fb02c4a976968b08d1b4ce134e720946ae F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8 F test/fts4content.test 6efc53b4fd03cab167e6998d2b0b7d4b7d419ee6 @@ -1022,7 +1025,7 @@ F tool/mkkeywordhash.c bb52064aa614e1426445e4b2b9b00eeecd23cc79 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 8bce31074e4cbe631bb7676526a048335f4c9f02 -F tool/mksqlite3c.tcl a570a0b0847f5659482087f70e409ac45ce93b91 +F tool/mksqlite3c.tcl a61fe62a2895ca6458c463fccf1211ca1c000fcf F tool/mksqlite3h.tcl ba24038056f51fde07c0079c41885ab85e2cff12 F tool/mksqlite3internalh.tcl 3dca7bb5374cee003379b8cbac73714f610ef795 F tool/mkvsix.tcl 0be7f7a591f1e83f9199cb82911b66668ca484c9 @@ -1051,7 +1054,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5f4907e1c6230e3dd904bd99e1c48c576c669f63 -R 3f851c92fc64d9ebdb6dc226b2109b9c -U drh -Z 928e878ceb923ac2280cddfe9d967aac +P 206caf21f6f6963aa38a9aa0f6ea11c7230bb726 +R 36a7a566fbf2ef5cbf69a5b7a4283f26 +U dan +Z 5ed9f520f4f2e158b055381052a84fbc diff --git a/manifest.uuid b/manifest.uuid index 77c2db74de..c65f3e21d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -206caf21f6f6963aa38a9aa0f6ea11c7230bb726 \ No newline at end of file +c5a4b21a15bd8a184380817ff41657acbccb866d \ No newline at end of file diff --git a/test/fts3tok1.test b/test/fts3tok1.test new file mode 100644 index 0000000000..89f799606b --- /dev/null +++ b/test/fts3tok1.test @@ -0,0 +1,125 @@ +# 2013 April 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the "fts3tokenize" virtual table +# that is part of the FTS3 module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +ifcapable !fts3 { finish_test ; return } +set ::testprefix fts3tok1 + +#------------------------------------------------------------------------- +# Simple test cases. Using the default (simple) tokenizer. +# +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE t1 USING fts3tokenize(simple); + CREATE VIRTUAL TABLE t2 USING fts3tokenize(); + CREATE VIRTUAL TABLE t3 USING fts3tokenize(simple, '', 'xyz '); +} + +foreach {tn tbl} {1 t1 2 t2 3 t3} { + do_execsql_test 1.$tn.1 "SELECT * FROM $tbl WHERE input = 'one two three'" { + {one two three} one 0 3 0 + {one two three} two 4 7 1 + {one two three} three 8 13 2 + } + + do_execsql_test 1.$tn.2 " + SELECT token FROM $tbl WHERE input = 'OnE tWo tHrEe' + " { + one two three + } +} + +do_execsql_test 1.4 { + SELECT token FROM t3 WHERE input = '1x2x3x' +} {1 2 3} + +do_execsql_test 1.5 { + SELECT token FROM t1 WHERE input = '1x2x3x' +} {1x2x3x} + +do_execsql_test 1.6 { + SELECT token FROM t3 WHERE input = '1''2x3x' +} {1'2 3} + +do_execsql_test 1.7 { + SELECT token FROM t3 WHERE input = '' +} {} + +do_execsql_test 1.8 { + SELECT token FROM t3 WHERE input = NULL +} {} + +do_execsql_test 1.9 { + SELECT * FROM t3 WHERE input = 123 +} {123 123 0 3 0} + +do_execsql_test 1.10 { + SELECT * FROM t1 WHERE input = 'a b c' AND token = 'b'; +} { + {a b c} b 2 3 1 +} + +do_execsql_test 1.11 { + SELECT * FROM t1 WHERE token = 'b' AND input = 'a b c'; +} { + {a b c} b 2 3 1 +} + +do_execsql_test 1.12 { + SELECT * FROM t1 WHERE input < 'b' AND input = 'a b c'; +} { + {a b c} a 0 1 0 + {a b c} b 2 3 1 + {a b c} c 4 5 2 +} + +do_execsql_test 1.13.1 { + CREATE TABLE c1(x); + INSERT INTO c1(x) VALUES('a b c'); + INSERT INTO c1(x) VALUES('d e f'); +} +breakpoint +do_execsql_test 1.13.2 { + SELECT * FROM c1, t1 WHERE input = x AND c1.rowid=t1.rowid; +} { + {a b c} {a b c} a 0 1 0 + {d e f} {d e f} e 2 3 1 +} + + +#------------------------------------------------------------------------- +# Error cases. +# +do_catchsql_test 2.0 { + CREATE VIRTUAL TABLE tX USING fts3tokenize(nosuchtokenizer); +} {1 {unknown tokenizer: nosuchtokenizer}} + +proc fts3_tokenizer {args} { return 1 } +db function fts3_tokenizer -argcount 1 fts3_tokenizer +do_catchsql_test 2.1 { + CREATE VIRTUAL TABLE tX USING fts3tokenize(simple); +} {1 {vtable constructor failed: tX}} + +db close +sqlite3 db test.db +do_catchsql_test 2.2 { + CREATE VIRTUAL TABLE t4 USING fts3tokenize; + SELECT * FROM t4; +} {1 {SQL logic error or missing database}} + + +finish_test + + diff --git a/test/fts3tok_err.test b/test/fts3tok_err.test new file mode 100644 index 0000000000..aaa7272b7b --- /dev/null +++ b/test/fts3tok_err.test @@ -0,0 +1,49 @@ +# 2013 April 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the "fts3tokenize" virtual table +# that is part of the FTS3 module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +ifcapable !fts3 { finish_test ; return } +set ::testprefix fts3tok_err + + +faultsim_save_and_close +do_faultsim_test fts3tok_err-1 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { CREATE VIRTUAL TABLE t1 USING fts3tokenize("simple"); } +} -test { + faultsim_test_result {0 {}} +} + +do_test fts3tok_err-2.prep { + faultsim_delete_and_reopen + execsql { CREATE VIRTUAL TABLE t1 USING fts3tokenize("simple"); } + faultsim_save_and_close +} {} + +do_faultsim_test fts3tok_err-2 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { SELECT token FROM t1 WHERE input = 'A galaxy far, far away' } +} -test { + faultsim_test_result {0 {a galaxy far far away}} +} + + +finish_test + + diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 7c8a7d54ad..c93bcea44d 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -311,6 +311,7 @@ foreach file { fts3_porter.c fts3_tokenizer.c fts3_tokenizer1.c + fts3_tokenize_vtab.c fts3_write.c fts3_snippet.c fts3_unicode.c From 4d648c777bc9ab75f4f25e31a4f206f469775189 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 22 Apr 2013 17:07:56 +0000 Subject: [PATCH 138/140] Update MSCV makefile for the fts3tokenize virtual table. Clear some harmless compiler warnings in the fts3tokenize code. FossilOrigin-Name: 2b3fed09eb89f1e9d6a5f7487ef526b46eb51490 --- Makefile.msc | 7 ++++++- ext/fts3/fts3Int.h | 3 +++ ext/fts3/fts3_tokenize_vtab.c | 6 +++--- manifest | 18 +++++++++--------- manifest.uuid | 2 +- 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 8c91e86bc8..47f815e677 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -478,7 +478,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ expr.lo fault.lo fkey.lo \ fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \ fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \ - fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \ + fts3_tokenize_vtab.lo fts3_unicode.lo fts3_unicode2.lo fts3_write.lo \ func.lo global.lo hash.lo \ icu.lo insert.lo journal.lo legacy.lo loadext.lo \ main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \ @@ -638,6 +638,7 @@ SRC = $(SRC) \ $(TOP)\ext\fts3\fts3_tokenizer.h \ $(TOP)\ext\fts3\fts3_tokenizer.c \ $(TOP)\ext\fts3\fts3_tokenizer1.c \ + $(TOP)\ext\fts3\fts3_tokenize_vtab.c \ $(TOP)\ext\fts3\fts3_unicode.c \ $(TOP)\ext\fts3\fts3_unicode2.c \ $(TOP)\ext\fts3\fts3_write.c @@ -748,6 +749,7 @@ TESTSRC2 = \ $(TOP)\ext\fts3\fts3_aux.c \ $(TOP)\ext\fts3\fts3_expr.c \ $(TOP)\ext\fts3\fts3_tokenizer.c \ + $(TOP)\ext\fts3\fts3_tokenize_vtab.c \ $(TOP)\ext\fts3\fts3_unicode.c \ $(TOP)\ext\fts3\fts3_unicode2.c \ $(TOP)\ext\fts3\fts3_write.c \ @@ -1176,6 +1178,9 @@ fts3_tokenizer.lo: $(TOP)\ext\fts3\fts3_tokenizer.c $(HDR) $(EXTHDR) fts3_tokenizer1.lo: $(TOP)\ext\fts3\fts3_tokenizer1.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_tokenizer1.c +fts3_tokenize_vtab.lo: $(TOP)\ext\fts3\fts3_tokenize_vtab.c $(HDR) $(EXTHDR) + $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_tokenize_vtab.c + fts3_unicode.lo: $(TOP)\ext\fts3\fts3_unicode.c $(HDR) $(EXTHDR) $(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3_unicode.c diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 77ca4704e8..a11c18a3b7 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -549,6 +549,9 @@ int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); +/* fts3_tokenize_vtab.c */ +int sqlite3Fts3InitTok(sqlite3*); + /* fts3_unicode2.c (functions generated by parsing unicode text files) */ #ifdef SQLITE_ENABLE_FTS4_UNICODE61 int sqlite3FtsUnicodeFold(int, int); diff --git a/ext/fts3/fts3_tokenize_vtab.c b/ext/fts3/fts3_tokenize_vtab.c index f00fefe7a9..0f2cef83c3 100644 --- a/ext/fts3/fts3_tokenize_vtab.c +++ b/ext/fts3/fts3_tokenize_vtab.c @@ -92,7 +92,7 @@ static int fts3tokQueryTokenizer( sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); if( SQLITE_ROW==sqlite3_step(pStmt) ){ if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ - memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); + memcpy((void*)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); } } @@ -125,7 +125,7 @@ static int fts3tokDequoteArray( char **azDequote; for(i=0; i Date: Mon, 22 Apr 2013 19:56:12 +0000 Subject: [PATCH 139/140] Do not allow a virtual table to cancel the ORDER BY clause unless all outer loops are guaranteed to return no more than one row result. Candidate fix for ticket [ba82a4a41eac1]. FossilOrigin-Name: 49cfa14fceeef2d55b449eb927c283ce6f650c07 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 11 +++++++++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index fe2a260a73..7c003c6acd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sMSCV\smakefile\sfor\sthe\sfts3tokenize\svirtual\stable.\s\sClear\ssome\nharmless\scompiler\swarnings\sin\sthe\sfts3tokenize\scode. -D 2013-04-22T17:07:56.175 +C Do\snot\sallow\sa\svirtual\stable\sto\scancel\sthe\sORDER\sBY\sclause\sunless\sall\nouter\sloops\sare\sguaranteed\sto\sreturn\sno\smore\sthan\sone\srow\sresult.\nCandidate\sfix\sfor\sticket\s[ba82a4a41eac1]. +D 2013-04-22T19:56:12.702 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 87591ea5bf7d6ed521ad42d5bc69c124debe11a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -259,7 +259,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b -F src/where.c c7c74fe8e354b60dee0810ea8b954c5ccb9e0f3c +F src/where.c d54e63087b52c309550aa2defdb20ef27add9f9a F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1054,7 +1054,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c5a4b21a15bd8a184380817ff41657acbccb866d -R 8cef88af587ec812976b521330718bb2 +P 2b3fed09eb89f1e9d6a5f7487ef526b46eb51490 +R c15aea6fcd9f9126da1de54ee794b8c9 U drh -Z 6d195d8624ede0ee30827cabebff2ea8 +Z 5c7507bd8ad3ff74a1cf8b492f137aec diff --git a/manifest.uuid b/manifest.uuid index 4f76ff333d..5a0d3a7ebc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2b3fed09eb89f1e9d6a5f7487ef526b46eb51490 \ No newline at end of file +49cfa14fceeef2d55b449eb927c283ce6f650c07 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 24b059e2d5..d70205205c 100644 --- a/src/where.c +++ b/src/where.c @@ -2400,6 +2400,17 @@ static void bestVirtualIndex(WhereBestIdx *p){ } if( i>=pIdxInfo->nConstraint ) break; } + + /* The orderByConsumed signal is only valid if all outer loops collectively + ** generate just a single row of output. + */ + if( pIdxInfo->orderByConsumed ){ + for(i=0; ii; i++){ + if( (p->aLevel[i].plan.wsFlags & WHERE_UNIQUE)==0 ){ + pIdxInfo->orderByConsumed = 0; + } + } + } /* If there is an ORDER BY clause, and the selected virtual table index ** does not satisfy it, increase the cost of the scan accordingly. This From da8caa0b2da85ebaf75fce1b19f5d645246f3eba Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 22 Apr 2013 23:38:50 +0000 Subject: [PATCH 140/140] Fix harmless compiler warnings. FossilOrigin-Name: 1a1cf5aa86734c832d845e07780262a178188d56 --- ext/fts3/fts3_tokenize_vtab.c | 7 ++++++- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/memjournal.c | 4 +++- src/os_unix.c | 1 + src/vdbetrace.c | 22 +++++++++++----------- 6 files changed, 31 insertions(+), 23 deletions(-) diff --git a/ext/fts3/fts3_tokenize_vtab.c b/ext/fts3/fts3_tokenize_vtab.c index 0f2cef83c3..9ec3b32aad 100644 --- a/ext/fts3/fts3_tokenize_vtab.c +++ b/ext/fts3/fts3_tokenize_vtab.c @@ -121,7 +121,7 @@ static int fts3tokDequoteArray( *pazDequote = 0; }else{ int i; - int nByte; + int nByte = 0; char **azDequote; for(i=0; inConstraint; i++){ if( pInfo->aConstraint[i].usable @@ -269,6 +271,7 @@ static int fts3tokBestIndexMethod( */ static int fts3tokOpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ Fts3tokCursor *pCsr; + UNUSED_PARAMETER(pVTab); pCsr = (Fts3tokCursor *)sqlite3_malloc(sizeof(Fts3tokCursor)); if( pCsr==0 ){ @@ -346,6 +349,8 @@ static int fts3tokFilterMethod( int rc = SQLITE_ERROR; Fts3tokCursor *pCsr = (Fts3tokCursor *)pCursor; Fts3tokTable *pTab = (Fts3tokTable *)(pCursor->pVtab); + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(nVal); fts3tokResetCursor(pCsr); if( idxNum==1 ){ diff --git a/manifest b/manifest index 7c003c6acd..f702eb82f2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sallow\sa\svirtual\stable\sto\scancel\sthe\sORDER\sBY\sclause\sunless\sall\nouter\sloops\sare\sguaranteed\sto\sreturn\sno\smore\sthan\sone\srow\sresult.\nCandidate\sfix\sfor\sticket\s[ba82a4a41eac1]. -D 2013-04-22T19:56:12.702 +C Fix\sharmless\scompiler\swarnings. +D 2013-04-22T23:38:50.007 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 87591ea5bf7d6ed521ad42d5bc69c124debe11a5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -67,7 +67,7 @@ F ext/fts3/fts3_porter.c a465b49fcb8249a755792f87516eff182efa42b3 F ext/fts3/fts3_snippet.c 5fcfcafff46a2a3a63b8e59fcb51987d01c74695 F ext/fts3/fts3_term.c a521f75132f9a495bdca1bdd45949b3191c52763 F ext/fts3/fts3_test.c f9a1a1702db1bfad3e2d0064746eeb808f125489 -F ext/fts3/fts3_tokenize_vtab.c e097c2e19126249b812a4002e76309d5fef3495b +F ext/fts3/fts3_tokenize_vtab.c a29f126b9e6c6a6f1021a8f7440bf125e68af1f9 F ext/fts3/fts3_tokenizer.c bbdc731bc91338050675c6d1da9ab82147391e16 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 @@ -157,7 +157,7 @@ F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa F src/mem2.c e307323e86b5da1853d7111b68fd6b84ad6f09cf F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 F src/mem5.c c2c63b7067570b00bf33d751c39af24182316f7f -F src/memjournal.c 0ebce851677a7ac035ba1512a7e65851b34530c6 +F src/memjournal.c 41a598445c8f249bd9b26ecae700c60dcf34cbf3 F src/mutex.c d3b66a569368015e0fcb1ac15f81c119f504d3bc F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 @@ -167,7 +167,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 0a561eae5965c9371300b0419027f5ae9f847af2 +F src/os_unix.c 5a9ac4a566fb566a2ff9b9e3a9d723075d9d26a7 F src/os_win.c 673b3e3d1fa3040d8d95a7f1f5e0e553aed56cfb F src/pager.c 6c3a8a5d665498b0344395a2c9f82d5abc4cc771 F src/pager.h 5cb78b8e1adfd5451e600be7719f5a99d87ac3b1 @@ -254,7 +254,7 @@ F src/vdbeaux.c ecb43014bcd3019e5aa2b5561e5c3a447f007a08 F src/vdbeblob.c 5dc79627775bd9a9b494dd956e26297946417d69 F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab F src/vdbesort.c 4fad64071ae120c25f39dcac572d716b9cadeb7f -F src/vdbetrace.c a22263ab47f6ba4fcd176515cec1e732866b25f0 +F src/vdbetrace.c 3ad1b4e92b60c082a02ac563da4a2735cc7d297c F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83 F src/wal.c 436bfceb141b9423c45119e68e444358ee0ed35d F src/wal.h a4d3da523d55a226a0b28e9058ef88d0a8051887 @@ -1054,7 +1054,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 2b3fed09eb89f1e9d6a5f7487ef526b46eb51490 -R c15aea6fcd9f9126da1de54ee794b8c9 +P 49cfa14fceeef2d55b449eb927c283ce6f650c07 +R 83c630a254d9cc209b10e1a83d85002b U drh -Z 5c7507bd8ad3ff74a1cf8b492f137aec +Z a7a8588b07b6fce1e57eced049fe41f1 diff --git a/manifest.uuid b/manifest.uuid index 5a0d3a7ebc..fa38fa5b2c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -49cfa14fceeef2d55b449eb927c283ce6f650c07 \ No newline at end of file +1a1cf5aa86734c832d845e07780262a178188d56 \ No newline at end of file diff --git a/src/memjournal.c b/src/memjournal.c index 3e66e215b2..05725948f6 100644 --- a/src/memjournal.c +++ b/src/memjournal.c @@ -230,7 +230,9 @@ static const struct sqlite3_io_methods MemJournalMethods = { 0, /* xShmMap */ 0, /* xShmLock */ 0, /* xShmBarrier */ - 0 /* xShmUnlock */ + 0, /* xShmUnmap */ + 0, /* xFetch */ + 0 /* xUnfetch */ }; /* diff --git a/src/os_unix.c b/src/os_unix.c index 8dfbe61814..7448afe4c7 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4782,6 +4782,7 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ */ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + UNUSED_PARAMETER(iOff); /* If p==0 (unmap the entire file) then there must be no outstanding ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), diff --git a/src/vdbetrace.c b/src/vdbetrace.c index 91554d6717..88935beeac 100644 --- a/src/vdbetrace.c +++ b/src/vdbetrace.c @@ -128,7 +128,7 @@ char *sqlite3VdbeExpandSql( }else if( pVar->flags & MEM_Real ){ sqlite3XPrintf(&out, "%!.15g", pVar->r); }else if( pVar->flags & MEM_Str ){ - int n; /* Number of bytes of the string text to include in output */ + int nOut; /* Number of bytes of the string text to include in output */ #ifndef SQLITE_OMIT_UTF16 u8 enc = ENC(db); Mem utf8; @@ -140,16 +140,16 @@ char *sqlite3VdbeExpandSql( pVar = &utf8; } #endif - n = pVar->n; + nOut = pVar->n; #ifdef SQLITE_TRACE_SIZE_LIMIT if( n>SQLITE_TRACE_SIZE_LIMIT ){ - n = SQLITE_TRACE_SIZE_LIMIT; - while( nn && (pVar->z[n]&0xc0)==0x80 ){ n++; } + nOut = SQLITE_TRACE_SIZE_LIMIT; + while( nOutn && (pVar->z[n]&0xc0)==0x80 ){ n++; } } #endif - sqlite3XPrintf(&out, "'%.*q'", n, pVar->z); + sqlite3XPrintf(&out, "'%.*q'", nOut, pVar->z); #ifdef SQLITE_TRACE_SIZE_LIMIT - if( nn ) sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-n); + if( nOutn ) sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-n); #endif #ifndef SQLITE_OMIT_UTF16 if( enc!=SQLITE_UTF8 ) sqlite3VdbeMemRelease(&utf8); @@ -157,19 +157,19 @@ char *sqlite3VdbeExpandSql( }else if( pVar->flags & MEM_Zero ){ sqlite3XPrintf(&out, "zeroblob(%d)", pVar->u.nZero); }else{ - int n; /* Number of bytes of the blob to include in output */ + int nOut; /* Number of bytes of the blob to include in output */ assert( pVar->flags & MEM_Blob ); sqlite3StrAccumAppend(&out, "x'", 2); - n = pVar->n; + nOut = pVar->n; #ifdef SQLITE_TRACE_SIZE_LIMIT - if( n>SQLITE_TRACE_SIZE_LIMIT ) n = SQLITE_TRACE_SIZE_LIMIT; + if( nOut>SQLITE_TRACE_SIZE_LIMIT ) nOut = SQLITE_TRACE_SIZE_LIMIT; #endif - for(i=0; iz[i]&0xff); } sqlite3StrAccumAppend(&out, "'", 1); #ifdef SQLITE_TRACE_SIZE_LIMIT - if( nn ) sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-n); + if( nOutn ) sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-n); #endif } }