From 13a3cb82ce7937bbb7e482c4ab4289d9b777d7c3 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 11 Jun 2010 19:04:21 +0000 Subject: [PATCH 1/8] Experimental change to the xShmXXX parts of the VFS interface. FossilOrigin-Name: ca68472db01c14a899892007d1cbaff5e86ae193 --- manifest | 45 ++-- manifest.uuid | 2 +- src/os.c | 9 + src/os.h | 1 + src/os_unix.c | 80 ++++++- src/sqlite.h.in | 1 + src/test_devsym.c | 14 +- src/test_vfs.c | 130 ++++++------ src/wal.c | 472 ++++++++++++++++++----------------------- test/permutations.test | 1 + test/wal2.test | 66 +++--- test/wal3.test | 2 +- 12 files changed, 428 insertions(+), 395 deletions(-) diff --git a/manifest b/manifest index c374482ea8..9c9da2bd3d 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,5 @@ ------BEGIN PGP SIGNED MESSAGE----- -Hash: SHA1 - -C Refactor\sand\ssimplify\sthe\slogic\sused\sto\schange\sjournalmode. -D 2010-06-11T17:01:25 +C Experimental\schange\sto\sthe\sxShmXXX\sparts\sof\sthe\sVFS\sinterface. +D 2010-06-11T19:04:21 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in a5cad1f8f3e021356bfcc6c77dc16f6f1952bbc3 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -153,11 +150,11 @@ F src/mutex_os2.c 6a62583e374ba3ac1a3fcc0da2bfdac7d3942689 F src/mutex_unix.c cf84466b4fdd2baa0d5a10bb19f08b2abc1ce42e F src/mutex_w32.c 1fe0e735897be20e09dd6f53c3fb516c6b48c0eb F src/notify.c cbfa66a836da3a51567209636e6a94059c137930 -F src/os.c 1516984144e26734f97748f891f1a04f9e294c2e -F src/os.h 6f529984a29511c7a3479cfe549c10bfa131532f +F src/os.c 00ab9dcdee6e33ff3d060744c86af25200c51e0f +F src/os.h a0d2c1436cb6003e6da16001499a0b828f1edb34 F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 665876d5eec7585226b0a1cf5e18098de2b2da19 -F src/os_unix.c 12051d37e533cdaa8bb13c9d9fe2a13e08552187 +F src/os_unix.c 29dac62790ccea7db1516be3abb007988accb165 F src/os_win.c 0cf1f571546f165001e2391b5d4a4a16d86977d3 F src/pager.c 2964185d4356d0dc159b8340e52d2538d32394e5 F src/pager.h ca1f23c0cf137ac26f8908df2427c8b308361efd @@ -173,7 +170,7 @@ F src/resolve.c ac5f1a713cd1ae77f08b83cc69581e11bf5ae6f9 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c c03d8a0565febcde8c6a12c5d77d065fddae889b F src/shell.c fd4ccdb37c3b68de0623eb938a649e0990710714 -F src/sqlite.h.in b6a64327e174cf725e57dd93ddf1e97c52dd41e2 +F src/sqlite.h.in 092df034f4b426ffbb9e5bb905958fa35bbb7f7a F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89 F src/sqliteInt.h 242987ebd2366ea36650a09cdab04a9163c62109 F src/sqliteLimit.h 196e2f83c3b444c4548fc1874f52f84fdbda40f3 @@ -195,7 +192,7 @@ F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de F src/test_btree.c 47cd771250f09cdc6e12dda5bc71bc0b3abc96e2 F src/test_config.c 6210f501d358bde619ae761f06f123529c6ba24f F src/test_demovfs.c da81a5f7785bb352bda7911c332a983ec4f17f27 -F src/test_devsym.c 709712f5157667410cd0dad1b7b1b54319c122c5 +F src/test_devsym.c cf64a4b602ccde10c9261283d1b9be12f4c4a0ea F src/test_func.c 13b582345fb1185a93e46c53310fae8547dcce20 F src/test_hexio.c 1237f000ec7a491009b1233f5c626ea71bce1ea2 F src/test_init.c 5d624ffd0409d424cf9adbfe1f056b200270077c @@ -212,7 +209,7 @@ F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c bbba05c144b5fc4b52ff650a4328027b3fa5fcc6 F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c aa9919c885a1fe53eafc73492f0898ee6c0a0726 -F src/test_vfs.c d329e3ea93624f65d7b6a46209861ddecea4e21d +F src/test_vfs.c b83206d2c04b3ba84d8d85420c4c7573c58feba5 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 25ceb0f0a746ea1d0f9553787f3f0a56853cfaeb F src/trigger.c 8927588cb9e6d47f933b53bfe74200fbb504100d @@ -229,7 +226,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c 2cdfea9a5e50e4dde48767e69e1fead2ff1781cd +F src/wal.c 0aa364734d6daca75771944fc2b4a8f36e63fc4e F src/wal.h 4ace25262452d17e7d3ec970c89ee17794004008 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 1c895bef33d0dfc7ed90fb1f74120435d210ea56 @@ -540,7 +537,7 @@ F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806 F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb F test/pcache.test eebc4420b37cb07733ae9b6e99c9da7c40dd6d58 F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16 -F test/permutations.test ad10d7b31b4a585977380886c832e2ac13c41237 +F test/permutations.test 64fbafa685149be54a1ceb545942911f998c604d F test/pragma.test 6960f9efbce476f70ba9ee2171daf5042f9e3d8a F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea @@ -768,8 +765,8 @@ F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d F test/wal.test 0a599c3c4812ed92bc7ad9efcc2c4007fe4cc99a -F test/wal2.test 854a2b409450f1cb756c2bbd1e87e30740094357 -F test/wal3.test ae876ff988af5b2b34d27474e0dd1a8c84e9bbcb +F test/wal2.test f9dce93acecff697fc1935869b1ae4cb7dc14587 +F test/wal3.test 1d3aee1a0295db941a0323c0ce5ac16bd5b7689d F test/wal_common.tcl 3e953ae60919281688ea73e4d0aa0e1bc94becd9 F test/walbak.test e7650a26eb4b8abeca9b145b1af1e63026dde432 F test/walcksum.test 4efa8fb88c32bed8288ea4385a9cc113a5c8f0bf @@ -823,14 +820,10 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P af353bd89e5ec89f113d217225cc59cbc8373d64 -R 3e2d7a9cc84ff16ee3e947d8ce602cce -U drh -Z e14f3acbecbf2df0abe6fde12480ec15 ------BEGIN PGP SIGNATURE----- -Version: GnuPG v1.4.6 (GNU/Linux) - -iD8DBQFMEmvooxKgR168RlERAg5RAJ9VmP08NyW1of8QWkDMnAiHK8A/xgCeLBcs -hbn4o1Zj2vKA/g5KFcfNycs= -=Li8t ------END PGP SIGNATURE----- +P 95cc3f6fdec5494560c3cd4439d06870d1c62506 +R 05bb7be31606a33c54980de0a121cf7e +T *branch * experimental +T *sym-experimental * +T -sym-trunk * +U dan +Z 7e046a655e9f9112461a59731e31d3dc diff --git a/manifest.uuid b/manifest.uuid index cc28afffbf..447d7ce948 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -95cc3f6fdec5494560c3cd4439d06870d1c62506 \ No newline at end of file +ca68472db01c14a899892007d1cbaff5e86ae193 \ No newline at end of file diff --git a/src/os.c b/src/os.c index 0b17a6b633..697233be0c 100644 --- a/src/os.c +++ b/src/os.c @@ -119,6 +119,15 @@ void sqlite3OsShmBarrier(sqlite3_file *id){ int sqlite3OsShmClose(sqlite3_file *id, int deleteFlag){ return id->pMethods->xShmClose(id, deleteFlag); } +int sqlite3OsShmPage( + sqlite3_file *id, + int iPage, + int pgsz, + int isWrite, + void volatile **pp +){ + return id->pMethods->xShmPage(id, iPage, pgsz, isWrite, pp); +} /* ** The next group of routines are convenience wrappers around the diff --git a/src/os.h b/src/os.h index 001732798e..d1d6bde058 100644 --- a/src/os.h +++ b/src/os.h @@ -254,6 +254,7 @@ int sqlite3OsShmRelease(sqlite3_file *id); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmClose(sqlite3_file *id, int); +int sqlite3OsShmPage(sqlite3_file *,int,int,int,void volatile **); /* ** Functions for accessing sqlite3_vfs methods diff --git a/src/os_unix.c b/src/os_unix.c index dadc3c98af..612d5c4366 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3141,8 +3141,14 @@ struct unixShmNode { sqlite3_mutex *mutexBuf; /* Mutex to access zBuf[] */ char *zFilename; /* Name of the mmapped file */ int h; /* Open file descriptor */ + int szMap; /* Size of the mapping into memory */ char *pMMapBuf; /* Where currently mmapped(). NULL if unmapped */ + + int pgsz; /* Size of shared-memory pages */ + int nPage; /* Size of array apPage */ + char **apPage; /* Array of mapped shared-memory pages */ + int nRef; /* Number of unixShm objects pointing to this */ unixShm *pFirst; /* All unixShm objects pointing to this */ #ifdef SQLITE_DEBUG @@ -3266,10 +3272,15 @@ static void unixShmPurge(unixFile *pFd){ unixShmNode *p = pFd->pInode->pShmNode; assert( unixMutexHeld() ); if( p && p->nRef==0 ){ + int i; assert( p->pInode==pFd->pInode ); if( p->mutex ) sqlite3_mutex_free(p->mutex); if( p->mutexBuf ) sqlite3_mutex_free(p->mutexBuf); if( p->pMMapBuf ) munmap(p->pMMapBuf, p->szMap); + for(i=0; inPage; i++){ + munmap(p->apPage[i], p->pgsz); + } + sqlite3_free(p->apPage); if( p->h>=0 ) close(p->h); p->pInode->pShmNode = 0; sqlite3_free(p); @@ -3706,6 +3717,71 @@ static void unixShmBarrier( unixLeaveMutex(); } +static int unixShmPage( + sqlite3_file *fd, /* Handle open on database file */ + int iPage, /* Page to retrieve */ + int pgsz, /* Size of pages */ + int isWrite, /* True to extend file if necessary */ + void volatile **pp /* OUT: Mapped memory */ +){ + unixFile *pDbFd = (unixFile*)fd; + unixShm *p = pDbFd->pShm; + unixShmNode *pShmNode = p->pShmNode; + int rc = SQLITE_OK; + + assert( p->hasMutexBuf==0 ); + sqlite3_mutex_enter(pShmNode->mutexBuf); + assert( pgsz==pShmNode->pgsz || pShmNode->nPage==0 ); + + if( pShmNode->nPage<=iPage ){ + char **apNew; /* New apPage[] array */ + int nByte = (iPage+1)*pgsz; /* Minimum required file size */ + struct stat sStat; + + pShmNode->pgsz = pgsz; + + /* Make sure the underlying file is large enough (or fail) */ + if( fstat(pShmNode->h, &sStat) ){ + rc = SQLITE_IOERR_SHMSIZE; + goto shmpage_out; + }else if( sStat.st_sizeh, nByte) ){ + rc = SQLITE_IOERR_SHMSIZE; + goto shmpage_out; + } + } + + apNew = (char**)sqlite3_realloc(pShmNode->apPage, (iPage+1)*sizeof(char *)); + if( !apNew ){ + rc = SQLITE_IOERR_NOMEM; + goto shmpage_out; + } + pShmNode->apPage = apNew; + + while(pShmNode->nPage<=iPage){ + void *pMem = mmap( + 0, pgsz, PROT_READ|PROT_WRITE, MAP_SHARED, pShmNode->h, iPage*pgsz + ); + if( pMem==MAP_FAILED ){ + assert(0); + rc = SQLITE_IOERR; + goto shmpage_out; + } + pShmNode->apPage[pShmNode->nPage] = pMem; + pShmNode->nPage++; + } + } + +shmpage_out: + if( pShmNode->nPage>iPage ){ + *pp = pShmNode->apPage[iPage]; + }else{ + *pp = 0; + } + sqlite3_mutex_leave(pShmNode->mutexBuf); + return rc; +} #else # define unixShmOpen 0 @@ -3715,6 +3791,7 @@ static void unixShmBarrier( # define unixShmLock 0 # define unixShmBarrier 0 # define unixShmClose 0 +# define unixShmPage 0 #endif /* #ifndef SQLITE_OMIT_WAL */ /* @@ -3778,7 +3855,8 @@ static const sqlite3_io_methods METHOD = { \ unixShmRelease, /* xShmRelease */ \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ - unixShmClose /* xShmClose */ \ + unixShmClose, /* xShmClose */ \ + unixShmPage /* xShmPage */ \ }; \ static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index e583e47d95..94b994ee39 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -666,6 +666,7 @@ struct sqlite3_io_methods { int (*xShmLock)(sqlite3_file*, int offset, int n, int flags); void (*xShmBarrier)(sqlite3_file*); int (*xShmClose)(sqlite3_file*, int deleteFlag); + int (*xShmPage)(sqlite3_file*, int iPage, int pgsz, int, void volatile**); /* Methods above are valid for version 2 */ /* Additional methods may be added in future releases */ }; diff --git a/src/test_devsym.c b/src/test_devsym.c index 0464804939..e27c00176e 100644 --- a/src/test_devsym.c +++ b/src/test_devsym.c @@ -57,6 +57,7 @@ static int devsymShmRelease(sqlite3_file*); static int devsymShmLock(sqlite3_file*,int,int,int); static void devsymShmBarrier(sqlite3_file*); static int devsymShmClose(sqlite3_file*,int); +static int devsymShmPage(sqlite3_file*,int,int,int, void volatile **); /* ** Method declarations for devsym_vfs. @@ -125,7 +126,8 @@ static sqlite3_io_methods devsym_io_methods = { devsymShmRelease, /* xShmRelease */ devsymShmLock, /* xShmLock */ devsymShmBarrier, /* xShmBarrier */ - devsymShmClose /* xShmClose */ + devsymShmClose, /* xShmClose */ + devsymShmPage /* xShmPage */ }; struct DevsymGlobal { @@ -275,6 +277,16 @@ static int devsymShmClose(sqlite3_file *pFile, int delFlag){ devsym_file *p = (devsym_file *)pFile; return sqlite3OsShmClose(p->pReal, delFlag); } +static int devsymShmPage( + sqlite3_file *pFile, + int iPage, + int pgsz, + int isWrite, + void volatile **pp +){ + devsym_file *p = (devsym_file *)pFile; + return sqlite3OsShmPage(p->pReal, iPage, pgsz, isWrite, pp); +} diff --git a/src/test_vfs.c b/src/test_vfs.c index 1083080eb9..745a417a7a 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -75,10 +75,14 @@ struct Testvfs { #define TESTVFS_SHMLOCK_MASK 0x00000010 #define TESTVFS_SHMBARRIER_MASK 0x00000020 #define TESTVFS_SHMCLOSE_MASK 0x00000040 +#define TESTVFS_SHMPAGE_MASK 0x00000080 -#define TESTVFS_OPEN_MASK 0x00000080 -#define TESTVFS_SYNC_MASK 0x00000100 -#define TESTVFS_ALL_MASK 0x000001FF +#define TESTVFS_OPEN_MASK 0x00000100 +#define TESTVFS_SYNC_MASK 0x00000200 +#define TESTVFS_ALL_MASK 0x000003FF + + +#define TESTVFS_MAX_PAGES 256 /* ** A shared-memory buffer. There is one of these objects for each shared @@ -87,8 +91,8 @@ struct Testvfs { */ struct TestvfsBuffer { char *zFile; /* Associated file name */ - int n; /* Size of allocated buffer in bytes */ - u8 *a; /* Buffer allocated using ckalloc() */ + int pgsz; /* Page size */ + u8 *aPage[TESTVFS_MAX_PAGES]; /* Array of ckalloc'd pages */ TestvfsFile *pFile; /* List of open handles */ TestvfsBuffer *pNext; /* Next in linked list of all buffers */ }; @@ -139,6 +143,7 @@ static int tvfsShmRelease(sqlite3_file*); static int tvfsShmLock(sqlite3_file*, int , int, int); static void tvfsShmBarrier(sqlite3_file*); static int tvfsShmClose(sqlite3_file*, int); +static int tvfsShmPage(sqlite3_file*,int,int,int, void volatile **); static sqlite3_io_methods tvfs_io_methods = { 2, /* iVersion */ @@ -160,7 +165,8 @@ static sqlite3_io_methods tvfs_io_methods = { tvfsShmRelease, /* xShmRelease */ tvfsShmLock, /* xShmLock */ tvfsShmBarrier, /* xShmBarrier */ - tvfsShmClose /* xShmClose */ + tvfsShmClose, /* xShmClose */ + tvfsShmPage /* xShmPage */ }; static int tvfsResultCode(Testvfs *p, int *pRc){ @@ -547,16 +553,6 @@ static int tvfsCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ return PARENTVFS(pVfs)->xCurrentTime(PARENTVFS(pVfs), pTimeOut); } -static void tvfsGrowBuffer(TestvfsFile *pFd, int reqSize, int *pNewSize){ - TestvfsBuffer *pBuffer = pFd->pShm; - if( reqSize>pBuffer->n ){ - pBuffer->a = (u8 *)ckrealloc((char *)pBuffer->a, reqSize); - memset(&pBuffer->a[pBuffer->n], 0x55, reqSize-pBuffer->n); - pBuffer->n = reqSize; - } - *pNewSize = pBuffer->n; -} - static int tvfsInjectIoerr(Testvfs *p){ int ret = 0; if( p->ioerr ){ @@ -624,66 +620,66 @@ static int tvfsShmSize( int reqSize, int *pNewSize ){ - int rc = SQLITE_OK; - TestvfsFile *pFd = (TestvfsFile *)pFile; - Testvfs *p = (Testvfs *)(pFd->pVfs->pAppData); - - if( p->pScript && p->mask&TESTVFS_SHMSIZE_MASK ){ - tvfsExecTcl(p, "xShmSize", - Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0 - ); - tvfsResultCode(p, &rc); - } - if( rc==SQLITE_OK && p->mask&TESTVFS_SHMSIZE_MASK && tvfsInjectIoerr(p) ){ - rc = SQLITE_IOERR; - } - if( rc==SQLITE_OK ){ - tvfsGrowBuffer(pFd, reqSize, pNewSize); - } - return rc; + assert(0); + return SQLITE_OK; } - static int tvfsShmGet( sqlite3_file *pFile, int reqMapSize, int *pMapSize, volatile void **pp +){ + assert(0); + return SQLITE_OK; +} +static int tvfsShmRelease(sqlite3_file *pFile){ + assert(0); + return SQLITE_OK; +} + +static void tvfsAllocPage(TestvfsBuffer *p, int iPage, int pgsz){ + assert( iPageaPage[iPage]==0 ){ + p->aPage[iPage] = ckalloc(pgsz); + memset(p->aPage[iPage], 0, pgsz); + p->pgsz = pgsz; + } +} + +static int tvfsShmPage( + sqlite3_file *pFile, /* Handle open on database file */ + int iPage, /* Page to retrieve */ + int pgsz, /* Size of pages */ + int isWrite, /* True to extend file if necessary */ + void volatile **pp /* OUT: Mapped memory */ ){ int rc = SQLITE_OK; TestvfsFile *pFd = (TestvfsFile *)pFile; Testvfs *p = (Testvfs *)(pFd->pVfs->pAppData); - if( p->pScript && p->mask&TESTVFS_SHMGET_MASK ){ - tvfsExecTcl(p, "xShmGet", - Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, - Tcl_NewIntObj(reqMapSize) + if( p->pScript && p->mask&TESTVFS_SHMPAGE_MASK ){ + Tcl_Obj *pArg = Tcl_NewObj(); + Tcl_ListObjAppendElement(p->interp, pArg, Tcl_NewIntObj(iPage)); + Tcl_ListObjAppendElement(p->interp, pArg, Tcl_NewIntObj(pgsz)); + Tcl_ListObjAppendElement(p->interp, pArg, Tcl_NewIntObj(isWrite)); + tvfsExecTcl(p, "xShmPage", + Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, pArg ); tvfsResultCode(p, &rc); } - if( rc==SQLITE_OK && p->mask&TESTVFS_SHMGET_MASK && tvfsInjectIoerr(p) ){ + if( rc==SQLITE_OK && p->mask&TESTVFS_SHMPAGE_MASK && tvfsInjectIoerr(p) ){ rc = SQLITE_IOERR; } - *pMapSize = pFd->pShm->n; - *pp = pFd->pShm->a; - return rc; -} - -static int tvfsShmRelease(sqlite3_file *pFile){ - int rc = SQLITE_OK; - TestvfsFile *pFd = (TestvfsFile *)pFile; - Testvfs *p = (Testvfs *)(pFd->pVfs->pAppData); - - if( p->pScript && p->mask&TESTVFS_SHMRELEASE_MASK ){ - tvfsExecTcl(p, "xShmRelease", - Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0 - ); - tvfsResultCode(p, &rc); + if( rc==SQLITE_OK && isWrite && !pFd->pShm->aPage[iPage] ){ + tvfsAllocPage(pFd->pShm, iPage, pgsz); } + *pp = (void volatile *)pFd->pShm->aPage[iPage]; return rc; } + static int tvfsShmLock( sqlite3_file *pFile, int ofst, @@ -782,10 +778,13 @@ static int tvfsShmClose( *ppFd = pFd->pNext; if( pBuffer->pFile==0 ){ + int i; TestvfsBuffer **pp; for(pp=&p->pBuffer; *pp!=pBuffer; pp=&((*pp)->pNext)); *pp = (*pp)->pNext; - ckfree((char *)pBuffer->a); + for(i=0; pBuffer->aPage[i]; i++){ + ckfree((char *)pBuffer->aPage[i]); + } ckfree((char *)pBuffer); } pFd->pShm = 0; @@ -821,6 +820,8 @@ static int testvfs_obj_cmd( switch( (enum DB_enum)i ){ case CMD_SHM: { + Tcl_Obj *pObj; + int i; TestvfsBuffer *pBuffer; char *zName; if( objc!=3 && objc!=4 ){ @@ -838,11 +839,22 @@ static int testvfs_obj_cmd( if( objc==4 ){ int n; u8 *a = Tcl_GetByteArrayFromObj(objv[3], &n); - pBuffer->a = (u8 *)ckrealloc((char *)pBuffer->a, n); - pBuffer->n = n; - memcpy(pBuffer->a, a, n); + assert( pBuffer->pgsz==0 || pBuffer->pgsz==32768 ); + for(i=0; i*32768aPage[i], &a[i*32768], nByte); + } } - Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(pBuffer->a, pBuffer->n)); + + pObj = Tcl_NewObj(); + for(i=0; pBuffer->aPage[i]; i++){ + Tcl_AppendObjToObj(pObj, Tcl_NewByteArrayObj(pBuffer->aPage[i], 32768)); + } + Tcl_SetObjResult(interp, pObj); break; } diff --git a/src/wal.c b/src/wal.c index 017b844908..a05eec0b82 100644 --- a/src/wal.c +++ b/src/wal.c @@ -370,8 +370,8 @@ struct Wal { sqlite3_file *pDbFd; /* File handle for the database file */ sqlite3_file *pWalFd; /* File handle for WAL file */ u32 iCallback; /* Value to pass to log callback (or 0) */ - int szWIndex; /* Size of the wal-index that is mapped in mem */ - volatile u32 *pWiData; /* Pointer to wal-index content in memory */ + int nWiData; /* Size of array apWiData */ + volatile u32 **apWiData; /* Pointer to wal-index content in memory */ u16 szPage; /* Database page size */ i16 readLock; /* Which read lock is being held. -1 for none */ u8 exclusiveMode; /* Non-zero if connection is in exclusive mode */ @@ -386,15 +386,77 @@ struct Wal { #endif }; +/* +** Define the parameters of the hash tables in the wal-index file. There +** is a hash-table following every HASHTABLE_NPAGE page numbers in the +** wal-index. +** +** Changing any of these constants will alter the wal-index format and +** create incompatibilities. +*/ +#define HASHTABLE_NPAGE 4096 /* Must be power of 2 and multiple of 256 */ +#define HASHTABLE_DATATYPE u16 +#define HASHTABLE_HASH_1 383 /* Should be prime */ +#define HASHTABLE_NSLOT (HASHTABLE_NPAGE*2) /* Must be a power of 2 */ +#define HASHTABLE_NBYTE (sizeof(HASHTABLE_DATATYPE)*HASHTABLE_NSLOT) + +/* The block of page numbers associated with the first hash-table in a +** wal-index is smaller than usual. This is so that there is a complete +** hash-table on each aligned 32KB page of the wal-index. +*/ +#define HASHTABLE_NPAGE_ONE (4096 - (WALINDEX_HDR_SIZE/sizeof(u32))) + +/* The wal-index is divided into pages of HASHTABLE_PAGESIZE bytes each. */ +#define HASHTABLE_PAGESIZE (HASHTABLE_NBYTE + HASHTABLE_NPAGE*sizeof(u32)) + +/* +** Obtain a pointer to the iPage'th page of the wal-index. The wal-index +** is broken into pages of HASHTABLE_PAGESIZE bytes. Wal-index pages are +** numbered from zero. +** +** If this call is successful, *ppPage is set to point to the wal-index +** page and SQLITE_OK is returned. If an error (an OOM or VFS error) occurs, +** then an SQLite error code is returned and *ppPage is set to 0. +*/ +static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ + int rc = SQLITE_OK; + + /* Enlarge the pWal->apWiData[] array if required */ + if( pWal->nWiData<=iPage ){ + int nByte = sizeof(u32 *)*(iPage+1); + volatile u32 **apNew; + apNew = (volatile u32 **)sqlite3_realloc(pWal->apWiData, nByte); + if( !apNew ){ + *ppPage = 0; + return SQLITE_NOMEM; + } + memset(&apNew[pWal->nWiData], 0, sizeof(u32 *)*(iPage+1-pWal->nWiData)); + pWal->apWiData = apNew; + pWal->nWiData = iPage+1; + } + + /* Request a pointer to the required page from the VFS */ + if( pWal->apWiData[iPage]==0 ){ + rc = sqlite3OsShmPage(pWal->pDbFd, iPage, HASHTABLE_PAGESIZE, + pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] + ); + } + + *ppPage = pWal->apWiData[iPage]; + assert( iPage==0 || *ppPage || rc!=SQLITE_OK ); + return rc; +} + /* ** Return a pointer to the WalCkptInfo structure in the wal-index. */ static volatile WalCkptInfo *walCkptInfo(Wal *pWal){ - assert( pWal->pWiData!=0 ); - return (volatile WalCkptInfo*)&pWal->pWiData[sizeof(WalIndexHdr)/2]; + volatile u32 *page1 = 0; + walIndexPage(pWal, 0, &page1); + assert( page1 ); + return (volatile WalCkptInfo*)&page1[sizeof(WalIndexHdr)/2]; } - /* ** This structure is used to implement an iterator that loops through ** all frames in the WAL in database page order. Where two or more frames @@ -413,12 +475,13 @@ static volatile WalCkptInfo *walCkptInfo(Wal *pWal){ struct WalIterator { int iPrior; /* Last result returned from the iterator */ int nSegment; /* Size of the aSegment[] array */ - int nFinal; /* Elements in aSegment[nSegment-1] */ struct WalSegment { - int iNext; /* Next slot in aIndex[] not previously returned */ - u8 *aIndex; /* i0, i1, i2... such that aPgno[iN] ascending */ - u32 *aPgno; /* 256 page numbers. Pointer to Wal.pWiData */ - } aSegment[1]; /* One for every 256 entries in the WAL */ + int iNext; /* Next slot in aIndex[] not yet returned */ + HASHTABLE_DATATYPE *aIndex; /* i0, i1, i2... such that aPgno[iN] ascend */ + u32 *aPgno; /* Array of page numbers. */ + int nEntry; /* Max size of aPgno[] and aIndex[] arrays */ + int iZero; /* Frame number associated with aPgno[0] */ + } aSegment[1]; /* One for every 32KB page in the WAL */ }; /* @@ -492,7 +555,7 @@ static void walIndexWriteHdr(Wal *pWal){ pWal->hdr.isInit = 1; walChecksumBytes(1, (u8*)&pWal->hdr, offsetof(WalIndexHdr, aCksum), 0, pWal->hdr.aCksum); - aHdr = (WalIndexHdr*)pWal->pWiData; + walIndexPage(pWal, 0, (volatile u32 **)&aHdr); memcpy(&aHdr[1], &pWal->hdr, sizeof(WalIndexHdr)); sqlite3OsShmBarrier(pWal->pDbFd); memcpy(&aHdr[0], &pWal->hdr, sizeof(WalIndexHdr)); @@ -586,19 +649,6 @@ static int walDecodeFrame( return 1; } -/* -** Define the parameters of the hash tables in the wal-index file. There -** is a hash-table following every HASHTABLE_NPAGE page numbers in the -** wal-index. -** -** Changing any of these constants will alter the wal-index format and -** create incompatibilities. -*/ -#define HASHTABLE_NPAGE 4096 /* Must be power of 2 and multiple of 256 */ -#define HASHTABLE_DATATYPE u16 -#define HASHTABLE_HASH_1 383 /* Should be prime */ -#define HASHTABLE_NSLOT (HASHTABLE_NPAGE*2) /* Must be a power of 2 */ -#define HASHTABLE_NBYTE (sizeof(HASHTABLE_DATATYPE)*HASHTABLE_NSLOT) #if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) /* @@ -663,96 +713,6 @@ static void walUnlockExclusive(Wal *pWal, int lockIdx, int n){ walLockName(lockIdx), n)); } -/* -** Return the index in the Wal.pWiData array that corresponds to -** frame iFrame. -** -** Wal.pWiData is an array of u32 elements that is the wal-index. -** The array begins with a header and is then followed by alternating -** "map" and "hash-table" blocks. Each "map" block consists of -** HASHTABLE_NPAGE u32 elements which are page numbers corresponding -** to frames in the WAL file. -** -** This routine returns an index X such that Wal.pWiData[X] is part -** of a "map" block that contains the page number of the iFrame-th -** frame in the WAL file. -*/ -static int walIndexEntry(u32 iFrame){ - return ( - (WALINDEX_LOCK_OFFSET+WALINDEX_LOCK_RESERVED)/sizeof(u32) - + (((iFrame-1)/HASHTABLE_NPAGE) * HASHTABLE_NBYTE)/sizeof(u32) - + (iFrame-1) - ); -} - -/* -** Return the minimum size of the shared-memory, in bytes, that is needed -** to support a wal-index containing frame iFrame. The value returned -** includes the wal-index header and the complete "block" containing iFrame, -** including the hash table segment that follows the block. -*/ -static int walMappingSize(u32 iFrame){ - const int nByte = (sizeof(u32)*HASHTABLE_NPAGE + HASHTABLE_NBYTE) ; - return ( WALINDEX_LOCK_OFFSET - + WALINDEX_LOCK_RESERVED - + nByte * ((iFrame + HASHTABLE_NPAGE - 1)/HASHTABLE_NPAGE) - ); -} - -/* -** Release our reference to the wal-index memory map, if we are holding -** it. -*/ -static void walIndexUnmap(Wal *pWal){ - if( pWal->pWiData ){ - sqlite3OsShmRelease(pWal->pDbFd); - } - pWal->pWiData = 0; - pWal->szWIndex = -1; -} - -/* -** Map the wal-index file into memory if it isn't already. -** -** The reqSize parameter is the requested size of the mapping. The -** mapping will be at least this big if the underlying storage is -** that big. But the mapping will never grow larger than the underlying -** storage. Use the walIndexRemap() to enlarget the storage space. -*/ -static int walIndexMap(Wal *pWal, int reqSize){ - int rc = SQLITE_OK; - if( pWal->pWiData==0 || reqSize>pWal->szWIndex ){ - walIndexUnmap(pWal); - rc = sqlite3OsShmGet(pWal->pDbFd, reqSize, &pWal->szWIndex, - (void volatile**)(char volatile*)&pWal->pWiData); - if( rc!=SQLITE_OK ){ - walIndexUnmap(pWal); - } - } - return rc; -} - -/* -** Enlarge the wal-index to be at least enlargeTo bytes in size and -** Remap the wal-index so that the mapping covers the full size -** of the underlying file. -** -** If enlargeTo is non-negative, then increase the size of the underlying -** storage to be at least as big as enlargeTo before remapping. -*/ -static int walIndexRemap(Wal *pWal, int enlargeTo){ - int rc; - int sz; - assert( pWal->writeLock ); - rc = sqlite3OsShmSize(pWal->pDbFd, enlargeTo, &sz); - if( rc==SQLITE_OK && sz>pWal->szWIndex ){ - walIndexUnmap(pWal); - rc = walIndexMap(pWal, sz); - } - assert( pWal->szWIndex>=enlargeTo || rc!=SQLITE_OK ); - return rc; -} - /* ** Compute a hash on a page number. The resulting hash value must land ** between 0 and (HASHTABLE_NSLOT-1). The walHashNext() function advances @@ -767,6 +727,54 @@ static int walNextHash(int iPriorHash){ return (iPriorHash+1)&(HASHTABLE_NSLOT-1); } +static void walHashGet( + Wal *pWal, /* WAL handle */ + int iHash, /* Find the iHash'th table */ + volatile HASHTABLE_DATATYPE **paHash, /* OUT: Pointer to hash index */ + volatile u32 **paPgno, /* OUT: Pointer to page number array */ + u32 *piZero /* OUT: Frame associated with *paPgno[0] */ +){ + u32 iZero; + volatile u32 *aPgno; + volatile HASHTABLE_DATATYPE *aHash; + + walIndexPage(pWal, iHash, &aPgno); + aHash = (volatile HASHTABLE_DATATYPE *)&aPgno[HASHTABLE_NPAGE]; + + if( iHash==0 ){ + aPgno = &aPgno[WALINDEX_HDR_SIZE/sizeof(u32)-1]; + iZero = 0; + }else{ + iZero = HASHTABLE_NPAGE_ONE + (iHash-1)*HASHTABLE_NPAGE; + aPgno = &aPgno[-1*iZero-1]; + } + + *paPgno = aPgno; + *paHash = aHash; + *piZero = iZero; +} + +static int walFramePage(u32 iFrame){ + int iHash = (iFrame+HASHTABLE_NPAGE-HASHTABLE_NPAGE_ONE-1) / HASHTABLE_NPAGE; + assert( (iHash==0 || iFrame>HASHTABLE_NPAGE_ONE) + && (iHash>=1 || iFrame<=HASHTABLE_NPAGE_ONE) + && (iHash<=1 || iFrame>(HASHTABLE_NPAGE_ONE+HASHTABLE_NPAGE)) + && (iHash>=2 || iFrame<=HASHTABLE_NPAGE_ONE+HASHTABLE_NPAGE) + && (iHash<=2 || iFrame>(HASHTABLE_NPAGE_ONE+2*HASHTABLE_NPAGE)) + ); + return iHash; +} + +/* +** Return the page number associated with frame iFrame in this WAL. +*/ +static u32 walFramePgno(Wal *pWal, u32 iFrame){ + int iHash = walFramePage(iFrame); + if( iHash==0 ){ + return pWal->apWiData[0][WALINDEX_HDR_SIZE/sizeof(u32) + iFrame - 1]; + } + return pWal->apWiData[iHash][(iFrame-1-HASHTABLE_NPAGE_ONE)%HASHTABLE_NPAGE]; +} /* ** Find the hash table and (section of the) page number array used to @@ -789,27 +797,8 @@ static void walHashFind( volatile u32 **paPgno, /* OUT: Pointer to page number array */ u32 *piZero /* OUT: Frame associated with *paPgno[0] */ ){ - u32 iZero; - volatile u32 *aPgno; - volatile HASHTABLE_DATATYPE *aHash; - - iZero = ((iFrame-1)/HASHTABLE_NPAGE) * HASHTABLE_NPAGE; - aPgno = &pWal->pWiData[walIndexEntry(iZero+1)-iZero-1]; - aHash = (HASHTABLE_DATATYPE *)&aPgno[iZero+HASHTABLE_NPAGE+1]; - - /* Assert that: - ** - ** + the mapping is large enough for this hash-table, and - ** - ** + that aPgno[iZero+1] really is the database page number associated - ** with the first frame indexed by this hash table. - */ - assert( (u32*)(&aHash[HASHTABLE_NSLOT])<=&pWal->pWiData[pWal->szWIndex/4] ); - assert( walIndexEntry(iZero+1)==(&aPgno[iZero+1] - pWal->pWiData) ); - - *paHash = aHash; - *paPgno = aPgno; - *piZero = iZero; + int iHash = walFramePage(iFrame); + walHashGet(pWal, iHash, paHash, paPgno, piZero); } /* @@ -829,16 +818,16 @@ static void walCleanupHash(Wal *pWal){ volatile u32 *aPgno; /* Unused return from walHashFind() */ u32 iZero; /* frame == (aHash[x]+iZero) */ int iLimit = 0; /* Zero values greater than this */ + int nByte; /* Number of bytes to zero in aPgno[] */ + int i; /* Used to iterate through aHash[] */ assert( pWal->writeLock ); testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE-1 ); testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE ); testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE+1 ); - if( (pWal->hdr.mxFrame % HASHTABLE_NPAGE)>0 ){ - int nByte; /* Number of bytes to zero in aPgno[] */ - int i; /* Used to iterate through aHash[] */ - walHashFind(pWal, pWal->hdr.mxFrame+1, &aHash, &aPgno, &iZero); + walHashFind(pWal, pWal->hdr.mxFrame+1, &aHash, &aPgno, &iZero); + if( iZero!=pWal->hdr.mxFrame ){ iLimit = pWal->hdr.mxFrame - iZero; assert( iLimit>0 ); for(i=0; ihdr.mxFrame. */ - nByte = sizeof(u32) * (HASHTABLE_NPAGE-iLimit); - memset((void *)&aPgno[iZero+iLimit+1], 0, nByte); - assert( &((u8 *)&aPgno[iZero+iLimit+1])[nByte]==(u8 *)aHash ); + nByte = ((char *)aHash - (char *)&aPgno[pWal->hdr.mxFrame+1]); + memset((void *)&aPgno[pWal->hdr.mxFrame+1], 0, nByte); } #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT @@ -878,15 +866,7 @@ static void walCleanupHash(Wal *pWal){ ** pPage into WAL frame iFrame. */ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ - int rc; /* Return code */ - int nMapping; /* Required mapping size in bytes */ - - /* Make sure the wal-index is mapped. Enlarge the mapping if required. */ - nMapping = walMappingSize(iFrame); - rc = walIndexMap(pWal, nMapping); - while( rc==SQLITE_OK && nMapping>pWal->szWIndex ){ - rc = walIndexRemap(pWal, nMapping); - } + int rc = SQLITE_OK; /* Return code */ /* Assuming the wal-index file was successfully mapped, find the hash ** table and section of of the page number array that pertain to frame @@ -904,8 +884,8 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ walHashFind(pWal, iFrame, &aHash, &aPgno, &iZero); idx = iFrame - iZero; if( idx==1 ){ - memset((void*)&aPgno[iZero+1], 0, HASHTABLE_NPAGE*sizeof(u32)); - memset((void*)aHash, 0, HASHTABLE_NBYTE); + int nByte = (u8 *)&aHash[HASHTABLE_NSLOT] - (u8 *)&aPgno[1+iZero]; + memset((void*)&aPgno[1+iZero], 0, nByte); } assert( idx <= HASHTABLE_NSLOT/2 + 1 ); @@ -1076,9 +1056,6 @@ static int walIndexRecover(Wal *pWal){ } finished: - if( rc==SQLITE_OK && pWal->hdr.mxFrame==0 ){ - rc = walIndexRemap(pWal, walMappingSize(1)); - } if( rc==SQLITE_OK ){ volatile WalCkptInfo *pInfo; int i; @@ -1164,7 +1141,6 @@ int sqlite3WalOpen( pRet->pVfs = pVfs; pRet->pWalFd = (sqlite3_file *)&pRet[1]; pRet->pDbFd = pDbFd; - pRet->szWIndex = -1; pRet->readLock = -1; sqlite3_randomness(8, &pRet->hdr.aSalt); pRet->zWalName = zWal = pVfs->szOsFile + (char*)pRet->pWalFd; @@ -1207,24 +1183,22 @@ static int walIteratorNext( u32 iMin; /* Result pgno must be greater than iMin */ u32 iRet = 0xFFFFFFFF; /* 0xffffffff is never a valid page number */ int i; /* For looping through segments */ - int nBlock = p->nFinal; /* Number of entries in current segment */ iMin = p->iPrior; assert( iMin<0xffffffff ); for(i=p->nSegment-1; i>=0; i--){ struct WalSegment *pSegment = &p->aSegment[i]; - while( pSegment->iNextiNextnEntry ){ u32 iPg = pSegment->aPgno[pSegment->aIndex[pSegment->iNext]]; if( iPg>iMin ){ if( iPgaIndex[pSegment->iNext]; + *piFrame = pSegment->iZero + pSegment->aIndex[pSegment->iNext]; } break; } pSegment->iNext++; } - nBlock = 256; } *piPage = p->iPrior = iRet; @@ -1232,28 +1206,28 @@ static int walIteratorNext( } -static void walMergesort8( - Pgno *aContent, /* Pages in wal */ - u8 *aBuffer, /* Buffer of at least *pnList items to use */ - u8 *aList, /* IN/OUT: List to sort */ +static void walMergesort( + u32 *aContent, /* Pages in wal */ + HASHTABLE_DATATYPE *aBuffer, /* Buffer of at least *pnList items to use */ + HASHTABLE_DATATYPE *aList, /* IN/OUT: List to sort */ int *pnList /* IN/OUT: Number of elements in aList[] */ ){ int nList = *pnList; if( nList>1 ){ int nLeft = nList / 2; /* Elements in left list */ int nRight = nList - nLeft; /* Elements in right list */ - u8 *aLeft = aList; /* Left list */ - u8 *aRight = &aList[nLeft]; /* Right list */ int iLeft = 0; /* Current index in aLeft */ int iRight = 0; /* Current index in aright */ int iOut = 0; /* Current index in output buffer */ + HASHTABLE_DATATYPE *aLeft = aList; /* Left list */ + HASHTABLE_DATATYPE *aRight = &aList[nLeft]; /* Right list */ /* TODO: Change to non-recursive version. */ - walMergesort8(aContent, aBuffer, aLeft, &nLeft); - walMergesort8(aContent, aBuffer, aRight, &nRight); + walMergesort(aContent, aBuffer, aLeft, &nLeft); + walMergesort(aContent, aBuffer, aRight, &nRight); while( iRightpWiData && pWal->szWIndex>=walMappingSize(pWal->hdr.mxFrame) ); + HASHTABLE_DATATYPE *aTmp; /* Temp space used by merge-sort */ + HASHTABLE_DATATYPE *aSpace; /* Space at the end of the allocation */ /* This routine only runs while holding SQLITE_SHM_CHECKPOINT. No other ** thread is able to write to shared memory while this routine is ** running (or, indeed, while the WalIterator object exists). Hence, - ** we can cast off the volatile qualifacation from shared memory + ** we can cast off the volatile qualification from shared memory */ assert( pWal->ckptLock ); - aData = (u32*)pWal->pWiData; + iLast = pWal->hdr.mxFrame; /* Allocate space for the WalIterator object */ - iLast = pWal->hdr.mxFrame; - nSegment = (iLast >> 8) + 1; - nFinal = (iLast & 0x000000FF); - nByte = sizeof(WalIterator) + (nSegment+1)*(sizeof(struct WalSegment)+256); + nSegment = walFramePage(iLast) + 1; + nByte = sizeof(WalIterator) + + nSegment*(sizeof(struct WalSegment)) + + (nSegment+1)*(HASHTABLE_NPAGE * sizeof(HASHTABLE_DATATYPE)); p = (WalIterator *)sqlite3_malloc(nByte); if( !p ){ return SQLITE_NOMEM; } memset(p, 0, nByte); - /* Initialize the WalIterator object. Each 256-entry segment is - ** presorted in order to make iterating through all entries much - ** faster. - */ + /* Allocate space for the WalIterator object */ p->nSegment = nSegment; - aSpace = (u8 *)&p->aSegment[nSegment]; - aTmp = &aSpace[nSegment*256]; + aSpace = (HASHTABLE_DATATYPE *)&p->aSegment[nSegment]; + aTmp = &aSpace[HASHTABLE_NPAGE*nSegment]; for(i=0; iaSegment[i].aPgno = &aData[walIndexEntry(i*256+1)]; - p->aSegment[i].aIndex = aSpace; - for(j=0; jaSegment[i].aPgno, aTmp, aSpace, &nIndex); - memset(&aSpace[nIndex], aSpace[nIndex-1], 256-nIndex); - aSpace += 256; - p->nFinal = nIndex; + walMergesort((u32 *)aPgno, aTmp, aSpace, &nEntry); + p->aSegment[i].iZero = iZero; + p->aSegment[i].nEntry = nEntry; + p->aSegment[i].aIndex = aSpace; + p->aSegment[i].aPgno = (u32 *)aPgno; + aSpace += HASHTABLE_NPAGE; } + assert( aSpace==aTmp ); - /* Return the fully initializd WalIterator object */ + /* Return the fully initialized WalIterator object */ *pp = p; return SQLITE_OK ; } @@ -1430,8 +1412,8 @@ static int walCheckpoint( ** cannot be backfilled from the WAL. */ mxSafeFrame = pWal->hdr.mxFrame; - pHdr = (volatile WalIndexHdr*)pWal->pWiData; - pInfo = (volatile WalCkptInfo*)&pHdr[2]; + walIndexPage(pWal, 0, (volatile u32 **)&pHdr); + pInfo = walCkptInfo(pWal); assert( pInfo==walCkptInfo(pWal) ); for(i=1; iaReadMark[i]; @@ -1461,6 +1443,7 @@ static int walCheckpoint( /* Iterate through the contents of the WAL, copying data to the db file. */ while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ + assert( walFramePgno(pWal, iFrame)==iDbpage ); if( iFrame<=nBackfill || iFrame>mxSafeFrame ) continue; rc = sqlite3OsRead(pWal->pWalFd, zBuf, szPage, walFrameOffset(iFrame, szPage) + WAL_FRAME_HDRSIZE @@ -1525,7 +1508,6 @@ int sqlite3WalClose( if( rc==SQLITE_OK ){ isDelete = 1; } - walIndexUnmap(pWal); } walIndexClose(pWal, isDelete); @@ -1534,6 +1516,7 @@ int sqlite3WalClose( sqlite3OsDelete(pWal->pVfs, pWal->zWalName, 0); } WALTRACE(("WAL%p: closed\n", pWal)); + sqlite3_free(pWal->apWiData); sqlite3_free(pWal); } return rc; @@ -1560,13 +1543,14 @@ int walIndexTryHdr(Wal *pWal, int *pChanged){ u32 aCksum[2]; /* Checksum on the header content */ WalIndexHdr h1, h2; /* Two copies of the header content */ WalIndexHdr *aHdr; /* Header in shared memory */ + volatile u32 *page1 = 0; - if( pWal->szWIndex < WALINDEX_HDR_SIZE ){ + walIndexPage(pWal, 0, &page1); + if( !page1 ){ /* The wal-index is not large enough to hold the header, then assume ** header is invalid. */ return 1; } - assert( pWal->pWiData ); /* Read the header. This might happen currently with a write to the ** same area of shared memory on a different CPU in a SMP, @@ -1578,7 +1562,7 @@ int walIndexTryHdr(Wal *pWal, int *pChanged){ ** Memory barriers are used to prevent the compiler or the hardware from ** reordering the reads and writes. */ - aHdr = (WalIndexHdr*)pWal->pWiData; + aHdr = (WalIndexHdr*)page1; memcpy(&h1, &aHdr[0], sizeof(h1)); sqlite3OsShmBarrier(pWal->pDbFd); memcpy(&h2, &aHdr[1], sizeof(h2)); @@ -1625,9 +1609,10 @@ int walIndexTryHdr(Wal *pWal, int *pChanged){ static int walIndexReadHdr(Wal *pWal, int *pChanged){ int rc; /* Return code */ int badHdr; /* True if a header read failed */ + volatile u32 *dummy; assert( pChanged ); - rc = walIndexMap(pWal, walMappingSize(1)); + rc = walIndexPage(pWal, 0, &dummy); if( rc!=SQLITE_OK ){ return rc; } @@ -1659,14 +1644,6 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ } } - /* Make sure the mapping is large enough to cover the entire wal-index */ - if( rc==SQLITE_OK ){ - int szWanted = walMappingSize(pWal->hdr.mxFrame); - if( pWal->szWIndexreadLock<0 ); /* Not currently locked */ @@ -1739,16 +1716,14 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ rc = SQLITE_BUSY_RECOVERY; } } - }else{ - rc = walIndexMap(pWal, walMappingSize(pWal->hdr.mxFrame)); } if( rc!=SQLITE_OK ){ return rc; } - pHdr = (volatile WalIndexHdr*)pWal->pWiData; - pInfo = (volatile WalCkptInfo*)&pHdr[2]; - assert( pInfo==walCkptInfo(pWal) ); + walIndexPage(pWal, 0, (volatile u32 **)&pHdr); + pInfo = walCkptInfo(pWal); + assert( pInfo==(volatile WalCkptInfo *)&pHdr[2] ); if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame ){ /* The WAL has been completely backfilled (or it is empty). ** and can be safely ignored. @@ -1883,7 +1858,6 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ do{ rc = walTryBeginRead(pWal, pChanged, 0, ++cnt); }while( rc==WAL_RETRY ); - walIndexUnmap(pWal); return rc; } @@ -1913,7 +1887,6 @@ int sqlite3WalRead( int nOut, /* Size of buffer pOut in bytes */ u8 *pOut /* Buffer to write page data to */ ){ - int rc; /* Return code */ u32 iRead = 0; /* If !=0, WAL frame to return data from */ u32 iLast = pWal->hdr.mxFrame; /* Last page in WAL for this reader */ int iHash; /* Used to loop through N hash tables */ @@ -1932,12 +1905,6 @@ int sqlite3WalRead( return SQLITE_OK; } - /* Ensure the wal-index is mapped. */ - rc = walIndexMap(pWal, walMappingSize(iLast)); - if( rc!=SQLITE_OK ){ - return rc; - } - /* Search the hash table or tables for an entry matching page number ** pgno. Each iteration of the following for() loop searches one ** hash table (each hash table indexes up to HASHTABLE_NPAGE frames). @@ -1963,16 +1930,13 @@ int sqlite3WalRead( ** This condition filters out entries that were added to the hash ** table after the current read-transaction had started. */ - for(iHash=iLast; iHash>0 && iRead==0; iHash-=HASHTABLE_NPAGE){ + for(iHash=walFramePage(iLast); iHash>=0 && iRead==0; iHash--){ volatile HASHTABLE_DATATYPE *aHash; /* Pointer to hash table */ volatile u32 *aPgno; /* Pointer to array of page numbers */ u32 iZero; /* Frame number corresponding to aPgno[0] */ int iKey; /* Hash slot index */ - int mxHash; /* upper bound on aHash[] values */ - walHashFind(pWal, iHash, &aHash, &aPgno, &iZero); - mxHash = iLast - iZero; - if( mxHash > HASHTABLE_NPAGE ) mxHash = HASHTABLE_NPAGE; + walHashGet(pWal, iHash, &aHash, &aPgno, &iZero); for(iKey=walHash(pgno); aHash[iKey]; iKey=walNextHash(iKey)){ u32 iFrame = aHash[iKey] + iZero; if( iFrame<=iLast && aPgno[iFrame]==pgno ){ @@ -1981,7 +1945,6 @@ int sqlite3WalRead( } } } - assert( iRead==0 || pWal->pWiData[walIndexEntry(iRead)]==pgno ); #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT /* If expensive assert() statements are available, do a linear search @@ -1991,7 +1954,7 @@ int sqlite3WalRead( u32 iRead2 = 0; u32 iTest; for(iTest=iLast; iTest>0; iTest--){ - if( pWal->pWiData[walIndexEntry(iTest)]==pgno ){ + if( walFramePgno(pWal, iTest)==pgno ){ iRead2 = iTest; break; } @@ -2003,7 +1966,6 @@ int sqlite3WalRead( /* If iRead is non-zero, then it is the log frame number that contains the ** required page. Read and return data from the log file. */ - walIndexUnmap(pWal); if( iRead ){ i64 iOffset = walFrameOffset(iRead, pWal->hdr.szPage) + WAL_FRAME_HDRSIZE; *pInWal = 1; @@ -2039,6 +2001,7 @@ void sqlite3WalDbsize(Wal *pWal, Pgno *pPgno){ */ int sqlite3WalBeginWriteTransaction(Wal *pWal){ int rc; + volatile u32 *page1; /* Cannot start a write transaction without first holding a read ** transaction. */ @@ -2057,19 +2020,13 @@ int sqlite3WalBeginWriteTransaction(Wal *pWal){ ** time the read transaction on this connection was started, then ** the write is disallowed. */ - rc = walIndexMap(pWal, walMappingSize(pWal->hdr.mxFrame)); - if( rc ){ - walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); - pWal->writeLock = 0; - return rc; - } - if( memcmp(&pWal->hdr, (void*)pWal->pWiData, sizeof(WalIndexHdr))!=0 ){ + walIndexPage(pWal, 0, &page1); + if( memcmp(&pWal->hdr, (void*)page1, sizeof(WalIndexHdr))!=0 ){ walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); pWal->writeLock = 0; rc = SQLITE_BUSY; } - walIndexUnmap(pWal); return rc; } @@ -2102,11 +2059,7 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ Pgno iMax = pWal->hdr.mxFrame; Pgno iFrame; - assert( pWal->pWiData==0 ); rc = walIndexReadHdr(pWal, &unused); - if( rc==SQLITE_OK ){ - rc = walIndexMap(pWal, walMappingSize(iMax)); - } if( rc==SQLITE_OK ){ for(iFrame=pWal->hdr.mxFrame+1; ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; @@ -2124,12 +2077,11 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ ** committed. As a result, the call to xUndo may not fail. */ assert( pWal->writeLock ); - assert( pWal->pWiData[walIndexEntry(iFrame)]!=1 ); - rc = xUndo(pUndoCtx, pWal->pWiData[walIndexEntry(iFrame)]); + assert( walFramePgno(pWal, iFrame)!=1 ); + rc = xUndo(pUndoCtx, walFramePgno(pWal, iFrame)); } walCleanupHash(pWal); } - walIndexUnmap(pWal); } return rc; } @@ -2170,7 +2122,6 @@ int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ } if( aWalData[0]hdr.mxFrame ){ - rc = walIndexMap(pWal, walMappingSize(pWal->hdr.mxFrame)); pWal->hdr.mxFrame = aWalData[0]; pWal->hdr.aFrameCksum[0] = aWalData[1]; pWal->hdr.aFrameCksum[1] = aWalData[2]; @@ -2179,7 +2130,6 @@ int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ } } - walIndexUnmap(pWal); return rc; } @@ -2199,9 +2149,7 @@ static int walRestartLog(Wal *pWal){ int rc = SQLITE_OK; int cnt; - if( pWal->readLock==0 - && SQLITE_OK==(rc = walIndexMap(pWal, walMappingSize(pWal->hdr.mxFrame))) - ){ + if( pWal->readLock==0 ){ volatile WalCkptInfo *pInfo = walCkptInfo(pWal); assert( pInfo->nBackfill==pWal->hdr.mxFrame ); if( pInfo->nBackfill>0 ){ @@ -2237,11 +2185,6 @@ static int walRestartLog(Wal *pWal){ int notUsed; rc = walTryBeginRead(pWal, ¬Used, 1, ++cnt); }while( rc==WAL_RETRY ); - - /* Unmap the wal-index before returning. Otherwise the VFS layer may - ** hold a mutex for the duration of the IO performed by WalFrames(). - */ - walIndexUnmap(pWal); } return rc; } @@ -2267,7 +2210,6 @@ int sqlite3WalFrames( assert( pList ); assert( pWal->writeLock ); - assert( pWal->pWiData==0 ); #if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) { int cnt; for(cnt=0, p=pList; p; p=p->pDirty, cnt++){} @@ -2280,10 +2222,8 @@ int sqlite3WalFrames( ** log file, instead of appending to it at pWal->hdr.mxFrame. */ if( SQLITE_OK!=(rc = walRestartLog(pWal)) ){ - assert( pWal->pWiData==0 ); return rc; } - assert( pWal->pWiData==0 && pWal->readLock>0 ); /* If this is the first frame written into the log, write the WAL ** header to the start of the WAL file. See comments at the top of @@ -2358,7 +2298,6 @@ int sqlite3WalFrames( rc = sqlite3OsSync(pWal->pWalFd, sync_flags); } - assert( pWal->pWiData==0 ); /* Append data to the wal-index. It is not necessary to lock the ** wal-index to do this as the SQLITE_SHM_WRITE lock held on the wal-index @@ -2391,7 +2330,6 @@ int sqlite3WalFrames( } } - walIndexUnmap(pWal); WALTRACE(("WAL%p: frame write %s\n", pWal, rc ? "failed" : "ok")); return rc; } @@ -2412,7 +2350,6 @@ int sqlite3WalCheckpoint( int rc; /* Return code */ int isChanged = 0; /* True if a new wal-index header is loaded */ - assert( pWal->pWiData==0 ); assert( pWal->ckptLock==0 ); WALTRACE(("WAL%p: checkpoint begins\n", pWal)); @@ -2441,7 +2378,6 @@ int sqlite3WalCheckpoint( } /* Release the locks. */ - walIndexUnmap(pWal); walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); pWal->ckptLock = 0; WALTRACE(("WAL%p: checkpoint %s\n", pWal, rc ? "failed" : "ok")); diff --git a/test/permutations.test b/test/permutations.test index 13c1ae4c63..00329acb1f 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -13,6 +13,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +db close # Argument processing. # diff --git a/test/wal2.test b/test/wal2.test index c3f98a4b24..fdbcae7d1e 100644 --- a/test/wal2.test +++ b/test/wal2.test @@ -75,9 +75,14 @@ proc incr_tvfs_hdr {file idx incrval} { # database content. # do_test wal2-1.0 { - proc tvfs_cb {method args} { return SQLITE_OK } + proc tvfs_cb {method filename args} { + set ::filename $filename + return SQLITE_OK + } + testvfs tvfs tvfs script tvfs_cb + tvfs filter xShmOpen sqlite3 db test.db -vfs tvfs sqlite3 db2 test.db -vfs tvfs @@ -123,21 +128,15 @@ foreach {tn iInsert res wal_index_hdr_mod wal_locks} " do_test wal2-1.$tn.1 { execsql { INSERT INTO t1 VALUES($iInsert) } - set ::locks [list] - set ::cb_done 0 - proc tvfs_cb {method args} { - if {$::cb_done == 0 && $method == "xShmGet"} { - set ::cb_done 1 - if {$::wal_index_hdr_mod >= 0} { - incr_tvfs_hdr [lindex $args 0] $::wal_index_hdr_mod 1 - } - } - if {$method == "xShmLock"} { lappend ::locks [lindex $args 2] } + lappend ::locks [lindex $args 2] return SQLITE_OK } - + tvfs filter xShmLock + if {$::wal_index_hdr_mod >= 0} { + incr_tvfs_hdr $::filename $::wal_index_hdr_mod 1 + } execsql { SELECT count(a), sum(a) FROM t1 } db2 } $res @@ -174,8 +173,9 @@ do_test wal2-2.0 { testvfs tvfs tvfs script tvfs_cb + tvfs filter xShmOpen proc tvfs_cb {method args} { - if {$method == "xShmOpen"} { set ::shm_file [lindex $args 0] } + set ::filename [lindex $args 0] return SQLITE_OK } @@ -208,32 +208,28 @@ foreach {tn iInsert res0 res1 wal_index_hdr_mod} { 8 11 {10 55} {11 66} 6 9 12 {11 66} {12 78} 7 } { + tvfs filter xShmLock + do_test wal2-2.$tn.1 { - set oldhdr [set_tvfs_hdr $::shm_file] + set oldhdr [set_tvfs_hdr $::filename] execsql { INSERT INTO t1 VALUES($iInsert) } execsql { SELECT count(a), sum(a) FROM t1 } } $res1 do_test wal2-2.$tn.2 { set ::locks [list] - set ::cb_done 0 proc tvfs_cb {method args} { - if {$::cb_done == 0 && $method == "xShmGet"} { - set ::cb_done 1 - if {$::wal_index_hdr_mod >= 0} { - incr_tvfs_hdr $::shm_file $::wal_index_hdr_mod 1 - } - } - if {$method == "xShmLock"} { - set lock [lindex $args 2] - lappend ::locks $lock - if {$lock == $::WRITER} { - set_tvfs_hdr $::shm_file $::oldhdr - } + set lock [lindex $args 2] + lappend ::locks $lock + if {$lock == $::WRITER} { + set_tvfs_hdr $::filename $::oldhdr } return SQLITE_OK } + if {$::wal_index_hdr_mod >= 0} { + incr_tvfs_hdr $::filename $::wal_index_hdr_mod 1 + } execsql { SELECT count(a), sum(a) FROM t1 } db2 } $res0 @@ -243,21 +239,15 @@ foreach {tn iInsert res0 res1 wal_index_hdr_mod} { do_test wal2-2.$tn.4 { set ::locks [list] - set ::cb_done 0 proc tvfs_cb {method args} { - if {$::cb_done == 0 && $method == "xShmGet"} { - set ::cb_done 1 - if {$::wal_index_hdr_mod >= 0} { - incr_tvfs_hdr $::shm_file $::wal_index_hdr_mod 1 - } - } - if {$method == "xShmLock"} { - set lock [lindex $args 2] - lappend ::locks $lock - } + set lock [lindex $args 2] + lappend ::locks $lock return SQLITE_OK } + if {$::wal_index_hdr_mod >= 0} { + incr_tvfs_hdr $::filename $::wal_index_hdr_mod 1 + } execsql { SELECT count(a), sum(a) FROM t1 } db2 } $res1 } diff --git a/test/wal3.test b/test/wal3.test index 6a5898297d..bc4ffa8b8f 100644 --- a/test/wal3.test +++ b/test/wal3.test @@ -353,7 +353,7 @@ T script method_callback proc method_callback {method args} { if {$method == "xShmBarrier"} { incr ::barrier_count - if {$::barrier_count == 1} { + if {$::barrier_count == 2} { # This code is executed within the xShmBarrier() callback invoked # by the client running recovery as part of writing the recovered # wal-index header. If a second client attempts to access the From 4280eb30049f180f7a49b449f6b0eb554df6540f Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 12 Jun 2010 12:02:35 +0000 Subject: [PATCH 2/8] Fix some problems with handling IO errors on the experimental branch. FossilOrigin-Name: eade8bc238df580412f5cf1b91a91532ae671e46 --- manifest | 17 ++- manifest.uuid | 2 +- src/test_vfs.c | 2 +- src/wal.c | 277 ++++++++++++++++++++++++++----------------------- 4 files changed, 156 insertions(+), 142 deletions(-) diff --git a/manifest b/manifest index 9c9da2bd3d..8fef7bc0d7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\schange\sto\sthe\sxShmXXX\sparts\sof\sthe\sVFS\sinterface. -D 2010-06-11T19:04:21 +C Fix\ssome\sproblems\swith\shandling\sIO\serrors\son\sthe\sexperimental\sbranch. +D 2010-06-12T12:02:36 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in a5cad1f8f3e021356bfcc6c77dc16f6f1952bbc3 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -209,7 +209,7 @@ F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c bbba05c144b5fc4b52ff650a4328027b3fa5fcc6 F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c aa9919c885a1fe53eafc73492f0898ee6c0a0726 -F src/test_vfs.c b83206d2c04b3ba84d8d85420c4c7573c58feba5 +F src/test_vfs.c c3643e985b000e1f7555bd843a508512a33ab60e F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 25ceb0f0a746ea1d0f9553787f3f0a56853cfaeb F src/trigger.c 8927588cb9e6d47f933b53bfe74200fbb504100d @@ -226,7 +226,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c 0aa364734d6daca75771944fc2b4a8f36e63fc4e +F src/wal.c ca3e0bf68c78005dee4e0d44d112e26975476d10 F src/wal.h 4ace25262452d17e7d3ec970c89ee17794004008 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 1c895bef33d0dfc7ed90fb1f74120435d210ea56 @@ -820,10 +820,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 95cc3f6fdec5494560c3cd4439d06870d1c62506 -R 05bb7be31606a33c54980de0a121cf7e -T *branch * experimental -T *sym-experimental * -T -sym-trunk * +P ca68472db01c14a899892007d1cbaff5e86ae193 +R 8c9eec4fce430c14365187b617d56771 U dan -Z 7e046a655e9f9112461a59731e31d3dc +Z 808594079cf8cc56f07144318e40eca2 diff --git a/manifest.uuid b/manifest.uuid index 447d7ce948..a212b4e19c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca68472db01c14a899892007d1cbaff5e86ae193 \ No newline at end of file +eade8bc238df580412f5cf1b91a91532ae671e46 \ No newline at end of file diff --git a/src/test_vfs.c b/src/test_vfs.c index 745a417a7a..4d9df73d79 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -640,7 +640,7 @@ static int tvfsShmRelease(sqlite3_file *pFile){ static void tvfsAllocPage(TestvfsBuffer *p, int iPage, int pgsz){ assert( iPageaPage[iPage]==0 ){ - p->aPage[iPage] = ckalloc(pgsz); + p->aPage[iPage] = (u8 *)ckalloc(pgsz); memset(p->aPage[iPage], 0, pgsz); p->pgsz = pgsz; } diff --git a/src/wal.c b/src/wal.c index a05eec0b82..197d0c0222 100644 --- a/src/wal.c +++ b/src/wal.c @@ -451,10 +451,16 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ ** Return a pointer to the WalCkptInfo structure in the wal-index. */ static volatile WalCkptInfo *walCkptInfo(Wal *pWal){ - volatile u32 *page1 = 0; - walIndexPage(pWal, 0, &page1); - assert( page1 ); - return (volatile WalCkptInfo*)&page1[sizeof(WalIndexHdr)/2]; + assert( pWal->nWiData>0 && pWal->apWiData[0] ); + return (volatile WalCkptInfo*)&(pWal->apWiData[0][sizeof(WalIndexHdr)/2]); +} + +/* +** Return a pointer to the WalIndexHdr structure in the wal-index. +*/ +static volatile WalIndexHdr *walIndexHdr(Wal *pWal){ + assert( pWal->nWiData>0 && pWal->apWiData[0] ); + return (volatile WalIndexHdr*)pWal->apWiData[0]; } /* @@ -549,16 +555,15 @@ static void walChecksumBytes( ** The checksum on pWal->hdr is updated before it is written. */ static void walIndexWriteHdr(Wal *pWal){ - WalIndexHdr *aHdr; + volatile WalIndexHdr *aHdr = walIndexHdr(pWal); + const int nCksum = offsetof(WalIndexHdr, aCksum); assert( pWal->writeLock ); pWal->hdr.isInit = 1; - walChecksumBytes(1, (u8*)&pWal->hdr, offsetof(WalIndexHdr, aCksum), - 0, pWal->hdr.aCksum); - walIndexPage(pWal, 0, (volatile u32 **)&aHdr); - memcpy(&aHdr[1], &pWal->hdr, sizeof(WalIndexHdr)); + walChecksumBytes(1, (u8*)&pWal->hdr, nCksum, 0, pWal->hdr.aCksum); + memcpy((void *)&aHdr[1], (void *)&pWal->hdr, sizeof(WalIndexHdr)); sqlite3OsShmBarrier(pWal->pDbFd); - memcpy(&aHdr[0], &pWal->hdr, sizeof(WalIndexHdr)); + memcpy((void *)&aHdr[0], (void *)&pWal->hdr, sizeof(WalIndexHdr)); } /* @@ -727,33 +732,60 @@ static int walNextHash(int iPriorHash){ return (iPriorHash+1)&(HASHTABLE_NSLOT-1); } -static void walHashGet( +/* +** Return pointers to the hash table and page number array stored on +** page iHash of the wal-index. The wal-index is broken into 32KB pages +** numbered starting from 0. +** +** Set output variable *paHash to point to the start of the hash table +** in the wal-index file. Set *piZero to one less than the frame +** number of the first frame indexed by this hash table. If a +** slot in the hash table is set to N, it refers to frame number +** (*piZero+N) in the log. +** +** Finally, set *paPgno such that for all frames F between (*piZero+1) and +** (*piZero+HASHTABLE_NPAGE), (*paPgno)[F] is the database page number +** associated with frame F. +*/ +static int walHashGet( Wal *pWal, /* WAL handle */ int iHash, /* Find the iHash'th table */ volatile HASHTABLE_DATATYPE **paHash, /* OUT: Pointer to hash index */ volatile u32 **paPgno, /* OUT: Pointer to page number array */ u32 *piZero /* OUT: Frame associated with *paPgno[0] */ ){ - u32 iZero; + int rc; /* Return code */ volatile u32 *aPgno; - volatile HASHTABLE_DATATYPE *aHash; - walIndexPage(pWal, iHash, &aPgno); - aHash = (volatile HASHTABLE_DATATYPE *)&aPgno[HASHTABLE_NPAGE]; + rc = walIndexPage(pWal, iHash, &aPgno); + assert( rc==SQLITE_OK || iHash>0 ); - if( iHash==0 ){ - aPgno = &aPgno[WALINDEX_HDR_SIZE/sizeof(u32)-1]; - iZero = 0; - }else{ - iZero = HASHTABLE_NPAGE_ONE + (iHash-1)*HASHTABLE_NPAGE; - aPgno = &aPgno[-1*iZero-1]; + if( rc==SQLITE_OK ){ + u32 iZero; + volatile HASHTABLE_DATATYPE *aHash; + + aHash = (volatile HASHTABLE_DATATYPE *)&aPgno[HASHTABLE_NPAGE]; + if( iHash==0 ){ + aPgno = &aPgno[WALINDEX_HDR_SIZE/sizeof(u32)-1]; + iZero = 0; + }else{ + iZero = HASHTABLE_NPAGE_ONE + (iHash-1)*HASHTABLE_NPAGE; + aPgno = &aPgno[-1*iZero-1]; + } + + *paPgno = aPgno; + *paHash = aHash; + *piZero = iZero; } - - *paPgno = aPgno; - *paHash = aHash; - *piZero = iZero; + return rc; } +/* +** Return the number of the wal-index page that contains the hash-table +** and page-number array that contain entries corresponding to WAL frame +** iFrame. The wal-index is broken up into 32KB pages. Wal-index pages +** are numbered starting from 0. +*/ static int walFramePage(u32 iFrame){ int iHash = (iFrame+HASHTABLE_NPAGE-HASHTABLE_NPAGE_ONE-1) / HASHTABLE_NPAGE; assert( (iHash==0 || iFrame>HASHTABLE_NPAGE_ONE) @@ -776,31 +808,6 @@ static u32 walFramePgno(Wal *pWal, u32 iFrame){ return pWal->apWiData[iHash][(iFrame-1-HASHTABLE_NPAGE_ONE)%HASHTABLE_NPAGE]; } -/* -** Find the hash table and (section of the) page number array used to -** store data for WAL frame iFrame. -** -** Set output variable *paHash to point to the start of the hash table -** in the wal-index file. Set *piZero to one less than the frame -** number of the first frame indexed by this hash table. If a -** slot in the hash table is set to N, it refers to frame number -** (*piZero+N) in the log. -** -** Finally, set *paPgno such that for all frames F between (*piZero+1) and -** (*piZero+HASHTABLE_NPAGE), (*paPgno)[F] is the database page number -** associated with frame F. -*/ -static void walHashFind( - Wal *pWal, /* WAL handle */ - u32 iFrame, /* Find the hash table indexing this frame */ - volatile HASHTABLE_DATATYPE **paHash, /* OUT: Pointer to hash index */ - volatile u32 **paPgno, /* OUT: Pointer to page number array */ - u32 *piZero /* OUT: Frame associated with *paPgno[0] */ -){ - int iHash = walFramePage(iFrame); - walHashGet(pWal, iHash, paHash, paPgno, piZero); -} - /* ** Remove entries from the hash table that point to WAL slots greater ** than pWal->hdr.mxFrame. @@ -815,7 +822,7 @@ static void walHashFind( */ static void walCleanupHash(Wal *pWal){ volatile HASHTABLE_DATATYPE *aHash; /* Pointer to hash table to clear */ - volatile u32 *aPgno; /* Unused return from walHashFind() */ + volatile u32 *aPgno; /* Page number array for hash table */ u32 iZero; /* frame == (aHash[x]+iZero) */ int iLimit = 0; /* Zero values greater than this */ int nByte; /* Number of bytes to zero in aPgno[] */ @@ -826,22 +833,32 @@ static void walCleanupHash(Wal *pWal){ testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE ); testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE+1 ); - walHashFind(pWal, pWal->hdr.mxFrame+1, &aHash, &aPgno, &iZero); - if( iZero!=pWal->hdr.mxFrame ){ - iLimit = pWal->hdr.mxFrame - iZero; - assert( iLimit>0 ); - for(i=0; iiLimit ){ - aHash[i] = 0; - } + if( pWal->hdr.mxFrame==0 ) return; + + /* Obtain pointers to the hash-table and page-number array containing + ** the entry that corresponds to frame pWal->hdr.mxFrame. It is guaranteed + ** that the page said hash-table and array reside on is already mapped. + */ + assert( pWal->nWiData>walFramePage(pWal->hdr.mxFrame) ); + assert( pWal->apWiData[walFramePage(pWal->hdr.mxFrame)] ); + walHashGet(pWal, walFramePage(pWal->hdr.mxFrame), &aHash, &aPgno, &iZero); + + /* Zero all hash-table entries that correspond to frame numbers greater + ** than pWal->hdr.mxFrame. + */ + iLimit = pWal->hdr.mxFrame - iZero; + assert( iLimit>0 ); + for(i=0; iiLimit ){ + aHash[i] = 0; } - - /* Zero the entries in the aPgno array that correspond to frames with - ** frame numbers greater than pWal->hdr.mxFrame. - */ - nByte = ((char *)aHash - (char *)&aPgno[pWal->hdr.mxFrame+1]); - memset((void *)&aPgno[pWal->hdr.mxFrame+1], 0, nByte); } + + /* Zero the entries in the aPgno array that correspond to frames with + ** frame numbers greater than pWal->hdr.mxFrame. + */ + nByte = ((char *)aHash - (char *)&aPgno[pWal->hdr.mxFrame+1]); + memset((void *)&aPgno[pWal->hdr.mxFrame+1], 0, nByte); #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT /* Verify that the every entry in the mapping region is still reachable @@ -866,43 +883,48 @@ static void walCleanupHash(Wal *pWal){ ** pPage into WAL frame iFrame. */ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ - int rc = SQLITE_OK; /* Return code */ + int rc; /* Return code */ + u32 iZero; /* One less than frame number of aPgno[1] */ + volatile u32 *aPgno; /* Page number array */ + volatile HASHTABLE_DATATYPE *aHash; /* Hash table */ - /* Assuming the wal-index file was successfully mapped, find the hash - ** table and section of of the page number array that pertain to frame - ** iFrame of the WAL. Then populate the page number array and the hash - ** table entry. + rc = walHashGet(pWal, walFramePage(iFrame), &aHash, &aPgno, &iZero); + + /* Assuming the wal-index file was successfully mapped, populate the + ** page number array and hash table entry. */ if( rc==SQLITE_OK ){ int iKey; /* Hash table key */ - u32 iZero; /* One less than frame number of aPgno[1] */ - volatile u32 *aPgno; /* Page number array */ - volatile HASHTABLE_DATATYPE *aHash; /* Hash table */ - int idx; /* Value to write to hash-table slot */ - TESTONLY( int nCollide = 0; /* Number of hash collisions */ ) + int idx; /* Value to write to hash-table slot */ + TESTONLY( int nCollide = 0; /* Number of hash collisions */ ) - walHashFind(pWal, iFrame, &aHash, &aPgno, &iZero); idx = iFrame - iZero; + assert( idx <= HASHTABLE_NSLOT/2 + 1 ); + + /* If this is the first entry to be added to this hash-table, zero the + ** entire hash table and aPgno[] array before proceding. + */ if( idx==1 ){ int nByte = (u8 *)&aHash[HASHTABLE_NSLOT] - (u8 *)&aPgno[1+iZero]; memset((void*)&aPgno[1+iZero], 0, nByte); } - assert( idx <= HASHTABLE_NSLOT/2 + 1 ); + /* If the entry in aPgno[] is already set, then the previous writer + ** must have exited unexpectedly in the middle of a transaction (after + ** writing one or more dirty pages to the WAL to free up memory). + ** Remove the remnants of that writers uncommitted transaction from + ** the hash-table before writing any new entries. + */ if( aPgno[iFrame] ){ - /* If the entry in aPgno[] is already set, then the previous writer - ** must have exited unexpectedly in the middle of a transaction (after - ** writing one or more dirty pages to the WAL to free up memory). - ** Remove the remnants of that writers uncommitted transaction from - ** the hash-table before writing any new entries. - */ walCleanupHash(pWal); assert( !aPgno[iFrame] ); } - aPgno[iFrame] = iPage; + + /* Write the aPgno[] array entry and the hash-table slot. */ for(iKey=walHash(iPage); aHash[iKey]; iKey=walNextHash(iKey)){ assert( nCollide++ < idx ); } + aPgno[iFrame] = iPage; aHash[iKey] = idx; #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT @@ -1306,20 +1328,19 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ aSpace = (HASHTABLE_DATATYPE *)&p->aSegment[nSegment]; aTmp = &aSpace[HASHTABLE_NPAGE*nSegment]; for(i=0; ihdr.mxFrame; - walIndexPage(pWal, 0, (volatile u32 **)&pHdr); pInfo = walCkptInfo(pWal); - assert( pInfo==walCkptInfo(pWal) ); for(i=1; iaReadMark[i]; if( mxSafeFrame>=y ){ @@ -1455,7 +1473,7 @@ static int walCheckpoint( /* If work was actually accomplished... */ if( rc==SQLITE_OK ){ - if( mxSafeFrame==pHdr[0].mxFrame ){ + if( mxSafeFrame==walIndexHdr(pWal)->mxFrame ){ rc = sqlite3OsTruncate(pWal->pDbFd, ((i64)pWal->hdr.nPage*(i64)szPage)); if( rc==SQLITE_OK && sync_flags ){ rc = sqlite3OsSync(pWal->pDbFd, sync_flags); @@ -1540,17 +1558,12 @@ int sqlite3WalClose( ** is read successfully and the checksum verified, return zero. */ int walIndexTryHdr(Wal *pWal, int *pChanged){ - u32 aCksum[2]; /* Checksum on the header content */ - WalIndexHdr h1, h2; /* Two copies of the header content */ - WalIndexHdr *aHdr; /* Header in shared memory */ - volatile u32 *page1 = 0; + u32 aCksum[2]; /* Checksum on the header content */ + WalIndexHdr h1, h2; /* Two copies of the header content */ + WalIndexHdr volatile *aHdr; /* Header in shared memory */ - walIndexPage(pWal, 0, &page1); - if( !page1 ){ - /* The wal-index is not large enough to hold the header, then assume - ** header is invalid. */ - return 1; - } + /* The first page of the wal-index must be mapped at this point. */ + assert( pWal->nWiData>0 && pWal->apWiData[0] ); /* Read the header. This might happen currently with a write to the ** same area of shared memory on a different CPU in a SMP, @@ -1562,10 +1575,10 @@ int walIndexTryHdr(Wal *pWal, int *pChanged){ ** Memory barriers are used to prevent the compiler or the hardware from ** reordering the reads and writes. */ - aHdr = (WalIndexHdr*)page1; - memcpy(&h1, &aHdr[0], sizeof(h1)); + aHdr = walIndexHdr(pWal); + memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); sqlite3OsShmBarrier(pWal->pDbFd); - memcpy(&h2, &aHdr[1], sizeof(h2)); + memcpy(&h2, (void *)&aHdr[1], sizeof(h2)); if( memcmp(&h1, &h2, sizeof(h1))!=0 ){ return 1; /* Dirty read */ @@ -1609,27 +1622,32 @@ int walIndexTryHdr(Wal *pWal, int *pChanged){ static int walIndexReadHdr(Wal *pWal, int *pChanged){ int rc; /* Return code */ int badHdr; /* True if a header read failed */ - volatile u32 *dummy; + volatile u32 *page0; + /* Ensure that page 0 of the wal-index (the page that contains the + ** wal-index header) is mapped. Return early if an error occurs here. + */ assert( pChanged ); - rc = walIndexPage(pWal, 0, &dummy); + rc = walIndexPage(pWal, 0, &page0); if( rc!=SQLITE_OK ){ return rc; - } + }; + assert( page0 || pWal->writeLock==0 ); - /* Try once to read the header straight out. This works most of the - ** time. + /* If the first page of the wal-index has been mapped, try to read the + ** wal-index header immediately, without holding any lock. This usually + ** works, but may fail if the wal-index header is corrupt or currently + ** being modified by another user. */ - badHdr = walIndexTryHdr(pWal, pChanged); + badHdr = (page0 ? walIndexTryHdr(pWal, pChanged) : 1); /* If the first attempt failed, it might have been due to a race ** with a writer. So get a WRITE lock and try again. */ assert( badHdr==0 || pWal->writeLock==0 ); - if( badHdr ){ - rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); - if( rc==SQLITE_OK ){ - pWal->writeLock = 1; + if( badHdr && SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) ){ + pWal->writeLock = 1; + if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ badHdr = walIndexTryHdr(pWal, pChanged); if( badHdr ){ /* If the wal-index header is still malformed even while holding @@ -1639,9 +1657,9 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ rc = walIndexRecover(pWal); *pChanged = 1; } - walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); - pWal->writeLock = 0; } + pWal->writeLock = 0; + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); } return rc; @@ -1682,7 +1700,6 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ ** WAL_READ_LOCK() while changing values. */ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ - volatile WalIndexHdr *pHdr; /* Header of the wal-index */ volatile WalCkptInfo *pInfo; /* Checkpoint information in wal-index */ u32 mxReadMark; /* Largest aReadMark[] value */ int mxI; /* Index of largest aReadMark[] value */ @@ -1721,9 +1738,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ return rc; } - walIndexPage(pWal, 0, (volatile u32 **)&pHdr); pInfo = walCkptInfo(pWal); - assert( pInfo==(volatile WalCkptInfo *)&pHdr[2] ); if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame ){ /* The WAL has been completely backfilled (or it is empty). ** and can be safely ignored. @@ -1731,7 +1746,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ rc = walLockShared(pWal, WAL_READ_LOCK(0)); sqlite3OsShmBarrier(pWal->pDbFd); if( rc==SQLITE_OK ){ - if( memcmp((void *)pHdr, &pWal->hdr, sizeof(WalIndexHdr)) ){ + if( memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ /* It is not safe to allow the reader to continue here if frames ** may have been appended to the log before READ_LOCK(0) was obtained. ** When holding READ_LOCK(0), the reader ignores the entire log file, @@ -1825,7 +1840,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ */ sqlite3OsShmBarrier(pWal->pDbFd); if( pInfo->aReadMark[mxI]!=mxReadMark - || memcmp((void *)pHdr, &pWal->hdr, sizeof(WalIndexHdr)) + || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ walUnlockShared(pWal, WAL_READ_LOCK(mxI)); return WAL_RETRY; @@ -1935,8 +1950,12 @@ int sqlite3WalRead( volatile u32 *aPgno; /* Pointer to array of page numbers */ u32 iZero; /* Frame number corresponding to aPgno[0] */ int iKey; /* Hash slot index */ + int rc; - walHashGet(pWal, iHash, &aHash, &aPgno, &iZero); + rc = walHashGet(pWal, iHash, &aHash, &aPgno, &iZero); + if( rc!=SQLITE_OK ){ + return rc; + } for(iKey=walHash(pgno); aHash[iKey]; iKey=walNextHash(iKey)){ u32 iFrame = aHash[iKey] + iZero; if( iFrame<=iLast && aPgno[iFrame]==pgno ){ @@ -2001,7 +2020,6 @@ void sqlite3WalDbsize(Wal *pWal, Pgno *pPgno){ */ int sqlite3WalBeginWriteTransaction(Wal *pWal){ int rc; - volatile u32 *page1; /* Cannot start a write transaction without first holding a read ** transaction. */ @@ -2020,8 +2038,7 @@ int sqlite3WalBeginWriteTransaction(Wal *pWal){ ** time the read transaction on this connection was started, then ** the write is disallowed. */ - walIndexPage(pWal, 0, &page1); - if( memcmp(&pWal->hdr, (void*)page1, sizeof(WalIndexHdr))!=0 ){ + if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){ walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); pWal->writeLock = 0; rc = SQLITE_BUSY; From 5d656852f700427e396d101a938ca6f989206a5c Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 14 Jun 2010 07:53:26 +0000 Subject: [PATCH 3/8] Add some fault-injection tests to improve coverage. FossilOrigin-Name: 37b26d125f4b1d8e75bb38800fefd145611f94aa --- manifest | 16 ++++----- manifest.uuid | 2 +- src/test_vfs.c | 21 +++++++++-- src/wal.c | 66 ++++++++++++++++++----------------- test/walfault.test | 86 ++++++++++++++++++++++++++++++++++++++++++++-- 5 files changed, 144 insertions(+), 47 deletions(-) diff --git a/manifest b/manifest index 8fef7bc0d7..0ef8f22436 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sproblems\swith\shandling\sIO\serrors\son\sthe\sexperimental\sbranch. -D 2010-06-12T12:02:36 +C Add\ssome\sfault-injection\stests\sto\simprove\scoverage. +D 2010-06-14T07:53:26 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in a5cad1f8f3e021356bfcc6c77dc16f6f1952bbc3 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -209,7 +209,7 @@ F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c bbba05c144b5fc4b52ff650a4328027b3fa5fcc6 F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c aa9919c885a1fe53eafc73492f0898ee6c0a0726 -F src/test_vfs.c c3643e985b000e1f7555bd843a508512a33ab60e +F src/test_vfs.c 15bddcddf6b1bf6360130e09aee950f5f563d5f3 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 25ceb0f0a746ea1d0f9553787f3f0a56853cfaeb F src/trigger.c 8927588cb9e6d47f933b53bfe74200fbb504100d @@ -226,7 +226,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c ca3e0bf68c78005dee4e0d44d112e26975476d10 +F src/wal.c 75c55049c0923418f10bbef842b846374637b595 F src/wal.h 4ace25262452d17e7d3ec970c89ee17794004008 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 1c895bef33d0dfc7ed90fb1f74120435d210ea56 @@ -772,7 +772,7 @@ F test/walbak.test e7650a26eb4b8abeca9b145b1af1e63026dde432 F test/walcksum.test 4efa8fb88c32bed8288ea4385a9cc113a5c8f0bf F test/walcrash.test f6d5fb2bb108876f04848720a488065d9deef69f F test/walcrash2.test 14585ad1a2c85da2de721caa3b4deeea55213008 -F test/walfault.test 0f9524cedbba0e5d48b58cabdc2dd7ae2c375476 +F test/walfault.test f6c1d0eade34fd0102e083726092d9bebd48bf7b F test/walhook.test 67e675127f4acb72f061a12667ce6e5460b06b78 F test/walmode.test 6ca9d710cc9f6545b913abcded6d6b0b15641048 F test/walslow.test d21625e2e99e11c032ce949e8a94661576548933 @@ -820,7 +820,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P ca68472db01c14a899892007d1cbaff5e86ae193 -R 8c9eec4fce430c14365187b617d56771 +P eade8bc238df580412f5cf1b91a91532ae671e46 +R d24014573c5813c3e5f096c2f4031503 U dan -Z 808594079cf8cc56f07144318e40eca2 +Z e43653a42fa5b8fa5bb9c7db25e1bbd8 diff --git a/manifest.uuid b/manifest.uuid index a212b4e19c..c8b6f5bf05 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eade8bc238df580412f5cf1b91a91532ae671e46 \ No newline at end of file +37b26d125f4b1d8e75bb38800fefd145611f94aa \ No newline at end of file diff --git a/src/test_vfs.c b/src/test_vfs.c index 4d9df73d79..9b212099c6 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -659,6 +659,7 @@ static int tvfsShmPage( if( p->pScript && p->mask&TESTVFS_SHMPAGE_MASK ){ Tcl_Obj *pArg = Tcl_NewObj(); + Tcl_IncrRefCount(pArg); Tcl_ListObjAppendElement(p->interp, pArg, Tcl_NewIntObj(iPage)); Tcl_ListObjAppendElement(p->interp, pArg, Tcl_NewIntObj(pgsz)); Tcl_ListObjAppendElement(p->interp, pArg, Tcl_NewIntObj(isWrite)); @@ -666,6 +667,7 @@ static int tvfsShmPage( Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, pArg ); tvfsResultCode(p, &rc); + Tcl_DecrRefCount(pArg); } if( rc==SQLITE_OK && p->mask&TESTVFS_SHMPAGE_MASK && tvfsInjectIoerr(p) ){ rc = SQLITE_IOERR; @@ -828,12 +830,17 @@ static int testvfs_obj_cmd( Tcl_WrongNumArgs(interp, 2, objv, "FILE ?VALUE?"); return TCL_ERROR; } - zName = Tcl_GetString(objv[2]); + zName = ckalloc(p->pParent->mxPathname); + p->pParent->xFullPathname( + p->pParent, Tcl_GetString(objv[2]), + p->pParent->mxPathname, zName + ); for(pBuffer=p->pBuffer; pBuffer; pBuffer=pBuffer->pNext){ if( 0==strcmp(pBuffer->zFile, zName) ) break; } + ckfree(zName); if( !pBuffer ){ - Tcl_AppendResult(interp, "no such file: ", zName, 0); + Tcl_AppendResult(interp, "no such file: ", Tcl_GetString(objv[2]), 0); return TCL_ERROR; } if( objc==4 ){ @@ -870,6 +877,7 @@ static int testvfs_obj_cmd( { "xShmLock", TESTVFS_SHMLOCK_MASK }, { "xShmBarrier", TESTVFS_SHMBARRIER_MASK }, { "xShmClose", TESTVFS_SHMCLOSE_MASK }, + { "xShmPage", TESTVFS_SHMPAGE_MASK }, { "xSync", TESTVFS_SYNC_MASK }, { "xOpen", TESTVFS_OPEN_MASK }, }; @@ -911,6 +919,7 @@ static int testvfs_obj_cmd( ckfree((char *)p->apScript); p->apScript = 0; p->nScript = 0; + p->pScript = 0; } Tcl_GetStringFromObj(objv[2], &nByte); if( nByte>0 ){ @@ -1083,6 +1092,13 @@ static int testvfs_cmd( p = (Testvfs *)ckalloc(nByte); memset(p, 0, nByte); + /* Create the new object command before querying SQLite for a default VFS + ** to use for 'real' IO operations. This is because creating the new VFS + ** may delete an existing [testvfs] VFS of the same name. If such a VFS + ** is currently the default, the new [testvfs] may end up calling the + ** methods of a deleted object. + */ + Tcl_CreateObjCommand(interp, zVfs, testvfs_obj_cmd, p, testvfs_obj_del); p->pParent = sqlite3_vfs_find(0); p->interp = interp; @@ -1099,7 +1115,6 @@ static int testvfs_cmd( p->isNoshm = isNoshm; p->mask = TESTVFS_ALL_MASK; - Tcl_CreateObjCommand(interp, zVfs, testvfs_obj_cmd, p, testvfs_obj_del); sqlite3_vfs_register(pVfs, isDefault); return TCL_OK; diff --git a/src/wal.c b/src/wal.c index 197d0c0222..2f35c20f96 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1281,6 +1281,13 @@ static void walMergesort( #endif } +/* +** Free an iterator allocated by walIteratorInit(). +*/ +static void walIteratorFree(WalIterator *p){ + sqlite3_free(p); +} + /* ** Map the wal-index into memory owned by this thread, if it is not ** mapped already. Then construct a WalInterator object that can be @@ -1337,6 +1344,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ rc = walHashGet(pWal, i, &aHash, &aPgno, &iZero); if( rc!=SQLITE_OK ){ + walIteratorFree(p); return rc; } nEntry = ((i+1)==nSegment)?iLast-iZero:(u32 *)aHash-(u32 *)&aPgno[iZero+1]; @@ -1361,13 +1369,6 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ return SQLITE_OK ; } -/* -** Free an iterator allocated by walIteratorInit(). -*/ -static void walIteratorFree(WalIterator *p){ - sqlite3_free(p); -} - /* ** Copy as much content as we can from the WAL back into the database file ** in response to an sqlite3_wal_checkpoint() request or the equivalent. @@ -2076,30 +2077,33 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ Pgno iMax = pWal->hdr.mxFrame; Pgno iFrame; - rc = walIndexReadHdr(pWal, &unused); - if( rc==SQLITE_OK ){ - for(iFrame=pWal->hdr.mxFrame+1; - ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; - iFrame++ - ){ - /* This call cannot fail. Unless the page for which the page number - ** is passed as the second argument is (a) in the cache and - ** (b) has an outstanding reference, then xUndo is either a no-op - ** (if (a) is false) or simply expels the page from the cache (if (b) - ** is false). - ** - ** If the upper layer is doing a rollback, it is guaranteed that there - ** are no outstanding references to any page other than page 1. And - ** page 1 is never written to the log until the transaction is - ** committed. As a result, the call to xUndo may not fail. - */ - assert( pWal->writeLock ); - assert( walFramePgno(pWal, iFrame)!=1 ); - rc = xUndo(pUndoCtx, walFramePgno(pWal, iFrame)); - } - walCleanupHash(pWal); + /* Restore the clients cache of the wal-index header to the state it + ** was in before the client began writing to the database. + */ + memcpy(&pWal->hdr, walIndexHdr(pWal), sizeof(WalIndexHdr)); + + for(iFrame=pWal->hdr.mxFrame+1; + ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; + iFrame++ + ){ + /* This call cannot fail. Unless the page for which the page number + ** is passed as the second argument is (a) in the cache and + ** (b) has an outstanding reference, then xUndo is either a no-op + ** (if (a) is false) or simply expels the page from the cache (if (b) + ** is false). + ** + ** If the upper layer is doing a rollback, it is guaranteed that there + ** are no outstanding references to any page other than page 1. And + ** page 1 is never written to the log until the transaction is + ** committed. As a result, the call to xUndo may not fail. + */ + assert( pWal->writeLock ); + assert( walFramePgno(pWal, iFrame)!=1 ); + rc = xUndo(pUndoCtx, walFramePgno(pWal, iFrame)); } + walCleanupHash(pWal); } + assert( rc==SQLITE_OK ); return rc; } @@ -2142,9 +2146,7 @@ int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ pWal->hdr.mxFrame = aWalData[0]; pWal->hdr.aFrameCksum[0] = aWalData[1]; pWal->hdr.aFrameCksum[1] = aWalData[2]; - if( rc==SQLITE_OK ){ - walCleanupHash(pWal); - } + walCleanupHash(pWal); } return rc; diff --git a/test/walfault.test b/test/walfault.test index ad7bf97cb2..3b64f8dc73 100644 --- a/test/walfault.test +++ b/test/walfault.test @@ -120,6 +120,7 @@ do_faultsim_test walfault-3 -prep { faultsim_test_result {0 {}} } + #-------------------------------------------------------------------------- # faultsim_delete_and_reopen @@ -152,7 +153,7 @@ do_test walfault-5-pre-1 { do_faultsim_test walfault-5 -faults shmerr* -prep { faultsim_restore_and_reopen execsql { PRAGMA wal_autocheckpoint = 0 } - shmfault filter xShmSize + shmfault filter xShmPage } -body { execsql { CREATE TABLE t1(x); @@ -211,7 +212,7 @@ do_test walfault-6-pre-1 { } {} do_faultsim_test walfault-6 -faults shmerr* -prep { faultsim_restore_and_reopen - shmfault filter xShmSize + shmfault filter xShmPage } -body { execsql { SELECT count(*) FROM t1 } } -test { @@ -326,7 +327,7 @@ do_test walfault-10-pre1 { faultsim_delete_and_reopen execsql { PRAGMA journal_mode = WAL; - PRAGMA wal_checkpoint = 0; + PRAGMA wal_autocheckpoint = 0; CREATE TABLE z(zz INTEGER PRIMARY KEY, zzz BLOB); CREATE INDEX zzzz ON z(zzz); INSERT INTO z VALUES(NULL, randomblob(800)); @@ -363,5 +364,84 @@ do_faultsim_test walfault-10 -prep { if {$n != "64 51200"} { error "Incorrect data: $n" } } +#-------------------------------------------------------------------------- +# Test fault injection while checkpointing a large WAL file, if the +# checkpoint is the first operation run after opening the database. +# This means that some of the required wal-index pages are mapped as part of +# the checkpoint process, which means there are a few more opportunities +# for IO errors. +# +# To speed this up, IO errors are only simulated within xShmPage() calls. +# +do_test walfault-11-pre-1 { + sqlite3 db test.db + execsql { + PRAGMA journal_mode = WAL; + PRAGMA wal_autocheckpoint = 0; + BEGIN; + CREATE TABLE abc(a PRIMARY KEY); + INSERT INTO abc VALUES(randomblob(1500)); + INSERT INTO abc VALUES(randomblob(1500)); + INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 4 + INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 8 + INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 16 + INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 32 + INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 64 + INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 128 + INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 256 + INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 512 + INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 1024 + INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 2048 + INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 4096 + COMMIT; + } + faultsim_save_and_close +} {} +do_faultsim_test walfault-11 -faults shmerr* -prep { + catch { db2 close } + faultsim_restore_and_reopen + shmfault filter xShmPage +} -body { + db eval { SELECT count(*) FROM abc } + sqlite3 db2 test.db -vfs shmfault + db2 eval { PRAGMA wal_checkpoint } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# Test the handling of the various IO/OOM/SHM errors that may occur during +# a log recovery operation undertaken as part of a call to +# sqlite3_wal_checkpoint(). +# +do_test walfault-12-pre-1 { + faultsim_delete_and_reopen + execsql { + PRAGMA journal_mode = WAL; + PRAGMA wal_autocheckpoint = 0; + BEGIN; + CREATE TABLE abc(a PRIMARY KEY); + INSERT INTO abc VALUES(randomblob(1500)); + INSERT INTO abc VALUES(randomblob(1500)); + COMMIT; + } + faultsim_save_and_close +} {} +do_faultsim_test walfault-12 -prep { + if {[info commands shmfault] == ""} { + testvfs shmfault -default true + } + faultsim_restore_and_reopen + db eval { SELECT * FROM sqlite_master } + shmfault shm test.db [string repeat "\000" 40] +} -body { + set rc [sqlite3_wal_checkpoint db] + if {$rc != "SQLITE_OK"} { error [sqlite3_errmsg db] } +} -test { + db close + faultsim_test_result {0 {}} +} + + finish_test From 067f3165d8df022c1b23683dbca59e0baecb6f7a Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 14 Jun 2010 10:30:12 +0000 Subject: [PATCH 4/8] Add the xShmPage method to the "crash" vfs in test6.c. FossilOrigin-Name: 1008f536440840da7d56c01ec147a25295fd1fd4 --- manifest | 14 ++++---- manifest.uuid | 2 +- src/test6.c | 12 ++++++- src/wal.c | 88 +++++++++++++++++++++++++++------------------------ 4 files changed, 65 insertions(+), 51 deletions(-) diff --git a/manifest b/manifest index 0ef8f22436..1842e90493 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssome\sfault-injection\stests\sto\simprove\scoverage. -D 2010-06-14T07:53:26 +C Add\sthe\sxShmPage\smethod\sto\sthe\s"crash"\svfs\sin\stest6.c. +D 2010-06-14T10:30:12 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in a5cad1f8f3e021356bfcc6c77dc16f6f1952bbc3 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -182,7 +182,7 @@ F src/test2.c e3f564ab1e9fd0b47b0c9e23e7054e38bf0836cf F src/test3.c 4c21700c73a890a47fc685c1097bfb661346ac94 F src/test4.c ad03bb987ddedce928f4258c1e7fa4109a73497d F src/test5.c cc55900118fa4add8ec9cf69fc4225a4662f76b1 -F src/test6.c d3e1a771a7b445ec771e64f2af70df5285da8e4c +F src/test6.c 1b312541883ec9413734035deada4794ee339e67 F src/test7.c 3f2d63e4ccf97f8c2cf1a7fa0a3c8e2e2a354e6e F src/test8.c f959db9a22d882013b64c92753fa793b2ce3bdea F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 @@ -226,7 +226,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c 75c55049c0923418f10bbef842b846374637b595 +F src/wal.c 7cf566b5c1cb9e1094bbbd9b188605ed83330e40 F src/wal.h 4ace25262452d17e7d3ec970c89ee17794004008 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 1c895bef33d0dfc7ed90fb1f74120435d210ea56 @@ -820,7 +820,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P eade8bc238df580412f5cf1b91a91532ae671e46 -R d24014573c5813c3e5f096c2f4031503 +P 37b26d125f4b1d8e75bb38800fefd145611f94aa +R dea40a6fff442af044dc9e87dfed9d5b U dan -Z e43653a42fa5b8fa5bb9c7db25e1bbd8 +Z 06678e8838f3de5fe68cbedec2758b53 diff --git a/manifest.uuid b/manifest.uuid index c8b6f5bf05..13d1c158aa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -37b26d125f4b1d8e75bb38800fefd145611f94aa \ No newline at end of file +1008f536440840da7d56c01ec147a25295fd1fd4 \ No newline at end of file diff --git a/src/test6.c b/src/test6.c index 1dded82efc..7acc1196e3 100644 --- a/src/test6.c +++ b/src/test6.c @@ -549,6 +549,15 @@ static void cfShmBarrier(sqlite3_file *pFile){ static int cfShmClose(sqlite3_file *pFile, int delFlag){ return sqlite3OsShmClose(((CrashFile*)pFile)->pRealFile, delFlag); } +static int cfShmPage( + sqlite3_file *pFile, /* Handle open on database file */ + int iPage, /* Page to retrieve */ + int pgsz, /* Size of pages */ + int w, /* True to extend file if necessary */ + void volatile **pp /* OUT: Mapped memory */ +){ + return sqlite3OsShmPage(((CrashFile*)pFile)->pRealFile, iPage, pgsz, w, pp); +} static const sqlite3_io_methods CrashFileVtab = { @@ -571,7 +580,8 @@ static const sqlite3_io_methods CrashFileVtab = { cfShmRelease, /* xShmRelease */ cfShmLock, /* xShmLock */ cfShmBarrier, /* xShmBarrier */ - cfShmClose /* xShmClose */ + cfShmClose, /* xShmClose */ + cfShmPage /* xShmPage */ }; /* diff --git a/src/wal.c b/src/wal.c index 2f35c20f96..e4be0abf62 100644 --- a/src/wal.c +++ b/src/wal.c @@ -386,6 +386,12 @@ struct Wal { #endif }; +/* +** Each page of the wal-index mapping contains a hash-table made up of +** an array of HASHTABLE_NSLOT elements of the following type. +*/ +typedef u16 ht_slot; + /* ** Define the parameters of the hash tables in the wal-index file. There ** is a hash-table following every HASHTABLE_NPAGE page numbers in the @@ -394,24 +400,24 @@ struct Wal { ** Changing any of these constants will alter the wal-index format and ** create incompatibilities. */ -#define HASHTABLE_NPAGE 4096 /* Must be power of 2 and multiple of 256 */ -#define HASHTABLE_DATATYPE u16 +#define HASHTABLE_NPAGE 4096 /* Must be power of 2 */ #define HASHTABLE_HASH_1 383 /* Should be prime */ #define HASHTABLE_NSLOT (HASHTABLE_NPAGE*2) /* Must be a power of 2 */ -#define HASHTABLE_NBYTE (sizeof(HASHTABLE_DATATYPE)*HASHTABLE_NSLOT) /* The block of page numbers associated with the first hash-table in a ** wal-index is smaller than usual. This is so that there is a complete ** hash-table on each aligned 32KB page of the wal-index. */ -#define HASHTABLE_NPAGE_ONE (4096 - (WALINDEX_HDR_SIZE/sizeof(u32))) +#define HASHTABLE_NPAGE_ONE (HASHTABLE_NPAGE - (WALINDEX_HDR_SIZE/sizeof(u32))) -/* The wal-index is divided into pages of HASHTABLE_PAGESIZE bytes each. */ -#define HASHTABLE_PAGESIZE (HASHTABLE_NBYTE + HASHTABLE_NPAGE*sizeof(u32)) +/* The wal-index is divided into pages of WALINDEX_PGSZ bytes each. */ +#define WALINDEX_PGSZ ( \ + sizeof(ht_slot)*HASHTABLE_NSLOT + HASHTABLE_NPAGE*sizeof(u32) \ +) /* ** Obtain a pointer to the iPage'th page of the wal-index. The wal-index -** is broken into pages of HASHTABLE_PAGESIZE bytes. Wal-index pages are +** is broken into pages of WALINDEX_PGSZ bytes. Wal-index pages are ** numbered from zero. ** ** If this call is successful, *ppPage is set to point to the wal-index @@ -437,7 +443,7 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ /* Request a pointer to the required page from the VFS */ if( pWal->apWiData[iPage]==0 ){ - rc = sqlite3OsShmPage(pWal->pDbFd, iPage, HASHTABLE_PAGESIZE, + rc = sqlite3OsShmPage(pWal->pDbFd, iPage, WALINDEX_PGSZ, pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] ); } @@ -479,15 +485,15 @@ static volatile WalIndexHdr *walIndexHdr(Wal *pWal){ ** This functionality is used by the checkpoint code (see walCheckpoint()). */ struct WalIterator { - int iPrior; /* Last result returned from the iterator */ - int nSegment; /* Size of the aSegment[] array */ + int iPrior; /* Last result returned from the iterator */ + int nSegment; /* Size of the aSegment[] array */ struct WalSegment { int iNext; /* Next slot in aIndex[] not yet returned */ - HASHTABLE_DATATYPE *aIndex; /* i0, i1, i2... such that aPgno[iN] ascend */ + ht_slot *aIndex; /* i0, i1, i2... such that aPgno[iN] ascend */ u32 *aPgno; /* Array of page numbers. */ int nEntry; /* Max size of aPgno[] and aIndex[] arrays */ int iZero; /* Frame number associated with aPgno[0] */ - } aSegment[1]; /* One for every 32KB page in the WAL */ + } aSegment[1]; /* One for every 32KB page in the WAL */ }; /* @@ -750,7 +756,7 @@ static int walNextHash(int iPriorHash){ static int walHashGet( Wal *pWal, /* WAL handle */ int iHash, /* Find the iHash'th table */ - volatile HASHTABLE_DATATYPE **paHash, /* OUT: Pointer to hash index */ + volatile ht_slot **paHash, /* OUT: Pointer to hash index */ volatile u32 **paPgno, /* OUT: Pointer to page number array */ u32 *piZero /* OUT: Frame associated with *paPgno[0] */ ){ @@ -762,9 +768,9 @@ static int walHashGet( if( rc==SQLITE_OK ){ u32 iZero; - volatile HASHTABLE_DATATYPE *aHash; + volatile ht_slot *aHash; - aHash = (volatile HASHTABLE_DATATYPE *)&aPgno[HASHTABLE_NPAGE]; + aHash = (volatile ht_slot *)&aPgno[HASHTABLE_NPAGE]; if( iHash==0 ){ aPgno = &aPgno[WALINDEX_HDR_SIZE/sizeof(u32)-1]; iZero = 0; @@ -821,12 +827,12 @@ static u32 walFramePgno(Wal *pWal, u32 iFrame){ ** actually needed. */ static void walCleanupHash(Wal *pWal){ - volatile HASHTABLE_DATATYPE *aHash; /* Pointer to hash table to clear */ - volatile u32 *aPgno; /* Page number array for hash table */ - u32 iZero; /* frame == (aHash[x]+iZero) */ - int iLimit = 0; /* Zero values greater than this */ - int nByte; /* Number of bytes to zero in aPgno[] */ - int i; /* Used to iterate through aHash[] */ + volatile ht_slot *aHash; /* Pointer to hash table to clear */ + volatile u32 *aPgno; /* Page number array for hash table */ + u32 iZero; /* frame == (aHash[x]+iZero) */ + int iLimit = 0; /* Zero values greater than this */ + int nByte; /* Number of bytes to zero in aPgno[] */ + int i; /* Used to iterate through aHash[] */ assert( pWal->writeLock ); testcase( pWal->hdr.mxFrame==HASHTABLE_NPAGE-1 ); @@ -886,7 +892,7 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ int rc; /* Return code */ u32 iZero; /* One less than frame number of aPgno[1] */ volatile u32 *aPgno; /* Page number array */ - volatile HASHTABLE_DATATYPE *aHash; /* Hash table */ + volatile ht_slot *aHash; /* Hash table */ rc = walHashGet(pWal, walFramePage(iFrame), &aHash, &aPgno, &iZero); @@ -1230,8 +1236,8 @@ static int walIteratorNext( static void walMergesort( u32 *aContent, /* Pages in wal */ - HASHTABLE_DATATYPE *aBuffer, /* Buffer of at least *pnList items to use */ - HASHTABLE_DATATYPE *aList, /* IN/OUT: List to sort */ + ht_slot *aBuffer, /* Buffer of at least *pnList items to use */ + ht_slot *aList, /* IN/OUT: List to sort */ int *pnList /* IN/OUT: Number of elements in aList[] */ ){ int nList = *pnList; @@ -1241,15 +1247,15 @@ static void walMergesort( int iLeft = 0; /* Current index in aLeft */ int iRight = 0; /* Current index in aright */ int iOut = 0; /* Current index in output buffer */ - HASHTABLE_DATATYPE *aLeft = aList; /* Left list */ - HASHTABLE_DATATYPE *aRight = &aList[nLeft]; /* Right list */ + ht_slot *aLeft = aList; /* Left list */ + ht_slot *aRight = aList+nLeft;/* Right list */ /* TODO: Change to non-recursive version. */ walMergesort(aContent, aBuffer, aLeft, &nLeft); walMergesort(aContent, aBuffer, aRight, &nRight); while( iRightnSegment = nSegment; - aSpace = (HASHTABLE_DATATYPE *)&p->aSegment[nSegment]; + aSpace = (ht_slot *)&p->aSegment[nSegment]; aTmp = &aSpace[HASHTABLE_NPAGE*nSegment]; for(i=0; i=0 && iRead==0; iHash--){ - volatile HASHTABLE_DATATYPE *aHash; /* Pointer to hash table */ - volatile u32 *aPgno; /* Pointer to array of page numbers */ + volatile ht_slot *aHash; /* Pointer to hash table */ + volatile u32 *aPgno; /* Pointer to array of page numbers */ u32 iZero; /* Frame number corresponding to aPgno[0] */ int iKey; /* Hash slot index */ int rc; @@ -2073,14 +2079,13 @@ int sqlite3WalEndWriteTransaction(Wal *pWal){ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ int rc = SQLITE_OK; if( pWal->writeLock ){ - int unused; Pgno iMax = pWal->hdr.mxFrame; Pgno iFrame; /* Restore the clients cache of the wal-index header to the state it ** was in before the client began writing to the database. */ - memcpy(&pWal->hdr, walIndexHdr(pWal), sizeof(WalIndexHdr)); + memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr)); for(iFrame=pWal->hdr.mxFrame+1; ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; @@ -2097,7 +2102,6 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ ** page 1 is never written to the log until the transaction is ** committed. As a result, the call to xUndo may not fail. */ - assert( pWal->writeLock ); assert( walFramePgno(pWal, iFrame)!=1 ); rc = xUndo(pUndoCtx, walFramePgno(pWal, iFrame)); } From d60bf110363bc0805ba25cd581c01c60110c3f3c Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 14 Jun 2010 11:18:50 +0000 Subject: [PATCH 5/8] Change the interface to internal function walGetHash() to make it easier to follow. FossilOrigin-Name: 5e8e2e978ea48ce4ad93a936c838934f33d665df --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 35 ++++++++++++++++------------------- 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index 1842e90493..6f09fdd48d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sxShmPage\smethod\sto\sthe\s"crash"\svfs\sin\stest6.c. -D 2010-06-14T10:30:12 +C Change\sthe\sinterface\sto\sinternal\sfunction\swalGetHash()\sto\smake\sit\seasier\sto\sfollow. +D 2010-06-14T11:18:51 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in a5cad1f8f3e021356bfcc6c77dc16f6f1952bbc3 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -226,7 +226,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c 7cf566b5c1cb9e1094bbbd9b188605ed83330e40 +F src/wal.c eb0a433af578bccd8c3742806d8f2748976aa894 F src/wal.h 4ace25262452d17e7d3ec970c89ee17794004008 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 1c895bef33d0dfc7ed90fb1f74120435d210ea56 @@ -820,7 +820,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 37b26d125f4b1d8e75bb38800fefd145611f94aa -R dea40a6fff442af044dc9e87dfed9d5b +P 1008f536440840da7d56c01ec147a25295fd1fd4 +R 912042aef26c7174a859c811eae607e9 U dan -Z 06678e8838f3de5fe68cbedec2758b53 +Z 1a6bfab282147a6b78cbcbccc32bce51 diff --git a/manifest.uuid b/manifest.uuid index 13d1c158aa..4411035668 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1008f536440840da7d56c01ec147a25295fd1fd4 \ No newline at end of file +5e8e2e978ea48ce4ad93a936c838934f33d665df \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index e4be0abf62..4a3ed99ff6 100644 --- a/src/wal.c +++ b/src/wal.c @@ -749,9 +749,8 @@ static int walNextHash(int iPriorHash){ ** slot in the hash table is set to N, it refers to frame number ** (*piZero+N) in the log. ** -** Finally, set *paPgno such that for all frames F between (*piZero+1) and -** (*piZero+HASHTABLE_NPAGE), (*paPgno)[F] is the database page number -** associated with frame F. +** Finally, set *paPgno so that *paPgno[1] is the page number of the +** first frame indexed by the hash table, frame (*piZero+1). */ static int walHashGet( Wal *pWal, /* WAL handle */ @@ -772,14 +771,13 @@ static int walHashGet( aHash = (volatile ht_slot *)&aPgno[HASHTABLE_NPAGE]; if( iHash==0 ){ - aPgno = &aPgno[WALINDEX_HDR_SIZE/sizeof(u32)-1]; + aPgno = &aPgno[WALINDEX_HDR_SIZE/sizeof(u32)]; iZero = 0; }else{ iZero = HASHTABLE_NPAGE_ONE + (iHash-1)*HASHTABLE_NPAGE; - aPgno = &aPgno[-1*iZero-1]; } - *paPgno = aPgno; + *paPgno = &aPgno[-1]; *paHash = aHash; *piZero = iZero; } @@ -863,8 +861,8 @@ static void walCleanupHash(Wal *pWal){ /* Zero the entries in the aPgno array that correspond to frames with ** frame numbers greater than pWal->hdr.mxFrame. */ - nByte = ((char *)aHash - (char *)&aPgno[pWal->hdr.mxFrame+1]); - memset((void *)&aPgno[pWal->hdr.mxFrame+1], 0, nByte); + nByte = ((char *)aHash - (char *)&aPgno[iLimit+1]); + memset((void *)&aPgno[iLimit+1], 0, nByte); #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT /* Verify that the every entry in the mapping region is still reachable @@ -874,7 +872,7 @@ static void walCleanupHash(Wal *pWal){ int i; /* Loop counter */ int iKey; /* Hash key */ for(i=1; i<=iLimit; i++){ - for(iKey=walHash(aPgno[i+iZero]); aHash[iKey]; iKey=walNextHash(iKey)){ + for(iKey=walHash(aPgno[i]); aHash[iKey]; iKey=walNextHash(iKey)){ if( aHash[iKey]==i ) break; } assert( aHash[iKey]==i ); @@ -911,8 +909,8 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ ** entire hash table and aPgno[] array before proceding. */ if( idx==1 ){ - int nByte = (u8 *)&aHash[HASHTABLE_NSLOT] - (u8 *)&aPgno[1+iZero]; - memset((void*)&aPgno[1+iZero], 0, nByte); + int nByte = (u8 *)&aHash[HASHTABLE_NSLOT] - (u8 *)&aPgno[1]; + memset((void*)&aPgno[1], 0, nByte); } /* If the entry in aPgno[] is already set, then the previous writer @@ -921,16 +919,16 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ ** Remove the remnants of that writers uncommitted transaction from ** the hash-table before writing any new entries. */ - if( aPgno[iFrame] ){ + if( aPgno[idx] ){ walCleanupHash(pWal); - assert( !aPgno[iFrame] ); + assert( !aPgno[idx] ); } /* Write the aPgno[] array entry and the hash-table slot. */ for(iKey=walHash(iPage); aHash[iKey]; iKey=walNextHash(iKey)){ assert( nCollide++ < idx ); } - aPgno[iFrame] = iPage; + aPgno[idx] = iPage; aHash[iKey] = idx; #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT @@ -952,7 +950,7 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ if( (idx&0x3ff)==0 ){ int i; /* Loop counter */ for(i=1; i<=idx; i++){ - for(iKey=walHash(aPgno[i+iZero]); aHash[iKey]; iKey=walNextHash(iKey)){ + for(iKey=walHash(aPgno[i]); aHash[iKey]; iKey=walNextHash(iKey)){ if( aHash[iKey]==i ) break; } assert( aHash[iKey]==i ); @@ -1353,10 +1351,9 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ walIteratorFree(p); return rc; } - nEntry = ((i+1)==nSegment)?iLast-iZero:(u32 *)aHash-(u32 *)&aPgno[iZero+1]; - + aPgno++; + nEntry = ((i+1)==nSegment)?iLast-iZero:(u32 *)aHash-(u32 *)aPgno; iZero++; - aPgno += iZero; for(j=0; jiRead ); iRead = iFrame; } From ad3cadd8b2b737c5190b805360f773383843172e Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 14 Jun 2010 11:49:26 +0000 Subject: [PATCH 6/8] Update some comments in wal.c. No code changes. FossilOrigin-Name: 1ce9c92bffa5d7f8431c005b29d698b0f5b95875 --- manifest | 12 +++---- manifest.uuid | 2 +- src/wal.c | 95 +++++++++++++++++++++++++++++---------------------- 3 files changed, 61 insertions(+), 48 deletions(-) diff --git a/manifest b/manifest index 6f09fdd48d..ecd7aedf0c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sinterface\sto\sinternal\sfunction\swalGetHash()\sto\smake\sit\seasier\sto\sfollow. -D 2010-06-14T11:18:51 +C Update\ssome\scomments\sin\swal.c.\sNo\scode\schanges. +D 2010-06-14T11:49:26 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in a5cad1f8f3e021356bfcc6c77dc16f6f1952bbc3 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -226,7 +226,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c eb0a433af578bccd8c3742806d8f2748976aa894 +F src/wal.c f8ba403da99ed5f2260f03cd01c3f10fe8155e67 F src/wal.h 4ace25262452d17e7d3ec970c89ee17794004008 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 1c895bef33d0dfc7ed90fb1f74120435d210ea56 @@ -820,7 +820,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 1008f536440840da7d56c01ec147a25295fd1fd4 -R 912042aef26c7174a859c811eae607e9 +P 5e8e2e978ea48ce4ad93a936c838934f33d665df +R 4612706a9d4bf144a165ca75ff4d4a89 U dan -Z 1a6bfab282147a6b78cbcbccc32bce51 +Z d61a90eb1d7a0bbf02b4e4817b07d279 diff --git a/manifest.uuid b/manifest.uuid index 4411035668..5cb0339b07 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5e8e2e978ea48ce4ad93a936c838934f33d665df \ No newline at end of file +1ce9c92bffa5d7f8431c005b29d698b0f5b95875 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 4a3ed99ff6..cd31bc3f81 100644 --- a/src/wal.c +++ b/src/wal.c @@ -141,21 +141,33 @@ ** more index blocks. ** ** The wal-index header contains the total number of frames within the WAL -** in the the mxFrame field. Each index block contains information on -** HASHTABLE_NPAGE frames. Each index block contains two sections, a -** mapping which is a database page number for each frame, and a hash -** table used to look up frames by page number. The mapping section is -** an array of HASHTABLE_NPAGE 32-bit page numbers. The first entry on the -** array is the page number for the first frame; the second entry is the -** page number for the second frame; and so forth. The last index block -** holds a total of (mxFrame%HASHTABLE_NPAGE) page numbers. All index -** blocks other than the last are completely full with HASHTABLE_NPAGE -** page numbers. All index blocks are the same size; the mapping section -** of the last index block merely contains unused entries if mxFrame is -** not an even multiple of HASHTABLE_NPAGE. +** in the the mxFrame field. +** +** Each index block except for the first contains information on +** HASHTABLE_NPAGE frames. The first index block contains information on +** HASHTABLE_NPAGE_ONE frames. The values of HASHTABLE_NPAGE_ONE and +** HASHTABLE_NPAGE are selected so that together the wal-index header and +** first index block are the same size as all other index blocks in the +** wal-index. +** +** Each index block contains two sections, a page-mapping that contains the +** database page number associated with each wal frame, and a hash-table +** that allows users to query an index block for a specific page number. +** The page-mapping is an array of HASHTABLE_NPAGE (or HASHTABLE_NPAGE_ONE +** for the first index block) 32-bit page numbers. The first entry in the +** first index-block contains the database page number corresponding to the +** first frame in the WAL file. The first entry in the second index block +** in the WAL file corresponds to the (HASHTABLE_NPAGE_ONE+1)th frame in +** the log, and so on. +** +** The last index block in a wal-index usually contains less than the full +** complement of HASHTABLE_NPAGE (or HASHTABLE_NPAGE_ONE) page-numbers, +** depending on the contents of the WAL file. This does not change the +** allocated size of the page-mapping array - the page-mapping array merely +** contains unused entries. ** ** Even without using the hash table, the last frame for page P -** can be found by scanning the mapping sections of each index block +** can be found by scanning the page-mapping sections of each index block ** starting with the last index block and moving toward the first, and ** within each index block, starting at the end and moving toward the ** beginning. The first entry that equals P corresponds to the frame @@ -392,6 +404,33 @@ struct Wal { */ typedef u16 ht_slot; +/* +** This structure is used to implement an iterator that loops through +** all frames in the WAL in database page order. Where two or more frames +** correspond to the same database page, the iterator visits only the +** frame most recently written to the WAL (in other words, the frame with +** the largest index). +** +** The internals of this structure are only accessed by: +** +** walIteratorInit() - Create a new iterator, +** walIteratorNext() - Step an iterator, +** walIteratorFree() - Free an iterator. +** +** This functionality is used by the checkpoint code (see walCheckpoint()). +*/ +struct WalIterator { + int iPrior; /* Last result returned from the iterator */ + int nSegment; /* Size of the aSegment[] array */ + struct WalSegment { + int iNext; /* Next slot in aIndex[] not yet returned */ + ht_slot *aIndex; /* i0, i1, i2... such that aPgno[iN] ascend */ + u32 *aPgno; /* Array of page numbers. */ + int nEntry; /* Max size of aPgno[] and aIndex[] arrays */ + int iZero; /* Frame number associated with aPgno[0] */ + } aSegment[1]; /* One for every 32KB page in the WAL */ +}; + /* ** Define the parameters of the hash tables in the wal-index file. There ** is a hash-table following every HASHTABLE_NPAGE page numbers in the @@ -404,7 +443,8 @@ typedef u16 ht_slot; #define HASHTABLE_HASH_1 383 /* Should be prime */ #define HASHTABLE_NSLOT (HASHTABLE_NPAGE*2) /* Must be a power of 2 */ -/* The block of page numbers associated with the first hash-table in a +/* +** The block of page numbers associated with the first hash-table in a ** wal-index is smaller than usual. This is so that there is a complete ** hash-table on each aligned 32KB page of the wal-index. */ @@ -469,33 +509,6 @@ static volatile WalIndexHdr *walIndexHdr(Wal *pWal){ return (volatile WalIndexHdr*)pWal->apWiData[0]; } -/* -** This structure is used to implement an iterator that loops through -** all frames in the WAL in database page order. Where two or more frames -** correspond to the same database page, the iterator visits only the -** frame most recently written to the WAL (in other words, the frame with -** the largest index). -** -** The internals of this structure are only accessed by: -** -** walIteratorInit() - Create a new iterator, -** walIteratorNext() - Step an iterator, -** walIteratorFree() - Free an iterator. -** -** This functionality is used by the checkpoint code (see walCheckpoint()). -*/ -struct WalIterator { - int iPrior; /* Last result returned from the iterator */ - int nSegment; /* Size of the aSegment[] array */ - struct WalSegment { - int iNext; /* Next slot in aIndex[] not yet returned */ - ht_slot *aIndex; /* i0, i1, i2... such that aPgno[iN] ascend */ - u32 *aPgno; /* Array of page numbers. */ - int nEntry; /* Max size of aPgno[] and aIndex[] arrays */ - int iZero; /* Frame number associated with aPgno[0] */ - } aSegment[1]; /* One for every 32KB page in the WAL */ -}; - /* ** The argument to this macro must be of type u32. On a little-endian ** architecture, it returns the u32 value that results from interpreting From 188019153dcf33ffd9d63ba17ab4a29e95561cce Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 14 Jun 2010 14:07:50 +0000 Subject: [PATCH 7/8] Remove xShmGet/Size/Release from the sqlite3_vfs structure. Change the name of xShmPage to xShmMap. Remove some code that is now unused from os_unix.c and some of the test VFS implementations. FossilOrigin-Name: fc0cabc15c97dde6a852b4f07df6d30f1d2c04bc --- manifest | 28 ++--- manifest.uuid | 2 +- src/os.c | 13 +-- src/os.h | 5 +- src/os_unix.c | 261 +++++++++++----------------------------------- src/sqlite.h.in | 5 +- src/test6.c | 28 +---- src/test_devsym.c | 35 ++----- src/test_osinst.c | 69 ++++-------- src/test_vfs.c | 38 +------ src/wal.c | 2 +- 11 files changed, 114 insertions(+), 372 deletions(-) diff --git a/manifest b/manifest index ecd7aedf0c..b211bcfee4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\ssome\scomments\sin\swal.c.\sNo\scode\schanges. -D 2010-06-14T11:49:26 +C Remove\sxShmGet/Size/Release\sfrom\sthe\ssqlite3_vfs\sstructure.\sChange\sthe\sname\sof\sxShmPage\sto\sxShmMap.\sRemove\ssome\scode\sthat\sis\snow\sunused\sfrom\sos_unix.c\sand\ssome\sof\sthe\stest\sVFS\simplementations. +D 2010-06-14T14:07:51 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in a5cad1f8f3e021356bfcc6c77dc16f6f1952bbc3 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -150,11 +150,11 @@ F src/mutex_os2.c 6a62583e374ba3ac1a3fcc0da2bfdac7d3942689 F src/mutex_unix.c cf84466b4fdd2baa0d5a10bb19f08b2abc1ce42e F src/mutex_w32.c 1fe0e735897be20e09dd6f53c3fb516c6b48c0eb F src/notify.c cbfa66a836da3a51567209636e6a94059c137930 -F src/os.c 00ab9dcdee6e33ff3d060744c86af25200c51e0f -F src/os.h a0d2c1436cb6003e6da16001499a0b828f1edb34 +F src/os.c 9c4a2f82a50306a33907678ec0187b6ad1486bfe +F src/os.h d7775504a51e6e0d40315aa427b3e229ff9ff9ca F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 665876d5eec7585226b0a1cf5e18098de2b2da19 -F src/os_unix.c 29dac62790ccea7db1516be3abb007988accb165 +F src/os_unix.c 0a57b86241011ca25f2fb83e4825b7af7fe6d920 F src/os_win.c 0cf1f571546f165001e2391b5d4a4a16d86977d3 F src/pager.c 2964185d4356d0dc159b8340e52d2538d32394e5 F src/pager.h ca1f23c0cf137ac26f8908df2427c8b308361efd @@ -170,7 +170,7 @@ F src/resolve.c ac5f1a713cd1ae77f08b83cc69581e11bf5ae6f9 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c c03d8a0565febcde8c6a12c5d77d065fddae889b F src/shell.c fd4ccdb37c3b68de0623eb938a649e0990710714 -F src/sqlite.h.in 092df034f4b426ffbb9e5bb905958fa35bbb7f7a +F src/sqlite.h.in 6e8e9454f6ada979405543b19e0c18ca16fd020e F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89 F src/sqliteInt.h 242987ebd2366ea36650a09cdab04a9163c62109 F src/sqliteLimit.h 196e2f83c3b444c4548fc1874f52f84fdbda40f3 @@ -182,7 +182,7 @@ F src/test2.c e3f564ab1e9fd0b47b0c9e23e7054e38bf0836cf F src/test3.c 4c21700c73a890a47fc685c1097bfb661346ac94 F src/test4.c ad03bb987ddedce928f4258c1e7fa4109a73497d F src/test5.c cc55900118fa4add8ec9cf69fc4225a4662f76b1 -F src/test6.c 1b312541883ec9413734035deada4794ee339e67 +F src/test6.c 43d6ce22d77b22f8c580be40990be6fa204d7f4d F src/test7.c 3f2d63e4ccf97f8c2cf1a7fa0a3c8e2e2a354e6e F src/test8.c f959db9a22d882013b64c92753fa793b2ce3bdea F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 @@ -192,7 +192,7 @@ F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de F src/test_btree.c 47cd771250f09cdc6e12dda5bc71bc0b3abc96e2 F src/test_config.c 6210f501d358bde619ae761f06f123529c6ba24f F src/test_demovfs.c da81a5f7785bb352bda7911c332a983ec4f17f27 -F src/test_devsym.c cf64a4b602ccde10c9261283d1b9be12f4c4a0ea +F src/test_devsym.c b2c821471efb0f3680de0c60fbe1ac673517ddd6 F src/test_func.c 13b582345fb1185a93e46c53310fae8547dcce20 F src/test_hexio.c 1237f000ec7a491009b1233f5c626ea71bce1ea2 F src/test_init.c 5d624ffd0409d424cf9adbfe1f056b200270077c @@ -203,13 +203,13 @@ F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 2842c922b8e8d992aba722214952204ca025b411 F src/test_mutex.c ce06b59aca168cd8c520b77159a24352a7469bd3 F src/test_onefile.c 4ce8c753c0240f010f0f2af89604875967d20945 -F src/test_osinst.c a0df974ab32f9aae97da5ce094301926cffcdc17 +F src/test_osinst.c 0d3d47cf3a8a0b4db9a5d0255b9d74c671ca19b4 F src/test_pcache.c 7bf828972ac0d2403f5cfa4cd14da41f8ebe73d8 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c bbba05c144b5fc4b52ff650a4328027b3fa5fcc6 F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c aa9919c885a1fe53eafc73492f0898ee6c0a0726 -F src/test_vfs.c 15bddcddf6b1bf6360130e09aee950f5f563d5f3 +F src/test_vfs.c 8e125e01b4f88c6a2e131d5a65d15f5c96d665f1 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 25ceb0f0a746ea1d0f9553787f3f0a56853cfaeb F src/trigger.c 8927588cb9e6d47f933b53bfe74200fbb504100d @@ -226,7 +226,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1 F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2 F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda -F src/wal.c f8ba403da99ed5f2260f03cd01c3f10fe8155e67 +F src/wal.c c261aeb28c9b40f4713bace3c1c0cee9dcba87ab F src/wal.h 4ace25262452d17e7d3ec970c89ee17794004008 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 1c895bef33d0dfc7ed90fb1f74120435d210ea56 @@ -820,7 +820,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 5e8e2e978ea48ce4ad93a936c838934f33d665df -R 4612706a9d4bf144a165ca75ff4d4a89 +P 1ce9c92bffa5d7f8431c005b29d698b0f5b95875 +R 8ee76bbe532e6db3f47bf2104ddf1a48 U dan -Z d61a90eb1d7a0bbf02b4e4817b07d279 +Z 95a9a1111fae03b142846d91ecde0c57 diff --git a/manifest.uuid b/manifest.uuid index 5cb0339b07..6a3db6fb58 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1ce9c92bffa5d7f8431c005b29d698b0f5b95875 \ No newline at end of file +fc0cabc15c97dde6a852b4f07df6d30f1d2c04bc \ No newline at end of file diff --git a/src/os.c b/src/os.c index 697233be0c..8fa10a9c02 100644 --- a/src/os.c +++ b/src/os.c @@ -101,15 +101,6 @@ int sqlite3OsDeviceCharacteristics(sqlite3_file *id){ int sqlite3OsShmOpen(sqlite3_file *id){ return id->pMethods->xShmOpen(id); } -int sqlite3OsShmSize(sqlite3_file *id, int reqSize, int *pNewSize){ - return id->pMethods->xShmSize(id, reqSize, pNewSize); -} -int sqlite3OsShmGet(sqlite3_file *id,int reqSize,int *pSize,void volatile **pp){ - return id->pMethods->xShmGet(id, reqSize, pSize, pp); -} -int sqlite3OsShmRelease(sqlite3_file *id){ - return id->pMethods->xShmRelease(id); -} int sqlite3OsShmLock(sqlite3_file *id, int offset, int n, int flags){ return id->pMethods->xShmLock(id, offset, n, flags); } @@ -119,14 +110,14 @@ void sqlite3OsShmBarrier(sqlite3_file *id){ int sqlite3OsShmClose(sqlite3_file *id, int deleteFlag){ return id->pMethods->xShmClose(id, deleteFlag); } -int sqlite3OsShmPage( +int sqlite3OsShmMap( sqlite3_file *id, int iPage, int pgsz, int isWrite, void volatile **pp ){ - return id->pMethods->xShmPage(id, iPage, pgsz, isWrite, pp); + return id->pMethods->xShmMap(id, iPage, pgsz, isWrite, pp); } /* diff --git a/src/os.h b/src/os.h index d1d6bde058..d51eec3abc 100644 --- a/src/os.h +++ b/src/os.h @@ -248,13 +248,10 @@ int sqlite3OsFileControl(sqlite3_file*,int,void*); int sqlite3OsSectorSize(sqlite3_file *id); int sqlite3OsDeviceCharacteristics(sqlite3_file *id); int sqlite3OsShmOpen(sqlite3_file *id); -int sqlite3OsShmSize(sqlite3_file *id, int, int*); -int sqlite3OsShmGet(sqlite3_file *id, int, int*, void volatile**); -int sqlite3OsShmRelease(sqlite3_file *id); int sqlite3OsShmLock(sqlite3_file *id, int, int, int); void sqlite3OsShmBarrier(sqlite3_file *id); int sqlite3OsShmClose(sqlite3_file *id, int); -int sqlite3OsShmPage(sqlite3_file *,int,int,int,void volatile **); +int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); /* ** Functions for accessing sqlite3_vfs methods diff --git a/src/os_unix.c b/src/os_unix.c index 612d5c4366..22c9a92826 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3128,27 +3128,15 @@ static int unixDeviceCharacteristics(sqlite3_file *NotUsed){ ** Either unixShmNode.mutex must be held or unixShmNode.nRef==0 and ** unixMutexHeld() is true when reading or writing any other field ** in this structure. -** -** To avoid deadlocks, mutex and mutexBuf are always released in the -** reverse order that they are acquired. mutexBuf is always acquired -** first and released last. This invariant is check by asserting -** sqlite3_mutex_notheld() on mutex whenever mutexBuf is acquired or -** released. */ struct unixShmNode { unixInodeInfo *pInode; /* unixInodeInfo that owns this SHM node */ sqlite3_mutex *mutex; /* Mutex to access this object */ - sqlite3_mutex *mutexBuf; /* Mutex to access zBuf[] */ char *zFilename; /* Name of the mmapped file */ int h; /* Open file descriptor */ - - int szMap; /* Size of the mapping into memory */ - char *pMMapBuf; /* Where currently mmapped(). NULL if unmapped */ - - int pgsz; /* Size of shared-memory pages */ - int nPage; /* Size of array apPage */ - char **apPage; /* Array of mapped shared-memory pages */ - + int szRegion; /* Size of shared-memory regions */ + int nRegion; /* Size of array apRegion */ + char **apRegion; /* Array of mapped shared-memory regions */ int nRef; /* Number of unixShm objects pointing to this */ unixShm *pFirst; /* All unixShm objects pointing to this */ #ifdef SQLITE_DEBUG @@ -3175,7 +3163,6 @@ struct unixShm { unixShmNode *pShmNode; /* The underlying unixShmNode object */ unixShm *pNext; /* Next unixShm with the same unixShmNode */ u8 hasMutex; /* True if holding the unixShmNode mutex */ - u8 hasMutexBuf; /* True if holding pFile->mutexBuf */ u16 sharedMask; /* Mask of shared locks held */ u16 exclMask; /* Mask of exclusive locks held */ #ifdef SQLITE_DEBUG @@ -3275,12 +3262,10 @@ static void unixShmPurge(unixFile *pFd){ int i; assert( p->pInode==pFd->pInode ); if( p->mutex ) sqlite3_mutex_free(p->mutex); - if( p->mutexBuf ) sqlite3_mutex_free(p->mutexBuf); - if( p->pMMapBuf ) munmap(p->pMMapBuf, p->szMap); - for(i=0; inPage; i++){ - munmap(p->apPage[i], p->pgsz); + for(i=0; inRegion; i++){ + munmap(p->apRegion[i], p->szRegion); } - sqlite3_free(p->apPage); + sqlite3_free(p->apRegion); if( p->h>=0 ) close(p->h); p->pInode->pShmNode = 0; sqlite3_free(p); @@ -3356,11 +3341,6 @@ static int unixShmOpen( rc = SQLITE_NOMEM; goto shm_open_err; } - pShmNode->mutexBuf = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutexBuf==0 ){ - rc = SQLITE_NOMEM; - goto shm_open_err; - } pShmNode->h = open(pShmNode->zFilename, O_RDWR|O_CREAT, 0664); if( pShmNode->h<0 ){ @@ -3431,7 +3411,6 @@ static int unixShmClose( *pp = p->pNext; /* Free the connection p */ - assert( p->hasMutexBuf==0 ); sqlite3_free(p); pDbFd->pShm = 0; sqlite3_mutex_leave(pShmNode->mutex); @@ -3450,148 +3429,6 @@ static int unixShmClose( return SQLITE_OK; } -/* -** Changes the size of the underlying storage for a shared-memory segment. -** -** The reqSize parameter is the new requested size of the shared memory. -** This implementation is free to increase the shared memory size to -** any amount greater than or equal to reqSize. If the shared memory is -** already as big or bigger as reqSize, this routine is a no-op. -** -** The reqSize parameter is the minimum size requested. The implementation -** is free to expand the storage to some larger amount if it chooses. -*/ -static int unixShmSize( - sqlite3_file *fd, /* The open database file holding SHM */ - int reqSize, /* Requested size. -1 for query only */ - int *pNewSize /* Write new size here */ -){ - unixFile *pDbFd = (unixFile*)fd; - unixShm *p = pDbFd->pShm; - unixShmNode *pShmNode = p->pShmNode; - int rc = SQLITE_OK; - struct stat sStat; - - assert( pShmNode==pDbFd->pInode->pShmNode ); - assert( pShmNode->pInode==pDbFd->pInode ); - - while( 1 ){ - if( fstat(pShmNode->h, &sStat)==0 ){ - *pNewSize = (int)sStat.st_size; - if( reqSize<=(int)sStat.st_size ) break; - }else{ - *pNewSize = 0; - rc = SQLITE_IOERR_SHMSIZE; - break; - } - rc = ftruncate(pShmNode->h, reqSize); - reqSize = -1; - } - return rc; -} - -/* -** Release the lock held on the shared memory segment to that other -** threads are free to resize it if necessary. -** -** If the lock is not currently held, this routine is a harmless no-op. -** -** If the shared-memory object is in lock state RECOVER, then we do not -** really want to release the lock, so in that case too, this routine -** is a no-op. -*/ -static int unixShmRelease(sqlite3_file *fd){ - unixFile *pDbFd = (unixFile*)fd; - unixShm *p = pDbFd->pShm; - - if( p->hasMutexBuf ){ - assert( sqlite3_mutex_notheld(p->pShmNode->mutex) ); - sqlite3_mutex_leave(p->pShmNode->mutexBuf); - p->hasMutexBuf = 0; - } - return SQLITE_OK; -} - -/* -** Map the shared storage into memory. -** -** If reqMapSize is positive, then an attempt is made to make the -** mapping at least reqMapSize bytes in size. However, the mapping -** will never be larger than the size of the underlying shared memory -** as set by prior calls to xShmSize(). -** -** *ppBuf is made to point to the memory which is a mapping of the -** underlying storage. A mutex is acquired to prevent other threads -** from running while *ppBuf is in use in order to prevent other threads -** remapping *ppBuf out from under this thread. The unixShmRelease() -** call will release the mutex. However, if the lock state is CHECKPOINT, -** the mutex is not acquired because CHECKPOINT will never remap the -** buffer. RECOVER might remap, though, so CHECKPOINT will acquire -** the mutex if and when it promotes to RECOVER. -** -** RECOVER needs to be atomic. The same mutex that prevents *ppBuf from -** being remapped also prevents more than one thread from being in -** RECOVER at a time. But, RECOVER sometimes wants to remap itself. -** To prevent RECOVER from losing its lock while remapping, the -** mutex is not released by unixShmRelease() when in RECOVER. -** -** *pNewMapSize is set to the size of the mapping. Usually *pNewMapSize -** will be reqMapSize or larger, though it could be smaller if the -** underlying shared memory has never been enlarged to reqMapSize bytes -** by prior calls to xShmSize(). -** -** *ppBuf might be NULL and zero if no space has -** yet been allocated to the underlying storage. -*/ -static int unixShmGet( - sqlite3_file *fd, /* Database file holding shared memory */ - int reqMapSize, /* Requested size of mapping. -1 means don't care */ - int *pNewMapSize, /* Write new size of mapping here */ - void volatile **ppBuf /* Write mapping buffer origin here */ -){ - unixFile *pDbFd = (unixFile*)fd; - unixShm *p = pDbFd->pShm; - unixShmNode *pShmNode = p->pShmNode; - int rc = SQLITE_OK; - - assert( pShmNode==pDbFd->pInode->pShmNode ); - assert( pShmNode->pInode==pDbFd->pInode ); - - if( p->hasMutexBuf==0 ){ - assert( sqlite3_mutex_notheld(pShmNode->mutex) ); - sqlite3_mutex_enter(pShmNode->mutexBuf); - p->hasMutexBuf = 1; - } - sqlite3_mutex_enter(pShmNode->mutex); - if( pShmNode->szMap==0 || reqMapSize>pShmNode->szMap ){ - int actualSize; - if( unixShmSize(fd, -1, &actualSize)!=SQLITE_OK ){ - actualSize = 0; - } - reqMapSize = actualSize; - if( pShmNode->pMMapBuf || reqMapSize<=0 ){ - munmap(pShmNode->pMMapBuf, pShmNode->szMap); - } - if( reqMapSize>0 ){ - pShmNode->pMMapBuf = mmap(0, reqMapSize, PROT_READ|PROT_WRITE, MAP_SHARED, - pShmNode->h, 0); - pShmNode->szMap = pShmNode->pMMapBuf ? reqMapSize : 0; - }else{ - pShmNode->pMMapBuf = 0; - pShmNode->szMap = 0; - } - } - *pNewMapSize = pShmNode->szMap; - *ppBuf = pShmNode->pMMapBuf; - sqlite3_mutex_leave(pShmNode->mutex); - if( *ppBuf==0 ){ - /* Do not hold the mutex if a NULL pointer is being returned. */ - unixShmRelease(fd); - } - return rc; -} - - /* ** Change the lock state for a shared-memory segment. ** @@ -3711,16 +3548,35 @@ static int unixShmLock( ** any load or store begun after the barrier. */ static void unixShmBarrier( - sqlite3_file *fd /* Database file holding the shared memory */ + sqlite3_file *fd /* Database file holding the shared memory */ ){ unixEnterMutex(); unixLeaveMutex(); } +/* +** This function is called to obtain a pointer to region iRegion of the +** shared-memory associated with the database file fd. Shared-memory regions +** are numbered starting from zero. Each shared-memory region is szRegion +** bytes in size. +** +** If an error occurs, an error code is returned and *pp is set to NULL. +** +** Otherwise, if the isWrite parameter is 0 and the requested shared-memory +** region has not been allocated (by any client, including one running in a +** separate process), then *pp is set to NULL and SQLITE_OK returned. If +** isWrite is non-zero and the requested shared-memory region has not yet +** been allocated, it is allocated by this function. +** +** If the shared-memory region has already been allocated or is allocated by +** this call as described above, then it is mapped into this processes +** address space (if it is not already), *pp is set to point to the mapped +** memory and SQLITE_OK returned. +*/ static int unixShmPage( sqlite3_file *fd, /* Handle open on database file */ - int iPage, /* Page to retrieve */ - int pgsz, /* Size of pages */ + int iRegion, /* Region to retrieve */ + int szRegion, /* Size of regions */ int isWrite, /* True to extend file if necessary */ void volatile **pp /* OUT: Mapped memory */ ){ @@ -3729,22 +3585,32 @@ static int unixShmPage( unixShmNode *pShmNode = p->pShmNode; int rc = SQLITE_OK; - assert( p->hasMutexBuf==0 ); - sqlite3_mutex_enter(pShmNode->mutexBuf); - assert( pgsz==pShmNode->pgsz || pShmNode->nPage==0 ); + sqlite3_mutex_enter(pShmNode->mutex); + assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); - if( pShmNode->nPage<=iPage ){ - char **apNew; /* New apPage[] array */ - int nByte = (iPage+1)*pgsz; /* Minimum required file size */ - struct stat sStat; + if( pShmNode->nRegion<=iRegion ){ + char **apNew; /* New apRegion[] array */ + int nByte = (iRegion+1)*szRegion; /* Minimum required file size */ + struct stat sStat; /* Used by fstat() */ - pShmNode->pgsz = pgsz; + pShmNode->szRegion = szRegion; - /* Make sure the underlying file is large enough (or fail) */ + /* The requested region is not mapped into this processes address space. + ** Check to see if it has been allocated (i.e. if the wal-index file is + ** large enough to contain the requested region). + */ if( fstat(pShmNode->h, &sStat) ){ rc = SQLITE_IOERR_SHMSIZE; goto shmpage_out; - }else if( sStat.st_sizeh, nByte) ){ rc = SQLITE_IOERR_SHMSIZE; @@ -3752,42 +3618,40 @@ static int unixShmPage( } } - apNew = (char**)sqlite3_realloc(pShmNode->apPage, (iPage+1)*sizeof(char *)); + /* Map the requested memory region into this processes address space. */ + apNew = (char **)sqlite3_realloc( + pShmNode->apRegion, (iRegion+1)*sizeof(char *) + ); if( !apNew ){ rc = SQLITE_IOERR_NOMEM; goto shmpage_out; } - pShmNode->apPage = apNew; - - while(pShmNode->nPage<=iPage){ - void *pMem = mmap( - 0, pgsz, PROT_READ|PROT_WRITE, MAP_SHARED, pShmNode->h, iPage*pgsz + pShmNode->apRegion = apNew; + while(pShmNode->nRegion<=iRegion){ + void *pMem = mmap(0, szRegion, PROT_READ|PROT_WRITE, + MAP_SHARED, pShmNode->h, iRegion*szRegion ); if( pMem==MAP_FAILED ){ - assert(0); rc = SQLITE_IOERR; goto shmpage_out; } - pShmNode->apPage[pShmNode->nPage] = pMem; - pShmNode->nPage++; + pShmNode->apRegion[pShmNode->nRegion] = pMem; + pShmNode->nRegion++; } } shmpage_out: - if( pShmNode->nPage>iPage ){ - *pp = pShmNode->apPage[iPage]; + if( pShmNode->nRegion>iRegion ){ + *pp = pShmNode->apRegion[iRegion]; }else{ *pp = 0; } - sqlite3_mutex_leave(pShmNode->mutexBuf); + sqlite3_mutex_leave(pShmNode->mutex); return rc; } #else # define unixShmOpen 0 -# define unixShmSize 0 -# define unixShmGet 0 -# define unixShmRelease 0 # define unixShmLock 0 # define unixShmBarrier 0 # define unixShmClose 0 @@ -3850,9 +3714,6 @@ static const sqlite3_io_methods METHOD = { \ unixSectorSize, /* xSectorSize */ \ unixDeviceCharacteristics, /* xDeviceCapabilities */ \ unixShmOpen, /* xShmOpen */ \ - unixShmSize, /* xShmSize */ \ - unixShmGet, /* xShmGet */ \ - unixShmRelease, /* xShmRelease */ \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ unixShmClose, /* xShmClose */ \ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 94b994ee39..f6ec12b988 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -660,13 +660,10 @@ struct sqlite3_io_methods { int (*xDeviceCharacteristics)(sqlite3_file*); /* Methods above are valid for version 1 */ int (*xShmOpen)(sqlite3_file*); - int (*xShmSize)(sqlite3_file*, int reqSize, int *pNewSize); - int (*xShmGet)(sqlite3_file*, int reqSize, int *pSize, void volatile**); - int (*xShmRelease)(sqlite3_file*); int (*xShmLock)(sqlite3_file*, int offset, int n, int flags); void (*xShmBarrier)(sqlite3_file*); int (*xShmClose)(sqlite3_file*, int deleteFlag); - int (*xShmPage)(sqlite3_file*, int iPage, int pgsz, int, void volatile**); + int (*xShmMap)(sqlite3_file*, int iPage, int pgsz, int, void volatile**); /* Methods above are valid for version 2 */ /* Additional methods may be added in future releases */ }; diff --git a/src/test6.c b/src/test6.c index 7acc1196e3..d6e6db2c18 100644 --- a/src/test6.c +++ b/src/test6.c @@ -526,20 +526,6 @@ static int cfDeviceCharacteristics(sqlite3_file *pFile){ static int cfShmOpen(sqlite3_file *pFile){ return sqlite3OsShmOpen(((CrashFile*)pFile)->pRealFile); } -static int cfShmSize(sqlite3_file *pFile, int reqSize, int *pNew){ - return sqlite3OsShmSize(((CrashFile*)pFile)->pRealFile, reqSize, pNew); -} -static int cfShmGet( - sqlite3_file *pFile, - int reqSize, - int *pSize, - void volatile **pp -){ - return sqlite3OsShmGet(((CrashFile*)pFile)->pRealFile, reqSize, pSize, pp); -} -static int cfShmRelease(sqlite3_file *pFile){ - return sqlite3OsShmRelease(((CrashFile*)pFile)->pRealFile); -} static int cfShmLock(sqlite3_file *pFile, int ofst, int n, int flags){ return sqlite3OsShmLock(((CrashFile*)pFile)->pRealFile, ofst, n, flags); } @@ -549,17 +535,16 @@ static void cfShmBarrier(sqlite3_file *pFile){ static int cfShmClose(sqlite3_file *pFile, int delFlag){ return sqlite3OsShmClose(((CrashFile*)pFile)->pRealFile, delFlag); } -static int cfShmPage( +static int cfShmMap( sqlite3_file *pFile, /* Handle open on database file */ - int iPage, /* Page to retrieve */ - int pgsz, /* Size of pages */ + int iRegion, /* Region to retrieve */ + int sz, /* Size of regions */ int w, /* True to extend file if necessary */ void volatile **pp /* OUT: Mapped memory */ ){ - return sqlite3OsShmPage(((CrashFile*)pFile)->pRealFile, iPage, pgsz, w, pp); + return sqlite3OsShmMap(((CrashFile*)pFile)->pRealFile, iRegion, sz, w, pp); } - static const sqlite3_io_methods CrashFileVtab = { 2, /* iVersion */ cfClose, /* xClose */ @@ -575,13 +560,10 @@ static const sqlite3_io_methods CrashFileVtab = { cfSectorSize, /* xSectorSize */ cfDeviceCharacteristics, /* xDeviceCharacteristics */ cfShmOpen, /* xShmOpen */ - cfShmSize, /* xShmSize */ - cfShmGet, /* xShmGet */ - cfShmRelease, /* xShmRelease */ cfShmLock, /* xShmLock */ cfShmBarrier, /* xShmBarrier */ cfShmClose, /* xShmClose */ - cfShmPage /* xShmPage */ + cfShmMap /* xShmMap */ }; /* diff --git a/src/test_devsym.c b/src/test_devsym.c index e27c00176e..98d6e2a30e 100644 --- a/src/test_devsym.c +++ b/src/test_devsym.c @@ -51,13 +51,10 @@ static int devsymFileControl(sqlite3_file*, int op, void *pArg); static int devsymSectorSize(sqlite3_file*); static int devsymDeviceCharacteristics(sqlite3_file*); static int devsymShmOpen(sqlite3_file*); -static int devsymShmSize(sqlite3_file*,int,int*); -static int devsymShmGet(sqlite3_file*,int,int*,volatile void**); -static int devsymShmRelease(sqlite3_file*); static int devsymShmLock(sqlite3_file*,int,int,int); static void devsymShmBarrier(sqlite3_file*); static int devsymShmClose(sqlite3_file*,int); -static int devsymShmPage(sqlite3_file*,int,int,int, void volatile **); +static int devsymShmMap(sqlite3_file*,int,int,int, void volatile **); /* ** Method declarations for devsym_vfs. @@ -121,13 +118,10 @@ static sqlite3_io_methods devsym_io_methods = { devsymSectorSize, /* xSectorSize */ devsymDeviceCharacteristics, /* xDeviceCharacteristics */ devsymShmOpen, /* xShmOpen */ - devsymShmSize, /* xShmSize */ - devsymShmGet, /* xShmGet */ - devsymShmRelease, /* xShmRelease */ devsymShmLock, /* xShmLock */ devsymShmBarrier, /* xShmBarrier */ devsymShmClose, /* xShmClose */ - devsymShmPage /* xShmPage */ + devsymShmMap /* xShmMap */ }; struct DevsymGlobal { @@ -248,23 +242,6 @@ static int devsymShmOpen(sqlite3_file *pFile){ devsym_file *p = (devsym_file *)pFile; return sqlite3OsShmOpen(p->pReal); } -static int devsymShmSize(sqlite3_file *pFile, int reqSize, int *pSize){ - devsym_file *p = (devsym_file *)pFile; - return sqlite3OsShmSize(p->pReal, reqSize, pSize); -} -static int devsymShmGet( - sqlite3_file *pFile, - int reqSz, - int *pSize, - void volatile **pp -){ - devsym_file *p = (devsym_file *)pFile; - return sqlite3OsShmGet(p->pReal, reqSz, pSize, pp); -} -static int devsymShmRelease(sqlite3_file *pFile){ - devsym_file *p = (devsym_file *)pFile; - return sqlite3OsShmRelease(p->pReal); -} static int devsymShmLock(sqlite3_file *pFile, int ofst, int n, int flags){ devsym_file *p = (devsym_file *)pFile; return sqlite3OsShmLock(p->pReal, ofst, n, flags); @@ -277,15 +254,15 @@ static int devsymShmClose(sqlite3_file *pFile, int delFlag){ devsym_file *p = (devsym_file *)pFile; return sqlite3OsShmClose(p->pReal, delFlag); } -static int devsymShmPage( +static int devsymShmMap( sqlite3_file *pFile, - int iPage, - int pgsz, + int iRegion, + int szRegion, int isWrite, void volatile **pp ){ devsym_file *p = (devsym_file *)pFile; - return sqlite3OsShmPage(p->pReal, iPage, pgsz, isWrite, pp); + return sqlite3OsShmMap(p->pReal, iRegion, szRegion, isWrite, pp); } diff --git a/src/test_osinst.c b/src/test_osinst.c index b60f84a572..f97822a1ab 100644 --- a/src/test_osinst.c +++ b/src/test_osinst.c @@ -100,11 +100,9 @@ #define OS_WRITE 20 #define OS_SHMOPEN 21 #define OS_SHMCLOSE 22 -#define OS_SHMGET 23 -#define OS_SHMRELEASE 24 +#define OS_SHMMAP 23 #define OS_SHMLOCK 25 #define OS_SHMBARRIER 26 -#define OS_SHMSIZE 27 #define OS_ANNOTATE 28 #define OS_NUMEVENTS 29 @@ -152,12 +150,10 @@ static int vfslogSectorSize(sqlite3_file*); static int vfslogDeviceCharacteristics(sqlite3_file*); static int vfslogShmOpen(sqlite3_file *pFile); -static int vfslogShmSize(sqlite3_file *pFile, int reqSize, int *pNewSize); -static int vfslogShmGet(sqlite3_file *pFile, int,int*,volatile void **); -static int vfslogShmRelease(sqlite3_file *pFile); static int vfslogShmLock(sqlite3_file *pFile, int ofst, int n, int flags); static void vfslogShmBarrier(sqlite3_file*); static int vfslogShmClose(sqlite3_file *pFile, int deleteFlag); +static int vfslogShmMap(sqlite3_file *pFile,int,int,int,volatile void **); /* ** Method declarations for vfslog_vfs. @@ -216,12 +212,10 @@ static sqlite3_io_methods vfslog_io_methods = { vfslogSectorSize, /* xSectorSize */ vfslogDeviceCharacteristics, /* xDeviceCharacteristics */ vfslogShmOpen, /* xShmOpen */ - vfslogShmSize, /* xShmSize */ - vfslogShmGet, /* xShmGet */ - vfslogShmRelease, /* xShmRelease */ vfslogShmLock, /* xShmLock */ vfslogShmBarrier, /* xShmBarrier */ - vfslogShmClose /* xShmClose */ + vfslogShmClose, /* xShmClose */ + vfslogShmMap /* xShmMap */ }; #if defined(SQLITE_OS_UNIX) && !defined(NO_GETTOD) @@ -441,41 +435,6 @@ static int vfslogShmOpen(sqlite3_file *pFile){ vfslog_call(p->pVfslog, OS_SHMOPEN, p->iFileId, t, rc, 0, 0); return rc; } -static int vfslogShmSize(sqlite3_file *pFile, int reqSize, int *pNewSize){ - int rc; - sqlite3_uint64 t; - VfslogFile *p = (VfslogFile *)pFile; - t = vfslog_time(); - rc = p->pReal->pMethods->xShmSize(p->pReal, reqSize, pNewSize); - t = vfslog_time() - t; - vfslog_call(p->pVfslog, OS_SHMSIZE, p->iFileId, t, rc, 0, 0); - return rc; -} -static int vfslogShmGet( - sqlite3_file *pFile, - int req, - int *pSize, - volatile void **pp -){ - int rc; - sqlite3_uint64 t; - VfslogFile *p = (VfslogFile *)pFile; - t = vfslog_time(); - rc = p->pReal->pMethods->xShmGet(p->pReal, req, pSize, pp); - t = vfslog_time() - t; - vfslog_call(p->pVfslog, OS_SHMGET, p->iFileId, t, rc, 0, 0); - return rc; -} -static int vfslogShmRelease(sqlite3_file *pFile){ - int rc; - sqlite3_uint64 t; - VfslogFile *p = (VfslogFile *)pFile; - t = vfslog_time(); - rc = p->pReal->pMethods->xShmRelease(p->pReal); - t = vfslog_time() - t; - vfslog_call(p->pVfslog, OS_SHMRELEASE, p->iFileId, t, rc, 0, 0); - return rc; -} static int vfslogShmLock(sqlite3_file *pFile, int ofst, int n, int flags){ int rc; sqlite3_uint64 t; @@ -504,6 +463,22 @@ static int vfslogShmClose(sqlite3_file *pFile, int deleteFlag){ vfslog_call(p->pVfslog, OS_SHMCLOSE, p->iFileId, t, rc, 0, 0); return rc; } +static int vfslogShmMap( + sqlite3_file *pFile, + int iRegion, + int szRegion, + int isWrite, + volatile void **pp +){ + int rc; + sqlite3_uint64 t; + VfslogFile *p = (VfslogFile *)pFile; + t = vfslog_time(); + rc = p->pReal->pMethods->xShmMap(p->pReal, iRegion, szRegion, isWrite, pp); + t = vfslog_time() - t; + vfslog_call(p->pVfslog, OS_SHMMAP, p->iFileId, t, rc, 0, 0); + return rc; +} /* @@ -826,11 +801,9 @@ static const char *vfslog_eventname(int eEvent){ case OS_SHMCLOSE: zEvent = "xShmClose"; break; case OS_SHMOPEN: zEvent = "xShmOpen"; break; - case OS_SHMGET: zEvent = "xShmGet"; break; - case OS_SHMSIZE: zEvent = "xShmSize"; break; - case OS_SHMRELEASE: zEvent = "xShmRelease"; break; case OS_SHMLOCK: zEvent = "xShmLock"; break; case OS_SHMBARRIER: zEvent = "xShmBarrier"; break; + case OS_SHMMAP: zEvent = "xShmMap"; break; case OS_ANNOTATE: zEvent = "annotation"; break; } diff --git a/src/test_vfs.c b/src/test_vfs.c index 9b212099c6..89cc842caf 100644 --- a/src/test_vfs.c +++ b/src/test_vfs.c @@ -69,9 +69,6 @@ struct Testvfs { ** + Invoking the Tcl callback script. */ #define TESTVFS_SHMOPEN_MASK 0x00000001 -#define TESTVFS_SHMSIZE_MASK 0x00000002 -#define TESTVFS_SHMGET_MASK 0x00000004 -#define TESTVFS_SHMRELEASE_MASK 0x00000008 #define TESTVFS_SHMLOCK_MASK 0x00000010 #define TESTVFS_SHMBARRIER_MASK 0x00000020 #define TESTVFS_SHMCLOSE_MASK 0x00000040 @@ -137,9 +134,6 @@ static int tvfsSleep(sqlite3_vfs*, int microseconds); static int tvfsCurrentTime(sqlite3_vfs*, double*); static int tvfsShmOpen(sqlite3_file*); -static int tvfsShmSize(sqlite3_file*, int , int *); -static int tvfsShmGet(sqlite3_file*, int , int *, volatile void **); -static int tvfsShmRelease(sqlite3_file*); static int tvfsShmLock(sqlite3_file*, int , int, int); static void tvfsShmBarrier(sqlite3_file*); static int tvfsShmClose(sqlite3_file*, int); @@ -160,9 +154,6 @@ static sqlite3_io_methods tvfs_io_methods = { tvfsSectorSize, /* xSectorSize */ tvfsDeviceCharacteristics, /* xDeviceCharacteristics */ tvfsShmOpen, /* xShmOpen */ - tvfsShmSize, /* xShmSize */ - tvfsShmGet, /* xShmGet */ - tvfsShmRelease, /* xShmRelease */ tvfsShmLock, /* xShmLock */ tvfsShmBarrier, /* xShmBarrier */ tvfsShmClose, /* xShmClose */ @@ -449,12 +440,10 @@ static int tvfsOpen( memcpy(pMethods, &tvfs_io_methods, sizeof(sqlite3_io_methods)); if( ((Testvfs *)pVfs->pAppData)->isNoshm ){ pMethods->xShmOpen = 0; - pMethods->xShmGet = 0; - pMethods->xShmSize = 0; - pMethods->xShmRelease = 0; pMethods->xShmClose = 0; pMethods->xShmLock = 0; pMethods->xShmBarrier = 0; + pMethods->xShmMap = 0; } pFile->pMethods = pMethods; } @@ -615,28 +604,6 @@ static int tvfsShmOpen( return SQLITE_OK; } -static int tvfsShmSize( - sqlite3_file *pFile, - int reqSize, - int *pNewSize -){ - assert(0); - return SQLITE_OK; -} -static int tvfsShmGet( - sqlite3_file *pFile, - int reqMapSize, - int *pMapSize, - volatile void **pp -){ - assert(0); - return SQLITE_OK; -} -static int tvfsShmRelease(sqlite3_file *pFile){ - assert(0); - return SQLITE_OK; -} - static void tvfsAllocPage(TestvfsBuffer *p, int iPage, int pgsz){ assert( iPageaPage[iPage]==0 ){ @@ -871,9 +838,6 @@ static int testvfs_obj_cmd( int mask; } vfsmethod [] = { { "xShmOpen", TESTVFS_SHMOPEN_MASK }, - { "xShmSize", TESTVFS_SHMSIZE_MASK }, - { "xShmGet", TESTVFS_SHMGET_MASK }, - { "xShmRelease", TESTVFS_SHMRELEASE_MASK }, { "xShmLock", TESTVFS_SHMLOCK_MASK }, { "xShmBarrier", TESTVFS_SHMBARRIER_MASK }, { "xShmClose", TESTVFS_SHMCLOSE_MASK }, diff --git a/src/wal.c b/src/wal.c index cd31bc3f81..775f9556bc 100644 --- a/src/wal.c +++ b/src/wal.c @@ -483,7 +483,7 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ /* Request a pointer to the required page from the VFS */ if( pWal->apWiData[iPage]==0 ){ - rc = sqlite3OsShmPage(pWal->pDbFd, iPage, WALINDEX_PGSZ, + rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] ); } From 9785fc9510ac710fc15f4e6a5e4012de48a47c98 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 14 Jun 2010 16:16:33 +0000 Subject: [PATCH 8/8] Add the new xShmMap (formerly xShmPage) to os_win.c. FossilOrigin-Name: 13e7a8242206bca4b5bf356ef074e66474d39609 --- manifest | 14 +-- manifest.uuid | 2 +- src/os_unix.c | 6 +- src/os_win.c | 272 +++++++++++++++++++------------------------------- 4 files changed, 115 insertions(+), 179 deletions(-) diff --git a/manifest b/manifest index b211bcfee4..86656a4e33 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sxShmGet/Size/Release\sfrom\sthe\ssqlite3_vfs\sstructure.\sChange\sthe\sname\sof\sxShmPage\sto\sxShmMap.\sRemove\ssome\scode\sthat\sis\snow\sunused\sfrom\sos_unix.c\sand\ssome\sof\sthe\stest\sVFS\simplementations. -D 2010-06-14T14:07:51 +C Add\sthe\snew\sxShmMap\s(formerly\sxShmPage)\sto\sos_win.c. +D 2010-06-14T16:16:34 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0 F Makefile.in a5cad1f8f3e021356bfcc6c77dc16f6f1952bbc3 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654 @@ -154,8 +154,8 @@ F src/os.c 9c4a2f82a50306a33907678ec0187b6ad1486bfe F src/os.h d7775504a51e6e0d40315aa427b3e229ff9ff9ca F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 665876d5eec7585226b0a1cf5e18098de2b2da19 -F src/os_unix.c 0a57b86241011ca25f2fb83e4825b7af7fe6d920 -F src/os_win.c 0cf1f571546f165001e2391b5d4a4a16d86977d3 +F src/os_unix.c 710418915cc3b59e596ee7ae916deca5dc740ca7 +F src/os_win.c 3d761e33687cb24645fe66e5efd11244390e8e9c F src/pager.c 2964185d4356d0dc159b8340e52d2538d32394e5 F src/pager.h ca1f23c0cf137ac26f8908df2427c8b308361efd F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e @@ -820,7 +820,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 1ce9c92bffa5d7f8431c005b29d698b0f5b95875 -R 8ee76bbe532e6db3f47bf2104ddf1a48 +P fc0cabc15c97dde6a852b4f07df6d30f1d2c04bc +R 25e35232d2e784a0633c25f463199a04 U dan -Z 95a9a1111fae03b142846d91ecde0c57 +Z 06f3bcfd0aba7d97df7e9b4406c194ab diff --git a/manifest.uuid b/manifest.uuid index 6a3db6fb58..13f5f78d56 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fc0cabc15c97dde6a852b4f07df6d30f1d2c04bc \ No newline at end of file +13e7a8242206bca4b5bf356ef074e66474d39609 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 22c9a92826..fcccead725 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3573,7 +3573,7 @@ static void unixShmBarrier( ** address space (if it is not already), *pp is set to point to the mapped ** memory and SQLITE_OK returned. */ -static int unixShmPage( +static int unixShmMap( sqlite3_file *fd, /* Handle open on database file */ int iRegion, /* Region to retrieve */ int szRegion, /* Size of regions */ @@ -3655,7 +3655,7 @@ shmpage_out: # define unixShmLock 0 # define unixShmBarrier 0 # define unixShmClose 0 -# define unixShmPage 0 +# define unixShmMap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ /* @@ -3717,7 +3717,7 @@ static const sqlite3_io_methods METHOD = { \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ unixShmClose, /* xShmClose */ \ - unixShmPage /* xShmPage */ \ + unixShmMap /* xShmMap */ \ }; \ static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \ UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \ diff --git a/src/os_win.c b/src/os_win.c index 1a9994b08d..3a6b47771a 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1216,13 +1216,17 @@ static int winShmMutexHeld(void) { */ struct winShmNode { sqlite3_mutex *mutex; /* Mutex to access this object */ - sqlite3_mutex *mutexBuf; /* Mutex to access zBuf[] */ char *zFilename; /* Name of the file */ winFile hFile; /* File handle from winOpen */ - HANDLE hMap; /* File handle from CreateFileMapping */ + + int szRegion; /* Size of shared-memory regions */ + int nRegion; /* Size of array apRegion */ + struct ShmRegion { + HANDLE hMap; /* File handle from CreateFileMapping */ + void *pMap; + } *aRegion; DWORD lastErrno; /* The Windows errno from the last I/O error */ - int szMap; /* Size of the mapping of file into memory */ - char *pMMapBuf; /* Where currently mmapped(). NULL if unmapped */ + int nRef; /* Number of winShm objects pointing to this */ winShm *pFirst; /* All winShm objects pointing to this */ winShmNode *pNext; /* Next in list of all winShmNode objects */ @@ -1325,19 +1329,18 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ pp = &winShmNodeList; while( (p = *pp)!=0 ){ if( p->nRef==0 ){ + int i; if( p->mutex ) sqlite3_mutex_free(p->mutex); - if( p->mutexBuf ) sqlite3_mutex_free(p->mutexBuf); - if( p->pMMapBuf ){ - UnmapViewOfFile(p->pMMapBuf); - } - if( INVALID_HANDLE_VALUE != p->hMap ){ - CloseHandle(p->hMap); + for(i=0; inRegion; i++){ + UnmapViewOfFile(p->aRegion[i].pMap); + CloseHandle(p->aRegion[i].hMap); } if( p->hFile.h != INVALID_HANDLE_VALUE ) { winClose((sqlite3_file *)&p->hFile); } if( deleteFlag ) winDelete(pVfs, p->zFilename, 0); *pp = p->pNext; + sqlite3_free(p->aRegion); sqlite3_free(p); }else{ pp = &p->pNext; @@ -1404,8 +1407,6 @@ static int winShmOpen( }else{ pShmNode = pNew; pNew = 0; - pShmNode->pMMapBuf = NULL; - pShmNode->hMap = INVALID_HANDLE_VALUE; ((winFile*)(&pShmNode->hFile))->h = INVALID_HANDLE_VALUE; pShmNode->pNext = winShmNodeList; winShmNodeList = pShmNode; @@ -1415,11 +1416,6 @@ static int winShmOpen( rc = SQLITE_NOMEM; goto shm_open_err; } - pShmNode->mutexBuf = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutexBuf==0 ){ - rc = SQLITE_NOMEM; - goto shm_open_err; - } rc = winOpen(pDbFd->pVfs, pShmNode->zFilename, /* Name of the file (UTF-8) */ (sqlite3_file*)&pShmNode->hFile, /* File handle here */ @@ -1507,173 +1503,115 @@ static int winShmClose( } /* -** Increase the size of the underlying storage for a shared-memory segment. +** This function is called to obtain a pointer to region iRegion of the +** shared-memory associated with the database file fd. Shared-memory regions +** are numbered starting from zero. Each shared-memory region is szRegion +** bytes in size. ** -** The reqSize parameter is the new requested minimum size of the underlying -** shared memory. This routine may choose to make the shared memory larger -** than this value (for example to round the shared memory size up to an -** operating-system dependent page size.) +** If an error occurs, an error code is returned and *pp is set to NULL. ** -** This routine will only grow the size of shared memory. A request for -** a smaller size is a no-op. +** Otherwise, if the isWrite parameter is 0 and the requested shared-memory +** region has not been allocated (by any client, including one running in a +** separate process), then *pp is set to NULL and SQLITE_OK returned. If +** isWrite is non-zero and the requested shared-memory region has not yet +** been allocated, it is allocated by this function. +** +** If the shared-memory region has already been allocated or is allocated by +** this call as described above, then it is mapped into this processes +** address space (if it is not already), *pp is set to point to the mapped +** memory and SQLITE_OK returned. */ -static int winShmSize( - sqlite3_file *fd, /* Database holding the shared memory */ - int reqSize, /* Requested size. -1 for query only */ - int *pNewSize /* Write new size here */ +static int winShmMap( + sqlite3_file *fd, /* Handle open on database file */ + int iRegion, /* Region to retrieve */ + int szRegion, /* Size of regions */ + int isWrite, /* True to extend file if necessary */ + void volatile **pp /* OUT: Mapped memory */ ){ winFile *pDbFd = (winFile*)fd; winShm *p = pDbFd->pShm; winShmNode *pShmNode = p->pShmNode; int rc = SQLITE_OK; - *pNewSize = 0; - if( reqSize>=0 ){ - sqlite3_int64 sz; - rc = winFileSize((sqlite3_file *)&pShmNode->hFile, &sz); - if( SQLITE_OK==rc && reqSize>sz ){ - rc = winTruncate((sqlite3_file *)&pShmNode->hFile, reqSize); - } - } - if( SQLITE_OK==rc ){ - sqlite3_int64 sz; - rc = winFileSize((sqlite3_file *)&pShmNode->hFile, &sz); - if( SQLITE_OK==rc ){ - *pNewSize = (int)sz; - }else{ - rc = SQLITE_IOERR; - } - } - return rc; -} - - -/* -** Map the shared storage into memory. The minimum size of the -** mapping should be reqMapSize if reqMapSize is positive. If -** reqMapSize is zero or negative, the implementation can choose -** whatever mapping size is convenient. -** -** *ppBuf is made to point to the memory which is a mapping of the -** underlying storage. A mutex is acquired to prevent other threads -** from running while *ppBuf is in use in order to prevent other threads -** remapping *ppBuf out from under this thread. The winShmRelease() -** call will release the mutex. However, if the lock state is CHECKPOINT, -** the mutex is not acquired because CHECKPOINT will never remap the -** buffer. RECOVER might remap, though, so CHECKPOINT will acquire -** the mutex if and when it promotes to RECOVER. -** -** RECOVER needs to be atomic. The same mutex that prevents *ppBuf from -** being remapped also prevents more than one thread from being in -** RECOVER at a time. But, RECOVER sometimes wants to remap itself. -** To prevent RECOVER from losing its lock while remapping, the -** mutex is not released by winShmRelease() when in RECOVER. -** -** *pNewMapSize is set to the size of the mapping. -** -** *ppBuf and *pNewMapSize might be NULL and zero if no space has -** yet been allocated to the underlying storage. -*/ -static int winShmGet( - sqlite3_file *fd, /* The database file holding the shared memory */ - int reqMapSize, /* Requested size of mapping. -1 means don't care */ - int *pNewMapSize, /* Write new size of mapping here */ - void volatile **ppBuf /* Write mapping buffer origin here */ -){ - winFile *pDbFd = (winFile*)fd; - winShm *p = pDbFd->pShm; - winShmNode *pShmNode = p->pShmNode; - int rc = SQLITE_OK; - - if( p->hasMutexBuf==0 ){ - assert( sqlite3_mutex_notheld(pShmNode->mutex) ); - sqlite3_mutex_enter(pShmNode->mutexBuf); - p->hasMutexBuf = 1; - } sqlite3_mutex_enter(pShmNode->mutex); - if( pShmNode->szMap==0 || reqMapSize>pShmNode->szMap ){ - int actualSize; - if( winShmSize(fd, -1, &actualSize)==SQLITE_OK - && reqMapSizeszRegion || pShmNode->nRegion==0 ); + + if( pShmNode->nRegion<=iRegion ){ + struct ShmRegion *apNew; /* New aRegion[] array */ + int nByte = (iRegion+1)*szRegion; /* Minimum required file size */ + sqlite3_int64 sz; /* Current size of wal-index file */ + + pShmNode->szRegion = szRegion; + + /* The requested region is not mapped into this processes address space. + ** Check to see if it has been allocated (i.e. if the wal-index file is + ** large enough to contain the requested region). + */ + rc = winFileSize((sqlite3_file *)&pShmNode->hFile, &sz); + if( rc!=SQLITE_OK ){ + goto shmpage_out; } - if( pShmNode->pMMapBuf ){ - if( !UnmapViewOfFile(pShmNode->pMMapBuf) ){ + + if( szhFile, nByte); + if( rc!=SQLITE_OK ){ + goto shmpage_out; + } + } + + /* Map the requested memory region into this processes address space. */ + apNew = (struct ShmRegion *)sqlite3_realloc( + pShmNode->aRegion, (iRegion+1)*sizeof(apNew[0]) + ); + if( !apNew ){ + rc = SQLITE_IOERR_NOMEM; + goto shmpage_out; + } + pShmNode->aRegion = apNew; + + while( pShmNode->nRegion<=iRegion ){ + HANDLE hMap; /* file-mapping handle */ + void *pMap = 0; /* Mapped memory region */ + + hMap = CreateFileMapping(pShmNode->hFile.h, + NULL, PAGE_READWRITE, 0, nByte, NULL + ); + if( hMap ){ + pMap = MapViewOfFile(hMap, FILE_MAP_WRITE | FILE_MAP_READ, + 0, 0, nByte + ); + } + if( !pMap ){ pShmNode->lastErrno = GetLastError(); rc = SQLITE_IOERR; + if( hMap ) CloseHandle(hMap); + goto shmpage_out; } - CloseHandle(pShmNode->hMap); - pShmNode->hMap = INVALID_HANDLE_VALUE; - } - if( SQLITE_OK == rc ){ - pShmNode->pMMapBuf = 0; - if( reqMapSize == 0 ){ - /* can't create 0 byte file mapping in Windows */ - pShmNode->szMap = 0; - }else{ - /* create the file mapping object */ - if( INVALID_HANDLE_VALUE == pShmNode->hMap ){ - /* TBD provide an object name to each file - ** mapping so it can be re-used across processes. - */ - pShmNode->hMap = CreateFileMapping(pShmNode->hFile.h, - NULL, - PAGE_READWRITE, - 0, - reqMapSize, - NULL); - } - if( NULL==pShmNode->hMap ){ - pShmNode->lastErrno = GetLastError(); - rc = SQLITE_IOERR; - pShmNode->szMap = 0; - pShmNode->hMap = INVALID_HANDLE_VALUE; - }else{ - pShmNode->pMMapBuf = MapViewOfFile(pShmNode->hMap, - FILE_MAP_WRITE | FILE_MAP_READ, - 0, - 0, - reqMapSize); - if( !pShmNode->pMMapBuf ){ - pShmNode->lastErrno = GetLastError(); - rc = SQLITE_IOERR; - pShmNode->szMap = 0; - }else{ - pShmNode->szMap = reqMapSize; - } - } - } + + pShmNode->aRegion[pShmNode->nRegion].pMap = pMap; + pShmNode->aRegion[pShmNode->nRegion].hMap = hMap; + pShmNode->nRegion++; } } - *pNewMapSize = pShmNode->szMap; - *ppBuf = pShmNode->pMMapBuf; + +shmpage_out: + if( pShmNode->nRegion>iRegion ){ + char *p = (char *)pShmNode->aRegion[iRegion].pMap; + *pp = (void *)&p[iRegion*szRegion]; + }else{ + *pp = 0; + } sqlite3_mutex_leave(pShmNode->mutex); return rc; } -/* -** Release the lock held on the shared memory segment so that other -** threads are free to resize it if necessary. -** -** If the lock is not currently held, this routine is a harmless no-op. -** -** If the shared-memory object is in lock state RECOVER, then we do not -** really want to release the lock, so in that case too, this routine -** is a no-op. -*/ -static int winShmRelease(sqlite3_file *fd){ - winFile *pDbFd = (winFile*)fd; - winShm *p = pDbFd->pShm; - if( p->hasMutexBuf ){ - winShmNode *pShmNode = p->pShmNode; - assert( sqlite3_mutex_notheld(pShmNode->mutex) ); - sqlite3_mutex_leave(pShmNode->mutexBuf); - p->hasMutexBuf = 0; - } - return SQLITE_OK; -} - /* ** Change the lock state for a shared-memory segment. */ @@ -1756,12 +1694,10 @@ static const sqlite3_io_methods winIoMethod = { winSectorSize, winDeviceCharacteristics, winShmOpen, /* xShmOpen */ - winShmSize, /* xShmSize */ - winShmGet, /* xShmGet */ - winShmRelease, /* xShmRelease */ winShmLock, /* xShmLock */ winShmBarrier, /* xShmBarrier */ - winShmClose /* xShmClose */ + winShmClose, /* xShmClose */ + winShmMap /* xShmMap */ }; /***************************************************************************