Merge the experimental shared-memory mmap-by-chunk changes into the trunk.

FossilOrigin-Name: f295e7ed5f04f6b8bffdaff8b565be3836ce4e80
This commit is contained in:
drh
2010-06-14 17:09:53 +00:00
16 changed files with 848 additions and 992 deletions
+22 -22
View File
@@ -1,8 +1,8 @@
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
C Refactor\sand\ssimplify\sthe\slogic\sused\sto\schange\sjournalmode.
D 2010-06-11T17:01:25
C Merge\sthe\sexperimental\sshared-memory\smmap-by-chunk\schanges\sinto\sthe\strunk.
D 2010-06-14T17:09:53
F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
F Makefile.in a5cad1f8f3e021356bfcc6c77dc16f6f1952bbc3
F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -153,12 +153,12 @@ F src/mutex_os2.c 6a62583e374ba3ac1a3fcc0da2bfdac7d3942689
F src/mutex_unix.c cf84466b4fdd2baa0d5a10bb19f08b2abc1ce42e
F src/mutex_w32.c 1fe0e735897be20e09dd6f53c3fb516c6b48c0eb
F src/notify.c cbfa66a836da3a51567209636e6a94059c137930
F src/os.c 1516984144e26734f97748f891f1a04f9e294c2e
F src/os.h 6f529984a29511c7a3479cfe549c10bfa131532f
F src/os.c 9c4a2f82a50306a33907678ec0187b6ad1486bfe
F src/os.h d7775504a51e6e0d40315aa427b3e229ff9ff9ca
F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f
F src/os_os2.c 665876d5eec7585226b0a1cf5e18098de2b2da19
F src/os_unix.c 12051d37e533cdaa8bb13c9d9fe2a13e08552187
F src/os_win.c 0cf1f571546f165001e2391b5d4a4a16d86977d3
F src/os_unix.c 710418915cc3b59e596ee7ae916deca5dc740ca7
F src/os_win.c 3d761e33687cb24645fe66e5efd11244390e8e9c
F src/pager.c 2964185d4356d0dc159b8340e52d2538d32394e5
F src/pager.h ca1f23c0cf137ac26f8908df2427c8b308361efd
F src/parse.y ace5c7a125d9f2a410e431ee3209034105045f7e
@@ -173,7 +173,7 @@ F src/resolve.c ac5f1a713cd1ae77f08b83cc69581e11bf5ae6f9
F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697
F src/select.c c03d8a0565febcde8c6a12c5d77d065fddae889b
F src/shell.c fd4ccdb37c3b68de0623eb938a649e0990710714
F src/sqlite.h.in b6a64327e174cf725e57dd93ddf1e97c52dd41e2
F src/sqlite.h.in 6e8e9454f6ada979405543b19e0c18ca16fd020e
F src/sqlite3ext.h 69dfb8116af51b84a029cddb3b35062354270c89
F src/sqliteInt.h 242987ebd2366ea36650a09cdab04a9163c62109
F src/sqliteLimit.h 196e2f83c3b444c4548fc1874f52f84fdbda40f3
@@ -185,7 +185,7 @@ F src/test2.c e3f564ab1e9fd0b47b0c9e23e7054e38bf0836cf
F src/test3.c 4c21700c73a890a47fc685c1097bfb661346ac94
F src/test4.c ad03bb987ddedce928f4258c1e7fa4109a73497d
F src/test5.c cc55900118fa4add8ec9cf69fc4225a4662f76b1
F src/test6.c d3e1a771a7b445ec771e64f2af70df5285da8e4c
F src/test6.c 43d6ce22d77b22f8c580be40990be6fa204d7f4d
F src/test7.c 3f2d63e4ccf97f8c2cf1a7fa0a3c8e2e2a354e6e
F src/test8.c f959db9a22d882013b64c92753fa793b2ce3bdea
F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60
@@ -195,7 +195,7 @@ F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de
F src/test_btree.c 47cd771250f09cdc6e12dda5bc71bc0b3abc96e2
F src/test_config.c 6210f501d358bde619ae761f06f123529c6ba24f
F src/test_demovfs.c da81a5f7785bb352bda7911c332a983ec4f17f27
F src/test_devsym.c 709712f5157667410cd0dad1b7b1b54319c122c5
F src/test_devsym.c b2c821471efb0f3680de0c60fbe1ac673517ddd6
F src/test_func.c 13b582345fb1185a93e46c53310fae8547dcce20
F src/test_hexio.c 1237f000ec7a491009b1233f5c626ea71bce1ea2
F src/test_init.c 5d624ffd0409d424cf9adbfe1f056b200270077c
@@ -206,13 +206,13 @@ F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e
F src/test_malloc.c 2842c922b8e8d992aba722214952204ca025b411
F src/test_mutex.c ce06b59aca168cd8c520b77159a24352a7469bd3
F src/test_onefile.c 4ce8c753c0240f010f0f2af89604875967d20945
F src/test_osinst.c a0df974ab32f9aae97da5ce094301926cffcdc17
F src/test_osinst.c 0d3d47cf3a8a0b4db9a5d0255b9d74c671ca19b4
F src/test_pcache.c 7bf828972ac0d2403f5cfa4cd14da41f8ebe73d8
F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0
F src/test_server.c bbba05c144b5fc4b52ff650a4328027b3fa5fcc6
F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa
F src/test_thread.c aa9919c885a1fe53eafc73492f0898ee6c0a0726
F src/test_vfs.c d329e3ea93624f65d7b6a46209861ddecea4e21d
F src/test_vfs.c 8e125e01b4f88c6a2e131d5a65d15f5c96d665f1
F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
F src/tokenize.c 25ceb0f0a746ea1d0f9553787f3f0a56853cfaeb
F src/trigger.c 8927588cb9e6d47f933b53bfe74200fbb504100d
@@ -229,7 +229,7 @@ F src/vdbeblob.c 5327132a42a91e8b7acfb60b9d2c3b1c5c863e0e
F src/vdbemem.c 2a82f455f6ca6f78b59fb312f96054c04ae0ead1
F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2
F src/vtab.c a0f8a40274e4261696ef57aa806de2776ab72cda
F src/wal.c 2cdfea9a5e50e4dde48767e69e1fead2ff1781cd
F src/wal.c c261aeb28c9b40f4713bace3c1c0cee9dcba87ab
F src/wal.h 4ace25262452d17e7d3ec970c89ee17794004008
F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f
F src/where.c 1c895bef33d0dfc7ed90fb1f74120435d210ea56
@@ -540,7 +540,7 @@ F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806
F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb
F test/pcache.test eebc4420b37cb07733ae9b6e99c9da7c40dd6d58
F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16
F test/permutations.test ad10d7b31b4a585977380886c832e2ac13c41237
F test/permutations.test 64fbafa685149be54a1ceb545942911f998c604d
F test/pragma.test 6960f9efbce476f70ba9ee2171daf5042f9e3d8a
F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47
F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea
@@ -768,14 +768,14 @@ F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5
F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d
F test/wal.test 0a599c3c4812ed92bc7ad9efcc2c4007fe4cc99a
F test/wal2.test 854a2b409450f1cb756c2bbd1e87e30740094357
F test/wal3.test ae876ff988af5b2b34d27474e0dd1a8c84e9bbcb
F test/wal2.test f9dce93acecff697fc1935869b1ae4cb7dc14587
F test/wal3.test 1d3aee1a0295db941a0323c0ce5ac16bd5b7689d
F test/wal_common.tcl 3e953ae60919281688ea73e4d0aa0e1bc94becd9
F test/walbak.test e7650a26eb4b8abeca9b145b1af1e63026dde432
F test/walcksum.test 4efa8fb88c32bed8288ea4385a9cc113a5c8f0bf
F test/walcrash.test f6d5fb2bb108876f04848720a488065d9deef69f
F test/walcrash2.test 14585ad1a2c85da2de721caa3b4deeea55213008
F test/walfault.test 0f9524cedbba0e5d48b58cabdc2dd7ae2c375476
F test/walfault.test f6c1d0eade34fd0102e083726092d9bebd48bf7b
F test/walhook.test 67e675127f4acb72f061a12667ce6e5460b06b78
F test/walmode.test 6ca9d710cc9f6545b913abcded6d6b0b15641048
F test/walslow.test d21625e2e99e11c032ce949e8a94661576548933
@@ -823,14 +823,14 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
P af353bd89e5ec89f113d217225cc59cbc8373d64
R 3e2d7a9cc84ff16ee3e947d8ce602cce
P 95cc3f6fdec5494560c3cd4439d06870d1c62506 13e7a8242206bca4b5bf356ef074e66474d39609
R 25e35232d2e784a0633c25f463199a04
U drh
Z e14f3acbecbf2df0abe6fde12480ec15
Z 04fdb9faf7b89a9cf801831e630d8cdf
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.6 (GNU/Linux)
iD8DBQFMEmvooxKgR168RlERAg5RAJ9VmP08NyW1of8QWkDMnAiHK8A/xgCeLBcs
hbn4o1Zj2vKA/g5KFcfNycs=
=Li8t
iD8DBQFMFmJkoxKgR168RlERAk/eAJ9VllwzElSwe9gXOW0UoIt+bcEvsgCfSESl
TnMmecbXjWkPCx7cvVOMr4o=
=hImA
-----END PGP SIGNATURE-----
+1 -1
View File
@@ -1 +1 @@
95cc3f6fdec5494560c3cd4439d06870d1c62506
f295e7ed5f04f6b8bffdaff8b565be3836ce4e80
+9 -9
View File
@@ -101,15 +101,6 @@ int sqlite3OsDeviceCharacteristics(sqlite3_file *id){
int sqlite3OsShmOpen(sqlite3_file *id){
return id->pMethods->xShmOpen(id);
}
int sqlite3OsShmSize(sqlite3_file *id, int reqSize, int *pNewSize){
return id->pMethods->xShmSize(id, reqSize, pNewSize);
}
int sqlite3OsShmGet(sqlite3_file *id,int reqSize,int *pSize,void volatile **pp){
return id->pMethods->xShmGet(id, reqSize, pSize, pp);
}
int sqlite3OsShmRelease(sqlite3_file *id){
return id->pMethods->xShmRelease(id);
}
int sqlite3OsShmLock(sqlite3_file *id, int offset, int n, int flags){
return id->pMethods->xShmLock(id, offset, n, flags);
}
@@ -119,6 +110,15 @@ void sqlite3OsShmBarrier(sqlite3_file *id){
int sqlite3OsShmClose(sqlite3_file *id, int deleteFlag){
return id->pMethods->xShmClose(id, deleteFlag);
}
int sqlite3OsShmMap(
sqlite3_file *id,
int iPage,
int pgsz,
int isWrite,
void volatile **pp
){
return id->pMethods->xShmMap(id, iPage, pgsz, isWrite, pp);
}
/*
** The next group of routines are convenience wrappers around the
+1 -3
View File
@@ -248,12 +248,10 @@ int sqlite3OsFileControl(sqlite3_file*,int,void*);
int sqlite3OsSectorSize(sqlite3_file *id);
int sqlite3OsDeviceCharacteristics(sqlite3_file *id);
int sqlite3OsShmOpen(sqlite3_file *id);
int sqlite3OsShmSize(sqlite3_file *id, int, int*);
int sqlite3OsShmGet(sqlite3_file *id, int, int*, void volatile**);
int sqlite3OsShmRelease(sqlite3_file *id);
int sqlite3OsShmLock(sqlite3_file *id, int, int, int);
void sqlite3OsShmBarrier(sqlite3_file *id);
int sqlite3OsShmClose(sqlite3_file *id, int);
int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **);
/*
** Functions for accessing sqlite3_vfs methods
+107 -168
View File
@@ -3128,21 +3128,15 @@ static int unixDeviceCharacteristics(sqlite3_file *NotUsed){
** Either unixShmNode.mutex must be held or unixShmNode.nRef==0 and
** unixMutexHeld() is true when reading or writing any other field
** in this structure.
**
** To avoid deadlocks, mutex and mutexBuf are always released in the
** reverse order that they are acquired. mutexBuf is always acquired
** first and released last. This invariant is check by asserting
** sqlite3_mutex_notheld() on mutex whenever mutexBuf is acquired or
** released.
*/
struct unixShmNode {
unixInodeInfo *pInode; /* unixInodeInfo that owns this SHM node */
sqlite3_mutex *mutex; /* Mutex to access this object */
sqlite3_mutex *mutexBuf; /* Mutex to access zBuf[] */
char *zFilename; /* Name of the mmapped file */
int h; /* Open file descriptor */
int szMap; /* Size of the mapping into memory */
char *pMMapBuf; /* Where currently mmapped(). NULL if unmapped */
int szRegion; /* Size of shared-memory regions */
int nRegion; /* Size of array apRegion */
char **apRegion; /* Array of mapped shared-memory regions */
int nRef; /* Number of unixShm objects pointing to this */
unixShm *pFirst; /* All unixShm objects pointing to this */
#ifdef SQLITE_DEBUG
@@ -3169,7 +3163,6 @@ struct unixShm {
unixShmNode *pShmNode; /* The underlying unixShmNode object */
unixShm *pNext; /* Next unixShm with the same unixShmNode */
u8 hasMutex; /* True if holding the unixShmNode mutex */
u8 hasMutexBuf; /* True if holding pFile->mutexBuf */
u16 sharedMask; /* Mask of shared locks held */
u16 exclMask; /* Mask of exclusive locks held */
#ifdef SQLITE_DEBUG
@@ -3266,10 +3259,13 @@ static void unixShmPurge(unixFile *pFd){
unixShmNode *p = pFd->pInode->pShmNode;
assert( unixMutexHeld() );
if( p && p->nRef==0 ){
int i;
assert( p->pInode==pFd->pInode );
if( p->mutex ) sqlite3_mutex_free(p->mutex);
if( p->mutexBuf ) sqlite3_mutex_free(p->mutexBuf);
if( p->pMMapBuf ) munmap(p->pMMapBuf, p->szMap);
for(i=0; i<p->nRegion; i++){
munmap(p->apRegion[i], p->szRegion);
}
sqlite3_free(p->apRegion);
if( p->h>=0 ) close(p->h);
p->pInode->pShmNode = 0;
sqlite3_free(p);
@@ -3345,11 +3341,6 @@ static int unixShmOpen(
rc = SQLITE_NOMEM;
goto shm_open_err;
}
pShmNode->mutexBuf = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST);
if( pShmNode->mutexBuf==0 ){
rc = SQLITE_NOMEM;
goto shm_open_err;
}
pShmNode->h = open(pShmNode->zFilename, O_RDWR|O_CREAT, 0664);
if( pShmNode->h<0 ){
@@ -3420,7 +3411,6 @@ static int unixShmClose(
*pp = p->pNext;
/* Free the connection p */
assert( p->hasMutexBuf==0 );
sqlite3_free(p);
pDbFd->pShm = 0;
sqlite3_mutex_leave(pShmNode->mutex);
@@ -3439,148 +3429,6 @@ static int unixShmClose(
return SQLITE_OK;
}
/*
** Changes the size of the underlying storage for a shared-memory segment.
**
** The reqSize parameter is the new requested size of the shared memory.
** This implementation is free to increase the shared memory size to
** any amount greater than or equal to reqSize. If the shared memory is
** already as big or bigger as reqSize, this routine is a no-op.
**
** The reqSize parameter is the minimum size requested. The implementation
** is free to expand the storage to some larger amount if it chooses.
*/
static int unixShmSize(
sqlite3_file *fd, /* The open database file holding SHM */
int reqSize, /* Requested size. -1 for query only */
int *pNewSize /* Write new size here */
){
unixFile *pDbFd = (unixFile*)fd;
unixShm *p = pDbFd->pShm;
unixShmNode *pShmNode = p->pShmNode;
int rc = SQLITE_OK;
struct stat sStat;
assert( pShmNode==pDbFd->pInode->pShmNode );
assert( pShmNode->pInode==pDbFd->pInode );
while( 1 ){
if( fstat(pShmNode->h, &sStat)==0 ){
*pNewSize = (int)sStat.st_size;
if( reqSize<=(int)sStat.st_size ) break;
}else{
*pNewSize = 0;
rc = SQLITE_IOERR_SHMSIZE;
break;
}
rc = ftruncate(pShmNode->h, reqSize);
reqSize = -1;
}
return rc;
}
/*
** Release the lock held on the shared memory segment to that other
** threads are free to resize it if necessary.
**
** If the lock is not currently held, this routine is a harmless no-op.
**
** If the shared-memory object is in lock state RECOVER, then we do not
** really want to release the lock, so in that case too, this routine
** is a no-op.
*/
static int unixShmRelease(sqlite3_file *fd){
unixFile *pDbFd = (unixFile*)fd;
unixShm *p = pDbFd->pShm;
if( p->hasMutexBuf ){
assert( sqlite3_mutex_notheld(p->pShmNode->mutex) );
sqlite3_mutex_leave(p->pShmNode->mutexBuf);
p->hasMutexBuf = 0;
}
return SQLITE_OK;
}
/*
** Map the shared storage into memory.
**
** If reqMapSize is positive, then an attempt is made to make the
** mapping at least reqMapSize bytes in size. However, the mapping
** will never be larger than the size of the underlying shared memory
** as set by prior calls to xShmSize().
**
** *ppBuf is made to point to the memory which is a mapping of the
** underlying storage. A mutex is acquired to prevent other threads
** from running while *ppBuf is in use in order to prevent other threads
** remapping *ppBuf out from under this thread. The unixShmRelease()
** call will release the mutex. However, if the lock state is CHECKPOINT,
** the mutex is not acquired because CHECKPOINT will never remap the
** buffer. RECOVER might remap, though, so CHECKPOINT will acquire
** the mutex if and when it promotes to RECOVER.
**
** RECOVER needs to be atomic. The same mutex that prevents *ppBuf from
** being remapped also prevents more than one thread from being in
** RECOVER at a time. But, RECOVER sometimes wants to remap itself.
** To prevent RECOVER from losing its lock while remapping, the
** mutex is not released by unixShmRelease() when in RECOVER.
**
** *pNewMapSize is set to the size of the mapping. Usually *pNewMapSize
** will be reqMapSize or larger, though it could be smaller if the
** underlying shared memory has never been enlarged to reqMapSize bytes
** by prior calls to xShmSize().
**
** *ppBuf might be NULL and zero if no space has
** yet been allocated to the underlying storage.
*/
static int unixShmGet(
sqlite3_file *fd, /* Database file holding shared memory */
int reqMapSize, /* Requested size of mapping. -1 means don't care */
int *pNewMapSize, /* Write new size of mapping here */
void volatile **ppBuf /* Write mapping buffer origin here */
){
unixFile *pDbFd = (unixFile*)fd;
unixShm *p = pDbFd->pShm;
unixShmNode *pShmNode = p->pShmNode;
int rc = SQLITE_OK;
assert( pShmNode==pDbFd->pInode->pShmNode );
assert( pShmNode->pInode==pDbFd->pInode );
if( p->hasMutexBuf==0 ){
assert( sqlite3_mutex_notheld(pShmNode->mutex) );
sqlite3_mutex_enter(pShmNode->mutexBuf);
p->hasMutexBuf = 1;
}
sqlite3_mutex_enter(pShmNode->mutex);
if( pShmNode->szMap==0 || reqMapSize>pShmNode->szMap ){
int actualSize;
if( unixShmSize(fd, -1, &actualSize)!=SQLITE_OK ){
actualSize = 0;
}
reqMapSize = actualSize;
if( pShmNode->pMMapBuf || reqMapSize<=0 ){
munmap(pShmNode->pMMapBuf, pShmNode->szMap);
}
if( reqMapSize>0 ){
pShmNode->pMMapBuf = mmap(0, reqMapSize, PROT_READ|PROT_WRITE, MAP_SHARED,
pShmNode->h, 0);
pShmNode->szMap = pShmNode->pMMapBuf ? reqMapSize : 0;
}else{
pShmNode->pMMapBuf = 0;
pShmNode->szMap = 0;
}
}
*pNewMapSize = pShmNode->szMap;
*ppBuf = pShmNode->pMMapBuf;
sqlite3_mutex_leave(pShmNode->mutex);
if( *ppBuf==0 ){
/* Do not hold the mutex if a NULL pointer is being returned. */
unixShmRelease(fd);
}
return rc;
}
/*
** Change the lock state for a shared-memory segment.
**
@@ -3700,21 +3548,114 @@ static int unixShmLock(
** any load or store begun after the barrier.
*/
static void unixShmBarrier(
sqlite3_file *fd /* Database file holding the shared memory */
sqlite3_file *fd /* Database file holding the shared memory */
){
unixEnterMutex();
unixLeaveMutex();
}
/*
** This function is called to obtain a pointer to region iRegion of the
** shared-memory associated with the database file fd. Shared-memory regions
** are numbered starting from zero. Each shared-memory region is szRegion
** bytes in size.
**
** If an error occurs, an error code is returned and *pp is set to NULL.
**
** Otherwise, if the isWrite parameter is 0 and the requested shared-memory
** region has not been allocated (by any client, including one running in a
** separate process), then *pp is set to NULL and SQLITE_OK returned. If
** isWrite is non-zero and the requested shared-memory region has not yet
** been allocated, it is allocated by this function.
**
** If the shared-memory region has already been allocated or is allocated by
** this call as described above, then it is mapped into this processes
** address space (if it is not already), *pp is set to point to the mapped
** memory and SQLITE_OK returned.
*/
static int unixShmMap(
sqlite3_file *fd, /* Handle open on database file */
int iRegion, /* Region to retrieve */
int szRegion, /* Size of regions */
int isWrite, /* True to extend file if necessary */
void volatile **pp /* OUT: Mapped memory */
){
unixFile *pDbFd = (unixFile*)fd;
unixShm *p = pDbFd->pShm;
unixShmNode *pShmNode = p->pShmNode;
int rc = SQLITE_OK;
sqlite3_mutex_enter(pShmNode->mutex);
assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 );
if( pShmNode->nRegion<=iRegion ){
char **apNew; /* New apRegion[] array */
int nByte = (iRegion+1)*szRegion; /* Minimum required file size */
struct stat sStat; /* Used by fstat() */
pShmNode->szRegion = szRegion;
/* The requested region is not mapped into this processes address space.
** Check to see if it has been allocated (i.e. if the wal-index file is
** large enough to contain the requested region).
*/
if( fstat(pShmNode->h, &sStat) ){
rc = SQLITE_IOERR_SHMSIZE;
goto shmpage_out;
}
if( sStat.st_size<nByte ){
/* The requested memory region does not exist. If isWrite is set to
** zero, exit early. *pp will be set to NULL and SQLITE_OK returned.
**
** Alternatively, if isWrite is non-zero, use ftruncate() to allocate
** the requested memory region.
*/
if( !isWrite ) goto shmpage_out;
if( ftruncate(pShmNode->h, nByte) ){
rc = SQLITE_IOERR_SHMSIZE;
goto shmpage_out;
}
}
/* Map the requested memory region into this processes address space. */
apNew = (char **)sqlite3_realloc(
pShmNode->apRegion, (iRegion+1)*sizeof(char *)
);
if( !apNew ){
rc = SQLITE_IOERR_NOMEM;
goto shmpage_out;
}
pShmNode->apRegion = apNew;
while(pShmNode->nRegion<=iRegion){
void *pMem = mmap(0, szRegion, PROT_READ|PROT_WRITE,
MAP_SHARED, pShmNode->h, iRegion*szRegion
);
if( pMem==MAP_FAILED ){
rc = SQLITE_IOERR;
goto shmpage_out;
}
pShmNode->apRegion[pShmNode->nRegion] = pMem;
pShmNode->nRegion++;
}
}
shmpage_out:
if( pShmNode->nRegion>iRegion ){
*pp = pShmNode->apRegion[iRegion];
}else{
*pp = 0;
}
sqlite3_mutex_leave(pShmNode->mutex);
return rc;
}
#else
# define unixShmOpen 0
# define unixShmSize 0
# define unixShmGet 0
# define unixShmRelease 0
# define unixShmLock 0
# define unixShmBarrier 0
# define unixShmClose 0
# define unixShmMap 0
#endif /* #ifndef SQLITE_OMIT_WAL */
/*
@@ -3773,12 +3714,10 @@ static const sqlite3_io_methods METHOD = { \
unixSectorSize, /* xSectorSize */ \
unixDeviceCharacteristics, /* xDeviceCapabilities */ \
unixShmOpen, /* xShmOpen */ \
unixShmSize, /* xShmSize */ \
unixShmGet, /* xShmGet */ \
unixShmRelease, /* xShmRelease */ \
unixShmLock, /* xShmLock */ \
unixShmBarrier, /* xShmBarrier */ \
unixShmClose /* xShmClose */ \
unixShmClose, /* xShmClose */ \
unixShmMap /* xShmMap */ \
}; \
static const sqlite3_io_methods *FINDER##Impl(const char *z, unixFile *p){ \
UNUSED_PARAMETER(z); UNUSED_PARAMETER(p); \
+104 -168
View File
@@ -1216,13 +1216,17 @@ static int winShmMutexHeld(void) {
*/
struct winShmNode {
sqlite3_mutex *mutex; /* Mutex to access this object */
sqlite3_mutex *mutexBuf; /* Mutex to access zBuf[] */
char *zFilename; /* Name of the file */
winFile hFile; /* File handle from winOpen */
HANDLE hMap; /* File handle from CreateFileMapping */
int szRegion; /* Size of shared-memory regions */
int nRegion; /* Size of array apRegion */
struct ShmRegion {
HANDLE hMap; /* File handle from CreateFileMapping */
void *pMap;
} *aRegion;
DWORD lastErrno; /* The Windows errno from the last I/O error */
int szMap; /* Size of the mapping of file into memory */
char *pMMapBuf; /* Where currently mmapped(). NULL if unmapped */
int nRef; /* Number of winShm objects pointing to this */
winShm *pFirst; /* All winShm objects pointing to this */
winShmNode *pNext; /* Next in list of all winShmNode objects */
@@ -1325,19 +1329,18 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){
pp = &winShmNodeList;
while( (p = *pp)!=0 ){
if( p->nRef==0 ){
int i;
if( p->mutex ) sqlite3_mutex_free(p->mutex);
if( p->mutexBuf ) sqlite3_mutex_free(p->mutexBuf);
if( p->pMMapBuf ){
UnmapViewOfFile(p->pMMapBuf);
}
if( INVALID_HANDLE_VALUE != p->hMap ){
CloseHandle(p->hMap);
for(i=0; i<p->nRegion; i++){
UnmapViewOfFile(p->aRegion[i].pMap);
CloseHandle(p->aRegion[i].hMap);
}
if( p->hFile.h != INVALID_HANDLE_VALUE ) {
winClose((sqlite3_file *)&p->hFile);
}
if( deleteFlag ) winDelete(pVfs, p->zFilename, 0);
*pp = p->pNext;
sqlite3_free(p->aRegion);
sqlite3_free(p);
}else{
pp = &p->pNext;
@@ -1404,8 +1407,6 @@ static int winShmOpen(
}else{
pShmNode = pNew;
pNew = 0;
pShmNode->pMMapBuf = NULL;
pShmNode->hMap = INVALID_HANDLE_VALUE;
((winFile*)(&pShmNode->hFile))->h = INVALID_HANDLE_VALUE;
pShmNode->pNext = winShmNodeList;
winShmNodeList = pShmNode;
@@ -1415,11 +1416,6 @@ static int winShmOpen(
rc = SQLITE_NOMEM;
goto shm_open_err;
}
pShmNode->mutexBuf = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST);
if( pShmNode->mutexBuf==0 ){
rc = SQLITE_NOMEM;
goto shm_open_err;
}
rc = winOpen(pDbFd->pVfs,
pShmNode->zFilename, /* Name of the file (UTF-8) */
(sqlite3_file*)&pShmNode->hFile, /* File handle here */
@@ -1507,173 +1503,115 @@ static int winShmClose(
}
/*
** Increase the size of the underlying storage for a shared-memory segment.
** This function is called to obtain a pointer to region iRegion of the
** shared-memory associated with the database file fd. Shared-memory regions
** are numbered starting from zero. Each shared-memory region is szRegion
** bytes in size.
**
** The reqSize parameter is the new requested minimum size of the underlying
** shared memory. This routine may choose to make the shared memory larger
** than this value (for example to round the shared memory size up to an
** operating-system dependent page size.)
** If an error occurs, an error code is returned and *pp is set to NULL.
**
** This routine will only grow the size of shared memory. A request for
** a smaller size is a no-op.
** Otherwise, if the isWrite parameter is 0 and the requested shared-memory
** region has not been allocated (by any client, including one running in a
** separate process), then *pp is set to NULL and SQLITE_OK returned. If
** isWrite is non-zero and the requested shared-memory region has not yet
** been allocated, it is allocated by this function.
**
** If the shared-memory region has already been allocated or is allocated by
** this call as described above, then it is mapped into this processes
** address space (if it is not already), *pp is set to point to the mapped
** memory and SQLITE_OK returned.
*/
static int winShmSize(
sqlite3_file *fd, /* Database holding the shared memory */
int reqSize, /* Requested size. -1 for query only */
int *pNewSize /* Write new size here */
static int winShmMap(
sqlite3_file *fd, /* Handle open on database file */
int iRegion, /* Region to retrieve */
int szRegion, /* Size of regions */
int isWrite, /* True to extend file if necessary */
void volatile **pp /* OUT: Mapped memory */
){
winFile *pDbFd = (winFile*)fd;
winShm *p = pDbFd->pShm;
winShmNode *pShmNode = p->pShmNode;
int rc = SQLITE_OK;
*pNewSize = 0;
if( reqSize>=0 ){
sqlite3_int64 sz;
rc = winFileSize((sqlite3_file *)&pShmNode->hFile, &sz);
if( SQLITE_OK==rc && reqSize>sz ){
rc = winTruncate((sqlite3_file *)&pShmNode->hFile, reqSize);
}
}
if( SQLITE_OK==rc ){
sqlite3_int64 sz;
rc = winFileSize((sqlite3_file *)&pShmNode->hFile, &sz);
if( SQLITE_OK==rc ){
*pNewSize = (int)sz;
}else{
rc = SQLITE_IOERR;
}
}
return rc;
}
/*
** Map the shared storage into memory. The minimum size of the
** mapping should be reqMapSize if reqMapSize is positive. If
** reqMapSize is zero or negative, the implementation can choose
** whatever mapping size is convenient.
**
** *ppBuf is made to point to the memory which is a mapping of the
** underlying storage. A mutex is acquired to prevent other threads
** from running while *ppBuf is in use in order to prevent other threads
** remapping *ppBuf out from under this thread. The winShmRelease()
** call will release the mutex. However, if the lock state is CHECKPOINT,
** the mutex is not acquired because CHECKPOINT will never remap the
** buffer. RECOVER might remap, though, so CHECKPOINT will acquire
** the mutex if and when it promotes to RECOVER.
**
** RECOVER needs to be atomic. The same mutex that prevents *ppBuf from
** being remapped also prevents more than one thread from being in
** RECOVER at a time. But, RECOVER sometimes wants to remap itself.
** To prevent RECOVER from losing its lock while remapping, the
** mutex is not released by winShmRelease() when in RECOVER.
**
** *pNewMapSize is set to the size of the mapping.
**
** *ppBuf and *pNewMapSize might be NULL and zero if no space has
** yet been allocated to the underlying storage.
*/
static int winShmGet(
sqlite3_file *fd, /* The database file holding the shared memory */
int reqMapSize, /* Requested size of mapping. -1 means don't care */
int *pNewMapSize, /* Write new size of mapping here */
void volatile **ppBuf /* Write mapping buffer origin here */
){
winFile *pDbFd = (winFile*)fd;
winShm *p = pDbFd->pShm;
winShmNode *pShmNode = p->pShmNode;
int rc = SQLITE_OK;
if( p->hasMutexBuf==0 ){
assert( sqlite3_mutex_notheld(pShmNode->mutex) );
sqlite3_mutex_enter(pShmNode->mutexBuf);
p->hasMutexBuf = 1;
}
sqlite3_mutex_enter(pShmNode->mutex);
if( pShmNode->szMap==0 || reqMapSize>pShmNode->szMap ){
int actualSize;
if( winShmSize(fd, -1, &actualSize)==SQLITE_OK
&& reqMapSize<actualSize
){
reqMapSize = actualSize;
assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 );
if( pShmNode->nRegion<=iRegion ){
struct ShmRegion *apNew; /* New aRegion[] array */
int nByte = (iRegion+1)*szRegion; /* Minimum required file size */
sqlite3_int64 sz; /* Current size of wal-index file */
pShmNode->szRegion = szRegion;
/* The requested region is not mapped into this processes address space.
** Check to see if it has been allocated (i.e. if the wal-index file is
** large enough to contain the requested region).
*/
rc = winFileSize((sqlite3_file *)&pShmNode->hFile, &sz);
if( rc!=SQLITE_OK ){
goto shmpage_out;
}
if( pShmNode->pMMapBuf ){
if( !UnmapViewOfFile(pShmNode->pMMapBuf) ){
if( sz<nByte ){
/* The requested memory region does not exist. If isWrite is set to
** zero, exit early. *pp will be set to NULL and SQLITE_OK returned.
**
** Alternatively, if isWrite is non-zero, use ftruncate() to allocate
** the requested memory region.
*/
if( !isWrite ) goto shmpage_out;
rc = winTruncate((sqlite3_file *)&pShmNode->hFile, nByte);
if( rc!=SQLITE_OK ){
goto shmpage_out;
}
}
/* Map the requested memory region into this processes address space. */
apNew = (struct ShmRegion *)sqlite3_realloc(
pShmNode->aRegion, (iRegion+1)*sizeof(apNew[0])
);
if( !apNew ){
rc = SQLITE_IOERR_NOMEM;
goto shmpage_out;
}
pShmNode->aRegion = apNew;
while( pShmNode->nRegion<=iRegion ){
HANDLE hMap; /* file-mapping handle */
void *pMap = 0; /* Mapped memory region */
hMap = CreateFileMapping(pShmNode->hFile.h,
NULL, PAGE_READWRITE, 0, nByte, NULL
);
if( hMap ){
pMap = MapViewOfFile(hMap, FILE_MAP_WRITE | FILE_MAP_READ,
0, 0, nByte
);
}
if( !pMap ){
pShmNode->lastErrno = GetLastError();
rc = SQLITE_IOERR;
if( hMap ) CloseHandle(hMap);
goto shmpage_out;
}
CloseHandle(pShmNode->hMap);
pShmNode->hMap = INVALID_HANDLE_VALUE;
}
if( SQLITE_OK == rc ){
pShmNode->pMMapBuf = 0;
if( reqMapSize == 0 ){
/* can't create 0 byte file mapping in Windows */
pShmNode->szMap = 0;
}else{
/* create the file mapping object */
if( INVALID_HANDLE_VALUE == pShmNode->hMap ){
/* TBD provide an object name to each file
** mapping so it can be re-used across processes.
*/
pShmNode->hMap = CreateFileMapping(pShmNode->hFile.h,
NULL,
PAGE_READWRITE,
0,
reqMapSize,
NULL);
}
if( NULL==pShmNode->hMap ){
pShmNode->lastErrno = GetLastError();
rc = SQLITE_IOERR;
pShmNode->szMap = 0;
pShmNode->hMap = INVALID_HANDLE_VALUE;
}else{
pShmNode->pMMapBuf = MapViewOfFile(pShmNode->hMap,
FILE_MAP_WRITE | FILE_MAP_READ,
0,
0,
reqMapSize);
if( !pShmNode->pMMapBuf ){
pShmNode->lastErrno = GetLastError();
rc = SQLITE_IOERR;
pShmNode->szMap = 0;
}else{
pShmNode->szMap = reqMapSize;
}
}
}
pShmNode->aRegion[pShmNode->nRegion].pMap = pMap;
pShmNode->aRegion[pShmNode->nRegion].hMap = hMap;
pShmNode->nRegion++;
}
}
*pNewMapSize = pShmNode->szMap;
*ppBuf = pShmNode->pMMapBuf;
shmpage_out:
if( pShmNode->nRegion>iRegion ){
char *p = (char *)pShmNode->aRegion[iRegion].pMap;
*pp = (void *)&p[iRegion*szRegion];
}else{
*pp = 0;
}
sqlite3_mutex_leave(pShmNode->mutex);
return rc;
}
/*
** Release the lock held on the shared memory segment so that other
** threads are free to resize it if necessary.
**
** If the lock is not currently held, this routine is a harmless no-op.
**
** If the shared-memory object is in lock state RECOVER, then we do not
** really want to release the lock, so in that case too, this routine
** is a no-op.
*/
static int winShmRelease(sqlite3_file *fd){
winFile *pDbFd = (winFile*)fd;
winShm *p = pDbFd->pShm;
if( p->hasMutexBuf ){
winShmNode *pShmNode = p->pShmNode;
assert( sqlite3_mutex_notheld(pShmNode->mutex) );
sqlite3_mutex_leave(pShmNode->mutexBuf);
p->hasMutexBuf = 0;
}
return SQLITE_OK;
}
/*
** Change the lock state for a shared-memory segment.
*/
@@ -1756,12 +1694,10 @@ static const sqlite3_io_methods winIoMethod = {
winSectorSize,
winDeviceCharacteristics,
winShmOpen, /* xShmOpen */
winShmSize, /* xShmSize */
winShmGet, /* xShmGet */
winShmRelease, /* xShmRelease */
winShmLock, /* xShmLock */
winShmBarrier, /* xShmBarrier */
winShmClose /* xShmClose */
winShmClose, /* xShmClose */
winShmMap /* xShmMap */
};
/***************************************************************************
+1 -3
View File
@@ -660,12 +660,10 @@ struct sqlite3_io_methods {
int (*xDeviceCharacteristics)(sqlite3_file*);
/* Methods above are valid for version 1 */
int (*xShmOpen)(sqlite3_file*);
int (*xShmSize)(sqlite3_file*, int reqSize, int *pNewSize);
int (*xShmGet)(sqlite3_file*, int reqSize, int *pSize, void volatile**);
int (*xShmRelease)(sqlite3_file*);
int (*xShmLock)(sqlite3_file*, int offset, int n, int flags);
void (*xShmBarrier)(sqlite3_file*);
int (*xShmClose)(sqlite3_file*, int deleteFlag);
int (*xShmMap)(sqlite3_file*, int iPage, int pgsz, int, void volatile**);
/* Methods above are valid for version 2 */
/* Additional methods may be added in future releases */
};
+11 -19
View File
@@ -526,20 +526,6 @@ static int cfDeviceCharacteristics(sqlite3_file *pFile){
static int cfShmOpen(sqlite3_file *pFile){
return sqlite3OsShmOpen(((CrashFile*)pFile)->pRealFile);
}
static int cfShmSize(sqlite3_file *pFile, int reqSize, int *pNew){
return sqlite3OsShmSize(((CrashFile*)pFile)->pRealFile, reqSize, pNew);
}
static int cfShmGet(
sqlite3_file *pFile,
int reqSize,
int *pSize,
void volatile **pp
){
return sqlite3OsShmGet(((CrashFile*)pFile)->pRealFile, reqSize, pSize, pp);
}
static int cfShmRelease(sqlite3_file *pFile){
return sqlite3OsShmRelease(((CrashFile*)pFile)->pRealFile);
}
static int cfShmLock(sqlite3_file *pFile, int ofst, int n, int flags){
return sqlite3OsShmLock(((CrashFile*)pFile)->pRealFile, ofst, n, flags);
}
@@ -549,7 +535,15 @@ static void cfShmBarrier(sqlite3_file *pFile){
static int cfShmClose(sqlite3_file *pFile, int delFlag){
return sqlite3OsShmClose(((CrashFile*)pFile)->pRealFile, delFlag);
}
static int cfShmMap(
sqlite3_file *pFile, /* Handle open on database file */
int iRegion, /* Region to retrieve */
int sz, /* Size of regions */
int w, /* True to extend file if necessary */
void volatile **pp /* OUT: Mapped memory */
){
return sqlite3OsShmMap(((CrashFile*)pFile)->pRealFile, iRegion, sz, w, pp);
}
static const sqlite3_io_methods CrashFileVtab = {
2, /* iVersion */
@@ -566,12 +560,10 @@ static const sqlite3_io_methods CrashFileVtab = {
cfSectorSize, /* xSectorSize */
cfDeviceCharacteristics, /* xDeviceCharacteristics */
cfShmOpen, /* xShmOpen */
cfShmSize, /* xShmSize */
cfShmGet, /* xShmGet */
cfShmRelease, /* xShmRelease */
cfShmLock, /* xShmLock */
cfShmBarrier, /* xShmBarrier */
cfShmClose /* xShmClose */
cfShmClose, /* xShmClose */
cfShmMap /* xShmMap */
};
/*
+13 -24
View File
@@ -51,12 +51,10 @@ static int devsymFileControl(sqlite3_file*, int op, void *pArg);
static int devsymSectorSize(sqlite3_file*);
static int devsymDeviceCharacteristics(sqlite3_file*);
static int devsymShmOpen(sqlite3_file*);
static int devsymShmSize(sqlite3_file*,int,int*);
static int devsymShmGet(sqlite3_file*,int,int*,volatile void**);
static int devsymShmRelease(sqlite3_file*);
static int devsymShmLock(sqlite3_file*,int,int,int);
static void devsymShmBarrier(sqlite3_file*);
static int devsymShmClose(sqlite3_file*,int);
static int devsymShmMap(sqlite3_file*,int,int,int, void volatile **);
/*
** Method declarations for devsym_vfs.
@@ -120,12 +118,10 @@ static sqlite3_io_methods devsym_io_methods = {
devsymSectorSize, /* xSectorSize */
devsymDeviceCharacteristics, /* xDeviceCharacteristics */
devsymShmOpen, /* xShmOpen */
devsymShmSize, /* xShmSize */
devsymShmGet, /* xShmGet */
devsymShmRelease, /* xShmRelease */
devsymShmLock, /* xShmLock */
devsymShmBarrier, /* xShmBarrier */
devsymShmClose /* xShmClose */
devsymShmClose, /* xShmClose */
devsymShmMap /* xShmMap */
};
struct DevsymGlobal {
@@ -246,23 +242,6 @@ static int devsymShmOpen(sqlite3_file *pFile){
devsym_file *p = (devsym_file *)pFile;
return sqlite3OsShmOpen(p->pReal);
}
static int devsymShmSize(sqlite3_file *pFile, int reqSize, int *pSize){
devsym_file *p = (devsym_file *)pFile;
return sqlite3OsShmSize(p->pReal, reqSize, pSize);
}
static int devsymShmGet(
sqlite3_file *pFile,
int reqSz,
int *pSize,
void volatile **pp
){
devsym_file *p = (devsym_file *)pFile;
return sqlite3OsShmGet(p->pReal, reqSz, pSize, pp);
}
static int devsymShmRelease(sqlite3_file *pFile){
devsym_file *p = (devsym_file *)pFile;
return sqlite3OsShmRelease(p->pReal);
}
static int devsymShmLock(sqlite3_file *pFile, int ofst, int n, int flags){
devsym_file *p = (devsym_file *)pFile;
return sqlite3OsShmLock(p->pReal, ofst, n, flags);
@@ -275,6 +254,16 @@ static int devsymShmClose(sqlite3_file *pFile, int delFlag){
devsym_file *p = (devsym_file *)pFile;
return sqlite3OsShmClose(p->pReal, delFlag);
}
static int devsymShmMap(
sqlite3_file *pFile,
int iRegion,
int szRegion,
int isWrite,
void volatile **pp
){
devsym_file *p = (devsym_file *)pFile;
return sqlite3OsShmMap(p->pReal, iRegion, szRegion, isWrite, pp);
}
+21 -48
View File
@@ -100,11 +100,9 @@
#define OS_WRITE 20
#define OS_SHMOPEN 21
#define OS_SHMCLOSE 22
#define OS_SHMGET 23
#define OS_SHMRELEASE 24
#define OS_SHMMAP 23
#define OS_SHMLOCK 25
#define OS_SHMBARRIER 26
#define OS_SHMSIZE 27
#define OS_ANNOTATE 28
#define OS_NUMEVENTS 29
@@ -152,12 +150,10 @@ static int vfslogSectorSize(sqlite3_file*);
static int vfslogDeviceCharacteristics(sqlite3_file*);
static int vfslogShmOpen(sqlite3_file *pFile);
static int vfslogShmSize(sqlite3_file *pFile, int reqSize, int *pNewSize);
static int vfslogShmGet(sqlite3_file *pFile, int,int*,volatile void **);
static int vfslogShmRelease(sqlite3_file *pFile);
static int vfslogShmLock(sqlite3_file *pFile, int ofst, int n, int flags);
static void vfslogShmBarrier(sqlite3_file*);
static int vfslogShmClose(sqlite3_file *pFile, int deleteFlag);
static int vfslogShmMap(sqlite3_file *pFile,int,int,int,volatile void **);
/*
** Method declarations for vfslog_vfs.
@@ -216,12 +212,10 @@ static sqlite3_io_methods vfslog_io_methods = {
vfslogSectorSize, /* xSectorSize */
vfslogDeviceCharacteristics, /* xDeviceCharacteristics */
vfslogShmOpen, /* xShmOpen */
vfslogShmSize, /* xShmSize */
vfslogShmGet, /* xShmGet */
vfslogShmRelease, /* xShmRelease */
vfslogShmLock, /* xShmLock */
vfslogShmBarrier, /* xShmBarrier */
vfslogShmClose /* xShmClose */
vfslogShmClose, /* xShmClose */
vfslogShmMap /* xShmMap */
};
#if defined(SQLITE_OS_UNIX) && !defined(NO_GETTOD)
@@ -441,41 +435,6 @@ static int vfslogShmOpen(sqlite3_file *pFile){
vfslog_call(p->pVfslog, OS_SHMOPEN, p->iFileId, t, rc, 0, 0);
return rc;
}
static int vfslogShmSize(sqlite3_file *pFile, int reqSize, int *pNewSize){
int rc;
sqlite3_uint64 t;
VfslogFile *p = (VfslogFile *)pFile;
t = vfslog_time();
rc = p->pReal->pMethods->xShmSize(p->pReal, reqSize, pNewSize);
t = vfslog_time() - t;
vfslog_call(p->pVfslog, OS_SHMSIZE, p->iFileId, t, rc, 0, 0);
return rc;
}
static int vfslogShmGet(
sqlite3_file *pFile,
int req,
int *pSize,
volatile void **pp
){
int rc;
sqlite3_uint64 t;
VfslogFile *p = (VfslogFile *)pFile;
t = vfslog_time();
rc = p->pReal->pMethods->xShmGet(p->pReal, req, pSize, pp);
t = vfslog_time() - t;
vfslog_call(p->pVfslog, OS_SHMGET, p->iFileId, t, rc, 0, 0);
return rc;
}
static int vfslogShmRelease(sqlite3_file *pFile){
int rc;
sqlite3_uint64 t;
VfslogFile *p = (VfslogFile *)pFile;
t = vfslog_time();
rc = p->pReal->pMethods->xShmRelease(p->pReal);
t = vfslog_time() - t;
vfslog_call(p->pVfslog, OS_SHMRELEASE, p->iFileId, t, rc, 0, 0);
return rc;
}
static int vfslogShmLock(sqlite3_file *pFile, int ofst, int n, int flags){
int rc;
sqlite3_uint64 t;
@@ -504,6 +463,22 @@ static int vfslogShmClose(sqlite3_file *pFile, int deleteFlag){
vfslog_call(p->pVfslog, OS_SHMCLOSE, p->iFileId, t, rc, 0, 0);
return rc;
}
static int vfslogShmMap(
sqlite3_file *pFile,
int iRegion,
int szRegion,
int isWrite,
volatile void **pp
){
int rc;
sqlite3_uint64 t;
VfslogFile *p = (VfslogFile *)pFile;
t = vfslog_time();
rc = p->pReal->pMethods->xShmMap(p->pReal, iRegion, szRegion, isWrite, pp);
t = vfslog_time() - t;
vfslog_call(p->pVfslog, OS_SHMMAP, p->iFileId, t, rc, 0, 0);
return rc;
}
/*
@@ -826,11 +801,9 @@ static const char *vfslog_eventname(int eEvent){
case OS_SHMCLOSE: zEvent = "xShmClose"; break;
case OS_SHMOPEN: zEvent = "xShmOpen"; break;
case OS_SHMGET: zEvent = "xShmGet"; break;
case OS_SHMSIZE: zEvent = "xShmSize"; break;
case OS_SHMRELEASE: zEvent = "xShmRelease"; break;
case OS_SHMLOCK: zEvent = "xShmLock"; break;
case OS_SHMBARRIER: zEvent = "xShmBarrier"; break;
case OS_SHMMAP: zEvent = "xShmMap"; break;
case OS_ANNOTATE: zEvent = "annotation"; break;
}
+79 -88
View File
@@ -69,16 +69,17 @@ struct Testvfs {
** + Invoking the Tcl callback script.
*/
#define TESTVFS_SHMOPEN_MASK 0x00000001
#define TESTVFS_SHMSIZE_MASK 0x00000002
#define TESTVFS_SHMGET_MASK 0x00000004
#define TESTVFS_SHMRELEASE_MASK 0x00000008
#define TESTVFS_SHMLOCK_MASK 0x00000010
#define TESTVFS_SHMBARRIER_MASK 0x00000020
#define TESTVFS_SHMCLOSE_MASK 0x00000040
#define TESTVFS_SHMPAGE_MASK 0x00000080
#define TESTVFS_OPEN_MASK 0x00000080
#define TESTVFS_SYNC_MASK 0x00000100
#define TESTVFS_ALL_MASK 0x000001FF
#define TESTVFS_OPEN_MASK 0x00000100
#define TESTVFS_SYNC_MASK 0x00000200
#define TESTVFS_ALL_MASK 0x000003FF
#define TESTVFS_MAX_PAGES 256
/*
** A shared-memory buffer. There is one of these objects for each shared
@@ -87,8 +88,8 @@ struct Testvfs {
*/
struct TestvfsBuffer {
char *zFile; /* Associated file name */
int n; /* Size of allocated buffer in bytes */
u8 *a; /* Buffer allocated using ckalloc() */
int pgsz; /* Page size */
u8 *aPage[TESTVFS_MAX_PAGES]; /* Array of ckalloc'd pages */
TestvfsFile *pFile; /* List of open handles */
TestvfsBuffer *pNext; /* Next in linked list of all buffers */
};
@@ -133,12 +134,10 @@ static int tvfsSleep(sqlite3_vfs*, int microseconds);
static int tvfsCurrentTime(sqlite3_vfs*, double*);
static int tvfsShmOpen(sqlite3_file*);
static int tvfsShmSize(sqlite3_file*, int , int *);
static int tvfsShmGet(sqlite3_file*, int , int *, volatile void **);
static int tvfsShmRelease(sqlite3_file*);
static int tvfsShmLock(sqlite3_file*, int , int, int);
static void tvfsShmBarrier(sqlite3_file*);
static int tvfsShmClose(sqlite3_file*, int);
static int tvfsShmPage(sqlite3_file*,int,int,int, void volatile **);
static sqlite3_io_methods tvfs_io_methods = {
2, /* iVersion */
@@ -155,12 +154,10 @@ static sqlite3_io_methods tvfs_io_methods = {
tvfsSectorSize, /* xSectorSize */
tvfsDeviceCharacteristics, /* xDeviceCharacteristics */
tvfsShmOpen, /* xShmOpen */
tvfsShmSize, /* xShmSize */
tvfsShmGet, /* xShmGet */
tvfsShmRelease, /* xShmRelease */
tvfsShmLock, /* xShmLock */
tvfsShmBarrier, /* xShmBarrier */
tvfsShmClose /* xShmClose */
tvfsShmClose, /* xShmClose */
tvfsShmPage /* xShmPage */
};
static int tvfsResultCode(Testvfs *p, int *pRc){
@@ -443,12 +440,10 @@ static int tvfsOpen(
memcpy(pMethods, &tvfs_io_methods, sizeof(sqlite3_io_methods));
if( ((Testvfs *)pVfs->pAppData)->isNoshm ){
pMethods->xShmOpen = 0;
pMethods->xShmGet = 0;
pMethods->xShmSize = 0;
pMethods->xShmRelease = 0;
pMethods->xShmClose = 0;
pMethods->xShmLock = 0;
pMethods->xShmBarrier = 0;
pMethods->xShmMap = 0;
}
pFile->pMethods = pMethods;
}
@@ -547,16 +542,6 @@ static int tvfsCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){
return PARENTVFS(pVfs)->xCurrentTime(PARENTVFS(pVfs), pTimeOut);
}
static void tvfsGrowBuffer(TestvfsFile *pFd, int reqSize, int *pNewSize){
TestvfsBuffer *pBuffer = pFd->pShm;
if( reqSize>pBuffer->n ){
pBuffer->a = (u8 *)ckrealloc((char *)pBuffer->a, reqSize);
memset(&pBuffer->a[pBuffer->n], 0x55, reqSize-pBuffer->n);
pBuffer->n = reqSize;
}
*pNewSize = pBuffer->n;
}
static int tvfsInjectIoerr(Testvfs *p){
int ret = 0;
if( p->ioerr ){
@@ -619,71 +604,51 @@ static int tvfsShmOpen(
return SQLITE_OK;
}
static int tvfsShmSize(
sqlite3_file *pFile,
int reqSize,
int *pNewSize
static void tvfsAllocPage(TestvfsBuffer *p, int iPage, int pgsz){
assert( iPage<TESTVFS_MAX_PAGES );
if( p->aPage[iPage]==0 ){
p->aPage[iPage] = (u8 *)ckalloc(pgsz);
memset(p->aPage[iPage], 0, pgsz);
p->pgsz = pgsz;
}
}
static int tvfsShmPage(
sqlite3_file *pFile, /* Handle open on database file */
int iPage, /* Page to retrieve */
int pgsz, /* Size of pages */
int isWrite, /* True to extend file if necessary */
void volatile **pp /* OUT: Mapped memory */
){
int rc = SQLITE_OK;
TestvfsFile *pFd = (TestvfsFile *)pFile;
Testvfs *p = (Testvfs *)(pFd->pVfs->pAppData);
if( p->pScript && p->mask&TESTVFS_SHMSIZE_MASK ){
tvfsExecTcl(p, "xShmSize",
Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0
if( p->pScript && p->mask&TESTVFS_SHMPAGE_MASK ){
Tcl_Obj *pArg = Tcl_NewObj();
Tcl_IncrRefCount(pArg);
Tcl_ListObjAppendElement(p->interp, pArg, Tcl_NewIntObj(iPage));
Tcl_ListObjAppendElement(p->interp, pArg, Tcl_NewIntObj(pgsz));
Tcl_ListObjAppendElement(p->interp, pArg, Tcl_NewIntObj(isWrite));
tvfsExecTcl(p, "xShmPage",
Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, pArg
);
tvfsResultCode(p, &rc);
Tcl_DecrRefCount(pArg);
}
if( rc==SQLITE_OK && p->mask&TESTVFS_SHMSIZE_MASK && tvfsInjectIoerr(p) ){
rc = SQLITE_IOERR;
}
if( rc==SQLITE_OK ){
tvfsGrowBuffer(pFd, reqSize, pNewSize);
}
return rc;
}
static int tvfsShmGet(
sqlite3_file *pFile,
int reqMapSize,
int *pMapSize,
volatile void **pp
){
int rc = SQLITE_OK;
TestvfsFile *pFd = (TestvfsFile *)pFile;
Testvfs *p = (Testvfs *)(pFd->pVfs->pAppData);
if( p->pScript && p->mask&TESTVFS_SHMGET_MASK ){
tvfsExecTcl(p, "xShmGet",
Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId,
Tcl_NewIntObj(reqMapSize)
);
tvfsResultCode(p, &rc);
}
if( rc==SQLITE_OK && p->mask&TESTVFS_SHMGET_MASK && tvfsInjectIoerr(p) ){
if( rc==SQLITE_OK && p->mask&TESTVFS_SHMPAGE_MASK && tvfsInjectIoerr(p) ){
rc = SQLITE_IOERR;
}
*pMapSize = pFd->pShm->n;
*pp = pFd->pShm->a;
return rc;
}
static int tvfsShmRelease(sqlite3_file *pFile){
int rc = SQLITE_OK;
TestvfsFile *pFd = (TestvfsFile *)pFile;
Testvfs *p = (Testvfs *)(pFd->pVfs->pAppData);
if( p->pScript && p->mask&TESTVFS_SHMRELEASE_MASK ){
tvfsExecTcl(p, "xShmRelease",
Tcl_NewStringObj(pFd->pShm->zFile, -1), pFd->pShmId, 0
);
tvfsResultCode(p, &rc);
if( rc==SQLITE_OK && isWrite && !pFd->pShm->aPage[iPage] ){
tvfsAllocPage(pFd->pShm, iPage, pgsz);
}
*pp = (void volatile *)pFd->pShm->aPage[iPage];
return rc;
}
static int tvfsShmLock(
sqlite3_file *pFile,
int ofst,
@@ -782,10 +747,13 @@ static int tvfsShmClose(
*ppFd = pFd->pNext;
if( pBuffer->pFile==0 ){
int i;
TestvfsBuffer **pp;
for(pp=&p->pBuffer; *pp!=pBuffer; pp=&((*pp)->pNext));
*pp = (*pp)->pNext;
ckfree((char *)pBuffer->a);
for(i=0; pBuffer->aPage[i]; i++){
ckfree((char *)pBuffer->aPage[i]);
}
ckfree((char *)pBuffer);
}
pFd->pShm = 0;
@@ -821,28 +789,46 @@ static int testvfs_obj_cmd(
switch( (enum DB_enum)i ){
case CMD_SHM: {
Tcl_Obj *pObj;
int i;
TestvfsBuffer *pBuffer;
char *zName;
if( objc!=3 && objc!=4 ){
Tcl_WrongNumArgs(interp, 2, objv, "FILE ?VALUE?");
return TCL_ERROR;
}
zName = Tcl_GetString(objv[2]);
zName = ckalloc(p->pParent->mxPathname);
p->pParent->xFullPathname(
p->pParent, Tcl_GetString(objv[2]),
p->pParent->mxPathname, zName
);
for(pBuffer=p->pBuffer; pBuffer; pBuffer=pBuffer->pNext){
if( 0==strcmp(pBuffer->zFile, zName) ) break;
}
ckfree(zName);
if( !pBuffer ){
Tcl_AppendResult(interp, "no such file: ", zName, 0);
Tcl_AppendResult(interp, "no such file: ", Tcl_GetString(objv[2]), 0);
return TCL_ERROR;
}
if( objc==4 ){
int n;
u8 *a = Tcl_GetByteArrayFromObj(objv[3], &n);
pBuffer->a = (u8 *)ckrealloc((char *)pBuffer->a, n);
pBuffer->n = n;
memcpy(pBuffer->a, a, n);
assert( pBuffer->pgsz==0 || pBuffer->pgsz==32768 );
for(i=0; i*32768<n; i++){
int nByte = 32768;
tvfsAllocPage(pBuffer, i, 32768);
if( n-i*32768<32768 ){
nByte = n;
}
memcpy(pBuffer->aPage[i], &a[i*32768], nByte);
}
}
Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(pBuffer->a, pBuffer->n));
pObj = Tcl_NewObj();
for(i=0; pBuffer->aPage[i]; i++){
Tcl_AppendObjToObj(pObj, Tcl_NewByteArrayObj(pBuffer->aPage[i], 32768));
}
Tcl_SetObjResult(interp, pObj);
break;
}
@@ -852,12 +838,10 @@ static int testvfs_obj_cmd(
int mask;
} vfsmethod [] = {
{ "xShmOpen", TESTVFS_SHMOPEN_MASK },
{ "xShmSize", TESTVFS_SHMSIZE_MASK },
{ "xShmGet", TESTVFS_SHMGET_MASK },
{ "xShmRelease", TESTVFS_SHMRELEASE_MASK },
{ "xShmLock", TESTVFS_SHMLOCK_MASK },
{ "xShmBarrier", TESTVFS_SHMBARRIER_MASK },
{ "xShmClose", TESTVFS_SHMCLOSE_MASK },
{ "xShmPage", TESTVFS_SHMPAGE_MASK },
{ "xSync", TESTVFS_SYNC_MASK },
{ "xOpen", TESTVFS_OPEN_MASK },
};
@@ -899,6 +883,7 @@ static int testvfs_obj_cmd(
ckfree((char *)p->apScript);
p->apScript = 0;
p->nScript = 0;
p->pScript = 0;
}
Tcl_GetStringFromObj(objv[2], &nByte);
if( nByte>0 ){
@@ -1071,6 +1056,13 @@ static int testvfs_cmd(
p = (Testvfs *)ckalloc(nByte);
memset(p, 0, nByte);
/* Create the new object command before querying SQLite for a default VFS
** to use for 'real' IO operations. This is because creating the new VFS
** may delete an existing [testvfs] VFS of the same name. If such a VFS
** is currently the default, the new [testvfs] may end up calling the
** methods of a deleted object.
*/
Tcl_CreateObjCommand(interp, zVfs, testvfs_obj_cmd, p, testvfs_obj_del);
p->pParent = sqlite3_vfs_find(0);
p->interp = interp;
@@ -1087,7 +1079,6 @@ static int testvfs_cmd(
p->isNoshm = isNoshm;
p->mask = TESTVFS_ALL_MASK;
Tcl_CreateObjCommand(interp, zVfs, testvfs_obj_cmd, p, testvfs_obj_del);
sqlite3_vfs_register(pVfs, isDefault);
return TCL_OK;
+366 -397
View File
File diff suppressed because it is too large Load Diff
+1
View File
@@ -13,6 +13,7 @@
set testdir [file dirname $argv0]
source $testdir/tester.tcl
db close
# Argument processing.
#
+28 -38
View File
@@ -75,9 +75,14 @@ proc incr_tvfs_hdr {file idx incrval} {
# database content.
#
do_test wal2-1.0 {
proc tvfs_cb {method args} { return SQLITE_OK }
proc tvfs_cb {method filename args} {
set ::filename $filename
return SQLITE_OK
}
testvfs tvfs
tvfs script tvfs_cb
tvfs filter xShmOpen
sqlite3 db test.db -vfs tvfs
sqlite3 db2 test.db -vfs tvfs
@@ -123,21 +128,15 @@ foreach {tn iInsert res wal_index_hdr_mod wal_locks} "
do_test wal2-1.$tn.1 {
execsql { INSERT INTO t1 VALUES($iInsert) }
set ::locks [list]
set ::cb_done 0
proc tvfs_cb {method args} {
if {$::cb_done == 0 && $method == "xShmGet"} {
set ::cb_done 1
if {$::wal_index_hdr_mod >= 0} {
incr_tvfs_hdr [lindex $args 0] $::wal_index_hdr_mod 1
}
}
if {$method == "xShmLock"} { lappend ::locks [lindex $args 2] }
lappend ::locks [lindex $args 2]
return SQLITE_OK
}
tvfs filter xShmLock
if {$::wal_index_hdr_mod >= 0} {
incr_tvfs_hdr $::filename $::wal_index_hdr_mod 1
}
execsql { SELECT count(a), sum(a) FROM t1 } db2
} $res
@@ -174,8 +173,9 @@ do_test wal2-2.0 {
testvfs tvfs
tvfs script tvfs_cb
tvfs filter xShmOpen
proc tvfs_cb {method args} {
if {$method == "xShmOpen"} { set ::shm_file [lindex $args 0] }
set ::filename [lindex $args 0]
return SQLITE_OK
}
@@ -208,32 +208,28 @@ foreach {tn iInsert res0 res1 wal_index_hdr_mod} {
8 11 {10 55} {11 66} 6
9 12 {11 66} {12 78} 7
} {
tvfs filter xShmLock
do_test wal2-2.$tn.1 {
set oldhdr [set_tvfs_hdr $::shm_file]
set oldhdr [set_tvfs_hdr $::filename]
execsql { INSERT INTO t1 VALUES($iInsert) }
execsql { SELECT count(a), sum(a) FROM t1 }
} $res1
do_test wal2-2.$tn.2 {
set ::locks [list]
set ::cb_done 0
proc tvfs_cb {method args} {
if {$::cb_done == 0 && $method == "xShmGet"} {
set ::cb_done 1
if {$::wal_index_hdr_mod >= 0} {
incr_tvfs_hdr $::shm_file $::wal_index_hdr_mod 1
}
}
if {$method == "xShmLock"} {
set lock [lindex $args 2]
lappend ::locks $lock
if {$lock == $::WRITER} {
set_tvfs_hdr $::shm_file $::oldhdr
}
set lock [lindex $args 2]
lappend ::locks $lock
if {$lock == $::WRITER} {
set_tvfs_hdr $::filename $::oldhdr
}
return SQLITE_OK
}
if {$::wal_index_hdr_mod >= 0} {
incr_tvfs_hdr $::filename $::wal_index_hdr_mod 1
}
execsql { SELECT count(a), sum(a) FROM t1 } db2
} $res0
@@ -243,21 +239,15 @@ foreach {tn iInsert res0 res1 wal_index_hdr_mod} {
do_test wal2-2.$tn.4 {
set ::locks [list]
set ::cb_done 0
proc tvfs_cb {method args} {
if {$::cb_done == 0 && $method == "xShmGet"} {
set ::cb_done 1
if {$::wal_index_hdr_mod >= 0} {
incr_tvfs_hdr $::shm_file $::wal_index_hdr_mod 1
}
}
if {$method == "xShmLock"} {
set lock [lindex $args 2]
lappend ::locks $lock
}
set lock [lindex $args 2]
lappend ::locks $lock
return SQLITE_OK
}
if {$::wal_index_hdr_mod >= 0} {
incr_tvfs_hdr $::filename $::wal_index_hdr_mod 1
}
execsql { SELECT count(a), sum(a) FROM t1 } db2
} $res1
}
+1 -1
View File
@@ -353,7 +353,7 @@ T script method_callback
proc method_callback {method args} {
if {$method == "xShmBarrier"} {
incr ::barrier_count
if {$::barrier_count == 1} {
if {$::barrier_count == 2} {
# This code is executed within the xShmBarrier() callback invoked
# by the client running recovery as part of writing the recovered
# wal-index header. If a second client attempts to access the
+83 -3
View File
@@ -120,6 +120,7 @@ do_faultsim_test walfault-3 -prep {
faultsim_test_result {0 {}}
}
#--------------------------------------------------------------------------
#
faultsim_delete_and_reopen
@@ -152,7 +153,7 @@ do_test walfault-5-pre-1 {
do_faultsim_test walfault-5 -faults shmerr* -prep {
faultsim_restore_and_reopen
execsql { PRAGMA wal_autocheckpoint = 0 }
shmfault filter xShmSize
shmfault filter xShmPage
} -body {
execsql {
CREATE TABLE t1(x);
@@ -211,7 +212,7 @@ do_test walfault-6-pre-1 {
} {}
do_faultsim_test walfault-6 -faults shmerr* -prep {
faultsim_restore_and_reopen
shmfault filter xShmSize
shmfault filter xShmPage
} -body {
execsql { SELECT count(*) FROM t1 }
} -test {
@@ -326,7 +327,7 @@ do_test walfault-10-pre1 {
faultsim_delete_and_reopen
execsql {
PRAGMA journal_mode = WAL;
PRAGMA wal_checkpoint = 0;
PRAGMA wal_autocheckpoint = 0;
CREATE TABLE z(zz INTEGER PRIMARY KEY, zzz BLOB);
CREATE INDEX zzzz ON z(zzz);
INSERT INTO z VALUES(NULL, randomblob(800));
@@ -363,5 +364,84 @@ do_faultsim_test walfault-10 -prep {
if {$n != "64 51200"} { error "Incorrect data: $n" }
}
#--------------------------------------------------------------------------
# Test fault injection while checkpointing a large WAL file, if the
# checkpoint is the first operation run after opening the database.
# This means that some of the required wal-index pages are mapped as part of
# the checkpoint process, which means there are a few more opportunities
# for IO errors.
#
# To speed this up, IO errors are only simulated within xShmPage() calls.
#
do_test walfault-11-pre-1 {
sqlite3 db test.db
execsql {
PRAGMA journal_mode = WAL;
PRAGMA wal_autocheckpoint = 0;
BEGIN;
CREATE TABLE abc(a PRIMARY KEY);
INSERT INTO abc VALUES(randomblob(1500));
INSERT INTO abc VALUES(randomblob(1500));
INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 4
INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 8
INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 16
INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 32
INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 64
INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 128
INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 256
INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 512
INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 1024
INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 2048
INSERT INTO abc SELECT randomblob(1500) FROM abc; -- 4096
COMMIT;
}
faultsim_save_and_close
} {}
do_faultsim_test walfault-11 -faults shmerr* -prep {
catch { db2 close }
faultsim_restore_and_reopen
shmfault filter xShmPage
} -body {
db eval { SELECT count(*) FROM abc }
sqlite3 db2 test.db -vfs shmfault
db2 eval { PRAGMA wal_checkpoint }
} -test {
faultsim_test_result {0 {}}
}
#-------------------------------------------------------------------------
# Test the handling of the various IO/OOM/SHM errors that may occur during
# a log recovery operation undertaken as part of a call to
# sqlite3_wal_checkpoint().
#
do_test walfault-12-pre-1 {
faultsim_delete_and_reopen
execsql {
PRAGMA journal_mode = WAL;
PRAGMA wal_autocheckpoint = 0;
BEGIN;
CREATE TABLE abc(a PRIMARY KEY);
INSERT INTO abc VALUES(randomblob(1500));
INSERT INTO abc VALUES(randomblob(1500));
COMMIT;
}
faultsim_save_and_close
} {}
do_faultsim_test walfault-12 -prep {
if {[info commands shmfault] == ""} {
testvfs shmfault -default true
}
faultsim_restore_and_reopen
db eval { SELECT * FROM sqlite_master }
shmfault shm test.db [string repeat "\000" 40]
} -body {
set rc [sqlite3_wal_checkpoint db]
if {$rc != "SQLITE_OK"} { error [sqlite3_errmsg db] }
} -test {
db close
faultsim_test_result {0 {}}
}
finish_test