From f0b190d94c1d80b7c20d1a4d52aef9187142c864 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 26 Jul 2011 16:03:07 +0000 Subject: [PATCH 1/2] Prototype change for a new sqlite3_file_control() that will cause the -wal and -shm files to persist after the last database connection closes. FossilOrigin-Name: e34c553bf04761e86f3bd72f91439c05886caa5c --- manifest | 21 ++++++++++++--------- manifest.uuid | 2 +- src/os_unix.c | 25 +++++++++++++++++++------ src/os_win.c | 19 +++++++++++++++---- src/sqlite.h.in | 15 +++++++++++++++ src/wal.c | 4 +++- 6 files changed, 65 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index e1657062a6..38df8835a6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enable\sthe\sSQLITE_FCNTL_SIZE_HINT\son\sunix\seven\sif\sSQLITE_FCNTL_CHUNK_SIZE\nhas\snot\sbeen\sset. -D 2011-07-25T23:25:47.867 +C Prototype\schange\sfor\sa\snew\ssqlite3_file_control()\sthat\swill\scause\sthe\n-wal\sand\s-shm\sfiles\sto\spersist\safter\sthe\slast\sdatabase\sconnection\scloses. +D 2011-07-26T16:03:07.413 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 1e6988b3c11dee9bd5edc0c804bd4468d74a9cdc F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -165,8 +165,8 @@ F src/os.c fcc717427a80b2ed225373f07b642dc1aad7490b F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h 65a897143b64667d23ed329a7984b9b405accb58 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c b83e0924d1a832a89989d3d2770179205d0e066a -F src/os_win.c 8449cb4ce1cd50248b7133108532d9582cc6c042 +F src/os_unix.c 294e8e9803454d70e796d84c77f990cc6d946899 +F src/os_win.c 313f48b6f92890f2a42d5ff6fa0fb0501d2fff25 F src/pager.c 120550e7ef01dafaa2cbb4a0528c0d87c8f12b41 F src/pager.h 3f8c783de1d4706b40b1ac15b64f5f896bcc78d1 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 @@ -181,7 +181,7 @@ F src/resolve.c 36368f44569208fa074e61f4dd0b6c4fb60ca2b4 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c d219c4b68d603cc734b6f9b1e2780fee12a1fa0d F src/shell.c bbe7818ff5bc8614105ceb81ad67b8bdc0b671dd -F src/sqlite.h.in badc4f56b9b57a8731475c2ddbccd8198ce33b9b +F src/sqlite.h.in 0b3cab7b2ea51f58396e8871fa5f349cfece5330 F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 F src/sqliteInt.h ba4a6d6288efb25b84bc0d7d0aaf80f9b42523ba F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d @@ -247,7 +247,7 @@ F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 -F src/wal.c 0c70ad7b1cac6005fa5e2cbefd23ee05e391c290 +F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9 F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f F src/where.c 106cd9ab3eb410dfa7d0598194c277664bb2e9a3 @@ -952,7 +952,10 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 08d0e8799e1441ef063b1cdf9e4107071a0f81ca -R 513a198a27aca1f07c596d222f883111 +P 05c9832e5f6eb705f1dce4e65cf4e2d56512ff6b +R 89b4ef80e4aa70f9d9aab4e9774d951c +T *branch * persistent-wal-patch +T *sym-persistent-wal-patch * +T -sym-trunk * U drh -Z bf5f24c7092583c5f65553c09e369462 +Z fcc3b5975e3b68e4a98c8be3a9674ec5 diff --git a/manifest.uuid b/manifest.uuid index 3e9f01365c..c07e3ca2d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -05c9832e5f6eb705f1dce4e65cf4e2d56512ff6b \ No newline at end of file +e34c553bf04761e86f3bd72f91439c05886caa5c \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index dce13e5365..9f8204dfd2 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -250,8 +250,9 @@ struct unixFile { /* ** Allowed values for the unixFile.ctrlFlags bitmask: */ -#define UNIXFILE_EXCL 0x01 /* Connections from one process only */ -#define UNIXFILE_RDONLY 0x02 /* Connection is read only */ +#define UNIXFILE_EXCL 0x01 /* Connections from one process only */ +#define UNIXFILE_RDONLY 0x02 /* Connection is read only */ +#define UNIXFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ /* ** Include code that is common to all os_*.c files @@ -3450,21 +3451,33 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ ** Information and control of an open file handle. */ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ + unixFile *pFile = (unixFile*)id; switch( op ){ case SQLITE_FCNTL_LOCKSTATE: { - *(int*)pArg = ((unixFile*)id)->eFileLock; + *(int*)pArg = pFile->eFileLock; return SQLITE_OK; } case SQLITE_LAST_ERRNO: { - *(int*)pArg = ((unixFile*)id)->lastErrno; + *(int*)pArg = pFile->lastErrno; return SQLITE_OK; } case SQLITE_FCNTL_CHUNK_SIZE: { - ((unixFile*)id)->szChunk = *(int *)pArg; + pFile->szChunk = *(int *)pArg; return SQLITE_OK; } case SQLITE_FCNTL_SIZE_HINT: { - return fcntlSizeHint((unixFile *)id, *(i64 *)pArg); + return fcntlSizeHint(pFile, *(i64 *)pArg); + } + case SQLITE_FCNTL_PERSIST_WAL: { + int bPersist = *(int*)pArg; + if( bPersist<0 ){ + bPersist = (pFile->ctrlFlags & UNIXFILE_PERSIST_WAL)!=0; + }else if( bPersist==0 ){ + pFile->ctrlFlags &= ~UNIXFILE_PERSIST_WAL; + }else{ + pFile->ctrlFlags |= UNIXFILE_PERSIST_WAL; + } + return SQLITE_OK; } #ifndef NDEBUG /* The pager calls this method to signal that it has done diff --git a/src/os_win.c b/src/os_win.c index 5d9bf59843..4c0fc86869 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -102,8 +102,9 @@ struct winFile { const sqlite3_io_methods *pMethod; /*** Must be first ***/ sqlite3_vfs *pVfs; /* The VFS used to open this file */ HANDLE h; /* Handle for accessing the file */ - unsigned char locktype; /* Type of lock currently held on this file */ + u8 locktype; /* Type of lock currently held on this file */ short sharedLockByte; /* Randomly chosen byte used as a shared lock */ + u8 bPersistWal; /* True to persist WAL files */ DWORD lastErrno; /* The Windows errno from the last I/O error */ DWORD sectorSize; /* Sector size of the device file is on */ winShm *pShm; /* Instance of shared memory on this file */ @@ -1335,17 +1336,18 @@ static int winUnlock(sqlite3_file *id, int locktype){ ** Control and query of the open file handle. */ static int winFileControl(sqlite3_file *id, int op, void *pArg){ + winFile *pFile = (winFile*)id; switch( op ){ case SQLITE_FCNTL_LOCKSTATE: { - *(int*)pArg = ((winFile*)id)->locktype; + *(int*)pArg = pFile->locktype; return SQLITE_OK; } case SQLITE_LAST_ERRNO: { - *(int*)pArg = (int)((winFile*)id)->lastErrno; + *(int*)pArg = (int)pFile->lastErrno; return SQLITE_OK; } case SQLITE_FCNTL_CHUNK_SIZE: { - ((winFile*)id)->szChunk = *(int *)pArg; + pFile->szChunk = *(int *)pArg; return SQLITE_OK; } case SQLITE_FCNTL_SIZE_HINT: { @@ -1355,6 +1357,15 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ SimulateIOErrorBenign(0); return SQLITE_OK; } + case SQLITE_FCNTL_PERSIST_WAL: { + int bPersist = *(int*)pArg; + if( bPersist<0 ){ + bPersist = pFile->bPersistWal; + }else{ + pFile->bPersistWal = bPersist!=0; + } + return SQLITE_OK; + } case SQLITE_FCNTL_SYNC_OMITTED: { return SQLITE_OK; } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index b8e8bfbc46..eb5f7a02a0 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -752,6 +752,20 @@ struct sqlite3_io_methods { ** is not changed but instead the prior value of that setting is written ** into the array entry, allowing the current retry settings to be ** interrogated. The zDbName parameter is ignored. +** +** ^The [SQLITE_FCNTL_PERSIST_WAL] opcode is used to set or query the +** persistent [WAL | Write AHead Log] setting. By default, the auxiliary +** write ahead log and shared memory files used for transaction control +** are automatically deleted when the latest connection to the database +** closes. Setting persistent WAL mode causes those files to persist after +** close. Persisting the files is useful when other processes that do not +** have write permission on the directory containing the database file want +** to read the database file, as the WAL and shared memory files must exist +** in order for the database to be readable. The fourth parameter to +** [sqlite3_file_control()] for this opcode should be a pointer to an integer. +** That integer is 0 to disable persistent WAL mode or 1 to enable persistent +** WAL mode. If the integer is -1, then it is overwritten with the current +** WAL persistence setting. ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -763,6 +777,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_FILE_POINTER 7 #define SQLITE_FCNTL_SYNC_OMITTED 8 #define SQLITE_FCNTL_WIN32_AV_RETRY 9 +#define SQLITE_FCNTL_PERSIST_WAL 10 /* ** CAPI3REF: Mutex Handle diff --git a/src/wal.c b/src/wal.c index b9a03dff23..3bc42ffb17 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1804,13 +1804,15 @@ int sqlite3WalClose( */ rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE); if( rc==SQLITE_OK ){ + int bPersistWal = -1; if( pWal->exclusiveMode==WAL_NORMAL_MODE ){ pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; } rc = sqlite3WalCheckpoint( pWal, SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0 ); - if( rc==SQLITE_OK ){ + sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_PERSIST_WAL, &bPersistWal); + if( rc==SQLITE_OK && bPersistWal!=1 ){ isDelete = 1; } } From 253cea5c6f34b1be77e9cc9c9c5ad35587ec5b25 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 26 Jul 2011 16:23:25 +0000 Subject: [PATCH 2/2] Test cases added. Fix the query mode. FossilOrigin-Name: a9d8794ae949df570466a84836882bc8bed95c7c --- manifest | 20 ++++++------- manifest.uuid | 2 +- src/os_unix.c | 2 +- src/os_win.c | 2 +- src/test1.c | 33 +++++++++++++++++++++ test/walpersist.test | 68 ++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 113 insertions(+), 14 deletions(-) create mode 100644 test/walpersist.test diff --git a/manifest b/manifest index 38df8835a6..afa1dda3c6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Prototype\schange\sfor\sa\snew\ssqlite3_file_control()\sthat\swill\scause\sthe\n-wal\sand\s-shm\sfiles\sto\spersist\safter\sthe\slast\sdatabase\sconnection\scloses. -D 2011-07-26T16:03:07.413 +C Test\scases\sadded.\s\sFix\sthe\squery\smode. +D 2011-07-26T16:23:25.835 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 1e6988b3c11dee9bd5edc0c804bd4468d74a9cdc F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -165,8 +165,8 @@ F src/os.c fcc717427a80b2ed225373f07b642dc1aad7490b F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h 65a897143b64667d23ed329a7984b9b405accb58 F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c 294e8e9803454d70e796d84c77f990cc6d946899 -F src/os_win.c 313f48b6f92890f2a42d5ff6fa0fb0501d2fff25 +F src/os_unix.c 637fa969213812ba92f1f3c54e4be967bda60a13 +F src/os_win.c 7dd45f72a6c6ae5a985865b440ebd14d8ee7b874 F src/pager.c 120550e7ef01dafaa2cbb4a0528c0d87c8f12b41 F src/pager.h 3f8c783de1d4706b40b1ac15b64f5f896bcc78d1 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 @@ -188,7 +188,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c c355b33803fb756dc09297ec84d19604301fa622 -F src/test1.c 693d9a63dfe2c68b167080c99cab82f267f5a38e +F src/test1.c e7ea726d26b9007bae7907d8b40280c70ec6c504 F src/test2.c 80d323d11e909cf0eb1b6fbb4ac22276483bcf31 F src/test3.c 124ff9735fb6bb7d41de180d6bac90e7b1509432 F src/test4.c d1e5a5e904d4b444cf572391fdcb017638e36ff7 @@ -894,6 +894,7 @@ F test/walfault.test 58fce626359c9376fe35101b5c0f2df8040aa839 F test/walhook.test ed00a40ba7255da22d6b66433ab61fab16a63483 F test/walmode.test 22ddccd073c817ac9ead62b88ac446e8dedc7d2c F test/walnoshm.test a074428046408f4eb5c6a00e09df8cc97ff93317 +F test/walpersist.test 45fb0c94fb63908e2d66b1d99ce4645bfce0fa1e F test/walro.test 2d5d69e2e99da19ce6faab340330234fc4ca0720 F test/walshared.test 6dda2293880c300baf5d791c307f653094585761 F test/walslow.test d21625e2e99e11c032ce949e8a94661576548933 @@ -952,10 +953,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 05c9832e5f6eb705f1dce4e65cf4e2d56512ff6b -R 89b4ef80e4aa70f9d9aab4e9774d951c -T *branch * persistent-wal-patch -T *sym-persistent-wal-patch * -T -sym-trunk * +P e34c553bf04761e86f3bd72f91439c05886caa5c +R 19cfffdeb7e8194e5c58a90ee9f2b26d U drh -Z fcc3b5975e3b68e4a98c8be3a9674ec5 +Z c92a162aecc6522b9f0c0715256a714d diff --git a/manifest.uuid b/manifest.uuid index c07e3ca2d5..1a4145a1d2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e34c553bf04761e86f3bd72f91439c05886caa5c \ No newline at end of file +a9d8794ae949df570466a84836882bc8bed95c7c \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 9f8204dfd2..dbf3c10120 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3471,7 +3471,7 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ case SQLITE_FCNTL_PERSIST_WAL: { int bPersist = *(int*)pArg; if( bPersist<0 ){ - bPersist = (pFile->ctrlFlags & UNIXFILE_PERSIST_WAL)!=0; + *(int*)pArg = (pFile->ctrlFlags & UNIXFILE_PERSIST_WAL)!=0; }else if( bPersist==0 ){ pFile->ctrlFlags &= ~UNIXFILE_PERSIST_WAL; }else{ diff --git a/src/os_win.c b/src/os_win.c index 4c0fc86869..827091e84e 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1360,7 +1360,7 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ case SQLITE_FCNTL_PERSIST_WAL: { int bPersist = *(int*)pArg; if( bPersist<0 ){ - bPersist = pFile->bPersistWal; + *(int*)pArg = pFile->bPersistWal; }else{ pFile->bPersistWal = bPersist!=0; } diff --git a/src/test1.c b/src/test1.c index 3301ab7403..9a5a50103f 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5129,6 +5129,38 @@ static int file_control_win32_av_retry( return TCL_OK; } +/* +** tclcmd: file_control_persist_wal DB PERSIST-FLAG +** +** This TCL command runs the sqlite3_file_control interface with +** the SQLITE_FCNTL_PERSIST_WAL opcode. +*/ +static int file_control_persist_wal( + ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3 *db; + int rc; + int bPersist; + char z[100]; + + if( objc!=3 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", + Tcl_GetStringFromObj(objv[0], 0), " DB FLAG", 0); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[2], &bPersist) ) return TCL_ERROR; + rc = sqlite3_file_control(db, NULL, SQLITE_FCNTL_PERSIST_WAL, (void*)&bPersist); + sqlite3_snprintf(sizeof(z), z, "%d %d", rc, bPersist); + Tcl_AppendResult(interp, z, (char*)0); + return TCL_OK; +} + /* ** tclcmd: sqlite3_vfs_list @@ -5928,6 +5960,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "file_control_chunksize_test", file_control_chunksize_test, 0 }, { "file_control_sizehint_test", file_control_sizehint_test, 0 }, { "file_control_win32_av_retry", file_control_win32_av_retry, 0 }, + { "file_control_persist_wal", file_control_persist_wal, 0 }, { "sqlite3_vfs_list", vfs_list, 0 }, { "sqlite3_create_function_v2", test_create_function_v2, 0 }, diff --git a/test/walpersist.test b/test/walpersist.test new file mode 100644 index 0000000000..3f8b0b8930 --- /dev/null +++ b/test/walpersist.test @@ -0,0 +1,68 @@ +# 2011 July 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for using WAL with persistent WAL file mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +set ::testprefix walpersist + +do_test walpersist-1.0 { + db eval { + PRAGMA journal_mode=WAL; + CREATE TABLE t1(a); + INSERT INTO t1 VALUES(randomblob(5000)); + } + file exists test.db-wal +} {1} +do_test walpersist-1.1 { + file exists test.db-shm +} {1} +do_test walpersist-1.2 { + db close + list [file exists test.db] [file exists test.db-wal] [file exists test.db-shm] +} {1 0 0} +do_test walpersist-1.3 { + sqlite3 db test.db + db eval {SELECT length(a) FROM t1} +} {5000} +do_test walpersist-1.4 { + list [file exists test.db] [file exists test.db-wal] [file exists test.db-shm] +} {1 1 1} +do_test walpersist-1.5 { + file_control_persist_wal db -1 +} {0 0} +do_test walpersist-1.6 { + file_control_persist_wal db 1 +} {0 1} +do_test walpersist-1.7 { + file_control_persist_wal db -1 +} {0 1} +do_test walpersist-1.8 { + file_control_persist_wal db 0 +} {0 0} +do_test walpersist-1.9 { + file_control_persist_wal db -1 +} {0 0} +do_test walpersist-1.10 { + file_control_persist_wal db 1 +} {0 1} +do_test walpersist-1.11 { + db close + list [file exists test.db] [file exists test.db-wal] [file exists test.db-shm] +} {1 1 1} + + + + +finish_test