From ba6eb8769f4f22cbf0f0244e2d39ac887c85d935 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 15 Nov 2016 17:37:56 +0000 Subject: [PATCH 01/11] Experimental changes toward making snapshots serializable. FossilOrigin-Name: b6a81fa1fc1fb78a65894129851a4ec3986640de --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 4 +++- src/wal.c | 5 +++++ 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index ebd7485bf2..9c806b3a89 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sOP_Last\sopcode\sso\sthat\sit\sis\sa\sno-op\sif\sthe\scursor\sis\salready\npointing\sat\sthe\send\sof\sthe\sb-tree. -D 2016-11-15T04:00:24.244 +C Experimental\schanges\stoward\smaking\ssnapshots\sserializable. +D 2016-11-15T17:37:56.374 F Makefile.in 6b572807415d3f0a379cebc9461416d8df4a12c8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc bb4d970894abbbe0e88d00aac29bd52af8bc95f4 @@ -389,7 +389,7 @@ F src/resolve.c bb070cf5f23611c44ab7e4788803684e385fc3fb F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 672b1af237ad257149fc5189f3277dcbca036eeb F src/shell.c f04e4af75c5517735397d060ed0b4a874104bb41 -F src/sqlite.h.in 803f7050f69b2eea573fac219f3c92582c096027 +F src/sqlite.h.in dfe8c1b4ac66635495ec9a6a7205fcb0f27f4451 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae F src/sqliteInt.h c471d791b10c0f2164c8b7a87adc338e703c09cc @@ -465,7 +465,7 @@ F src/vdbesort.c 91fda3909326860382b0ca8aa251e609c6a9d62c F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 F src/vtab.c e02cacb5c7ae742631edeb9ae9f53d399f093fd8 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 7e8557314bcf867ffe3fff1d8ef2d207ace8e065 +F src/wal.c 88f8d8adcaecf6a225ae5098062fd151634fb672 F src/wal.h bf03a23da3100ab25e5c0363450233cfee09cfc2 F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 F src/where.c 952f76e7a03727480b274b66ca6641b1657cd591 @@ -1534,7 +1534,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5ed0bd387699a0f3b477d4f3cfcb460a6a44cb84 -R 666875522a5a69f5f9bb179805474981 +P 663473850c4274270445b3771911fa773a8c405f +R 1784ed6bcadef61e22ad39533122ebea +T *branch * serializable-snapshot +T *sym-serializable-snapshot * +T -sym-trunk * U drh -Z 3c6d6cf804e5301c81a1f9af34f5b4ed +Z 8a16f1f47c269929c7f968ee35b8cb55 diff --git a/manifest.uuid b/manifest.uuid index 53cf085748..5e74943d27 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -663473850c4274270445b3771911fa773a8c405f \ No newline at end of file +b6a81fa1fc1fb78a65894129851a4ec3986640de \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index dd70407c1b..7987cfae5e 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -8283,7 +8283,9 @@ int sqlite3_system_errno(sqlite3*); ** to an historical snapshot (if possible). The destructor for ** sqlite3_snapshot objects is [sqlite3_snapshot_free()]. */ -typedef struct sqlite3_snapshot sqlite3_snapshot; +typedef struct sqlite3_snapshot { + unsigned char hidden[48]; +} sqlite3_snapshot; /* ** CAPI3REF: Record A Database Snapshot diff --git a/src/wal.c b/src/wal.c index d71ee81a51..669c428efa 100644 --- a/src/wal.c +++ b/src/wal.c @@ -3393,9 +3393,14 @@ int sqlite3WalHeapMemory(Wal *pWal){ int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapshot){ int rc = SQLITE_OK; WalIndexHdr *pRet; + static const u32 aZero[4] = { 0, 0, 0, 0 }; assert( pWal->readLock>=0 && pWal->writeLock==0 ); + if( memcmp(&pWal->hdr.aFrameCksum[0],aZero,16)==0 ){ + *ppSnapshot = 0; + return SQLITE_ERROR; + } pRet = (WalIndexHdr*)sqlite3_malloc(sizeof(WalIndexHdr)); if( pRet==0 ){ rc = SQLITE_NOMEM_BKPT; From 25accbca7234545ce9d665a1f5672ab0bf04d38a Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 18 Nov 2016 14:38:41 +0000 Subject: [PATCH 02/11] Enhance existing snapshot tests to serialize/deserialize snapshots. No new tests. FossilOrigin-Name: 16b9bf92741e4c62874cffd7c6a61763c5054c7a --- manifest | 19 +- manifest.uuid | 2 +- src/test1.c | 110 +++++++ test/snapshot.test | 779 +++++++++++++++++++++++---------------------- 4 files changed, 526 insertions(+), 384 deletions(-) diff --git a/manifest b/manifest index 9c806b3a89..c5597fb646 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\schanges\stoward\smaking\ssnapshots\sserializable. -D 2016-11-15T17:37:56.374 +C Enhance\sexisting\ssnapshot\stests\sto\sserialize/deserialize\ssnapshots.\sNo\snew\ntests. +D 2016-11-18T14:38:41.852 F Makefile.in 6b572807415d3f0a379cebc9461416d8df4a12c8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc bb4d970894abbbe0e88d00aac29bd52af8bc95f4 @@ -397,7 +397,7 @@ F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 F src/tclsqlite.c aef87dcd8cb66564d560ab48d43d19ac812a1eab -F src/test1.c f52bed152c97be23be794efa234374d0de593eeb +F src/test1.c 58de30ed902f78531cf5cf52b883a26d107208c4 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c d03f5b5da9a2410b7a91c64b0d3306ed28ab6fee F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6 @@ -1102,7 +1102,7 @@ F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 F test/skipscan6.test 5866039d03a56f5bd0b3d172a012074a1d90a15b -F test/snapshot.test a19465046168b4420b5afeed37c3214e42a49f4a +F test/snapshot.test bb34cc38828bf2e86a49207995fa7dfacd303832 F test/snapshot_fault.test 062ff0438a074978d45e9f9a92e7ad459b74ee73 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 843cd84db9891b2d01b9ab64cef3e9020f98d087 @@ -1534,10 +1534,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 663473850c4274270445b3771911fa773a8c405f -R 1784ed6bcadef61e22ad39533122ebea -T *branch * serializable-snapshot -T *sym-serializable-snapshot * -T -sym-trunk * -U drh -Z 8a16f1f47c269929c7f968ee35b8cb55 +P b6a81fa1fc1fb78a65894129851a4ec3986640de +R 17ac6d49deba8d40db4fa5f6ce7fc2ca +U dan +Z cee5f11eb9093786088be3369dcd2153 diff --git a/manifest.uuid b/manifest.uuid index 5e74943d27..6af23f5732 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b6a81fa1fc1fb78a65894129851a4ec3986640de \ No newline at end of file +16b9bf92741e4c62874cffd7c6a61763c5054c7a \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 18b6b295c2..e31ad95f40 100644 --- a/src/test1.c +++ b/src/test1.c @@ -2388,6 +2388,113 @@ static int SQLITE_TCLAPI test_snapshot_cmp( } #endif /* SQLITE_ENABLE_SNAPSHOT */ +#ifdef SQLITE_ENABLE_SNAPSHOT +/* +** Usage: sqlite3_snapshot_get_blob DB DBNAME +*/ +static int SQLITE_TCLAPI test_snapshot_get_blob( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int rc; + sqlite3 *db; + char *zName; + sqlite3_snapshot *pSnapshot = 0; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zName = Tcl_GetString(objv[2]); + + rc = sqlite3_snapshot_get(db, zName, &pSnapshot); + if( rc!=SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + return TCL_ERROR; + }else{ + Tcl_SetObjResult(interp, + Tcl_NewByteArrayObj((unsigned char*)pSnapshot, sizeof(sqlite3_snapshot)) + ); + sqlite3_snapshot_free(pSnapshot); + } + return TCL_OK; +} +#endif /* SQLITE_ENABLE_SNAPSHOT */ + +#ifdef SQLITE_ENABLE_SNAPSHOT + /* + ** Usage: sqlite3_snapshot_open_blob DB DBNAME SNAPSHOT +*/ +static int SQLITE_TCLAPI test_snapshot_open_blob( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int rc; + sqlite3 *db; + char *zName; + unsigned char *pBlob; + int nBlob; + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME SNAPSHOT"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zName = Tcl_GetString(objv[2]); + pBlob = Tcl_GetByteArrayFromObj(objv[3], &nBlob); + if( nBlob!=sizeof(sqlite3_snapshot) ){ + Tcl_AppendResult(interp, "bad SNAPSHOT", 0); + return TCL_ERROR; + } + rc = sqlite3_snapshot_open(db, zName, (sqlite3_snapshot*)pBlob); + if( rc!=SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + return TCL_ERROR; + } + return TCL_OK; +} +#endif /* SQLITE_ENABLE_SNAPSHOT */ + +#ifdef SQLITE_ENABLE_SNAPSHOT +/* +** Usage: sqlite3_snapshot_cmp_blob SNAPSHOT1 SNAPSHOT2 +*/ +static int SQLITE_TCLAPI test_snapshot_cmp_blob( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int res; + unsigned char *p1; + unsigned char *p2; + int n1; + int n2; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "SNAPSHOT1 SNAPSHOT2"); + return TCL_ERROR; + } + + p1 = Tcl_GetByteArrayFromObj(objv[1], &n1); + p2 = Tcl_GetByteArrayFromObj(objv[2], &n2); + + if( n1!=sizeof(sqlite3_snapshot) || n1!=n2 ){ + Tcl_AppendResult(interp, "bad SNAPSHOT", 0); + return TCL_ERROR; + } + + res = sqlite3_snapshot_cmp((sqlite3_snapshot*)p1, (sqlite3_snapshot*)p2); + Tcl_SetObjResult(interp, Tcl_NewIntObj(res)); + return TCL_OK; +} +#endif /* SQLITE_ENABLE_SNAPSHOT */ + /* ** Usage: sqlite3_delete_database FILENAME */ @@ -7539,6 +7646,9 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_snapshot_open", test_snapshot_open, 0 }, { "sqlite3_snapshot_free", test_snapshot_free, 0 }, { "sqlite3_snapshot_cmp", test_snapshot_cmp, 0 }, + { "sqlite3_snapshot_get_blob", test_snapshot_get_blob, 0 }, + { "sqlite3_snapshot_open_blob", test_snapshot_open_blob, 0 }, + { "sqlite3_snapshot_cmp_blob", test_snapshot_cmp_blob, 0 }, #endif { "sqlite3_delete_database", test_delete_database, 0 }, }; diff --git a/test/snapshot.test b/test/snapshot.test index b2d61941a7..1143c14962 100644 --- a/test/snapshot.test +++ b/test/snapshot.test @@ -26,413 +26,448 @@ if {[permutation]=="inmemory_journal"} { return } -#------------------------------------------------------------------------- -# Check some error conditions in snapshot_get(). It is an error if: -# -# 1) snapshot_get() is called on a non-WAL database, or -# 2) there is an open write transaction on the database. -# -do_execsql_test 1.0 { - CREATE TABLE t1(a, b); - INSERT INTO t1 VALUES(1, 2); - INSERT INTO t1 VALUES(3, 4); -} - -do_test 1.1.1 { - execsql { BEGIN; SELECT * FROM t1; } - list [catch { sqlite3_snapshot_get db main } msg] $msg -} {1 SQLITE_ERROR} -do_execsql_test 1.1.2 COMMIT - -do_test 1.2.1 { - execsql { - PRAGMA journal_mode = WAL; - BEGIN; - INSERT INTO t1 VALUES(5, 6); - INSERT INTO t1 VALUES(7, 8); +foreach {tn tcl} { + 1 { + proc snapshot_get {DB DBNAME} { + uplevel [list sqlite3_snapshot_get $DB $DBNAME] + } + proc snapshot_open {DB DBNAME SNAPSHOT} { + uplevel [list sqlite3_snapshot_open $DB $DBNAME $SNAPSHOT] + } + proc snapshot_free {SNAPSHOT} { + uplevel [list sqlite3_snapshot_free $SNAPSHOT] + } + proc snapshot_cmp {SNAPSHOT1 SNAPSHOT2} { + uplevel [list sqlite3_snapshot_cmp $SNAPSHOT1 $SNAPSHOT2] + } } - list [catch { sqlite3_snapshot_get db main } msg] $msg -} {1 SQLITE_ERROR} -do_execsql_test 1.3.2 COMMIT -#------------------------------------------------------------------------- -# Check that a simple case works. Reuse the database created by the -# block of tests above. -# -do_execsql_test 2.1.0 { - BEGIN; - SELECT * FROM t1; -} {1 2 3 4 5 6 7 8} - -do_test 2.1.1 { - set snapshot [sqlite3_snapshot_get db main] - execsql { - COMMIT; - INSERT INTO t1 VALUES(9, 10); - SELECT * FROM t1; + 2 { + proc snapshot_get {DB DBNAME} { + uplevel [list sqlite3_snapshot_get_blob $DB $DBNAME] + } + proc snapshot_open {DB DBNAME SNAPSHOT} { + uplevel [list sqlite3_snapshot_open_blob $DB $DBNAME $SNAPSHOT] + } + proc snapshot_free {SNAPSHOT} { + } + proc snapshot_cmp {SNAPSHOT1 SNAPSHOT2} { + uplevel [list sqlite3_snapshot_cmp_blob $SNAPSHOT1 $SNAPSHOT2] + } } -} {1 2 3 4 5 6 7 8 9 10} +} { -do_test 2.1.2 { - execsql BEGIN - sqlite3_snapshot_open db main $snapshot - execsql { - SELECT * FROM t1; + reset_db + eval $tcl + + #------------------------------------------------------------------------- + # Check some error conditions in snapshot_get(). It is an error if: + # + # 1) snapshot_get() is called on a non-WAL database, or + # 2) there is an open write transaction on the database. + # + do_execsql_test $tn.1.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); } -} {1 2 3 4 5 6 7 8} -do_test 2.1.3 { - sqlite3_snapshot_free $snapshot - execsql COMMIT -} {} + do_test $tn.1.1.1 { + execsql { BEGIN; SELECT * FROM t1; } + list [catch { snapshot_get db main } msg] $msg + } {1 SQLITE_ERROR} + do_execsql_test 1.1.2 COMMIT -do_test 2.2.0 { - sqlite3 db2 test.db - execsql { + do_test $tn.1.2.1 { + execsql { + PRAGMA journal_mode = WAL; + BEGIN; + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t1 VALUES(7, 8); + } + list [catch { snapshot_get db main } msg] $msg + } {1 SQLITE_ERROR} + do_execsql_test $tn.1.3.2 COMMIT + + #------------------------------------------------------------------------- + # Check that a simple case works. Reuse the database created by the + # block of tests above. + # + do_execsql_test $tn.2.1.0 { BEGIN; SELECT * FROM t1; - } db2 -} {1 2 3 4 5 6 7 8 9 10} + } {1 2 3 4 5 6 7 8} -do_test 2.2.1 { - set snapshot [sqlite3_snapshot_get db2 main] - execsql { - INSERT INTO t1 VALUES(11, 12); - SELECT * FROM t1; - } -} {1 2 3 4 5 6 7 8 9 10 11 12} + do_test $tn.2.1.1 { + set snapshot [snapshot_get db main] + execsql { + COMMIT; + INSERT INTO t1 VALUES(9, 10); + SELECT * FROM t1; + } + } {1 2 3 4 5 6 7 8 9 10} -do_test 2.2.2 { - execsql BEGIN - sqlite3_snapshot_open db main $snapshot - execsql { - SELECT * FROM t1; - } -} {1 2 3 4 5 6 7 8 9 10} + do_test $tn.2.1.2 { + execsql BEGIN + snapshot_open db main $snapshot + execsql { + SELECT * FROM t1; + } + } {1 2 3 4 5 6 7 8} -do_test 2.2.3 { - sqlite3_snapshot_free $snapshot - execsql COMMIT - execsql COMMIT db2 - db2 close -} {} + do_test $tn.2.1.3 { + snapshot_free $snapshot + execsql COMMIT + } {} -do_test 2.3.1 { - execsql { DELETE FROM t1 WHERE a>6 } - set snapshot [sqlite3_snapshot_get db main] - execsql { - INSERT INTO t1 VALUES('a', 'b'); - INSERT INTO t1 VALUES('c', 'd'); - SELECT * FROM t1; - } -} {1 2 3 4 5 6 a b c d} -do_test 2.3.2 { - execsql BEGIN - sqlite3_snapshot_open db main $snapshot - execsql { SELECT * FROM t1 } -} {1 2 3 4 5 6} + do_test $tn.2.2.0 { + sqlite3 db2 test.db + execsql { + BEGIN; + SELECT * FROM t1; + } db2 + } {1 2 3 4 5 6 7 8 9 10} -do_test 2.3.3 { - catchsql { - INSERT INTO t1 VALUES('x','y') - } -} {1 {database is locked}} -do_test 2.3.4 { - execsql COMMIT - sqlite3_snapshot_free $snapshot -} {} + do_test $tn.2.2.1 { + set snapshot [snapshot_get db2 main] + execsql { + INSERT INTO t1 VALUES(11, 12); + SELECT * FROM t1; + } + } {1 2 3 4 5 6 7 8 9 10 11 12} -#------------------------------------------------------------------------- -# Check some errors in sqlite3_snapshot_open(). It is an error if: -# -# 1) the db is in auto-commit mode, -# 2) the db has an open (read or write) transaction, -# 3) the db is not a wal database, -# -# Reuse the database created by earlier tests. -# -do_execsql_test 3.0.0 { - CREATE TABLE t2(x, y); - INSERT INTO t2 VALUES('a', 'b'); - INSERT INTO t2 VALUES('c', 'd'); - BEGIN; - SELECT * FROM t2; -} {a b c d} -do_test 3.0.1 { - set snapshot [sqlite3_snapshot_get db main] - execsql { COMMIT } - execsql { INSERT INTO t2 VALUES('e', 'f'); } -} {} + do_test $tn.2.2.2 { + execsql BEGIN + snapshot_open db main $snapshot + execsql { + SELECT * FROM t1; + } + } {1 2 3 4 5 6 7 8 9 10} -do_test 3.1 { - list [catch {sqlite3_snapshot_open db main $snapshot } msg] $msg -} {1 SQLITE_ERROR} + do_test $tn.2.2.3 { + snapshot_free $snapshot + execsql COMMIT + execsql COMMIT db2 + db2 close + } {} -do_test 3.2.1 { - execsql { + do_test $tn.2.3.1 { + execsql { DELETE FROM t1 WHERE a>6 } + set snapshot [snapshot_get db main] + execsql { + INSERT INTO t1 VALUES('a', 'b'); + INSERT INTO t1 VALUES('c', 'd'); + SELECT * FROM t1; + } + } {1 2 3 4 5 6 a b c d} + do_test $tn.2.3.2 { + execsql BEGIN + snapshot_open db main $snapshot + execsql { SELECT * FROM t1 } + } {1 2 3 4 5 6} + + do_test $tn.2.3.3 { + catchsql { + INSERT INTO t1 VALUES('x','y') + } + } {1 {database is locked}} + do_test $tn.2.3.4 { + execsql COMMIT + snapshot_free $snapshot + } {} + + #------------------------------------------------------------------------- + # Check some errors in snapshot_open(). It is an error if: + # + # 1) the db is in auto-commit mode, + # 2) the db has an open (read or write) transaction, + # 3) the db is not a wal database, + # + # Reuse the database created by earlier tests. + # + do_execsql_test $tn.3.0.0 { + CREATE TABLE t2(x, y); + INSERT INTO t2 VALUES('a', 'b'); + INSERT INTO t2 VALUES('c', 'd'); BEGIN; SELECT * FROM t2; - } -} {a b c d e f} -do_test 3.2.2 { - list [catch {sqlite3_snapshot_open db main $snapshot } msg] $msg -} {1 SQLITE_ERROR} + } {a b c d} + do_test $tn.3.0.1 { + set snapshot [snapshot_get db main] + execsql { COMMIT } + execsql { INSERT INTO t2 VALUES('e', 'f'); } + } {} -do_test 3.2.3 { - execsql { - COMMIT; - BEGIN; - INSERT INTO t2 VALUES('g', 'h'); - } - list [catch {sqlite3_snapshot_open db main $snapshot } msg] $msg -} {1 SQLITE_ERROR} -do_execsql_test 3.2.4 COMMIT + do_test $tn.3.1 { + list [catch {snapshot_open db main $snapshot } msg] $msg + } {1 SQLITE_ERROR} -do_test 3.3.1 { - execsql { PRAGMA journal_mode = DELETE } - execsql { BEGIN } - list [catch {sqlite3_snapshot_open db main $snapshot } msg] $msg -} {1 SQLITE_ERROR} + do_test $tn.3.2.1 { + execsql { + BEGIN; + SELECT * FROM t2; + } + } {a b c d e f} + do_test $tn.3.2.2 { + list [catch {snapshot_open db main $snapshot } msg] $msg + } {1 SQLITE_ERROR} -do_test 3.3.2 { - sqlite3_snapshot_free $snapshot - execsql COMMIT -} {} + do_test $tn.3.2.3 { + execsql { + COMMIT; + BEGIN; + INSERT INTO t2 VALUES('g', 'h'); + } + list [catch {snapshot_open db main $snapshot } msg] $msg + } {1 SQLITE_ERROR} + do_execsql_test 3.2.4 COMMIT -#------------------------------------------------------------------------- -# Check that SQLITE_BUSY_SNAPSHOT is returned if the specified snapshot -# no longer exists because the wal file has been checkpointed. -# -# 1. Reading a snapshot from the middle of a wal file is not possible -# after the wal file has been checkpointed. -# -# 2. That a snapshot from the end of a wal file can not be read once -# the wal file has been wrapped. -# -do_execsql_test 4.1.0 { - PRAGMA journal_mode = wal; - CREATE TABLE t3(i, j); - INSERT INTO t3 VALUES('o', 't'); - INSERT INTO t3 VALUES('t', 'f'); - BEGIN; - SELECT * FROM t3; -} {wal o t t f} + do_test $tn.3.3.1 { + execsql { PRAGMA journal_mode = DELETE } + execsql { BEGIN } + list [catch {snapshot_open db main $snapshot } msg] $msg + } {1 SQLITE_ERROR} -do_test 4.1.1 { - set snapshot [sqlite3_snapshot_get db main] - execsql COMMIT -} {} -do_test 4.1.2 { - execsql { - INSERT INTO t3 VALUES('f', 's'); - BEGIN; - } - sqlite3_snapshot_open db main $snapshot - execsql { SELECT * FROM t3 } -} {o t t f} + do_test $tn.$tn.3.3.2 { + snapshot_free $snapshot + execsql COMMIT + } {} -do_test 4.1.3 { - execsql { - COMMIT; - PRAGMA wal_checkpoint; - BEGIN; - } - list [catch {sqlite3_snapshot_open db main $snapshot} msg] $msg -} {1 SQLITE_BUSY_SNAPSHOT} -do_test 4.1.4 { - sqlite3_snapshot_free $snapshot - execsql COMMIT -} {} - -do_test 4.2.1 { - execsql { - INSERT INTO t3 VALUES('s', 'e'); - INSERT INTO t3 VALUES('n', 't'); + #------------------------------------------------------------------------- + # Check that SQLITE_BUSY_SNAPSHOT is returned if the specified snapshot + # no longer exists because the wal file has been checkpointed. + # + # 1. Reading a snapshot from the middle of a wal file is not possible + # after the wal file has been checkpointed. + # + # 2. That a snapshot from the end of a wal file can not be read once + # the wal file has been wrapped. + # + do_execsql_test $tn.4.1.0 { + PRAGMA journal_mode = wal; + CREATE TABLE t3(i, j); + INSERT INTO t3 VALUES('o', 't'); + INSERT INTO t3 VALUES('t', 'f'); BEGIN; SELECT * FROM t3; - } -} {o t t f f s s e n t} -do_test 4.2.2 { - set snapshot [sqlite3_snapshot_get db main] - execsql { - COMMIT; - PRAGMA wal_checkpoint; + } {wal o t t f} + + do_test $tn.4.1.1 { + set snapshot [snapshot_get db main] + execsql COMMIT + } {} + do_test $tn.4.1.2 { + execsql { + INSERT INTO t3 VALUES('f', 's'); + BEGIN; + } + snapshot_open db main $snapshot + execsql { SELECT * FROM t3 } + } {o t t f} + + do_test $tn.4.1.3 { + execsql { + COMMIT; + PRAGMA wal_checkpoint; + BEGIN; + } + list [catch {snapshot_open db main $snapshot} msg] $msg + } {1 SQLITE_BUSY_SNAPSHOT} + do_test $tn.4.1.4 { + snapshot_free $snapshot + execsql COMMIT + } {} + + do_test $tn.4.2.1 { + execsql { + INSERT INTO t3 VALUES('s', 'e'); + INSERT INTO t3 VALUES('n', 't'); + BEGIN; + SELECT * FROM t3; + } + } {o t t f f s s e n t} + do_test $tn.4.2.2 { + set snapshot [snapshot_get db main] + execsql { + COMMIT; + PRAGMA wal_checkpoint; + BEGIN; + } + snapshot_open db main $snapshot + execsql { SELECT * FROM t3 } + } {o t t f f s s e n t} + do_test $tn.4.2.3 { + execsql { + COMMIT; + INSERT INTO t3 VALUES('e', 't'); + BEGIN; + } + list [catch {snapshot_open db main $snapshot} msg] $msg + } {1 SQLITE_BUSY_SNAPSHOT} + do_test $tn.4.2.4 { + snapshot_free $snapshot + } {} + + #------------------------------------------------------------------------- + # Check that SQLITE_BUSY is returned if a checkpoint is running when + # sqlite3_snapshot_open() is called. + # + reset_db + db close + testvfs tvfs + sqlite3 db test.db -vfs tvfs + + do_execsql_test $tn.5.1 { + PRAGMA journal_mode = wal; + CREATE TABLE x1(x, xx, xxx); + INSERT INTO x1 VALUES('z', 'zz', 'zzz'); BEGIN; + SELECT * FROM x1; + } {wal z zz zzz} + + do_test $tn.5.2 { + set ::snapshot [snapshot_get db main] + sqlite3 db2 test.db -vfs tvfs + execsql { + INSERT INTO x1 VALUES('a', 'aa', 'aaa'); + COMMIT; + } + } {} + + set t53 0 + proc write_callback {args} { + do_test $tn.5.3.[incr ::t53] { + execsql BEGIN + list [catch { snapshot_open db main $::snapshot } msg] $msg + } {1 SQLITE_BUSY} + catchsql COMMIT } - sqlite3_snapshot_open db main $snapshot - execsql { SELECT * FROM t3 } -} {o t t f f s s e n t} -do_test 4.2.3 { - execsql { - COMMIT; - INSERT INTO t3 VALUES('e', 't'); - BEGIN; - } - list [catch {sqlite3_snapshot_open db main $snapshot} msg] $msg -} {1 SQLITE_BUSY_SNAPSHOT} -do_test 4.2.4 { - sqlite3_snapshot_free $snapshot -} {} -#------------------------------------------------------------------------- -# Check that SQLITE_BUSY is returned if a checkpoint is running when -# sqlite3_snapshot_open() is called. -# -reset_db -db close -testvfs tvfs -sqlite3 db test.db -vfs tvfs - -do_execsql_test 5.1 { - PRAGMA journal_mode = wal; - CREATE TABLE x1(x, xx, xxx); - INSERT INTO x1 VALUES('z', 'zz', 'zzz'); - BEGIN; - SELECT * FROM x1; -} {wal z zz zzz} - -do_test 5.2 { - set ::snapshot [sqlite3_snapshot_get db main] - sqlite3 db2 test.db -vfs tvfs - execsql { - INSERT INTO x1 VALUES('a', 'aa', 'aaa'); - COMMIT; - } -} {} - -set t53 0 -proc write_callback {args} { - do_test 5.3.[incr ::t53] { - execsql BEGIN - list [catch { sqlite3_snapshot_open db main $::snapshot } msg] $msg - } {1 SQLITE_BUSY} - catchsql COMMIT -} - -tvfs filter xWrite -tvfs script write_callback -db2 eval { PRAGMA wal_checkpoint } -db close -db2 close -tvfs delete -sqlite3_snapshot_free $snapshot - -#------------------------------------------------------------------------- -# Test that sqlite3_snapshot_get() may be called immediately after -# "BEGIN; PRAGMA user_version;". And that sqlite3_snapshot_open() may -# be called after opening the db handle and running the script -# "PRAGMA user_version; BEGIN". -reset_db -do_execsql_test 6.1 { - PRAGMA journal_mode = wal; - CREATE TABLE x1(x, xx, xxx); - INSERT INTO x1 VALUES('z', 'zz', 'zzz'); - BEGIN; - PRAGMA user_version; -} {wal 0} -do_test 6.2 { - set ::snapshot [sqlite3_snapshot_get db main] - execsql { - INSERT INTO x1 VALUES('a', 'aa', 'aaa'); - COMMIT; - } -} {} -do_test 6.3 { - sqlite3 db2 test.db - db2 eval "PRAGMA user_version ; BEGIN" - sqlite3_snapshot_open db2 main $::snapshot - db2 eval { SELECT * FROM x1 } -} {z zz zzz} -do_test 6.4 { + tvfs filter xWrite + tvfs script write_callback + db2 eval { PRAGMA wal_checkpoint } + db close db2 close - sqlite3 db2 test.db - db2 eval "PRAGMA application_id" - db2 eval "BEGIN" - sqlite3_snapshot_open db2 main $::snapshot - db2 eval { SELECT * FROM x1 } -} {z zz zzz} + tvfs delete + snapshot_free $snapshot -do_test 6.5 { - db2 close - sqlite3 db2 test.db - db2 eval "BEGIN" - list [catch {sqlite3_snapshot_open db2 main $::snapshot} msg] $msg -} {1 SQLITE_ERROR} + #------------------------------------------------------------------------- + # Test that sqlite3_snapshot_get() may be called immediately after + # "BEGIN; PRAGMA user_version;". And that sqlite3_snapshot_open() may + # be called after opening the db handle and running the script + # "PRAGMA user_version; BEGIN". + reset_db + do_execsql_test $tn.6.1 { + PRAGMA journal_mode = wal; + CREATE TABLE x1(x, xx, xxx); + INSERT INTO x1 VALUES('z', 'zz', 'zzz'); + BEGIN; + PRAGMA user_version; + } {wal 0} + do_test $tn.6.2 { + set ::snapshot [snapshot_get db main] + execsql { + INSERT INTO x1 VALUES('a', 'aa', 'aaa'); + COMMIT; + } + } {} + do_test $tn.6.3 { + sqlite3 db2 test.db + db2 eval "PRAGMA user_version ; BEGIN" + snapshot_open db2 main $::snapshot + db2 eval { SELECT * FROM x1 } + } {z zz zzz} + do_test $tn.6.4 { + db2 close + sqlite3 db2 test.db + db2 eval "PRAGMA application_id" + db2 eval "BEGIN" + snapshot_open db2 main $::snapshot + db2 eval { SELECT * FROM x1 } + } {z zz zzz} -sqlite3_snapshot_free $snapshot + do_test $tn.6.5 { + db2 close + sqlite3 db2 test.db + db2 eval "BEGIN" + list [catch {snapshot_open db2 main $::snapshot} msg] $msg + } {1 SQLITE_ERROR} -#------------------------------------------------------------------------- -# The following tests investigate the sqlite3_snapshot_cmp() API. -# + snapshot_free $snapshot -# Compare snapshots $p1 and $p2, checking that the result is $r. -# -proc do_snapshot_cmp_test {tn p1 p2 r} { - uplevel [list do_test $tn.1 [list sqlite3_snapshot_cmp $p1 $p2] $r] - uplevel [list do_test $tn.2 [list sqlite3_snapshot_cmp $p2 $p1] [expr $r*-1]] - uplevel [list do_test $tn.3 [list sqlite3_snapshot_cmp $p1 $p1] 0] - uplevel [list do_test $tn.4 [list sqlite3_snapshot_cmp $p2 $p2] 0] + #------------------------------------------------------------------------- + # The following tests investigate the sqlite3_snapshot_cmp() API. + # + + # Compare snapshots $p1 and $p2, checking that the result is $r. + # + proc do_snapshot_cmp_test {tn p1 p2 r} { + uplevel [list do_test $tn.1 [list snapshot_cmp $p1 $p2] $r] + uplevel [list do_test $tn.2 [list snapshot_cmp $p2 $p1] [expr $r*-1]] + uplevel [list do_test $tn.3 [list snapshot_cmp $p1 $p1] 0] + uplevel [list do_test $tn.4 [list snapshot_cmp $p2 $p2] 0] + } + + catch { db2 close } + reset_db + + do_execsql_test $tn.7.1 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x); + } wal + + do_test $tn.7.1.2 { + execsql { BEGIN ; PRAGMA application_id } + set p1 [snapshot_get db main] + execsql { + INSERT INTO t1 VALUES(10); + COMMIT; + } + execsql { BEGIN ; PRAGMA application_id } + set p2 [snapshot_get db main] + execsql COMMIT + } {} + + do_snapshot_cmp_test $tn.7.1.3 $p1 $p2 -1 + snapshot_free $p1 + snapshot_free $p2 + + do_execsql_test $tn.7.2.1 { + INSERT INTO t1 VALUES(11); + INSERT INTO t1 VALUES(12); + INSERT INTO t1 VALUES(13); + BEGIN; + PRAGMA application_id; + } {0} + do_test $tn.7.2.2 { + set p1 [snapshot_get db main] + execsql { + COMMIT; + INSERT INTO t1 VALUES(14); + PRAGMA wal_checkpoint; + BEGIN; + PRAGMA application_id; + } + set p2 [snapshot_get db main] + execsql COMMIT + } {} + + do_snapshot_cmp_test $tn.7.2.3 $p1 $p2 -1 + snapshot_free $p2 + + do_test $tn.7.3.1 { + execsql { + INSERT INTO t1 VALUES(14); + BEGIN; + PRAGMA application_id; + } + set p2 [snapshot_get db main] + execsql COMMIT + } {} + + do_snapshot_cmp_test $tn.7.3.2 $p1 $p2 -1 + snapshot_free $p1 + snapshot_free $p2 } -catch { db2 close } -reset_db - -do_execsql_test 7.1 { - PRAGMA journal_mode = wal; - CREATE TABLE t1(x); -} wal - -do_test 7.1.2 { - execsql { BEGIN ; PRAGMA application_id } - set p1 [sqlite3_snapshot_get db main] - execsql { - INSERT INTO t1 VALUES(10); - COMMIT; - } - execsql { BEGIN ; PRAGMA application_id } - set p2 [sqlite3_snapshot_get db main] - execsql COMMIT -} {} - -do_snapshot_cmp_test 7.1.3 $p1 $p2 -1 -sqlite3_snapshot_free $p1 -sqlite3_snapshot_free $p2 - -do_execsql_test 7.2.1 { - INSERT INTO t1 VALUES(11); - INSERT INTO t1 VALUES(12); - INSERT INTO t1 VALUES(13); - BEGIN; - PRAGMA application_id; -} {0} -do_test 7.2.2 { - set p1 [sqlite3_snapshot_get db main] - execsql { - COMMIT; - INSERT INTO t1 VALUES(14); - PRAGMA wal_checkpoint; - BEGIN; - PRAGMA application_id; - } - set p2 [sqlite3_snapshot_get db main] - execsql COMMIT -} {} - -do_snapshot_cmp_test 7.2.3 $p1 $p2 -1 -sqlite3_snapshot_free $p2 - -do_test 7.3.1 { - execsql { - INSERT INTO t1 VALUES(14); - BEGIN; - PRAGMA application_id; - } - set p2 [sqlite3_snapshot_get db main] - execsql COMMIT -} {} - -do_snapshot_cmp_test 7.3.2 $p1 $p2 -1 -sqlite3_snapshot_free $p1 -sqlite3_snapshot_free $p2 - finish_test From 93006cdd853eb84752c0803928afdd25ce4cdb53 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 18 Nov 2016 18:22:05 +0000 Subject: [PATCH 03/11] Add tests for snapshot interfaces. FossilOrigin-Name: 1f7ee7af7b620262ae663d65889b6a87415d4a34 --- manifest | 11 +++--- manifest.uuid | 2 +- test/snapshot2.test | 82 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 6 deletions(-) create mode 100644 test/snapshot2.test diff --git a/manifest b/manifest index c5597fb646..066a12a0ba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sexisting\ssnapshot\stests\sto\sserialize/deserialize\ssnapshots.\sNo\snew\ntests. -D 2016-11-18T14:38:41.852 +C Add\stests\sfor\ssnapshot\sinterfaces. +D 2016-11-18T18:22:05.618 F Makefile.in 6b572807415d3f0a379cebc9461416d8df4a12c8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc bb4d970894abbbe0e88d00aac29bd52af8bc95f4 @@ -1103,6 +1103,7 @@ F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 F test/skipscan6.test 5866039d03a56f5bd0b3d172a012074a1d90a15b F test/snapshot.test bb34cc38828bf2e86a49207995fa7dfacd303832 +F test/snapshot2.test 30bd95f6fefa8be7f29421e27745cac90b66c74d F test/snapshot_fault.test 062ff0438a074978d45e9f9a92e7ad459b74ee73 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 843cd84db9891b2d01b9ab64cef3e9020f98d087 @@ -1534,7 +1535,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b6a81fa1fc1fb78a65894129851a4ec3986640de -R 17ac6d49deba8d40db4fa5f6ce7fc2ca +P 16b9bf92741e4c62874cffd7c6a61763c5054c7a +R aa1ba4f87ce4a1d41c6deb879752303b U dan -Z cee5f11eb9093786088be3369dcd2153 +Z ee11b90501c467e7c32bf0deb14250e6 diff --git a/manifest.uuid b/manifest.uuid index 6af23f5732..891d62480c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -16b9bf92741e4c62874cffd7c6a61763c5054c7a \ No newline at end of file +1f7ee7af7b620262ae663d65889b6a87415d4a34 \ No newline at end of file diff --git a/test/snapshot2.test b/test/snapshot2.test new file mode 100644 index 0000000000..aa5d385893 --- /dev/null +++ b/test/snapshot2.test @@ -0,0 +1,82 @@ +# 2016 November 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The focus +# of this file is the sqlite3_snapshot_xxx() APIs. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +ifcapable !snapshot {finish_test; return} +set testprefix snapshot2 + +# This test does not work with the inmemory_journal permutation. The reason +# is that each connection opened as part of this permutation executes +# "PRAGMA journal_mode=memory", which fails if the database is in wal mode +# and there are one or more existing connections. +if {[permutation]=="inmemory_journal"} { + finish_test + return +} + +#------------------------------------------------------------------------- +# Check that it is not possible to obtain a snapshot immediately after +# a wal mode database with an empty wal file is opened. But it is after +# the file has been written, even by some other connection. +# +do_execsql_test 1.0 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); +} {wal} + +db close +do_test 1.1.1 { list [file exists test.db] [file exists test.db-wal] } {1 0} + +sqlite3 db test.db +do_execsql_test 1.1.2 { SELECT * FROM t1 } {1 2 3 4 5 6} + +do_test 1.1.3 { + execsql BEGIN + list [catch { sqlite3_snapshot_get_blob db main } msg] $msg +} {1 SQLITE_ERROR} +execsql COMMIT + +do_test 1.1.4 { + execsql { INSERT INTO t1 VALUES(7, 8, 9) } + execsql BEGIN + string length [sqlite3_snapshot_get_blob db main] +} 48 +execsql COMMIT + +db close +do_test 1.2.1 { list [file exists test.db] [file exists test.db-wal] } {1 0} + +sqlite3 db test.db +do_execsql_test 1.2.2 { SELECT * FROM t1 } {1 2 3 4 5 6 7 8 9} + +do_test 1.2.3 { + execsql BEGIN + list [catch { sqlite3_snapshot_get_blob db main } msg] $msg +} {1 SQLITE_ERROR} +execsql COMMIT + +do_test 1.2.4 { + sqlite3 db2 test.db + execsql { INSERT INTO t1 VALUES(10, 11, 12) } db2 + execsql BEGIN + string length [sqlite3_snapshot_get_blob db main] +} 48 +execsql COMMIT +db2 close + + +finish_test From edace5d4f16897f813564259afe5200acd03ce0d Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 18 Nov 2016 18:43:39 +0000 Subject: [PATCH 04/11] Require that the database handle be in autocommit mode for sqlite3_snapshot_get() to succeed. This is because it may open a read transaction on the database file. FossilOrigin-Name: 83b658dad091211ade3594d1e8d00ce525882506 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/main.c | 17 +++++++++-------- src/sqlite.h.in | 29 ++++++++++++++++++++++++++--- test/snapshot.test | 17 +++++++++++++---- 5 files changed, 57 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 066a12a0ba..955a7b77e9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\ssnapshot\sinterfaces. -D 2016-11-18T18:22:05.618 +C Require\sthat\sthe\sdatabase\shandle\sbe\sin\sautocommit\smode\sfor\nsqlite3_snapshot_get()\sto\ssucceed.\sThis\sis\sbecause\sit\smay\sopen\sa\sread\ntransaction\son\sthe\sdatabase\sfile. +D 2016-11-18T18:43:39.100 F Makefile.in 6b572807415d3f0a379cebc9461416d8df4a12c8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc bb4d970894abbbe0e88d00aac29bd52af8bc95f4 @@ -352,7 +352,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 0d6e59f9eea62db772eabc0f07901ec26fe01968 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e F src/loadext.c 5d6642d141c07d366e43d359e94ec9de47add41d -F src/main.c 694ac90557abdaa62151a6090670e107b0f2c2ab +F src/main.c 602d7179fda1879d688174dcd74c69c8579045b5 F src/malloc.c 5ee7c2d3dcb1b0a902c9c6d0115deef54736bdfa F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b @@ -389,7 +389,7 @@ F src/resolve.c bb070cf5f23611c44ab7e4788803684e385fc3fb F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 672b1af237ad257149fc5189f3277dcbca036eeb F src/shell.c f04e4af75c5517735397d060ed0b4a874104bb41 -F src/sqlite.h.in dfe8c1b4ac66635495ec9a6a7205fcb0f27f4451 +F src/sqlite.h.in cc6e3f38d4c1e4df4f569af49c5deb7c32f1ea10 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae F src/sqliteInt.h c471d791b10c0f2164c8b7a87adc338e703c09cc @@ -1102,7 +1102,7 @@ F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 F test/skipscan6.test 5866039d03a56f5bd0b3d172a012074a1d90a15b -F test/snapshot.test bb34cc38828bf2e86a49207995fa7dfacd303832 +F test/snapshot.test 85735bd997a4f6d710140c28fd860519a299649f F test/snapshot2.test 30bd95f6fefa8be7f29421e27745cac90b66c74d F test/snapshot_fault.test 062ff0438a074978d45e9f9a92e7ad459b74ee73 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f @@ -1535,7 +1535,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 16b9bf92741e4c62874cffd7c6a61763c5054c7a -R aa1ba4f87ce4a1d41c6deb879752303b +P 1f7ee7af7b620262ae663d65889b6a87415d4a34 +R 94e3e6505e79390151b9cd467eca8236 U dan -Z ee11b90501c467e7c32bf0deb14250e6 +Z 084fdf6a46a9563f3a18164c0feb0d49 diff --git a/manifest.uuid b/manifest.uuid index 891d62480c..0a12e01e25 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1f7ee7af7b620262ae663d65889b6a87415d4a34 \ No newline at end of file +83b658dad091211ade3594d1e8d00ce525882506 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 666a72547b..29e166447e 100644 --- a/src/main.c +++ b/src/main.c @@ -3981,7 +3981,6 @@ int sqlite3_snapshot_get( ){ int rc = SQLITE_ERROR; #ifndef SQLITE_OMIT_WAL - int iDb; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ){ @@ -3990,13 +3989,15 @@ int sqlite3_snapshot_get( #endif sqlite3_mutex_enter(db->mutex); - iDb = sqlite3FindDbName(db, zDb); - if( iDb==0 || iDb>1 ){ - Btree *pBt = db->aDb[iDb].pBt; - if( 0==sqlite3BtreeIsInTrans(pBt) ){ - rc = sqlite3BtreeBeginTrans(pBt, 0); - if( rc==SQLITE_OK ){ - rc = sqlite3PagerSnapshotGet(sqlite3BtreePager(pBt), ppSnapshot); + if( db->autoCommit==0 ){ + int iDb = sqlite3FindDbName(db, zDb); + if( iDb==0 || iDb>1 ){ + Btree *pBt = db->aDb[iDb].pBt; + if( 0==sqlite3BtreeIsInTrans(pBt) ){ + rc = sqlite3BtreeBeginTrans(pBt, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSnapshotGet(sqlite3BtreePager(pBt), ppSnapshot); + } } } } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 7987cfae5e..8f3b40292d 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -8296,9 +8296,32 @@ typedef struct sqlite3_snapshot { ** schema S in database connection D. ^On success, the ** [sqlite3_snapshot_get(D,S,P)] interface writes a pointer to the newly ** created [sqlite3_snapshot] object into *P and returns SQLITE_OK. -** ^If schema S of [database connection] D is not a [WAL mode] database -** that is in a read transaction, then [sqlite3_snapshot_get(D,S,P)] -** leaves the *P value unchanged and returns an appropriate [error code]. +** If there is not already a read-transaction open on schema S when +** this function is called, one is opened automatically. +** +** The following must be true for this function to succeed. If any of +** the following statements are false when sqlite3_snapshot_get() is +** called, SQLITE_ERROR is returned. The final value of *P is undefined +** in this case. +** +**
    +**
  • The database handle must be in [autocommit mode]. +** +**
  • Schema S of [database connection] D must be a [WAL mode] database. +** +**
  • There must not be a write transaction open on schema S of database +** connection D. +** +**
  • One or more transactions must have been written to the current wal +** file since it was created on disk (by any connection). This means +** that a snapshot cannot be taken on a wal mode database with no wal +** file immediately after it is first opened. At least one transaction +** must be written to it first. +**
+** +** This function may also return SQLITE_NOMEM. If it is called with the +** database handle in autocommit mode but fails for some other reason, +** whether or not a read transaction is opened on schema S is undefined. ** ** The [sqlite3_snapshot] object returned from a successful call to ** [sqlite3_snapshot_get()] must be freed using [sqlite3_snapshot_free()] diff --git a/test/snapshot.test b/test/snapshot.test index 1143c14962..99d3ed47c3 100644 --- a/test/snapshot.test +++ b/test/snapshot.test @@ -65,6 +65,7 @@ foreach {tn tcl} { # # 1) snapshot_get() is called on a non-WAL database, or # 2) there is an open write transaction on the database. + # 3) the database handle is in auto-commit mode # do_execsql_test $tn.1.0 { CREATE TABLE t1(a, b); @@ -76,7 +77,7 @@ foreach {tn tcl} { execsql { BEGIN; SELECT * FROM t1; } list [catch { snapshot_get db main } msg] $msg } {1 SQLITE_ERROR} - do_execsql_test 1.1.2 COMMIT + do_execsql_test $tn.1.1.2 COMMIT do_test $tn.1.2.1 { execsql { @@ -87,7 +88,15 @@ foreach {tn tcl} { } list [catch { snapshot_get db main } msg] $msg } {1 SQLITE_ERROR} - do_execsql_test $tn.1.3.2 COMMIT + do_execsql_test $tn.1.2.2 COMMIT + + do_test $tn.1.3.1 { + list [catch { snapshot_get db main } msg] $msg + } {1 SQLITE_ERROR} + do_test $tn.1.3.2 { + db trans { set snap [snapshot_get db main] } + snapshot_free $snap + } {} #------------------------------------------------------------------------- # Check that a simple case works. Reuse the database created by the @@ -153,7 +162,7 @@ foreach {tn tcl} { do_test $tn.2.3.1 { execsql { DELETE FROM t1 WHERE a>6 } - set snapshot [snapshot_get db main] + db trans { set snapshot [snapshot_get db main] } execsql { INSERT INTO t1 VALUES('a', 'b'); INSERT INTO t1 VALUES('c', 'd'); @@ -220,7 +229,7 @@ foreach {tn tcl} { } list [catch {snapshot_open db main $snapshot } msg] $msg } {1 SQLITE_ERROR} - do_execsql_test 3.2.4 COMMIT + do_execsql_test $tn.3.2.4 COMMIT do_test $tn.3.3.1 { execsql { PRAGMA journal_mode = DELETE } From 1158498dce3f73e3666c06f7067f69f06fe14fb4 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 18 Nov 2016 20:49:43 +0000 Subject: [PATCH 05/11] Add experimental sqlite3_snapshot_recover() API. FossilOrigin-Name: 174a6076a8d7bebe5efebf55f3fdc5d87c589cc7 --- manifest | 26 +++++++++--------- manifest.uuid | 2 +- src/main.c | 28 +++++++++++++++++++ src/pager.c | 14 ++++++++++ src/pager.h | 1 + src/sqlite.h.in | 6 +++++ src/test1.c | 33 +++++++++++++++++++++++ src/wal.c | 65 ++++++++++++++++++++++++++++++++++++++++++++- src/wal.h | 1 + test/snapshot2.test | 60 +++++++++++++++++++++++++++++++++++++++++ 10 files changed, 221 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 955a7b77e9..8ffa3b7ee3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Require\sthat\sthe\sdatabase\shandle\sbe\sin\sautocommit\smode\sfor\nsqlite3_snapshot_get()\sto\ssucceed.\sThis\sis\sbecause\sit\smay\sopen\sa\sread\ntransaction\son\sthe\sdatabase\sfile. -D 2016-11-18T18:43:39.100 +C Add\sexperimental\ssqlite3_snapshot_recover()\sAPI. +D 2016-11-18T20:49:43.736 F Makefile.in 6b572807415d3f0a379cebc9461416d8df4a12c8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc bb4d970894abbbe0e88d00aac29bd52af8bc95f4 @@ -352,7 +352,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 0d6e59f9eea62db772eabc0f07901ec26fe01968 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e F src/loadext.c 5d6642d141c07d366e43d359e94ec9de47add41d -F src/main.c 602d7179fda1879d688174dcd74c69c8579045b5 +F src/main.c de55e68145758d5512e9397728c9a8f4ff0ffa78 F src/malloc.c 5ee7c2d3dcb1b0a902c9c6d0115deef54736bdfa F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b @@ -374,8 +374,8 @@ F src/os_setup.h 0dbaea40a7d36bf311613d31342e0b99e2536586 F src/os_unix.c be9ca0f901a2b6c1bc93dc338f4863675180c189 F src/os_win.c cf90abd4e50d9f56d2c20ce8e005aff55d7bd8e9 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a -F src/pager.c a31e2c25563065ebfc9308f2ba3a061901fd60a8 -F src/pager.h 07d6938df0b74e4abe8f57807a8b0e1084321d8b +F src/pager.c 4e4aea7ced5734753ccbff4cf4bb4d032cf2173e +F src/pager.h d1e944291030351f362a0a7da9b5c3e34e603e39 F src/parse.y 0338f906b61e311c2b7e11a3f89b0092c780b664 F src/pcache.c 5ff2a08f76a9c1b22f43eb063b7068fb085465ac F src/pcache.h 2cedcd8407eb23017d92790b112186886e179490 @@ -389,7 +389,7 @@ F src/resolve.c bb070cf5f23611c44ab7e4788803684e385fc3fb F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 672b1af237ad257149fc5189f3277dcbca036eeb F src/shell.c f04e4af75c5517735397d060ed0b4a874104bb41 -F src/sqlite.h.in cc6e3f38d4c1e4df4f569af49c5deb7c32f1ea10 +F src/sqlite.h.in d9b7b5942a18bb83b560c7699aff3925b4f9af90 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae F src/sqliteInt.h c471d791b10c0f2164c8b7a87adc338e703c09cc @@ -397,7 +397,7 @@ F src/sqliteLimit.h c0373387c287c8d0932510b5547ecde31b5da247 F src/status.c a9e66593dfb28a9e746cba7153f84d49c1ddc4b1 F src/table.c 5226df15ab9179b9ed558d89575ea0ce37b03fc9 F src/tclsqlite.c aef87dcd8cb66564d560ab48d43d19ac812a1eab -F src/test1.c 58de30ed902f78531cf5cf52b883a26d107208c4 +F src/test1.c d6a047ea534fb68fedcb5a47f4db3baef6748294 F src/test2.c 3efb99ab7f1fc8d154933e02ae1378bac9637da5 F src/test3.c d03f5b5da9a2410b7a91c64b0d3306ed28ab6fee F src/test4.c 18ec393bb4d0ad1de729f0b94da7267270f3d8e6 @@ -465,8 +465,8 @@ F src/vdbesort.c 91fda3909326860382b0ca8aa251e609c6a9d62c F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 F src/vtab.c e02cacb5c7ae742631edeb9ae9f53d399f093fd8 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 88f8d8adcaecf6a225ae5098062fd151634fb672 -F src/wal.h bf03a23da3100ab25e5c0363450233cfee09cfc2 +F src/wal.c 5ef877f34c1becb8aaff337bb24544462117a79b +F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 F src/where.c 952f76e7a03727480b274b66ca6641b1657cd591 F src/whereInt.h 2bcc3d176e6091cb8f50a30b65c006e88a73614d @@ -1103,7 +1103,7 @@ F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 F test/skipscan6.test 5866039d03a56f5bd0b3d172a012074a1d90a15b F test/snapshot.test 85735bd997a4f6d710140c28fd860519a299649f -F test/snapshot2.test 30bd95f6fefa8be7f29421e27745cac90b66c74d +F test/snapshot2.test 26949814860534949672dc007877062719b57c39 F test/snapshot_fault.test 062ff0438a074978d45e9f9a92e7ad459b74ee73 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 843cd84db9891b2d01b9ab64cef3e9020f98d087 @@ -1535,7 +1535,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1f7ee7af7b620262ae663d65889b6a87415d4a34 -R 94e3e6505e79390151b9cd467eca8236 +P 83b658dad091211ade3594d1e8d00ce525882506 +R 3db63b187961b7fe3d252e007ca73669 U dan -Z 084fdf6a46a9563f3a18164c0feb0d49 +Z 5c1ddc1fb889cebc1f3b9de248414272 diff --git a/manifest.uuid b/manifest.uuid index 0a12e01e25..3e4851d960 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83b658dad091211ade3594d1e8d00ce525882506 \ No newline at end of file +174a6076a8d7bebe5efebf55f3fdc5d87c589cc7 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 29e166447e..c0494135ca 100644 --- a/src/main.c +++ b/src/main.c @@ -4044,6 +4044,34 @@ int sqlite3_snapshot_open( return rc; } +/* +** Recover as many snapshots as possible from the wal file associated with +** schema zDb of database db. +*/ +int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb){ + int rc = SQLITE_ERROR; + int iDb; +#ifndef SQLITE_OMIT_WAL + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + + sqlite3_mutex_enter(db->mutex); + iDb = sqlite3FindDbName(db, zDb); + if( iDb==0 || iDb>1 ){ + Btree *pBt = db->aDb[iDb].pBt; + if( 0==sqlite3BtreeIsInReadTrans(pBt) ){ + rc = sqlite3PagerSnapshotRecover(sqlite3BtreePager(pBt)); + } + } + sqlite3_mutex_leave(db->mutex); +#endif /* SQLITE_OMIT_WAL */ + return rc; +} + /* ** Free a snapshot handle obtained from sqlite3_snapshot_get(). */ diff --git a/src/pager.c b/src/pager.c index 04ce19547b..9975f3fcbb 100644 --- a/src/pager.c +++ b/src/pager.c @@ -7405,6 +7405,20 @@ int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSnapshot){ } return rc; } + +/* +** If this is a WAL database, call sqlite3WalSnapshotRecover(). If this +** is not a WAL database, return an error. +*/ +int sqlite3PagerSnapshotRecover(Pager *pPager){ + int rc; + if( pPager->pWal ){ + rc = sqlite3WalSnapshotRecover(pPager->pWal); + }else{ + rc = SQLITE_ERROR; + } + return rc; +} #endif /* SQLITE_ENABLE_SNAPSHOT */ #endif /* !SQLITE_OMIT_WAL */ diff --git a/src/pager.h b/src/pager.h index 3003c21ecc..dd57f598bd 100644 --- a/src/pager.h +++ b/src/pager.h @@ -182,6 +182,7 @@ int sqlite3PagerSharedLock(Pager *pPager); # ifdef SQLITE_ENABLE_SNAPSHOT int sqlite3PagerSnapshotGet(Pager *pPager, sqlite3_snapshot **ppSnapshot); int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSnapshot); + int sqlite3PagerSnapshotRecover(Pager *pPager); # endif #else # define sqlite3PagerUseWal(x) 0 diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 8f3b40292d..9c921b8c99 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -8414,6 +8414,12 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( sqlite3_snapshot *p2 ); +/* +** CAPI3REF: Recover snapshots from a wal file +** EXPERIMENTAL +*/ +SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); + /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. diff --git a/src/test1.c b/src/test1.c index e31ad95f40..de16246b56 100644 --- a/src/test1.c +++ b/src/test1.c @@ -2310,6 +2310,38 @@ static int SQLITE_TCLAPI test_snapshot_get( } #endif /* SQLITE_ENABLE_SNAPSHOT */ +#ifdef SQLITE_ENABLE_SNAPSHOT +/* +** Usage: sqlite3_snapshot_recover DB DBNAME +*/ +static int SQLITE_TCLAPI test_snapshot_recover( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int rc; + sqlite3 *db; + char *zName; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB DBNAME"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zName = Tcl_GetString(objv[2]); + + rc = sqlite3_snapshot_recover(db, zName); + if( rc!=SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewStringObj(sqlite3ErrName(rc), -1)); + return TCL_ERROR; + }else{ + Tcl_ResetResult(interp); + } + return TCL_OK; +} +#endif /* SQLITE_ENABLE_SNAPSHOT */ + #ifdef SQLITE_ENABLE_SNAPSHOT /* ** Usage: sqlite3_snapshot_open DB DBNAME SNAPSHOT @@ -7646,6 +7678,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_snapshot_open", test_snapshot_open, 0 }, { "sqlite3_snapshot_free", test_snapshot_free, 0 }, { "sqlite3_snapshot_cmp", test_snapshot_cmp, 0 }, + { "sqlite3_snapshot_recover", test_snapshot_recover, 0 }, { "sqlite3_snapshot_get_blob", test_snapshot_get_blob, 0 }, { "sqlite3_snapshot_open_blob", test_snapshot_open_blob, 0 }, { "sqlite3_snapshot_cmp_blob", test_snapshot_cmp_blob, 0 }, diff --git a/src/wal.c b/src/wal.c index 669c428efa..178278816a 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2379,6 +2379,65 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ return rc; } +/* +** Recover as many snapshots as possible from the wal file. +*/ +int sqlite3WalSnapshotRecover(Wal *pWal){ + int dummy; + int rc; + + rc = sqlite3WalBeginReadTransaction(pWal, &dummy); + if( rc==SQLITE_OK ){ + rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); + if( rc==SQLITE_OK ){ + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + int szPage = (int)pWal->szPage; + void *pBuf1 = sqlite3_malloc(szPage); + void *pBuf2 = sqlite3_malloc(szPage); + + if( pBuf1==0 || pBuf2==0 ){ + rc = SQLITE_NOMEM; + }else{ + u32 i = pInfo->nBackfillAttempted; + for(i=pInfo->nBackfillAttempted; i>pInfo->nBackfill; i--){ + volatile ht_slot *dummy; + volatile u32 *aPgno; /* Array of page numbers */ + u32 iZero; /* Frame corresponding to aPgno[0] */ + u32 pgno; /* Page number in db file */ + i64 iDbOff; /* Offset of db file entry */ + i64 iWalOff; /* Offset of wal file entry */ + rc = walHashGet(pWal, walFramePage(i), &dummy, &aPgno, &iZero); + + if( rc==SQLITE_OK ){ + pgno = aPgno[i-iZero]; + iDbOff = (i64)(pgno-1) * szPage; + iWalOff = walFrameOffset(i, szPage) + WAL_FRAME_HDRSIZE; + rc = sqlite3OsRead(pWal->pWalFd, pBuf1, szPage, iWalOff); + } + + if( rc==SQLITE_OK ){ + rc = sqlite3OsRead(pWal->pDbFd, pBuf2, szPage, iDbOff); + } + + if( rc!=SQLITE_OK || 0==memcmp(pBuf1, pBuf2, szPage) ){ + break; + } + + pInfo->nBackfillAttempted = i-1; + } + } + + sqlite3_free(pBuf1); + sqlite3_free(pBuf2); + walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); + } + + sqlite3WalEndReadTransaction(pWal); + } + + return rc; +} + /* ** Begin a read transaction on the database. ** @@ -2441,7 +2500,11 @@ int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ ** has not yet set the pInfo->nBackfillAttempted variable to indicate ** its intent. To avoid the race condition this leads to, ensure that ** there is no checkpointer process by taking a shared CKPT lock - ** before checking pInfo->nBackfillAttempted. */ + ** before checking pInfo->nBackfillAttempted. + ** + ** TODO: Does the aReadMark[] lock prevent a checkpointer from doing + ** this already? + */ rc = walLockShared(pWal, WAL_CKPT_LOCK); if( rc==SQLITE_OK ){ diff --git a/src/wal.h b/src/wal.h index 16d9d6e0d4..4f6d01dad6 100644 --- a/src/wal.h +++ b/src/wal.h @@ -131,6 +131,7 @@ int sqlite3WalHeapMemory(Wal *pWal); #ifdef SQLITE_ENABLE_SNAPSHOT int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapshot); void sqlite3WalSnapshotOpen(Wal *pWal, sqlite3_snapshot *pSnapshot); +int sqlite3WalSnapshotRecover(Wal *pWal); #endif #ifdef SQLITE_ENABLE_ZIPVFS diff --git a/test/snapshot2.test b/test/snapshot2.test index aa5d385893..8d37a1499f 100644 --- a/test/snapshot2.test +++ b/test/snapshot2.test @@ -78,5 +78,65 @@ do_test 1.2.4 { execsql COMMIT db2 close +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(x); + PRAGMA journal_mode = wal; + INSERT INTO t1 VALUES(1); + INSERT INTO t1 VALUES(2); +} {wal} + +do_test 2.1 { + db trans { set snap [sqlite3_snapshot_get_blob db main] } + sqlite3_db_config db NO_CKPT_ON_CLOSE 1 + db close + sqlite3 db test.db + + execsql {SELECT * FROM sqlite_master} + execsql BEGIN + sqlite3_snapshot_open_blob db main $snap + execsql COMMIT; + execsql { INSERT INTO t1 VALUES(3); } +} {} + +do_test 2.2 { + sqlite3_db_config db NO_CKPT_ON_CLOSE 1 + db close + sqlite3 db test.db + + execsql {SELECT * FROM sqlite_master} + execsql BEGIN + list [catch { sqlite3_snapshot_open_blob db main $snap } msg] $msg +} {1 SQLITE_BUSY_SNAPSHOT} + +do_test 2.3 { + execsql COMMIT + sqlite3_snapshot_recover db main + execsql BEGIN + sqlite3_snapshot_open_blob db main $snap + execsql { SELECT * FROM t1 } +} {1 2} + +do_test 2.4 { + execsql COMMIT + execsql { SELECT * FROM t1 } +} {1 2 3} + +do_test 2.5 { + execsql { PRAGMA wal_checkpoint } + sqlite3_db_config db NO_CKPT_ON_CLOSE 1 + db close + sqlite3 db test.db + + execsql {SELECT * FROM sqlite_master} + sqlite3_snapshot_recover db main + execsql BEGIN + list [catch { sqlite3_snapshot_open_blob db main $snap } msg] $msg +} {1 SQLITE_BUSY_SNAPSHOT} + finish_test + + From 5b4009f6376fb7b7433e859ce647a4262d2bcf07 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 19 Nov 2016 14:53:22 +0000 Subject: [PATCH 06/11] Fix a bug in sqlite3_snapshot_recover() that could cause subsequent read transactions to use out-of-data cache entries. FossilOrigin-Name: 9abeb7980a34cec11a3420e14ad98a4ec0d9c599 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/wal.c | 5 +++++ test/snapshot2.test | 21 +++++++++++++++++++++ 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 8ffa3b7ee3..e549e4dfd0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sexperimental\ssqlite3_snapshot_recover()\sAPI. -D 2016-11-18T20:49:43.736 +C Fix\sa\sbug\sin\ssqlite3_snapshot_recover()\sthat\scould\scause\ssubsequent\sread\ntransactions\sto\suse\sout-of-data\scache\sentries. +D 2016-11-19T14:53:22.181 F Makefile.in 6b572807415d3f0a379cebc9461416d8df4a12c8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc bb4d970894abbbe0e88d00aac29bd52af8bc95f4 @@ -465,7 +465,7 @@ F src/vdbesort.c 91fda3909326860382b0ca8aa251e609c6a9d62c F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 F src/vtab.c e02cacb5c7ae742631edeb9ae9f53d399f093fd8 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 5ef877f34c1becb8aaff337bb24544462117a79b +F src/wal.c df90cfbeb84788a05947da40bc821c834a29b085 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 F src/where.c 952f76e7a03727480b274b66ca6641b1657cd591 @@ -1103,7 +1103,7 @@ F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 F test/skipscan6.test 5866039d03a56f5bd0b3d172a012074a1d90a15b F test/snapshot.test 85735bd997a4f6d710140c28fd860519a299649f -F test/snapshot2.test 26949814860534949672dc007877062719b57c39 +F test/snapshot2.test aeacd61be9ad4aafdab183e9137eeca87623edad F test/snapshot_fault.test 062ff0438a074978d45e9f9a92e7ad459b74ee73 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 843cd84db9891b2d01b9ab64cef3e9020f98d087 @@ -1535,7 +1535,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 83b658dad091211ade3594d1e8d00ce525882506 -R 3db63b187961b7fe3d252e007ca73669 +P 174a6076a8d7bebe5efebf55f3fdc5d87c589cc7 +R d8309c01b710e62ecca9fb17e1bbb17f U dan -Z 5c1ddc1fb889cebc1f3b9de248414272 +Z 62bf95225417162d453d1041ff22e883 diff --git a/manifest.uuid b/manifest.uuid index 3e4851d960..a6da6c0bc7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -174a6076a8d7bebe5efebf55f3fdc5d87c589cc7 \ No newline at end of file +9abeb7980a34cec11a3420e14ad98a4ec0d9c599 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 178278816a..279d751d7f 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2432,7 +2432,12 @@ int sqlite3WalSnapshotRecover(Wal *pWal){ walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); } + /* End the read transaction opened above. Also zero the cache of the + ** wal-index header to force the pager-cache to be flushed when the next + ** read transaction is open, as it may not match the current contents of + ** pWal->hdr. */ sqlite3WalEndReadTransaction(pWal); + memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); } return rc; diff --git a/test/snapshot2.test b/test/snapshot2.test index 8d37a1499f..5c21b31005 100644 --- a/test/snapshot2.test +++ b/test/snapshot2.test @@ -79,6 +79,7 @@ execsql COMMIT db2 close #------------------------------------------------------------------------- +# Simple tests for sqlite3_snapshot_recover(). # reset_db do_execsql_test 2.0 { @@ -136,6 +137,26 @@ do_test 2.5 { list [catch { sqlite3_snapshot_open_blob db main $snap } msg] $msg } {1 SQLITE_BUSY_SNAPSHOT} +#------------------------------------------------------------------------- +# Check that calling sqlite3_snapshot_recover() does not confuse the +# pager cache. +reset_db +do_execsql_test 3.0 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES('a', 'b'); + INSERT INTO t1 VALUES('c', 'd'); +} {wal} +do_test 3.1 { + sqlite3 db2 test.db + execsql { INSERT INTO t1 VALUES('e', 'f') } db2 + db2 close + sqlite3_snapshot_recover db main +} {} +breakpoint +do_execsql_test 3.2 { + SELECT * FROM t1; +} {a b c d e f} finish_test From 6a9e7f16ebab46e2f8ec2be8b6f834535787a164 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 19 Nov 2016 16:35:53 +0000 Subject: [PATCH 07/11] Fix a problem causing sqlite3_snapshot_recover() to return SQLITE_IOERR_SHORT_READ. FossilOrigin-Name: 525f75fa9fd4a95acc3fb3b0a01dabe2be39b383 --- manifest | 14 ++++----- manifest.uuid | 2 +- src/wal.c | 62 ++++++++++++++++++++++------------------ test/snapshot_fault.test | 31 ++++++++++++++++++++ 4 files changed, 73 insertions(+), 36 deletions(-) diff --git a/manifest b/manifest index e549e4dfd0..8f0fbcafe4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sin\ssqlite3_snapshot_recover()\sthat\scould\scause\ssubsequent\sread\ntransactions\sto\suse\sout-of-data\scache\sentries. -D 2016-11-19T14:53:22.181 +C Fix\sa\sproblem\scausing\ssqlite3_snapshot_recover()\sto\sreturn\nSQLITE_IOERR_SHORT_READ. +D 2016-11-19T16:35:53.322 F Makefile.in 6b572807415d3f0a379cebc9461416d8df4a12c8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc bb4d970894abbbe0e88d00aac29bd52af8bc95f4 @@ -465,7 +465,7 @@ F src/vdbesort.c 91fda3909326860382b0ca8aa251e609c6a9d62c F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 F src/vtab.c e02cacb5c7ae742631edeb9ae9f53d399f093fd8 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c df90cfbeb84788a05947da40bc821c834a29b085 +F src/wal.c 006eec60874fc502afc9e95639bbd446125cfab5 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 F src/where.c 952f76e7a03727480b274b66ca6641b1657cd591 @@ -1104,7 +1104,7 @@ F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 F test/skipscan6.test 5866039d03a56f5bd0b3d172a012074a1d90a15b F test/snapshot.test 85735bd997a4f6d710140c28fd860519a299649f F test/snapshot2.test aeacd61be9ad4aafdab183e9137eeca87623edad -F test/snapshot_fault.test 062ff0438a074978d45e9f9a92e7ad459b74ee73 +F test/snapshot_fault.test 566e8a45a2ba71c67bad2077072c85ac54e9dded F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 843cd84db9891b2d01b9ab64cef3e9020f98d087 F test/sort.test c2adc635c2564241fefec0b3a68391ef6868fd3b @@ -1535,7 +1535,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 174a6076a8d7bebe5efebf55f3fdc5d87c589cc7 -R d8309c01b710e62ecca9fb17e1bbb17f +P 9abeb7980a34cec11a3420e14ad98a4ec0d9c599 +R 7b6eb1e1f049ba4c986979f464295360 U dan -Z 62bf95225417162d453d1041ff22e883 +Z 9f95743e0b02ecca5491496b02c4041a diff --git a/manifest.uuid b/manifest.uuid index a6da6c0bc7..51ffaef2ce 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9abeb7980a34cec11a3420e14ad98a4ec0d9c599 \ No newline at end of file +525f75fa9fd4a95acc3fb3b0a01dabe2be39b383 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 279d751d7f..4819d65c27 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2392,43 +2392,49 @@ int sqlite3WalSnapshotRecover(Wal *pWal){ if( rc==SQLITE_OK ){ volatile WalCkptInfo *pInfo = walCkptInfo(pWal); int szPage = (int)pWal->szPage; - void *pBuf1 = sqlite3_malloc(szPage); - void *pBuf2 = sqlite3_malloc(szPage); + i64 szDb; /* Size of db file in bytes */ - if( pBuf1==0 || pBuf2==0 ){ - rc = SQLITE_NOMEM; - }else{ - u32 i = pInfo->nBackfillAttempted; - for(i=pInfo->nBackfillAttempted; i>pInfo->nBackfill; i--){ - volatile ht_slot *dummy; - volatile u32 *aPgno; /* Array of page numbers */ - u32 iZero; /* Frame corresponding to aPgno[0] */ - u32 pgno; /* Page number in db file */ - i64 iDbOff; /* Offset of db file entry */ - i64 iWalOff; /* Offset of wal file entry */ - rc = walHashGet(pWal, walFramePage(i), &dummy, &aPgno, &iZero); + rc = sqlite3OsFileSize(pWal->pDbFd, &szDb); + if( rc==SQLITE_OK ){ + void *pBuf1 = sqlite3_malloc(szPage); + void *pBuf2 = sqlite3_malloc(szPage); + if( pBuf1==0 || pBuf2==0 ){ + rc = SQLITE_NOMEM; + }else{ + u32 i = pInfo->nBackfillAttempted; + for(i=pInfo->nBackfillAttempted; i>pInfo->nBackfill; i--){ + volatile ht_slot *dummy; + volatile u32 *aPgno; /* Array of page numbers */ + u32 iZero; /* Frame corresponding to aPgno[0] */ + u32 pgno; /* Page number in db file */ + i64 iDbOff; /* Offset of db file entry */ + i64 iWalOff; /* Offset of wal file entry */ - if( rc==SQLITE_OK ){ + rc = walHashGet(pWal, walFramePage(i), &dummy, &aPgno, &iZero); + if( rc!=SQLITE_OK ) break; pgno = aPgno[i-iZero]; iDbOff = (i64)(pgno-1) * szPage; - iWalOff = walFrameOffset(i, szPage) + WAL_FRAME_HDRSIZE; - rc = sqlite3OsRead(pWal->pWalFd, pBuf1, szPage, iWalOff); - } - if( rc==SQLITE_OK ){ - rc = sqlite3OsRead(pWal->pDbFd, pBuf2, szPage, iDbOff); - } + if( iDbOff+szPage<=szDb ){ + iWalOff = walFrameOffset(i, szPage) + WAL_FRAME_HDRSIZE; + rc = sqlite3OsRead(pWal->pWalFd, pBuf1, szPage, iWalOff); - if( rc!=SQLITE_OK || 0==memcmp(pBuf1, pBuf2, szPage) ){ - break; - } + if( rc==SQLITE_OK ){ + rc = sqlite3OsRead(pWal->pDbFd, pBuf2, szPage, iDbOff); + } - pInfo->nBackfillAttempted = i-1; + if( rc!=SQLITE_OK || 0==memcmp(pBuf1, pBuf2, szPage) ){ + break; + } + } + + pInfo->nBackfillAttempted = i-1; + } } - } - sqlite3_free(pBuf1); - sqlite3_free(pBuf2); + sqlite3_free(pBuf1); + sqlite3_free(pBuf2); + } walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); } diff --git a/test/snapshot_fault.test b/test/snapshot_fault.test index f90690e3f6..f4f45249a4 100644 --- a/test/snapshot_fault.test +++ b/test/snapshot_fault.test @@ -17,6 +17,8 @@ source $testdir/tester.tcl ifcapable !snapshot {finish_test; return} set testprefix snapshot_fault +if 1 { + #------------------------------------------------------------------------- # Check that an sqlite3_snapshot_open() client cannot be tricked into # reading a corrupt snapshot even if a second client fails while @@ -159,6 +161,35 @@ do_faultsim_test 3.0 -prep { sqlite3_snapshot_free $::snapshot } +} + +#------------------------------------------------------------------------- +# Test the handling of faults that occur within sqlite3_snapshot_recover(). +# +reset_db +do_execsql_test 4.0 { + PRAGMA journal_mode = wal; + CREATE TABLE t1(zzz); + INSERT INTO t1 VALUES('abc'); + INSERT INTO t1 VALUES('def'); +} {wal} +faultsim_save_and_close + +do_test 4.1 { + faultsim_restore_and_reopen + db eval { SELECT * FROM sqlite_master } + sqlite3_snapshot_recover db main +} {} +db close + +do_faultsim_test 4 -faults oom* -prep { + faultsim_restore_and_reopen + db eval { SELECT * FROM sqlite_master } +} -body { + sqlite3_snapshot_recover db main +} -test { + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} {1 SQLITE_IOERR_NOMEM} +} finish_test From 20ba3d7fcb6d8f7e21149b01e181fc1a9532c050 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 19 Nov 2016 17:20:28 +0000 Subject: [PATCH 08/11] Add another fault-injection test for sqlite3_snapshot_recover(). FossilOrigin-Name: 7e040406138669bd67dd6ecae016b3e50dbfaaf8 --- manifest | 12 +++++------ manifest.uuid | 2 +- test/snapshot_fault.test | 43 ++++++++++++++++++++++++++++++++++------ 3 files changed, 44 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 8f0fbcafe4..c1d99f809d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\scausing\ssqlite3_snapshot_recover()\sto\sreturn\nSQLITE_IOERR_SHORT_READ. -D 2016-11-19T16:35:53.322 +C Add\sanother\sfault-injection\stest\sfor\ssqlite3_snapshot_recover(). +D 2016-11-19T17:20:28.983 F Makefile.in 6b572807415d3f0a379cebc9461416d8df4a12c8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc bb4d970894abbbe0e88d00aac29bd52af8bc95f4 @@ -1104,7 +1104,7 @@ F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 F test/skipscan6.test 5866039d03a56f5bd0b3d172a012074a1d90a15b F test/snapshot.test 85735bd997a4f6d710140c28fd860519a299649f F test/snapshot2.test aeacd61be9ad4aafdab183e9137eeca87623edad -F test/snapshot_fault.test 566e8a45a2ba71c67bad2077072c85ac54e9dded +F test/snapshot_fault.test 52c5e97ebd218846a8ae2da4d147d3e77d71f963 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 843cd84db9891b2d01b9ab64cef3e9020f98d087 F test/sort.test c2adc635c2564241fefec0b3a68391ef6868fd3b @@ -1535,7 +1535,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9abeb7980a34cec11a3420e14ad98a4ec0d9c599 -R 7b6eb1e1f049ba4c986979f464295360 +P 525f75fa9fd4a95acc3fb3b0a01dabe2be39b383 +R 540803770026fc0e2bba54ecd3e455de U dan -Z 9f95743e0b02ecca5491496b02c4041a +Z a16f9ea1f945ee0adbd8c25a4885f6d9 diff --git a/manifest.uuid b/manifest.uuid index 51ffaef2ce..cb29e68423 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -525f75fa9fd4a95acc3fb3b0a01dabe2be39b383 \ No newline at end of file +7e040406138669bd67dd6ecae016b3e50dbfaaf8 \ No newline at end of file diff --git a/test/snapshot_fault.test b/test/snapshot_fault.test index f4f45249a4..5c6cf40a6f 100644 --- a/test/snapshot_fault.test +++ b/test/snapshot_fault.test @@ -17,8 +17,6 @@ source $testdir/tester.tcl ifcapable !snapshot {finish_test; return} set testprefix snapshot_fault -if 1 { - #------------------------------------------------------------------------- # Check that an sqlite3_snapshot_open() client cannot be tricked into # reading a corrupt snapshot even if a second client fails while @@ -161,8 +159,6 @@ do_faultsim_test 3.0 -prep { sqlite3_snapshot_free $::snapshot } -} - #------------------------------------------------------------------------- # Test the handling of faults that occur within sqlite3_snapshot_recover(). # @@ -175,14 +171,14 @@ do_execsql_test 4.0 { } {wal} faultsim_save_and_close -do_test 4.1 { +do_test 4.0.1 { faultsim_restore_and_reopen db eval { SELECT * FROM sqlite_master } sqlite3_snapshot_recover db main } {} db close -do_faultsim_test 4 -faults oom* -prep { +do_faultsim_test 4.0 -faults oom* -prep { faultsim_restore_and_reopen db eval { SELECT * FROM sqlite_master } } -body { @@ -191,5 +187,40 @@ do_faultsim_test 4 -faults oom* -prep { faultsim_test_result {0 {}} {1 SQLITE_NOMEM} {1 SQLITE_IOERR_NOMEM} } +# The following test cases contrive to call sqlite3_snapshot_recover() +# before all pages of the *-shm file have been mapped. This tests an +# extra branch of error handling logic in snapshot_recover(). +# +reset_db +do_execsql_test 4.1.0 { + PRAGMA page_size = 512; + PRAGMA journal_mode = wal; + PRAGMA wal_autocheckpoint = 0; + CREATE TABLE t1(zzz); + INSERT INTO t1 VALUES(randomblob( 500 * 9500 )); + PRAGMA user_version = 211; +} {wal 0} + +do_test 4.1.1 { + list [file size test.db-shm] [file size test.db] +} {98304 512} + +faultsim_save_and_close +do_faultsim_test 4.1 -faults shm* -prep { + catch { db2 close } + catch { db close } + faultsim_restore_and_reopen + sqlite3 db2 test.db + db2 eval { SELECT * FROM sqlite_master } + db eval BEGIN + sqlite3_snapshot_get_blob db main + db eval COMMIT +} -body { + sqlite3_snapshot_recover db main +} -test { + faultsim_test_result {0 {}} {1 SQLITE_IOERR} +} + + finish_test From 4d4757fcc86fd7014b58b08aa06b2c0f1b88105e Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 19 Nov 2016 17:30:57 +0000 Subject: [PATCH 09/11] Test some extra error conditions in sqlite3_recover_snapshot(). FossilOrigin-Name: db314213c08f27dd0ff5ede3c6a8eda36560809a --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/snapshot2.test | 42 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c1d99f809d..c697bbdef6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sanother\sfault-injection\stest\sfor\ssqlite3_snapshot_recover(). -D 2016-11-19T17:20:28.983 +C Test\ssome\sextra\serror\sconditions\sin\ssqlite3_recover_snapshot(). +D 2016-11-19T17:30:57.131 F Makefile.in 6b572807415d3f0a379cebc9461416d8df4a12c8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc bb4d970894abbbe0e88d00aac29bd52af8bc95f4 @@ -1103,7 +1103,7 @@ F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 F test/skipscan6.test 5866039d03a56f5bd0b3d172a012074a1d90a15b F test/snapshot.test 85735bd997a4f6d710140c28fd860519a299649f -F test/snapshot2.test aeacd61be9ad4aafdab183e9137eeca87623edad +F test/snapshot2.test 706498fdd1f03163473965faedd56b137ff7ec50 F test/snapshot_fault.test 52c5e97ebd218846a8ae2da4d147d3e77d71f963 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 843cd84db9891b2d01b9ab64cef3e9020f98d087 @@ -1535,7 +1535,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 525f75fa9fd4a95acc3fb3b0a01dabe2be39b383 -R 540803770026fc0e2bba54ecd3e455de +P 7e040406138669bd67dd6ecae016b3e50dbfaaf8 +R 8677d727bb420bd9cae1a19aef039ec1 U dan -Z a16f9ea1f945ee0adbd8c25a4885f6d9 +Z 6dc3dd1ccc30316098a970a98f898ad5 diff --git a/manifest.uuid b/manifest.uuid index cb29e68423..a26ea4d37d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7e040406138669bd67dd6ecae016b3e50dbfaaf8 \ No newline at end of file +db314213c08f27dd0ff5ede3c6a8eda36560809a \ No newline at end of file diff --git a/test/snapshot2.test b/test/snapshot2.test index 5c21b31005..b13bbe8862 100644 --- a/test/snapshot2.test +++ b/test/snapshot2.test @@ -153,11 +153,51 @@ do_test 3.1 { db2 close sqlite3_snapshot_recover db main } {} -breakpoint do_execsql_test 3.2 { SELECT * FROM t1; } {a b c d e f} +#------------------------------------------------------------------------- +# Check that sqlite3_snapshot_recover() returns an error if it is called +# with an open read-transaction. Or on a database that does not exist. Or +# on the temp database. Or on a db that is not in wal mode. +# +do_test 4.1 { + sqlite3_snapshot_recover db main +} {} +do_test 4.2 { + execsql { + BEGIN; + SELECT * FROM sqlite_master; + } + list [catch { sqlite3_snapshot_recover db main } msg] $msg +} {1 SQLITE_ERROR} +do_test 4.3 { + execsql COMMIT + sqlite3_snapshot_recover db main +} {} +do_test 4.4 { + list [catch { sqlite3_snapshot_recover db aux } msg] $msg +} {1 SQLITE_ERROR} +do_test 4.5 { + forcedelete test.db2 + execsql { + ATTACH 'test.db2' AS aux; + PRAGMA aux.journal_mode = wal; + CREATE TABLE aux.t2(x, y); + } + list [catch { sqlite3_snapshot_recover db aux } msg] $msg +} {0 {}} +do_test 4.6 { + list [catch { sqlite3_snapshot_recover db temp } msg] $msg +} {1 SQLITE_ERROR} +do_test 4.7 { + execsql { + PRAGMA aux.journal_mode = delete; + } + list [catch { sqlite3_snapshot_recover db aux } msg] $msg +} {1 SQLITE_ERROR} + finish_test From 93f5113290a42f48386e17cee1a737ed1e5fe0d2 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 19 Nov 2016 18:31:37 +0000 Subject: [PATCH 10/11] Remove the requirement to open the wal file before sqlite3_snapshot_recover() is called. Also add some comments to new functions. FossilOrigin-Name: 28393c413cc4505b94411730e728583c5d4baaae --- manifest | 18 ++++---- manifest.uuid | 2 +- src/main.c | 6 ++- src/sqlite.h.in | 16 +++++++ src/wal.c | 104 +++++++++++++++++++++++--------------------- test/snapshot2.test | 1 - 6 files changed, 86 insertions(+), 61 deletions(-) diff --git a/manifest b/manifest index c697bbdef6..21ea07385c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\ssome\sextra\serror\sconditions\sin\ssqlite3_recover_snapshot(). -D 2016-11-19T17:30:57.131 +C Remove\sthe\srequirement\sto\sopen\sthe\swal\sfile\sbefore\ssqlite3_snapshot_recover()\nis\scalled.\sAlso\sadd\ssome\scomments\sto\snew\sfunctions. +D 2016-11-19T18:31:37.017 F Makefile.in 6b572807415d3f0a379cebc9461416d8df4a12c8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc bb4d970894abbbe0e88d00aac29bd52af8bc95f4 @@ -352,7 +352,7 @@ F src/in-operator.md 10cd8f4bcd225a32518407c2fb2484089112fd71 F src/insert.c 0d6e59f9eea62db772eabc0f07901ec26fe01968 F src/legacy.c 75d3023be8f0d2b99d60f905090341a03358c58e F src/loadext.c 5d6642d141c07d366e43d359e94ec9de47add41d -F src/main.c de55e68145758d5512e9397728c9a8f4ff0ffa78 +F src/main.c 5669ae83c6bd47f090aee26a6b548a882d69e9e1 F src/malloc.c 5ee7c2d3dcb1b0a902c9c6d0115deef54736bdfa F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 6919bcf12f221868ea066eec27e579fed95ce98b @@ -389,7 +389,7 @@ F src/resolve.c bb070cf5f23611c44ab7e4788803684e385fc3fb F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac F src/select.c 672b1af237ad257149fc5189f3277dcbca036eeb F src/shell.c f04e4af75c5517735397d060ed0b4a874104bb41 -F src/sqlite.h.in d9b7b5942a18bb83b560c7699aff3925b4f9af90 +F src/sqlite.h.in 631cc94108c1c03190d5f1edb70e7797c522a37a F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 8648034aa702469afb553231677306cc6492a1ae F src/sqliteInt.h c471d791b10c0f2164c8b7a87adc338e703c09cc @@ -465,7 +465,7 @@ F src/vdbesort.c 91fda3909326860382b0ca8aa251e609c6a9d62c F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 F src/vtab.c e02cacb5c7ae742631edeb9ae9f53d399f093fd8 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 006eec60874fc502afc9e95639bbd446125cfab5 +F src/wal.c 56bba6f4101b55054145ce164951bea4a1b09548 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 F src/where.c 952f76e7a03727480b274b66ca6641b1657cd591 @@ -1103,7 +1103,7 @@ F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 F test/skipscan6.test 5866039d03a56f5bd0b3d172a012074a1d90a15b F test/snapshot.test 85735bd997a4f6d710140c28fd860519a299649f -F test/snapshot2.test 706498fdd1f03163473965faedd56b137ff7ec50 +F test/snapshot2.test eb083df2e617708a4a93d70965f14268d4934120 F test/snapshot_fault.test 52c5e97ebd218846a8ae2da4d147d3e77d71f963 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 843cd84db9891b2d01b9ab64cef3e9020f98d087 @@ -1535,7 +1535,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7e040406138669bd67dd6ecae016b3e50dbfaaf8 -R 8677d727bb420bd9cae1a19aef039ec1 +P db314213c08f27dd0ff5ede3c6a8eda36560809a +R 0c1e63a3653bf6d9b725483fda2de4a1 U dan -Z 6dc3dd1ccc30316098a970a98f898ad5 +Z 573fa32a4bef0fb7a3cf7415ded5b3fe diff --git a/manifest.uuid b/manifest.uuid index a26ea4d37d..d4754cb8fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -db314213c08f27dd0ff5ede3c6a8eda36560809a \ No newline at end of file +28393c413cc4505b94411730e728583c5d4baaae \ No newline at end of file diff --git a/src/main.c b/src/main.c index c0494135ca..5980dbcc37 100644 --- a/src/main.c +++ b/src/main.c @@ -4064,7 +4064,11 @@ int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb){ if( iDb==0 || iDb>1 ){ Btree *pBt = db->aDb[iDb].pBt; if( 0==sqlite3BtreeIsInReadTrans(pBt) ){ - rc = sqlite3PagerSnapshotRecover(sqlite3BtreePager(pBt)); + rc = sqlite3BtreeBeginTrans(pBt, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSnapshotRecover(sqlite3BtreePager(pBt)); + sqlite3BtreeCommit(pBt); + } } } sqlite3_mutex_leave(db->mutex); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 9c921b8c99..563b21f24e 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -8417,6 +8417,22 @@ SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( /* ** CAPI3REF: Recover snapshots from a wal file ** EXPERIMENTAL +** +** If all connections disconnect from a database file but do not perform +** a checkpoint, the existing wal file is opened along with the database +** file the next time the database is opened. At this point it is only +** possible to successfully call sqlite3_snapshot_open() to open the most +** recent snapshot of the database (the one at the head of the wal file), +** even though the wal file may contain other valid snapshots for which +** clients have sqlite3_snapshot handles. +** +** This function attempts to scan the wal file associated with database zDb +** of database handle db and make all valid snapshots available to +** sqlite3_snapshot_open(). It is an error if there is already a read +** transaction open on the database, or if the database is not a wal mode +** database. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); diff --git a/src/wal.c b/src/wal.c index 4819d65c27..52e44ad550 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2380,70 +2380,76 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } /* -** Recover as many snapshots as possible from the wal file. +** Attempt to reduce the value of the WalCkptInfo.nBackfillAttempted +** variable so that older snapshots can be accessed. To do this, loop +** through all wal frames from nBackfillAttempted to (nBackfill+1), +** comparing their content to the corresponding page with the database +** file, if any. Set nBackfillAttempted to the frame number of the +** first frame for which the wal file content matches the db file. +** +** This is only really safe if the file-system is such that any page +** writes made by earlier checkpointers were atomic operations, which +** is not always true. It is also possible that nBackfillAttempted +** may be left set to a value larger than expected, if a wal frame +** contains content that duplicate of an earlier version of the same +** page. +** +** SQLITE_OK is returned if successful, or an SQLite error code if an +** error occurs. It is not an error if nBackfillAttempted cannot be +** decreased at all. */ int sqlite3WalSnapshotRecover(Wal *pWal){ - int dummy; int rc; - rc = sqlite3WalBeginReadTransaction(pWal, &dummy); + assert( pWal->readLock>=0 ); + rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); if( rc==SQLITE_OK ){ - rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + int szPage = (int)pWal->szPage; + i64 szDb; /* Size of db file in bytes */ + + rc = sqlite3OsFileSize(pWal->pDbFd, &szDb); if( rc==SQLITE_OK ){ - volatile WalCkptInfo *pInfo = walCkptInfo(pWal); - int szPage = (int)pWal->szPage; - i64 szDb; /* Size of db file in bytes */ + void *pBuf1 = sqlite3_malloc(szPage); + void *pBuf2 = sqlite3_malloc(szPage); + if( pBuf1==0 || pBuf2==0 ){ + rc = SQLITE_NOMEM; + }else{ + u32 i = pInfo->nBackfillAttempted; + for(i=pInfo->nBackfillAttempted; i>pInfo->nBackfill; i--){ + volatile ht_slot *dummy; + volatile u32 *aPgno; /* Array of page numbers */ + u32 iZero; /* Frame corresponding to aPgno[0] */ + u32 pgno; /* Page number in db file */ + i64 iDbOff; /* Offset of db file entry */ + i64 iWalOff; /* Offset of wal file entry */ - rc = sqlite3OsFileSize(pWal->pDbFd, &szDb); - if( rc==SQLITE_OK ){ - void *pBuf1 = sqlite3_malloc(szPage); - void *pBuf2 = sqlite3_malloc(szPage); - if( pBuf1==0 || pBuf2==0 ){ - rc = SQLITE_NOMEM; - }else{ - u32 i = pInfo->nBackfillAttempted; - for(i=pInfo->nBackfillAttempted; i>pInfo->nBackfill; i--){ - volatile ht_slot *dummy; - volatile u32 *aPgno; /* Array of page numbers */ - u32 iZero; /* Frame corresponding to aPgno[0] */ - u32 pgno; /* Page number in db file */ - i64 iDbOff; /* Offset of db file entry */ - i64 iWalOff; /* Offset of wal file entry */ + rc = walHashGet(pWal, walFramePage(i), &dummy, &aPgno, &iZero); + if( rc!=SQLITE_OK ) break; + pgno = aPgno[i-iZero]; + iDbOff = (i64)(pgno-1) * szPage; - rc = walHashGet(pWal, walFramePage(i), &dummy, &aPgno, &iZero); - if( rc!=SQLITE_OK ) break; - pgno = aPgno[i-iZero]; - iDbOff = (i64)(pgno-1) * szPage; + if( iDbOff+szPage<=szDb ){ + iWalOff = walFrameOffset(i, szPage) + WAL_FRAME_HDRSIZE; + rc = sqlite3OsRead(pWal->pWalFd, pBuf1, szPage, iWalOff); - if( iDbOff+szPage<=szDb ){ - iWalOff = walFrameOffset(i, szPage) + WAL_FRAME_HDRSIZE; - rc = sqlite3OsRead(pWal->pWalFd, pBuf1, szPage, iWalOff); - - if( rc==SQLITE_OK ){ - rc = sqlite3OsRead(pWal->pDbFd, pBuf2, szPage, iDbOff); - } - - if( rc!=SQLITE_OK || 0==memcmp(pBuf1, pBuf2, szPage) ){ - break; - } + if( rc==SQLITE_OK ){ + rc = sqlite3OsRead(pWal->pDbFd, pBuf2, szPage, iDbOff); } - pInfo->nBackfillAttempted = i-1; + if( rc!=SQLITE_OK || 0==memcmp(pBuf1, pBuf2, szPage) ){ + break; + } } + + pInfo->nBackfillAttempted = i-1; } - - sqlite3_free(pBuf1); - sqlite3_free(pBuf2); } - walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); - } - /* End the read transaction opened above. Also zero the cache of the - ** wal-index header to force the pager-cache to be flushed when the next - ** read transaction is open, as it may not match the current contents of - ** pWal->hdr. */ - sqlite3WalEndReadTransaction(pWal); - memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); + sqlite3_free(pBuf1); + sqlite3_free(pBuf2); + } + walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); } return rc; diff --git a/test/snapshot2.test b/test/snapshot2.test index b13bbe8862..62d3070ecd 100644 --- a/test/snapshot2.test +++ b/test/snapshot2.test @@ -131,7 +131,6 @@ do_test 2.5 { db close sqlite3 db test.db - execsql {SELECT * FROM sqlite_master} sqlite3_snapshot_recover db main execsql BEGIN list [catch { sqlite3_snapshot_open_blob db main $snap } msg] $msg From bc88711d08602b00b591bf1f127b1d4a9d8b310d Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 22 Nov 2016 21:11:59 +0000 Subject: [PATCH 11/11] Enclose the sqlite3WalSnapshotRecover() routine within FossilOrigin-Name: e7be3183eb25e0f9f04b9e251ff37fa5e50cc1a7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/wal.c | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 21ea07385c..8112830311 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\srequirement\sto\sopen\sthe\swal\sfile\sbefore\ssqlite3_snapshot_recover()\nis\scalled.\sAlso\sadd\ssome\scomments\sto\snew\sfunctions. -D 2016-11-19T18:31:37.017 +C Enclose\sthe\ssqlite3WalSnapshotRecover()\sroutine\swithin +D 2016-11-22T21:11:59.294 F Makefile.in 6b572807415d3f0a379cebc9461416d8df4a12c8 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc bb4d970894abbbe0e88d00aac29bd52af8bc95f4 @@ -465,7 +465,7 @@ F src/vdbesort.c 91fda3909326860382b0ca8aa251e609c6a9d62c F src/vdbetrace.c 41963d5376f0349842b5fc4aaaaacd7d9cdc0834 F src/vtab.c e02cacb5c7ae742631edeb9ae9f53d399f093fd8 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 -F src/wal.c 56bba6f4101b55054145ce164951bea4a1b09548 +F src/wal.c 40c543f0a2195d1b0dc88ef12142bea690009344 F src/wal.h 06b2a0b599cc0f53ea97f497cf8c6b758c999f71 F src/walker.c 91a6df7435827e41cff6bb7df50ea00934ee78b0 F src/where.c 952f76e7a03727480b274b66ca6641b1657cd591 @@ -1535,7 +1535,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P db314213c08f27dd0ff5ede3c6a8eda36560809a -R 0c1e63a3653bf6d9b725483fda2de4a1 -U dan -Z 573fa32a4bef0fb7a3cf7415ded5b3fe +P 28393c413cc4505b94411730e728583c5d4baaae +R 87af84aaf5faeefdf69427a408fd9884 +U drh +Z eaed2a9cb4277f14d0de83d53d00fc85 diff --git a/manifest.uuid b/manifest.uuid index d4754cb8fa..f2d1cc2d07 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -28393c413cc4505b94411730e728583c5d4baaae \ No newline at end of file +e7be3183eb25e0f9f04b9e251ff37fa5e50cc1a7 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 52e44ad550..09f605fe57 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2379,6 +2379,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ return rc; } +#ifdef SQLITE_ENABLE_SNAPSHOT /* ** Attempt to reduce the value of the WalCkptInfo.nBackfillAttempted ** variable so that older snapshots can be accessed. To do this, loop @@ -2454,6 +2455,7 @@ int sqlite3WalSnapshotRecover(Wal *pWal){ return rc; } +#endif /* SQLITE_ENABLE_SNAPSHOT */ /* ** Begin a read transaction on the database.