From f89157653a8a4586b2498cc6b8f64cd8896e1e9f Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 15 Apr 2016 20:46:41 +0000 Subject: [PATCH 01/15] Add the experimental sqlite3rbu_vacuum() API function. For opening an RBU handle that rebuilds a database from scratch. FossilOrigin-Name: 0216b48f28042ad86711e00802c2da8ce9be3044 --- ext/rbu/rbuvacuum.test | 181 +++++++++++++++ ext/rbu/sqlite3rbu.c | 501 +++++++++++++++++++++++++++++------------ ext/rbu/sqlite3rbu.h | 8 + ext/rbu/test_rbu.c | 29 +++ manifest | 22 +- manifest.uuid | 2 +- 6 files changed, 595 insertions(+), 148 deletions(-) create mode 100644 ext/rbu/rbuvacuum.test diff --git a/ext/rbu/rbuvacuum.test b/ext/rbu/rbuvacuum.test new file mode 100644 index 0000000000..045b2d1bb7 --- /dev/null +++ b/ext/rbu/rbuvacuum.test @@ -0,0 +1,181 @@ +# 2016 April 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the RBU module. More specifically, it +# contains tests to ensure that the sqlite3rbu_vacuum() API works as +# expected. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +set ::testprefix rbuvacuum + +proc do_rbu_vacuum_test {tn} { + uplevel [list do_test $tn.1 { + forcedelete state.db + if {$::step==0} { sqlite3rbu_vacuum rbu test.db state.db } + while 1 { + if {$::step==1} { sqlite3rbu_vacuum rbu test.db state.db } + set rc [rbu step] + if {$rc!="SQLITE_OK"} break + if {$::step==1} { rbu close } + } + rbu close + } {SQLITE_DONE}] + + uplevel [list do_execsql_test $tn.2 { + PRAGMA integrity_check + } ok] +} + +foreach step {0 1} { + + set ::testprefix rbuvacuum-step=$step + reset_db + + # Simplest possible vacuum. + do_execsql_test 1.0 { + PRAGMA page_size = 1024; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1 VALUES(7, 8, 9); + PRAGMA integrity_check; + } {ok} + do_rbu_vacuum_test 1.1 + + # A vacuum that actually reclaims space. + do_execsql_test 1.2.1 { + INSERT INTO t1 VALUES(8, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(9, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(10, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(11, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(12, randomblob(900), randomblob(900)); + PRAGMA page_count; + } {12} + do_execsql_test 1.2.2 { + DELETE FROM t1 WHERE rowid BETWEEN 8 AND 11; + PRAGMA page_count; + } {12} + do_rbu_vacuum_test 1.2.3 + do_execsql_test 1.2.4 { + PRAGMA page_count; + } {3} + + # Add an index to the table. + do_execsql_test 1.3.1 { + CREATE INDEX t1b ON t1(b); + INSERT INTO t1 VALUES(13, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(14, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(15, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(16, randomblob(900), randomblob(900)); + PRAGMA page_count; + } {18} + do_execsql_test 1.3.2 { + DELETE FROM t1 WHERE rowid BETWEEN 12 AND 15; + PRAGMA page_count; + } {18} + do_rbu_vacuum_test 1.3.3 + do_execsql_test 1.3.4 { + PRAGMA page_count; + } {5} + + # WITHOUT ROWID table. + do_execsql_test 1.4.1 { + CREATE TABLE t2(a, b, c, PRIMARY KEY(a, b)) WITHOUT ROWID; + + INSERT INTO t2 VALUES(randomblob(900), 1, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 2, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 3, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 4, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 6, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 7, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 8, randomblob(900)); + + DELETE FROM t2 WHERE b BETWEEN 2 AND 7; + PRAGMA page_count; + } {20} + do_rbu_vacuum_test 1.4.2 + do_execsql_test 1.4.3 { + PRAGMA page_count; + } {10} + + # WITHOUT ROWID table with an index. + do_execsql_test 1.4.1 { + CREATE INDEX t2c ON t2(c); + + INSERT INTO t2 VALUES(randomblob(900), 9, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 10, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 11, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 12, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 13, randomblob(900)); + + DELETE FROM t2 WHERE b BETWEEN 8 AND 12; + PRAGMA page_count; + } {35} + do_rbu_vacuum_test 1.4.2 + do_execsql_test 1.4.3 { + PRAGMA page_count; + } {15} + do_execsql_test 1.4.4 { + VACUUM; + PRAGMA page_count; + } {15} + + do_execsql_test 1.5.1 { + CREATE TABLE t3(a, b, c); + INSERT INTO t3 VALUES('a', 'b', 'c'); + INSERT INTO t3 VALUES('d', 'e', 'f'); + INSERT INTO t3 VALUES('g', 'h', 'i'); + } + do_rbu_vacuum_test 1.5.2 + do_execsql_test 1.5.3 { + SELECT * FROM t3 + } {a b c d e f g h i} + do_execsql_test 1.5.4 { + CREATE INDEX t3a ON t3(a); + CREATE INDEX t3b ON t3(b); + CREATE INDEX t3c ON t3(c); + INSERT INTO t3 VALUES('j', 'k', 'l'); + DELETE FROM t3 WHERE a = 'g'; + } + do_rbu_vacuum_test 1.5.5 + do_execsql_test 1.5.6 { + SELECT rowid, * FROM t3 ORDER BY b + } {1 a b c 2 d e f 4 j k l} + + do_execsql_test 1.6.1 { + CREATE TABLE t4(a PRIMARY KEY, b, c); + INSERT INTO t4 VALUES('a', 'b', 'c'); + INSERT INTO t4 VALUES('d', 'e', 'f'); + INSERT INTO t4 VALUES('g', 'h', 'i'); + } + do_rbu_vacuum_test 1.6.2 + do_execsql_test 1.6.3 { + SELECT * FROM t4 + } {a b c d e f g h i} + do_execsql_test 1.6.4 { + CREATE INDEX t4a ON t4(a); + CREATE INDEX t4b ON t4(b); + CREATE INDEX t4c ON t4(c); + + INSERT INTO t4 VALUES('j', 'k', 'l'); + DELETE FROM t4 WHERE a='g'; + } + do_rbu_vacuum_test 1.6.5 + do_execsql_test 1.6.6 { + SELECT * FROM t4 ORDER BY b + } {a b c d e f j k l} + +} + +catch { db close } +finish_test + diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index f553ebdac5..e71690769b 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -176,6 +176,7 @@ typedef struct RbuUpdateStmt RbuUpdateStmt; #if !defined(SQLITE_AMALGAMATION) typedef unsigned int u32; +typedef unsigned short u16; typedef unsigned char u8; typedef sqlite3_int64 i64; #endif @@ -402,6 +403,11 @@ struct rbu_file { rbu_file *pMainNext; /* Next MAIN_DB file */ }; +/* +** True for an RBU vacuum handle, or false otherwise. +*/ +#define rbuIsVacuum(p) ((p)->zTarget==0) + /************************************************************************* ** The following three functions, found below: @@ -850,8 +856,11 @@ static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ /* ** The implementation of the rbu_target_name() SQL function. This function -** accepts one argument - the name of a table in the RBU database. If the -** table name matches the pattern: +** accepts one or two arguments. The first argument is the name of a table - +** the name of a table in the RBU database. The second, if it is present, is 1 +** for a view or 0 for a table. +** +** For a non-vacuum RBU handle, if the table name matches the pattern: ** ** data[0-9]_ ** @@ -862,21 +871,33 @@ static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ ** "data_t1" -> "t1" ** "data0123_t2" -> "t2" ** "dataAB_t3" -> NULL +** +** For an rbu vacuum handle, a copy of the first argument is returned if +** the second argument is either missing or 0 (not a view). */ static void rbuTargetNameFunc( - sqlite3_context *context, + sqlite3_context *pCtx, int argc, sqlite3_value **argv ){ + sqlite3rbu *p = sqlite3_user_data(pCtx); const char *zIn; - assert( argc==1 ); + assert( argc==1 || argc==2 ); zIn = (const char*)sqlite3_value_text(argv[0]); - if( zIn && strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ - int i; - for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); - if( zIn[i]=='_' && zIn[i+1] ){ - sqlite3_result_text(context, &zIn[i+1], -1, SQLITE_STATIC); + if( zIn ){ + if( rbuIsVacuum(p) ){ + if( argc==1 || 0==sqlite3_value_int(argv[1]) ){ + sqlite3_result_text(pCtx, zIn, -1, SQLITE_STATIC); + } + }else{ + if( strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ + int i; + for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); + if( zIn[i]=='_' && zIn[i+1] ){ + sqlite3_result_text(pCtx, &zIn[i+1], -1, SQLITE_STATIC); + } + } } } } @@ -894,7 +915,8 @@ static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){ memset(pIter, 0, sizeof(RbuObjIter)); rc = prepareAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, - "SELECT rbu_target_name(name) AS target, name FROM sqlite_master " + "SELECT rbu_target_name(name, type='view') AS target, name " + "FROM sqlite_master " "WHERE type IN ('table', 'view') AND target IS NOT NULL " "ORDER BY name" ); @@ -1270,6 +1292,7 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ pStmt = 0; if( p->rc==SQLITE_OK + && rbuIsVacuum(p)==0 && bRbuRowid!=(pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) ){ p->rc = SQLITE_ERROR; @@ -1409,6 +1432,8 @@ static char *rbuObjIterGetIndexCols( for(i=0; pIter->abTblPk[i]==0; i++); assert( inTblCol ); zCol = pIter->azTblCol[i]; + }else if( rbuIsVacuum(p) ){ + zCol = "_rowid_"; }else{ zCol = "rbu_rowid"; } @@ -1949,7 +1974,7 @@ static int rbuObjIterPrepareAll( } /* And to delete index entries */ - if( p->rc==SQLITE_OK ){ + if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError( p->dbMain, &pIter->pDelete, &p->zErrmsg, sqlite3_mprintf("DELETE FROM \"rbu_imp_%w\" WHERE %s", zTbl, zWhere) @@ -1959,6 +1984,15 @@ static int rbuObjIterPrepareAll( /* Create the SELECT statement to read keys in sorted order */ if( p->rc==SQLITE_OK ){ char *zSql; + if( rbuIsVacuum(p) ){ + zSql = sqlite3_mprintf( + "SELECT %s, 0 AS rbu_control FROM '%q' ORDER BY %s%s", + zCollist, + pIter->zDataTbl, + zCollist, zLimit + ); + }else + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ zSql = sqlite3_mprintf( "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' ORDER BY %s%s", @@ -1985,7 +2019,9 @@ static int rbuObjIterPrepareAll( sqlite3_free(zWhere); sqlite3_free(zBind); }else{ - int bRbuRowid = (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE); + int bRbuRowid = (pIter->eType==RBU_PK_VTAB) + ||(pIter->eType==RBU_PK_NONE) + ||(pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p)); const char *zTbl = pIter->zTbl; /* Table this step applies to */ const char *zWrite; /* Imposter table name */ @@ -2012,8 +2048,10 @@ static int rbuObjIterPrepareAll( ); } - /* Create the DELETE statement to write to the target PK b-tree */ - if( p->rc==SQLITE_OK ){ + /* Create the DELETE statement to write to the target PK b-tree. + ** Because it only performs INSERT operations, this is not required for + ** an rbu vacuum handle. */ + if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pDelete, pz, sqlite3_mprintf( "DELETE FROM \"%s%w\" WHERE %s", zWrite, zTbl, zWhere @@ -2021,7 +2059,7 @@ static int rbuObjIterPrepareAll( ); } - if( pIter->abIndexed ){ + if( rbuIsVacuum(p)==0 && pIter->abIndexed ){ const char *zRbuRowid = ""; if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ zRbuRowid = ", rbu_rowid"; @@ -2071,10 +2109,16 @@ static int rbuObjIterPrepareAll( /* Create the SELECT statement to read keys from data_xxx */ if( p->rc==SQLITE_OK ){ + const char *zRbuRowid = ""; + if( bRbuRowid ){ + zRbuRowid = rbuIsVacuum(p) ? ",_rowid_ " : ",rbu_rowid"; + } p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, sqlite3_mprintf( - "SELECT %s, rbu_control%s FROM '%q'%s", - zCollist, (bRbuRowid ? ", rbu_rowid" : ""), + "SELECT %s,%s rbu_control%s FROM '%q'%s", + zCollist, + (rbuIsVacuum(p) ? "0 AS " : ""), + zRbuRowid, pIter->zDataTbl, zLimit ) ); @@ -2183,6 +2227,95 @@ static sqlite3 *rbuOpenDbhandle(sqlite3rbu *p, const char *zName){ return db; } +/* +** Free an RbuState object allocated by rbuLoadState(). +*/ +static void rbuFreeState(RbuState *p){ + if( p ){ + sqlite3_free(p->zTbl); + sqlite3_free(p->zIdx); + sqlite3_free(p); + } +} + +/* +** Allocate an RbuState object and load the contents of the rbu_state +** table into it. Return a pointer to the new object. It is the +** responsibility of the caller to eventually free the object using +** sqlite3_free(). +** +** If an error occurs, leave an error code and message in the rbu handle +** and return NULL. +*/ +static RbuState *rbuLoadState(sqlite3rbu *p){ + RbuState *pRet = 0; + sqlite3_stmt *pStmt = 0; + int rc; + int rc2; + + pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState)); + if( pRet==0 ) return 0; + + rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb) + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + switch( sqlite3_column_int(pStmt, 0) ){ + case RBU_STATE_STAGE: + pRet->eStage = sqlite3_column_int(pStmt, 1); + if( pRet->eStage!=RBU_STAGE_OAL + && pRet->eStage!=RBU_STAGE_MOVE + && pRet->eStage!=RBU_STAGE_CKPT + ){ + p->rc = SQLITE_CORRUPT; + } + break; + + case RBU_STATE_TBL: + pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + + case RBU_STATE_IDX: + pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + + case RBU_STATE_ROW: + pRet->nRow = sqlite3_column_int(pStmt, 1); + break; + + case RBU_STATE_PROGRESS: + pRet->nProgress = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_CKPT: + pRet->iWalCksum = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_COOKIE: + pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_OALSZ: + pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_PHASEONESTEP: + pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1); + break; + + default: + rc = SQLITE_CORRUPT; + break; + } + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + + p->rc = rc; + return pRet; +} + + /* ** Open the database handle and attach the RBU database as "rbu". If an ** error occurs, leave an error code and message in the RBU handle. @@ -2191,8 +2324,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ assert( p->rc==SQLITE_OK ); assert( p->dbMain==0 && p->dbRbu==0 ); - p->eStage = 0; - p->dbMain = rbuOpenDbhandle(p, p->zTarget); + /* Open the RBU database */ p->dbRbu = rbuOpenDbhandle(p, p->zRbu); /* If using separate RBU and state databases, attach the state database to @@ -2204,6 +2336,38 @@ static void rbuOpenDatabase(sqlite3rbu *p){ memcpy(p->zStateDb, "main", 4); } + /* If it has not already been created, create the rbu_state table */ + rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb); + + if( rbuIsVacuum(p) ){ + int bOpen = 0; + if( p->eStage>=RBU_STAGE_MOVE ){ + bOpen = 1; + }else{ + RbuState *pState = rbuLoadState(p); + if( pState ){ + bOpen = (pState->eStage>RBU_STAGE_MOVE); + rbuFreeState(pState); + } + } + if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu); + } + + p->eStage = 0; + if( p->dbMain==0 ){ + if( p->zTarget ){ + p->dbMain = rbuOpenDbhandle(p, p->zTarget); + }else{ + char *zTarget = sqlite3_mprintf("%s-vacuum", p->zRbu); + if( zTarget==0 ){ + p->rc = SQLITE_NOMEM; + return; + } + p->dbMain = rbuOpenDbhandle(p, zTarget); + sqlite3_free(zTarget); + } + } + if( p->rc==SQLITE_OK ){ p->rc = sqlite3_create_function(p->dbMain, "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0 @@ -2218,7 +2382,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ if( p->rc==SQLITE_OK ){ p->rc = sqlite3_create_function(p->dbRbu, - "rbu_target_name", 1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 + "rbu_target_name", -1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 ); } @@ -2477,9 +2641,14 @@ static LPWSTR rbuWinUtf8ToUnicode(const char *zFilename){ */ static void rbuMoveOalFile(sqlite3rbu *p){ const char *zBase = sqlite3_db_filename(p->dbMain, "main"); - - char *zWal = sqlite3_mprintf("%s-wal", zBase); char *zOal = sqlite3_mprintf("%s-oal", zBase); + char *zWal; + + if( rbuIsVacuum(p) ){ + zWal = sqlite3_mprintf("%s-wal", sqlite3_db_filename(p->dbRbu, "main")); + }else{ + zWal = sqlite3_mprintf("%s-wal", zBase); + } assert( p->eStage==RBU_STAGE_MOVE ); assert( p->rc==SQLITE_OK && p->zErrmsg==0 ); @@ -2500,8 +2669,8 @@ static void rbuMoveOalFile(sqlite3rbu *p){ /* Re-open the databases. */ rbuObjIterFinalize(&p->objiter); - sqlite3_close(p->dbMain); sqlite3_close(p->dbRbu); + sqlite3_close(p->dbMain); p->dbMain = 0; p->dbRbu = 0; @@ -2663,19 +2832,24 @@ static void rbuStepOneOp(sqlite3rbu *p, int eType){ p->rc = sqlite3_bind_value(pWriter, i+1, pVal); if( p->rc ) return; } - if( pIter->zIdx==0 - && (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) - ){ - /* For a virtual table, or a table with no primary key, the - ** SELECT statement is: - ** - ** SELECT , rbu_control, rbu_rowid FROM .... - ** - ** Hence column_value(pIter->nCol+1). - */ - assertColumnName(pIter->pSelect, pIter->nCol+1, "rbu_rowid"); - pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1); - p->rc = sqlite3_bind_value(pWriter, pIter->nCol+1, pVal); + if( pIter->zIdx==0 ){ + if( pIter->eType==RBU_PK_VTAB + || pIter->eType==RBU_PK_NONE + || (pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p)) + ){ + /* For a virtual table, or a table with no primary key, the + ** SELECT statement is: + ** + ** SELECT , rbu_control, rbu_rowid FROM .... + ** + ** Hence column_value(pIter->nCol+1). + */ + assertColumnName(pIter->pSelect, pIter->nCol+1, + rbuIsVacuum(p) ? "rowid" : "rbu_rowid" + ); + pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1); + p->rc = sqlite3_bind_value(pWriter, pIter->nCol+1, pVal); + } } if( p->rc==SQLITE_OK ){ sqlite3_step(pWriter); @@ -2840,7 +3014,7 @@ int sqlite3rbu_step(sqlite3rbu *p){ /* Clean up the rbu_tmp_xxx table for the previous table. It ** cannot be dropped as there are currently active SQL statements. ** But the contents can be deleted. */ - if( pIter->abIndexed ){ + if( rbuIsVacuum(p)==0 && pIter->abIndexed ){ rbuMPrintfExec(p, p->dbRbu, "DELETE FROM %s.'rbu_tmp_%q'", p->zStateDb, pIter->zDataTbl ); @@ -2927,94 +3101,6 @@ int sqlite3rbu_step(sqlite3rbu *p){ } } -/* -** Free an RbuState object allocated by rbuLoadState(). -*/ -static void rbuFreeState(RbuState *p){ - if( p ){ - sqlite3_free(p->zTbl); - sqlite3_free(p->zIdx); - sqlite3_free(p); - } -} - -/* -** Allocate an RbuState object and load the contents of the rbu_state -** table into it. Return a pointer to the new object. It is the -** responsibility of the caller to eventually free the object using -** sqlite3_free(). -** -** If an error occurs, leave an error code and message in the rbu handle -** and return NULL. -*/ -static RbuState *rbuLoadState(sqlite3rbu *p){ - RbuState *pRet = 0; - sqlite3_stmt *pStmt = 0; - int rc; - int rc2; - - pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState)); - if( pRet==0 ) return 0; - - rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, - sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb) - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - switch( sqlite3_column_int(pStmt, 0) ){ - case RBU_STATE_STAGE: - pRet->eStage = sqlite3_column_int(pStmt, 1); - if( pRet->eStage!=RBU_STAGE_OAL - && pRet->eStage!=RBU_STAGE_MOVE - && pRet->eStage!=RBU_STAGE_CKPT - ){ - p->rc = SQLITE_CORRUPT; - } - break; - - case RBU_STATE_TBL: - pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); - break; - - case RBU_STATE_IDX: - pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); - break; - - case RBU_STATE_ROW: - pRet->nRow = sqlite3_column_int(pStmt, 1); - break; - - case RBU_STATE_PROGRESS: - pRet->nProgress = sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_CKPT: - pRet->iWalCksum = sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_COOKIE: - pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_OALSZ: - pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1); - break; - - case RBU_STATE_PHASEONESTEP: - pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1); - break; - - default: - rc = SQLITE_CORRUPT; - break; - } - } - rc2 = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) rc = rc2; - - p->rc = rc; - return pRet; -} - /* ** Compare strings z1 and z2, returning 0 if they are identical, or non-zero ** otherwise. Either or both argument may be NULL. Two NULL values are @@ -3205,15 +3291,63 @@ static void rbuInitPhaseOneSteps(sqlite3rbu *p){ } /* -** Open and return a new RBU handle. +** The RBU handle passed as the only argument has just been opened and +** the state database is empty. If this RBU handle was opened for an +** RBU vacuum operation, create the schema in the target db. */ -sqlite3rbu *sqlite3rbu_open( +static void rbuCreateTargetSchema(sqlite3rbu *p){ + sqlite3_stmt *pSql = 0; + sqlite3_stmt *pInsert = 0; + int rc2; + + assert( rbuIsVacuum(p) ); + + p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, + "SELECT sql FROM sqlite_master WHERE sql!='' AND rootpage!=0" + " ORDER BY type DESC" + ); + while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){ + const char *zSql = sqlite3_column_text(pSql, 0); + p->rc = sqlite3_exec(p->dbMain, zSql, 0, 0, &p->zErrmsg); + } + rbuFinalize(p, pSql); + if( p->rc!=SQLITE_OK ) return; + + p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=1", 0,0, &p->zErrmsg); + + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, + "SELECT * FROM sqlite_master WHERE rootpage=0 OR rootpage IS NULL" + ); + } + + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbMain, &pInsert, &p->zErrmsg, + "INSERT INTO sqlite_master VALUES(?,?,?,?,?)" + ); + } + + while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){ + int i; + for(i=0; i<5; i++){ + sqlite3_bind_value(pInsert, i+1, sqlite3_column_value(pSql, i)); + } + sqlite3_step(pInsert); + p->rc = sqlite3_reset(pInsert); + } + + rbuFinalize(p, pSql); + rbuFinalize(p, pInsert); +} + + +static sqlite3rbu *openRbuHandle( const char *zTarget, const char *zRbu, const char *zState ){ sqlite3rbu *p; - size_t nTarget = strlen(zTarget); + size_t nTarget = zTarget ? strlen(zTarget) : 0; size_t nRbu = strlen(zRbu); size_t nState = zState ? strlen(zState) : 0; size_t nByte = sizeof(sqlite3rbu) + nTarget+1 + nRbu+1+ nState+1; @@ -3226,22 +3360,24 @@ sqlite3rbu *sqlite3rbu_open( memset(p, 0, sizeof(sqlite3rbu)); rbuCreateVfs(p); - /* Open the target database */ + /* Open the target, RBU and state databases */ if( p->rc==SQLITE_OK ){ - p->zTarget = (char*)&p[1]; - memcpy(p->zTarget, zTarget, nTarget+1); - p->zRbu = &p->zTarget[nTarget+1]; + char *pCsr = (char*)&p[1]; + if( zTarget ){ + p->zTarget = pCsr; + memcpy(p->zTarget, zTarget, nTarget+1); + pCsr += nTarget+1; + } + p->zRbu = pCsr; memcpy(p->zRbu, zRbu, nRbu+1); + pCsr += nRbu+1; if( zState ){ - p->zState = &p->zRbu[nRbu+1]; + p->zState = pCsr; memcpy(p->zState, zState, nState+1); } rbuOpenDatabase(p); } - /* If it has not already been created, create the rbu_state table */ - rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb); - if( p->rc==SQLITE_OK ){ pState = rbuLoadState(p); assert( pState || p->rc!=SQLITE_OK ); @@ -3304,6 +3440,12 @@ sqlite3rbu *sqlite3rbu_open( } } + /* If this is an RBU vacuum operation and the state table was empty + ** when this handle was opened, create the target database schema. */ + if( pState->eStage==0 && rbuIsVacuum(p) ){ + rbuCreateTargetSchema(p); + } + /* Point the object iterator at the first object */ if( p->rc==SQLITE_OK ){ p->rc = rbuObjIterFirst(p, &p->objiter); @@ -3336,6 +3478,28 @@ sqlite3rbu *sqlite3rbu_open( return p; } +/* +** Open and return a new RBU handle. +*/ +sqlite3rbu *sqlite3rbu_open( + const char *zTarget, + const char *zRbu, + const char *zState +){ + /* TODO: Check that zTarget and zRbu are non-NULL */ + return openRbuHandle(zTarget, zRbu, zState); +} + +/* +** Open a handle to begin or resume an RBU VACUUM operation. +*/ +sqlite3rbu *sqlite3rbu_vacuum( + const char *zTarget, + const char *zState +){ + /* TODO: Check that both arguments are non-NULL */ + return openRbuHandle(0, zTarget, zState); +} /* ** Return the database handle used by pRbu. @@ -3391,8 +3555,8 @@ int sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){ rbuObjIterFinalize(&p->objiter); /* Close the open database handle and VFS object. */ - sqlite3_close(p->dbMain); sqlite3_close(p->dbRbu); + sqlite3_close(p->dbMain); rbuDeleteVfs(p); sqlite3_free(p->aBuf); sqlite3_free(p->aFrame); @@ -3594,6 +3758,26 @@ static u32 rbuGetU32(u8 *aBuf){ + ((u32)aBuf[3]); } +/* +** Write an unsigned 32-bit value in big-endian format to the supplied +** buffer. +*/ +static void rbuPutU32(u8 *aBuf, u32 iVal){ + aBuf[0] = (iVal >> 24) & 0xFF; + aBuf[1] = (iVal >> 16) & 0xFF; + aBuf[2] = (iVal >> 8) & 0xFF; + aBuf[3] = (iVal >> 0) & 0xFF; +} + +/* +** Write an unsigned 16-bit value in big-endian format to the supplied +** buffer. +*/ +static void rbuPutU16(u8 *aBuf, u16 iVal){ + aBuf[0] = (iVal >> 8) & 0xFF; + aBuf[1] = (iVal >> 0) & 0xFF; +} + /* ** Read data from an rbuVfs-file. */ @@ -3619,6 +3803,34 @@ static int rbuVfsRead( memset(zBuf, 0, iAmt); }else{ rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst); + /* If this is being called to read the first page of the target + ** database as part of an rbu vacuum operation, synthesize the + ** contents of the first page if it does not yet exist. Otherwise, + ** SQLite will not check for a *-wal file. */ + if( p->pRbu && rbuIsVacuum(p->pRbu) + && rc==SQLITE_IOERR_SHORT_READ && iOfst==0 + && (p->openFlags & SQLITE_OPEN_MAIN_DB) + ){ + sqlite3_file *pFd = 0; + rc = sqlite3_file_control( + p->pRbu->dbRbu, "main", SQLITE_FCNTL_FILE_POINTER, (void*)&pFd + ); + if( rc==SQLITE_OK ){ + rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst); + } + if( rc==SQLITE_OK ){ + rbuPutU32(&zBuf[52], 0); /* largest root page number */ + rbuPutU32(&zBuf[36], 0); /* number of free pages */ + rbuPutU32(&zBuf[32], 0); /* first page on free list trunk */ + rbuPutU32(&zBuf[28], 1); /* size of db file in pages */ + + if( iAmt>100 ){ + assert( iAmt>=101 ); + memset(&zBuf[101], 0, iAmt-101); + rbuPutU16(&zBuf[105], iAmt & 0xFFFF); + } + } + } } if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ /* These look like magic numbers. But they are stable, as they are part @@ -3693,7 +3905,20 @@ static int rbuVfsSync(sqlite3_file *pFile, int flags){ */ static int rbuVfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ rbu_file *p = (rbu_file *)pFile; - return p->pReal->pMethods->xFileSize(p->pReal, pSize); + int rc; + rc = p->pReal->pMethods->xFileSize(p->pReal, pSize); + + /* If this is an RBU vacuum operation and this is the target database, + ** pretend that it has at least one page. Otherwise, SQLite will not + ** check for the existance of a *-wal file. rbuVfsRead() contains + ** similar logic. */ + if( rc==SQLITE_OK && *pSize==0 + && p->pRbu && rbuIsVacuum(p->pRbu) + && (p->openFlags & SQLITE_OPEN_MAIN_DB) + ){ + *pSize = 1024; + } + return rc; } /* diff --git a/ext/rbu/sqlite3rbu.h b/ext/rbu/sqlite3rbu.h index f379bb5b41..b910866293 100644 --- a/ext/rbu/sqlite3rbu.h +++ b/ext/rbu/sqlite3rbu.h @@ -314,6 +314,14 @@ sqlite3rbu *sqlite3rbu_open( const char *zState ); +/* +** Open an RBU handle to perform an RBU vacuum database file zTarget. +*/ +sqlite3rbu *sqlite3rbu_vacuum( + const char *zTarget, + const char *zState +); + /* ** Internally, each RBU connection uses a separate SQLite database ** connection to access the target and rbu update databases. This diff --git a/ext/rbu/test_rbu.c b/ext/rbu/test_rbu.c index 5e8640a9f8..629a33cd0e 100644 --- a/ext/rbu/test_rbu.c +++ b/ext/rbu/test_rbu.c @@ -187,6 +187,34 @@ static int test_sqlite3rbu( return TCL_OK; } +/* +** Tclcmd: sqlite3rbu_vacuum CMD +*/ +static int test_sqlite3rbu_vacuum( + ClientData clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3rbu *pRbu = 0; + const char *zCmd; + const char *zTarget; + const char *zStateDb = 0; + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "NAME TARGET-DB STATE-DB"); + return TCL_ERROR; + } + zCmd = Tcl_GetString(objv[1]); + zTarget = Tcl_GetString(objv[2]); + zStateDb = Tcl_GetString(objv[3]); + + pRbu = sqlite3rbu_vacuum(zTarget, zStateDb); + Tcl_CreateObjCommand(interp, zCmd, test_sqlite3rbu_cmd, (ClientData)pRbu, 0); + Tcl_SetObjResult(interp, objv[1]); + return TCL_OK; +} + /* ** Tclcmd: sqlite3rbu_create_vfs ?-default? NAME PARENT */ @@ -274,6 +302,7 @@ int SqliteRbu_Init(Tcl_Interp *interp){ Tcl_ObjCmdProc *xProc; } aObjCmd[] = { { "sqlite3rbu", test_sqlite3rbu }, + { "sqlite3rbu_vacuum", test_sqlite3rbu_vacuum }, { "sqlite3rbu_create_vfs", test_sqlite3rbu_create_vfs }, { "sqlite3rbu_destroy_vfs", test_sqlite3rbu_destroy_vfs }, { "sqlite3rbu_internal_test", test_sqlite3rbu_internal_test }, diff --git a/manifest b/manifest index c039180bbf..91d7041221 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C CLI\senhancement:\s\sAdd\sthe\s".eqp\sfull"\soption,\sthat\sshows\sboth\sthe\sEXPLAIN\nQUERY\sPLAN\sand\sthe\sEXPLAIN\soutput\sfor\seach\scommand\srun.\s\sAlso\sdisable\nany\s".wheretrace"\sand\s".selecttrace"\swhen\sshowing\sEQP\soutput. -D 2016-04-15T15:03:27.144 +C Add\sthe\sexperimental\ssqlite3rbu_vacuum()\sAPI\sfunction.\sFor\sopening\san\sRBU\shandle\sthat\srebuilds\sa\sdatabase\sfrom\sscratch. +D 2016-04-15T20:46:41.467 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -246,9 +246,10 @@ F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 -F ext/rbu/sqlite3rbu.c 9097f1d95666dbef72ca61d5b6a13a84660735ac -F ext/rbu/sqlite3rbu.h d7cc99350c10134f358fe1a8997d9225b3f712b2 -F ext/rbu/test_rbu.c 3505641a78b723589b8780d5f9b2faeeb73e037d +F ext/rbu/rbuvacuum.test 75b4231f85622859e814c7f028afad0303f72f60 +F ext/rbu/sqlite3rbu.c 79b8be4a0c8276b2b2b24c88edf3944216ccd35b +F ext/rbu/sqlite3rbu.h 1342ab6121e715b8da59ec35c5b5c16060be7a6b +F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 0b870ccb7b58b734a2a8e1e2755a7c0ded070920 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1482,7 +1483,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d23e581351fb8eea28e7b13b3dcadfc817c3a05f -R e880e1f26ac9fef78137656188a6e0aa -U drh -Z dc2294cde78bca18f07e3e2fb59a4dd5 +P 3e217d6265ecd16db783bed7ce1d9d0f9c4828bb +R d2c575fe1215bc1b298a105940b3f837 +T *branch * rbu-vacuum +T *sym-rbu-vacuum * +T -sym-trunk * +U dan +Z af92587ff7c9268b9a60aaab80a1a468 diff --git a/manifest.uuid b/manifest.uuid index 453393af12..66b85c22cc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3e217d6265ecd16db783bed7ce1d9d0f9c4828bb \ No newline at end of file +0216b48f28042ad86711e00802c2da8ce9be3044 \ No newline at end of file From ee65eea4af7d4eec9f0c17d39236faffcb10678c Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 16 Apr 2016 15:03:20 +0000 Subject: [PATCH 02/15] Fix a couple of assert() statements that were failing with OOM error tests. FossilOrigin-Name: 8eb3d7d8360530f364bbbebac53e1f0e6753d924 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/expr.c | 4 ++-- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 91d7041221..eeb8cf07ef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sexperimental\ssqlite3rbu_vacuum()\sAPI\sfunction.\sFor\sopening\san\sRBU\shandle\sthat\srebuilds\sa\sdatabase\sfrom\sscratch. -D 2016-04-15T20:46:41.467 +C Fix\sa\scouple\sof\sassert()\sstatements\sthat\swere\sfailing\swith\sOOM\serror\stests. +D 2016-04-16T15:03:20.537 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -330,7 +330,7 @@ F src/ctime.c 60e135af364d777a9ab41c97e5e89cd224da6198 F src/date.c 1cc9fb516ec9932c6fd4d2a0d2f8bc4480145c39 F src/dbstat.c c845548d4346e606e2f2b7d2e714ace2b8a7dd1b F src/delete.c 78eb999114ec04fcf1b7d123ccedb4b5b734930e -F src/expr.c 17d4e745ef6a3fd2e4ef863f5f9a4912f1ba1198 +F src/expr.c 8796c0739b2ad091e6779253f62aad6e767e2be1 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 4c0bd09e602b8ae8d36d81e31e4872d0b53c87bb F src/func.c 552d300265aed09eea21f68ac742a440550c0062 @@ -1483,10 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3e217d6265ecd16db783bed7ce1d9d0f9c4828bb -R d2c575fe1215bc1b298a105940b3f837 -T *branch * rbu-vacuum -T *sym-rbu-vacuum * -T -sym-trunk * +P 0216b48f28042ad86711e00802c2da8ce9be3044 +R 0603f47373db6aeb2d2c22dfe403e0e2 U dan -Z af92587ff7c9268b9a60aaab80a1a468 +Z 6e5ab10ab6efe323abcf341183f13408 diff --git a/manifest.uuid b/manifest.uuid index 66b85c22cc..773e6bf755 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0216b48f28042ad86711e00802c2da8ce9be3044 \ No newline at end of file +8eb3d7d8360530f364bbbebac53e1f0e6753d924 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 094a454ab1..9e3b664e8d 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2360,7 +2360,7 @@ static void cacheEntryClear(Parse *pParse, struct yColCache *p){ } p->iReg = 0; pParse->nColCache--; - assert( cacheIsValid(pParse) ); + assert( pParse->db->mallocFailed || cacheIsValid(pParse) ); } @@ -2405,7 +2405,7 @@ void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ p->tempReg = 0; p->lru = pParse->iCacheCnt++; pParse->nColCache++; - assert( cacheIsValid(pParse) ); + assert( pParse->db->mallocFailed || cacheIsValid(pParse) ); return; } } From 977cbe78b3945be39c0e353c77e25d6e1d161d35 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 16 Apr 2016 17:53:14 +0000 Subject: [PATCH 03/15] Avoid creating any extra files ("-vacuum") when running an RBU vacuum. Ensure that the OAL file created is "-oal", not "-vacuum-oal". FossilOrigin-Name: dc19aacc7e99213edca9bb57b5c11a8a1ac99113 --- ext/rbu/sqlite3rbu.c | 147 ++++++++++++++++++++++--------------------- manifest | 12 ++-- manifest.uuid | 2 +- 3 files changed, 84 insertions(+), 77 deletions(-) diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index e71690769b..ff01f30725 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -2323,6 +2323,7 @@ static RbuState *rbuLoadState(sqlite3rbu *p){ static void rbuOpenDatabase(sqlite3rbu *p){ assert( p->rc==SQLITE_OK ); assert( p->dbMain==0 && p->dbRbu==0 ); + assert( rbuIsVacuum(p) || p->zTarget!=0 ); /* Open the RBU database */ p->dbRbu = rbuOpenDbhandle(p, p->zRbu); @@ -2355,16 +2356,21 @@ static void rbuOpenDatabase(sqlite3rbu *p){ p->eStage = 0; if( p->dbMain==0 ){ - if( p->zTarget ){ + if( !rbuIsVacuum(p) ){ p->dbMain = rbuOpenDbhandle(p, p->zTarget); }else{ - char *zTarget = sqlite3_mprintf("%s-vacuum", p->zRbu); + char *zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1", p->zRbu); if( zTarget==0 ){ p->rc = SQLITE_NOMEM; return; } p->dbMain = rbuOpenDbhandle(p, zTarget); sqlite3_free(zTarget); + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbMain, + "PRAGMA journal_mode=off; BEGIN EXCLUSIVE; COMMIT;", 0, 0, 0 + ); + } } } @@ -2641,14 +2647,15 @@ static LPWSTR rbuWinUtf8ToUnicode(const char *zFilename){ */ static void rbuMoveOalFile(sqlite3rbu *p){ const char *zBase = sqlite3_db_filename(p->dbMain, "main"); - char *zOal = sqlite3_mprintf("%s-oal", zBase); + const char *zMove = zBase; + char *zOal; char *zWal; if( rbuIsVacuum(p) ){ - zWal = sqlite3_mprintf("%s-wal", sqlite3_db_filename(p->dbRbu, "main")); - }else{ - zWal = sqlite3_mprintf("%s-wal", zBase); + zMove = sqlite3_db_filename(p->dbRbu, "main"); } + zOal = sqlite3_mprintf("%s-oal", zMove); + zWal = sqlite3_mprintf("%s-wal", zMove); assert( p->eStage==RBU_STAGE_MOVE ); assert( p->rc==SQLITE_OK && p->zErrmsg==0 ); @@ -2928,13 +2935,18 @@ static int rbuStep(sqlite3rbu *p){ /* ** Increment the schema cookie of the main database opened by p->dbMain. +** +** Or, if this is an RBU vacuum, set the schema cookie of the main db +** opened by p->dbMain to one more than the schema cookie of the main +** db opened by p->dbRbu. */ static void rbuIncrSchemaCookie(sqlite3rbu *p){ if( p->rc==SQLITE_OK ){ + sqlite3 *dbread = (rbuIsVacuum(p) ? p->dbRbu : p->dbMain); int iCookie = 1000000; sqlite3_stmt *pStmt; - p->rc = prepareAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, + p->rc = prepareAndCollectError(dbread, &pStmt, &p->zErrmsg, "PRAGMA schema_version" ); if( p->rc==SQLITE_OK ){ @@ -3408,6 +3420,7 @@ static sqlite3rbu *openRbuHandle( } if( p->rc==SQLITE_OK + && !rbuIsVacuum(p) && (p->eStage==RBU_STAGE_OAL || p->eStage==RBU_STAGE_MOVE) && pState->eStage!=0 && p->pTargetFd->iCookie!=pState->iCookie ){ @@ -3803,34 +3816,6 @@ static int rbuVfsRead( memset(zBuf, 0, iAmt); }else{ rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst); - /* If this is being called to read the first page of the target - ** database as part of an rbu vacuum operation, synthesize the - ** contents of the first page if it does not yet exist. Otherwise, - ** SQLite will not check for a *-wal file. */ - if( p->pRbu && rbuIsVacuum(p->pRbu) - && rc==SQLITE_IOERR_SHORT_READ && iOfst==0 - && (p->openFlags & SQLITE_OPEN_MAIN_DB) - ){ - sqlite3_file *pFd = 0; - rc = sqlite3_file_control( - p->pRbu->dbRbu, "main", SQLITE_FCNTL_FILE_POINTER, (void*)&pFd - ); - if( rc==SQLITE_OK ){ - rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst); - } - if( rc==SQLITE_OK ){ - rbuPutU32(&zBuf[52], 0); /* largest root page number */ - rbuPutU32(&zBuf[36], 0); /* number of free pages */ - rbuPutU32(&zBuf[32], 0); /* first page on free list trunk */ - rbuPutU32(&zBuf[28], 1); /* size of db file in pages */ - - if( iAmt>100 ){ - assert( iAmt>=101 ); - memset(&zBuf[101], 0, iAmt-101); - rbuPutU16(&zBuf[105], iAmt & 0xFFFF); - } - } - } } if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ /* These look like magic numbers. But they are stable, as they are part @@ -3838,6 +3823,13 @@ static int rbuVfsRead( u8 *pBuf = (u8*)zBuf; p->iCookie = rbuGetU32(&pBuf[24]); p->iWriteVer = pBuf[19]; + if( pRbu && rbuIsVacuum(p->pRbu) ){ + rbu_file *pRbuFd = 0; + sqlite3_file_control(pRbu->dbRbu, "main", + SQLITE_FCNTL_FILE_POINTER, (void*)&pRbuFd + ); + rbuPutU32(&pBuf[24], pRbuFd->iCookie+1); + } } } return rc; @@ -3905,20 +3897,7 @@ static int rbuVfsSync(sqlite3_file *pFile, int flags){ */ static int rbuVfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ rbu_file *p = (rbu_file *)pFile; - int rc; - rc = p->pReal->pMethods->xFileSize(p->pReal, pSize); - - /* If this is an RBU vacuum operation and this is the target database, - ** pretend that it has at least one page. Otherwise, SQLite will not - ** check for the existance of a *-wal file. rbuVfsRead() contains - ** similar logic. */ - if( rc==SQLITE_OK && *pSize==0 - && p->pRbu && rbuIsVacuum(p->pRbu) - && (p->openFlags & SQLITE_OPEN_MAIN_DB) - ){ - *pSize = 1024; - } - return rc; + return p->pReal->pMethods->xFileSize(p->pReal, pSize); } /* @@ -4156,6 +4135,33 @@ static rbu_file *rbuFindMaindb(rbu_vfs *pRbuVfs, const char *zWal){ return pDb; } +/* +** A main database named zName has just been opened. The following +** function returns a pointer to a buffer owned by SQLite that contains +** the name of the *-wal file this db connection will use. SQLite +** happens to pass a pointer to this buffer when using xAccess() +** or xOpen() to operate on the *-wal file. +*/ +static const char *rbuMainToWal(const char *zName, int flags){ + int n = (int)strlen(zName); + const char *z = &zName[n]; + if( flags & SQLITE_OPEN_URI ){ + int odd = 0; + while( 1 ){ + if( z[0]==0 ){ + odd = 1 - odd; + if( odd && z[1]==0 ) break; + } + z++; + } + z += 2; + }else{ + while( *z==0 ) z++; + } + z += (n + 8 + 1); + return z; +} + /* ** Open an rbu file handle. */ @@ -4191,6 +4197,7 @@ static int rbuVfsOpen( rbu_file *pFd = (rbu_file *)pFile; int rc = SQLITE_OK; const char *zOpen = zName; + int oflags = flags; memset(pFd, 0, sizeof(rbu_file)); pFd->pReal = (sqlite3_file*)&pFd[1]; @@ -4203,23 +4210,7 @@ static int rbuVfsOpen( ** the name of the *-wal file this db connection will use. SQLite ** happens to pass a pointer to this buffer when using xAccess() ** or xOpen() to operate on the *-wal file. */ - int n = (int)strlen(zName); - const char *z = &zName[n]; - if( flags & SQLITE_OPEN_URI ){ - int odd = 0; - while( 1 ){ - if( z[0]==0 ){ - odd = 1 - odd; - if( odd && z[1]==0 ) break; - } - z++; - } - z += 2; - }else{ - while( *z==0 ) z++; - } - z += (n + 8 + 1); - pFd->zWal = z; + pFd->zWal = rbuMainToWal(zName, flags); } else if( flags & SQLITE_OPEN_WAL ){ rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName); @@ -4229,10 +4220,17 @@ static int rbuVfsOpen( ** code ensures that the string passed to xOpen() is terminated by a ** pair of '\0' bytes in case the VFS attempts to extract a URI ** parameter from it. */ - size_t nCopy = strlen(zName); - char *zCopy = sqlite3_malloc64(nCopy+2); + const char *zBase = zName; + size_t nCopy; + char *zCopy; + if( rbuIsVacuum(pDb->pRbu) ){ + zBase = sqlite3_db_filename(pDb->pRbu->dbRbu, "main"); + zBase = rbuMainToWal(zBase, SQLITE_OPEN_URI); + } + nCopy = strlen(zBase); + zCopy = sqlite3_malloc64(nCopy+2); if( zCopy ){ - memcpy(zCopy, zName, nCopy); + memcpy(zCopy, zBase, nCopy); zCopy[nCopy-3] = 'o'; zCopy[nCopy] = '\0'; zCopy[nCopy+1] = '\0'; @@ -4247,8 +4245,17 @@ static int rbuVfsOpen( } } + if( oflags & SQLITE_OPEN_MAIN_DB + && sqlite3_uri_boolean(zName, "rbu_memory", 0) + ){ + assert( oflags & SQLITE_OPEN_MAIN_DB ); + oflags = SQLITE_OPEN_TEMP_DB | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | + SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE; + zOpen = 0; + } + if( rc==SQLITE_OK ){ - rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, flags, pOutFlags); + rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, oflags, pOutFlags); } if( pFd->pReal->pMethods ){ /* The xOpen() operation has succeeded. Set the sqlite3_file.pMethods diff --git a/manifest b/manifest index eeb8cf07ef..0e404f7022 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scouple\sof\sassert()\sstatements\sthat\swere\sfailing\swith\sOOM\serror\stests. -D 2016-04-16T15:03:20.537 +C Avoid\screating\sany\sextra\sfiles\s("-vacuum")\swhen\srunning\san\sRBU\svacuum.\sEnsure\sthat\sthe\sOAL\sfile\screated\sis\s"-oal",\snot\s"-vacuum-oal". +D 2016-04-16T17:53:14.824 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -247,7 +247,7 @@ F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test 75b4231f85622859e814c7f028afad0303f72f60 -F ext/rbu/sqlite3rbu.c 79b8be4a0c8276b2b2b24c88edf3944216ccd35b +F ext/rbu/sqlite3rbu.c 471b4055618473612e9ae7d7e4f1922559b59aaf F ext/rbu/sqlite3rbu.h 1342ab6121e715b8da59ec35c5b5c16060be7a6b F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1483,7 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0216b48f28042ad86711e00802c2da8ce9be3044 -R 0603f47373db6aeb2d2c22dfe403e0e2 +P 8eb3d7d8360530f364bbbebac53e1f0e6753d924 +R 4d03a7f5bef5f70852d283bb46ff80e2 U dan -Z 6e5ab10ab6efe323abcf341183f13408 +Z 0a7d529e6f2d77bae387935d2996f1da diff --git a/manifest.uuid b/manifest.uuid index 773e6bf755..56172c7c76 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8eb3d7d8360530f364bbbebac53e1f0e6753d924 \ No newline at end of file +dc19aacc7e99213edca9bb57b5c11a8a1ac99113 \ No newline at end of file From 2bb5f4478ea2c4fc2e5889a1586153e394f37ffd Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 16 Apr 2016 19:02:53 +0000 Subject: [PATCH 04/15] Changes so that rbu vacuum works with zipvfs databases. FossilOrigin-Name: bae7f875f476f6e01112751cb404fef42ba0a01c --- ext/rbu/sqlite3rbu.c | 32 ++++++++++++++------------------ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 21 insertions(+), 25 deletions(-) diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index ff01f30725..280b64bb2a 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -2213,11 +2213,15 @@ static int rbuGetUpdateStmt( return p->rc; } -static sqlite3 *rbuOpenDbhandle(sqlite3rbu *p, const char *zName){ +static sqlite3 *rbuOpenDbhandle( + sqlite3rbu *p, + const char *zName, + int bUseVfs +){ sqlite3 *db = 0; if( p->rc==SQLITE_OK ){ const int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_URI; - p->rc = sqlite3_open_v2(zName, &db, flags, p->zVfsName); + p->rc = sqlite3_open_v2(zName, &db, flags, bUseVfs ? p->zVfsName : 0); if( p->rc ){ p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); sqlite3_close(db); @@ -2326,7 +2330,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ assert( rbuIsVacuum(p) || p->zTarget!=0 ); /* Open the RBU database */ - p->dbRbu = rbuOpenDbhandle(p, p->zRbu); + p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1); /* If using separate RBU and state databases, attach the state database to ** the RBU db handle now. */ @@ -2351,24 +2355,26 @@ static void rbuOpenDatabase(sqlite3rbu *p){ rbuFreeState(pState); } } - if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu); + if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, 1); } p->eStage = 0; if( p->dbMain==0 ){ if( !rbuIsVacuum(p) ){ - p->dbMain = rbuOpenDbhandle(p, p->zTarget); + p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1); }else{ + int frc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_ZIPVFS, 0); char *zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1", p->zRbu); if( zTarget==0 ){ p->rc = SQLITE_NOMEM; return; } - p->dbMain = rbuOpenDbhandle(p, zTarget); + p->dbMain = rbuOpenDbhandle(p, zTarget, frc!=SQLITE_OK); sqlite3_free(zTarget); if( p->rc==SQLITE_OK ){ p->rc = sqlite3_exec(p->dbMain, - "PRAGMA journal_mode=off; BEGIN EXCLUSIVE; COMMIT;", 0, 0, 0 + "PRAGMA journal_mode=off; PRAGMA zipvfs_journal_mode = off;" + "BEGIN EXCLUSIVE; COMMIT;", 0, 0, 0 ); } } @@ -3310,7 +3316,6 @@ static void rbuInitPhaseOneSteps(sqlite3rbu *p){ static void rbuCreateTargetSchema(sqlite3rbu *p){ sqlite3_stmt *pSql = 0; sqlite3_stmt *pInsert = 0; - int rc2; assert( rbuIsVacuum(p) ); @@ -3319,7 +3324,7 @@ static void rbuCreateTargetSchema(sqlite3rbu *p){ " ORDER BY type DESC" ); while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){ - const char *zSql = sqlite3_column_text(pSql, 0); + const char *zSql = (const char*)sqlite3_column_text(pSql, 0); p->rc = sqlite3_exec(p->dbMain, zSql, 0, 0, &p->zErrmsg); } rbuFinalize(p, pSql); @@ -3782,15 +3787,6 @@ static void rbuPutU32(u8 *aBuf, u32 iVal){ aBuf[3] = (iVal >> 0) & 0xFF; } -/* -** Write an unsigned 16-bit value in big-endian format to the supplied -** buffer. -*/ -static void rbuPutU16(u8 *aBuf, u16 iVal){ - aBuf[0] = (iVal >> 8) & 0xFF; - aBuf[1] = (iVal >> 0) & 0xFF; -} - /* ** Read data from an rbuVfs-file. */ diff --git a/manifest b/manifest index 0e404f7022..bcd6ca601c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\screating\sany\sextra\sfiles\s("-vacuum")\swhen\srunning\san\sRBU\svacuum.\sEnsure\sthat\sthe\sOAL\sfile\screated\sis\s"-oal",\snot\s"-vacuum-oal". -D 2016-04-16T17:53:14.824 +C Changes\sso\sthat\srbu\svacuum\sworks\swith\szipvfs\sdatabases. +D 2016-04-16T19:02:53.248 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -247,7 +247,7 @@ F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test 75b4231f85622859e814c7f028afad0303f72f60 -F ext/rbu/sqlite3rbu.c 471b4055618473612e9ae7d7e4f1922559b59aaf +F ext/rbu/sqlite3rbu.c 721c6c116018b5d02f6318b6bbb7834098bc6a07 F ext/rbu/sqlite3rbu.h 1342ab6121e715b8da59ec35c5b5c16060be7a6b F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1483,7 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8eb3d7d8360530f364bbbebac53e1f0e6753d924 -R 4d03a7f5bef5f70852d283bb46ff80e2 +P dc19aacc7e99213edca9bb57b5c11a8a1ac99113 +R 51e4311e92cd324079b6a63abfa03656 U dan -Z 0a7d529e6f2d77bae387935d2996f1da +Z c0a4c5c182b536f7f5ddad73d82153a8 diff --git a/manifest.uuid b/manifest.uuid index 56172c7c76..3a2c90ce4f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dc19aacc7e99213edca9bb57b5c11a8a1ac99113 \ No newline at end of file +bae7f875f476f6e01112751cb404fef42ba0a01c \ No newline at end of file From 303a094466c7c5bb76e80cac61c79123b4b95c41 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 18 Apr 2016 09:17:05 +0000 Subject: [PATCH 05/15] Add the -vacuum switch to the "rbu" demonstration program. FossilOrigin-Name: 9a0078a538c7e73a009960347b8953c5af99fefd --- ext/rbu/rbu.c | 56 ++++++++++++++++++++++++++++++++++++--------------- manifest | 12 +++++------ manifest.uuid | 2 +- 3 files changed, 47 insertions(+), 23 deletions(-) diff --git a/ext/rbu/rbu.c b/ext/rbu/rbu.c index 018296b8dd..a3efad0d44 100644 --- a/ext/rbu/rbu.c +++ b/ext/rbu/rbu.c @@ -24,12 +24,22 @@ */ void usage(const char *zArgv0){ fprintf(stderr, -"Usage: %s [-step NSTEP] TARGET-DB RBU-DB\n" +"Usage: %s ?OPTIONS? TARGET-DB RBU-DB\n" "\n" -" Argument RBU-DB must be an RBU database containing an update suitable for\n" -" target database TARGET-DB. If NSTEP is set to less than or equal to zero\n" -" (the default value), this program attempts to apply the entire update to\n" -" the target database.\n" +"Where options are:\n" +"\n" +" -step NSTEP\n" +" -vacuum\n" +"\n" +" If the -vacuum switch is not present, argument RBU-DB must be an RBU\n" +" database containing an update suitable for target database TARGET-DB.\n" +" Or, if -vacuum is specified, then TARGET-DB is a database to vacuum using\n" +" RBU, and RBU-DB is used as the state database for the vacuum (refer to\n" +" API documentation for details).\n" +"\n" +" If NSTEP is set to less than or equal to zero (the default value), this \n" +" program attempts to perform the entire update or vacuum operation before\n" +" exiting\n" "\n" " If NSTEP is greater than zero, then a maximum of NSTEP calls are made\n" " to sqlite3rbu_step(). If the RBU update has not been completely applied\n" @@ -69,29 +79,43 @@ int main(int argc, char **argv){ char *zErrmsg; /* Error message, if any */ sqlite3rbu *pRbu; /* RBU handle */ int nStep = 0; /* Maximum number of step() calls */ + int bVacuum = 0; int rc; sqlite3_int64 nProgress = 0; + int nArg = argc-2; - /* Process command line arguments. Following this block local variables - ** zTarget, zRbu and nStep are all set. */ - if( argc==5 ){ - size_t nArg1 = strlen(argv[1]); - if( nArg1>5 || nArg1<2 || memcmp("-step", argv[1], nArg1) ) usage(argv[0]); - nStep = atoi(argv[2]); - }else if( argc!=3 ){ - usage(argv[0]); + if( argc<3 ) usage(argv[0]); + for(i=1; i1 && nArg<=8 && 0==memcmp(zArg, "-vacuum", nArg) ){ + bVacuum = 1; + }else if( nArg>1 && nArg<=5 && 0==memcmp(zArg, "-step", nArg) && i Date: Mon, 18 Apr 2016 18:18:18 +0000 Subject: [PATCH 06/15] Fix some zipvfs related problems in RBU vacuum. FossilOrigin-Name: d76f4aaa4caab713460421bd27365a82ac986c20 --- ext/rbu/sqlite3rbu.c | 81 +++++++++++++++++++++++++++++++++++++------- manifest | 12 +++---- manifest.uuid | 2 +- 3 files changed, 76 insertions(+), 19 deletions(-) diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 280b64bb2a..f98a836852 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -190,6 +190,8 @@ typedef sqlite3_int64 i64; #define WAL_LOCK_CKPT 1 #define WAL_LOCK_READ0 3 +#define SQLITE_FCNTL_RBUCNT 5149216 + /* ** A structure to store values read from the rbu_state table in memory. */ @@ -393,6 +395,7 @@ struct rbu_file { int openFlags; /* Flags this file was opened with */ u32 iCookie; /* Cookie value for main db files */ u8 iWriteVer; /* "write-version" value for main db files */ + u8 bNolock; int nShm; /* Number of entries in apShm[] array */ char **apShm; /* Array of mmap'd *-shm regions */ @@ -2325,6 +2328,7 @@ static RbuState *rbuLoadState(sqlite3rbu *p){ ** error occurs, leave an error code and message in the RBU handle. */ static void rbuOpenDatabase(sqlite3rbu *p){ + int nRbu = 0; assert( p->rc==SQLITE_OK ); assert( p->dbMain==0 && p->dbRbu==0 ); assert( rbuIsVacuum(p) || p->zTarget!=0 ); @@ -2332,6 +2336,10 @@ static void rbuOpenDatabase(sqlite3rbu *p){ /* Open the RBU database */ p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1); + if( rbuIsVacuum(p) ){ + sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, &nRbu); + } + /* If using separate RBU and state databases, attach the state database to ** the RBU db handle now. */ if( p->zState ){ @@ -2355,7 +2363,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ rbuFreeState(pState); } } - if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, 1); + if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, nRbu<=1); } p->eStage = 0; @@ -2363,20 +2371,13 @@ static void rbuOpenDatabase(sqlite3rbu *p){ if( !rbuIsVacuum(p) ){ p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1); }else{ - int frc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_ZIPVFS, 0); char *zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1", p->zRbu); if( zTarget==0 ){ p->rc = SQLITE_NOMEM; return; } - p->dbMain = rbuOpenDbhandle(p, zTarget, frc!=SQLITE_OK); + p->dbMain = rbuOpenDbhandle(p, zTarget, nRbu<=1); sqlite3_free(zTarget); - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(p->dbMain, - "PRAGMA journal_mode=off; PRAGMA zipvfs_journal_mode = off;" - "BEGIN EXCLUSIVE; COMMIT;", 0, 0, 0 - ); - } } } @@ -3445,7 +3446,7 @@ static sqlite3rbu *openRbuHandle( ** created at this point. */ p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(p->dbRbu, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); + p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, &p->zErrmsg); } /* Check if the main database is a zipvfs db. If it is, set the upper @@ -3787,6 +3788,11 @@ static void rbuPutU32(u8 *aBuf, u32 iVal){ aBuf[3] = (iVal >> 0) & 0xFF; } +static void rbuPutU16(u8 *aBuf, u16 iVal){ + aBuf[0] = (iVal >> 8) & 0xFF; + aBuf[1] = (iVal >> 0) & 0xFF; +} + /* ** Read data from an rbuVfs-file. */ @@ -3812,6 +3818,37 @@ static int rbuVfsRead( memset(zBuf, 0, iAmt); }else{ rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst); +#if 1 + /* If this is being called to read the first page of the target + ** database as part of an rbu vacuum operation, synthesize the + ** contents of the first page if it does not yet exist. Otherwise, + ** SQLite will not check for a *-wal file. */ + if( p->pRbu && rbuIsVacuum(p->pRbu) + && rc==SQLITE_IOERR_SHORT_READ && iOfst==0 + && (p->openFlags & SQLITE_OPEN_MAIN_DB) + ){ + sqlite3_file *pFd = 0; + rc = sqlite3_file_control( + p->pRbu->dbRbu, "main", SQLITE_FCNTL_FILE_POINTER, (void*)&pFd + ); + if( rc==SQLITE_OK ){ + rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst); + } + if( rc==SQLITE_OK ){ + u8 *aBuf = (u8*)zBuf; + rbuPutU32(&aBuf[52], 0); /* largest root page number */ + rbuPutU32(&aBuf[36], 0); /* number of free pages */ + rbuPutU32(&aBuf[32], 0); /* first page on free list trunk */ + rbuPutU32(&aBuf[28], 1); /* size of db file in pages */ + + if( iAmt>100 ){ + assert( iAmt>=101 ); + memset(&aBuf[101], 0, iAmt-101); + rbuPutU16(&aBuf[105], iAmt & 0xFFFF); + } + } + } +#endif } if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ /* These look like magic numbers. But they are stable, as they are part @@ -3893,7 +3930,20 @@ static int rbuVfsSync(sqlite3_file *pFile, int flags){ */ static int rbuVfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ rbu_file *p = (rbu_file *)pFile; - return p->pReal->pMethods->xFileSize(p->pReal, pSize); + int rc; + rc = p->pReal->pMethods->xFileSize(p->pReal, pSize); + + /* If this is an RBU vacuum operation and this is the target database, + ** pretend that it has at least one page. Otherwise, SQLite will not + ** check for the existance of a *-wal file. rbuVfsRead() contains + ** similar logic. */ + if( rc==SQLITE_OK && *pSize==0 + && p->pRbu && rbuIsVacuum(p->pRbu) + && (p->openFlags & SQLITE_OPEN_MAIN_DB) + ){ + *pSize = 1024; + } + return rc; } /* @@ -3905,7 +3955,9 @@ static int rbuVfsLock(sqlite3_file *pFile, int eLock){ int rc = SQLITE_OK; assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) ); - if( pRbu && eLock==SQLITE_LOCK_EXCLUSIVE && pRbu->eStage!=RBU_STAGE_DONE ){ + if( eLock==SQLITE_LOCK_EXCLUSIVE + && (p->bNolock || (pRbu && pRbu->eStage!=RBU_STAGE_DONE)) + ){ /* Do not allow EXCLUSIVE locks. Preventing SQLite from taking this ** prevents it from checkpointing the database from sqlite3_close(). */ rc = SQLITE_BUSY; @@ -3968,6 +4020,11 @@ static int rbuVfsFileControl(sqlite3_file *pFile, int op, void *pArg){ } return rc; } + else if( op==SQLITE_FCNTL_RBUCNT ){ + int *pnRbu = (int*)pArg; + (*pnRbu)++; + p->bNolock = 1; + } rc = xControl(p->pReal, op, pArg); if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ diff --git a/manifest b/manifest index 6840069cfa..d23cec1ffd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s-vacuum\sswitch\sto\sthe\s"rbu"\sdemonstration\sprogram. -D 2016-04-18T09:17:05.073 +C Fix\ssome\szipvfs\srelated\sproblems\sin\sRBU\svacuum. +D 2016-04-18T18:18:18.881 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -247,7 +247,7 @@ F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test 75b4231f85622859e814c7f028afad0303f72f60 -F ext/rbu/sqlite3rbu.c 721c6c116018b5d02f6318b6bbb7834098bc6a07 +F ext/rbu/sqlite3rbu.c 372ed3aaa396e3edfacdb8976d49aafdbe330cfa F ext/rbu/sqlite3rbu.h 1342ab6121e715b8da59ec35c5b5c16060be7a6b F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1483,7 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bae7f875f476f6e01112751cb404fef42ba0a01c -R 446c70a62698b2fec86a1f84d13dd5af +P 9a0078a538c7e73a009960347b8953c5af99fefd +R dc7cff23cf20a157412c6c371d56e10b U dan -Z be7942fa7aee76e8d89cc36d886361a8 +Z 72aacc0891d9c46e37fe5c65eb96b678 diff --git a/manifest.uuid b/manifest.uuid index c5a45627c9..e3fa7de1a3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9a0078a538c7e73a009960347b8953c5af99fefd \ No newline at end of file +d76f4aaa4caab713460421bd27365a82ac986c20 \ No newline at end of file From 58e3812988e4eea3140ae365d0325254555c71bc Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 18 Apr 2016 21:00:01 +0000 Subject: [PATCH 07/15] Another fix to rbu vacuum for a zipvfs case. FossilOrigin-Name: 29407d70e44ad9ea5ddaf1011d0e212b602a3ddf --- ext/rbu/sqlite3rbu.c | 30 ++++++++++++++++-------------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index f98a836852..60773b79f8 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -370,6 +370,10 @@ struct sqlite3rbu { int pgsz; u8 *aBuf; i64 iWalCksum; + + /* Used in RBU vacuum mode only */ + int nRbu; /* Number of RBU VFS in the stack */ + rbu_file *pRbuFd; /* Fd for main db of dbRbu */ }; /* @@ -2328,7 +2332,6 @@ static RbuState *rbuLoadState(sqlite3rbu *p){ ** error occurs, leave an error code and message in the RBU handle. */ static void rbuOpenDatabase(sqlite3rbu *p){ - int nRbu = 0; assert( p->rc==SQLITE_OK ); assert( p->dbMain==0 && p->dbRbu==0 ); assert( rbuIsVacuum(p) || p->zTarget!=0 ); @@ -2337,7 +2340,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1); if( rbuIsVacuum(p) ){ - sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, &nRbu); + sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); } /* If using separate RBU and state databases, attach the state database to @@ -2354,6 +2357,9 @@ static void rbuOpenDatabase(sqlite3rbu *p){ if( rbuIsVacuum(p) ){ int bOpen = 0; + p->nRbu = 0; + p->pRbuFd = 0; + sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); if( p->eStage>=RBU_STAGE_MOVE ){ bOpen = 1; }else{ @@ -2363,7 +2369,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ rbuFreeState(pState); } } - if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, nRbu<=1); + if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, p->nRbu<=1); } p->eStage = 0; @@ -2376,7 +2382,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ p->rc = SQLITE_NOMEM; return; } - p->dbMain = rbuOpenDbhandle(p, zTarget, nRbu<=1); + p->dbMain = rbuOpenDbhandle(p, zTarget, p->nRbu<=1); sqlite3_free(zTarget); } } @@ -3823,17 +3829,12 @@ static int rbuVfsRead( ** database as part of an rbu vacuum operation, synthesize the ** contents of the first page if it does not yet exist. Otherwise, ** SQLite will not check for a *-wal file. */ - if( p->pRbu && rbuIsVacuum(p->pRbu) + if( pRbu && rbuIsVacuum(pRbu) && rc==SQLITE_IOERR_SHORT_READ && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ - sqlite3_file *pFd = 0; - rc = sqlite3_file_control( - p->pRbu->dbRbu, "main", SQLITE_FCNTL_FILE_POINTER, (void*)&pFd - ); - if( rc==SQLITE_OK ){ - rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst); - } + sqlite3_file *pFd = (sqlite3_file*)pRbu->pRbuFd; + rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst); if( rc==SQLITE_OK ){ u8 *aBuf = (u8*)zBuf; rbuPutU32(&aBuf[52], 0); /* largest root page number */ @@ -4021,8 +4022,9 @@ static int rbuVfsFileControl(sqlite3_file *pFile, int op, void *pArg){ return rc; } else if( op==SQLITE_FCNTL_RBUCNT ){ - int *pnRbu = (int*)pArg; - (*pnRbu)++; + sqlite3rbu *pRbu = (sqlite3rbu*)pArg; + pRbu->nRbu++; + pRbu->pRbuFd = p; p->bNolock = 1; } diff --git a/manifest b/manifest index d23cec1ffd..068554e465 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\szipvfs\srelated\sproblems\sin\sRBU\svacuum. -D 2016-04-18T18:18:18.881 +C Another\sfix\sto\srbu\svacuum\sfor\sa\szipvfs\scase. +D 2016-04-18T21:00:01.258 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -247,7 +247,7 @@ F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test 75b4231f85622859e814c7f028afad0303f72f60 -F ext/rbu/sqlite3rbu.c 372ed3aaa396e3edfacdb8976d49aafdbe330cfa +F ext/rbu/sqlite3rbu.c ac4621fc62e0bc3e9e92fceb483c0207af25addf F ext/rbu/sqlite3rbu.h 1342ab6121e715b8da59ec35c5b5c16060be7a6b F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1483,7 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9a0078a538c7e73a009960347b8953c5af99fefd -R dc7cff23cf20a157412c6c371d56e10b +P d76f4aaa4caab713460421bd27365a82ac986c20 +R 02609be0d1cf35faad7663d479babac3 U dan -Z 72aacc0891d9c46e37fe5c65eb96b678 +Z ecb53ed56b3181cf4b46470b55ef0f36 diff --git a/manifest.uuid b/manifest.uuid index e3fa7de1a3..74ea0959b8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d76f4aaa4caab713460421bd27365a82ac986c20 \ No newline at end of file +29407d70e44ad9ea5ddaf1011d0e212b602a3ddf \ No newline at end of file From bff4b6367c4a216c71f3e6f7ef6c613da82f6341 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 19 Apr 2016 16:20:24 +0000 Subject: [PATCH 08/15] Detect attempts to use rbu vacuum on a wal mode database (not allowed). And attempts to write to a database in the middle of an rbu vacuum (which prevents the vacuum from resuming). FossilOrigin-Name: 00b2f4b09ffca5156e43c4db2bfe0b2c979c49b0 --- ext/rbu/rbuvacuum.test | 82 +++++++++++++++++++++++++++++++++++------- ext/rbu/sqlite3rbu.c | 42 +++++++++++----------- manifest | 14 ++++---- manifest.uuid | 2 +- 4 files changed, 99 insertions(+), 41 deletions(-) diff --git a/ext/rbu/rbuvacuum.test b/ext/rbu/rbuvacuum.test index 045b2d1bb7..4380b78fbf 100644 --- a/ext/rbu/rbuvacuum.test +++ b/ext/rbu/rbuvacuum.test @@ -17,15 +17,15 @@ source [file join [file dirname [info script]] rbu_common.tcl] set ::testprefix rbuvacuum -proc do_rbu_vacuum_test {tn} { +proc do_rbu_vacuum_test {tn step} { uplevel [list do_test $tn.1 { forcedelete state.db - if {$::step==0} { sqlite3rbu_vacuum rbu test.db state.db } + if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db } while 1 { - if {$::step==1} { sqlite3rbu_vacuum rbu test.db state.db } + if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db } set rc [rbu step] if {$rc!="SQLITE_OK"} break - if {$::step==1} { rbu close } + if {$step==1} { rbu close } } rbu close } {SQLITE_DONE}] @@ -49,7 +49,7 @@ foreach step {0 1} { INSERT INTO t1 VALUES(7, 8, 9); PRAGMA integrity_check; } {ok} - do_rbu_vacuum_test 1.1 + do_rbu_vacuum_test 1.1 $step # A vacuum that actually reclaims space. do_execsql_test 1.2.1 { @@ -64,7 +64,7 @@ foreach step {0 1} { DELETE FROM t1 WHERE rowid BETWEEN 8 AND 11; PRAGMA page_count; } {12} - do_rbu_vacuum_test 1.2.3 + do_rbu_vacuum_test 1.2.3 $step do_execsql_test 1.2.4 { PRAGMA page_count; } {3} @@ -82,7 +82,7 @@ foreach step {0 1} { DELETE FROM t1 WHERE rowid BETWEEN 12 AND 15; PRAGMA page_count; } {18} - do_rbu_vacuum_test 1.3.3 + do_rbu_vacuum_test 1.3.3 $step do_execsql_test 1.3.4 { PRAGMA page_count; } {5} @@ -102,7 +102,7 @@ foreach step {0 1} { DELETE FROM t2 WHERE b BETWEEN 2 AND 7; PRAGMA page_count; } {20} - do_rbu_vacuum_test 1.4.2 + do_rbu_vacuum_test 1.4.2 $step do_execsql_test 1.4.3 { PRAGMA page_count; } {10} @@ -120,7 +120,7 @@ foreach step {0 1} { DELETE FROM t2 WHERE b BETWEEN 8 AND 12; PRAGMA page_count; } {35} - do_rbu_vacuum_test 1.4.2 + do_rbu_vacuum_test 1.4.2 $step do_execsql_test 1.4.3 { PRAGMA page_count; } {15} @@ -135,7 +135,7 @@ foreach step {0 1} { INSERT INTO t3 VALUES('d', 'e', 'f'); INSERT INTO t3 VALUES('g', 'h', 'i'); } - do_rbu_vacuum_test 1.5.2 + do_rbu_vacuum_test 1.5.2 $step do_execsql_test 1.5.3 { SELECT * FROM t3 } {a b c d e f g h i} @@ -146,7 +146,7 @@ foreach step {0 1} { INSERT INTO t3 VALUES('j', 'k', 'l'); DELETE FROM t3 WHERE a = 'g'; } - do_rbu_vacuum_test 1.5.5 + do_rbu_vacuum_test 1.5.5 $step do_execsql_test 1.5.6 { SELECT rowid, * FROM t3 ORDER BY b } {1 a b c 2 d e f 4 j k l} @@ -157,7 +157,7 @@ foreach step {0 1} { INSERT INTO t4 VALUES('d', 'e', 'f'); INSERT INTO t4 VALUES('g', 'h', 'i'); } - do_rbu_vacuum_test 1.6.2 + do_rbu_vacuum_test 1.6.2 $step do_execsql_test 1.6.3 { SELECT * FROM t4 } {a b c d e f g h i} @@ -169,13 +169,69 @@ foreach step {0 1} { INSERT INTO t4 VALUES('j', 'k', 'l'); DELETE FROM t4 WHERE a='g'; } - do_rbu_vacuum_test 1.6.5 + do_rbu_vacuum_test 1.6.5 $step do_execsql_test 1.6.6 { SELECT * FROM t4 ORDER BY b } {a b c d e f j k l} } +#------------------------------------------------------------------------- +# Test some error cases: +# +# 2.1.* the db being vacuumed being in wal mode already. +# 2.2.* database modified mid vacuum. +# +reset_db +do_execsql_test 2.1.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t1 VALUES(7, 8); + PRAGMA journal_mode = wal; + INSERT INTO t1 VALUES(9, 10); +} wal +do_test 2.1.1 { + forcedelete state.db + sqlite3rbu_vacuum rbu test.db state.db + rbu step +} {SQLITE_ERROR} +do_test 2.1.2 { + list [catch { rbu close } msg] $msg +} {1 {SQLITE_ERROR - cannot vacuum wal mode database}} + +reset_db +do_execsql_test 2.2.0 { + CREATE TABLE tx(a PRIMARY KEY, b BLOB); + INSERT INTO tx VALUES(1, randomblob(900)); + INSERT INTO tx SELECT a+1, randomblob(900) FROM tx; + INSERT INTO tx SELECT a+2, randomblob(900) FROM tx; + INSERT INTO tx SELECT a+4, randomblob(900) FROM tx; + INSERT INTO tx SELECT a+8, randomblob(900) FROM tx; +} +db_save_and_close +for {set i 1} 1 {incr i} { + forcedelete state.db + db_restore_and_reopen + + sqlite3rbu_vacuum rbu test.db state.db + for {set step 0} {$step<$i} {incr step} { rbu step } + rbu close + if {[file exists test.db-wal]} break + + execsql { INSERT INTO tx VALUES(20, 20) } + + do_test 2.2.$i.1 { + sqlite3rbu_vacuum rbu test.db state.db + rbu step + } {SQLITE_BUSY} + do_test 2.2.$i.2 { + list [catch { rbu close } msg] $msg + } {1 {SQLITE_BUSY - database modified during rbu vacuum}} + +} + catch { db close } finish_test diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 60773b79f8..6ccf54826c 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -399,7 +399,7 @@ struct rbu_file { int openFlags; /* Flags this file was opened with */ u32 iCookie; /* Cookie value for main db files */ u8 iWriteVer; /* "write-version" value for main db files */ - u8 bNolock; + u8 bNolock; /* True to fail EXCLUSIVE locks */ int nShm; /* Number of entries in apShm[] array */ char **apShm; /* Array of mmap'd *-shm regions */ @@ -2373,9 +2373,12 @@ static void rbuOpenDatabase(sqlite3rbu *p){ } p->eStage = 0; - if( p->dbMain==0 ){ + if( p->rc==SQLITE_OK && p->dbMain==0 ){ if( !rbuIsVacuum(p) ){ p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1); + }else if( p->pRbuFd->pWalFd ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database"); }else{ char *zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1", p->zRbu); if( zTarget==0 ){ @@ -2987,6 +2990,7 @@ static void rbuIncrSchemaCookie(sqlite3rbu *p){ static void rbuSaveState(sqlite3rbu *p, int eStage){ if( p->rc==SQLITE_OK || p->rc==SQLITE_DONE ){ sqlite3_stmt *pInsert = 0; + rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd); int rc; assert( p->zErrmsg==0 ); @@ -3009,7 +3013,7 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ RBU_STATE_ROW, p->nStep, RBU_STATE_PROGRESS, p->nProgress, RBU_STATE_CKPT, p->iWalCksum, - RBU_STATE_COOKIE, (i64)p->pTargetFd->iCookie, + RBU_STATE_COOKIE, (i64)pFd->iCookie, RBU_STATE_OALSZ, p->iOalSz, RBU_STATE_PHASEONESTEP, p->nPhaseOneStep ) @@ -3431,17 +3435,21 @@ static sqlite3rbu *openRbuHandle( } } - if( p->rc==SQLITE_OK - && !rbuIsVacuum(p) + if( p->rc==SQLITE_OK && (p->eStage==RBU_STAGE_OAL || p->eStage==RBU_STAGE_MOVE) - && pState->eStage!=0 && p->pTargetFd->iCookie!=pState->iCookie - ){ - /* At this point (pTargetFd->iCookie) contains the value of the - ** change-counter cookie (the thing that gets incremented when a - ** transaction is committed in rollback mode) currently stored on - ** page 1 of the database file. */ - p->rc = SQLITE_BUSY; - p->zErrmsg = sqlite3_mprintf("database modified during rbu update"); + && pState->eStage!=0 + ){ + rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd); + if( pFd->iCookie!=pState->iCookie ){ + /* At this point (pTargetFd->iCookie) contains the value of the + ** change-counter cookie (the thing that gets incremented when a + ** transaction is committed in rollback mode) currently stored on + ** page 1 of the database file. */ + p->rc = SQLITE_BUSY; + p->zErrmsg = sqlite3_mprintf("database modified during rbu %s", + (rbuIsVacuum(p) ? "vacuum" : "update") + ); + } } if( p->rc==SQLITE_OK ){ @@ -3841,6 +3849,7 @@ static int rbuVfsRead( rbuPutU32(&aBuf[36], 0); /* number of free pages */ rbuPutU32(&aBuf[32], 0); /* first page on free list trunk */ rbuPutU32(&aBuf[28], 1); /* size of db file in pages */ + rbuPutU32(&aBuf[24], pRbu->pRbuFd->iCookie+1); /* Change counter */ if( iAmt>100 ){ assert( iAmt>=101 ); @@ -3857,13 +3866,6 @@ static int rbuVfsRead( u8 *pBuf = (u8*)zBuf; p->iCookie = rbuGetU32(&pBuf[24]); p->iWriteVer = pBuf[19]; - if( pRbu && rbuIsVacuum(p->pRbu) ){ - rbu_file *pRbuFd = 0; - sqlite3_file_control(pRbu->dbRbu, "main", - SQLITE_FCNTL_FILE_POINTER, (void*)&pRbuFd - ); - rbuPutU32(&pBuf[24], pRbuFd->iCookie+1); - } } } return rc; diff --git a/manifest b/manifest index 068554e465..fe489077ea 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Another\sfix\sto\srbu\svacuum\sfor\sa\szipvfs\scase. -D 2016-04-18T21:00:01.258 +C Detect\sattempts\sto\suse\srbu\svacuum\son\sa\swal\smode\sdatabase\s(not\sallowed).\sAnd\sattempts\sto\swrite\sto\sa\sdatabase\sin\sthe\smiddle\sof\san\srbu\svacuum\s(which\sprevents\sthe\svacuum\sfrom\sresuming). +D 2016-04-19T16:20:24.513 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -246,8 +246,8 @@ F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 -F ext/rbu/rbuvacuum.test 75b4231f85622859e814c7f028afad0303f72f60 -F ext/rbu/sqlite3rbu.c ac4621fc62e0bc3e9e92fceb483c0207af25addf +F ext/rbu/rbuvacuum.test b7194617c22f69e2015082a5414585aae2827a29 +F ext/rbu/sqlite3rbu.c d269d60bddb770fc9bbe747c47522f946562878d F ext/rbu/sqlite3rbu.h 1342ab6121e715b8da59ec35c5b5c16060be7a6b F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1483,7 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d76f4aaa4caab713460421bd27365a82ac986c20 -R 02609be0d1cf35faad7663d479babac3 +P 29407d70e44ad9ea5ddaf1011d0e212b602a3ddf +R da49b342c9cd61c56db951e40b34afc9 U dan -Z ecb53ed56b3181cf4b46470b55ef0f36 +Z 3a78d0d54b42b6cec75432abbb63bed7 diff --git a/manifest.uuid b/manifest.uuid index 74ea0959b8..4483a9f107 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -29407d70e44ad9ea5ddaf1011d0e212b602a3ddf \ No newline at end of file +00b2f4b09ffca5156e43c4db2bfe0b2c979c49b0 \ No newline at end of file From 46f0035b4829af743b83b6b9efd431031a716d35 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 19 Apr 2016 17:11:05 +0000 Subject: [PATCH 09/15] When an RBU vacuum is started on a db identified using a URI filename, pass the same URI parameters when creating the new version of the db. This ensures that RBU vacuum works with password protected databases. FossilOrigin-Name: ca021ba88106500f347ed65199a4832bc9eb5ef8 --- ext/rbu/sqlite3rbu.c | 16 +++++++++++++++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 6ccf54826c..0670b9b8b8 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -2380,7 +2380,21 @@ static void rbuOpenDatabase(sqlite3rbu *p){ p->rc = SQLITE_ERROR; p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database"); }else{ - char *zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1", p->zRbu); + char *zTarget; + char *zExtra = 0; + if( strlen(p->zRbu)>=5 && 0==memcmp("file:", p->zRbu, 5) ){ + zExtra = &p->zRbu[5]; + while( *zExtra ){ + if( *zExtra++=='?' ) break; + } + if( *zExtra=='\0' ) zExtra = 0; + } + + zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1%s%s", + sqlite3_db_filename(p->dbRbu, "main"), + (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra) + ); + if( zTarget==0 ){ p->rc = SQLITE_NOMEM; return; diff --git a/manifest b/manifest index fe489077ea..2edcbd1077 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Detect\sattempts\sto\suse\srbu\svacuum\son\sa\swal\smode\sdatabase\s(not\sallowed).\sAnd\sattempts\sto\swrite\sto\sa\sdatabase\sin\sthe\smiddle\sof\san\srbu\svacuum\s(which\sprevents\sthe\svacuum\sfrom\sresuming). -D 2016-04-19T16:20:24.513 +C When\san\sRBU\svacuum\sis\sstarted\son\sa\sdb\sidentified\susing\sa\sURI\sfilename,\spass\sthe\ssame\sURI\sparameters\swhen\screating\sthe\snew\sversion\sof\sthe\sdb.\sThis\sensures\sthat\sRBU\svacuum\sworks\swith\spassword\sprotected\sdatabases. +D 2016-04-19T17:11:05.418 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -247,7 +247,7 @@ F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test b7194617c22f69e2015082a5414585aae2827a29 -F ext/rbu/sqlite3rbu.c d269d60bddb770fc9bbe747c47522f946562878d +F ext/rbu/sqlite3rbu.c 0447a8a306411e585d8595465abbed7709f1feb7 F ext/rbu/sqlite3rbu.h 1342ab6121e715b8da59ec35c5b5c16060be7a6b F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1483,7 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 29407d70e44ad9ea5ddaf1011d0e212b602a3ddf -R da49b342c9cd61c56db951e40b34afc9 +P 00b2f4b09ffca5156e43c4db2bfe0b2c979c49b0 +R e7445097e6cb895f2eea40503c71f11e U dan -Z 3a78d0d54b42b6cec75432abbb63bed7 +Z 90c42b90d6761cf98f1c67d89391cb68 diff --git a/manifest.uuid b/manifest.uuid index 4483a9f107..f1fc314990 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -00b2f4b09ffca5156e43c4db2bfe0b2c979c49b0 \ No newline at end of file +ca021ba88106500f347ed65199a4832bc9eb5ef8 \ No newline at end of file From 1aea63032a02a2f38b673243b63a71a80bb4aadb Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 19 Apr 2016 19:27:09 +0000 Subject: [PATCH 10/15] Updates to ensure the values of PRAGMA settings like "page_size", "auto_vacuum", "user_version" and "application_id" are not lost when a database is RBU vacuumed. FossilOrigin-Name: 74ffea76b8c9ddd7699ce6ca77dce005bd7922ac --- ext/rbu/rbuvacuum.test | 68 +++++++++++++++++++++++++++++++++++++++++- ext/rbu/sqlite3rbu.c | 57 ++++++++++++++++++++++++++++++----- manifest | 14 ++++----- manifest.uuid | 2 +- 4 files changed, 125 insertions(+), 16 deletions(-) diff --git a/ext/rbu/rbuvacuum.test b/ext/rbu/rbuvacuum.test index 4380b78fbf..85ecfc4c13 100644 --- a/ext/rbu/rbuvacuum.test +++ b/ext/rbu/rbuvacuum.test @@ -174,6 +174,73 @@ foreach step {0 1} { SELECT * FROM t4 ORDER BY b } {a b c d e f j k l} + reset_db + do_execsql_test 1.7.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b); + INSERT INTO t1 VALUES(NULL, 'one'); + INSERT INTO t1 VALUES(NULL, 'two'); + DELETE FROM t1 WHERE a=2; + INSERT INTO t1 VALUES(NULL, 'three'); + INSERT INTO t1 VALUES(NULL, 'four'); + DELETE FROM t1 WHERE a=4; + INSERT INTO t1 VALUES(NULL, 'five'); + INSERT INTO t1 VALUES(NULL, 'six'); + DELETE FROM t1 WHERE a=6; + SELECT * FROM t1; + } {1 one 3 three 5 five} + do_rbu_vacuum_test 1.7.1 $step + do_execsql_test 1.7.2 { + INSERT INTO t1 VALUES(NULL, 'seven'); + SELECT * FROM t1; + } {1 one 3 three 5 five 7 seven} + + reset_db + do_execsql_test 1.8.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(NULL, 'one'); + INSERT INTO t1 VALUES(NULL, 'two'); + INSERT INTO t1 VALUES(NULL, 'three'); + INSERT INTO t1 VALUES(NULL, 'four'); + INSERT INTO t1 VALUES(NULL, 'five'); + INSERT INTO t1 VALUES(NULL, 'six'); + ANALYZE; + SELECT * FROM sqlite_stat1; + } {t1 i1 {6 1}} + do_rbu_vacuum_test 1.8.1 $step + do_execsql_test 1.7.2 { + SELECT * FROM sqlite_stat1; + } {t1 i1 {6 1}} + + reset_db + do_execsql_test 1.9.0 { + PRAGMA page_size = 8192; + PRAGMA auto_vacuum = 2; + PRAGMA user_version = 412; + PRAGMA application_id = 413; + + CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(NULL, 'one'); + INSERT INTO t1 VALUES(NULL, 'two'); + INSERT INTO t1 VALUES(NULL, 'three'); + INSERT INTO t1 VALUES(NULL, 'four'); + INSERT INTO t1 VALUES(NULL, 'five'); + INSERT INTO t1 VALUES(NULL, 'six'); + + PRAGMA main.page_size; + PRAGMA main.auto_vacuum; + PRAGMA main.user_version; + PRAGMA main.application_id; + } {8192 2 412 413} + + do_rbu_vacuum_test 1.9.1 $step + do_execsql_test 1.9.2 { + PRAGMA main.page_size; + PRAGMA main.auto_vacuum; + PRAGMA main.user_version; + PRAGMA main.application_id; + } {8192 2 412 413} } #------------------------------------------------------------------------- @@ -229,7 +296,6 @@ for {set i 1} 1 {incr i} { do_test 2.2.$i.2 { list [catch { rbu close } msg] $msg } {1 {SQLITE_BUSY - database modified during rbu vacuum}} - } catch { db close } diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 0670b9b8b8..8c7682abea 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -3333,6 +3333,38 @@ static void rbuInitPhaseOneSteps(sqlite3rbu *p){ } } +/* +** The second argument passed to this function is the name of a PRAGMA +** setting - "page_size", "auto_vacuum", "user_version" or "application_id". +** This function executes the following on sqlite3rbu.dbRbu: +** +** "PRAGMA main.$zPragma" +** +** where $zPragma is the string passed as the second argument, then +** on sqlite3rbu.dbMain: +** +** "PRAGMA main.$zPragma = $val" +** +** where $val is the value returned by the first PRAGMA invocation. +** +** In short, it copies the value of the specified PRAGMA setting from +** dbRbu to dbMain. +*/ +static void rbuCopyPragma(sqlite3rbu *p, const char *zPragma){ + if( p->rc==SQLITE_OK ){ + sqlite3_stmt *pPragma = 0; + p->rc = prepareFreeAndCollectError(p->dbRbu, &pPragma, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.%s", zPragma) + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPragma) ){ + p->rc = rbuMPrintfExec(p, p->dbMain, "PRAGMA main.%s = %d", + zPragma, sqlite3_column_int(pPragma, 0) + ); + } + rbuFinalize(p, pPragma); + } +} + /* ** The RBU handle passed as the only argument has just been opened and ** the state database is empty. If this RBU handle was opened for an @@ -3343,11 +3375,15 @@ static void rbuCreateTargetSchema(sqlite3rbu *p){ sqlite3_stmt *pInsert = 0; assert( rbuIsVacuum(p) ); + p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=1", 0,0, &p->zErrmsg); + if( p->rc==SQLITE_OK ){ + p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, + "SELECT sql FROM sqlite_master WHERE sql!='' AND rootpage!=0" + " AND name!='sqlite_sequence' " + " ORDER BY type DESC" + ); + } - p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, - "SELECT sql FROM sqlite_master WHERE sql!='' AND rootpage!=0" - " ORDER BY type DESC" - ); while( p->rc==SQLITE_OK && sqlite3_step(pSql)==SQLITE_ROW ){ const char *zSql = (const char*)sqlite3_column_text(pSql, 0); p->rc = sqlite3_exec(p->dbMain, zSql, 0, 0, &p->zErrmsg); @@ -3355,8 +3391,6 @@ static void rbuCreateTargetSchema(sqlite3rbu *p){ rbuFinalize(p, pSql); if( p->rc!=SQLITE_OK ) return; - p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=1", 0,0, &p->zErrmsg); - if( p->rc==SQLITE_OK ){ p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, "SELECT * FROM sqlite_master WHERE rootpage=0 OR rootpage IS NULL" @@ -3470,6 +3504,11 @@ static sqlite3rbu *openRbuHandle( if( p->eStage==RBU_STAGE_OAL ){ sqlite3 *db = p->dbMain; + if( pState->eStage==0 && rbuIsVacuum(p) ){ + rbuCopyPragma(p, "page_size"); + rbuCopyPragma(p, "auto_vacuum"); + } + /* Open transactions both databases. The *-oal file is opened or ** created at this point. */ p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); @@ -3491,6 +3530,8 @@ static sqlite3rbu *openRbuHandle( ** when this handle was opened, create the target database schema. */ if( pState->eStage==0 && rbuIsVacuum(p) ){ rbuCreateTargetSchema(p); + rbuCopyPragma(p, "user_version"); + rbuCopyPragma(p, "application_id"); } /* Point the object iterator at the first object */ @@ -3854,12 +3895,14 @@ static int rbuVfsRead( if( pRbu && rbuIsVacuum(pRbu) && rc==SQLITE_IOERR_SHORT_READ && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) + && pRbu->pRbuFd->base.pMethods ){ sqlite3_file *pFd = (sqlite3_file*)pRbu->pRbuFd; rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst); if( rc==SQLITE_OK ){ u8 *aBuf = (u8*)zBuf; - rbuPutU32(&aBuf[52], 0); /* largest root page number */ + u32 iRoot = rbuGetU32(&aBuf[52]) ? 1 : 0; + rbuPutU32(&aBuf[52], iRoot); /* largest root page number */ rbuPutU32(&aBuf[36], 0); /* number of free pages */ rbuPutU32(&aBuf[32], 0); /* first page on free list trunk */ rbuPutU32(&aBuf[28], 1); /* size of db file in pages */ diff --git a/manifest b/manifest index 2edcbd1077..d233334b3c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\san\sRBU\svacuum\sis\sstarted\son\sa\sdb\sidentified\susing\sa\sURI\sfilename,\spass\sthe\ssame\sURI\sparameters\swhen\screating\sthe\snew\sversion\sof\sthe\sdb.\sThis\sensures\sthat\sRBU\svacuum\sworks\swith\spassword\sprotected\sdatabases. -D 2016-04-19T17:11:05.418 +C Updates\sto\sensure\sthe\svalues\sof\sPRAGMA\ssettings\slike\s"page_size",\s"auto_vacuum",\s"user_version"\sand\s"application_id"\sare\snot\slost\swhen\sa\sdatabase\sis\sRBU\svacuumed. +D 2016-04-19T19:27:09.289 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -246,8 +246,8 @@ F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 -F ext/rbu/rbuvacuum.test b7194617c22f69e2015082a5414585aae2827a29 -F ext/rbu/sqlite3rbu.c 0447a8a306411e585d8595465abbed7709f1feb7 +F ext/rbu/rbuvacuum.test ecd7ce1cd8528d6230b56f4840bbb7188cee6946 +F ext/rbu/sqlite3rbu.c a6c02a67834f9e6c2adb18a9191aa0ae65057cfb F ext/rbu/sqlite3rbu.h 1342ab6121e715b8da59ec35c5b5c16060be7a6b F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1483,7 +1483,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 00b2f4b09ffca5156e43c4db2bfe0b2c979c49b0 -R e7445097e6cb895f2eea40503c71f11e +P ca021ba88106500f347ed65199a4832bc9eb5ef8 +R 8122ed3aabc11f4475527c60a5badfa5 U dan -Z 90c42b90d6761cf98f1c67d89391cb68 +Z 8cbed13b6f72dec5a4cc65417abbee63 diff --git a/manifest.uuid b/manifest.uuid index f1fc314990..590abb9da7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca021ba88106500f347ed65199a4832bc9eb5ef8 \ No newline at end of file +74ffea76b8c9ddd7699ce6ca77dce005bd7922ac \ No newline at end of file From 132dde5cdf413bc8b0c809c3ec3fbc09a665a2c3 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 20 Apr 2016 17:47:52 +0000 Subject: [PATCH 11/15] Add fault injection tests for rbu vacuum. Fix some problems revealed by the same. FossilOrigin-Name: f042fdd1ea7febec7228e51efc2b0281805e196a --- ext/rbu/rbufault3.test | 98 ++++++++++++++++++++++++++++++++++++++++++ ext/rbu/sqlite3rbu.c | 19 +++++--- manifest | 13 +++--- manifest.uuid | 2 +- 4 files changed, 119 insertions(+), 13 deletions(-) create mode 100644 ext/rbu/rbufault3.test diff --git a/ext/rbu/rbufault3.test b/ext/rbu/rbufault3.test new file mode 100644 index 0000000000..e72eb14f7d --- /dev/null +++ b/ext/rbu/rbufault3.test @@ -0,0 +1,98 @@ +# 2016 April 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains fault injection tests for RBU vacuum operations. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +source $testdir/malloc_common.tcl +set ::testprefix rbufault3 + +foreach {fault errlist} { + oom-* { + {1 SQLITE_NOMEM} + {1 SQLITE_IOERR_NOMEM} + {1 {SQLITE_NOMEM - out of memory}} + } + + ioerr-* { + {1 {SQLITE_IOERR - disk I/O error}} + {1 SQLITE_IOERR} + {1 SQLITE_IOERR_WRITE} + {1 SQLITE_IOERR_FSYNC} + {1 SQLITE_IOERR_READ} + {1 {SQLITE_IOERR - unable to open database: test.db2}} + {1 {SQLITE_ERROR - unable to open database: test.db2}} + {1 {SQLITE_ERROR - SQL logic error or missing database}} + } + + cantopen* { + {1 {SQLITE_CANTOPEN - unable to open database: test.db2}} + {1 {SQLITE_CANTOPEN - unable to open database: test.db2}} + {1 {SQLITE_CANTOPEN - unable to open database file}} + {1 SQLITE_CANTOPEN} + } + +} { + + reset_db + do_execsql_test 0 { + CREATE TABLE target(x UNIQUE, y, z, PRIMARY KEY(y)); + INSERT INTO target VALUES(1, 2, 3); + INSERT INTO target VALUES(4, 5, 6); + INSERT INTO target VALUES(7, 8, 9); + CREATE INDEX i1 ON target(z); + } + faultsim_save_and_close + + do_faultsim_test 1 -faults $fault -prep { + faultsim_restore_and_reopen + forcedelete test.db2 + } -body { + sqlite3rbu_vacuum rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } -test { + eval [list faultsim_test_result {0 SQLITE_DONE} {*}$::errlist] + } + + do_faultsim_test 2 -faults $fault -prep { + faultsim_restore_and_reopen + forcedelete test.db2 + } -body { + sqlite3rbu_vacuum rbu test.db test.db2 + rbu step + rbu close + } -test { + eval [list faultsim_test_result {0 SQLITE_OK} {*}$::errlist] + } + + forcedelete test.db2 + sqlite3rbu_vacuum rbu test.db test.db2 + rbu step + rbu close + faultsim_save_and_close + + do_faultsim_test 3 -faults $fault -prep { + faultsim_restore_and_reopen + forcedelete test.db2 + } -body { + sqlite3rbu_vacuum rbu test.db test.db2 + rbu step + rbu close + } -test { + eval [list faultsim_test_result {0 SQLITE_OK} {*}$::errlist] + } + +} + +finish_test + diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 8c7682abea..8a1b06dd62 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -2339,7 +2339,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ /* Open the RBU database */ p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1); - if( rbuIsVacuum(p) ){ + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); } @@ -2355,11 +2355,13 @@ static void rbuOpenDatabase(sqlite3rbu *p){ /* If it has not already been created, create the rbu_state table */ rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb); - if( rbuIsVacuum(p) ){ + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ int bOpen = 0; + int rc; p->nRbu = 0; p->pRbuFd = 0; - sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); + rc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); + if( rc!=SQLITE_NOTFOUND ) p->rc = rc; if( p->eStage>=RBU_STAGE_MOVE ){ bOpen = 1; }else{ @@ -3411,6 +3413,9 @@ static void rbuCreateTargetSchema(sqlite3rbu *p){ sqlite3_step(pInsert); p->rc = sqlite3_reset(pInsert); } + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=0",0,0,&p->zErrmsg); + } rbuFinalize(p, pSql); rbuFinalize(p, pInsert); @@ -3511,7 +3516,9 @@ static sqlite3rbu *openRbuHandle( /* Open transactions both databases. The *-oal file is opened or ** created at this point. */ - p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); + } if( p->rc==SQLITE_OK ){ p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, &p->zErrmsg); } @@ -3528,7 +3535,7 @@ static sqlite3rbu *openRbuHandle( /* If this is an RBU vacuum operation and the state table was empty ** when this handle was opened, create the target database schema. */ - if( pState->eStage==0 && rbuIsVacuum(p) ){ + if( p->rc==SQLITE_OK && pState->eStage==0 && rbuIsVacuum(p) ){ rbuCreateTargetSchema(p); rbuCopyPragma(p, "user_version"); rbuCopyPragma(p, "application_id"); @@ -3895,7 +3902,7 @@ static int rbuVfsRead( if( pRbu && rbuIsVacuum(pRbu) && rc==SQLITE_IOERR_SHORT_READ && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) - && pRbu->pRbuFd->base.pMethods + && pRbu->rc==SQLITE_OK ){ sqlite3_file *pFd = (sqlite3_file*)pRbu->pRbuFd; rc = pFd->pMethods->xRead(pFd, zBuf, iAmt, iOfst); diff --git a/manifest b/manifest index d233334b3c..e640d710d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\sensure\sthe\svalues\sof\sPRAGMA\ssettings\slike\s"page_size",\s"auto_vacuum",\s"user_version"\sand\s"application_id"\sare\snot\slost\swhen\sa\sdatabase\sis\sRBU\svacuumed. -D 2016-04-19T19:27:09.289 +C Add\sfault\sinjection\stests\sfor\srbu\svacuum.\sFix\ssome\sproblems\srevealed\sby\sthe\ssame. +D 2016-04-20T17:47:52.967 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -243,11 +243,12 @@ F ext/rbu/rbucrash.test 8d2ed5d4b05fef6c00c2a6b5f7ead71fa172a695 F ext/rbu/rbudiff.test 2df0a8a7d998ecf81764c21eeda3cde5611c5091 F ext/rbu/rbufault.test cc0be8d5d392d98b0c2d6a51be377ea989250a89 F ext/rbu/rbufault2.test 9a7f19edd6ea35c4c9f807d8a3db0a03a5670c06 +F ext/rbu/rbufault3.test 54a399888ac4af44c68f9f58afbed23149428bca F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test ecd7ce1cd8528d6230b56f4840bbb7188cee6946 -F ext/rbu/sqlite3rbu.c a6c02a67834f9e6c2adb18a9191aa0ae65057cfb +F ext/rbu/sqlite3rbu.c cc44505ef06053077f35b08c4ba11dc8bcf310dc F ext/rbu/sqlite3rbu.h 1342ab6121e715b8da59ec35c5b5c16060be7a6b F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1483,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ca021ba88106500f347ed65199a4832bc9eb5ef8 -R 8122ed3aabc11f4475527c60a5badfa5 +P 74ffea76b8c9ddd7699ce6ca77dce005bd7922ac +R f86f4403b5c8adf842292353673eca81 U dan -Z 8cbed13b6f72dec5a4cc65417abbee63 +Z 6e60b344cf662b4bf49e9c442360b79f diff --git a/manifest.uuid b/manifest.uuid index 590abb9da7..ce2dab7023 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -74ffea76b8c9ddd7699ce6ca77dce005bd7922ac \ No newline at end of file +f042fdd1ea7febec7228e51efc2b0281805e196a \ No newline at end of file From 94d49697b506e9c612352809d0ef100d6c5d4fc8 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 20 Apr 2016 20:08:58 +0000 Subject: [PATCH 12/15] Add a documentation comment for sqlite3rbu_vacuum() to sqlite3rbu.h. FossilOrigin-Name: da5c753ddac3cbfdf03710a82f5fd9fa2e29e819 --- ext/rbu/sqlite3rbu.c | 36 ++++++++++++++++++++++++++++++++++++ ext/rbu/sqlite3rbu.h | 22 +++++++++++++++++++++- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 65 insertions(+), 9 deletions(-) diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 8a1b06dd62..022e682bde 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -2352,9 +2352,45 @@ static void rbuOpenDatabase(sqlite3rbu *p){ memcpy(p->zStateDb, "main", 4); } +#if 0 + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, 0); + } +#endif + /* If it has not already been created, create the rbu_state table */ rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb); +#if 0 + if( rbuIsVacuum(p) ){ + if( p->rc==SQLITE_OK ){ + int rc2; + int bOk = 0; + sqlite3_stmt *pCnt = 0; + p->rc = prepareAndCollectError(p->dbRbu, &pCnt, &p->zErrmsg, + "SELECT count(*) FROM stat.sqlite_master" + ); + if( p->rc==SQLITE_OK + && sqlite3_step(pCnt)==SQLITE_ROW + && 1==sqlite3_column_int(pCnt, 0) + ){ + bOk = 1; + } + rc2 = sqlite3_finalize(pCnt); + if( p->rc==SQLITE_OK ) p->rc = rc2; + + if( p->rc==SQLITE_OK && bOk==0 ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("invalid state database"); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0); + } + } + } +#endif + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ int bOpen = 0; int rc; diff --git a/ext/rbu/sqlite3rbu.h b/ext/rbu/sqlite3rbu.h index b910866293..c0c95e51fd 100644 --- a/ext/rbu/sqlite3rbu.h +++ b/ext/rbu/sqlite3rbu.h @@ -315,7 +315,27 @@ sqlite3rbu *sqlite3rbu_open( ); /* -** Open an RBU handle to perform an RBU vacuum database file zTarget. +** Open an RBU handle to perform an RBU vacuum on database file zTarget. +** An RBU vacuum is similar to SQLite's built-in VACUUM command, except +** that it can be suspended and resumed like an RBU update. +** +** The second argument to this function, which may not be NULL, identifies +** a database in which to store the state of the RBU vacuum operation if +** it is suspended. The first time sqlite3rbu_vacuum() is called, to start +** an RBU vacuum operation, the state database should either not exist or +** be empty (contain no tables). If an RBU vacuum is suspended by calling +** sqlite3rbu_close() on the RBU handle before sqlite3rbu_step() has +** returned SQLITE_DONE, the vacuum state is stored in the state database. +** The vacuum can be resumed by calling this function to open a new RBU +** handle specifying the same target and state databases. +** +** This function does not delete the state database after an RBU vacuum +** is completed, even if it created it. +** +** As with sqlite3rbu_open(), Zipvfs users should rever to the comment +** describing the sqlite3rbu_create_vfs() API function below for +** a description of the complications associated with using RBU with +** zipvfs databases. */ sqlite3rbu *sqlite3rbu_vacuum( const char *zTarget, diff --git a/manifest b/manifest index e640d710d0..1ec956e979 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfault\sinjection\stests\sfor\srbu\svacuum.\sFix\ssome\sproblems\srevealed\sby\sthe\ssame. -D 2016-04-20T17:47:52.967 +C Add\sa\sdocumentation\scomment\sfor\ssqlite3rbu_vacuum()\sto\ssqlite3rbu.h. +D 2016-04-20T20:08:58.990 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -248,8 +248,8 @@ F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 F ext/rbu/rbuvacuum.test ecd7ce1cd8528d6230b56f4840bbb7188cee6946 -F ext/rbu/sqlite3rbu.c cc44505ef06053077f35b08c4ba11dc8bcf310dc -F ext/rbu/sqlite3rbu.h 1342ab6121e715b8da59ec35c5b5c16060be7a6b +F ext/rbu/sqlite3rbu.c 80ab68658a1dc1921a3eee7c666c948ea640087c +F ext/rbu/sqlite3rbu.h efcafd0e36861943a88deb181753236f84dbc244 F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 0b870ccb7b58b734a2a8e1e2755a7c0ded070920 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 74ffea76b8c9ddd7699ce6ca77dce005bd7922ac -R f86f4403b5c8adf842292353673eca81 +P f042fdd1ea7febec7228e51efc2b0281805e196a +R 20401012c91c212b91bec1accca1e5a8 U dan -Z 6e60b344cf662b4bf49e9c442360b79f +Z ebff3ad2f969914e02c51eda888da39a diff --git a/manifest.uuid b/manifest.uuid index ce2dab7023..ffe63d9829 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f042fdd1ea7febec7228e51efc2b0281805e196a \ No newline at end of file +da5c753ddac3cbfdf03710a82f5fd9fa2e29e819 \ No newline at end of file From 2e8bfb67d4a6de7029f62573bdc44b6ce9c47d78 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 21 Apr 2016 15:26:26 +0000 Subject: [PATCH 13/15] Fix a problem in rbu vacuum on tables with a large sqlite_master table. FossilOrigin-Name: 23eac52e987996afe8db847921aef434cebd25b5 --- ext/rbu/rbuvacuum.test | 15 +++++++++++++++ ext/rbu/sqlite3rbu.c | 4 ++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/ext/rbu/rbuvacuum.test b/ext/rbu/rbuvacuum.test index 85ecfc4c13..c67162c567 100644 --- a/ext/rbu/rbuvacuum.test +++ b/ext/rbu/rbuvacuum.test @@ -241,6 +241,21 @@ foreach step {0 1} { PRAGMA main.user_version; PRAGMA main.application_id; } {8192 2 412 413} + + # Vacuum a database with a large sqlite_master table. + # + reset_db + do_test 1.10.1 { + for {set i 1} {$i < 50} {incr i} { + execsql "PRAGMA page_size = 1024" + execsql "CREATE TABLE t$i (a, b, c, PRIMARY KEY(a, b));" + execsql " + INSERT INTO t$i VALUES(1, 2, 3); + INSERT INTO t$i VALUES(4, 5, 6); + " + } + } {} + do_rbu_vacuum_test 1.10.2 $step } #------------------------------------------------------------------------- diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 022e682bde..04dc9944d7 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -3952,9 +3952,9 @@ static int rbuVfsRead( rbuPutU32(&aBuf[24], pRbu->pRbuFd->iCookie+1); /* Change counter */ if( iAmt>100 ){ - assert( iAmt>=101 ); - memset(&aBuf[101], 0, iAmt-101); + memset(&aBuf[100], 0, iAmt-100); rbuPutU16(&aBuf[105], iAmt & 0xFFFF); + aBuf[100] = 0x0D; } } } diff --git a/manifest b/manifest index 1ec956e979..dbe7f79adc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sdocumentation\scomment\sfor\ssqlite3rbu_vacuum()\sto\ssqlite3rbu.h. -D 2016-04-20T20:08:58.990 +C Fix\sa\sproblem\sin\srbu\svacuum\son\stables\swith\sa\slarge\ssqlite_master\stable. +D 2016-04-21T15:26:26.501 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -247,8 +247,8 @@ F ext/rbu/rbufault3.test 54a399888ac4af44c68f9f58afbed23149428bca F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 -F ext/rbu/rbuvacuum.test ecd7ce1cd8528d6230b56f4840bbb7188cee6946 -F ext/rbu/sqlite3rbu.c 80ab68658a1dc1921a3eee7c666c948ea640087c +F ext/rbu/rbuvacuum.test aa405cff38b15a89f6c97f91f1c6a67b2140b358 +F ext/rbu/sqlite3rbu.c 8310553b029e749329a89b3880a21f80876755c3 F ext/rbu/sqlite3rbu.h efcafd0e36861943a88deb181753236f84dbc244 F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f042fdd1ea7febec7228e51efc2b0281805e196a -R 20401012c91c212b91bec1accca1e5a8 +P da5c753ddac3cbfdf03710a82f5fd9fa2e29e819 +R e2636288f7318e2301180a849b5ff72b U dan -Z ebff3ad2f969914e02c51eda888da39a +Z d9d3e7dacedd64ed84574ca4c0316d8e diff --git a/manifest.uuid b/manifest.uuid index ffe63d9829..8f13cab8e3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -da5c753ddac3cbfdf03710a82f5fd9fa2e29e819 \ No newline at end of file +23eac52e987996afe8db847921aef434cebd25b5 \ No newline at end of file From b7a208686ae0a1cbe8aac5919e9502a023ec8409 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 21 Apr 2016 16:44:38 +0000 Subject: [PATCH 14/15] If a call to sqlite3rbu_close() on an rbu handle opened by sqlite3rbu_vacuum() returns other than SQLITE_OK, delete the contents of the rbu_state table. This ensures that if an RBU vacuum operation either completes successfully or encounters an error, the next call to sqlite3rbu_vacuum() with the same parameters starts a new RBU vacuum operation. FossilOrigin-Name: 6e4ac704f51bdd1e0b16814f6a45ab29f26400ee --- ext/rbu/rbuvacuum.test | 3 --- ext/rbu/sqlite3rbu.c | 10 ++++++++++ ext/rbu/sqlite3rbu.h | 6 +++++- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/ext/rbu/rbuvacuum.test b/ext/rbu/rbuvacuum.test index c67162c567..96a37d1f71 100644 --- a/ext/rbu/rbuvacuum.test +++ b/ext/rbu/rbuvacuum.test @@ -19,7 +19,6 @@ set ::testprefix rbuvacuum proc do_rbu_vacuum_test {tn step} { uplevel [list do_test $tn.1 { - forcedelete state.db if {$step==0} { sqlite3rbu_vacuum rbu test.db state.db } while 1 { if {$step==1} { sqlite3rbu_vacuum rbu test.db state.db } @@ -275,7 +274,6 @@ do_execsql_test 2.1.0 { INSERT INTO t1 VALUES(9, 10); } wal do_test 2.1.1 { - forcedelete state.db sqlite3rbu_vacuum rbu test.db state.db rbu step } {SQLITE_ERROR} @@ -294,7 +292,6 @@ do_execsql_test 2.2.0 { } db_save_and_close for {set i 1} 1 {incr i} { - forcedelete state.db db_restore_and_reopen sqlite3rbu_vacuum rbu test.db state.db diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c index 04dc9944d7..e7e4ddef58 100644 --- a/ext/rbu/sqlite3rbu.c +++ b/ext/rbu/sqlite3rbu.c @@ -3685,6 +3685,16 @@ int sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){ /* Close any open statement handles. */ rbuObjIterFinalize(&p->objiter); + /* If this is an RBU vacuum handle and the vacuum has either finished + ** successfully or encountered an error, delete the contents of the + ** state table. This causes the next call to sqlite3rbu_vacuum() + ** specifying the current target and state databases to start a new + ** vacuum from scratch. */ + if( rbuIsVacuum(p) && p->rc!=SQLITE_OK && p->dbRbu ){ + int rc2 = sqlite3_exec(p->dbRbu, "DELETE FROM stat.rbu_state", 0, 0, 0); + if( p->rc==SQLITE_DONE && rc2!=SQLITE_OK ) p->rc = rc2; + } + /* Close the open database handle and VFS object. */ sqlite3_close(p->dbRbu); sqlite3_close(p->dbMain); diff --git a/ext/rbu/sqlite3rbu.h b/ext/rbu/sqlite3rbu.h index c0c95e51fd..fce9e493db 100644 --- a/ext/rbu/sqlite3rbu.h +++ b/ext/rbu/sqlite3rbu.h @@ -330,7 +330,11 @@ sqlite3rbu *sqlite3rbu_open( ** handle specifying the same target and state databases. ** ** This function does not delete the state database after an RBU vacuum -** is completed, even if it created it. +** is completed, even if it created it. However, if the call to +** sqlite3rbu_close() returns any value other than SQLITE_OK, the contents +** of the state tables within the state database are zeroed. This way, +** the next call to sqlite3rbu_vacuum() opens a handle that starts a +** new RBU vacuum operation. ** ** As with sqlite3rbu_open(), Zipvfs users should rever to the comment ** describing the sqlite3rbu_create_vfs() API function below for diff --git a/manifest b/manifest index dbe7f79adc..a1b6be3a3e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\srbu\svacuum\son\stables\swith\sa\slarge\ssqlite_master\stable. -D 2016-04-21T15:26:26.501 +C If\sa\scall\sto\ssqlite3rbu_close()\son\san\srbu\shandle\sopened\sby\ssqlite3rbu_vacuum()\sreturns\sother\sthan\sSQLITE_OK,\sdelete\sthe\scontents\sof\sthe\srbu_state\stable.\sThis\sensures\sthat\sif\san\sRBU\svacuum\soperation\seither\scompletes\ssuccessfully\sor\sencounters\san\serror,\sthe\snext\scall\sto\ssqlite3rbu_vacuum()\swith\sthe\ssame\sparameters\sstarts\sa\snew\sRBU\svacuum\soperation. +D 2016-04-21T16:44:38.130 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -247,9 +247,9 @@ F ext/rbu/rbufault3.test 54a399888ac4af44c68f9f58afbed23149428bca F ext/rbu/rbufts.test 828cd689da825f0a7b7c53ffc1f6f7fdb6fa5bda F ext/rbu/rbuprogress.test 2023a7df2c523e3df1cb532eff811cda385a789a F ext/rbu/rbusave.test 0f43b6686084f426ddd040b878426452fd2c2f48 -F ext/rbu/rbuvacuum.test aa405cff38b15a89f6c97f91f1c6a67b2140b358 -F ext/rbu/sqlite3rbu.c 8310553b029e749329a89b3880a21f80876755c3 -F ext/rbu/sqlite3rbu.h efcafd0e36861943a88deb181753236f84dbc244 +F ext/rbu/rbuvacuum.test bcbc1dcd8e2a46a811e46477692ae1c0e8917a85 +F ext/rbu/sqlite3rbu.c 20922328dcebe89589638923bb46840df8bc7733 +F ext/rbu/sqlite3rbu.h 2acd0a6344a6079de15c8dc9d84d3df83a665930 F ext/rbu/test_rbu.c 430b8b9520c233505371d564d3561e0b554355f4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 0b870ccb7b58b734a2a8e1e2755a7c0ded070920 @@ -1484,7 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P da5c753ddac3cbfdf03710a82f5fd9fa2e29e819 -R e2636288f7318e2301180a849b5ff72b +P 23eac52e987996afe8db847921aef434cebd25b5 +R 9f2af869b142a7169421543b9ca51ae8 U dan -Z d9d3e7dacedd64ed84574ca4c0316d8e +Z 717b7e491d25b95d9f80b6c2df00ade7 diff --git a/manifest.uuid b/manifest.uuid index 8f13cab8e3..ef1798e1f0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -23eac52e987996afe8db847921aef434cebd25b5 \ No newline at end of file +6e4ac704f51bdd1e0b16814f6a45ab29f26400ee \ No newline at end of file From 7694e06461203a466865fc8ed3068e0990a7ca50 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 21 Apr 2016 23:37:24 +0000 Subject: [PATCH 15/15] Change the temporary directory search algorithm on unix so that directories with only -wx permission are allowed. And do not allow "." to be returned if it lacks -wx permission. FossilOrigin-Name: 67985761aa93fb613b87d340e75371fa55b0f778 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/os_unix.c | 8 +++++--- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 084953dacb..df9a5ec95f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssqlite3rbu_vacuum()\sAPI,\sto\screate\san\sRBU\shandle\sthat\scan\sbe\sused\sto\srebuild\sa\sdatabase\sin\sthe\ssame\sway\sas\sthe\sVACUUM\scommand. -D 2016-04-21T18:13:37.597 +C Change\sthe\stemporary\sdirectory\ssearch\salgorithm\son\sunix\sso\sthat\sdirectories\nwith\sonly\s-wx\spermission\sare\sallowed.\s\sAnd\sdo\snot\sallow\s"."\sto\sbe\sreturned\sif\nit\slacks\s-wx\spermission. +D 2016-04-21T23:37:24.378 F Makefile.in eba680121821b8a60940a81454316f47a341487a F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 71b8b16cf9393f68e2e2035486ca104872558836 @@ -361,7 +361,7 @@ F src/os.c 4d83917f072ad958fba9235136fa2ed43df47905 F src/os.h 8e976e59eb4ca1c0fca6d35ee803e38951cb0343 F src/os_common.h b2f4707a603e36811d9b1a13278bffd757857b85 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c d0b41a47eb5f0dc00e423a1723aadeab0e78c85f +F src/os_unix.c c96826e21e897bf9c6ae896be9de8d55d63a4ed5 F src/os_win.c 1997a873bfc8296a701bd8e2df8c3d5da5afe956 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca F src/pager.c d20fa46d5beda38095be6070dd4c59e502817c72 @@ -1484,8 +1484,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c4f165c460c4244ed434107feac005efa3c386cf 6e4ac704f51bdd1e0b16814f6a45ab29f26400ee -R 4089c7bed4eb395c7a148d5eb16b1be4 -T +closed 6e4ac704f51bdd1e0b16814f6a45ab29f26400ee -U dan -Z 0e5a7e5d2cf2990714a283851c232d22 +P bd4a6f8313eb71399a7141ebf7f2617d7b1464c3 +R 90c3ca98d0d4d69b4119d1fe5513a916 +U drh +Z f4304caf089d5c38523c57ca13360c36 diff --git a/manifest.uuid b/manifest.uuid index a983c0514a..65f89ba88b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bd4a6f8313eb71399a7141ebf7f2617d7b1464c3 \ No newline at end of file +67985761aa93fb613b87d340e75371fa55b0f778 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index aa86f00d70..9a97515fdc 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5420,10 +5420,10 @@ static const char *unixTempFileDir(void){ if( zDir==0 ) continue; if( osStat(zDir, &buf) ) continue; if( !S_ISDIR(buf.st_mode) ) continue; - if( osAccess(zDir, 07) ) continue; - break; + if( osAccess(zDir, 03) ) continue; + return zDir; } - return zDir; + return 0; } /* @@ -5439,9 +5439,11 @@ static int unixGetTempname(int nBuf, char *zBuf){ ** using the io-error infrastructure to test that SQLite handles this ** function failing. */ + zBuf[0] = 0; SimulateIOError( return SQLITE_IOERR ); zDir = unixTempFileDir(); + if( zDir==0 ) return SQLITE_IOERR_GETTEMPPATH; do{ u64 r; sqlite3_randomness(sizeof(r), &r);