From bde13e26949ea15d639724a0d8d301ead3ab8005 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 27 Oct 2017 20:53:16 +0000 Subject: [PATCH 01/20] Add the start of the "incremental_index_check" virtual table in ext/repair/checkindex.c. For incremental verification of index contents. FossilOrigin-Name: d5b9dada471358a2864727759648b763bf6890fc2521fac53c0d8216017d39b7 --- ext/repair/checkindex.c | 442 ++++++++++++++++++++++++++++++++++++++++ manifest | 17 +- manifest.uuid | 2 +- test/checkindex.test | 91 +++++++++ 4 files changed, 545 insertions(+), 7 deletions(-) create mode 100644 ext/repair/checkindex.c create mode 100644 test/checkindex.test diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c new file mode 100644 index 0000000000..a74d1d4b75 --- /dev/null +++ b/ext/repair/checkindex.c @@ -0,0 +1,442 @@ +/* +** 2017 October 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 + +#ifndef SQLITE_AMALGAMATION +# include +# include +# include +# include +# define ALWAYS(X) 1 +# define NEVER(X) 0 + typedef unsigned char u8; + typedef unsigned short u16; + typedef unsigned int u32; +#define get4byte(x) ( \ + ((u32)((x)[0])<<24) + \ + ((u32)((x)[1])<<16) + \ + ((u32)((x)[2])<<8) + \ + ((u32)((x)[3])) \ +) +#endif + +typedef struct CidxTable CidxTable; +typedef struct CidxCursor CidxCursor; + +struct CidxTable { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; +}; + +struct CidxCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + sqlite3_stmt *pStmt; +}; + +static void *cidxMalloc(int *pRc, int n){ + void *pRet = 0; + assert( n!=0 ); + if( *pRc==SQLITE_OK ){ + pRet = sqlite3_malloc(n); + if( pRet ){ + memset(pRet, 0, n); + }else{ + *pRc = SQLITE_NOMEM; + } + } + return pRet; +} + +static void cidxCursorError(CidxCursor *pCsr, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + assert( pCsr->base.pVtab->zErrMsg==0 ); + pCsr->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap); + va_end(ap); +} + +/* +** Connect to then incremental_index_check virtual table. +*/ +static int cidxConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + int rc = SQLITE_OK; + CidxTable *pRet; + + rc = sqlite3_declare_vtab(db, + "CREATE TABLE xyz(" + " errmsg TEXT, current_key TEXT," + " index_name HIDDEN, after_key HIDDEN" + ")" + ); + pRet = cidxMalloc(&rc, sizeof(CidxTable)); + if( pRet ){ + pRet->db = db; + } + + *ppVtab = (sqlite3_vtab*)pRet; + return rc; +} + +/* +** Disconnect from or destroy an incremental_index_check virtual table. +*/ +static int cidxDisconnect(sqlite3_vtab *pVtab){ + CidxTable *pTab = (CidxTable*)pVtab; + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* +** xBestIndex method. +*/ +static int cidxBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pInfo){ + int iIdxName = -1; + int iAfterKey = -1; + int i; + + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; + if( p->usable==0 ) continue; + if( p->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + + if( p->iColumn==2 ){ + iIdxName = i; + } + if( p->iColumn==3 ){ + iAfterKey = i; + } + } + + if( iIdxName<0 ){ + pInfo->estimatedCost = 1000000000.0; + }else{ + pInfo->aConstraintUsage[iIdxName].argvIndex = 1; + pInfo->aConstraintUsage[iIdxName].omit = 1; + if( iAfterKey<0 ){ + pInfo->estimatedCost = 1000000.0; + }else{ + pInfo->aConstraintUsage[iAfterKey].argvIndex = 2; + pInfo->aConstraintUsage[iAfterKey].omit = 1; + pInfo->estimatedCost = 1000.0; + } + } + + return SQLITE_OK; +} + +/* +** Open a new btreeinfo cursor. +*/ +static int cidxOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + CidxCursor *pRet; + int rc = SQLITE_OK; + + pRet = cidxMalloc(&rc, sizeof(CidxCursor)); + + *ppCursor = (sqlite3_vtab_cursor*)pRet; + return rc; +} + +/* +** Close a btreeinfo cursor. +*/ +static int cidxClose(sqlite3_vtab_cursor *pCursor){ + CidxCursor *pCsr = (CidxCursor*)pCursor; + sqlite3_finalize(pCsr->pStmt); + pCsr->pStmt = 0; + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Move a btreeinfo cursor to the next entry in the file. +*/ +static int cidxNext(sqlite3_vtab_cursor *pCursor){ + CidxCursor *pCsr = (CidxCursor*)pCursor; + int rc = sqlite3_step(pCsr->pStmt); + if( rc!=SQLITE_ROW ){ + rc = sqlite3_finalize(pCsr->pStmt); + pCsr->pStmt = 0; + if( rc!=SQLITE_OK ){ + sqlite3 *db = ((CidxTable*)pCsr->base.pVtab)->db; + cidxCursorError(pCsr, "Cursor error: %s", sqlite3_errmsg(db)); + } + }else{ + rc = SQLITE_OK; + } + return rc; +} + +/* We have reached EOF if previous sqlite3_step() returned +** anything other than SQLITE_ROW; +*/ +static int cidxEof(sqlite3_vtab_cursor *pCursor){ + CidxCursor *pCsr = (CidxCursor*)pCursor; + return pCsr->pStmt==0; +} + +static sqlite3_stmt *cidxPrepare( + int *pRc, CidxCursor *pCsr, const char *zFmt, ... +){ + sqlite3_stmt *pRet = 0; + char *zSql; + va_list ap; /* ... printf arguments */ + va_start(ap, zFmt); + + zSql = sqlite3_vmprintf(zFmt, ap); + if( *pRc==SQLITE_OK ){ + if( zSql==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + sqlite3 *db = ((CidxTable*)pCsr->base.pVtab)->db; + *pRc = sqlite3_prepare_v2(db, zSql, -1, &pRet, 0); + if( *pRc!=SQLITE_OK ){ + cidxCursorError(pCsr, "SQL error: %s", sqlite3_errmsg(db)); + } + } + } + sqlite3_free(zSql); + va_end(ap); + + return pRet; +} + +static void cidxFinalize(int *pRc, sqlite3_stmt *pStmt){ + int rc = sqlite3_finalize(pStmt); + if( *pRc==SQLITE_OK ) *pRc = rc; +} + +char *cidxStrdup(int *pRc, const char *zStr){ + char *zRet = 0; + if( *pRc==SQLITE_OK ){ + int n = strlen(zStr); + zRet = cidxMalloc(pRc, n+1); + if( zRet ) memcpy(zRet, zStr, n+1); + } + return zRet; +} + +static int cidxLookupIndex( + CidxCursor *pCsr, /* Cursor object */ + const char *zIdx, /* Name of index to look up */ + char **pzTab, /* OUT: Table name */ + char **pzCurrentKey, /* OUT: Expression for current_key */ + char **pzOrderBy, /* OUT: ORDER BY expression list */ + char **pzSubWhere, /* OUT: sub-query WHERE clause */ + char **pzSubExpr /* OUT: sub-query WHERE clause */ +){ + int rc = SQLITE_OK; + char *zTab = 0; + char *zCurrentKey = 0; + char *zOrderBy = 0; + char *zSubWhere = 0; + char *zSubExpr = 0; + + sqlite3_stmt *pFindTab = 0; + sqlite3_stmt *pGroup = 0; + + /* Find the table */ + pFindTab = cidxPrepare(&rc, pCsr, + "SELECT tbl_name FROM sqlite_master WHERE name=%Q AND type='index'", + zIdx + ); + if( rc==SQLITE_OK && sqlite3_step(pFindTab)==SQLITE_ROW ){ + zTab = cidxStrdup(&rc, (const char*)sqlite3_column_text(pFindTab, 0)); + } + cidxFinalize(&rc, pFindTab); + if( rc==SQLITE_OK && zTab==0 ){ + rc = SQLITE_ERROR; + } + + pGroup = cidxPrepare(&rc, pCsr, + "SELECT group_concat(" + " coalesce(name, 'rowid'), '|| '','' ||'" + ") AS zCurrentKey," + " group_concat(" + " coalesce(name, 'rowid') || CASE WHEN desc THEN ' DESC' ELSE '' END," + " ', '" + ") AS zOrderBy," + " group_concat(" + " CASE WHEN key==1 THEN NULL ELSE " + " coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') " + " END," + " 'AND '" + ") AS zSubWhere," + " group_concat(" + " CASE WHEN key==0 THEN NULL ELSE " + " coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') " + " END," + " 'AND '" + ") AS zSubExpr " + " FROM pragma_index_xinfo(%Q);" + , zIdx, zIdx, zIdx + ); + if( rc==SQLITE_OK && sqlite3_step(pGroup)==SQLITE_ROW ){ + zCurrentKey = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 0)); + zOrderBy = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 1)); + zSubWhere = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 2)); + zSubExpr = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 3)); + } + cidxFinalize(&rc, pGroup); + + if( rc!=SQLITE_OK ){ + sqlite3_free(zTab); + sqlite3_free(zCurrentKey); + sqlite3_free(zOrderBy); + sqlite3_free(zSubWhere); + sqlite3_free(zSubExpr); + }else{ + *pzTab = zTab; + *pzCurrentKey = zCurrentKey; + *pzOrderBy = zOrderBy; + *pzSubWhere = zSubWhere; + *pzSubExpr = zSubExpr; + } + + return rc; +} + +/* +** Position a cursor back to the beginning. +*/ +static int cidxFilter( + sqlite3_vtab_cursor *pCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + int rc = SQLITE_OK; + CidxCursor *pCsr = (CidxCursor*)pCursor; + const char *zIdxName = 0; + const char *zAfterKey = 0; + + if( argc>0 ){ + zIdxName = (const char*)sqlite3_value_text(argv[0]); + if( argc>1 ){ + zAfterKey = (const char*)sqlite3_value_text(argv[1]); + } + } + + if( zIdxName ){ + char *zTab = 0; + char *zCurrentKey = 0; + char *zOrderBy = 0; + char *zSubWhere = 0; + char *zSubExpr = 0; + + rc = cidxLookupIndex(pCsr, zIdxName, + &zTab, &zCurrentKey, &zOrderBy, &zSubWhere, &zSubExpr + ); + pCsr->pStmt = cidxPrepare(&rc, pCsr, + "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM %Q AS %Q ORDER BY %s", + zSubExpr, zTab, zSubWhere, zCurrentKey, zTab, zIdxName, zOrderBy + ); + + sqlite3_free(zTab); + sqlite3_free(zCurrentKey); + sqlite3_free(zOrderBy); + sqlite3_free(zSubWhere); + sqlite3_free(zSubExpr); + } + + if( pCsr->pStmt ){ + assert( rc==SQLITE_OK ); + rc = cidxNext(pCursor); + } + return rc; +} + +/* Return a column for the sqlite_btreeinfo table */ +static int cidxColumn( + sqlite3_vtab_cursor *pCursor, + sqlite3_context *ctx, + int iCol +){ + CidxCursor *pCsr = (CidxCursor*)pCursor; + assert( iCol==0 || iCol==1 ); + if( iCol==0 ){ + const char *zVal = 0; + if( sqlite3_column_type(pCsr->pStmt, 0)==SQLITE_INTEGER ){ + if( sqlite3_column_int(pCsr->pStmt, 0)==0 ){ + zVal = "row data mismatch"; + } + }else{ + zVal = "row missing"; + } + sqlite3_result_text(ctx, zVal, -1, SQLITE_STATIC); + }else{ + sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, 1)); + } + return SQLITE_OK; +} + +/* Return the ROWID for the sqlite_btreeinfo table */ +static int cidxRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + *pRowid = 0; + return SQLITE_OK; +} + +/* +** Register the virtual table modules with the database handle passed +** as the only argument. +*/ +static int ciInit(sqlite3 *db){ + static sqlite3_module cidx_module = { + 0, /* iVersion */ + 0, /* xCreate */ + cidxConnect, /* xConnect */ + cidxBestIndex, /* xBestIndex */ + cidxDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + cidxOpen, /* xOpen - open a cursor */ + cidxClose, /* xClose - close a cursor */ + cidxFilter, /* xFilter - configure scan constraints */ + cidxNext, /* xNext - advance a cursor */ + cidxEof, /* xEof - check for end of scan */ + cidxColumn, /* xColumn - read data */ + cidxRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + }; + return sqlite3_create_module(db, "incremental_index_check", &cidx_module, 0); +} + +/* +** Extension load function. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_checkindex_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + return ciInit(db); +} diff --git a/manifest b/manifest index 2d13f91b8d..e0740cdd7b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Handle\sthe\sCtrl-C\sevent\sfor\sthe\sshell\son\sWin32. -D 2017-10-27T17:09:44.546 +C Add\sthe\sstart\sof\sthe\s"incremental_index_check"\svirtual\stable\sin\next/repair/checkindex.c.\sFor\sincremental\sverification\sof\sindex\scontents. +D 2017-10-27T20:53:16.451 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -328,6 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 +F ext/repair/checkindex.c 924432d01fabff8df8a758ef29d7124483653cd7874787564664e0eea8e267b1 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -653,6 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b +F test/checkindex.test 2dc7bd4c0de8ba7a8af0b6d3beaa6759d57b88c62e10ae4d158e9f544982d5d4 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1666,7 +1668,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8d3cc928a8f0c7b2616c5c75af9d3a50bd4f0fe8e4ccab545ab8648cbfbb4b7f -R b04098ed5c4b7451669ce8c4fddf24e2 -U mistachkin -Z 04b0a33e7f8b0430861d645ce26e5f19 +P c8aaf37117ed4a23bbd15dc481788735efcb77bff98be423eca3521c0a9270de +R 38fb5f31777282a52c65b4e266007341 +T *branch * checkindex +T *sym-checkindex * +T -sym-trunk * +U dan +Z 12607f1e180da6aa96255bb6397017d4 diff --git a/manifest.uuid b/manifest.uuid index 478c761165..9ee1c84b8f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c8aaf37117ed4a23bbd15dc481788735efcb77bff98be423eca3521c0a9270de \ No newline at end of file +d5b9dada471358a2864727759648b763bf6890fc2521fac53c0d8216017d39b7 \ No newline at end of file diff --git a/test/checkindex.test b/test/checkindex.test new file mode 100644 index 0000000000..841dca75a2 --- /dev/null +++ b/test/checkindex.test @@ -0,0 +1,91 @@ +# 2017-10-11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the checkindex extension. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix checkindex + +ifcapable !vtab||!compound { + finish_test + return +} + +if {[file exists ../checkindex.so]==0} { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); + INSERT INTO t1 VALUES('one', 2); + INSERT INTO t1 VALUES('two', 4); + INSERT INTO t1 VALUES('three', 6); + INSERT INTO t1 VALUES('four', 8); + INSERT INTO t1 VALUES('five', 10); +} + +db enable_load_extension 1 +do_execsql_test 1.1 { + SELECT load_extension('../checkindex.so'); +} {{}} + +do_execsql_test 1.2 { + SELECT errmsg IS NULL, current_key FROM incremental_index_check('i1'); +} { + 1 five,5 + 1 four,4 + 1 one,1 + 1 three,3 + 1 two,2 +} + + +do_test 1.3 { + set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t1' }] + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot + db eval {CREATE TABLE xt1(a, b)} + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 + + execsql { + UPDATE xt1 SET a='six' WHERE rowid=3; + DELETE FROM xt1 WHERE rowid = 5; + } + + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1 +} {} + +do_execsql_test 1.4 { + SELECT errmsg IS NULL, current_key FROM incremental_index_check('i1'); +} { + 0 five,5 + 1 four,4 + 1 one,1 + 0 three,3 + 1 two,2 +} +do_execsql_test 1.5 { + SELECT errmsg, current_key FROM incremental_index_check('i1'); +} { + {row missing} five,5 + {} four,4 + {} one,1 + {row data mismatch} three,3 + {} two,2 +} + + + +finish_test + From 19465e7380f725f9bada9c338e55d7e95bf46ef3 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 28 Oct 2017 20:31:25 +0000 Subject: [PATCH 02/20] Fix "after" parameter handling in the incremental_index_check code. FossilOrigin-Name: c40c3c62e996044f31ca49ffc2edb2cc0320e69956f7ee6fe3e9012200e0d9a0 --- ext/repair/checkindex.c | 238 ++++++++++++++++++++++++++++++++++++++-- manifest | 17 ++- manifest.uuid | 2 +- test/checkindex.test | 175 +++++++++++++++++++++++++---- 4 files changed, 389 insertions(+), 43 deletions(-) diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c index a74d1d4b75..f86cfdae7a 100644 --- a/ext/repair/checkindex.c +++ b/ext/repair/checkindex.c @@ -45,6 +45,13 @@ struct CidxCursor { sqlite3_stmt *pStmt; }; +typedef struct CidxColumn CidxColumn; +struct CidxColumn { + char *zName; + char *zColl; + int bDesc; +}; + static void *cidxMalloc(int *pRc, int n){ void *pRet = 0; assert( n!=0 ); @@ -193,6 +200,23 @@ static int cidxEof(sqlite3_vtab_cursor *pCursor){ return pCsr->pStmt==0; } +static char *cidxMprintf(int *pRc, const char *zFmt, ...){ + char *zRet = 0; + va_list ap; + va_start(ap, zFmt); + zRet = sqlite3_vmprintf(zFmt, ap); + if( *pRc==SQLITE_OK ){ + if( zRet==0 ){ + *pRc = SQLITE_NOMEM; + } + }else{ + sqlite3_free(zRet); + zRet = 0; + } + va_end(ap); + return zRet; +} + static sqlite3_stmt *cidxPrepare( int *pRc, CidxCursor *pCsr, const char *zFmt, ... ){ @@ -237,6 +261,8 @@ char *cidxStrdup(int *pRc, const char *zStr){ static int cidxLookupIndex( CidxCursor *pCsr, /* Cursor object */ const char *zIdx, /* Name of index to look up */ + int *pnCol, /* OUT: Number of columns in index */ + CidxColumn **paCol, /* OUT: Columns */ char **pzTab, /* OUT: Table name */ char **pzCurrentKey, /* OUT: Expression for current_key */ char **pzOrderBy, /* OUT: ORDER BY expression list */ @@ -249,6 +275,7 @@ static int cidxLookupIndex( char *zOrderBy = 0; char *zSubWhere = 0; char *zSubExpr = 0; + CidxColumn *aCol = 0; sqlite3_stmt *pFindTab = 0; sqlite3_stmt *pGroup = 0; @@ -268,7 +295,7 @@ static int cidxLookupIndex( pGroup = cidxPrepare(&rc, pCsr, "SELECT group_concat(" - " coalesce(name, 'rowid'), '|| '','' ||'" + " coalesce('quote(' || name || ')', 'rowid'), '|| '','' ||'" ") AS zCurrentKey," " group_concat(" " coalesce(name, 'rowid') || CASE WHEN desc THEN ' DESC' ELSE '' END," @@ -278,14 +305,15 @@ static int cidxLookupIndex( " CASE WHEN key==1 THEN NULL ELSE " " coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') " " END," - " 'AND '" + " ' AND '" ") AS zSubWhere," " group_concat(" " CASE WHEN key==0 THEN NULL ELSE " " coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') " " END," - " 'AND '" - ") AS zSubExpr " + " ' AND '" + ") AS zSubExpr," + " count(*) AS nCol" " FROM pragma_index_xinfo(%Q);" , zIdx, zIdx, zIdx ); @@ -294,8 +322,49 @@ static int cidxLookupIndex( zOrderBy = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 1)); zSubWhere = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 2)); zSubExpr = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 3)); + *pnCol = sqlite3_column_int(pGroup, 4); } cidxFinalize(&rc, pGroup); + + pGroup = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx); + if( rc==SQLITE_OK ){ + int nByte = 0; + int nCol = 0; + while( sqlite3_step(pGroup)==SQLITE_ROW ){ + const char *zName = (const char*)sqlite3_column_text(pGroup, 2); + const char *zColl = (const char*)sqlite3_column_text(pGroup, 4); + if( zName==0 ) zName = "rowid"; + nCol++; + nByte += strlen(zName)+1 + strlen(zColl)+1; + } + rc = sqlite3_reset(pGroup); + aCol = (CidxColumn*)cidxMalloc(&rc, sizeof(CidxColumn)*nCol + nByte); + + if( rc==SQLITE_OK ){ + int iCol = 0; + char *z = (char*)&aCol[nCol]; + while( sqlite3_step(pGroup)==SQLITE_ROW ){ + int nName, nColl; + const char *zName = (const char*)sqlite3_column_text(pGroup, 2); + const char *zColl = (const char*)sqlite3_column_text(pGroup, 4); + if( zName==0 ) zName = "rowid"; + + nName = strlen(zName); + nColl = strlen(zColl); + memcpy(z, zName, nName); + aCol[iCol].zName = z; + z += nName+1; + + memcpy(z, zColl, nColl); + aCol[iCol].zColl = z; + z += nColl+1; + + aCol[iCol].bDesc = sqlite3_column_int(pGroup, 3); + iCol++; + } + } + cidxFinalize(&rc, pGroup); + } if( rc!=SQLITE_OK ){ sqlite3_free(zTab); @@ -303,17 +372,128 @@ static int cidxLookupIndex( sqlite3_free(zOrderBy); sqlite3_free(zSubWhere); sqlite3_free(zSubExpr); + sqlite3_free(aCol); }else{ *pzTab = zTab; *pzCurrentKey = zCurrentKey; *pzOrderBy = zOrderBy; *pzSubWhere = zSubWhere; *pzSubExpr = zSubExpr; + *paCol = aCol; } return rc; } +static int cidxDecodeAfter( + CidxCursor *pCsr, + int nCol, + const char *zAfterKey, + char ***pazAfter +){ + char **azAfter; + int rc = SQLITE_OK; + int nAfterKey = strlen(zAfterKey); + + azAfter = cidxMalloc(&rc, sizeof(char*)*nCol + nAfterKey+1); + if( rc==SQLITE_OK ){ + int i; + char *zCopy = (char*)&azAfter[nCol]; + char *p = zCopy; + memcpy(zCopy, zAfterKey, nAfterKey+1); + for(i=0; i='0' && *p<='9') + || *p=='.' || *p=='+' || *p=='-' || *p=='e' || *p=='E' + ){ + p++; + } + } + + while( *p==' ' ) p++; + if( *p!=(i==(nCol-1) ? '\0' : ',') ){ + goto parse_error; + } + *p++ = '\0'; + } + } + + *pazAfter = azAfter; + return rc; + + parse_error: + sqlite3_free(azAfter); + *pazAfter = 0; + cidxCursorError(pCsr, "%s", "error parsing after value"); + return SQLITE_ERROR; +} + +static char *cidxWhere( + int *pRc, CidxColumn *aCol, char **azAfter, int iGt, int bLastIsNull +){ + char *zRet = 0; + const char *zSep = ""; + int i; + + for(i=0; i"), + azAfter[iGt] + ); + }else{ + zRet = cidxMprintf(pRc, "%z%s%s IS NOT NULL", zRet, zSep, aCol[iGt].zName); + } + + return zRet; +} + +static char *cidxColumnList(int *pRc, CidxColumn *aCol, int nCol){ + int i; + char *zRet = 0; + const char *zSep = ""; + for(i=0; ipStmt = cidxPrepare(&rc, pCsr, - "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM %Q AS %Q ORDER BY %s", - zSubExpr, zTab, zSubWhere, zCurrentKey, zTab, zIdxName, zOrderBy + &nCol, &aCol, &zTab, &zCurrentKey, &zOrderBy, &zSubWhere, &zSubExpr ); + if( rc==SQLITE_OK && zAfterKey ){ + rc = cidxDecodeAfter(pCsr, nCol, zAfterKey, &azAfter); + } + + if( rc || zAfterKey==0 ){ + pCsr->pStmt = cidxPrepare(&rc, pCsr, + "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM %Q AS %Q ORDER BY %s", + zSubExpr, zTab, zSubWhere, zCurrentKey, zTab, zIdxName, zOrderBy + ); + /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt)); */ + }else{ + char *zList = cidxColumnList(&rc, aCol, nCol); + const char *zSep = ""; + char *zSql; + int i; + + zSql = cidxMprintf(&rc, "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (", + zSubExpr, zTab, zSubWhere, zCurrentKey + ); + for(i=nCol-1; i>=0; i--){ + int j; + if( aCol[i].bDesc && azAfter[i]==0 ) continue; + for(j=0; j<2; j++){ + char *zWhere = cidxWhere(&rc, aCol, azAfter, i, j); + zSql = cidxMprintf(&rc, + "%z%s SELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)", + zSql, zSep, zList, zTab, zWhere, zOrderBy + ); + zSep = " UNION ALL "; + if( aCol[i].bDesc==0 ) break; + } + } + zSql = cidxMprintf(&rc, "%z) AS %Q", zSql, zIdxName); + sqlite3_free(zList); + + /* printf("SQL: %s\n", zSql); */ + pCsr->pStmt = cidxPrepare(&rc, pCsr, "%z", zSql); + } + sqlite3_free(zTab); sqlite3_free(zCurrentKey); sqlite3_free(zOrderBy); sqlite3_free(zSubWhere); sqlite3_free(zSubExpr); + sqlite3_free(aCol); + sqlite3_free(azAfter); } if( pCsr->pStmt ){ diff --git a/manifest b/manifest index e0740cdd7b..8f56822a50 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sstart\sof\sthe\s"incremental_index_check"\svirtual\stable\sin\next/repair/checkindex.c.\sFor\sincremental\sverification\sof\sindex\scontents. -D 2017-10-27T20:53:16.451 +C Fix\s"after"\sparameter\shandling\sin\sthe\sincremental_index_check\scode. +D 2017-10-28T20:31:25.564 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -328,7 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 -F ext/repair/checkindex.c 924432d01fabff8df8a758ef29d7124483653cd7874787564664e0eea8e267b1 +F ext/repair/checkindex.c 62df3d58ddf2988ab5bc9a7e3a9f16d16c3844a988df5cb702a2d0f4d2dd5bf4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -654,7 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b -F test/checkindex.test 2dc7bd4c0de8ba7a8af0b6d3beaa6759d57b88c62e10ae4d158e9f544982d5d4 +F test/checkindex.test 48cf7b237756f4691e7a12be78e1719d5b0456e7e205b87da380f3a6bec11966 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1668,10 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c8aaf37117ed4a23bbd15dc481788735efcb77bff98be423eca3521c0a9270de -R 38fb5f31777282a52c65b4e266007341 -T *branch * checkindex -T *sym-checkindex * -T -sym-trunk * +P d5b9dada471358a2864727759648b763bf6890fc2521fac53c0d8216017d39b7 +R 31f10dd93d0cbe363e20ad50a747a726 U dan -Z 12607f1e180da6aa96255bb6397017d4 +Z 019388109f5e5cd896dce5eabbc0252a diff --git a/manifest.uuid b/manifest.uuid index 9ee1c84b8f..45cec907d3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d5b9dada471358a2864727759648b763bf6890fc2521fac53c0d8216017d39b7 \ No newline at end of file +c40c3c62e996044f31ca49ffc2edb2cc0320e69956f7ee6fe3e9012200e0d9a0 \ No newline at end of file diff --git a/test/checkindex.test b/test/checkindex.test index 841dca75a2..bafa42334c 100644 --- a/test/checkindex.test +++ b/test/checkindex.test @@ -34,6 +34,8 @@ do_execsql_test 1.0 { INSERT INTO t1 VALUES('three', 6); INSERT INTO t1 VALUES('four', 8); INSERT INTO t1 VALUES('five', 10); + + CREATE INDEX i2 ON t1(a DESC); } db enable_load_extension 1 @@ -41,18 +43,61 @@ do_execsql_test 1.1 { SELECT load_extension('../checkindex.so'); } {{}} +proc incr_index_check {idx nStep} { + set Q { + SELECT errmsg, current_key FROM incremental_index_check($idx, $after) + LIMIT $nStep + } + + set res [list] + while {1} { + unset -nocomplain current_key + set res1 [db eval $Q] + if {[llength $res1]==0} break + set res [concat $res $res1] + set after [lindex $res end] + } + + return $res +} + +proc do_index_check_test {tn idx res} { + uplevel [list do_execsql_test $tn.1 " + SELECT errmsg, current_key FROM incremental_index_check('$idx'); + " $res] + + uplevel [list do_test $tn.2 "incr_index_check $idx 1" [list {*}$res]] + #uplevel [list do_test $tn.3 "incr_index_check $idx 2" [list {*}$res]] + #uplevel [list do_test $tn.4 "incr_index_check $idx 5" [list {*}$res]] +} + do_execsql_test 1.2 { SELECT errmsg IS NULL, current_key FROM incremental_index_check('i1'); } { - 1 five,5 - 1 four,4 - 1 one,1 - 1 three,3 - 1 two,2 + 1 'five',5 + 1 'four',4 + 1 'one',1 + 1 'three',3 + 1 'two',2 } +do_index_check_test 1.3 i1 { + {} 'five',5 + {} 'four',4 + {} 'one',1 + {} 'three',3 + {} 'two',2 +} -do_test 1.3 { +do_index_check_test 1.4 i2 { + {} 'two',2 + {} 'three',3 + {} 'one',1 + {} 'four',4 + {} 'five',5 +} + +do_test 1.5 { set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t1' }] sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot db eval {CREATE TABLE xt1(a, b)} @@ -66,26 +111,110 @@ do_test 1.3 { sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1 } {} -do_execsql_test 1.4 { - SELECT errmsg IS NULL, current_key FROM incremental_index_check('i1'); -} { - 0 five,5 - 1 four,4 - 1 one,1 - 0 three,3 - 1 two,2 -} -do_execsql_test 1.5 { - SELECT errmsg, current_key FROM incremental_index_check('i1'); -} { - {row missing} five,5 - {} four,4 - {} one,1 - {row data mismatch} three,3 - {} two,2 +do_index_check_test 1.6 i1 { + {row missing} 'five',5 + {} 'four',4 + {} 'one',1 + {row data mismatch} 'three',3 + {} 'two',2 } +do_index_check_test 1.7 i2 { + {} 'two',2 + {row data mismatch} 'three',3 + {} 'one',1 + {} 'four',4 + {row missing} 'five',5 +} +#-------------------------------------------------------------------------- +do_execsql_test 2.0 { + + CREATE TABLE t2(a INTEGER PRIMARY KEY, b, c, d); + + INSERT INTO t2 VALUES(1, NULL, 1, 1); + INSERT INTO t2 VALUES(2, 1, NULL, 1); + INSERT INTO t2 VALUES(3, 1, 1, NULL); + + INSERT INTO t2 VALUES(4, 2, 2, 1); + INSERT INTO t2 VALUES(5, 2, 2, 2); + INSERT INTO t2 VALUES(6, 2, 2, 3); + + INSERT INTO t2 VALUES(7, 2, 2, 1); + INSERT INTO t2 VALUES(8, 2, 2, 2); + INSERT INTO t2 VALUES(9, 2, 2, 3); + + CREATE INDEX i3 ON t2(b, c, d); + CREATE INDEX i4 ON t2(b DESC, c DESC, d DESC); + CREATE INDEX i5 ON t2(d, c DESC, b); +} + +do_index_check_test 2.1 i3 { + {} NULL,1,1,1 + {} 1,NULL,1,2 + {} 1,1,NULL,3 + {} 2,2,1,4 + {} 2,2,1,7 + {} 2,2,2,5 + {} 2,2,2,8 + {} 2,2,3,6 + {} 2,2,3,9 +} + +do_index_check_test 2.2 i4 { + {} 2,2,3,6 + {} 2,2,3,9 + {} 2,2,2,5 + {} 2,2,2,8 + {} 2,2,1,4 + {} 2,2,1,7 + {} 1,1,NULL,3 + {} 1,NULL,1,2 + {} NULL,1,1,1 +} + +do_index_check_test 2.3 i5 { + {} NULL,1,1,3 + {} 1,2,2,4 + {} 1,2,2,7 + {} 1,1,NULL,1 + {} 1,NULL,1,2 + {} 2,2,2,5 + {} 2,2,2,8 + {} 3,2,2,6 + {} 3,2,2,9 +} + +do_execsql_test 3.0 { + + CREATE TABLE t3(w, x, y, z PRIMARY KEY) WITHOUT ROWID; + CREATE INDEX t3wxy ON t3(w, x, y); + CREATE INDEX t3wxy2 ON t3(w DESC, x DESC, y DESC); + + INSERT INTO t3 VALUES(NULL, NULL, NULL, 1); + INSERT INTO t3 VALUES(NULL, NULL, NULL, 2); + INSERT INTO t3 VALUES(NULL, NULL, NULL, 3); + + INSERT INTO t3 VALUES('a', NULL, NULL, 4); + INSERT INTO t3 VALUES('a', NULL, NULL, 5); + INSERT INTO t3 VALUES('a', NULL, NULL, 6); + + INSERT INTO t3 VALUES('a', 'b', NULL, 7); + INSERT INTO t3 VALUES('a', 'b', NULL, 8); + INSERT INTO t3 VALUES('a', 'b', NULL, 9); + +} + +do_index_check_test 3.1 t3wxy { + {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3 + {} 'a',NULL,NULL,4 {} 'a',NULL,NULL,5 {} 'a',NULL,NULL,6 + {} 'a','b',NULL,7 {} 'a','b',NULL,8 {} 'a','b',NULL,9 +} +do_index_check_test 3.2 t3wxy2 { + {} 'a','b',NULL,7 {} 'a','b',NULL,8 {} 'a','b',NULL,9 + {} 'a',NULL,NULL,4 {} 'a',NULL,NULL,5 {} 'a',NULL,NULL,6 + {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3 +} finish_test From 6fc66179092c843b8f352579709417f6f5e5190d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 30 Oct 2017 08:04:38 +0000 Subject: [PATCH 03/20] Fix an issue in incremental_index_check with indexes that use non-default collation sequences. FossilOrigin-Name: 3ebb2351e2650d263029d2c0042683cba3529c9d3f76b5f994f2e737b84d3f67 --- ext/repair/checkindex.c | 5 +++-- manifest | 14 ++++++------- manifest.uuid | 2 +- test/checkindex.test | 44 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 10 deletions(-) diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c index f86cfdae7a..b70b57122c 100644 --- a/ext/repair/checkindex.c +++ b/ext/repair/checkindex.c @@ -298,7 +298,8 @@ static int cidxLookupIndex( " coalesce('quote(' || name || ')', 'rowid'), '|| '','' ||'" ") AS zCurrentKey," " group_concat(" - " coalesce(name, 'rowid') || CASE WHEN desc THEN ' DESC' ELSE '' END," + " coalesce(name, 'rowid') || ' COLLATE ' || coll " + " || CASE WHEN desc THEN ' DESC' ELSE '' END," " ', '" ") AS zOrderBy," " group_concat(" @@ -313,7 +314,7 @@ static int cidxLookupIndex( " END," " ' AND '" ") AS zSubExpr," - " count(*) AS nCol" + " count(*) AS nCol" " FROM pragma_index_xinfo(%Q);" , zIdx, zIdx, zIdx ); diff --git a/manifest b/manifest index 8f56822a50..1aefe38098 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\s"after"\sparameter\shandling\sin\sthe\sincremental_index_check\scode. -D 2017-10-28T20:31:25.564 +C Fix\san\sissue\sin\sincremental_index_check\swith\sindexes\sthat\suse\snon-default\ncollation\ssequences. +D 2017-10-30T08:04:38.448 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -328,7 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 -F ext/repair/checkindex.c 62df3d58ddf2988ab5bc9a7e3a9f16d16c3844a988df5cb702a2d0f4d2dd5bf4 +F ext/repair/checkindex.c ff736821b84286ace872a5fa793aee39232fa804cb79c40c3086686e94d2f7e0 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -654,7 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b -F test/checkindex.test 48cf7b237756f4691e7a12be78e1719d5b0456e7e205b87da380f3a6bec11966 +F test/checkindex.test ea3ae087539e36cf8f63f65afe7347c3325147f6ddd873fb6bbb2214804d08f2 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P d5b9dada471358a2864727759648b763bf6890fc2521fac53c0d8216017d39b7 -R 31f10dd93d0cbe363e20ad50a747a726 +P c40c3c62e996044f31ca49ffc2edb2cc0320e69956f7ee6fe3e9012200e0d9a0 +R 35b9a81e4b8b6b857354a831ff9c29c1 U dan -Z 019388109f5e5cd896dce5eabbc0252a +Z cd0262c8d2e550da2f2d47bdfb3d2467 diff --git a/manifest.uuid b/manifest.uuid index 45cec907d3..7ea946e219 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c40c3c62e996044f31ca49ffc2edb2cc0320e69956f7ee6fe3e9012200e0d9a0 \ No newline at end of file +3ebb2351e2650d263029d2c0042683cba3529c9d3f76b5f994f2e737b84d3f67 \ No newline at end of file diff --git a/test/checkindex.test b/test/checkindex.test index bafa42334c..706171ff19 100644 --- a/test/checkindex.test +++ b/test/checkindex.test @@ -185,6 +185,7 @@ do_index_check_test 2.3 i5 { {} 3,2,2,9 } +#-------------------------------------------------------------------------- do_execsql_test 3.0 { CREATE TABLE t3(w, x, y, z PRIMARY KEY) WITHOUT ROWID; @@ -216,5 +217,48 @@ do_index_check_test 3.2 t3wxy2 { {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3 } +#-------------------------------------------------------------------------- +# Test with an index that uses non-default collation sequences. +# +do_execsql_test 4.0 { + CREATE TABLE t4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT); + INSERT INTO t4 VALUES(1, 'aaa', 'bbb'); + INSERT INTO t4 VALUES(2, 'AAA', 'CCC'); + INSERT INTO t4 VALUES(3, 'aab', 'ddd'); + INSERT INTO t4 VALUES(4, 'AAB', 'EEE'); + + CREATE INDEX t4cc ON t4(c1 COLLATE nocase, c2 COLLATE nocase); +} + +do_index_check_test 4.1 t4cc { + {} 'aaa','bbb',1 + {} 'AAA','CCC',2 + {} 'aab','ddd',3 + {} 'AAB','EEE',4 +} + +do_test 4.2 { + set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t4' }] + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot + db eval {CREATE TABLE xt4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT)} + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 + + execsql { + UPDATE xt4 SET c1='hello' WHERE rowid=2; + DELETE FROM xt4 WHERE rowid = 3; + } + + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1 +} {} + +do_index_check_test 4.3 t4cc { + {} 'aaa','bbb',1 + {row data mismatch} 'AAA','CCC',2 + {row missing} 'aab','ddd',3 + {} 'AAB','EEE',4 +} + + + finish_test From 75650d4f9453db917f8198a8d276ed96590ffb8a Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 30 Oct 2017 17:05:18 +0000 Subject: [PATCH 04/20] In checkindex.c, use C code instead of SQL/group_concat() to compose various SQL clauses. This is to make it easier to support indexes on expressions. FossilOrigin-Name: 940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269 --- ext/repair/checkindex.c | 255 +++++++++++++++++++++------------------- manifest | 14 +-- manifest.uuid | 2 +- test/checkindex.test | 5 +- 4 files changed, 144 insertions(+), 132 deletions(-) diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c index b70b57122c..0393358567 100644 --- a/ext/repair/checkindex.c +++ b/ext/repair/checkindex.c @@ -47,9 +47,15 @@ struct CidxCursor { typedef struct CidxColumn CidxColumn; struct CidxColumn { - char *zName; - char *zColl; - int bDesc; + char *zExpr; /* Text for indexed expression */ + int bDesc; /* True for DESC columns, otherwise false */ + int bKey; /* Part of index, not PK */ +}; + +typedef struct CidxIndex CidxIndex; +struct CidxIndex { + int nCol; /* Elements in aCol[] array */ + CidxColumn aCol[1]; /* Array of indexed columns */ }; static void *cidxMalloc(int *pRc, int n){ @@ -258,29 +264,30 @@ char *cidxStrdup(int *pRc, const char *zStr){ return zRet; } +static void cidxFreeIndex(CidxIndex *pIdx){ + if( pIdx ){ + int i; + for(i=0; inCol; i++){ + sqlite3_free(pIdx->aCol[i].zExpr); + } + sqlite3_free(pIdx); + } +} + static int cidxLookupIndex( CidxCursor *pCsr, /* Cursor object */ const char *zIdx, /* Name of index to look up */ - int *pnCol, /* OUT: Number of columns in index */ - CidxColumn **paCol, /* OUT: Columns */ - char **pzTab, /* OUT: Table name */ - char **pzCurrentKey, /* OUT: Expression for current_key */ - char **pzOrderBy, /* OUT: ORDER BY expression list */ - char **pzSubWhere, /* OUT: sub-query WHERE clause */ - char **pzSubExpr /* OUT: sub-query WHERE clause */ + CidxIndex **ppIdx, /* OUT: Description of columns */ + char **pzTab /* OUT: Table name */ ){ int rc = SQLITE_OK; char *zTab = 0; - char *zCurrentKey = 0; - char *zOrderBy = 0; - char *zSubWhere = 0; - char *zSubExpr = 0; - CidxColumn *aCol = 0; + CidxIndex *pIdx = 0; sqlite3_stmt *pFindTab = 0; - sqlite3_stmt *pGroup = 0; + sqlite3_stmt *pInfo = 0; - /* Find the table */ + /* Find the table for this index. */ pFindTab = cidxPrepare(&rc, pCsr, "SELECT tbl_name FROM sqlite_master WHERE name=%Q AND type='index'", zIdx @@ -293,94 +300,35 @@ static int cidxLookupIndex( rc = SQLITE_ERROR; } - pGroup = cidxPrepare(&rc, pCsr, - "SELECT group_concat(" - " coalesce('quote(' || name || ')', 'rowid'), '|| '','' ||'" - ") AS zCurrentKey," - " group_concat(" - " coalesce(name, 'rowid') || ' COLLATE ' || coll " - " || CASE WHEN desc THEN ' DESC' ELSE '' END," - " ', '" - ") AS zOrderBy," - " group_concat(" - " CASE WHEN key==1 THEN NULL ELSE " - " coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') " - " END," - " ' AND '" - ") AS zSubWhere," - " group_concat(" - " CASE WHEN key==0 THEN NULL ELSE " - " coalesce(name, 'rowid') || ' IS \"%w\".' || coalesce(name, 'rowid') " - " END," - " ' AND '" - ") AS zSubExpr," - " count(*) AS nCol" - " FROM pragma_index_xinfo(%Q);" - , zIdx, zIdx, zIdx - ); - if( rc==SQLITE_OK && sqlite3_step(pGroup)==SQLITE_ROW ){ - zCurrentKey = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 0)); - zOrderBy = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 1)); - zSubWhere = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 2)); - zSubExpr = cidxStrdup(&rc, (const char*)sqlite3_column_text(pGroup, 3)); - *pnCol = sqlite3_column_int(pGroup, 4); - } - cidxFinalize(&rc, pGroup); - - pGroup = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx); + pInfo = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx); if( rc==SQLITE_OK ){ - int nByte = 0; - int nCol = 0; - while( sqlite3_step(pGroup)==SQLITE_ROW ){ - const char *zName = (const char*)sqlite3_column_text(pGroup, 2); - const char *zColl = (const char*)sqlite3_column_text(pGroup, 4); + int nAlloc = 0; + int iCol = 0; + + while( sqlite3_step(pInfo)==SQLITE_ROW ){ + const char *zName = (const char*)sqlite3_column_text(pInfo, 2); + const char *zColl = (const char*)sqlite3_column_text(pInfo, 4); + CidxColumn *p; if( zName==0 ) zName = "rowid"; - nCol++; - nByte += strlen(zName)+1 + strlen(zColl)+1; - } - rc = sqlite3_reset(pGroup); - aCol = (CidxColumn*)cidxMalloc(&rc, sizeof(CidxColumn)*nCol + nByte); - - if( rc==SQLITE_OK ){ - int iCol = 0; - char *z = (char*)&aCol[nCol]; - while( sqlite3_step(pGroup)==SQLITE_ROW ){ - int nName, nColl; - const char *zName = (const char*)sqlite3_column_text(pGroup, 2); - const char *zColl = (const char*)sqlite3_column_text(pGroup, 4); - if( zName==0 ) zName = "rowid"; - - nName = strlen(zName); - nColl = strlen(zColl); - memcpy(z, zName, nName); - aCol[iCol].zName = z; - z += nName+1; - - memcpy(z, zColl, nColl); - aCol[iCol].zColl = z; - z += nColl+1; - - aCol[iCol].bDesc = sqlite3_column_int(pGroup, 3); - iCol++; + if( iCol==nAlloc ){ + int nByte = sizeof(CidxIndex) + sizeof(CidxColumn)*(nAlloc+8); + pIdx = (CidxIndex*)sqlite3_realloc(pIdx, nByte); } + p = &pIdx->aCol[iCol++]; + p->zExpr = cidxMprintf(&rc, "\"%w\" COLLATE %s",zName,zColl); + p->bDesc = sqlite3_column_int(pInfo, 3); + p->bKey = sqlite3_column_int(pInfo, 5); + pIdx->nCol = iCol; } - cidxFinalize(&rc, pGroup); + cidxFinalize(&rc, pInfo); } if( rc!=SQLITE_OK ){ sqlite3_free(zTab); - sqlite3_free(zCurrentKey); - sqlite3_free(zOrderBy); - sqlite3_free(zSubWhere); - sqlite3_free(zSubExpr); - sqlite3_free(aCol); + cidxFreeIndex(pIdx); }else{ *pzTab = zTab; - *pzCurrentKey = zCurrentKey; - *pzOrderBy = zOrderBy; - *pzSubWhere = zSubWhere; - *pzSubExpr = zSubExpr; - *paCol = aCol; + *ppIdx = pIdx; } return rc; @@ -463,35 +411,91 @@ static char *cidxWhere( int i; for(i=0; i"), + zRet = cidxMprintf(pRc, "%z%s%s %s %s", zRet, + zSep, aCol[iGt].zExpr, (aCol[iGt].bDesc ? "<" : ">"), azAfter[iGt] ); }else{ - zRet = cidxMprintf(pRc, "%z%s%s IS NOT NULL", zRet, zSep, aCol[iGt].zName); + zRet = cidxMprintf(pRc, "%z%s%s IS NOT NULL", zRet, zSep, aCol[iGt].zExpr); } return zRet; } -static char *cidxColumnList(int *pRc, CidxColumn *aCol, int nCol){ - int i; +#define CIDX_CLIST_ALL 0 +#define CIDX_CLIST_ORDERBY 1 +#define CIDX_CLIST_CURRENT_KEY 2 +#define CIDX_CLIST_SUBWHERE 3 +#define CIDX_CLIST_SUBEXPR 4 + +/* +** This function returns various strings based on the contents of the +** CidxIndex structure and the eType parameter. +*/ +static char *cidxColumnList( + int *pRc, /* IN/OUT: Error code */ + const char *zIdx, + CidxIndex *pIdx, /* Indexed columns */ + int eType /* True to include ASC/DESC */ +){ char *zRet = 0; - const char *zSep = ""; - for(i=0; inCol; i++){ + CidxColumn *p = &pIdx->aCol[i]; + assert( pIdx->aCol[i].bDesc==0 || pIdx->aCol[i].bDesc==1 ); + switch( eType ){ + + case CIDX_CLIST_ORDERBY: + zRet = cidxMprintf(pRc, "%z%s%s%s",zRet,zSep,p->zExpr,aDir[p->bDesc]); + zSep = ","; + break; + + case CIDX_CLIST_CURRENT_KEY: + zRet = cidxMprintf(pRc, "%z%squote(%s)", zRet, zSep, p->zExpr); + zSep = "||','||"; + break; + + case CIDX_CLIST_SUBWHERE: + if( p->bKey==0 ){ + zRet = cidxMprintf(pRc, "%z%s%s IS \"%w\".%s", zRet, + zSep, p->zExpr, zIdx, p->zExpr + ); + zSep = " AND "; + } + break; + + case CIDX_CLIST_SUBEXPR: + if( p->bKey==1 ){ + zRet = cidxMprintf(pRc, "%z%s%s IS \"%w\".%s", zRet, + zSep, p->zExpr, zIdx, p->zExpr + ); + zSep = " AND "; + } + break; + + default: + assert( eType==CIDX_CLIST_ALL ); + zRet = cidxMprintf(pRc, "%z%s%s", zRet, zSep, p->zExpr); + zSep = ","; + break; + } + } } + return zRet; } @@ -516,21 +520,26 @@ static int cidxFilter( } if( zIdxName ){ - int nCol = 0; char *zTab = 0; char *zCurrentKey = 0; char *zOrderBy = 0; char *zSubWhere = 0; char *zSubExpr = 0; - char **azAfter = 0; - CidxColumn *aCol = 0; + char *zSrcList = 0; - rc = cidxLookupIndex(pCsr, zIdxName, - &nCol, &aCol, &zTab, &zCurrentKey, &zOrderBy, &zSubWhere, &zSubExpr - ); + char **azAfter = 0; + CidxIndex *pIdx = 0; + + rc = cidxLookupIndex(pCsr, zIdxName, &pIdx, &zTab); + + zOrderBy = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ORDERBY); + zCurrentKey = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_CURRENT_KEY); + zSubWhere = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBWHERE); + zSubExpr = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBEXPR); + /* zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL); */ if( rc==SQLITE_OK && zAfterKey ){ - rc = cidxDecodeAfter(pCsr, nCol, zAfterKey, &azAfter); + rc = cidxDecodeAfter(pCsr, pIdx->nCol, zAfterKey, &azAfter); } if( rc || zAfterKey==0 ){ @@ -538,9 +547,9 @@ static int cidxFilter( "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM %Q AS %Q ORDER BY %s", zSubExpr, zTab, zSubWhere, zCurrentKey, zTab, zIdxName, zOrderBy ); - /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt)); */ + /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt)); */ }else{ - char *zList = cidxColumnList(&rc, aCol, nCol); + char *zList = cidxColumnList(&rc, zIdxName, pIdx, 0); const char *zSep = ""; char *zSql; int i; @@ -548,17 +557,17 @@ static int cidxFilter( zSql = cidxMprintf(&rc, "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (", zSubExpr, zTab, zSubWhere, zCurrentKey ); - for(i=nCol-1; i>=0; i--){ + for(i=pIdx->nCol-1; i>=0; i--){ int j; - if( aCol[i].bDesc && azAfter[i]==0 ) continue; + if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue; for(j=0; j<2; j++){ - char *zWhere = cidxWhere(&rc, aCol, azAfter, i, j); + char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j); zSql = cidxMprintf(&rc, "%z%s SELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)", zSql, zSep, zList, zTab, zWhere, zOrderBy ); zSep = " UNION ALL "; - if( aCol[i].bDesc==0 ) break; + if( pIdx->aCol[i].bDesc==0 ) break; } } zSql = cidxMprintf(&rc, "%z) AS %Q", zSql, zIdxName); @@ -573,7 +582,7 @@ static int cidxFilter( sqlite3_free(zOrderBy); sqlite3_free(zSubWhere); sqlite3_free(zSubExpr); - sqlite3_free(aCol); + cidxFreeIndex(pIdx); sqlite3_free(azAfter); } @@ -584,7 +593,9 @@ static int cidxFilter( return rc; } -/* Return a column for the sqlite_btreeinfo table */ +/* +** Return a column value. +*/ static int cidxColumn( sqlite3_vtab_cursor *pCursor, sqlite3_context *ctx, diff --git a/manifest b/manifest index 1aefe38098..48fe0a91a2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sissue\sin\sincremental_index_check\swith\sindexes\sthat\suse\snon-default\ncollation\ssequences. -D 2017-10-30T08:04:38.448 +C In\scheckindex.c,\suse\sC\scode\sinstead\sof\sSQL/group_concat()\sto\scompose\svarious\nSQL\sclauses.\sThis\sis\sto\smake\sit\seasier\sto\ssupport\sindexes\son\sexpressions. +D 2017-10-30T17:05:18.290 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -328,7 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 -F ext/repair/checkindex.c ff736821b84286ace872a5fa793aee39232fa804cb79c40c3086686e94d2f7e0 +F ext/repair/checkindex.c 6168af2569681aba298f09a2bba358454f68c7de50df8d37a8b6f91924dd42c7 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -654,7 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b -F test/checkindex.test ea3ae087539e36cf8f63f65afe7347c3325147f6ddd873fb6bbb2214804d08f2 +F test/checkindex.test a5969b99755cb78129fe42bd50470a65e987d713ad7ff01e8c2a0c4d2333e8f4 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P c40c3c62e996044f31ca49ffc2edb2cc0320e69956f7ee6fe3e9012200e0d9a0 -R 35b9a81e4b8b6b857354a831ff9c29c1 +P 3ebb2351e2650d263029d2c0042683cba3529c9d3f76b5f994f2e737b84d3f67 +R 45b731228881e6db68d5d1d151d2789f U dan -Z cd0262c8d2e550da2f2d47bdfb3d2467 +Z 8f0b667700c59bebe92a45558d3d25e5 diff --git a/manifest.uuid b/manifest.uuid index 7ea946e219..5d8e25d480 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3ebb2351e2650d263029d2c0042683cba3529c9d3f76b5f994f2e737b84d3f67 \ No newline at end of file +940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269 \ No newline at end of file diff --git a/test/checkindex.test b/test/checkindex.test index 706171ff19..6648187559 100644 --- a/test/checkindex.test +++ b/test/checkindex.test @@ -67,10 +67,11 @@ proc do_index_check_test {tn idx res} { " $res] uplevel [list do_test $tn.2 "incr_index_check $idx 1" [list {*}$res]] - #uplevel [list do_test $tn.3 "incr_index_check $idx 2" [list {*}$res]] - #uplevel [list do_test $tn.4 "incr_index_check $idx 5" [list {*}$res]] + uplevel [list do_test $tn.3 "incr_index_check $idx 2" [list {*}$res]] + uplevel [list do_test $tn.4 "incr_index_check $idx 5" [list {*}$res]] } + do_execsql_test 1.2 { SELECT errmsg IS NULL, current_key FROM incremental_index_check('i1'); } { From e7a579d27d12ce936d5a536dffd7453bd0503235 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 30 Oct 2017 19:38:41 +0000 Subject: [PATCH 05/20] Add support for indexes on expressions to incremental_index_check. FossilOrigin-Name: 8c1c701fdbe0d56ee7f6f7d7b583aafde9fa14acc93ee8ecaddc8bb311e2bf52 --- ext/repair/checkindex.c | 215 +++++++++++++++++++++++++++++++--------- manifest | 14 +-- manifest.uuid | 2 +- test/checkindex.test | 58 +++++++++++ 4 files changed, 232 insertions(+), 57 deletions(-) diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c index 0393358567..fc91b74244 100644 --- a/ext/repair/checkindex.c +++ b/ext/repair/checkindex.c @@ -274,6 +274,110 @@ static void cidxFreeIndex(CidxIndex *pIdx){ } } +#define CIDX_PARSE_EOF 0 +#define CIDX_PARSE_COMMA 1 /* "," */ +#define CIDX_PARSE_OPEN 2 /* "(" */ +#define CIDX_PARSE_CLOSE 3 /* ")" */ + +static int cidxFindNext(const char *zIn, const char **pzOut){ + const char *z = zIn; + + while( 1 ){ + *pzOut = z; + switch( *z ){ + case '\0': + return CIDX_PARSE_EOF; + case '(': + return CIDX_PARSE_OPEN; + case ')': + return CIDX_PARSE_CLOSE; + case ',': + return CIDX_PARSE_COMMA; + + case '"': + case '\'': + case '`': { + char q = *z; + z++; + while( *z ){ + if( *z==q ){ + z++; + if( *z!=q ) break; + } + z++; + } + break; + } + + case '[': + while( *z++!=']' ); + break; + + default: + z++; + } + } + + assert( 0 ); + return -1; +} + +static int cidx_isspace(char c){ + return c==' ' || c=='\t' || c=='\r' || c=='\n'; +} + +static int cidx_isident(char c){ + return c<0 + || (c>='0' && c<='9') || (c>='a' && c<='z') + || (c>='A' && c<='Z') || c=='_'; +} + +static int cidxParseSQL(CidxCursor *pCsr, CidxIndex *pIdx, const char *zSql){ + const char *z = zSql; + const char *z1; + int e; + int rc = SQLITE_OK; + int nParen = 1; + CidxColumn *pCol = pIdx->aCol; + + e = cidxFindNext(z, &z); + if( e!=CIDX_PARSE_OPEN ) goto parse_error; + z1 = z+1; + z++; + while( nParen>0 ){ + e = cidxFindNext(z, &z); + if( e==CIDX_PARSE_EOF ) goto parse_error; + if( (e==CIDX_PARSE_COMMA || e==CIDX_PARSE_CLOSE) && nParen==1 ){ + const char *z2 = z; + if( pCol->zExpr ) goto parse_error; + + while( cidx_isspace(z[-1]) ) z--; + if( 0==sqlite3_strnicmp(&z[-3], "asc", 3) && 0==cidx_isident(z[-4]) ){ + z -= 3; + while( cidx_isspace(z[-1]) ) z--; + }else + if( 0==sqlite3_strnicmp(&z[-4], "desc", 4) && 0==cidx_isident(z[-5]) ){ + z -= 4; + while( cidx_isspace(z[-1]) ) z--; + } + + while( cidx_isspace(z1[0]) ) z1++; + pCol->zExpr = cidxMprintf(&rc, "%.*s", z-z1, z1); + pCol++; + z = z1 = z2+1; + } + if( e==CIDX_PARSE_OPEN ) nParen++; + if( e==CIDX_PARSE_CLOSE ) nParen--; + z++; + } + + return rc; + + parse_error: + cidxCursorError(pCsr, "Parse error in: %s", zSql); + return SQLITE_ERROR; +} + static int cidxLookupIndex( CidxCursor *pCsr, /* Cursor object */ const char *zIdx, /* Name of index to look up */ @@ -289,39 +393,50 @@ static int cidxLookupIndex( /* Find the table for this index. */ pFindTab = cidxPrepare(&rc, pCsr, - "SELECT tbl_name FROM sqlite_master WHERE name=%Q AND type='index'", + "SELECT tbl_name, sql FROM sqlite_master WHERE name=%Q AND type='index'", zIdx ); if( rc==SQLITE_OK && sqlite3_step(pFindTab)==SQLITE_ROW ){ + const char *zSql = (const char*)sqlite3_column_text(pFindTab, 1); zTab = cidxStrdup(&rc, (const char*)sqlite3_column_text(pFindTab, 0)); + + pInfo = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx); + if( rc==SQLITE_OK ){ + int nAlloc = 0; + int iCol = 0; + + while( sqlite3_step(pInfo)==SQLITE_ROW ){ + const char *zName = (const char*)sqlite3_column_text(pInfo, 2); + const char *zColl = (const char*)sqlite3_column_text(pInfo, 4); + CidxColumn *p; + if( zName==0 ) zName = "rowid"; + if( iCol==nAlloc ){ + int nByte = sizeof(CidxIndex) + sizeof(CidxColumn)*(nAlloc+8); + pIdx = (CidxIndex*)sqlite3_realloc(pIdx, nByte); + nAlloc += 8; + } + p = &pIdx->aCol[iCol++]; + p->bDesc = sqlite3_column_int(pInfo, 3); + p->bKey = sqlite3_column_int(pInfo, 5); + if( zSql==0 || p->bKey==0 ){ + p->zExpr = cidxMprintf(&rc, "\"%w\" COLLATE %s",zName,zColl); + }else{ + p->zExpr = 0; + } + pIdx->nCol = iCol; + } + cidxFinalize(&rc, pInfo); + } + + if( rc==SQLITE_OK && zSql ){ + rc = cidxParseSQL(pCsr, pIdx, zSql); + } } + cidxFinalize(&rc, pFindTab); if( rc==SQLITE_OK && zTab==0 ){ rc = SQLITE_ERROR; } - - pInfo = cidxPrepare(&rc, pCsr, "PRAGMA index_xinfo(%Q)", zIdx); - if( rc==SQLITE_OK ){ - int nAlloc = 0; - int iCol = 0; - - while( sqlite3_step(pInfo)==SQLITE_ROW ){ - const char *zName = (const char*)sqlite3_column_text(pInfo, 2); - const char *zColl = (const char*)sqlite3_column_text(pInfo, 4); - CidxColumn *p; - if( zName==0 ) zName = "rowid"; - if( iCol==nAlloc ){ - int nByte = sizeof(CidxIndex) + sizeof(CidxColumn)*(nAlloc+8); - pIdx = (CidxIndex*)sqlite3_realloc(pIdx, nByte); - } - p = &pIdx->aCol[iCol++]; - p->zExpr = cidxMprintf(&rc, "\"%w\" COLLATE %s",zName,zColl); - p->bDesc = sqlite3_column_int(pInfo, 3); - p->bKey = sqlite3_column_int(pInfo, 5); - pIdx->nCol = iCol; - } - cidxFinalize(&rc, pInfo); - } if( rc!=SQLITE_OK ){ sqlite3_free(zTab); @@ -411,22 +526,22 @@ static char *cidxWhere( int i; for(i=0; i"), azAfter[iGt] ); }else{ - zRet = cidxMprintf(pRc, "%z%s%s IS NOT NULL", zRet, zSep, aCol[iGt].zExpr); + zRet = cidxMprintf(pRc, "%z%s(%s) IS NOT NULL", zRet, zSep,aCol[iGt].zExpr); } return zRet; @@ -450,7 +565,7 @@ static char *cidxColumnList( ){ char *zRet = 0; if( *pRc==SQLITE_OK ){ - const char *aDir[2] = {" ASC", " DESC"}; + const char *aDir[2] = {"", " DESC"}; int i; const char *zSep = ""; @@ -460,19 +575,19 @@ static char *cidxColumnList( switch( eType ){ case CIDX_CLIST_ORDERBY: - zRet = cidxMprintf(pRc, "%z%s%s%s",zRet,zSep,p->zExpr,aDir[p->bDesc]); + zRet = cidxMprintf(pRc, "%z%s%d%s", zRet, zSep, i+1, aDir[p->bDesc]); zSep = ","; break; case CIDX_CLIST_CURRENT_KEY: - zRet = cidxMprintf(pRc, "%z%squote(%s)", zRet, zSep, p->zExpr); + zRet = cidxMprintf(pRc, "%z%squote(i%d)", zRet, zSep, i); zSep = "||','||"; break; case CIDX_CLIST_SUBWHERE: if( p->bKey==0 ){ - zRet = cidxMprintf(pRc, "%z%s%s IS \"%w\".%s", zRet, - zSep, p->zExpr, zIdx, p->zExpr + zRet = cidxMprintf(pRc, "%z%s%s IS i.i%d", zRet, + zSep, p->zExpr, i ); zSep = " AND "; } @@ -480,8 +595,8 @@ static char *cidxColumnList( case CIDX_CLIST_SUBEXPR: if( p->bKey==1 ){ - zRet = cidxMprintf(pRc, "%z%s%s IS \"%w\".%s", zRet, - zSep, p->zExpr, zIdx, p->zExpr + zRet = cidxMprintf(pRc, "%z%s%s IS i.i%d", zRet, + zSep, p->zExpr, i ); zSep = " AND "; } @@ -489,8 +604,8 @@ static char *cidxColumnList( default: assert( eType==CIDX_CLIST_ALL ); - zRet = cidxMprintf(pRc, "%z%s%s", zRet, zSep, p->zExpr); - zSep = ","; + zRet = cidxMprintf(pRc, "%z%s(%s) AS i%d", zRet, zSep, p->zExpr, i); + zSep = ", "; break; } } @@ -536,7 +651,7 @@ static int cidxFilter( zCurrentKey = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_CURRENT_KEY); zSubWhere = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBWHERE); zSubExpr = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_SUBEXPR); - /* zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL); */ + zSrcList = cidxColumnList(&rc, zIdxName, pIdx, CIDX_CLIST_ALL); if( rc==SQLITE_OK && zAfterKey ){ rc = cidxDecodeAfter(pCsr, pIdx->nCol, zAfterKey, &azAfter); @@ -544,17 +659,19 @@ static int cidxFilter( if( rc || zAfterKey==0 ){ pCsr->pStmt = cidxPrepare(&rc, pCsr, - "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM %Q AS %Q ORDER BY %s", - zSubExpr, zTab, zSubWhere, zCurrentKey, zTab, zIdxName, zOrderBy + "SELECT (SELECT %s FROM %Q AS t WHERE %s), %s " + "FROM (SELECT %s FROM %Q ORDER BY %s) AS i", + zSubExpr, zTab, zSubWhere, zCurrentKey, + zSrcList, zTab, zOrderBy ); - /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt)); */ + /* printf("SQL: %s\n", sqlite3_sql(pCsr->pStmt)); */ }else{ - char *zList = cidxColumnList(&rc, zIdxName, pIdx, 0); const char *zSep = ""; char *zSql; int i; - zSql = cidxMprintf(&rc, "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (", + zSql = cidxMprintf(&rc, + "SELECT (SELECT %s FROM %Q WHERE %s), %s FROM (", zSubExpr, zTab, zSubWhere, zCurrentKey ); for(i=pIdx->nCol-1; i>=0; i--){ @@ -562,16 +679,15 @@ static int cidxFilter( if( pIdx->aCol[i].bDesc && azAfter[i]==0 ) continue; for(j=0; j<2; j++){ char *zWhere = cidxWhere(&rc, pIdx->aCol, azAfter, i, j); - zSql = cidxMprintf(&rc, - "%z%s SELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)", - zSql, zSep, zList, zTab, zWhere, zOrderBy - ); + zSql = cidxMprintf(&rc, "%z" + "%sSELECT * FROM (SELECT %s FROM %Q WHERE %z ORDER BY %s)", + zSql, zSep, zSrcList, zTab, zWhere, zOrderBy + ); zSep = " UNION ALL "; if( pIdx->aCol[i].bDesc==0 ) break; } } - zSql = cidxMprintf(&rc, "%z) AS %Q", zSql, zIdxName); - sqlite3_free(zList); + zSql = cidxMprintf(&rc, "%z) AS i", zSql); /* printf("SQL: %s\n", zSql); */ pCsr->pStmt = cidxPrepare(&rc, pCsr, "%z", zSql); @@ -582,6 +698,7 @@ static int cidxFilter( sqlite3_free(zOrderBy); sqlite3_free(zSubWhere); sqlite3_free(zSubExpr); + sqlite3_free(zSrcList); cidxFreeIndex(pIdx); sqlite3_free(azAfter); } diff --git a/manifest b/manifest index 48fe0a91a2..59ee792f38 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\scheckindex.c,\suse\sC\scode\sinstead\sof\sSQL/group_concat()\sto\scompose\svarious\nSQL\sclauses.\sThis\sis\sto\smake\sit\seasier\sto\ssupport\sindexes\son\sexpressions. -D 2017-10-30T17:05:18.290 +C Add\ssupport\sfor\sindexes\son\sexpressions\sto\sincremental_index_check. +D 2017-10-30T19:38:41.046 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -328,7 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 -F ext/repair/checkindex.c 6168af2569681aba298f09a2bba358454f68c7de50df8d37a8b6f91924dd42c7 +F ext/repair/checkindex.c 9feaee9a393e11198aced072e81dfd4c38bda8b914a2a20aba126efbef445185 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -654,7 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b -F test/checkindex.test a5969b99755cb78129fe42bd50470a65e987d713ad7ff01e8c2a0c4d2333e8f4 +F test/checkindex.test f0a611472ac09f294766c8ece1f85b0545aa82207f516437416ef3390b00b8a3 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3ebb2351e2650d263029d2c0042683cba3529c9d3f76b5f994f2e737b84d3f67 -R 45b731228881e6db68d5d1d151d2789f +P 940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269 +R 32cef05dfb77ed0c73cb51913be9d3e3 U dan -Z 8f0b667700c59bebe92a45558d3d25e5 +Z 4d91b1f15edf1ab6cd873cde5a983874 diff --git a/manifest.uuid b/manifest.uuid index 5d8e25d480..f363d67abb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269 \ No newline at end of file +8c1c701fdbe0d56ee7f6f7d7b583aafde9fa14acc93ee8ecaddc8bb311e2bf52 \ No newline at end of file diff --git a/test/checkindex.test b/test/checkindex.test index 6648187559..a91455558f 100644 --- a/test/checkindex.test +++ b/test/checkindex.test @@ -259,6 +259,64 @@ do_index_check_test 4.3 t4cc { {} 'AAB','EEE',4 } +#-------------------------------------------------------------------------- +# Test an index on an expression. +# +do_execsql_test 5.0 { + CREATE TABLE t5(x INTEGER PRIMARY KEY, y TEXT, UNIQUE(y)); + INSERT INTO t5 VALUES(1, '{"x":1, "y":1}'); + INSERT INTO t5 VALUES(2, '{"x":2, "y":2}'); + INSERT INTO t5 VALUES(3, '{"x":3, "y":3}'); + INSERT INTO t5 VALUES(4, '{"w":4, "z":4}'); + INSERT INTO t5 VALUES(5, '{"x":5, "y":5}'); + + CREATE INDEX t5x ON t5( json_extract(y, '$.x') ); + CREATE INDEX t5y ON t5( json_extract(y, '$.y') DESC ); +} + +do_index_check_test 5.1.1 t5x { + {} NULL,4 {} 1,1 {} 2,2 {} 3,3 {} 5,5 +} + +do_index_check_test 5.1.2 t5y { + {} 5,5 {} 3,3 {} 2,2 {} 1,1 {} NULL,4 +} + +do_index_check_test 5.1.3 sqlite_autoindex_t5_1 { + {} {'{"w":4, "z":4}',4} + {} {'{"x":1, "y":1}',1} + {} {'{"x":2, "y":2}',2} + {} {'{"x":3, "y":3}',3} + {} {'{"x":5, "y":5}',5} +} + +do_test 5.2 { + set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t5' }] + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot + db eval {CREATE TABLE xt5(a INTEGER PRIMARY KEY, c1 TEXT);} + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 + execsql { + UPDATE xt5 SET c1='{"x":22, "y":11}' WHERE rowid=1; + DELETE FROM xt5 WHERE rowid = 4; + } + sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1 +} {} + +do_index_check_test 5.3.1 t5x { + {row missing} NULL,4 + {row data mismatch} 1,1 + {} 2,2 + {} 3,3 + {} 5,5 +} + +do_index_check_test 5.3.2 sqlite_autoindex_t5_1 { + {row missing} {'{"w":4, "z":4}',4} + {row data mismatch} {'{"x":1, "y":1}',1} + {} {'{"x":2, "y":2}',2} + {} {'{"x":3, "y":3}',3} + {} {'{"x":5, "y":5}',5} +} finish_test From 7eded5925b89222b4934a36a8c3d118dee4d7457 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 31 Oct 2017 12:01:01 +0000 Subject: [PATCH 06/20] Fix a couple of issues in incremental_index_check to do with CREATE INDEX statements that contain embedded SQL comments. FossilOrigin-Name: 2aef41815a9f1786ebdf09d8f6cfa59a8e7d733253eafeae24fa6e2a093bb1d8 --- ext/repair/checkindex.c | 146 ++++++++++++++++++++++++---------------- manifest | 14 ++-- manifest.uuid | 2 +- test/checkindex.test | 27 ++++++++ 4 files changed, 123 insertions(+), 66 deletions(-) diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c index fc91b74244..16fb3b327e 100644 --- a/ext/repair/checkindex.c +++ b/ext/repair/checkindex.c @@ -274,54 +274,6 @@ static void cidxFreeIndex(CidxIndex *pIdx){ } } -#define CIDX_PARSE_EOF 0 -#define CIDX_PARSE_COMMA 1 /* "," */ -#define CIDX_PARSE_OPEN 2 /* "(" */ -#define CIDX_PARSE_CLOSE 3 /* ")" */ - -static int cidxFindNext(const char *zIn, const char **pzOut){ - const char *z = zIn; - - while( 1 ){ - *pzOut = z; - switch( *z ){ - case '\0': - return CIDX_PARSE_EOF; - case '(': - return CIDX_PARSE_OPEN; - case ')': - return CIDX_PARSE_CLOSE; - case ',': - return CIDX_PARSE_COMMA; - - case '"': - case '\'': - case '`': { - char q = *z; - z++; - while( *z ){ - if( *z==q ){ - z++; - if( *z!=q ) break; - } - z++; - } - break; - } - - case '[': - while( *z++!=']' ); - break; - - default: - z++; - } - } - - assert( 0 ); - return -1; -} - static int cidx_isspace(char c){ return c==' ' || c=='\t' || c=='\r' || c=='\n'; } @@ -332,36 +284,114 @@ static int cidx_isident(char c){ || (c>='A' && c<='Z') || c=='_'; } +#define CIDX_PARSE_EOF 0 +#define CIDX_PARSE_COMMA 1 /* "," */ +#define CIDX_PARSE_OPEN 2 /* "(" */ +#define CIDX_PARSE_CLOSE 3 /* ")" */ + +static int cidxFindNext( + const char *zIn, + const char **pzOut, + int *pbDoNotTrim /* OUT: True if prev is -- comment */ +){ + const char *z = zIn; + + while( 1 ){ + if( z[0]=='-' && z[1]=='-' ){ + z += 2; + while( z[0]!='\n' ){ + if( z[0]=='\0' ) return CIDX_PARSE_EOF; + z++; + } + while( cidx_isspace(*z) ) z++; + *pbDoNotTrim = 1; + }else{ + *pzOut = z; + switch( *z ){ + case '\0': + return CIDX_PARSE_EOF; + case '(': + return CIDX_PARSE_OPEN; + case ')': + return CIDX_PARSE_CLOSE; + case ',': + return CIDX_PARSE_COMMA; + + case '"': + case '\'': + case '`': { + char q = *z; + z++; + while( *z ){ + if( *z==q ){ + z++; + if( *z!=q ) break; + } + z++; + } + break; + } + + case '[': + while( *z++!=']' ); + break; + + case '/': + if( z[1]=='*' ){ + z += 2; + while( z[0]!='*' || z[1]!='/' ){ + if( z[1]=='\0' ) return CIDX_PARSE_EOF; + z++; + } + z += 2; + break; + } + + default: + z++; + break; + } + *pbDoNotTrim = 0; + } + } + + assert( 0 ); + return -1; +} + static int cidxParseSQL(CidxCursor *pCsr, CidxIndex *pIdx, const char *zSql){ const char *z = zSql; const char *z1; int e; int rc = SQLITE_OK; int nParen = 1; + int bDoNotTrim = 0; CidxColumn *pCol = pIdx->aCol; - e = cidxFindNext(z, &z); + e = cidxFindNext(z, &z, &bDoNotTrim); if( e!=CIDX_PARSE_OPEN ) goto parse_error; z1 = z+1; z++; while( nParen>0 ){ - e = cidxFindNext(z, &z); + e = cidxFindNext(z, &z, &bDoNotTrim); if( e==CIDX_PARSE_EOF ) goto parse_error; if( (e==CIDX_PARSE_COMMA || e==CIDX_PARSE_CLOSE) && nParen==1 ){ const char *z2 = z; if( pCol->zExpr ) goto parse_error; - while( cidx_isspace(z[-1]) ) z--; - if( 0==sqlite3_strnicmp(&z[-3], "asc", 3) && 0==cidx_isident(z[-4]) ){ - z -= 3; - while( cidx_isspace(z[-1]) ) z--; - }else - if( 0==sqlite3_strnicmp(&z[-4], "desc", 4) && 0==cidx_isident(z[-5]) ){ - z -= 4; + if( bDoNotTrim==0 ){ while( cidx_isspace(z[-1]) ) z--; + if( !sqlite3_strnicmp(&z[-3], "asc", 3) && 0==cidx_isident(z[-4]) ){ + z -= 3; + while( cidx_isspace(z[-1]) ) z--; + }else + if( !sqlite3_strnicmp(&z[-4], "desc", 4) && 0==cidx_isident(z[-5]) ){ + z -= 4; + while( cidx_isspace(z[-1]) ) z--; + } + while( cidx_isspace(z1[0]) ) z1++; } - while( cidx_isspace(z1[0]) ) z1++; pCol->zExpr = cidxMprintf(&rc, "%.*s", z-z1, z1); pCol++; z = z1 = z2+1; diff --git a/manifest b/manifest index 59ee792f38..974a630385 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\sindexes\son\sexpressions\sto\sincremental_index_check. -D 2017-10-30T19:38:41.046 +C Fix\sa\scouple\sof\sissues\sin\sincremental_index_check\sto\sdo\swith\sCREATE\sINDEX\nstatements\sthat\scontain\sembedded\sSQL\scomments. +D 2017-10-31T12:01:01.861 F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 @@ -328,7 +328,7 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 -F ext/repair/checkindex.c 9feaee9a393e11198aced072e81dfd4c38bda8b914a2a20aba126efbef445185 +F ext/repair/checkindex.c f33d90ed6a556ad03511f7932891c2fd47ad93ddc998a4ab8bb56f4adf6fb206 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -654,7 +654,7 @@ F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b -F test/checkindex.test f0a611472ac09f294766c8ece1f85b0545aa82207f516437416ef3390b00b8a3 +F test/checkindex.test 77153b3d92492a186c947031ee0eb2e9b879c07192c0066f6152539b670dd237 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1668,7 +1668,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 940606b3af059eb3f79d71fec871ea88df8bce0349f5b33b79c147a85610e269 -R 32cef05dfb77ed0c73cb51913be9d3e3 +P 8c1c701fdbe0d56ee7f6f7d7b583aafde9fa14acc93ee8ecaddc8bb311e2bf52 +R 4a5b38f6ae42388faee270886368a889 U dan -Z 4d91b1f15edf1ab6cd873cde5a983874 +Z ee7435bc6b4be93625562670a465a6a7 diff --git a/manifest.uuid b/manifest.uuid index f363d67abb..be302e7d98 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8c1c701fdbe0d56ee7f6f7d7b583aafde9fa14acc93ee8ecaddc8bb311e2bf52 \ No newline at end of file +2aef41815a9f1786ebdf09d8f6cfa59a8e7d733253eafeae24fa6e2a093bb1d8 \ No newline at end of file diff --git a/test/checkindex.test b/test/checkindex.test index a91455558f..22463d3ffe 100644 --- a/test/checkindex.test +++ b/test/checkindex.test @@ -318,6 +318,33 @@ do_index_check_test 5.3.2 sqlite_autoindex_t5_1 { {} {'{"x":5, "y":5}',5} } +#------------------------------------------------------------------------- +# +do_execsql_test 6.0 { + CREATE TABLE t6(x INTEGER PRIMARY KEY, y, z); + CREATE INDEX t6x1 ON t6(y, /* one,two,three */ z); + CREATE INDEX t6x2 ON t6(z, -- hello,world, + y); + + CREATE INDEX t6x3 ON t6(z -- hello,world + , y); + + INSERT INTO t6 VALUES(1, 2, 3); + INSERT INTO t6 VALUES(4, 5, 6); +} + +do_index_check_test 6.1 t6x1 { + {} 2,3,1 + {} 5,6,4 +} +do_index_check_test 6.2 t6x2 { + {} 3,2,1 + {} 6,5,4 +} +do_index_check_test 6.2 t6x3 { + {} 3,2,1 + {} 6,5,4 +} finish_test From 2737fa037660b4694ff5c190f9de955c15673d47 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 31 Oct 2017 15:49:19 +0000 Subject: [PATCH 07/20] Add build infrastructure for the sqlite3_checker command-line utility. FossilOrigin-Name: 429e3c3c85d6b677038be93567f93e69eb756442bb421bb02ee5d5d69290fa09 --- Makefile.in | 15 ++++++ Makefile.msc | 17 +++++++ ext/misc/btreeinfo.c | 4 +- ext/repair/sqlite3_checker.c.in | 31 ++++++++++++ ext/repair/sqlite3_checker.tcl | 88 +++++++++++++++++++++++++++++++++ main.mk | 16 ++++++ manifest | 22 +++++---- manifest.uuid | 2 +- 8 files changed, 183 insertions(+), 12 deletions(-) create mode 100644 ext/repair/sqlite3_checker.c.in create mode 100644 ext/repair/sqlite3_checker.tcl diff --git a/Makefile.in b/Makefile.in index 5f6be57c37..74ade8241e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1187,6 +1187,21 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $ sqlite3_analyzer$(TEXE): sqlite3_analyzer.c $(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS) +CHECKER_DEPS =\ + $(TOP)/tool/mkccode.tcl \ + sqlite3.c \ + $(TOP)/src/tclsqlite.c \ + $(TOP)/ext/repair/sqlite3_checker.tcl \ + $(TOP)/ext/repair/checkindex.c \ + $(TOP)/ext/misc/btreeinfo.c \ + $(TOP)/ext/repair/sqlite3_checker.c.in + +sqlite3_checker.c: $(CHECKER_DEPS) + $(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@ + +sqlite3_checker$(TEXE): sqlite3_checker.c + $(LTLINK) sqlite3_checker.c -o $@ $(LIBTCL) $(TLIBS) + dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo $(LTLINK) -DDBDUMP_STANDALONE -o $@ \ $(TOP)/ext/misc/dbdump.c sqlite3.lo $(TLIBS) diff --git a/Makefile.msc b/Makefile.msc index 2804c25ab4..1e069cfe66 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1491,6 +1491,7 @@ TESTPROGS = \ testfixture.exe \ $(SQLITE3EXE) \ sqlite3_analyzer.exe \ + sqlite3_checker.exe \ sqldiff.exe \ dbhash.exe @@ -2198,6 +2199,22 @@ sqlite3_analyzer.exe: sqlite3_analyzer.c $(LIBRESOBJS) $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_analyzer.c \ /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) +CHECKER_DEPS =\ + $(TOP)/tool/mkccode.tcl \ + sqlite3.c \ + $(TOP)/src/tclsqlite.c \ + $(TOP)/ext/repair/sqlite3_checker.tcl \ + $(TOP)/ext/repair/checkindex.c \ + $(TOP)/ext/misc/btreeinfo.c \ + $(TOP)/ext/repair/sqlite3_checker.c.in + +sqlite3_checker.c: $(CHECKER_DEPS) + $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\misc\repair\sqlite3_checker.c.in > $@ + +sqlite3_checker.exe: sqlite3_checker.c $(LIBRESOBJS) + $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_checker.c \ + /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) $(TLIBS) + dbdump.exe: $(TOP)\ext\misc\dbdump.c $(SQLITE3C) $(SQLITE3H) $(LTLINK) $(NO_WARN) -DDBDUMP_STANDALONE $(TOP)\ext\misc\dbdump.c $(SQLITE3C) \ /link $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LTLIBS) diff --git a/ext/misc/btreeinfo.c b/ext/misc/btreeinfo.c index 131c63e835..d75f06277f 100644 --- a/ext/misc/btreeinfo.c +++ b/ext/misc/btreeinfo.c @@ -61,7 +61,9 @@ ** SELECT name FROM sqlite_btreeinfo ** WHERE type='table' AND NOT hasRowid; */ -#include +#if !defined(SQLITEINT_H) +#include "sqlite3ext.h" +#endif SQLITE_EXTENSION_INIT1 #include #include diff --git a/ext/repair/sqlite3_checker.c.in b/ext/repair/sqlite3_checker.c.in new file mode 100644 index 0000000000..af8b673ec5 --- /dev/null +++ b/ext/repair/sqlite3_checker.c.in @@ -0,0 +1,31 @@ +/* +** Read an SQLite database file and analyze its space utilization. Generate +** text on standard output. +*/ +#define TCLSH_INIT_PROC sqlite3_checker_init_proc +#define SQLITE_ENABLE_DBPAGE_VTAB 1 +#undef SQLITE_THREADSAFE +#define SQLITE_THREADSAFE 0 +#undef SQLITE_ENABLE_COLUMN_METADATA +#define SQLITE_OMIT_DECLTYPE 1 +#define SQLITE_OMIT_DEPRECATED 1 +#define SQLITE_OMIT_PROGRESS_CALLBACK 1 +#define SQLITE_OMIT_SHARED_CACHE 1 +#define SQLITE_DEFAULT_MEMSTATUS 0 +#define SQLITE_MAX_EXPR_DEPTH 0 +INCLUDE sqlite3.c +INCLUDE $ROOT/src/tclsqlite.c +INCLUDE $ROOT/ext/misc/btreeinfo.c +INCLUDE $ROOT/ext/repair/checkindex.c + +#include + +const char *sqlite3_checker_init_proc(Tcl_Interp *interp){ + sqlite3_auto_extension((void(*)(void))sqlite3_btreeinfo_init); + sqlite3_auto_extension((void(*)(void))sqlite3_checkindex_init); + return +BEGIN_STRING +INCLUDE $ROOT/ext/repair/sqlite3_checker.tcl +END_STRING +; +} diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl new file mode 100644 index 0000000000..86799741bc --- /dev/null +++ b/ext/repair/sqlite3_checker.tcl @@ -0,0 +1,88 @@ +# Read and run TCL commands from standard input. Used to implement +# the --tclsh option. +# This TCL script is the main driver script for the sqlite3_checker utility +# program. +# + +proc tclsh {} { + set line {} + while {![eof stdin]} { + if {$line!=""} { + puts -nonewline "> " + } else { + puts -nonewline "% " + } + flush stdout + append line [gets stdin] + if {[info complete $line]} { + if {[catch {uplevel #0 $line} result]} { + puts stderr "Error: $result" + } elseif {$result!=""} { + puts $result + } + set line {} + } else { + append line \n + } + } +} + +proc usage {} { + set argv0 [file rootname [file tail [info nameofexecutable]]] + puts stderr "Usage: $argv0 OPTIONS database-filename" + puts stderr { +Do sanity checking on a live SQLite3 database file specified by the +"database-filename" argument. + +Options: + + --tclsh Run the built-in TCL interpreter interactively (for debugging) + + --version Show the version number of SQLite +} + exit 1 +} + +set file_to_analyze {} +append argv {} +foreach arg $argv { + if {[regexp {^-+tclsh$} $arg]} { + tclsh + exit 0 + } + if {[regexp {^-+version$} $arg]} { + sqlite3 mem :memory: + puts [mem one {SELECT sqlite_version()||' '||sqlite_source_id()}] + mem close + exit 0 + } + if {[regexp {^-} $arg]} { + puts stderr "Unknown option: $arg" + usage + } + if {$file_to_analyze!=""} { + usage + } else { + set file_to_analyze $arg + } +} +if {$file_to_analyze==""} usage + +# If a TCL script is specified on the command-line, then run that +# script. +# +if {[file extension $file_to_analyze]==".tcl"} { + source $file_to_analyze + exit 0 +} + +set root_filename $file_to_analyze +regexp {^file:(//)?([^?]*)} $file_to_analyze all x1 root_filename +if {![file exists $root_filename]} { + puts stderr "No such file: $root_filename" + exit 1 +} +if {![file readable $root_filename]} { + puts stderr "File is not readable: $root_filename" + exit 1 +} diff --git a/main.mk b/main.mk index c7711fb6e9..b9072a8dcc 100644 --- a/main.mk +++ b/main.mk @@ -464,6 +464,7 @@ TESTPROGS = \ testfixture$(EXE) \ sqlite3$(EXE) \ sqlite3_analyzer$(EXE) \ + sqlite3_checker$(EXE) \ sqldiff$(EXE) \ dbhash$(EXE) @@ -786,6 +787,21 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $ sqlite3_analyzer$(EXE): sqlite3_analyzer.c $(TCCX) $(TCL_FLAGS) sqlite3_analyzer.c -o $@ $(LIBTCL) $(THREADLIB) +CHECKER_DEPS =\ + $(TOP)/tool/mkccode.tcl \ + sqlite3.c \ + $(TOP)/src/tclsqlite.c \ + $(TOP)/ext/repair/sqlite3_checker.tcl \ + $(TOP)/ext/repair/checkindex.c \ + $(TOP)/ext/misc/btreeinfo.c \ + $(TOP)/ext/repair/sqlite3_checker.c.in + +sqlite3_checker.c: $(CHECKER_DEPS) + tclsh $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@ + +sqlite3_checker$(TEXE): sqlite3_checker.c + $(TCCX) $(TCL_FLAGS) sqlite3_checker.c -o $@ $(LIBTCL) $(THREADLIB) + dbdump$(EXE): $(TOP)/ext/misc/dbdump.c sqlite3.o $(TCCX) -DDBDUMP_STANDALONE -o dbdump$(EXE) \ $(TOP)/ext/misc/dbdump.c sqlite3.o $(THREADLIB) diff --git a/manifest b/manifest index 974a630385..b28e60b264 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Fix\sa\scouple\sof\sissues\sin\sincremental_index_check\sto\sdo\swith\sCREATE\sINDEX\nstatements\sthat\scontain\sembedded\sSQL\scomments. -D 2017-10-31T12:01:01.861 -F Makefile.in e016061b23e60ac9ec27c65cb577292b6bde0307ca55abd874ab3487b3b1beb2 +C Add\sbuild\sinfrastructure\sfor\sthe\ssqlite3_checker\scommand-line\sutility. +D 2017-10-31T15:49:19.426 +F Makefile.in bc4705e00b28eb444ba6689c5b8d3bf3df2c58deb6d5093db6869e1f3ce416e5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 37740aba9c4bb359c627eadccf1cfd7be4f5f847078723777ea7763969e533b1 +F Makefile.msc ba20905d0169cf0734d205c0f9e18498158553129578d8f64d00cddf98559db0 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -258,7 +258,7 @@ F ext/lsm1/test/lsm1_simple.test ca949efefa102f4644231dcd9291d8cda7699a4ce1006b2 F ext/misc/README.md 8e008c8d2b02e09096b31dfba033253ac27c6c06a18aa5826e299fa7601d90b2 F ext/misc/amatch.c 6db4607cb17c54b853a2d7c7c36046d004853f65b9b733e6f019d543d5dfae87 F ext/misc/anycollseq.c 5ffdfde9829eeac52219136ad6aa7cd9a4edb3b15f4f2532de52f4a22525eddb -F ext/misc/btreeinfo.c 89488a065f5fc3c54f81c589460d69d83cd2f337ae59ed1cf813b8572b258516 +F ext/misc/btreeinfo.c d7fd9a2fe2fa33ba28488e2fce703ebecc759219ea9e0bb3b254784866c0a676 F ext/misc/carray.c ed96c218ea940b85c9a274c4d9c59fe9491c299147a38a8bba537687bd6c6005 F ext/misc/closure.c 0d2a038df8fbae7f19de42e7c7d71f2e4dc88704 F ext/misc/completion.c 52c3f01523e3e387eb321b4739a89d1fe47cbe6025aa1f2d8d3685e9e365df0f @@ -329,6 +329,8 @@ F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd782299 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/repair/checkindex.c f33d90ed6a556ad03511f7932891c2fd47ad93ddc998a4ab8bb56f4adf6fb206 +F ext/repair/sqlite3_checker.c.in cf1850c9fc819ec63a006799daabc1911e89248671b3ae5670c205067c9b2c92 +F ext/repair/sqlite3_checker.tcl 61437f3eb4de9ba044e0b72532dc13b7e0cf1a06e63c76198b1ef577cabe2de4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -387,7 +389,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk a39528d993afc1f0c0aebde2e3623ab4171d3bba484eea1e5241615c706c9ce8 +F main.mk 2bf2c463a52d7fad3b6b779810d09307ac484e7acd0ea044cfa6add5703c5030 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -1668,7 +1670,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8c1c701fdbe0d56ee7f6f7d7b583aafde9fa14acc93ee8ecaddc8bb311e2bf52 -R 4a5b38f6ae42388faee270886368a889 -U dan -Z ee7435bc6b4be93625562670a465a6a7 +P 2aef41815a9f1786ebdf09d8f6cfa59a8e7d733253eafeae24fa6e2a093bb1d8 +R 0fe5d7d5728508c17e7a673dce852490 +U drh +Z 5b634bacf5fe6d374b5cc3f0a57d52b2 diff --git a/manifest.uuid b/manifest.uuid index be302e7d98..58aa8b845a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2aef41815a9f1786ebdf09d8f6cfa59a8e7d733253eafeae24fa6e2a093bb1d8 \ No newline at end of file +429e3c3c85d6b677038be93567f93e69eb756442bb421bb02ee5d5d69290fa09 \ No newline at end of file From 2091f0fb3e52c493315796185b2329de6f55cfc6 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 31 Oct 2017 15:55:48 +0000 Subject: [PATCH 08/20] Fix an error in the Makefile.msc for the sqlite3_checker.exe target. FossilOrigin-Name: 49d0c6f1f62dccc3c4609760950f7b28bf94a9520996a80d10ff45096b0dd9a0 --- Makefile.msc | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 1e069cfe66..6d3f7c7ccd 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2209,7 +2209,7 @@ CHECKER_DEPS =\ $(TOP)/ext/repair/sqlite3_checker.c.in sqlite3_checker.c: $(CHECKER_DEPS) - $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\misc\repair\sqlite3_checker.c.in > $@ + $(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\ext\repair\sqlite3_checker.c.in > $@ sqlite3_checker.exe: sqlite3_checker.c $(LIBRESOBJS) $(LTLINK) $(NO_WARN) -DBUILD_sqlite -I$(TCLINCDIR) sqlite3_checker.c \ diff --git a/manifest b/manifest index b28e60b264..20118f2662 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Add\sbuild\sinfrastructure\sfor\sthe\ssqlite3_checker\scommand-line\sutility. -D 2017-10-31T15:49:19.426 +C Fix\san\serror\sin\sthe\sMakefile.msc\sfor\sthe\ssqlite3_checker.exe\starget. +D 2017-10-31T15:55:48.444 F Makefile.in bc4705e00b28eb444ba6689c5b8d3bf3df2c58deb6d5093db6869e1f3ce416e5 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc ba20905d0169cf0734d205c0f9e18498158553129578d8f64d00cddf98559db0 +F Makefile.msc 16ab000bdfd7578f8b762ac65c72112196d8f7353554498075093c08930a33c1 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -1670,7 +1670,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2aef41815a9f1786ebdf09d8f6cfa59a8e7d733253eafeae24fa6e2a093bb1d8 -R 0fe5d7d5728508c17e7a673dce852490 +P 429e3c3c85d6b677038be93567f93e69eb756442bb421bb02ee5d5d69290fa09 +R d3e35133a6b7ed531196c48b12a56714 U drh -Z 5b634bacf5fe6d374b5cc3f0a57d52b2 +Z 22a9f7b25cabce39b5f911e9801c9b67 diff --git a/manifest.uuid b/manifest.uuid index 58aa8b845a..c6585cbdd7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -429e3c3c85d6b677038be93567f93e69eb756442bb421bb02ee5d5d69290fa09 \ No newline at end of file +49d0c6f1f62dccc3c4609760950f7b28bf94a9520996a80d10ff45096b0dd9a0 \ No newline at end of file From 927dd51fce9e80feb2b3bda48bf9f77724cf0f89 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 31 Oct 2017 18:09:40 +0000 Subject: [PATCH 09/20] Add the checkfreelist extension to the sqlite3_checker binary. FossilOrigin-Name: 0c22f0d6d19e595c0577b0bad416522e71647f54d2eb5cda587e2ccc5f63dd6a --- Makefile.in | 1 + Makefile.msc | 1 + ext/repair/sqlite3_checker.c.in | 2 ++ main.mk | 1 + manifest | 18 +++++++++--------- manifest.uuid | 2 +- 6 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Makefile.in b/Makefile.in index 74ade8241e..0653ae8eae 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1193,6 +1193,7 @@ CHECKER_DEPS =\ $(TOP)/src/tclsqlite.c \ $(TOP)/ext/repair/sqlite3_checker.tcl \ $(TOP)/ext/repair/checkindex.c \ + $(TOP)/ext/repair/checkfreelist.c \ $(TOP)/ext/misc/btreeinfo.c \ $(TOP)/ext/repair/sqlite3_checker.c.in diff --git a/Makefile.msc b/Makefile.msc index 6d3f7c7ccd..320dacc411 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -2205,6 +2205,7 @@ CHECKER_DEPS =\ $(TOP)/src/tclsqlite.c \ $(TOP)/ext/repair/sqlite3_checker.tcl \ $(TOP)/ext/repair/checkindex.c \ + $(TOP)/ext/repair/checkfreelist.c \ $(TOP)/ext/misc/btreeinfo.c \ $(TOP)/ext/repair/sqlite3_checker.c.in diff --git a/ext/repair/sqlite3_checker.c.in b/ext/repair/sqlite3_checker.c.in index af8b673ec5..6045c36a9d 100644 --- a/ext/repair/sqlite3_checker.c.in +++ b/ext/repair/sqlite3_checker.c.in @@ -17,12 +17,14 @@ INCLUDE sqlite3.c INCLUDE $ROOT/src/tclsqlite.c INCLUDE $ROOT/ext/misc/btreeinfo.c INCLUDE $ROOT/ext/repair/checkindex.c +INCLUDE $ROOT/ext/repair/checkfreelist.c #include const char *sqlite3_checker_init_proc(Tcl_Interp *interp){ sqlite3_auto_extension((void(*)(void))sqlite3_btreeinfo_init); sqlite3_auto_extension((void(*)(void))sqlite3_checkindex_init); + sqlite3_auto_extension((void(*)(void))sqlite3_checkfreelist_init); return BEGIN_STRING INCLUDE $ROOT/ext/repair/sqlite3_checker.tcl diff --git a/main.mk b/main.mk index b9072a8dcc..266328ac8b 100644 --- a/main.mk +++ b/main.mk @@ -793,6 +793,7 @@ CHECKER_DEPS =\ $(TOP)/src/tclsqlite.c \ $(TOP)/ext/repair/sqlite3_checker.tcl \ $(TOP)/ext/repair/checkindex.c \ + $(TOP)/ext/repair/checkfreelist.c \ $(TOP)/ext/misc/btreeinfo.c \ $(TOP)/ext/repair/sqlite3_checker.c.in diff --git a/manifest b/manifest index 20118f2662..9b4d6af191 100644 --- a/manifest +++ b/manifest @@ -1,8 +1,8 @@ -C Fix\san\serror\sin\sthe\sMakefile.msc\sfor\sthe\ssqlite3_checker.exe\starget. -D 2017-10-31T15:55:48.444 -F Makefile.in bc4705e00b28eb444ba6689c5b8d3bf3df2c58deb6d5093db6869e1f3ce416e5 +C Add\sthe\scheckfreelist\sextension\sto\sthe\ssqlite3_checker\sbinary. +D 2017-10-31T18:09:40.959 +F Makefile.in 750ff35a98dd7f4ad585923d88f4309bb4cebb51c4a6622a32f134a138c9bd13 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 -F Makefile.msc 16ab000bdfd7578f8b762ac65c72112196d8f7353554498075093c08930a33c1 +F Makefile.msc 038f2c990966e6503f4960da574d4eef7b07c272a480983e10c4ebaa6d98cff9 F README.md f5c87359573c4d255425e588a56554b50fdcc2afba4e017a2e02a43701456afd F VERSION f81232df28e2d3ff049feefad5fbd5489cc33697f6bd2ecf61af7f0dde3b83d0 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 @@ -329,7 +329,7 @@ F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd782299 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/repair/checkindex.c f33d90ed6a556ad03511f7932891c2fd47ad93ddc998a4ab8bb56f4adf6fb206 -F ext/repair/sqlite3_checker.c.in cf1850c9fc819ec63a006799daabc1911e89248671b3ae5670c205067c9b2c92 +F ext/repair/sqlite3_checker.c.in 16d62615dfce1ff3eeac83d1a77fe376a7b660afa9db07e1fdd8b964dcc41510 F ext/repair/sqlite3_checker.tcl 61437f3eb4de9ba044e0b72532dc13b7e0cf1a06e63c76198b1ef577cabe2de4 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f @@ -389,7 +389,7 @@ F ext/userauth/userauth.c 3410be31283abba70255d71fd24734e017a4497f F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 2bf2c463a52d7fad3b6b779810d09307ac484e7acd0ea044cfa6add5703c5030 +F main.mk 4f59e62614c9e2d793cdafa25f7b24f357b48b009099185c66c52af4e31ee36f F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -1670,7 +1670,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 429e3c3c85d6b677038be93567f93e69eb756442bb421bb02ee5d5d69290fa09 -R d3e35133a6b7ed531196c48b12a56714 +P 49d0c6f1f62dccc3c4609760950f7b28bf94a9520996a80d10ff45096b0dd9a0 +R 0e924f8968e62617bb612a72ff226c2d U drh -Z 22a9f7b25cabce39b5f911e9801c9b67 +Z 18789d1f0a3682c9ed4749672ea58f7e diff --git a/manifest.uuid b/manifest.uuid index c6585cbdd7..99d2299f3b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -49d0c6f1f62dccc3c4609760950f7b28bf94a9520996a80d10ff45096b0dd9a0 \ No newline at end of file +0c22f0d6d19e595c0577b0bad416522e71647f54d2eb5cda587e2ccc5f63dd6a \ No newline at end of file From 4e8ad3bc1435e5202d1a59a9dd3e7eabfaff9469 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Nov 2017 00:10:34 +0000 Subject: [PATCH 10/20] Begin putting functionality into the sqlite3_checker binary. FossilOrigin-Name: e82e883b93128e4d1105a82abe8d1860c0a15505b6ca421e187b9bbbc2fdc659 --- ext/repair/sqlite3_checker.tcl | 57 ++++++++++++++++++++++++++++++++++ manifest | 12 +++---- manifest.uuid | 2 +- 3 files changed, 64 insertions(+), 7 deletions(-) diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl index 86799741bc..fa3ffd73ce 100644 --- a/ext/repair/sqlite3_checker.tcl +++ b/ext/repair/sqlite3_checker.tcl @@ -4,6 +4,21 @@ # program. # +# Special case: +# +# sqlite3_checker --test FILENAME ARGS +# +# uses FILENAME in place of this script. +# +if {[lindex $argv 0]=="--test" && [llength $argv]>2} { + set file [lindex $argv 1] + set argv [lrange $argv 2 end] + source $file + exit 0 +} + +# Emulate a TCL shell +# proc tclsh {} { set line {} while {![eof stdin]} { @@ -27,6 +42,8 @@ proc tclsh {} { } } +# Print a usage message on standard error, then quit. +# proc usage {} { set argv0 [file rootname [file tail [info nameofexecutable]]] puts stderr "Usage: $argv0 OPTIONS database-filename" @@ -36,6 +53,8 @@ Do sanity checking on a live SQLite3 database file specified by the Options: + --freelist Perform a freelist check + --tclsh Run the built-in TCL interpreter interactively (for debugging) --version Show the version number of SQLite @@ -45,6 +64,8 @@ Options: set file_to_analyze {} append argv {} +set bFreelistCheck 0 +set bSummary 1 foreach arg $argv { if {[regexp {^-+tclsh$} $arg]} { tclsh @@ -56,6 +77,11 @@ foreach arg $argv { mem close exit 0 } + if {[regexp {^-+freelist$} $arg]} { + set bFreelistCheck 1 + set bSummary 0 + continue + } if {[regexp {^-} $arg]} { puts stderr "Unknown option: $arg" usage @@ -86,3 +112,34 @@ if {![file readable $root_filename]} { puts stderr "File is not readable: $root_filename" exit 1 } + +if {[catch {sqlite3 db $file_to_analyze} res]} { + puts stderr "Cannot open datababase $root_filename: $res" + exit 1 +} + +if {$bFreelistCheck} { + puts "freelist-check:" + flush stdout + puts [db one {SELECT checkfreelist('main')}] +} +if {$bSummary} { + set scale 0 + set pgsz [db one {PRAGMA page_size}] + db eval {SELECT nPage*$pgsz AS sz, name, tbl_name + FROM sqlite_btreeinfo + WHERE type='index' + ORDER BY 1 DESC, name} { + if {$scale==0} { + if {$sz>10000000} { + set scale 1000000.0 + set unit MB + } else { + set scale 1000.0 + set unit KB + } + } + puts [format {%7.1f %s index %s of table %s} \ + [expr {$sz/$scale}] $unit $name $tbl_name] + } +} diff --git a/manifest b/manifest index 0976eed231..e4e6ce9964 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\srecent\senhancements\sfrom\strunk. -D 2017-10-31T18:15:36.138 +C Begin\sputting\sfunctionality\sinto\sthe\ssqlite3_checker\sbinary. +D 2017-11-01T00:10:34.144 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -330,7 +330,7 @@ F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b14469 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/repair/checkindex.c f33d90ed6a556ad03511f7932891c2fd47ad93ddc998a4ab8bb56f4adf6fb206 F ext/repair/sqlite3_checker.c.in 16d62615dfce1ff3eeac83d1a77fe376a7b660afa9db07e1fdd8b964dcc41510 -F ext/repair/sqlite3_checker.tcl 61437f3eb4de9ba044e0b72532dc13b7e0cf1a06e63c76198b1ef577cabe2de4 +F ext/repair/sqlite3_checker.tcl cc94d391dae61c5076cadd4789caa8a0cf19701a5f5cd19d329b77d2fe47a967 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1671,7 +1671,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0c22f0d6d19e595c0577b0bad416522e71647f54d2eb5cda587e2ccc5f63dd6a bb39744f4b2b25c10d293e85db7579e2a99c639fdab45e93d1de75952b68b2de -R 6d1ce2918085f767aa6a41a268922785 +P 24adf90ffb3ce9ff3c26efef0357f3a47312e8d11dc391ef2cc7e6873ef25895 +R f819f4cdc6b92e91586a5e2d567aa2dc U drh -Z 59d272efca4b8d35d600b1005fb03cf7 +Z e7d1ee3ba646c59ecaada1f36dfcbcb9 diff --git a/manifest.uuid b/manifest.uuid index 462cf20d71..f800dc1ed5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -24adf90ffb3ce9ff3c26efef0357f3a47312e8d11dc391ef2cc7e6873ef25895 \ No newline at end of file +e82e883b93128e4d1105a82abe8d1860c0a15505b6ca421e187b9bbbc2fdc659 \ No newline at end of file From 1fae37319e5a104161f9aa0d27e27508ac453ffa Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Nov 2017 01:01:20 +0000 Subject: [PATCH 11/20] Actually perform index checks when running sqlite3_checker FossilOrigin-Name: 54530020260ea9e4cfd021f5ffccb74d78c469b717dce377d3df6eaf84b63719 --- ext/repair/sqlite3_checker.tcl | 113 ++++++++++++++++++++++++++++++--- manifest | 12 ++-- manifest.uuid | 2 +- 3 files changed, 112 insertions(+), 15 deletions(-) diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl index fa3ffd73ce..d4aa84aa3b 100644 --- a/ext/repair/sqlite3_checker.tcl +++ b/ext/repair/sqlite3_checker.tcl @@ -42,6 +42,37 @@ proc tclsh {} { } } +# Do an incremental integrity check of a single index +# +proc check_index {idxname batchsize} { + set i 0 + set more 1 + set nerr 0 + puts -nonewline "$idxname: " + while {$more} { + set more 0 + db eval {SELECT errmsg, current_key AS key + FROM incremental_index_check($idxname) + WHERE after_key=$key + LIMIT $batchsize} { + set more 1 + if {$errmsg!=""} { + if {$nerr>0} { + puts -nonewline "$idxname: " + } + incr nerr + puts "row $i: $errmsg" + } + incr i + } + } + if {$nerr==0} { + puts "$i entries, ok" + } else { + puts "$idxname: $nerr errors out of $i entries" + } +} + # Print a usage message on standard error, then quit. # proc usage {} { @@ -53,11 +84,19 @@ Do sanity checking on a live SQLite3 database file specified by the Options: - --freelist Perform a freelist check + --batchsize N Number of rows to check per transaction - --tclsh Run the built-in TCL interpreter interactively (for debugging) + --freelist Perform a freelist check - --version Show the version number of SQLite + --index NAME Run a check of the index NAME + + --summary Print summary information about the database + + --table NAME Run a check of all indexes for table NAME + + --tclsh Run the built-in TCL interpreter (for debugging) + + --version Show the version number of SQLite } exit 1 } @@ -65,8 +104,14 @@ Options: set file_to_analyze {} append argv {} set bFreelistCheck 0 -set bSummary 1 -foreach arg $argv { +set bSummary 0 +set zIndex {} +set zTable {} +set batchsize 100 +set bAll 1 +set argc [llength $argv] +for {set i 0} {$i<$argc} {incr i} { + set arg [lindex $argv $i] if {[regexp {^-+tclsh$} $arg]} { tclsh exit 0 @@ -79,7 +124,41 @@ foreach arg $argv { } if {[regexp {^-+freelist$} $arg]} { set bFreelistCheck 1 - set bSummary 0 + set bAll 0 + continue + } + if {[regexp {^-+summary$} $arg]} { + set bSummary 1 + set bAll 0 + continue + } + if {[regexp {^-+batchsize$} $arg]} { + incr i + if {$i>=$argc} { + puts stderr "missing argument on $arg" + exit 1 + } + set batchsize [lindex $argv $i] + continue + } + if {[regexp {^-+index$} $arg]} { + incr i + if {$i>=$argc} { + puts stderr "missing argument on $arg" + exit 1 + } + set zIndex [lindex $argv $i] + set bAll 0 + continue + } + if {[regexp {^-+table$} $arg]} { + incr i + if {$i>=$argc} { + puts stderr "missing argument on $arg" + exit 1 + } + set zTable [lindex $argv $i] + set bAll 0 continue } if {[regexp {^-} $arg]} { @@ -118,8 +197,8 @@ if {[catch {sqlite3 db $file_to_analyze} res]} { exit 1 } -if {$bFreelistCheck} { - puts "freelist-check:" +if {$bFreelistCheck || $bAll} { + puts -nonewline "freelist-check: " flush stdout puts [db one {SELECT checkfreelist('main')}] } @@ -143,3 +222,21 @@ if {$bSummary} { [expr {$sz/$scale}] $unit $name $tbl_name] } } +if {$zIndex!=""} { + check_index $zIndex $batchsize +} +if {$zTable!=""} { + foreach idx [db eval {SELECT name FROM sqlite_master + WHERE type='index' AND rootpage>0 + AND tbl_name=$zTable}] { + check_index $idx $batchsize + } +} +if {$bAll} { + set allidx [db eval {SELECT name FROM sqlite_btreeinfo('main') + WHERE type='index' AND rootpage>0 + ORDER BY nEntry}] + foreach idx $allidx { + check_index $idx $batchsize + } +} diff --git a/manifest b/manifest index e4e6ce9964..a675da0f12 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Begin\sputting\sfunctionality\sinto\sthe\ssqlite3_checker\sbinary. -D 2017-11-01T00:10:34.144 +C Actually\sperform\sindex\schecks\swhen\srunning\ssqlite3_checker +D 2017-11-01T01:01:20.360 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -330,7 +330,7 @@ F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b14469 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/repair/checkindex.c f33d90ed6a556ad03511f7932891c2fd47ad93ddc998a4ab8bb56f4adf6fb206 F ext/repair/sqlite3_checker.c.in 16d62615dfce1ff3eeac83d1a77fe376a7b660afa9db07e1fdd8b964dcc41510 -F ext/repair/sqlite3_checker.tcl cc94d391dae61c5076cadd4789caa8a0cf19701a5f5cd19d329b77d2fe47a967 +F ext/repair/sqlite3_checker.tcl c7f68b0d2d2832d90a591c5cad936264c0d54cb2a06bee55d7e391d385fc7a1e F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1671,7 +1671,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 24adf90ffb3ce9ff3c26efef0357f3a47312e8d11dc391ef2cc7e6873ef25895 -R f819f4cdc6b92e91586a5e2d567aa2dc +P e82e883b93128e4d1105a82abe8d1860c0a15505b6ca421e187b9bbbc2fdc659 +R 6f23f39130aeaf4ea9f27fe6626dd3eb U drh -Z e7d1ee3ba646c59ecaada1f36dfcbcb9 +Z cc97ad1013bf49d4724282a689bb05ce diff --git a/manifest.uuid b/manifest.uuid index f800dc1ed5..3e765dc206 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e82e883b93128e4d1105a82abe8d1860c0a15505b6ca421e187b9bbbc2fdc659 \ No newline at end of file +54530020260ea9e4cfd021f5ffccb74d78c469b717dce377d3df6eaf84b63719 \ No newline at end of file From 24fa4d57a72de204c782e2e48664ae2484a7c2e8 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Nov 2017 01:05:41 +0000 Subject: [PATCH 12/20] When sqlite3_checker finds a problem, show the row key as part of the error message, not the row index number. FossilOrigin-Name: 6ffe917d10e298cd80cd3a8c8c5116a2819145a176fb8cfccd5dbd88b10f39df --- ext/repair/sqlite3_checker.tcl | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl index d4aa84aa3b..25a5d252e6 100644 --- a/ext/repair/sqlite3_checker.tcl +++ b/ext/repair/sqlite3_checker.tcl @@ -61,7 +61,7 @@ proc check_index {idxname batchsize} { puts -nonewline "$idxname: " } incr nerr - puts "row $i: $errmsg" + puts "key($key): $errmsg" } incr i } diff --git a/manifest b/manifest index a675da0f12..222cd77113 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Actually\sperform\sindex\schecks\swhen\srunning\ssqlite3_checker -D 2017-11-01T01:01:20.360 +C When\ssqlite3_checker\sfinds\sa\sproblem,\sshow\sthe\srow\skey\sas\spart\sof\sthe\serror\nmessage,\snot\sthe\srow\sindex\snumber. +D 2017-11-01T01:05:42.000 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -330,7 +330,7 @@ F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b14469 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/repair/checkindex.c f33d90ed6a556ad03511f7932891c2fd47ad93ddc998a4ab8bb56f4adf6fb206 F ext/repair/sqlite3_checker.c.in 16d62615dfce1ff3eeac83d1a77fe376a7b660afa9db07e1fdd8b964dcc41510 -F ext/repair/sqlite3_checker.tcl c7f68b0d2d2832d90a591c5cad936264c0d54cb2a06bee55d7e391d385fc7a1e +F ext/repair/sqlite3_checker.tcl 1eb23dcc262fb8b6e869775dc06bdfba3931f5c2ea5bfd41188c0b84fa6e35a7 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1671,7 +1671,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e82e883b93128e4d1105a82abe8d1860c0a15505b6ca421e187b9bbbc2fdc659 -R 6f23f39130aeaf4ea9f27fe6626dd3eb +P 54530020260ea9e4cfd021f5ffccb74d78c469b717dce377d3df6eaf84b63719 +R 84bd4abb8e94f0df6079dbff9fcf5a09 U drh -Z cc97ad1013bf49d4724282a689bb05ce +Z d157fe339c4385e31979ffec5d5c510b diff --git a/manifest.uuid b/manifest.uuid index 3e765dc206..69455d17b6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -54530020260ea9e4cfd021f5ffccb74d78c469b717dce377d3df6eaf84b63719 \ No newline at end of file +6ffe917d10e298cd80cd3a8c8c5116a2819145a176fb8cfccd5dbd88b10f39df \ No newline at end of file From 02c21b7e34a35755b3e90bf5fb352af46e7df187 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Nov 2017 13:09:02 +0000 Subject: [PATCH 13/20] Minor cleanup in checkindex.c. Add progress displays when checking a single index in the top-level TCL script for sqlite3_checker. FossilOrigin-Name: 3ca31cc3ffe1cce4a9961d29801eebd47f0093d1b53d0ea63386338a1d434fd3 --- ext/repair/checkindex.c | 17 ++++++++++++----- ext/repair/sqlite3_checker.tcl | 25 ++++++++++++++----------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 34 insertions(+), 24 deletions(-) diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c index 16fb3b327e..362e199168 100644 --- a/ext/repair/checkindex.c +++ b/ext/repair/checkindex.c @@ -42,6 +42,7 @@ struct CidxTable { struct CidxCursor { sqlite3_vtab_cursor base; /* Base class. Must be first */ + sqlite3_int64 iRowid; sqlite3_stmt *pStmt; }; @@ -93,6 +94,10 @@ static int cidxConnect( int rc = SQLITE_OK; CidxTable *pRet; +#define IIC_ERRMSG 0 +#define IIC_CURRENT_KEY 1 +#define IIC_INDEX_NAME 2 +#define IIC_AFTER_KEY 3 rc = sqlite3_declare_vtab(db, "CREATE TABLE xyz(" " errmsg TEXT, current_key TEXT," @@ -130,10 +135,10 @@ static int cidxBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pInfo){ if( p->usable==0 ) continue; if( p->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; - if( p->iColumn==2 ){ + if( p->iColumn==IIC_INDEX_NAME ){ iIdxName = i; } - if( p->iColumn==3 ){ + if( p->iColumn==IIC_AFTER_KEY ){ iAfterKey = i; } } @@ -193,6 +198,7 @@ static int cidxNext(sqlite3_vtab_cursor *pCursor){ cidxCursorError(pCsr, "Cursor error: %s", sqlite3_errmsg(db)); } }else{ + pCsr->iRowid++; rc = SQLITE_OK; } return rc; @@ -737,6 +743,7 @@ static int cidxFilter( assert( rc==SQLITE_OK ); rc = cidxNext(pCursor); } + pCsr->iRowid = 1; return rc; } @@ -749,8 +756,8 @@ static int cidxColumn( int iCol ){ CidxCursor *pCsr = (CidxCursor*)pCursor; - assert( iCol==0 || iCol==1 ); - if( iCol==0 ){ + assert( iCol>=IIC_ERRMSG && iCol<=IIC_AFTER_KEY ); + if( iCol==IIC_ERRMSG ){ const char *zVal = 0; if( sqlite3_column_type(pCsr->pStmt, 0)==SQLITE_INTEGER ){ if( sqlite3_column_int(pCsr->pStmt, 0)==0 ){ @@ -760,7 +767,7 @@ static int cidxColumn( zVal = "row missing"; } sqlite3_result_text(ctx, zVal, -1, SQLITE_STATIC); - }else{ + }else if( iCol==IIC_CURRENT_KEY ){ sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, 1)); } return SQLITE_OK; diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl index 25a5d252e6..a29d0332a6 100644 --- a/ext/repair/sqlite3_checker.tcl +++ b/ext/repair/sqlite3_checker.tcl @@ -48,7 +48,11 @@ proc check_index {idxname batchsize} { set i 0 set more 1 set nerr 0 - puts -nonewline "$idxname: " + set pct 00.0 + set max [db one {SELECT nEntry FROM sqlite_btreeinfo('main') + WHERE name=$idxname}] + puts -nonewline "$idxname: $i of $max rows ($pct%)\r" + flush stdout while {$more} { set more 0 db eval {SELECT errmsg, current_key AS key @@ -57,20 +61,19 @@ proc check_index {idxname batchsize} { LIMIT $batchsize} { set more 1 if {$errmsg!=""} { - if {$nerr>0} { - puts -nonewline "$idxname: " - } incr nerr - puts "key($key): $errmsg" + puts "$idxname: key($key): $errmsg" } incr i } + set x [format {%.1f} [expr {($i*100.0)/$max}]] + if {$x!=$pct} { + puts -nonewline "$idxname: $i of $max rows ($pct%)\r" + flush stdout + set pct $x + } } - if {$nerr==0} { - puts "$i entries, ok" - } else { - puts "$idxname: $nerr errors out of $i entries" - } + puts "$idxname: $nerr errors out of $i entries" } # Print a usage message on standard error, then quit. @@ -107,7 +110,7 @@ set bFreelistCheck 0 set bSummary 0 set zIndex {} set zTable {} -set batchsize 100 +set batchsize 1000 set bAll 1 set argc [llength $argv] for {set i 0} {$i<$argc} {incr i} { diff --git a/manifest b/manifest index 222cd77113..bf8e3e2133 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\ssqlite3_checker\sfinds\sa\sproblem,\sshow\sthe\srow\skey\sas\spart\sof\sthe\serror\nmessage,\snot\sthe\srow\sindex\snumber. -D 2017-11-01T01:05:42.000 +C Minor\scleanup\sin\scheckindex.c.\s\sAdd\sprogress\sdisplays\swhen\schecking\sa\nsingle\sindex\sin\sthe\stop-level\sTCL\sscript\sfor\ssqlite3_checker. +D 2017-11-01T13:09:02.677 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -328,9 +328,9 @@ F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d0 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 -F ext/repair/checkindex.c f33d90ed6a556ad03511f7932891c2fd47ad93ddc998a4ab8bb56f4adf6fb206 +F ext/repair/checkindex.c a013a0a165b2e6f2b278a31566da04913856c88d6ed5457d477f89e78d5979d9 F ext/repair/sqlite3_checker.c.in 16d62615dfce1ff3eeac83d1a77fe376a7b660afa9db07e1fdd8b964dcc41510 -F ext/repair/sqlite3_checker.tcl 1eb23dcc262fb8b6e869775dc06bdfba3931f5c2ea5bfd41188c0b84fa6e35a7 +F ext/repair/sqlite3_checker.tcl 32d474decb6bb65e60bd3660dba0b75e7e0719d6f6fb6dba97e5b9249eec94a1 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -1671,7 +1671,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 54530020260ea9e4cfd021f5ffccb74d78c469b717dce377d3df6eaf84b63719 -R 84bd4abb8e94f0df6079dbff9fcf5a09 +P 6ffe917d10e298cd80cd3a8c8c5116a2819145a176fb8cfccd5dbd88b10f39df +R 9a8d975a41c4365cfebb7312db115daa U drh -Z d157fe339c4385e31979ffec5d5c510b +Z d02ff4965492ed83b018c8e578de7329 diff --git a/manifest.uuid b/manifest.uuid index 69455d17b6..86f32e4c10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6ffe917d10e298cd80cd3a8c8c5116a2819145a176fb8cfccd5dbd88b10f39df \ No newline at end of file +3ca31cc3ffe1cce4a9961d29801eebd47f0093d1b53d0ea63386338a1d434fd3 \ No newline at end of file From 37ab9523fa97daea32e40e5ed111acf1f91dc78b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Nov 2017 18:05:32 +0000 Subject: [PATCH 14/20] Move the test scripts for checkfreelist and checkindex over into the ext/repair/test directory. Run them now using the sqlite3_checker utility with the --test option. Some tests are currently failing due to an incomplete port. This is an incremental check-in. FossilOrigin-Name: 17f8d5e111a9fe5b074f946e23936ae5a2a7a8d8018bad4212660d8eb81c04b7 --- ext/repair/sqlite3_checker.tcl | 8 +-- ext/repair/test/README.md | 13 ++++ .../repair/test/checkfreelist01.test | 33 +-------- .../repair/test/checkindex01.test | 38 +---------- ext/repair/test/test.tcl | 67 +++++++++++++++++++ manifest | 18 ++--- manifest.uuid | 2 +- 7 files changed, 98 insertions(+), 81 deletions(-) create mode 100644 ext/repair/test/README.md rename test/checkfreelist.test => ext/repair/test/checkfreelist01.test (75%) rename test/checkindex.test => ext/repair/test/checkindex01.test (90%) create mode 100644 ext/repair/test/test.tcl diff --git a/ext/repair/sqlite3_checker.tcl b/ext/repair/sqlite3_checker.tcl index a29d0332a6..ab8d05e600 100644 --- a/ext/repair/sqlite3_checker.tcl +++ b/ext/repair/sqlite3_checker.tcl @@ -1,5 +1,3 @@ -# Read and run TCL commands from standard input. Used to implement -# the --tclsh option. # This TCL script is the main driver script for the sqlite3_checker utility # program. # @@ -10,10 +8,10 @@ # # uses FILENAME in place of this script. # -if {[lindex $argv 0]=="--test" && [llength $argv]>2} { - set file [lindex $argv 1] +if {[lindex $argv 0]=="--test" && [llength $argv]>1} { + set ::argv0 [lindex $argv 1] set argv [lrange $argv 2 end] - source $file + source $argv0 exit 0 } diff --git a/ext/repair/test/README.md b/ext/repair/test/README.md new file mode 100644 index 0000000000..8cc954adf5 --- /dev/null +++ b/ext/repair/test/README.md @@ -0,0 +1,13 @@ +To run these tests, first build sqlite3_checker: + + +> make sqlite3_checker + + +Then run the "test.tcl" script using: + + +> ./sqlite3_checker --test $path/test.tcl + + +Optionally add the full pathnames of individual *.test modules diff --git a/test/checkfreelist.test b/ext/repair/test/checkfreelist01.test similarity index 75% rename from test/checkfreelist.test rename to ext/repair/test/checkfreelist01.test index 93e4ecc234..7c33862583 100644 --- a/test/checkfreelist.test +++ b/ext/repair/test/checkfreelist01.test @@ -1,40 +1,12 @@ # 2017-10-11 -# -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# This file implements regression tests for SQLite library. The -# focus of this file is testing the checkfreelist extension. -# -set testdir [file dirname $argv0] -source $testdir/tester.tcl set testprefix checkfreelist -ifcapable !vtab||!compound { - finish_test - return -} - -if {[file exists ../checkfreelist.so]==0} { - finish_test - return -} - do_execsql_test 1.0 { + PRAGMA page_size=1024; CREATE TABLE t1(a, b); } -db enable_load_extension 1 -do_execsql_test 1.1 { - SELECT load_extension('../checkfreelist.so'); -} {{}} - do_execsql_test 1.2 { SELECT checkfreelist('main') } {ok} do_execsql_test 1.3 { WITH s(i) AS ( @@ -118,6 +90,3 @@ do_execsql_test 1.11 { SELECT checkfreelist('main'); ROLLBACK; } {{leaf count out of range (249) on trunk page 5}} - -finish_test - diff --git a/test/checkindex.test b/ext/repair/test/checkindex01.test similarity index 90% rename from test/checkindex.test rename to ext/repair/test/checkindex01.test index 22463d3ffe..36bae886d2 100644 --- a/test/checkindex.test +++ b/ext/repair/test/checkindex01.test @@ -1,31 +1,7 @@ # 2017-10-11 # -# The author disclaims copyright to this source code. In place of -# a legal notice, here is a blessing: -# -# May you do good and not evil. -# May you find forgiveness for yourself and forgive others. -# May you share freely, never taking more than you give. -# -#*********************************************************************** -# This file implements regression tests for SQLite library. The -# focus of this file is testing the checkindex extension. -# - -set testdir [file dirname $argv0] -source $testdir/tester.tcl set testprefix checkindex -ifcapable !vtab||!compound { - finish_test - return -} - -if {[file exists ../checkindex.so]==0} { - finish_test - return -} - do_execsql_test 1.0 { CREATE TABLE t1(a, b); CREATE INDEX i1 ON t1(a); @@ -36,12 +12,7 @@ do_execsql_test 1.0 { INSERT INTO t1 VALUES('five', 10); CREATE INDEX i2 ON t1(a DESC); -} - -db enable_load_extension 1 -do_execsql_test 1.1 { - SELECT load_extension('../checkindex.so'); -} {{}} +} {} proc incr_index_check {idx nStep} { set Q { @@ -148,7 +119,7 @@ do_execsql_test 2.0 { CREATE INDEX i3 ON t2(b, c, d); CREATE INDEX i4 ON t2(b DESC, c DESC, d DESC); CREATE INDEX i5 ON t2(d, c DESC, b); -} +} {} do_index_check_test 2.1 i3 { {} NULL,1,1,1 @@ -205,7 +176,7 @@ do_execsql_test 3.0 { INSERT INTO t3 VALUES('a', 'b', NULL, 8); INSERT INTO t3 VALUES('a', 'b', NULL, 9); -} +} {} do_index_check_test 3.1 t3wxy { {} NULL,NULL,NULL,1 {} NULL,NULL,NULL,2 {} NULL,NULL,NULL,3 @@ -345,6 +316,3 @@ do_index_check_test 6.2 t6x3 { {} 3,2,1 {} 6,5,4 } - -finish_test - diff --git a/ext/repair/test/test.tcl b/ext/repair/test/test.tcl new file mode 100644 index 0000000000..c073bb73c5 --- /dev/null +++ b/ext/repair/test/test.tcl @@ -0,0 +1,67 @@ +# Run this script using +# +# sqlite3_checker --test $thisscript $testscripts +# +# The $testscripts argument is optional. If omitted, all *.test files +# in the same directory as $thisscript are run. +# +set NTEST 0 +set NERR 0 + + +# Invoke the do_test procedure to run a single test +# +# The $expected parameter is the expected result. The result is the return +# value from the last TCL command in $cmd. +# +# Normally, $expected must match exactly. But if $expected is of the form +# "/regexp/" then regular expression matching is used. If $expected is +# "~/regexp/" then the regular expression must NOT match. If $expected is +# of the form "#/value-list/" then each term in value-list must be numeric +# and must approximately match the corresponding numeric term in $result. +# Values must match within 10%. Or if the $expected term is A..B then the +# $result term must be in between A and B. +# +proc do_test {name cmd expected} { + if {[info exists ::testprefix]} { + set name "$::testprefix$name" + } + + incr ::NTEST + puts -nonewline $name... + flush stdout + + if {[catch {uplevel #0 "$cmd;\n"} result]} { + puts -nonewline $name... + puts "\nError: $result" + incr ::NERR + } else { + set ok [expr {[string compare $result $expected]==0}] + if {!$ok} { + puts "\n! $name expected: \[$expected\]\n! $name got: \[$result\]" + incr ::NERR + } else { + puts " Ok" + } + } + flush stdout +} + +# +# do_execsql_test TESTNAME SQL RES +# +proc do_execsql_test {testname sql {result {}}} { + uplevel [list do_test $testname [list db eval $sql] [list {*}$result]] +} + +if {[llength $argv]==0} { + set dir [file dirname $argv0] + set argv [glob -nocomplain $dir/*.test] +} +foreach testfile $argv { + file delete -force test.db + sqlite3 db test.db + source $testfile + catch {db close} +} +puts "$NERR errors out of $NTEST tests" diff --git a/manifest b/manifest index bf8e3e2133..ff5e3f094d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\scleanup\sin\scheckindex.c.\s\sAdd\sprogress\sdisplays\swhen\schecking\sa\nsingle\sindex\sin\sthe\stop-level\sTCL\sscript\sfor\ssqlite3_checker. -D 2017-11-01T13:09:02.677 +C Move\sthe\stest\sscripts\sfor\scheckfreelist\sand\scheckindex\sover\sinto\sthe\next/repair/test\sdirectory.\s\sRun\sthem\snow\susing\sthe\ssqlite3_checker\sutility\nwith\sthe\s--test\soption.\s\sSome\stests\sare\scurrently\sfailing\sdue\sto\san\nincomplete\sport.\s\sThis\sis\san\sincremental\scheck-in. +D 2017-11-01T18:05:32.697 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -330,7 +330,11 @@ F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b14469 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/repair/checkindex.c a013a0a165b2e6f2b278a31566da04913856c88d6ed5457d477f89e78d5979d9 F ext/repair/sqlite3_checker.c.in 16d62615dfce1ff3eeac83d1a77fe376a7b660afa9db07e1fdd8b964dcc41510 -F ext/repair/sqlite3_checker.tcl 32d474decb6bb65e60bd3660dba0b75e7e0719d6f6fb6dba97e5b9249eec94a1 +F ext/repair/sqlite3_checker.tcl 4820d7f58428d47336874b5a148a95b4dad38fe5da72286c01a861590b8f8337 +F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e +F ext/repair/test/checkfreelist01.test 185648c046c0e269dc3b5141d525b87a38d72f0f55495d46c65108a5dde1428f +F ext/repair/test/checkindex01.test 7be2299647ac7be41b6a7e249eb78e3aba1b57a7b20943ad199a1dc457edb4f7 +F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -655,8 +659,6 @@ F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/cffault.test 9d6b20606afe712374952eec4f8fd74b1a8097ef F test/check.test 33a698e8c63613449d85d624a38ef669bf20331daabebe3891c9405dd6df463a -F test/checkfreelist.test 100283a3e6b8a3018c7fab7cfdaf03d1d6540fc66453114e248cf82b25784d3b -F test/checkindex.test 77153b3d92492a186c947031ee0eb2e9b879c07192c0066f6152539b670dd237 F test/close.test 799ea4599d2f5704b0a30f477d17c2c760d8523fa5d0c8be4a7df2a8cad787d8 F test/closure01.test b1703ba40639cfc9b295cf478d70739415eec6a4 F test/coalesce.test cee0dccb9fbd2d494b77234bccf9dc6c6786eb91 @@ -1671,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 6ffe917d10e298cd80cd3a8c8c5116a2819145a176fb8cfccd5dbd88b10f39df -R 9a8d975a41c4365cfebb7312db115daa +P 3ca31cc3ffe1cce4a9961d29801eebd47f0093d1b53d0ea63386338a1d434fd3 +R 9a0909f5ab1dfc6f8f86ede9c43722ca U drh -Z d02ff4965492ed83b018c8e578de7329 +Z 17e4966dab1f438179fdac475786b16e diff --git a/manifest.uuid b/manifest.uuid index 86f32e4c10..1cd31e3773 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3ca31cc3ffe1cce4a9961d29801eebd47f0093d1b53d0ea63386338a1d434fd3 \ No newline at end of file +17f8d5e111a9fe5b074f946e23936ae5a2a7a8d8018bad4212660d8eb81c04b7 \ No newline at end of file From 3680e89b1e0fe927445e7046dee037957818fe1b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Nov 2017 18:31:34 +0000 Subject: [PATCH 15/20] Add the "sqlite3_imposter DB SCHEMA ROOT SQL" command to sqlite3_checker. Use it to fix the checkindex01.test module. There are still errors reported by the checkfreelist01.test module. FossilOrigin-Name: 0593a2ba74c886afe8a65cea1310025bb9777c320d093278044719210c9f6ba2 --- ext/repair/sqlite3_checker.c.in | 52 +++++++++++++++++++++++++++++++ ext/repair/test/checkindex01.test | 29 +++++++---------- manifest | 14 ++++----- manifest.uuid | 2 +- 4 files changed, 71 insertions(+), 26 deletions(-) diff --git a/ext/repair/sqlite3_checker.c.in b/ext/repair/sqlite3_checker.c.in index 6045c36a9d..96b15f2713 100644 --- a/ext/repair/sqlite3_checker.c.in +++ b/ext/repair/sqlite3_checker.c.in @@ -19,9 +19,61 @@ INCLUDE $ROOT/ext/misc/btreeinfo.c INCLUDE $ROOT/ext/repair/checkindex.c INCLUDE $ROOT/ext/repair/checkfreelist.c +/* +** Decode a pointer to an sqlite3 object. +*/ +int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb){ + struct SqliteDb *p; + Tcl_CmdInfo cmdInfo; + if( Tcl_GetCommandInfo(interp, zA, &cmdInfo) ){ + p = (struct SqliteDb*)cmdInfo.objClientData; + *ppDb = p->db; + return TCL_OK; + }else{ + *ppDb = 0; + return TCL_ERROR; + } + return TCL_OK; +} + +/* +** sqlite3_imposter db main rootpage {CREATE TABLE...} ;# setup an imposter +** sqlite3_imposter db main ;# rm all imposters +*/ +static int sqlite3_imposter( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; + const char *zSchema; + int iRoot; + const char *zSql; + + if( objc!=3 && objc!=5 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB SCHEMA [ROOTPAGE SQL]"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zSchema = Tcl_GetString(objv[2]); + if( objc==3 ){ + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 0, 1); + }else{ + if( Tcl_GetIntFromObj(interp, objv[3], &iRoot) ) return TCL_ERROR; + zSql = Tcl_GetString(objv[4]); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 1, iRoot); + sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, db, zSchema, 0, 0); + } + return TCL_OK; +} + #include const char *sqlite3_checker_init_proc(Tcl_Interp *interp){ + Tcl_CreateObjCommand(interp, "sqlite3_imposter", + (Tcl_ObjCmdProc*)sqlite3_imposter, 0, 0); sqlite3_auto_extension((void(*)(void))sqlite3_btreeinfo_init); sqlite3_auto_extension((void(*)(void))sqlite3_checkindex_init); sqlite3_auto_extension((void(*)(void))sqlite3_checkfreelist_init); diff --git a/ext/repair/test/checkindex01.test b/ext/repair/test/checkindex01.test index 36bae886d2..744a21c69e 100644 --- a/ext/repair/test/checkindex01.test +++ b/ext/repair/test/checkindex01.test @@ -71,16 +71,12 @@ do_index_check_test 1.4 i2 { do_test 1.5 { set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t1' }] - sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot - db eval {CREATE TABLE xt1(a, b)} - sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 - - execsql { + sqlite3_imposter db main $tblroot {CREATE TABLE xt1(a,b)} + db eval { UPDATE xt1 SET a='six' WHERE rowid=3; DELETE FROM xt1 WHERE rowid = 5; } - - sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1 + sqlite3_imposter db main } {} do_index_check_test 1.6 i1 { @@ -211,16 +207,14 @@ do_index_check_test 4.1 t4cc { do_test 4.2 { set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t4' }] - sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot - db eval {CREATE TABLE xt4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT)} - sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 + sqlite3_imposter db main $tblroot \ + {CREATE TABLE xt4(a INTEGER PRIMARY KEY, c1 TEXT, c2 TEXT)} - execsql { + db eval { UPDATE xt4 SET c1='hello' WHERE rowid=2; DELETE FROM xt4 WHERE rowid = 3; } - - sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1 + sqlite3_imposter db main } {} do_index_check_test 4.3 t4cc { @@ -263,14 +257,13 @@ do_index_check_test 5.1.3 sqlite_autoindex_t5_1 { do_test 5.2 { set tblroot [db one { SELECT rootpage FROM sqlite_master WHERE name='t5' }] - sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 1 $tblroot - db eval {CREATE TABLE xt5(a INTEGER PRIMARY KEY, c1 TEXT);} - sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 0 - execsql { + sqlite3_imposter db main $tblroot \ + {CREATE TABLE xt5(a INTEGER PRIMARY KEY, c1 TEXT);} + db eval { UPDATE xt5 SET c1='{"x":22, "y":11}' WHERE rowid=1; DELETE FROM xt5 WHERE rowid = 4; } - sqlite3_test_control SQLITE_TESTCTRL_IMPOSTER db main 0 1 + sqlite3_imposter db main } {} do_index_check_test 5.3.1 t5x { diff --git a/manifest b/manifest index ff5e3f094d..1e2056b04c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Move\sthe\stest\sscripts\sfor\scheckfreelist\sand\scheckindex\sover\sinto\sthe\next/repair/test\sdirectory.\s\sRun\sthem\snow\susing\sthe\ssqlite3_checker\sutility\nwith\sthe\s--test\soption.\s\sSome\stests\sare\scurrently\sfailing\sdue\sto\san\nincomplete\sport.\s\sThis\sis\san\sincremental\scheck-in. -D 2017-11-01T18:05:32.697 +C Add\sthe\s"sqlite3_imposter\sDB\sSCHEMA\sROOT\sSQL"\scommand\sto\ssqlite3_checker.\nUse\sit\sto\sfix\sthe\scheckindex01.test\smodule.\s\sThere\sare\sstill\serrors\sreported\nby\sthe\scheckfreelist01.test\smodule. +D 2017-11-01T18:31:34.119 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -329,11 +329,11 @@ F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd782299 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/repair/checkindex.c a013a0a165b2e6f2b278a31566da04913856c88d6ed5457d477f89e78d5979d9 -F ext/repair/sqlite3_checker.c.in 16d62615dfce1ff3eeac83d1a77fe376a7b660afa9db07e1fdd8b964dcc41510 +F ext/repair/sqlite3_checker.c.in 445118c5f7fea958b36fba1b2c464283e60ed4842039ddee3265f1698115ebf7 F ext/repair/sqlite3_checker.tcl 4820d7f58428d47336874b5a148a95b4dad38fe5da72286c01a861590b8f8337 F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e F ext/repair/test/checkfreelist01.test 185648c046c0e269dc3b5141d525b87a38d72f0f55495d46c65108a5dde1428f -F ext/repair/test/checkindex01.test 7be2299647ac7be41b6a7e249eb78e3aba1b57a7b20943ad199a1dc457edb4f7 +F ext/repair/test/checkindex01.test 98bfac50822da9681d75570087aac92a905290ffdaddf95ab6f69212fb4c7b14 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c cc91b6905bf55512c6ebc7dfdd37ac81c86f1753db8cfa6d62f0ee864464044f @@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3ca31cc3ffe1cce4a9961d29801eebd47f0093d1b53d0ea63386338a1d434fd3 -R 9a0909f5ab1dfc6f8f86ede9c43722ca +P 17f8d5e111a9fe5b074f946e23936ae5a2a7a8d8018bad4212660d8eb81c04b7 +R 789d2ddc9f5813af408c7050793744d3 U drh -Z 17e4966dab1f438179fdac475786b16e +Z 00e34b3226a7a4c513950643ceed7a0f diff --git a/manifest.uuid b/manifest.uuid index 1cd31e3773..6e77603c23 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -17f8d5e111a9fe5b074f946e23936ae5a2a7a8d8018bad4212660d8eb81c04b7 \ No newline at end of file +0593a2ba74c886afe8a65cea1310025bb9777c320d093278044719210c9f6ba2 \ No newline at end of file From 9b37b366c2624de4a15e216b34c3d4620bfa42da Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Nov 2017 19:22:08 +0000 Subject: [PATCH 16/20] Adjust page numbers in the ext/repair/test/checkfreelist01.test module due to the fact that the pending-byte page is no longer shifted down to a low-numbered page but is in its rightful place. FossilOrigin-Name: c1641affae31a4350727ce940c92499263880e672dc2c3f47e78e1c23ae99b78 --- ext/repair/test/checkfreelist01.test | 14 +++++++------- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/ext/repair/test/checkfreelist01.test b/ext/repair/test/checkfreelist01.test index 7c33862583..7e2dd51c37 100644 --- a/ext/repair/test/checkfreelist01.test +++ b/ext/repair/test/checkfreelist01.test @@ -27,8 +27,8 @@ do_execsql_test 1.5 { ) SELECT i FROM freelist_trunk WHERE i!=1; } { - 10010 9716 9344 8970 8596 8223 7848 7475 7103 6728 6355 5983 5609 5235 - 4861 4488 4113 3741 3368 2993 2620 2248 1873 1500 1126 753 378 5 + 10009 9715 9343 8969 8595 8222 7847 7474 7102 6727 6354 5982 5608 5234 + 4860 4487 4112 3740 3367 2992 2619 2247 1872 1499 1125 752 377 5 } do_execsql_test 1.6 { SELECT checkfreelist('main') } {ok} @@ -50,7 +50,7 @@ do_execsql_test 1.7 { BEGIN; UPDATE sqlite_dbpage SET data = set_int(data, 1, get_int(data, 1)-1) - WHERE pgno=4861; + WHERE pgno=4860; SELECT checkfreelist('main'); ROLLBACK; } {{free-list count mismatch: actual=6725 header=6726}} @@ -59,19 +59,19 @@ do_execsql_test 1.8 { BEGIN; UPDATE sqlite_dbpage SET data = set_int(data, 5, (SELECT * FROM pragma_page_count)+1) - WHERE pgno=4861; + WHERE pgno=4860; SELECT checkfreelist('main'); ROLLBACK; -} {{leaf page 10093 is out of range (child 3 of trunk page 4861)}} +} {{leaf page 10092 is out of range (child 3 of trunk page 4860)}} do_execsql_test 1.9 { BEGIN; UPDATE sqlite_dbpage SET data = set_int(data, 5, 0) - WHERE pgno=4861; + WHERE pgno=4860; SELECT checkfreelist('main'); ROLLBACK; -} {{leaf page 0 is out of range (child 3 of trunk page 4861)}} +} {{leaf page 0 is out of range (child 3 of trunk page 4860)}} do_execsql_test 1.10 { BEGIN; diff --git a/manifest b/manifest index 1e2056b04c..4d5258d691 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"sqlite3_imposter\sDB\sSCHEMA\sROOT\sSQL"\scommand\sto\ssqlite3_checker.\nUse\sit\sto\sfix\sthe\scheckindex01.test\smodule.\s\sThere\sare\sstill\serrors\sreported\nby\sthe\scheckfreelist01.test\smodule. -D 2017-11-01T18:31:34.119 +C Adjust\spage\snumbers\sin\sthe\sext/repair/test/checkfreelist01.test\smodule\sdue\sto\nthe\sfact\sthat\sthe\spending-byte\spage\sis\sno\slonger\sshifted\sdown\sto\sa\slow-numbered\npage\sbut\sis\sin\sits\srightful\splace. +D 2017-11-01T19:22:08.991 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -332,7 +332,7 @@ F ext/repair/checkindex.c a013a0a165b2e6f2b278a31566da04913856c88d6ed5457d477f89 F ext/repair/sqlite3_checker.c.in 445118c5f7fea958b36fba1b2c464283e60ed4842039ddee3265f1698115ebf7 F ext/repair/sqlite3_checker.tcl 4820d7f58428d47336874b5a148a95b4dad38fe5da72286c01a861590b8f8337 F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e -F ext/repair/test/checkfreelist01.test 185648c046c0e269dc3b5141d525b87a38d72f0f55495d46c65108a5dde1428f +F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc78249442da72ff3f8297398a69 F ext/repair/test/checkindex01.test 98bfac50822da9681d75570087aac92a905290ffdaddf95ab6f69212fb4c7b14 F ext/repair/test/test.tcl 686d76d888dffd021f64260abf29a55c57b2cedfa7fc69150b42b1d6119aac3c F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 @@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 17f8d5e111a9fe5b074f946e23936ae5a2a7a8d8018bad4212660d8eb81c04b7 -R 789d2ddc9f5813af408c7050793744d3 +P 0593a2ba74c886afe8a65cea1310025bb9777c320d093278044719210c9f6ba2 +R e97da7e6f34a0c98621feff75b5fc1d7 U drh -Z 00e34b3226a7a4c513950643ceed7a0f +Z 33cbf67c611e121ff7dcf902c08c95ea diff --git a/manifest.uuid b/manifest.uuid index 6e77603c23..6eea893785 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0593a2ba74c886afe8a65cea1310025bb9777c320d093278044719210c9f6ba2 \ No newline at end of file +c1641affae31a4350727ce940c92499263880e672dc2c3f47e78e1c23ae99b78 \ No newline at end of file From 7dcde38f59eb0b5d858597147f7f8725d2672764 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Nov 2017 19:50:11 +0000 Subject: [PATCH 17/20] Make sure the JSON1 extension is available to sqlite3_checker.exe FossilOrigin-Name: a946a8ed46ba76b9a09bcbac58000d3d92db1f08a98f6cc12365be696d8921de --- ext/repair/sqlite3_checker.c.in | 1 + manifest | 13 ++++++------- manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/repair/sqlite3_checker.c.in b/ext/repair/sqlite3_checker.c.in index 96b15f2713..76e9708787 100644 --- a/ext/repair/sqlite3_checker.c.in +++ b/ext/repair/sqlite3_checker.c.in @@ -4,6 +4,7 @@ */ #define TCLSH_INIT_PROC sqlite3_checker_init_proc #define SQLITE_ENABLE_DBPAGE_VTAB 1 +#define SQLITE_ENABLE_JSON1 1 #undef SQLITE_THREADSAFE #define SQLITE_THREADSAFE 0 #undef SQLITE_ENABLE_COLUMN_METADATA diff --git a/manifest b/manifest index 86482e8b0c..47e05b0082 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\scheckindex.c\sextension\sand\sthe\ssqlite3_checker\sutility\sprogram\sused\nfor\sdoing\slive\svalidation\sof\slarge\sdatabases. -D 2017-11-01T19:44:19.216 +C Make\ssure\sthe\sJSON1\sextension\sis\savailable\sto\ssqlite3_checker.exe +D 2017-11-01T19:50:11.569 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -329,7 +329,7 @@ F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd782299 F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 F ext/repair/checkindex.c a013a0a165b2e6f2b278a31566da04913856c88d6ed5457d477f89e78d5979d9 -F ext/repair/sqlite3_checker.c.in 445118c5f7fea958b36fba1b2c464283e60ed4842039ddee3265f1698115ebf7 +F ext/repair/sqlite3_checker.c.in 4a5a3af3f450fe503e5a2985e98516dc2a6b9ad247449e284c1cf140fc91720f F ext/repair/sqlite3_checker.tcl 4820d7f58428d47336874b5a148a95b4dad38fe5da72286c01a861590b8f8337 F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e F ext/repair/test/checkfreelist01.test 3e8aa6aeb4007680c94a8d07b41c339aa635cc78249442da72ff3f8297398a69 @@ -1673,8 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bb39744f4b2b25c10d293e85db7579e2a99c639fdab45e93d1de75952b68b2de c1641affae31a4350727ce940c92499263880e672dc2c3f47e78e1c23ae99b78 -R e97da7e6f34a0c98621feff75b5fc1d7 -T +closed c1641affae31a4350727ce940c92499263880e672dc2c3f47e78e1c23ae99b78 +P 0c5d18a01ec77f784d5434c5465ab8da9a0c365a58d4bd8551872ca90aaf42d6 +R f83c5c1798f6f791a77e395a421e7535 U drh -Z 29d052b9d1f9214dbd3b25e5b7544e56 +Z b827e3b3bff13979721cde0c77e4ccc0 diff --git a/manifest.uuid b/manifest.uuid index b0b6dde090..05400f1a8f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0c5d18a01ec77f784d5434c5465ab8da9a0c365a58d4bd8551872ca90aaf42d6 \ No newline at end of file +a946a8ed46ba76b9a09bcbac58000d3d92db1f08a98f6cc12365be696d8921de \ No newline at end of file From a48a290bcc6f7f81d0378aa5fc4ad51ee6e182cc Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Nov 2017 19:58:25 +0000 Subject: [PATCH 18/20] Fix harmless compiler warnings in sqlite3_checker.exe FossilOrigin-Name: 491f867b377b3b9e00bd713fb07df00207673f9eca0e7d5b7af7974082c8e3f0 --- ext/repair/checkfreelist.c | 2 +- ext/repair/checkindex.c | 4 ++-- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/dbpage.c | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ext/repair/checkfreelist.c b/ext/repair/checkfreelist.c index cd2801e040..990be4afa7 100644 --- a/ext/repair/checkfreelist.c +++ b/ext/repair/checkfreelist.c @@ -166,7 +166,7 @@ static int checkFreelist( u32 i; u32 iTrunk = (u32)sqlite3_column_int(pTrunk, 0); const u8 *aData = (const u8*)sqlite3_column_blob(pTrunk, 1); - int nData = sqlite3_column_bytes(pTrunk, 1); + u32 nData = (u32)sqlite3_column_bytes(pTrunk, 1); u32 iNext = get4byte(&aData[0]); u32 nLeaf = get4byte(&aData[4]); diff --git a/ext/repair/checkindex.c b/ext/repair/checkindex.c index 362e199168..fa8c713f7c 100644 --- a/ext/repair/checkindex.c +++ b/ext/repair/checkindex.c @@ -263,7 +263,7 @@ static void cidxFinalize(int *pRc, sqlite3_stmt *pStmt){ char *cidxStrdup(int *pRc, const char *zStr){ char *zRet = 0; if( *pRc==SQLITE_OK ){ - int n = strlen(zStr); + int n = (int)strlen(zStr); zRet = cidxMalloc(pRc, n+1); if( zRet ) memcpy(zRet, zStr, n+1); } @@ -493,7 +493,7 @@ static int cidxDecodeAfter( ){ char **azAfter; int rc = SQLITE_OK; - int nAfterKey = strlen(zAfterKey); + int nAfterKey = (int)strlen(zAfterKey); azAfter = cidxMalloc(&rc, sizeof(char*)*nCol + nAfterKey+1); if( rc==SQLITE_OK ){ diff --git a/manifest b/manifest index 47e05b0082..9e0fd087b8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\sJSON1\sextension\sis\savailable\sto\ssqlite3_checker.exe -D 2017-11-01T19:50:11.569 +C Fix\sharmless\scompiler\swarnings\sin\ssqlite3_checker.exe +D 2017-11-01T19:58:25.196 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -327,8 +327,8 @@ F ext/rbu/sqlite3rbu.c 64bd08c1011456f90564ed167abce3a9c2af421a924b21eb57231e078 F ext/rbu/sqlite3rbu.h b42bcd4d8357268c6c39ab2a60b29c091e89328fa8cc49c8fac5ab8d007e79b2 F ext/rbu/test_rbu.c 7073979b9cc80912bb03599ac8d85ab5d3bf03cfacd3463f2dcdd7822997533a F ext/repair/README.md 92f5e8aae749a4dae14f02eea8e1bb42d4db2b6ce5e83dbcdd6b1446997e0c15 -F ext/repair/checkfreelist.c 0abb84b4545016d57ba1a2aa8884c72c73ed838968909858c03bc1f38fb6b054 -F ext/repair/checkindex.c a013a0a165b2e6f2b278a31566da04913856c88d6ed5457d477f89e78d5979d9 +F ext/repair/checkfreelist.c 0dbae18c1b552f58d64f8969e4fb1e7f11930c60a8c2a9a8d50b7f15bdfd54bd +F ext/repair/checkindex.c 73f26fc1e2e17d68ede5db5b0aaf4869f2d6182f45f3d3624befc503c0f04a70 F ext/repair/sqlite3_checker.c.in 4a5a3af3f450fe503e5a2985e98516dc2a6b9ad247449e284c1cf140fc91720f F ext/repair/sqlite3_checker.tcl 4820d7f58428d47336874b5a148a95b4dad38fe5da72286c01a861590b8f8337 F ext/repair/test/README.md 34b2f542cf5be7bffe479242b33ee3492cea30711e447cc4a1a86cb5915f419e @@ -420,7 +420,7 @@ F src/callback.c 28a8ede982fde4129b828350f78f2c01fe7d12c74d1a0a05d7108ab36f30868 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c ff1be3eed7bdd75aaca61ca8dc848f7c9f850ef2fb9cb56f2734e922a098f9c0 F src/date.c 48f743d88bbe88f848532d333cca84f26e52a4f217e86f86be7fc1b919c33d74 -F src/dbpage.c 003755140e21b917c0a39e70907c1e4612b25d1e24b9a3c64b0d638c8ecb5dc2 +F src/dbpage.c 432f46fb47c99ae1f0e2863fe595038a6f9d3b59f4a3dabe5f86ec2a88758fde F src/dbstat.c 7a4ba8518b6369ef3600c49cf9c918ad979acba610b2aebef1b656d649b96720 F src/delete.c 21a5f1812fdb599e9f7afb9f650bdabab60a3afd51d7e94e539c982f647b0023 F src/expr.c 0016b95aed1df47ebc0eb7415bd621841d72dbffd3bfb62210e50e71d83b4691 @@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 0c5d18a01ec77f784d5434c5465ab8da9a0c365a58d4bd8551872ca90aaf42d6 -R f83c5c1798f6f791a77e395a421e7535 +P a946a8ed46ba76b9a09bcbac58000d3d92db1f08a98f6cc12365be696d8921de +R 7ee45d8c7d2f00dd882d0691ca6a1bfb U drh -Z b827e3b3bff13979721cde0c77e4ccc0 +Z 0f28cc7179e20c43d298dd1935e3a35c diff --git a/manifest.uuid b/manifest.uuid index 05400f1a8f..7f5fcadcf3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a946a8ed46ba76b9a09bcbac58000d3d92db1f08a98f6cc12365be696d8921de \ No newline at end of file +491f867b377b3b9e00bd713fb07df00207673f9eca0e7d5b7af7974082c8e3f0 \ No newline at end of file diff --git a/src/dbpage.c b/src/dbpage.c index f287e72229..75e4c42e67 100644 --- a/src/dbpage.c +++ b/src/dbpage.c @@ -329,7 +329,7 @@ static int dbpageUpdate( goto update_fail; } pBt = pTab->db->aDb[iDb].pBt; - if( pgno<1 || pBt==0 || pgno>sqlite3BtreeLastPage(pBt) ){ + if( pgno<1 || pBt==0 || pgno>(int)sqlite3BtreeLastPage(pBt) ){ zErr = "bad page number"; goto update_fail; } From 53be36b026469d665ed238dc4f65f37161862c7b Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 3 Nov 2017 06:45:37 +0000 Subject: [PATCH 19/20] Fix harmless compiler warnings. FossilOrigin-Name: d088c5a3f1a803118cb67560fc35c11178e1b7e54c0e1511677559906b9da980 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/whereexpr.c | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 9e0fd087b8..c6be978982 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sin\ssqlite3_checker.exe -D 2017-11-01T19:58:25.196 +C Fix\sharmless\scompiler\swarnings. +D 2017-11-03T06:45:37.503 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -556,7 +556,7 @@ F src/walker.c d591e8a9ccf60abb010966b354fcea4aa08eba4d83675c2b281a8764c76cc22f F src/where.c b7a075f5fb3d912a891dcc3257f538372bb4a1622dd8ca7d752ad95ce8949ba4 F src/whereInt.h 82c04c5075308abbac59180c8bad5ecb45b07453981f60a53f3c7dee21e1e971 F src/wherecode.c 4a117dd5886616d074f7b6589c23bf742f5a9858d6ffdaf8b9d1f76ab06245d2 -F src/whereexpr.c afcac9cccfc0fdaccbdda94034a398947b6dc47dbf821c1b496261722832a6a4 +F src/whereexpr.c 427ea8e96ec24f2a7814c67b8024ad664a9c7656264c4566c34743cb23186e46 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d @@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a946a8ed46ba76b9a09bcbac58000d3d92db1f08a98f6cc12365be696d8921de -R 7ee45d8c7d2f00dd882d0691ca6a1bfb -U drh -Z 0f28cc7179e20c43d298dd1935e3a35c +P 491f867b377b3b9e00bd713fb07df00207673f9eca0e7d5b7af7974082c8e3f0 +R 2898c172e35d6c4a6ab6316ba23aa96d +U mistachkin +Z 0198e72b5c24cd8ac4cd7a2d9de73913 diff --git a/manifest.uuid b/manifest.uuid index 7f5fcadcf3..d96908a8ac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -491f867b377b3b9e00bd713fb07df00207673f9eca0e7d5b7af7974082c8e3f0 \ No newline at end of file +d088c5a3f1a803118cb67560fc35c11178e1b7e54c0e1511677559906b9da980 \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index 9f83a84534..58f1908cf8 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -983,7 +983,7 @@ static void exprAnalyze( int op; /* Top-level operator. pExpr->op */ Parse *pParse = pWInfo->pParse; /* Parsing context */ sqlite3 *db = pParse->db; /* Database connection */ - unsigned char eOp2; /* op2 value for LIKE/REGEXP/GLOB */ + unsigned char eOp2 = 0; /* op2 value for LIKE/REGEXP/GLOB */ int nLeft; /* Number of elements on left side vector */ if( db->mallocFailed ){ @@ -1227,7 +1227,7 @@ static void exprAnalyze( ** to do anything with MATCH functions. */ if( pWC->op==TK_AND ){ - Expr *pRight, *pLeft; + Expr *pRight = 0, *pLeft = 0; int res = isAuxiliaryVtabOperator(pExpr, &eOp2, &pLeft, &pRight); while( res-- > 0 ){ int idxNew; From 537e70289ff4168476252458965ca0bccf70559b Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 3 Nov 2017 08:46:48 +0000 Subject: [PATCH 20/20] The extensions functions in spellfix are all deterministic. FossilOrigin-Name: 29ec855e13e0dcd675dcf12948b42f9e669d0a31c5d9efb95857888aba0beeee --- ext/misc/spellfix.c | 24 +++++++++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index 1ac1712f4e..4f17b88e1d 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -1122,15 +1122,17 @@ static int editDist3Install(sqlite3 *db){ if( pConfig==0 ) return SQLITE_NOMEM; memset(pConfig, 0, sizeof(*pConfig)); rc = sqlite3_create_function_v2(db, "editdist3", - 2, SQLITE_UTF8, pConfig, editDist3SqlFunc, 0, 0, 0); + 2, SQLITE_UTF8|SQLITE_DETERMINISTIC, pConfig, + editDist3SqlFunc, 0, 0, 0); if( rc==SQLITE_OK ){ rc = sqlite3_create_function_v2(db, "editdist3", - 3, SQLITE_UTF8, pConfig, editDist3SqlFunc, 0, 0, 0); + 3, SQLITE_UTF8|SQLITE_DETERMINISTIC, pConfig, + editDist3SqlFunc, 0, 0, 0); } if( rc==SQLITE_OK ){ rc = sqlite3_create_function_v2(db, "editdist3", - 1, SQLITE_UTF8, pConfig, editDist3SqlFunc, 0, 0, - editDist3ConfigDelete); + 1, SQLITE_UTF8|SQLITE_DETERMINISTIC, pConfig, + editDist3SqlFunc, 0, 0, editDist3ConfigDelete); }else{ sqlite3_free(pConfig); } @@ -2895,18 +2897,22 @@ static sqlite3_module spellfix1Module = { static int spellfix1Register(sqlite3 *db){ int rc = SQLITE_OK; int i; - rc = sqlite3_create_function(db, "spellfix1_translit", 1, SQLITE_UTF8, 0, - transliterateSqlFunc, 0, 0); + rc = sqlite3_create_function(db, "spellfix1_translit", 1, + SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, + transliterateSqlFunc, 0, 0); if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "spellfix1_editdist", 2, SQLITE_UTF8, 0, + rc = sqlite3_create_function(db, "spellfix1_editdist", 2, + SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, editdistSqlFunc, 0, 0); } if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "spellfix1_phonehash", 1, SQLITE_UTF8, 0, + rc = sqlite3_create_function(db, "spellfix1_phonehash", 1, + SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, phoneticHashSqlFunc, 0, 0); } if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "spellfix1_scriptcode", 1, SQLITE_UTF8, 0, + rc = sqlite3_create_function(db, "spellfix1_scriptcode", 1, + SQLITE_UTF8|SQLITE_DETERMINISTIC, 0, scriptCodeSqlFunc, 0, 0); } if( rc==SQLITE_OK ){ diff --git a/manifest b/manifest index c6be978982..a447c36649 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings. -D 2017-11-03T06:45:37.503 +C The\sextensions\sfunctions\sin\sspellfix\sare\sall\sdeterministic. +D 2017-11-03T08:46:48.968 F Makefile.in b142eb20482922153ebc77b261cdfd0a560ed05a81e9f6d9a2b0e8192922a1d2 F Makefile.linux-gcc 7bc79876b875010e8c8f9502eb935ca92aa3c434 F Makefile.msc a55372a22454e742ba7c8f6edf05b83213ec01125166ad7dcee0567e2f7fc81b @@ -282,7 +282,7 @@ F ext/misc/series.c f3c0dba5c5c749ce1782b53076108f87cf0b71041eb6023f727a9c50681d F ext/misc/sha1.c 0b9e9b855354910d3ca467bf39099d570e73db56 F ext/misc/shathree.c fa185d7aee0ad0aca5e091b4a2db7baff11796170e5793b5de99e511a13af448 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 -F ext/misc/spellfix.c a4723b6aff748a417b5091b68a46443265c40f0d +F ext/misc/spellfix.c 41cf26c6b89fcaa8798ae10ae64d39c1f1d9d6995152e545bd491c13058b8fac F ext/misc/stmt.c 6f16443abb3551e3f5813bb13ba19a30e7032830015b0f92fe0c0453045c0a11 F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/unionvtab.c 1e0ebc5078e1a916db191bcd88f87e94ea7ba4aa563ee30ff706261cb4b39461 @@ -1673,7 +1673,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 491f867b377b3b9e00bd713fb07df00207673f9eca0e7d5b7af7974082c8e3f0 -R 2898c172e35d6c4a6ab6316ba23aa96d -U mistachkin -Z 0198e72b5c24cd8ac4cd7a2d9de73913 +P d088c5a3f1a803118cb67560fc35c11178e1b7e54c0e1511677559906b9da980 +R b7838526febd368042ba32a4576c580c +U drh +Z 18ebbc1917110c35a4b2606f5ce6f533 diff --git a/manifest.uuid b/manifest.uuid index d96908a8ac..5035b0e5a7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d088c5a3f1a803118cb67560fc35c11178e1b7e54c0e1511677559906b9da980 \ No newline at end of file +29ec855e13e0dcd675dcf12948b42f9e669d0a31c5d9efb95857888aba0beeee \ No newline at end of file