From f1173b69b1c3bfc68ea2877e6fabf15902b7bf72 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 17 Apr 2019 21:17:22 +0000 Subject: [PATCH 01/38] Add the experimental dbdata extension. FossilOrigin-Name: a3ab58832935e1399ecc7e4d8daefa3a6afa6b301792ce7176bc5d7c173510fb --- ext/misc/dbdata.c | 620 ++++++++++++++++++++++++++++++++++++++++++++++ manifest | 17 +- manifest.uuid | 2 +- test/dbdata.test | 54 ++++ 4 files changed, 686 insertions(+), 7 deletions(-) create mode 100644 ext/misc/dbdata.c create mode 100644 test/dbdata.test diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c new file mode 100644 index 0000000000..c0ad8c1f9e --- /dev/null +++ b/ext/misc/dbdata.c @@ -0,0 +1,620 @@ +/* +** 2019-04-17 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains an implementation of the eponymous "sqlite_dbdata" +** virtual table. sqlite_dbdata is used to extract data directly from a +** database b-tree page and its associated overflow pages, bypassing the b-tree +** layer. The table schema is equivalent to: +** +** CREATE TABLE sqlite_dbdata( +** pgno INTEGER, +** cell INTEGER, +** field INTEGER, +** value ANY, +** schema TEXT HIDDEN +** ); +** +** Each page of the database is inspected. If it cannot be interpreted as a +** b-tree page, or if it is a b-tree page containing 0 entries, the +** sqlite_dbdata table contains no rows for that page. Otherwise, the table +** contains one row for each field in the record associated with each +** cell on the page. For intkey b-trees, the key value is stored in field -1. +** +** For example, for the database: +** +** CREATE TABLE t1(a, b); -- root page is page 2 +** INSERT INTO t1(rowid, a, b) VALUES(5, 'v', 'five'); +** INSERT INTO t1(rowid, a, b) VALUES(10, 'x', 'ten'); +** +** the sqlite_dbdata table contains, as well as from entries related to +** page 1, content equivalent to: +** +** INSERT INTO sqlite_dbdata(pgno, cell, field, value) VALUES +** (2, 0, -1, 5 ), +** (2, 0, 0, 'v' ), +** (2, 0, 1, 'five'), +** (2, 1, -1, 10 ), +** (2, 1, 0, 'x' ), +** (2, 1, 1, 'ten' ); +** +** If database corruption is encountered, this module does not report an +** error. Instead, it attempts to extract as much data as possible and +** ignores the corruption. +** +** This module requires that the "sqlite_dbpage" eponymous virtual table be +** available. +*/ +#if !defined(SQLITEINT_H) +#include "sqlite3ext.h" + +typedef unsigned char u8; +typedef unsigned int u32; + +#endif +SQLITE_EXTENSION_INIT1 +#include +#include + +typedef struct DbdataTable DbdataTable; +typedef struct DbdataCursor DbdataCursor; + +/* A cursor for the sqlite_dbdata table */ +struct DbdataCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + sqlite3_stmt *pStmt; /* For fetching database pages */ + + int iPgno; /* Current page number */ + u8 *aPage; /* Buffer containing page */ + int nPage; /* Size of aPage[] in bytes */ + int nCell; /* Number of cells on aPage[] */ + int iCell; /* Current cell number */ + u8 *pRec; /* Buffer containing current record */ + int nRec; /* Size of pRec[] in bytes */ + int nField; /* Number of fields in pRec */ + int iField; /* Current field number */ + sqlite3_int64 iIntkey; /* Integer key value */ + + sqlite3_int64 iRowid; +}; + +/* The sqlite_dbdata table */ +struct DbdataTable { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; /* The database connection */ +}; + +#define DBDATA_COLUMN_PGNO 0 +#define DBDATA_COLUMN_CELL 1 +#define DBDATA_COLUMN_FIELD 2 +#define DBDATA_COLUMN_VALUE 3 +#define DBDATA_COLUMN_SCHEMA 4 + +#define DBDATA_SCHEMA \ + "CREATE TABLE x(" \ + " pgno INTEGER," \ + " cell INTEGER," \ + " field INTEGER," \ + " value ANY," \ + " schema TEXT HIDDEN" \ + ")" + +/* +** Connect to the sqlite_dbdata virtual table. +*/ +static int dbdataConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + DbdataTable *pTab = 0; + int rc = sqlite3_declare_vtab(db, DBDATA_SCHEMA); + + if( rc==SQLITE_OK ){ + pTab = (DbdataTable*)sqlite3_malloc64(sizeof(DbdataTable)); + if( pTab==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pTab, 0, sizeof(DbdataTable)); + pTab->db = db; + } + } + + *ppVtab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** Disconnect from or destroy a dbdata virtual table. +*/ +static int dbdataDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** +** This function interprets two types of constraints: +** +** schema=? +** pgno=? +** +** If neither are present, idxNum is set to 0. If schema=? is present, +** the 0x01 bit in idxNum is set. If pgno=? is present, the 0x02 bit +** in idxNum is set. +** +** If both parameters are present, schema is in position 0 and pgno in +** position 1. +*/ +static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int i; + int iSchema = -1; + int iPgno = -1; + + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i]; + if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + if( p->iColumn==DBDATA_COLUMN_SCHEMA ){ + if( p->usable==0 ) return SQLITE_CONSTRAINT; + iSchema = i; + } + if( p->iColumn==DBDATA_COLUMN_PGNO && p->usable ){ + iPgno = i; + } + } + } + + if( iSchema>=0 ){ + pIdxInfo->aConstraintUsage[iSchema].argvIndex = 1; + pIdxInfo->aConstraintUsage[iSchema].omit = 1; + } + if( iPgno>=0 ){ + pIdxInfo->aConstraintUsage[iPgno].argvIndex = 1 + (iSchema>=0); + pIdxInfo->aConstraintUsage[iPgno].omit = 1; + } + pIdxInfo->idxNum = (iSchema>=0 ? 0x01 : 0x00) | (iPgno>=0 ? 0x02 : 0x00); + return SQLITE_OK; +} + +/* +** Open a new dbdata cursor. +*/ +static int dbdataOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + DbdataCursor *pCsr; + + pCsr = (DbdataCursor*)sqlite3_malloc64(sizeof(DbdataCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + }else{ + memset(pCsr, 0, sizeof(DbdataCursor)); + pCsr->base.pVtab = pVTab; + } + + *ppCursor = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +static void dbdataResetCursor(DbdataCursor *pCsr){ + sqlite3_finalize(pCsr->pStmt); + pCsr->pStmt = 0; + pCsr->iPgno = 1; + pCsr->iCell = 0; + pCsr->iField = 0; +} + +/* +** Close a dbdata cursor. +*/ +static int dbdataClose(sqlite3_vtab_cursor *pCursor){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + dbdataResetCursor(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + + +/* Decode big-endian integers */ +static unsigned int get_uint16(unsigned char *a){ + return (a[0]<<8)|a[1]; +} +static unsigned int get_uint32(unsigned char *a){ + return (a[0]<<24)|(a[1]<<16)|(a[2]<<8)|a[3]; +} + +static int dbdataLoadPage( + DbdataCursor *pCsr, + u32 pgno, + u8 **ppPage, + int *pnPage +){ + int rc2; + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = pCsr->pStmt; + + *ppPage = 0; + *pnPage = 0; + sqlite3_bind_int64(pStmt, 2, pgno); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + int nCopy = sqlite3_column_bytes(pStmt, 0); + u8 *pPage = (u8*)sqlite3_malloc64(nCopy); + if( pPage==0 ){ + rc = SQLITE_NOMEM; + }else{ + const u8 *pCopy = sqlite3_column_blob(pStmt, 0); + memcpy(pPage, pCopy, nCopy); + *ppPage = pPage; + *pnPage = nCopy; + } + } + rc2 = sqlite3_reset(pStmt); + if( *ppPage==0 ) rc = rc2; + + return rc; +} + +/* +** Read a varint. Put the value in *pVal and return the number of bytes. +*/ +static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){ + sqlite3_int64 v = 0; + int i; + for(i=0; i<8; i++){ + v = (v<<7) + (z[i]&0x7f); + if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; } + } + v = (v<<8) + (z[i]&0xff); + *pVal = v; + return 9; +} + +/* +** Move a dbdata cursor to the next entry in the file. +*/ +static int dbdataNext(sqlite3_vtab_cursor *pCursor){ + DbdataCursor *pCsr = (DbdataCursor *)pCursor; + + pCsr->iRowid++; + while( 1 ){ + int rc; + + if( pCsr->aPage==0 ){ + rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage); + if( rc!=SQLITE_OK ) return rc; + pCsr->iCell = 0; + pCsr->nCell = get_uint16(&pCsr->aPage[pCsr->iPgno==1 ? 103 : 3]); + } + + /* If there is no record loaded, load it now. */ + if( pCsr->pRec==0 ){ + int iOff = (pCsr->iPgno==1 ? 100 : 0); + int bHasRowid = 0; + int nPointer = 0; + sqlite3_int64 nPayload = 0; + sqlite3_int64 nHdr = 0; + int iHdr; + int U, X; + int nLocal; + + switch( pCsr->aPage[iOff] ){ + case 0x02: + nPointer = 4; + break; + case 0x0a: + break; + case 0x0d: + bHasRowid = 1; + break; + default: + pCsr->iCell = pCsr->nCell; + break; + } + if( pCsr->iCell>=pCsr->nCell ){ + sqlite3_free(pCsr->aPage); + pCsr->aPage = 0; + return SQLITE_OK; + } + + iOff += 8 + nPointer + pCsr->iCell*2; + iOff = get_uint16(&pCsr->aPage[iOff]); + + /* For an interior node cell, skip past the child-page number */ + iOff += nPointer; + + /* Load the "byte of payload including overflow" field */ + iOff += dbdataGetVarint(&pCsr->aPage[iOff], &nPayload); + + /* If this is a leaf intkey cell, load the rowid */ + if( bHasRowid ){ + iOff += dbdataGetVarint(&pCsr->aPage[iOff], &pCsr->iIntkey); + } + + /* Allocate space for payload */ + pCsr->pRec = (u8*)sqlite3_malloc64(nPayload); + if( pCsr->pRec==0 ) return SQLITE_NOMEM; + pCsr->nRec = nPayload; + + U = pCsr->nPage; + if( bHasRowid ){ + X = U-35; + }else{ + X = ((U-12)*64/255)-23; + } + if( nPayload<=X ){ + nLocal = nPayload; + }else{ + int M, K; + M = ((U-12)*32/255)-23; + K = M+((nPayload-M)%(U-4)); + if( K<=X ){ + nLocal = K; + }else{ + nLocal = M; + } + } + + /* Load the nLocal bytes of payload */ + memcpy(pCsr->pRec, &pCsr->aPage[iOff], nLocal); + iOff += nLocal; + + /* Load content from overflow pages */ + if( nPayload>nLocal ){ + sqlite3_int64 nRem = nPayload - nLocal; + u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); + while( nRem>0 ){ + u8 *aOvfl = 0; + int nOvfl = 0; + int nCopy; + rc = dbdataLoadPage(pCsr, pgnoOvfl, &aOvfl, &nOvfl); + assert( rc!=SQLITE_OK || nOvfl==pCsr->nPage ); + if( rc!=SQLITE_OK ) return rc; + + nCopy = U-4; + if( nCopy>nRem ) nCopy = nRem; + memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy); + nRem -= nCopy; + + sqlite3_free(aOvfl); + } + } + + /* Figure out how many fields in the record */ + pCsr->nField = 0; + iHdr = dbdataGetVarint(pCsr->pRec, &nHdr); + while( iHdrpRec[iHdr], &iDummy); + pCsr->nField++; + } + + pCsr->iField = (bHasRowid ? -2 : -1); + } + + pCsr->iField++; + if( pCsr->iFieldnField ) return SQLITE_OK; + + /* Advance to the next cell. The next iteration of the loop will load + ** the record and so on. */ + sqlite3_free(pCsr->pRec); + pCsr->pRec = 0; + pCsr->iCell++; + } + + assert( !"can't get here" ); + return SQLITE_OK; +} + +/* We have reached EOF if previous sqlite3_step() returned +** anything other than SQLITE_ROW; +*/ +static int dbdataEof(sqlite3_vtab_cursor *pCursor){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + return pCsr->aPage==0; +} + +/* Position a cursor back to the beginning. +*/ +static int dbdataFilter( + sqlite3_vtab_cursor *pCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; + int rc; + const char *zSchema = "main"; + + dbdataResetCursor(pCsr); + assert( pCsr->iPgno==1 ); + if( idxNum & 0x01 ){ + zSchema = sqlite3_value_text(argv[0]); + } + if( idxNum & 0x02 ){ + pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]); + } + + rc = sqlite3_prepare_v2(pTab->db, + "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1, + &pCsr->pStmt, 0 + ); + if( rc==SQLITE_OK ){ + rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT); + } + if( rc==SQLITE_OK ){ + rc = dbdataNext(pCursor); + } + return rc; +} + +static int dbdataValueBytes(int eType){ + switch( eType ){ + case 0: case 8: case 9: + case 10: case 11: + return 0; + case 1: + return 1; + case 2: + return 2; + case 3: + return 3; + case 4: + return 4; + case 5: + return 6; + case 6: + case 7: + return 8; + default: + return ((eType-12) / 2); + } +} + +static void dbdataValue(sqlite3_context *pCtx, int eType, u8 *pData){ + switch( eType ){ + case 0: + case 10: + case 11: + sqlite3_result_null(pCtx); + break; + + case 8: + sqlite3_result_int(pCtx, 0); + break; + case 9: + sqlite3_result_int(pCtx, 1); + break; + + case 1: case 2: case 3: case 4: case 5: case 6: case 7: { + sqlite3_uint64 v = (signed char)pData[0]; + pData++; + switch( eType ){ + case 7: + case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 4: v = (v<<8) + pData[0]; pData++; + case 3: v = (v<<8) + pData[0]; pData++; + case 2: v = (v<<8) + pData[0]; pData++; + } + + if( eType==7 ){ + double r; + memcpy(&r, &v, sizeof(r)); + sqlite3_result_double(pCtx, r); + }else{ + sqlite3_result_int64(pCtx, (sqlite3_int64)v); + } + break; + } + + default: { + int n = ((eType-12) / 2); + if( eType % 2 ){ + sqlite3_result_text(pCtx, pData, n, SQLITE_TRANSIENT); + }else{ + sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); + } + } + } +} + +/* Return a column for the sqlite_dbdata table */ +static int dbdataColumn( + sqlite3_vtab_cursor *pCursor, + sqlite3_context *ctx, + int i +){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + switch( i ){ + case DBDATA_COLUMN_PGNO: + sqlite3_result_int64(ctx, pCsr->iPgno); + break; + case DBDATA_COLUMN_CELL: + sqlite3_result_int(ctx, pCsr->iCell); + break; + case DBDATA_COLUMN_FIELD: + sqlite3_result_int(ctx, pCsr->iField); + break; + case DBDATA_COLUMN_VALUE: { + if( pCsr->iField<0 ){ + sqlite3_result_int64(ctx, pCsr->iIntkey); + }else{ + int iHdr; + sqlite3_int64 iType; + sqlite3_int64 iOff; + int i; + iHdr = dbdataGetVarint(pCsr->pRec, &iOff); + for(i=0; iiField; i++){ + iHdr += dbdataGetVarint(&pCsr->pRec[iHdr], &iType); + iOff += dbdataValueBytes(iType); + } + dbdataGetVarint(&pCsr->pRec[iHdr], &iType); + + dbdataValue(ctx, iType, &pCsr->pRec[iOff]); + } + break; + } + } + return SQLITE_OK; +} + +/* Return the ROWID for the sqlite_dbdata table */ +static int dbdataRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + *pRowid = pCsr->iRowid; + return SQLITE_OK; +} + + +/* +** Invoke this routine to register the "sqlite_dbdata" virtual table module +*/ +static int sqlite3DbdataRegister(sqlite3 *db){ + static sqlite3_module dbdata_module = { + 0, /* iVersion */ + 0, /* xCreate */ + dbdataConnect, /* xConnect */ + dbdataBestIndex, /* xBestIndex */ + dbdataDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + dbdataOpen, /* xOpen - open a cursor */ + dbdataClose, /* xClose - close a cursor */ + dbdataFilter, /* xFilter - configure scan constraints */ + dbdataNext, /* xNext - advance a cursor */ + dbdataEof, /* xEof - check for end of scan */ + dbdataColumn, /* xColumn - read data */ + dbdataRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ + }; + return sqlite3_create_module(db, "sqlite_dbdata", &dbdata_module, 0); +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_dbdata_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3DbdataRegister(db); +} diff --git a/manifest b/manifest index b6c54ac76e..59bdc64972 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Faster\sand\ssmaller\simplementation\sof\ssqlite3StrICmp(). -D 2019-04-17T11:34:44.568 +C Add\sthe\sexperimental\sdbdata\sextension. +D 2019-04-17T21:17:22.795 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -284,6 +284,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c cec672d40604075bb341a7f11ac48393efdcd90a979269b8fe7977ea62d0547f F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189 F ext/misc/csv.c 7f047aeb68f5802e7ce6639292095d622a488bb43526ed04810e0649faa71ceb +F ext/misc/dbdata.c 436a7883a7f1455c5d2853bd927c5ac31826b21c6e16ca11a21d7262d25ff838 F ext/misc/dbdump.c baf6e37447c9d6968417b1cd34cbedb0b0ab3f91b5329501d8a8d5be3287c336 F ext/misc/eval.c 4b4757592d00fd32e44c7a067e6a0e4839c81a4d57abc4131ee7806d1be3104e F ext/misc/explain.c d5c12962d79913ef774b297006872af1fccda388f61a11d37758f9179a09551f @@ -786,6 +787,7 @@ F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee F test/dataversion1.test 6e5e86ac681f0782e766ebcb56c019ae001522d114e0e111e5ebf68ccf2a7bb8 F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10 +F test/dbdata.test 6e791619d18e0cff2c79392de980ca0594368cdaa05326f043f866beb0c82614 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e F test/dbfuzz001.test e32d14465f1c77712896fda6a1ccc0f037b481c191c1696a9c44f6c9e4964faf F test/dbfuzz2-seed1.db e6225c6f3d7b63f9c5b6867146a5f329d997ab105bee64644dc2b3a2f2aebaee @@ -1818,7 +1820,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 884b4b7e502b4e991677b53971277adfaf0a04a284f8e483e2553d0f83156b50 -R dcc66ec8a55d4cba7e12163fe5fbfbcf -U drh -Z 40a9f6f7fce76c8f72a2cb9229b22090 +P 7ac500fb5abfe1ad60f2ffdcc8fbe5ccc1c641bbeed53f00940e9ff78788e53d +R 9b65ccecf461800383c04a268416aaa8 +T *branch * dbdata +T *sym-dbdata * +T -sym-trunk * +U dan +Z f99e8e6e33bab5d12ccf1d6f1a7cae6f diff --git a/manifest.uuid b/manifest.uuid index c8e27168f5..dc49ef98d8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7ac500fb5abfe1ad60f2ffdcc8fbe5ccc1c641bbeed53f00940e9ff78788e53d \ No newline at end of file +a3ab58832935e1399ecc7e4d8daefa3a6afa6b301792ce7176bc5d7c173510fb \ No newline at end of file diff --git a/test/dbdata.test b/test/dbdata.test new file mode 100644 index 0000000000..c4412a5de9 --- /dev/null +++ b/test/dbdata.test @@ -0,0 +1,54 @@ +# 2019-04-11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the sqlite_dbpage virtual table. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix dbdata + +ifcapable !vtab||!compound { + finish_test + return +} +db enable_load_extension 1 +if { [catch { db eval { SELECT load_extension('../dbdata') } }] } { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE TABLE T1(a, b); + INSERT INTO t1(rowid, a ,b) VALUES(5, 'v', 'five'); + INSERT INTO t1(rowid, a, b) VALUES(10, 'x', 'ten'); +} + +do_execsql_test 1.1 { + SELECT pgno, cell, field, quote(value) FROM sqlite_dbdata WHERE pgno=2; +} { + 2 0 -1 5 + 2 0 0 'v' + 2 0 1 'five' + 2 1 -1 10 + 2 1 0 'x' + 2 1 1 'ten' +} + +set big [string repeat big 2000] +do_execsql_test 1.2 { + INSERT INTO t1 VALUES(NULL, $big); + SELECT value FROM sqlite_dbdata WHERE pgno=2 AND cell=2 AND field=1; +} $big + + + +finish_test From 3b412ac2473707fc7bc4ce29466d7b7dc4988905 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 18 Apr 2019 21:14:11 +0000 Subject: [PATCH 02/38] Add the sqlite_dbptr virtual table to the dbdata extension. For querying the links between b-tree pages. FossilOrigin-Name: 3213a15f2133afbb0a4fec3b8f6e0eeca8c0befafd6658c41074e84f589d5d32 --- ext/misc/dbdata.c | 347 ++++++++++++++++++++++++++++------------------ manifest | 17 +-- manifest.uuid | 2 +- test/dbdata.test | 52 ++++++- 4 files changed, 270 insertions(+), 148 deletions(-) diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c index c0ad8c1f9e..eb2bc01600 100644 --- a/ext/misc/dbdata.c +++ b/ext/misc/dbdata.c @@ -23,6 +23,9 @@ ** schema TEXT HIDDEN ** ); ** +** IMPORTANT: THE VIRTUAL TABLE SCHEMA ABOVE IS SUBJECT TO CHANGE. IN THE +** FUTURE NEW NON-HIDDEN COLUMNS MAY BE ADDED BETWEEN "value" AND "schema". +** ** Each page of the database is inspected. If it cannot be interpreted as a ** b-tree page, or if it is a b-tree page containing 0 entries, the ** sqlite_dbdata table contains no rows for that page. Otherwise, the table @@ -52,6 +55,13 @@ ** ** This module requires that the "sqlite_dbpage" eponymous virtual table be ** available. +** +** +** CREATE TABLE sqlite_dbptr( +** pgno INTEGER, +** child INTEGER, +** schema TEXT HIDDEN +** ); */ #if !defined(SQLITEINT_H) #include "sqlite3ext.h" @@ -67,6 +77,7 @@ SQLITE_EXTENSION_INIT1 typedef struct DbdataTable DbdataTable; typedef struct DbdataCursor DbdataCursor; + /* A cursor for the sqlite_dbdata table */ struct DbdataCursor { sqlite3_vtab_cursor base; /* Base class. Must be first */ @@ -77,19 +88,22 @@ struct DbdataCursor { int nPage; /* Size of aPage[] in bytes */ int nCell; /* Number of cells on aPage[] */ int iCell; /* Current cell number */ + int bOnePage; /* True to stop after one page */ + sqlite3_int64 iRowid; + + /* Only for the sqlite_dbdata table */ u8 *pRec; /* Buffer containing current record */ int nRec; /* Size of pRec[] in bytes */ int nField; /* Number of fields in pRec */ int iField; /* Current field number */ sqlite3_int64 iIntkey; /* Integer key value */ - - sqlite3_int64 iRowid; }; /* The sqlite_dbdata table */ struct DbdataTable { sqlite3_vtab base; /* Base class. Must be first */ sqlite3 *db; /* The database connection */ + int bPtr; /* True for sqlite3_dbptr table */ }; #define DBDATA_COLUMN_PGNO 0 @@ -98,6 +112,10 @@ struct DbdataTable { #define DBDATA_COLUMN_VALUE 3 #define DBDATA_COLUMN_SCHEMA 4 +#define DBPTR_COLUMN_PGNO 0 +#define DBPTR_COLUMN_CHILD 1 +#define DBPTR_COLUMN_SCHEMA 2 + #define DBDATA_SCHEMA \ "CREATE TABLE x(" \ " pgno INTEGER," \ @@ -107,6 +125,13 @@ struct DbdataTable { " schema TEXT HIDDEN" \ ")" +#define DBPTR_SCHEMA \ + "CREATE TABLE x(" \ + " pgno INTEGER," \ + " child INTEGER," \ + " schema TEXT HIDDEN" \ + ")" + /* ** Connect to the sqlite_dbdata virtual table. */ @@ -118,7 +143,7 @@ static int dbdataConnect( char **pzErr ){ DbdataTable *pTab = 0; - int rc = sqlite3_declare_vtab(db, DBDATA_SCHEMA); + int rc = sqlite3_declare_vtab(db, pAux ? DBPTR_SCHEMA : DBDATA_SCHEMA); if( rc==SQLITE_OK ){ pTab = (DbdataTable*)sqlite3_malloc64(sizeof(DbdataTable)); @@ -127,6 +152,7 @@ static int dbdataConnect( }else{ memset(pTab, 0, sizeof(DbdataTable)); pTab->db = db; + pTab->bPtr = (pAux!=0); } } @@ -157,14 +183,16 @@ static int dbdataDisconnect(sqlite3_vtab *pVtab){ ** position 1. */ static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + DbdataTable *pTab = (DbdataTable*)tab; int i; int iSchema = -1; int iPgno = -1; + int colSchema = (pTab->bPtr ? DBPTR_COLUMN_SCHEMA : DBDATA_COLUMN_SCHEMA); for(i=0; inConstraint; i++){ struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i]; if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ - if( p->iColumn==DBDATA_COLUMN_SCHEMA ){ + if( p->iColumn==colSchema ){ if( p->usable==0 ) return SQLITE_CONSTRAINT; iSchema = i; } @@ -210,6 +238,7 @@ static void dbdataResetCursor(DbdataCursor *pCsr){ pCsr->iPgno = 1; pCsr->iCell = 0; pCsr->iField = 0; + pCsr->bOnePage = 0; } /* @@ -281,132 +310,152 @@ static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){ ** Move a dbdata cursor to the next entry in the file. */ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ - DbdataCursor *pCsr = (DbdataCursor *)pCursor; + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; pCsr->iRowid++; while( 1 ){ int rc; + int iOff = (pCsr->iPgno==1 ? 100 : 0); if( pCsr->aPage==0 ){ rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage); - if( rc!=SQLITE_OK ) return rc; - pCsr->iCell = 0; - pCsr->nCell = get_uint16(&pCsr->aPage[pCsr->iPgno==1 ? 103 : 3]); + if( rc!=SQLITE_OK || pCsr->aPage==0 ) return rc; + pCsr->iCell = pTab->bPtr ? -2 : 0; + pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]); } - /* If there is no record loaded, load it now. */ - if( pCsr->pRec==0 ){ - int iOff = (pCsr->iPgno==1 ? 100 : 0); - int bHasRowid = 0; - int nPointer = 0; - sqlite3_int64 nPayload = 0; - sqlite3_int64 nHdr = 0; - int iHdr; - int U, X; - int nLocal; - - switch( pCsr->aPage[iOff] ){ - case 0x02: - nPointer = 4; - break; - case 0x0a: - break; - case 0x0d: - bHasRowid = 1; - break; - default: - pCsr->iCell = pCsr->nCell; - break; + if( pTab->bPtr ){ + if( pCsr->aPage[iOff]!=0x02 && pCsr->aPage[iOff]!=0x05 ){ + pCsr->iCell = pCsr->nCell; } + pCsr->iCell++; if( pCsr->iCell>=pCsr->nCell ){ sqlite3_free(pCsr->aPage); pCsr->aPage = 0; + if( pCsr->bOnePage ) return SQLITE_OK; + pCsr->iPgno++; + }else{ return SQLITE_OK; } + }else{ + /* If there is no record loaded, load it now. */ + if( pCsr->pRec==0 ){ + int bHasRowid = 0; + int nPointer = 0; + sqlite3_int64 nPayload = 0; + sqlite3_int64 nHdr = 0; + int iHdr; + int U, X; + int nLocal; + + switch( pCsr->aPage[iOff] ){ + case 0x02: + nPointer = 4; + break; + case 0x0a: + break; + case 0x0d: + bHasRowid = 1; + break; + default: + /* This is not a b-tree page with records on it. Continue. */ + pCsr->iCell = pCsr->nCell; + break; + } - iOff += 8 + nPointer + pCsr->iCell*2; - iOff = get_uint16(&pCsr->aPage[iOff]); - - /* For an interior node cell, skip past the child-page number */ - iOff += nPointer; - - /* Load the "byte of payload including overflow" field */ - iOff += dbdataGetVarint(&pCsr->aPage[iOff], &nPayload); - - /* If this is a leaf intkey cell, load the rowid */ - if( bHasRowid ){ - iOff += dbdataGetVarint(&pCsr->aPage[iOff], &pCsr->iIntkey); - } - - /* Allocate space for payload */ - pCsr->pRec = (u8*)sqlite3_malloc64(nPayload); - if( pCsr->pRec==0 ) return SQLITE_NOMEM; - pCsr->nRec = nPayload; - - U = pCsr->nPage; - if( bHasRowid ){ - X = U-35; - }else{ - X = ((U-12)*64/255)-23; - } - if( nPayload<=X ){ - nLocal = nPayload; - }else{ - int M, K; - M = ((U-12)*32/255)-23; - K = M+((nPayload-M)%(U-4)); - if( K<=X ){ - nLocal = K; + if( pCsr->iCell>=pCsr->nCell ){ + sqlite3_free(pCsr->aPage); + pCsr->aPage = 0; + if( pCsr->bOnePage ) return SQLITE_OK; + pCsr->iPgno++; + continue; + } + + iOff += 8 + nPointer + pCsr->iCell*2; + iOff = get_uint16(&pCsr->aPage[iOff]); + + /* For an interior node cell, skip past the child-page number */ + iOff += nPointer; + + /* Load the "byte of payload including overflow" field */ + iOff += dbdataGetVarint(&pCsr->aPage[iOff], &nPayload); + + /* If this is a leaf intkey cell, load the rowid */ + if( bHasRowid ){ + iOff += dbdataGetVarint(&pCsr->aPage[iOff], &pCsr->iIntkey); + } + + /* Allocate space for payload */ + pCsr->pRec = (u8*)sqlite3_malloc64(nPayload); + if( pCsr->pRec==0 ) return SQLITE_NOMEM; + pCsr->nRec = nPayload; + + U = pCsr->nPage; + if( bHasRowid ){ + X = U-35; }else{ - nLocal = M; + X = ((U-12)*64/255)-23; } - } - - /* Load the nLocal bytes of payload */ - memcpy(pCsr->pRec, &pCsr->aPage[iOff], nLocal); - iOff += nLocal; - - /* Load content from overflow pages */ - if( nPayload>nLocal ){ - sqlite3_int64 nRem = nPayload - nLocal; - u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); - while( nRem>0 ){ - u8 *aOvfl = 0; - int nOvfl = 0; - int nCopy; - rc = dbdataLoadPage(pCsr, pgnoOvfl, &aOvfl, &nOvfl); - assert( rc!=SQLITE_OK || nOvfl==pCsr->nPage ); - if( rc!=SQLITE_OK ) return rc; - - nCopy = U-4; - if( nCopy>nRem ) nCopy = nRem; - memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy); - nRem -= nCopy; - - sqlite3_free(aOvfl); + if( nPayload<=X ){ + nLocal = nPayload; + }else{ + int M, K; + M = ((U-12)*32/255)-23; + K = M+((nPayload-M)%(U-4)); + if( K<=X ){ + nLocal = K; + }else{ + nLocal = M; + } } + + /* Load the nLocal bytes of payload */ + memcpy(pCsr->pRec, &pCsr->aPage[iOff], nLocal); + iOff += nLocal; + + /* Load content from overflow pages */ + if( nPayload>nLocal ){ + sqlite3_int64 nRem = nPayload - nLocal; + u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); + while( nRem>0 ){ + u8 *aOvfl = 0; + int nOvfl = 0; + int nCopy; + rc = dbdataLoadPage(pCsr, pgnoOvfl, &aOvfl, &nOvfl); + assert( rc!=SQLITE_OK || nOvfl==pCsr->nPage ); + if( rc!=SQLITE_OK ) return rc; + + nCopy = U-4; + if( nCopy>nRem ) nCopy = nRem; + memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy); + nRem -= nCopy; + + sqlite3_free(aOvfl); + } + } + + /* Figure out how many fields in the record */ + pCsr->nField = 0; + iHdr = dbdataGetVarint(pCsr->pRec, &nHdr); + while( iHdrpRec[iHdr], &iDummy); + pCsr->nField++; + } + + pCsr->iField = (bHasRowid ? -2 : -1); } - - /* Figure out how many fields in the record */ - pCsr->nField = 0; - iHdr = dbdataGetVarint(pCsr->pRec, &nHdr); - while( iHdrpRec[iHdr], &iDummy); - pCsr->nField++; - } - - pCsr->iField = (bHasRowid ? -2 : -1); + + pCsr->iField++; + if( pCsr->iFieldnField ) return SQLITE_OK; + + /* Advance to the next cell. The next iteration of the loop will load + ** the record and so on. */ + sqlite3_free(pCsr->pRec); + pCsr->pRec = 0; + pCsr->iCell++; } - - pCsr->iField++; - if( pCsr->iFieldnField ) return SQLITE_OK; - - /* Advance to the next cell. The next iteration of the loop will load - ** the record and so on. */ - sqlite3_free(pCsr->pRec); - pCsr->pRec = 0; - pCsr->iCell++; } assert( !"can't get here" ); @@ -440,6 +489,7 @@ static int dbdataFilter( } if( idxNum & 0x02 ){ pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]); + pCsr->bOnePage = 1; } rc = sqlite3_prepare_v2(pTab->db, @@ -533,34 +583,54 @@ static int dbdataColumn( int i ){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; - switch( i ){ - case DBDATA_COLUMN_PGNO: - sqlite3_result_int64(ctx, pCsr->iPgno); - break; - case DBDATA_COLUMN_CELL: - sqlite3_result_int(ctx, pCsr->iCell); - break; - case DBDATA_COLUMN_FIELD: - sqlite3_result_int(ctx, pCsr->iField); - break; - case DBDATA_COLUMN_VALUE: { - if( pCsr->iField<0 ){ - sqlite3_result_int64(ctx, pCsr->iIntkey); - }else{ - int iHdr; - sqlite3_int64 iType; - sqlite3_int64 iOff; - int i; - iHdr = dbdataGetVarint(pCsr->pRec, &iOff); - for(i=0; iiField; i++){ - iHdr += dbdataGetVarint(&pCsr->pRec[iHdr], &iType); - iOff += dbdataValueBytes(iType); + DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; + if( pTab->bPtr ){ + switch( i ){ + case DBPTR_COLUMN_PGNO: + sqlite3_result_int64(ctx, pCsr->iPgno); + break; + case DBPTR_COLUMN_CHILD: { + int iOff = pCsr->iPgno==1 ? 100 : 0; + if( pCsr->iCell<0 ){ + iOff += 8; + }else{ + iOff += 12 + pCsr->iCell*2; + iOff = get_uint16(&pCsr->aPage[iOff]); } - dbdataGetVarint(&pCsr->pRec[iHdr], &iType); - - dbdataValue(ctx, iType, &pCsr->pRec[iOff]); + sqlite3_result_int64(ctx, get_uint32(&pCsr->aPage[iOff])); + break; + } + } + }else{ + switch( i ){ + case DBDATA_COLUMN_PGNO: + sqlite3_result_int64(ctx, pCsr->iPgno); + break; + case DBDATA_COLUMN_CELL: + sqlite3_result_int(ctx, pCsr->iCell); + break; + case DBDATA_COLUMN_FIELD: + sqlite3_result_int(ctx, pCsr->iField); + break; + case DBDATA_COLUMN_VALUE: { + if( pCsr->iField<0 ){ + sqlite3_result_int64(ctx, pCsr->iIntkey); + }else{ + int iHdr; + sqlite3_int64 iType; + sqlite3_int64 iOff; + int i; + iHdr = dbdataGetVarint(pCsr->pRec, &iOff); + for(i=0; iiField; i++){ + iHdr += dbdataGetVarint(&pCsr->pRec[iHdr], &iType); + iOff += dbdataValueBytes(iType); + } + dbdataGetVarint(&pCsr->pRec[iHdr], &iType); + + dbdataValue(ctx, iType, &pCsr->pRec[iOff]); + } + break; } - break; } } return SQLITE_OK; @@ -604,7 +674,12 @@ static int sqlite3DbdataRegister(sqlite3 *db){ 0, /* xRollbackTo */ 0 /* xShadowName */ }; - return sqlite3_create_module(db, "sqlite_dbdata", &dbdata_module, 0); + + int rc = sqlite3_create_module(db, "sqlite_dbdata", &dbdata_module, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "sqlite_dbptr", &dbdata_module, (void*)1); + } + return rc; } #ifdef _WIN32 diff --git a/manifest b/manifest index 59bdc64972..d6a2b045e8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sexperimental\sdbdata\sextension. -D 2019-04-17T21:17:22.795 +C Add\sthe\ssqlite_dbptr\svirtual\stable\sto\sthe\sdbdata\sextension.\sFor\squerying\sthe\slinks\sbetween\sb-tree\spages. +D 2019-04-18T21:14:11.260 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -284,7 +284,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c cec672d40604075bb341a7f11ac48393efdcd90a979269b8fe7977ea62d0547f F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189 F ext/misc/csv.c 7f047aeb68f5802e7ce6639292095d622a488bb43526ed04810e0649faa71ceb -F ext/misc/dbdata.c 436a7883a7f1455c5d2853bd927c5ac31826b21c6e16ca11a21d7262d25ff838 +F ext/misc/dbdata.c 20d85d7d7503817a5f9ac7d93c4ae7bf5fcc40570a77adc417acab0500058c99 F ext/misc/dbdump.c baf6e37447c9d6968417b1cd34cbedb0b0ab3f91b5329501d8a8d5be3287c336 F ext/misc/eval.c 4b4757592d00fd32e44c7a067e6a0e4839c81a4d57abc4131ee7806d1be3104e F ext/misc/explain.c d5c12962d79913ef774b297006872af1fccda388f61a11d37758f9179a09551f @@ -787,7 +787,7 @@ F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee F test/dataversion1.test 6e5e86ac681f0782e766ebcb56c019ae001522d114e0e111e5ebf68ccf2a7bb8 F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10 -F test/dbdata.test 6e791619d18e0cff2c79392de980ca0594368cdaa05326f043f866beb0c82614 +F test/dbdata.test 573fa3347744863e47d011e4e8e9b87c6795ba92a759bf5d5c68da975900ddd4 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e F test/dbfuzz001.test e32d14465f1c77712896fda6a1ccc0f037b481c191c1696a9c44f6c9e4964faf F test/dbfuzz2-seed1.db e6225c6f3d7b63f9c5b6867146a5f329d997ab105bee64644dc2b3a2f2aebaee @@ -1820,10 +1820,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7ac500fb5abfe1ad60f2ffdcc8fbe5ccc1c641bbeed53f00940e9ff78788e53d -R 9b65ccecf461800383c04a268416aaa8 -T *branch * dbdata -T *sym-dbdata * -T -sym-trunk * +P a3ab58832935e1399ecc7e4d8daefa3a6afa6b301792ce7176bc5d7c173510fb +R 901e9e476021499846a29c847f49c578 U dan -Z f99e8e6e33bab5d12ccf1d6f1a7cae6f +Z e0c2ac18f5bd53328c7a0485e8c7ef97 diff --git a/manifest.uuid b/manifest.uuid index dc49ef98d8..6ee3f97f4e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a3ab58832935e1399ecc7e4d8daefa3a6afa6b301792ce7176bc5d7c173510fb \ No newline at end of file +3213a15f2133afbb0a4fec3b8f6e0eeca8c0befafd6658c41074e84f589d5d32 \ No newline at end of file diff --git a/test/dbdata.test b/test/dbdata.test index c4412a5de9..5a0df5dc86 100644 --- a/test/dbdata.test +++ b/test/dbdata.test @@ -43,12 +43,62 @@ do_execsql_test 1.1 { 2 1 1 'ten' } -set big [string repeat big 2000] +breakpoint do_execsql_test 1.2 { + SELECT pgno, cell, field, quote(value) FROM sqlite_dbdata; +} { + 1 0 -1 1 + 1 0 0 'table' + 1 0 1 'T1' + 1 0 2 'T1' + 1 0 3 2 + 1 0 4 {'CREATE TABLE T1(a, b)'} + 2 0 -1 5 + 2 0 0 'v' + 2 0 1 'five' + 2 1 -1 10 + 2 1 0 'x' + 2 1 1 'ten' +} + +set big [string repeat big 2000] +do_execsql_test 1.3 { INSERT INTO t1 VALUES(NULL, $big); SELECT value FROM sqlite_dbdata WHERE pgno=2 AND cell=2 AND field=1; } $big +#------------------------------------------------------------------------- +reset_db +db enable_load_extension 1 +db eval { SELECT load_extension('../dbdata') } + +do_execsql_test 2.0 { + CREATE TABLE t1(a); + CREATE INDEX i1 ON t1(a); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10 + ) + INSERT INTO t1 SELECT randomblob(900) FROM s; +} + +do_execsql_test 2.1 { + SELECT * FROM sqlite_dbptr WHERE pgno=2; +} { + 2 25 2 6 2 7 2 9 2 11 2 13 2 15 2 17 2 19 2 21 +} + +do_execsql_test 2.2 { + SELECT * FROM sqlite_dbptr WHERE pgno=3; +} { + 3 24 3 23 +} + +do_execsql_test 2.3 { + SELECT * FROM sqlite_dbptr +} { + 2 25 2 6 2 7 2 9 2 11 2 13 2 15 2 17 2 19 2 21 + 3 24 3 23 +} finish_test From 68cb86ef23e3bc9e6362fec4937968a65ec025b5 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 20 Apr 2019 20:57:28 +0000 Subject: [PATCH 03/38] Add the ".recovery" command to the shell tool. For recovering the maximum amount data from corrupt databases. Still needs work. FossilOrigin-Name: 7461d2e120f2149315ddac2676d51d7445bcdb8e97543effd9c30603517ef9da --- ext/misc/dbdata.c | 194 ++++++++++++++++---------------- main.mk | 1 + manifest | 18 +-- manifest.uuid | 2 +- src/shell.c.in | 277 +++++++++++++++++++++++++++++++++++++++++++--- tool/mkshellc.tcl | 8 +- 6 files changed, 375 insertions(+), 125 deletions(-) diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c index eb2bc01600..aae862fc7f 100644 --- a/ext/misc/dbdata.c +++ b/ext/misc/dbdata.c @@ -63,11 +63,11 @@ ** schema TEXT HIDDEN ** ); */ -#if !defined(SQLITEINT_H) +#if !defined(SQLITEINT_H) #include "sqlite3ext.h" typedef unsigned char u8; -typedef unsigned int u32; +typedef unsigned long u32; #endif SQLITE_EXTENSION_INIT1 @@ -94,8 +94,11 @@ struct DbdataCursor { /* Only for the sqlite_dbdata table */ u8 *pRec; /* Buffer containing current record */ int nRec; /* Size of pRec[] in bytes */ - int nField; /* Number of fields in pRec */ + int nHdr; /* Size of header in bytes */ int iField; /* Current field number */ + u8 *pHdrPtr; + u8 *pPtr; + sqlite3_int64 iIntkey; /* Integer key value */ }; @@ -306,6 +309,78 @@ static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){ return 9; } +static int dbdataValueBytes(int eType){ + switch( eType ){ + case 0: case 8: case 9: + case 10: case 11: + return 0; + case 1: + return 1; + case 2: + return 2; + case 3: + return 3; + case 4: + return 4; + case 5: + return 6; + case 6: + case 7: + return 8; + default: + return ((eType-12) / 2); + } +} + +static void dbdataValue(sqlite3_context *pCtx, int eType, u8 *pData){ + switch( eType ){ + case 0: + case 10: + case 11: + sqlite3_result_null(pCtx); + break; + + case 8: + sqlite3_result_int(pCtx, 0); + break; + case 9: + sqlite3_result_int(pCtx, 1); + break; + + case 1: case 2: case 3: case 4: case 5: case 6: case 7: { + sqlite3_uint64 v = (signed char)pData[0]; + pData++; + switch( eType ){ + case 7: + case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 4: v = (v<<8) + pData[0]; pData++; + case 3: v = (v<<8) + pData[0]; pData++; + case 2: v = (v<<8) + pData[0]; pData++; + } + + if( eType==7 ){ + double r; + memcpy(&r, &v, sizeof(r)); + sqlite3_result_double(pCtx, r); + }else{ + sqlite3_result_int64(pCtx, (sqlite3_int64)v); + } + break; + } + + default: { + int n = ((eType-12) / 2); + if( eType % 2 ){ + sqlite3_result_text(pCtx, (const char*)pData, n, SQLITE_TRANSIENT); + }else{ + sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); + } + } + } +} + + /* ** Move a dbdata cursor to the next entry in the file. */ @@ -435,20 +510,23 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ } } - /* Figure out how many fields in the record */ - pCsr->nField = 0; iHdr = dbdataGetVarint(pCsr->pRec, &nHdr); - while( iHdrpRec[iHdr], &iDummy); - pCsr->nField++; + pCsr->nHdr = nHdr; + pCsr->pHdrPtr = &pCsr->pRec[iHdr]; + pCsr->pPtr = &pCsr->pRec[pCsr->nHdr]; + pCsr->iField = (bHasRowid ? -1 : 0); + }else{ + pCsr->iField++; + if( pCsr->iField>0 ){ + sqlite3_int64 iType; + pCsr->pHdrPtr += dbdataGetVarint(pCsr->pHdrPtr, &iType); + pCsr->pPtr += dbdataValueBytes(iType); } - - pCsr->iField = (bHasRowid ? -2 : -1); } - - pCsr->iField++; - if( pCsr->iFieldnField ) return SQLITE_OK; + + if( pCsr->iField<0 || pCsr->pHdrPtr<&pCsr->pRec[pCsr->nHdr] ){ + return SQLITE_OK; + } /* Advance to the next cell. The next iteration of the loop will load ** the record and so on. */ @@ -485,7 +563,7 @@ static int dbdataFilter( dbdataResetCursor(pCsr); assert( pCsr->iPgno==1 ); if( idxNum & 0x01 ){ - zSchema = sqlite3_value_text(argv[0]); + zSchema = (const char*)sqlite3_value_text(argv[0]); } if( idxNum & 0x02 ){ pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]); @@ -498,6 +576,8 @@ static int dbdataFilter( ); if( rc==SQLITE_OK ){ rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT); + }else{ + pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); } if( rc==SQLITE_OK ){ rc = dbdataNext(pCursor); @@ -505,77 +585,6 @@ static int dbdataFilter( return rc; } -static int dbdataValueBytes(int eType){ - switch( eType ){ - case 0: case 8: case 9: - case 10: case 11: - return 0; - case 1: - return 1; - case 2: - return 2; - case 3: - return 3; - case 4: - return 4; - case 5: - return 6; - case 6: - case 7: - return 8; - default: - return ((eType-12) / 2); - } -} - -static void dbdataValue(sqlite3_context *pCtx, int eType, u8 *pData){ - switch( eType ){ - case 0: - case 10: - case 11: - sqlite3_result_null(pCtx); - break; - - case 8: - sqlite3_result_int(pCtx, 0); - break; - case 9: - sqlite3_result_int(pCtx, 1); - break; - - case 1: case 2: case 3: case 4: case 5: case 6: case 7: { - sqlite3_uint64 v = (signed char)pData[0]; - pData++; - switch( eType ){ - case 7: - case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; - case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; - case 4: v = (v<<8) + pData[0]; pData++; - case 3: v = (v<<8) + pData[0]; pData++; - case 2: v = (v<<8) + pData[0]; pData++; - } - - if( eType==7 ){ - double r; - memcpy(&r, &v, sizeof(r)); - sqlite3_result_double(pCtx, r); - }else{ - sqlite3_result_int64(pCtx, (sqlite3_int64)v); - } - break; - } - - default: { - int n = ((eType-12) / 2); - if( eType % 2 ){ - sqlite3_result_text(pCtx, pData, n, SQLITE_TRANSIENT); - }else{ - sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); - } - } - } -} - /* Return a column for the sqlite_dbdata table */ static int dbdataColumn( sqlite3_vtab_cursor *pCursor, @@ -616,18 +625,9 @@ static int dbdataColumn( if( pCsr->iField<0 ){ sqlite3_result_int64(ctx, pCsr->iIntkey); }else{ - int iHdr; sqlite3_int64 iType; - sqlite3_int64 iOff; - int i; - iHdr = dbdataGetVarint(pCsr->pRec, &iOff); - for(i=0; iiField; i++){ - iHdr += dbdataGetVarint(&pCsr->pRec[iHdr], &iType); - iOff += dbdataValueBytes(iType); - } - dbdataGetVarint(&pCsr->pRec[iHdr], &iType); - - dbdataValue(ctx, iType, &pCsr->pRec[iOff]); + dbdataGetVarint(pCsr->pHdrPtr, &iType); + dbdataValue(ctx, iType, pCsr->pPtr); } break; } diff --git a/main.mk b/main.mk index f418eec68a..508554d733 100644 --- a/main.mk +++ b/main.mk @@ -738,6 +738,7 @@ SHELL_SRC = \ $(TOP)/ext/expert/sqlite3expert.h \ $(TOP)/ext/misc/zipfile.c \ $(TOP)/ext/misc/memtrace.c \ + $(TOP)/ext/misc/dbdata.c \ $(TOP)/src/test_windirent.c shell.c: $(SHELL_SRC) $(TOP)/tool/mkshellc.tcl diff --git a/manifest b/manifest index d6a2b045e8..c103da1624 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssqlite_dbptr\svirtual\stable\sto\sthe\sdbdata\sextension.\sFor\squerying\sthe\slinks\sbetween\sb-tree\spages. -D 2019-04-18T21:14:11.260 +C Add\sthe\s".recovery"\scommand\sto\sthe\sshell\stool.\sFor\srecovering\sthe\smaximum\samount\sdata\sfrom\scorrupt\sdatabases.\sStill\sneeds\swork. +D 2019-04-20T20:57:28.136 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -284,7 +284,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c cec672d40604075bb341a7f11ac48393efdcd90a979269b8fe7977ea62d0547f F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189 F ext/misc/csv.c 7f047aeb68f5802e7ce6639292095d622a488bb43526ed04810e0649faa71ceb -F ext/misc/dbdata.c 20d85d7d7503817a5f9ac7d93c4ae7bf5fcc40570a77adc417acab0500058c99 +F ext/misc/dbdata.c 8f74f25565e1f57942f75134ef94ab73058849bed87d92be619be7d3b8a7d054 F ext/misc/dbdump.c baf6e37447c9d6968417b1cd34cbedb0b0ab3f91b5329501d8a8d5be3287c336 F ext/misc/eval.c 4b4757592d00fd32e44c7a067e6a0e4839c81a4d57abc4131ee7806d1be3104e F ext/misc/explain.c d5c12962d79913ef774b297006872af1fccda388f61a11d37758f9179a09551f @@ -441,7 +441,7 @@ F ext/userauth/userauth.c f81aa5a3ecacf406f170c62a144405858f6f6de51dbdc0920134e6 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 23d3660f7053d196aef76938bf78b10fc3ce1831a85d96bd71565758788f34d4 +F main.mk 125adda36bb32c99dc3a11340bd029ef373b9523eac2b2af76087bfe82d4fdf8 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 9263f5c30dd44c7ac2eb29f40a7ec64322a96885b71c00de6bc30b756c2e1c49 -F src/shell.c.in c1986496062f9dba4ed5b70db06b5e0f32e1954cdcfab0b30372c6c186796810 +F src/shell.c.in 3646e448cc207fa3118266c2a23f177306b068e42aae6fee64323ffb13307680 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1754,7 +1754,7 @@ F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl 352a4319c0ad869eb26442bf7c3b015aa15594c21f1cce5a6420dbe999367c21 F tool/mkopts.tcl 680f785fdb09729fd9ac50632413da4eadbdf9071535e3f26d03795828ab07fa F tool/mkpragmatab.tcl 49039adedafbc430d2959400da2e0e8f20ef8dcf6898e447c946e7d50ef5906b -F tool/mkshellc.tcl 1f45770aea226ac093a9c72f718efbb88a2a2833409ec2e1c4cecae4202626f5 +F tool/mkshellc.tcl d18a6769f766bd5a0fd5fd60e6bbde38ed6d1a41fced5fe32ac01ab9d2e0d5ec F tool/mksourceid.c d458f9004c837bee87a6382228ac20d3eae3c49ea3b0a5aace936f8b60748d3b F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 4f7cfef5152b0c91920355cbfc1d608a4ad242cb819f1aea07f6d0274f584a7f @@ -1820,7 +1820,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a3ab58832935e1399ecc7e4d8daefa3a6afa6b301792ce7176bc5d7c173510fb -R 901e9e476021499846a29c847f49c578 +P 3213a15f2133afbb0a4fec3b8f6e0eeca8c0befafd6658c41074e84f589d5d32 +R 9a7e323fe9ebfe9eb0759e1c70136795 U dan -Z e0c2ac18f5bd53328c7a0485e8c7ef97 +Z 100e3ccaddf1d19d0bacd74d773a93b7 diff --git a/manifest.uuid b/manifest.uuid index 6ee3f97f4e..0b854fa40a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3213a15f2133afbb0a4fec3b8f6e0eeca8c0befafd6658c41074e84f589d5d32 \ No newline at end of file +7461d2e120f2149315ddac2676d51d7445bcdb8e97543effd9c30603517ef9da \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index d8c57b481e..c38bfb64b9 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -948,6 +948,8 @@ INCLUDE ../ext/misc/sqlar.c INCLUDE ../ext/expert/sqlite3expert.h INCLUDE ../ext/expert/sqlite3expert.c +INCLUDE ../ext/misc/dbdata.c + #if defined(SQLITE_ENABLE_SESSION) /* ** State information for a single open session @@ -1102,6 +1104,7 @@ struct ShellState { #define SHFLG_Newlines 0x00000010 /* .dump --newline flag */ #define SHFLG_CountChanges 0x00000020 /* .changes setting */ #define SHFLG_Echo 0x00000040 /* .echo or --echo setting */ +#define SHFLG_Recover 0x00000080 /* .dump is --recover */ /* ** Macros for testing and setting shellFlgs @@ -3999,6 +4002,7 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_fileio_init(p->db, 0, 0); sqlite3_shathree_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); + sqlite3_dbdata_init(p->db, 0, 0); #ifdef SQLITE_HAVE_ZLIB sqlite3_zipfile_init(p->db, 0, 0); sqlite3_sqlar_init(p->db, 0, 0); @@ -6026,6 +6030,238 @@ end_ar_command: **********************************************************************************/ #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */ +static void shellExec(sqlite3 *db, int *pRc, const char *zSql){ + int rc = *pRc; + if( rc==SQLITE_OK ){ + char *zErr = 0; + rc = sqlite3_exec(db, zSql, 0, 0, &zErr); + if( rc!=SQLITE_OK ){ + raw_printf(stderr, "SQL error: %s\n", zErr); + } + *pRc = rc; + } +} + +static void *shellMalloc(int *pRc, sqlite3_int64 nByte){ + void *pRet = 0; + if( *pRc==SQLITE_OK ){ + pRet = sqlite3_malloc64(nByte); + if( pRet==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, nByte); + } + } + return pRet; +} + +static char *shellMPrintf(int *pRc, const char *zFmt, ...){ + char *z = 0; + if( *pRc==SQLITE_OK ){ + va_list ap; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( z==0 ){ + *pRc = SQLITE_NOMEM; + } + } + return z; +} + +typedef struct RecoverTable RecoverTable; +struct RecoverTable { + char *zName; /* Name of table */ + char *zQuoted; /* Quoted version of zName */ + char *zCreate; /* SQL to create table in default schema */ + int nCol; /* Number of columns in table */ + char **azlCol; /* Array of column lists */ +}; + +/* +** Free a RecoverTable object allocated by recoverNewTable() +*/ +static void recoverFreeTable(RecoverTable *pTab){ + if( pTab ){ + sqlite3_free(pTab->zName); + sqlite3_free(pTab->zQuoted); + sqlite3_free(pTab->zCreate); + if( pTab->azlCol ){ + int i; + for(i=0; inCol; i++){ + sqlite3_free(pTab->azlCol[i]); + } + sqlite3_free(pTab->azlCol); + } + sqlite3_free(pTab); + } +} + +static RecoverTable *recoverNewTable( + ShellState *pState, + int *pRc, + int iRoot, + int nCol +){ + RecoverTable *pRet = 0; + + pRet = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable)); + if( pRet ){ + sqlite3_stmt *pStmt = 0; + pRet->zName = shellMPrintf(pRc, "orphan_%d_%d", nCol, iRoot); + pRet->zQuoted = shellMPrintf(pRc, "%Q", pRet->zName); + pRet->azlCol = (char**)shellMalloc(pRc, sizeof(char*) * nCol); + pRet->nCol = nCol; + + shellPreparePrintf(pState->db, pRc, &pStmt, + "WITH s(i) AS (" + " SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<%d" + ")" + "SELECT i-1, group_concat('c' || i, ', ') OVER (ORDER BY i) FROM s", + nCol + ); + while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + int idx = sqlite3_column_int(pStmt, 0); + const char *zText = (const char*)sqlite3_column_text(pStmt, 1); + pRet->azlCol[idx] = shellMPrintf(pRc, "%s", zText); + } + shellFinalize(pRc, pStmt); + + pRet->zCreate = shellMPrintf(pRc, "CREATE TABLE %Q (id, %s)", + pRet->zName, pRet->azlCol[nCol-1] + ); + } + + if( *pRc!=SQLITE_OK ){ + recoverFreeTable(pRet); + pRet = 0; + } + + return pRet; +} + +/* +** This function is called to recover data from the database. A script +** to construct a new database containing all recovered data is output +** on stream pState->out. +*/ +static int recoverDatabaseCmd(ShellState *pState){ + const char *zSql; + int rc = SQLITE_OK; + sqlite3_stmt *pLoop = 0; /* Loop through all root pages */ + + shellExec(pState->db, &rc, + /* Attach an in-memory database named 'recovery'. Create an indexed + ** cache of the sqlite_dbptr virtual table. */ + "ATTACH '' AS recovery;" + "CREATE TABLE recovery.dbptr(" + " pgno, child, PRIMARY KEY(child, pgno)" + ") WITHOUT ROWID;" + "INSERT OR IGNORE INTO dbptr(pgno, child) SELECT * FROM sqlite_dbptr;" + + /* Delete any pointer to page 1. This ensures that page 1 is considered + ** a root page, regardless of how corrupt the db is. */ + "DELETE FROM recovery.dbptr WHERE child = 1;" + + /* Delete all pointers to any pages that have more than one pointer + ** to them. Such pages will be treated as root pages when recovering + ** data. */ + "DELETE FROM recovery.dbptr WHERE child IN (" + " SELECT child FROM recovery.dbptr GROUP BY child HAVING count(*)>1" + ");" + + /* Create the "map" table that will (eventually) contain instructions + ** for dealing with each page in the db that contains one or more + ** records. */ + "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, maxlen INT, root INT);" + + /* Populate table [map]. If there are circular loops of pages in the + ** database, the following adds all pages in such a loop to the map + ** as individual root pages. This could be handled better. */ + "WITH pages(i, maxlen) AS (" + " SELECT page_count, max(field+1) " + " FROM pragma_page_count, sqlite_dbdata WHERE pgno=page_count" + " UNION ALL" + " SELECT * FROM (SELECT i-1, max(field+1)" + " FROM pages, sqlite_dbdata WHERE pgno=i-1 AND i>=2)" + ")" + "INSERT INTO recovery.map(pgno, maxlen, root) SELECT i, maxlen, (" + " WITH p(orig, pgno, parent) AS (" + " SELECT 0, i, (SELECT pgno FROM recovery.dbptr WHERE child=i)" + " UNION ALL" + " SELECT i, p.parent, " + " (SELECT pgno FROM recovery.dbptr WHERE child=p.parent) FROM p" + " )" + " SELECT pgno FROM p WHERE (parent IS NULL OR pgno = orig)" + ") " + "FROM pages WHERE maxlen > 0;" + + /* Extract data from page 1 and any linked pages into table + ** recovery.schema. With the same schema as an sqlite_master table. */ + "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" + "INSERT INTO recovery.schema SELECT " + " max(CASE WHEN field=0 THEN value ELSE NULL END)," + " max(CASE WHEN field=1 THEN value ELSE NULL END)," + " max(CASE WHEN field=2 THEN value ELSE NULL END)," + " max(CASE WHEN field=3 THEN value ELSE NULL END)," + " max(CASE WHEN field=4 THEN value ELSE NULL END)" + "FROM sqlite_dbdata WHERE pgno IN (" + " SELECT pgno FROM recovery.map WHERE root=1" + ")" + "GROUP BY pgno, cell;" + ); + +#if 0 + zSql = "SELECT type ||','|| name ||','|| tbl_name ||','|| rootpage ||','|| sql FROM recovery.schema;"; + shellPrepare(pState->db, &rc, zSql, &pLoop); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ + raw_printf(pState->out, "%s\n", (const char*)sqlite3_column_text(pLoop, 0)); + } + shellFinalize(&rc, pLoop); + return rc; +#endif + + /* Loop through each root page. */ + zSql = "SELECT root,max(maxlen) FROM recovery.map WHERE root>1 GROUP BY root"; + shellPrepare(pState->db, &rc, zSql, &pLoop); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ + int iRoot = sqlite3_column_int(pLoop, 0); + int nCol = sqlite3_column_int(pLoop, 1); + RecoverTable *pTab; + + pTab = recoverNewTable(pState, &rc, iRoot, nCol); + if( pTab ){ + sqlite3_stmt *pData = 0; + raw_printf(pState->out, "%s;\n", pTab->zCreate); + shellPreparePrintf(pState->db, &rc, &pData, + "SELECT max(field), group_concat(quote(value), ', ') " + "FROM sqlite_dbdata WHERE pgno IN (" + " SELECT pgno FROM recovery.map WHERE root=%d" + ")" + "GROUP BY pgno, cell;", iRoot + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pData) ){ + int iMax = sqlite3_column_int(pData, 0); + const char *zVal = (const char*)sqlite3_column_text(pData, 1); + if( iMax+1==pTab->nCol ){ + raw_printf(pState->out, "INSERT INTO %s VALUES( %s );\n", + pTab->zQuoted, zVal); + }else{ + raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", + pTab->zQuoted, pTab->azlCol[iMax], zVal + ); + } + } + shellFinalize(&rc, pData); + } + recoverFreeTable(pTab); + } + shellFinalize(&rc, pLoop); + + sqlite3_exec(pState->db, "DETACH recovery", 0, 0, 0); + return rc; +} + /* ** If an input line begins with "." then invoke this routine to @@ -6313,6 +6549,11 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = shell_dbinfo_command(p, nArg, azArg); }else + if( c=='r' && strncmp(azArg[0], "recover", n)==0 ){ + open_db(p, 0); + rc = recoverDatabaseCmd(p); + }else + if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){ const char *zLike = 0; int i; @@ -6350,7 +6591,9 @@ static int do_meta_command(char *zLine, ShellState *p){ zLike = azArg[i]; } } + open_db(p, 0); + /* When playing back a "dump", the content might appear in an order ** which causes immediate foreign key constraints to be violated. ** So disable foreign-key constraint enforcement to prevent problems. */ @@ -6365,30 +6608,30 @@ static int do_meta_command(char *zLine, ShellState *p){ p->nErr = 0; if( zLike==0 ){ run_schema_dump_query(p, - "SELECT name, type, sql FROM sqlite_master " - "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'" - ); + "SELECT name, type, sql FROM sqlite_master " + "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'" + ); run_schema_dump_query(p, - "SELECT name, type, sql FROM sqlite_master " - "WHERE name=='sqlite_sequence'" - ); + "SELECT name, type, sql FROM sqlite_master " + "WHERE name=='sqlite_sequence'" + ); run_table_dump_query(p, - "SELECT sql FROM sqlite_master " - "WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0 - ); + "SELECT sql FROM sqlite_master " + "WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0 + ); }else{ char *zSql; zSql = sqlite3_mprintf( - "SELECT name, type, sql FROM sqlite_master " - "WHERE tbl_name LIKE %Q AND type=='table'" - " AND sql NOT NULL", zLike); + "SELECT name, type, sql FROM sqlite_master " + "WHERE tbl_name LIKE %Q AND type=='table'" + " AND sql NOT NULL", zLike); run_schema_dump_query(p,zSql); sqlite3_free(zSql); zSql = sqlite3_mprintf( - "SELECT sql FROM sqlite_master " - "WHERE sql NOT NULL" - " AND type IN ('index','trigger','view')" - " AND tbl_name LIKE %Q", zLike); + "SELECT sql FROM sqlite_master " + "WHERE sql NOT NULL" + " AND type IN ('index','trigger','view')" + " AND tbl_name LIKE %Q", zLike); run_table_dump_query(p, zSql, 0); sqlite3_free(zSql); } @@ -6398,7 +6641,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0); sqlite3_exec(p->db, "RELEASE dump;", 0, 0, 0); - raw_printf(p->out, p->nErr ? "ROLLBACK; -- due to errors\n" : "COMMIT;\n"); + raw_printf(p->out, p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n"); p->showHeader = savedShowHeader; p->shellFlgs = savedShellFlags; }else diff --git a/tool/mkshellc.tcl b/tool/mkshellc.tcl index 534ac6156a..73eacdb9da 100644 --- a/tool/mkshellc.tcl +++ b/tool/mkshellc.tcl @@ -40,16 +40,21 @@ proc omit_redundant_typedefs {line} { } return $line } +set iLine 0 while {1} { set lx [omit_redundant_typedefs [gets $in]] if {[eof $in]} break; + incr iLine if {[regexp {^INCLUDE } $lx]} { set cfile [lindex $lx 1] puts $out "/************************* Begin $cfile ******************/" + puts $out "#line 1 \"$cfile\"" set in2 [open $topdir/src/$cfile rb] while {![eof $in2]} { set lx [omit_redundant_typedefs [gets $in2]] - if {[regexp {^#include "sqlite} $lx]} continue + if {[regexp {^#include "sqlite} $lx]} { + set lx "/* $lx */" + } if {[regexp {^# *include "test_windirent.h"} $lx]} { set lx "/* $lx */" } @@ -58,6 +63,7 @@ while {1} { } close $in2 puts $out "/************************* End $cfile ********************/" + puts $out "#line [expr $iLine+1] \"shell.c.in\"" continue } puts $out $lx From b40af49d735f7ea8111bf37dab5c3f041e549290 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 22 Apr 2019 20:52:12 +0000 Subject: [PATCH 04/38] Enhance the ".recover" command. Fix a problem with overflow pages in dbdata.c. FossilOrigin-Name: f193ca587f9e4f925f4f2343b0b07053bd6f93dd87fc6f8f41cf4479e90cf562 --- ext/misc/dbdata.c | 1 + manifest | 16 ++-- manifest.uuid | 2 +- src/shell.c.in | 207 ++++++++++++++++++++++++++++++++++++++++------ test/dbdata.test | 10 +++ 5 files changed, 201 insertions(+), 35 deletions(-) diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c index aae862fc7f..7662417ae5 100644 --- a/ext/misc/dbdata.c +++ b/ext/misc/dbdata.c @@ -506,6 +506,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy); nRem -= nCopy; + pgnoOvfl = get_uint32(aOvfl); sqlite3_free(aOvfl); } } diff --git a/manifest b/manifest index c103da1624..b830ce2b1e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s".recovery"\scommand\sto\sthe\sshell\stool.\sFor\srecovering\sthe\smaximum\samount\sdata\sfrom\scorrupt\sdatabases.\sStill\sneeds\swork. -D 2019-04-20T20:57:28.136 +C Enhance\sthe\s".recover"\scommand.\sFix\sa\sproblem\swith\soverflow\spages\sin\sdbdata.c. +D 2019-04-22T20:52:12.850 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -284,7 +284,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c cec672d40604075bb341a7f11ac48393efdcd90a979269b8fe7977ea62d0547f F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189 F ext/misc/csv.c 7f047aeb68f5802e7ce6639292095d622a488bb43526ed04810e0649faa71ceb -F ext/misc/dbdata.c 8f74f25565e1f57942f75134ef94ab73058849bed87d92be619be7d3b8a7d054 +F ext/misc/dbdata.c 6a0ccc33e8c5ef3b6164eec7839cb4dff44e27bee0717f1660dbb8ee77ee4cf8 F ext/misc/dbdump.c baf6e37447c9d6968417b1cd34cbedb0b0ab3f91b5329501d8a8d5be3287c336 F ext/misc/eval.c 4b4757592d00fd32e44c7a067e6a0e4839c81a4d57abc4131ee7806d1be3104e F ext/misc/explain.c d5c12962d79913ef774b297006872af1fccda388f61a11d37758f9179a09551f @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 9263f5c30dd44c7ac2eb29f40a7ec64322a96885b71c00de6bc30b756c2e1c49 -F src/shell.c.in 3646e448cc207fa3118266c2a23f177306b068e42aae6fee64323ffb13307680 +F src/shell.c.in 7dd0babc42bc136b8dd070152981ee2325b787571f58326e68e6ad35d764aa33 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -787,7 +787,7 @@ F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee F test/dataversion1.test 6e5e86ac681f0782e766ebcb56c019ae001522d114e0e111e5ebf68ccf2a7bb8 F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10 -F test/dbdata.test 573fa3347744863e47d011e4e8e9b87c6795ba92a759bf5d5c68da975900ddd4 +F test/dbdata.test c8d97bafd1b2efb1e445871c4641208dcd91e686d2dfbb6463d83934adbd1ac5 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e F test/dbfuzz001.test e32d14465f1c77712896fda6a1ccc0f037b481c191c1696a9c44f6c9e4964faf F test/dbfuzz2-seed1.db e6225c6f3d7b63f9c5b6867146a5f329d997ab105bee64644dc2b3a2f2aebaee @@ -1820,7 +1820,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3213a15f2133afbb0a4fec3b8f6e0eeca8c0befafd6658c41074e84f589d5d32 -R 9a7e323fe9ebfe9eb0759e1c70136795 +P 7461d2e120f2149315ddac2676d51d7445bcdb8e97543effd9c30603517ef9da +R 14050e9b3d7e7a784dc8a127e6275222 U dan -Z 100e3ccaddf1d19d0bacd74d773a93b7 +Z abcbf2aebf4cbfcbf65f71e558364487 diff --git a/manifest.uuid b/manifest.uuid index 0b854fa40a..65b5fdd250 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7461d2e120f2149315ddac2676d51d7445bcdb8e97543effd9c30603517ef9da \ No newline at end of file +f193ca587f9e4f925f4f2343b0b07053bd6f93dd87fc6f8f41cf4479e90cf562 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index c38bfb64b9..abba5b0168 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6073,9 +6073,9 @@ typedef struct RecoverTable RecoverTable; struct RecoverTable { char *zName; /* Name of table */ char *zQuoted; /* Quoted version of zName */ - char *zCreate; /* SQL to create table in default schema */ int nCol; /* Number of columns in table */ char **azlCol; /* Array of column lists */ + int iPk; }; /* @@ -6085,7 +6085,6 @@ static void recoverFreeTable(RecoverTable *pTab){ if( pTab ){ sqlite3_free(pTab->zName); sqlite3_free(pTab->zQuoted); - sqlite3_free(pTab->zCreate); if( pTab->azlCol ){ int i; for(i=0; inCol; i++){ @@ -6097,17 +6096,141 @@ static void recoverFreeTable(RecoverTable *pTab){ } } +static void recoverOldTable( + int *pRc, /* IN/OUT: Error code */ + RecoverTable *pTab, + const char *zName, /* Name of table */ + const char *zSql, /* CREATE TABLE statement */ + int bIntkey, + int nCol +){ + sqlite3 *dbtmp = 0; /* sqlite3 handle for testing CREATE TABLE */ + int rc = *pRc; + + if( rc==SQLITE_OK ){ + int nSqlCol = 0; + int bSqlIntkey = 0; + sqlite3_stmt *pStmt = 0; + + rc = sqlite3_open("", &dbtmp); + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(dbtmp, zSql, 0, 0, 0); + if( rc==SQLITE_ERROR ){ + rc = SQLITE_OK; + goto finished; + } + } + shellPreparePrintf(dbtmp, &rc, &pStmt, + "SELECT count(*) FROM pragma_table_info(%Q)", zName + ); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + nSqlCol = sqlite3_column_int(pStmt, 0); + } + shellFinalize(&rc, pStmt); + + if( rc!=SQLITE_OK || nSqlColiPk = sqlite3_column_int(pPkFinder, 0); + zPk = (const char*)sqlite3_column_text(pPkFinder, 1); + } + + pTab->zName = shellMPrintf(&rc, "%s", zName); + pTab->zQuoted = shellMPrintf(&rc, "%Q", pTab->zName); + pTab->azlCol = (char**)shellMalloc(&rc, sizeof(char*) * nSqlCol); + pTab->nCol = nSqlCol; + + if( nSqlCol==1 && pTab->iPk==0 ){ + pTab->azlCol[0] = shellMPrintf(&rc, "%Q", zPk); + }else{ + shellPreparePrintf(dbtmp, &rc, &pStmt, + "SELECT -1+row_number() OVER (ORDER BY cid)," + " %Q||%Q||group_concat(name, ', ') FILTER (WHERE cid!=%d) " + " OVER (ORDER BY cid) " + "FROM pragma_table_info(%Q)", + (bIntkey ? zPk : ""), (bIntkey ? ", " : ""), + pTab->iPk, zName + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + int idx = sqlite3_column_int(pStmt, 0); + const char *zText = (const char*)sqlite3_column_text(pStmt, 1); + pTab->azlCol[idx] = shellMPrintf(&rc, "%s", zText); + } + shellFinalize(&rc, pStmt); + } + shellFinalize(&rc, pPkFinder); + } + } + + finished: + sqlite3_close(dbtmp); + *pRc = rc; +} + static RecoverTable *recoverNewTable( ShellState *pState, int *pRc, int iRoot, + int bIntkey, int nCol ){ + sqlite3_stmt *pStmt = 0; RecoverTable *pRet = 0; + int bNoop = 0; + const char *zSql = 0; + const char *zName = 0; pRet = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable)); - if( pRet ){ + if( pRet ) pRet->iPk = -2; + + /* Search the recovered schema for an object with root page iRoot. */ + shellPreparePrintf(pState->db, pRc, &pStmt, + "SELECT type, name, sql FROM recovery.schema WHERE rootpage=%d", iRoot + ); + while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zType = (const char*)sqlite3_column_text(pStmt, 0); + if( bIntkey==0 && sqlite3_stricmp(zType, "index")==0 ){ + bNoop = 1; + break; + } + if( sqlite3_stricmp(zType, "table")==0 ){ + zName = (const char*)sqlite3_column_text(pStmt, 1); + zSql = (const char*)sqlite3_column_text(pStmt, 2); + recoverOldTable(pRc, pRet, zName, zSql, bIntkey, nCol); + break; + } + } + shellFinalize(pRc, pStmt); + if( bNoop ){ + sqlite3_free(pRet); + return 0; + } + + if( pRet && pRet->zName==0 ){ sqlite3_stmt *pStmt = 0; + pRet->zName = shellMPrintf(pRc, "orphan_%d_%d", nCol, iRoot); pRet->zQuoted = shellMPrintf(pRc, "%Q", pRet->zName); pRet->azlCol = (char**)shellMalloc(pRc, sizeof(char*) * nCol); @@ -6117,8 +6240,8 @@ static RecoverTable *recoverNewTable( "WITH s(i) AS (" " SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<%d" ")" - "SELECT i-1, group_concat('c' || i, ', ') OVER (ORDER BY i) FROM s", - nCol + "SELECT i-1, %Q || group_concat('c' || i, ', ') OVER (ORDER BY i) FROM s", + nCol, (bIntkey ? "id, " : "") ); while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ int idx = sqlite3_column_int(pStmt, 0); @@ -6127,9 +6250,15 @@ static RecoverTable *recoverNewTable( } shellFinalize(pRc, pStmt); - pRet->zCreate = shellMPrintf(pRc, "CREATE TABLE %Q (id, %s)", + if( *pRc==SQLITE_OK ){ + char *zCreate = shellMPrintf(pRc, "CREATE TABLE %Q (%s)", pRet->zName, pRet->azlCol[nCol-1] - ); + ); + if( zCreate ){ + raw_printf(pState->out, "%s;\n", zCreate); + sqlite3_free(zCreate); + } + } } if( *pRc!=SQLITE_OK ){ @@ -6146,7 +6275,6 @@ static RecoverTable *recoverNewTable( ** on stream pState->out. */ static int recoverDatabaseCmd(ShellState *pState){ - const char *zSql; int rc = SQLITE_OK; sqlite3_stmt *pLoop = 0; /* Loop through all root pages */ @@ -6173,7 +6301,9 @@ static int recoverDatabaseCmd(ShellState *pState){ /* Create the "map" table that will (eventually) contain instructions ** for dealing with each page in the db that contains one or more ** records. */ - "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, maxlen INT, root INT);" + "CREATE TABLE recovery.map(" + "pgno INTEGER PRIMARY KEY, maxlen INT, intkey, root INT" + ");" /* Populate table [map]. If there are circular loops of pages in the ** database, the following adds all pages in such a loop to the map @@ -6182,10 +6312,12 @@ static int recoverDatabaseCmd(ShellState *pState){ " SELECT page_count, max(field+1) " " FROM pragma_page_count, sqlite_dbdata WHERE pgno=page_count" " UNION ALL" - " SELECT * FROM (SELECT i-1, max(field+1)" - " FROM pages, sqlite_dbdata WHERE pgno=i-1 AND i>=2)" + " SELECT i-1, (" + " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=i-1" + " ) FROM pages WHERE i>=2" ")" - "INSERT INTO recovery.map(pgno, maxlen, root) SELECT i, maxlen, (" + "INSERT INTO recovery.map(pgno, maxlen, intkey, root) " + " SELECT i, maxlen, NULL, (" " WITH p(orig, pgno, parent) AS (" " SELECT 0, i, (SELECT pgno FROM recovery.dbptr WHERE child=i)" " UNION ALL" @@ -6195,6 +6327,9 @@ static int recoverDatabaseCmd(ShellState *pState){ " SELECT pgno FROM p WHERE (parent IS NULL OR pgno = orig)" ") " "FROM pages WHERE maxlen > 0;" + "UPDATE recovery.map AS o SET intkey = (" + " SELECT substr(data, 1, 1)==X'0D' FROM sqlite_dbpage WHERE pgno=o.pgno" + ");" /* Extract data from page 1 and any linked pages into table ** recovery.schema. With the same schema as an sqlite_master table. */ @@ -6221,36 +6356,53 @@ static int recoverDatabaseCmd(ShellState *pState){ return rc; #endif + /* Open a transaction, then print out all non-virtual, non-"sqlite_%" + ** CREATE TABLE statements that extracted from the existing schema. */ + if( rc==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + raw_printf(pState->out, "BEGIN;\n"); + shellPrepare(pState->db, &rc, + "SELECT sql FROM recovery.schema " + "WHERE type='table' " + " AND length(sql)>6" + " AND sql LIKE 'create table%'" + " AND name NOT LIKE 'sqliteX_%' ESCAPE 'X'", &pStmt + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zCreateTable = (const char*)sqlite3_column_text(pStmt, 0); + raw_printf(pState->out, "%s;\n", zCreateTable); + } + shellFinalize(&rc, pStmt); + } + /* Loop through each root page. */ - zSql = "SELECT root,max(maxlen) FROM recovery.map WHERE root>1 GROUP BY root"; - shellPrepare(pState->db, &rc, zSql, &pLoop); + shellPrepare(pState->db, &rc, + "SELECT root, intkey, max(maxlen) FROM recovery.map" + " WHERE root>1 GROUP BY root, intkey", &pLoop + ); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ int iRoot = sqlite3_column_int(pLoop, 0); - int nCol = sqlite3_column_int(pLoop, 1); + int bIntkey = sqlite3_column_int(pLoop, 1); + int nCol = sqlite3_column_int(pLoop, 2); RecoverTable *pTab; - pTab = recoverNewTable(pState, &rc, iRoot, nCol); + pTab = recoverNewTable(pState, &rc, iRoot, bIntkey, nCol); if( pTab ){ sqlite3_stmt *pData = 0; - raw_printf(pState->out, "%s;\n", pTab->zCreate); shellPreparePrintf(pState->db, &rc, &pData, "SELECT max(field), group_concat(quote(value), ', ') " "FROM sqlite_dbdata WHERE pgno IN (" " SELECT pgno FROM recovery.map WHERE root=%d" ")" - "GROUP BY pgno, cell;", iRoot + " AND field!=%d " + "GROUP BY pgno, cell;", iRoot, pTab->iPk ); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pData) ){ int iMax = sqlite3_column_int(pData, 0); const char *zVal = (const char*)sqlite3_column_text(pData, 1); - if( iMax+1==pTab->nCol ){ - raw_printf(pState->out, "INSERT INTO %s VALUES( %s );\n", - pTab->zQuoted, zVal); - }else{ - raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", - pTab->zQuoted, pTab->azlCol[iMax], zVal - ); - } + raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", + pTab->zQuoted, pTab->azlCol[iMax>0?iMax:0], zVal + ); } shellFinalize(&rc, pData); } @@ -6258,6 +6410,9 @@ static int recoverDatabaseCmd(ShellState *pState){ } shellFinalize(&rc, pLoop); + if( rc==SQLITE_OK ){ + raw_printf(pState->out, "COMMIT;\n"); + } sqlite3_exec(pState->db, "DETACH recovery", 0, 0, 0); return rc; } diff --git a/test/dbdata.test b/test/dbdata.test index 5a0df5dc86..391b278025 100644 --- a/test/dbdata.test +++ b/test/dbdata.test @@ -67,6 +67,16 @@ do_execsql_test 1.3 { SELECT value FROM sqlite_dbdata WHERE pgno=2 AND cell=2 AND field=1; } $big +do_execsql_test 1.4 { + DELETE FROM t1; + INSERT INTO t1 VALUES(NULL, randomblob(5050)); +} +do_test 1.5 { + execsql { + SELECT quote(value) FROM sqlite_dbdata WHERE pgno=2 AND cell=0 AND field=1; + } +} [db one {SELECT quote(b) FROM t1}] + #------------------------------------------------------------------------- reset_db db enable_load_extension 1 From 38f9c7194df5d9c86cbeaa04c22bdee56c3c5635 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 23 Apr 2019 18:03:02 +0000 Subject: [PATCH 05/38] Fixes for the ".recover" shell command. FossilOrigin-Name: 8dcc1d89d955bf58c80a8c30a37960f0cf95719953951a92626cc332cc75ec60 --- manifest | 13 +++---- manifest.uuid | 2 +- src/shell.c.in | 47 +++++++++++++++++++++---- test/recover.test | 88 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 136 insertions(+), 14 deletions(-) create mode 100644 test/recover.test diff --git a/manifest b/manifest index b830ce2b1e..509212bede 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\s".recover"\scommand.\sFix\sa\sproblem\swith\soverflow\spages\sin\sdbdata.c. -D 2019-04-22T20:52:12.850 +C Fixes\sfor\sthe\s".recover"\sshell\scommand. +D 2019-04-23T18:03:02.318 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 9263f5c30dd44c7ac2eb29f40a7ec64322a96885b71c00de6bc30b756c2e1c49 -F src/shell.c.in 7dd0babc42bc136b8dd070152981ee2325b787571f58326e68e6ad35d764aa33 +F src/shell.c.in 6c02cbd1de3b878abc04b08f2f74cc4d65d0a8e52dd3e78194db2a79d686b841 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1226,6 +1226,7 @@ F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 +F test/recover.test a8fed5acf2742268e366abb76a01de6ddce278aae01658488b950c968ebef638 F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/regexp2.test 40e894223b3d6672655481493f1be12012f2b33c F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 @@ -1820,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7461d2e120f2149315ddac2676d51d7445bcdb8e97543effd9c30603517ef9da -R 14050e9b3d7e7a784dc8a127e6275222 +P f193ca587f9e4f925f4f2343b0b07053bd6f93dd87fc6f8f41cf4479e90cf562 +R 73bcd5cc8a48367ff5a018a5ccb0a79d U dan -Z abcbf2aebf4cbfcbf65f71e558364487 +Z e408dc486ac689ec1d61c890dc79708f diff --git a/manifest.uuid b/manifest.uuid index 65b5fdd250..680d92aac8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f193ca587f9e4f925f4f2343b0b07053bd6f93dd87fc6f8f41cf4479e90cf562 \ No newline at end of file +8dcc1d89d955bf58c80a8c30a37960f0cf95719953951a92626cc332cc75ec60 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index abba5b0168..8ac87fbfa2 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6113,6 +6113,9 @@ static void recoverOldTable( sqlite3_stmt *pStmt = 0; rc = sqlite3_open("", &dbtmp); + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(dbtmp, "PRAGMA writable_schema = on", 0, 0, 0); + } if( rc==SQLITE_OK ){ rc = sqlite3_exec(dbtmp, zSql, 0, 0, 0); if( rc==SQLITE_ERROR ){ @@ -6361,16 +6364,16 @@ static int recoverDatabaseCmd(ShellState *pState){ if( rc==SQLITE_OK ){ sqlite3_stmt *pStmt = 0; raw_printf(pState->out, "BEGIN;\n"); - shellPrepare(pState->db, &rc, + raw_printf(pState->out, "PRAGMA writable_schema = on;\n"); + shellPrepare(pState->db, &rc, "SELECT sql FROM recovery.schema " - "WHERE type='table' " - " AND length(sql)>6" - " AND sql LIKE 'create table%'" - " AND name NOT LIKE 'sqliteX_%' ESCAPE 'X'", &pStmt + "WHERE type='table' AND sql LIKE 'create table%'", &pStmt ); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ const char *zCreateTable = (const char*)sqlite3_column_text(pStmt, 0); - raw_printf(pState->out, "%s;\n", zCreateTable); + raw_printf(pState->out, "CREATE TABLE IF NOT EXISTS %s;\n", + &zCreateTable[12] + ); } shellFinalize(&rc, pStmt); } @@ -6378,7 +6381,9 @@ static int recoverDatabaseCmd(ShellState *pState){ /* Loop through each root page. */ shellPrepare(pState->db, &rc, "SELECT root, intkey, max(maxlen) FROM recovery.map" - " WHERE root>1 GROUP BY root, intkey", &pLoop + " WHERE root>1 GROUP BY root, intkey ORDER BY root=(" + " SELECT rootpage FROM recovery.schema WHERE name='sqlite_sequence'" + ")", &pLoop ); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ int iRoot = sqlite3_column_int(pLoop, 0); @@ -6389,6 +6394,9 @@ static int recoverDatabaseCmd(ShellState *pState){ pTab = recoverNewTable(pState, &rc, iRoot, bIntkey, nCol); if( pTab ){ sqlite3_stmt *pData = 0; + if( 0==sqlite3_stricmp(pTab->zName, "sqlite_sequence") ){ + raw_printf(pState->out, "DELETE FROM sqlite_sequence;\n"); + } shellPreparePrintf(pState->db, &rc, &pData, "SELECT max(field), group_concat(quote(value), ', ') " "FROM sqlite_dbdata WHERE pgno IN (" @@ -6410,7 +6418,32 @@ static int recoverDatabaseCmd(ShellState *pState){ } shellFinalize(&rc, pLoop); + /* The rest of the schema */ if( rc==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + shellPrepare(pState->db, &rc, + "SELECT sql, name FROM recovery.schema " + "WHERE (type='table' AND sql LIKE 'create table%') IS NOT TRUE", &pStmt + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zSql = (const char*)sqlite3_column_text(pStmt, 0); + if( sqlite3_strnicmp(zSql, "create virt", 11)==0 ){ + const char *zName = (const char*)sqlite3_column_text(pStmt, 1); + char *zPrint = shellMPrintf(&rc, + "INSERT INTO sqlite_master VALUES('table', %Q, %Q, 0, %Q)", + zName, zName, zSql + ); + raw_printf(pState->out, "%s;\n", zPrint); + sqlite3_free(zPrint); + }else{ + raw_printf(pState->out, "%s;\n", zSql); + } + } + shellFinalize(&rc, pStmt); + } + + if( rc==SQLITE_OK ){ + raw_printf(pState->out, "PRAGMA writable_schema = off;\n"); raw_printf(pState->out, "COMMIT;\n"); } sqlite3_exec(pState->db, "DETACH recovery", 0, 0, 0); diff --git a/test/recover.test b/test/recover.test new file mode 100644 index 0000000000..6ee35054f8 --- /dev/null +++ b/test/recover.test @@ -0,0 +1,88 @@ +# 2019 April 23 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test the shell tool ".ar" command. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix recover + +ifcapable !vtab { + finish_test; return +} +set CLI [test_find_cli] + +proc compare_result {db1 db2 sql} { + set r1 [$db1 eval $sql] + set r2 [$db2 eval $sql] + if {$r1 != $r2} { + puts "r1: $r1" + puts "r2: $r2" + error "mismatch for $sql" + } + return "" +} + +proc compare_dbs {db1 db2} { + compare_result $db1 $db2 "SELECT sql FROM sqlite_master ORDER BY 1" + foreach tbl [$db1 eval {SELECT name FROM sqlite_master WHERE type='table'}] { + compare_result $db1 $db2 "SELECT * FROM $tbl" + } +} + +proc do_recover_test {tn} { + set fd [open "|$::CLI test.db .recover"] + fconfigure $fd -encoding binary + fconfigure $fd -translation binary + set sql [read $fd] + close $fd + + forcedelete test.db2 + sqlite3 db2 test.db2 + breakpoint + execsql $sql db2 + uplevel [list do_test $tn [list compare_dbs db db2] {}] + db2 close +} + +set doc { + hello + world +} +do_execsql_test 1.1.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 4, X'1234567800'); + INSERT INTO t1 VALUES(2, 'test', 8.1); + INSERT INTO t1 VALUES(3, $doc, 8.4); +} +do_recover_test 1.1.2 + +do_execsql_test 1.2.1 " + DELETE FROM t1; + INSERT INTO t1 VALUES(13, 'hello\r\nworld', 13); +" +do_recover_test 1.2.2 + +do_execsql_test 1.3.1 " + CREATE TABLE t2(i INTEGER PRIMARY KEY AUTOINCREMENT, b, c); + INSERT INTO t2 VALUES(NULL, 1, 2); + INSERT INTO t2 VALUES(NULL, 3, 4); + INSERT INTO t2 VALUES(NULL, 5, 6); + CREATE TABLE t3(i INTEGER PRIMARY KEY AUTOINCREMENT, b, c); + INSERT INTO t3 VALUES(NULL, 1, 2); + INSERT INTO t3 VALUES(NULL, 3, 4); + INSERT INTO t3 VALUES(NULL, 5, 6); + DELETE FROM t2; +" +do_recover_test 1.3.2 + +finish_test From b182588c8e0cdbafc5987df0a919d29021d1ba58 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 23 Apr 2019 20:48:32 +0000 Subject: [PATCH 06/38] Have ".recover" handle "\r" and "\n" in the same way as ".dump". FossilOrigin-Name: f95f0f02ab6c6cf45f25b613c7ab57f68249689d0a9eddf4c9518ddf0edad365 --- manifest | 12 +++--- manifest.uuid | 2 +- src/shell.c.in | 106 +++++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 101 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 509212bede..e3ab13575f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sfor\sthe\s".recover"\sshell\scommand. -D 2019-04-23T18:03:02.318 +C Have\s".recover"\shandle\s"\\r"\sand\s"\\n"\sin\sthe\ssame\sway\sas\s".dump". +D 2019-04-23T20:48:32.366 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 9263f5c30dd44c7ac2eb29f40a7ec64322a96885b71c00de6bc30b756c2e1c49 -F src/shell.c.in 6c02cbd1de3b878abc04b08f2f74cc4d65d0a8e52dd3e78194db2a79d686b841 +F src/shell.c.in 6e56c60640410885a8c3c264971619407ae5c7f46165e2bfe5fb1ca1a6a58ad0 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f193ca587f9e4f925f4f2343b0b07053bd6f93dd87fc6f8f41cf4479e90cf562 -R 73bcd5cc8a48367ff5a018a5ccb0a79d +P 8dcc1d89d955bf58c80a8c30a37960f0cf95719953951a92626cc332cc75ec60 +R b9c18dd1095e3f7b028bdaea88f9fcb0 U dan -Z e408dc486ac689ec1d61c890dc79708f +Z edabfc3afbc2656b0ce1a8fb6b1e492b diff --git a/manifest.uuid b/manifest.uuid index 680d92aac8..fdcc68f9a9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8dcc1d89d955bf58c80a8c30a37960f0cf95719953951a92626cc332cc75ec60 \ No newline at end of file +f95f0f02ab6c6cf45f25b613c7ab57f68249689d0a9eddf4c9518ddf0edad365 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 8ac87fbfa2..7989c9dfde 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -3934,6 +3934,96 @@ readHexDb_error: } #endif /* SQLITE_ENABLE_DESERIALIZE */ +/* +** Scalar function "shell_escape_crnl" used by the .recover command. +** The argument passed to this function is the output of built-in +** function quote(). If the first character of the input is "'", +** indicating that the value passed to quote() was a text value, +** then this function searches the input for "\n" and "\r" characters +** and adds a wrapper similar to the following: +** +** replace(replace(, '\n', char(10), '\r', char(13)); +** +** Or, if the first character of the input is not "'", then a copy +** of the input is returned. +*/ +static void shellEscapeCrnl( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zText = (const char*)sqlite3_value_text(argv[0]); + if( zText[0]=='\'' ){ + int nText = sqlite3_value_bytes(argv[0]); + int i; + char zBuf1[20]; + char zBuf2[20]; + const char *zNL = 0; + const char *zCR = 0; + int nCR = 0; + int nNL = 0; + + for(i=0; zText[i]; i++){ + if( zNL==0 && zText[i]=='\n' ){ + zNL = unused_string(zText, "\\n", "\\012", zBuf1); + nNL = (int)strlen(zNL); + } + if( zCR==0 && zText[i]=='\r' ){ + zCR = unused_string(zText, "\\r", "\\015", zBuf2); + nCR = (int)strlen(zCR); + } + } + + if( zNL || zCR ){ + int iOut = 0; + i64 nMax = (nNL > nCR) ? nNL : nCR; + i64 nAlloc = nMax * nText + (nMax+12)*2; + char *zOut = (char*)sqlite3_malloc64(nAlloc); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + if( zNL && zCR ){ + memcpy(&zOut[iOut], "replace(replace(", 16); + iOut += 16; + }else{ + memcpy(&zOut[iOut], "replace(", 8); + iOut += 8; + } + for(i=0; zText[i]; i++){ + if( zText[i]=='\n' ){ + memcpy(&zOut[iOut], zNL, nNL); + iOut += nNL; + }else if( zText[i]=='\r' ){ + memcpy(&zOut[iOut], zCR, nCR); + iOut += nCR; + }else{ + zOut[iOut] = zText[i]; + iOut++; + } + } + + if( zNL ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zNL, nNL); iOut += nNL; + memcpy(&zOut[iOut], "', char(10))", 12); iOut += 12; + } + if( zCR ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zCR, nCR); iOut += nCR; + memcpy(&zOut[iOut], "', char(13))", 12); iOut += 12; + } + + sqlite3_result_text(context, zOut, iOut, SQLITE_TRANSIENT); + sqlite3_free(zOut); + return; + } + } + + sqlite3_result_value(context, argv[0]); +} + /* Flags for open_db(). ** ** The default behavior of open_db() is to exit(1) if the database fails to @@ -4013,6 +4103,8 @@ static void open_db(ShellState *p, int openFlags){ shellModuleSchema, 0, 0); sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p, shellPutsFunc, 0, 0); + sqlite3_create_function(p->db, "shell_escape_crnl", 1, SQLITE_UTF8, 0, + shellEscapeCrnl, 0, 0); #ifndef SQLITE_NOHAVE_SYSTEM sqlite3_create_function(p->db, "edit", 1, SQLITE_UTF8, 0, editFunc, 0, 0); @@ -6349,16 +6441,6 @@ static int recoverDatabaseCmd(ShellState *pState){ "GROUP BY pgno, cell;" ); -#if 0 - zSql = "SELECT type ||','|| name ||','|| tbl_name ||','|| rootpage ||','|| sql FROM recovery.schema;"; - shellPrepare(pState->db, &rc, zSql, &pLoop); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ - raw_printf(pState->out, "%s\n", (const char*)sqlite3_column_text(pLoop, 0)); - } - shellFinalize(&rc, pLoop); - return rc; -#endif - /* Open a transaction, then print out all non-virtual, non-"sqlite_%" ** CREATE TABLE statements that extracted from the existing schema. */ if( rc==SQLITE_OK ){ @@ -6398,7 +6480,7 @@ static int recoverDatabaseCmd(ShellState *pState){ raw_printf(pState->out, "DELETE FROM sqlite_sequence;\n"); } shellPreparePrintf(pState->db, &rc, &pData, - "SELECT max(field), group_concat(quote(value), ', ') " + "SELECT max(field), group_concat(shell_escape_crnl(quote(value)),', ')" "FROM sqlite_dbdata WHERE pgno IN (" " SELECT pgno FROM recovery.map WHERE root=%d" ")" @@ -6423,7 +6505,7 @@ static int recoverDatabaseCmd(ShellState *pState){ sqlite3_stmt *pStmt = 0; shellPrepare(pState->db, &rc, "SELECT sql, name FROM recovery.schema " - "WHERE (type='table' AND sql LIKE 'create table%') IS NOT TRUE", &pStmt + "WHERE sql NOT LIKE 'create table%'", &pStmt ); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ const char *zSql = (const char*)sqlite3_column_text(pStmt, 0); From efa363b84c43d63c706e7997085a2e5977aac64d Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 24 Apr 2019 20:48:55 +0000 Subject: [PATCH 07/38] Improve the performance of the .recover command. FossilOrigin-Name: a50768314d10d743a0cc013b434b516f0763e0a6c5b79655d8fefde7de53e869 --- ext/misc/dbdata.c | 60 +++++++++++++++++++++++++++++++++++------------ manifest | 14 +++++------ manifest.uuid | 2 +- src/shell.c.in | 42 ++++++++++++++++++++------------- 4 files changed, 79 insertions(+), 39 deletions(-) diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c index 7662417ae5..13bb51f08b 100644 --- a/ext/misc/dbdata.c +++ b/ext/misc/dbdata.c @@ -106,6 +106,7 @@ struct DbdataCursor { struct DbdataTable { sqlite3_vtab base; /* Base class. Must be first */ sqlite3 *db; /* The database connection */ + sqlite3_stmt *pStmt; /* For fetching database pages */ int bPtr; /* True for sqlite3_dbptr table */ }; @@ -167,7 +168,11 @@ static int dbdataConnect( ** Disconnect from or destroy a dbdata virtual table. */ static int dbdataDisconnect(sqlite3_vtab *pVtab){ - sqlite3_free(pVtab); + DbdataTable *pTab = (DbdataTable*)pVtab; + if( pTab ){ + sqlite3_finalize(pTab->pStmt); + sqlite3_free(pVtab); + } return SQLITE_OK; } @@ -185,15 +190,15 @@ static int dbdataDisconnect(sqlite3_vtab *pVtab){ ** If both parameters are present, schema is in position 0 and pgno in ** position 1. */ -static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ +static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdx){ DbdataTable *pTab = (DbdataTable*)tab; int i; int iSchema = -1; int iPgno = -1; int colSchema = (pTab->bPtr ? DBPTR_COLUMN_SCHEMA : DBDATA_COLUMN_SCHEMA); - for(i=0; inConstraint; i++){ - struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i]; + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pIdx->aConstraint[i]; if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ if( p->iColumn==colSchema ){ if( p->usable==0 ) return SQLITE_CONSTRAINT; @@ -206,14 +211,29 @@ static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ } if( iSchema>=0 ){ - pIdxInfo->aConstraintUsage[iSchema].argvIndex = 1; - pIdxInfo->aConstraintUsage[iSchema].omit = 1; + pIdx->aConstraintUsage[iSchema].argvIndex = 1; + pIdx->aConstraintUsage[iSchema].omit = 1; } if( iPgno>=0 ){ - pIdxInfo->aConstraintUsage[iPgno].argvIndex = 1 + (iSchema>=0); - pIdxInfo->aConstraintUsage[iPgno].omit = 1; + pIdx->aConstraintUsage[iPgno].argvIndex = 1 + (iSchema>=0); + pIdx->aConstraintUsage[iPgno].omit = 1; + pIdx->estimatedCost = 100; + pIdx->estimatedRows = 50; + + if( pTab->bPtr==0 && pIdx->nOrderBy && pIdx->aOrderBy[0].desc==0 ){ + int iCol = pIdx->aOrderBy[0].iColumn; + if( pIdx->nOrderBy==1 ){ + pIdx->orderByConsumed = (iCol==0 || iCol==1); + }else if( pIdx->nOrderBy==2 && pIdx->aOrderBy[1].desc==0 && iCol==0 ){ + pIdx->orderByConsumed = (pIdx->aOrderBy[1].iColumn==1); + } + } + + }else{ + pIdx->estimatedCost = 100000000; + pIdx->estimatedRows = 1000000000; } - pIdxInfo->idxNum = (iSchema>=0 ? 0x01 : 0x00) | (iPgno>=0 ? 0x02 : 0x00); + pIdx->idxNum = (iSchema>=0 ? 0x01 : 0x00) | (iPgno>=0 ? 0x02 : 0x00); return SQLITE_OK; } @@ -236,7 +256,12 @@ static int dbdataOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ } static void dbdataResetCursor(DbdataCursor *pCsr){ - sqlite3_finalize(pCsr->pStmt); + DbdataTable *pTab = (DbdataTable*)(pCsr->base.pVtab); + if( pTab->pStmt==0 ){ + pTab->pStmt = pCsr->pStmt; + }else{ + sqlite3_finalize(pCsr->pStmt); + } pCsr->pStmt = 0; pCsr->iPgno = 1; pCsr->iCell = 0; @@ -558,7 +583,7 @@ static int dbdataFilter( ){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; - int rc; + int rc = SQLITE_OK; const char *zSchema = "main"; dbdataResetCursor(pCsr); @@ -571,10 +596,15 @@ static int dbdataFilter( pCsr->bOnePage = 1; } - rc = sqlite3_prepare_v2(pTab->db, - "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1, - &pCsr->pStmt, 0 - ); + if( pTab->pStmt ){ + pCsr->pStmt = pTab->pStmt; + pTab->pStmt = 0; + }else{ + rc = sqlite3_prepare_v2(pTab->db, + "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1, + &pCsr->pStmt, 0 + ); + } if( rc==SQLITE_OK ){ rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT); }else{ diff --git a/manifest b/manifest index e3ab13575f..039fa63685 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\s".recover"\shandle\s"\\r"\sand\s"\\n"\sin\sthe\ssame\sway\sas\s".dump". -D 2019-04-23T20:48:32.366 +C Improve\sthe\sperformance\sof\sthe\s.recover\scommand. +D 2019-04-24T20:48:55.802 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -284,7 +284,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c cec672d40604075bb341a7f11ac48393efdcd90a979269b8fe7977ea62d0547f F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189 F ext/misc/csv.c 7f047aeb68f5802e7ce6639292095d622a488bb43526ed04810e0649faa71ceb -F ext/misc/dbdata.c 6a0ccc33e8c5ef3b6164eec7839cb4dff44e27bee0717f1660dbb8ee77ee4cf8 +F ext/misc/dbdata.c 0c80f0757c3a1b5c57027328ab0ccfe30fcef1a4875b41dd9ddc0f53c7087e97 F ext/misc/dbdump.c baf6e37447c9d6968417b1cd34cbedb0b0ab3f91b5329501d8a8d5be3287c336 F ext/misc/eval.c 4b4757592d00fd32e44c7a067e6a0e4839c81a4d57abc4131ee7806d1be3104e F ext/misc/explain.c d5c12962d79913ef774b297006872af1fccda388f61a11d37758f9179a09551f @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 9263f5c30dd44c7ac2eb29f40a7ec64322a96885b71c00de6bc30b756c2e1c49 -F src/shell.c.in 6e56c60640410885a8c3c264971619407ae5c7f46165e2bfe5fb1ca1a6a58ad0 +F src/shell.c.in 569ee1f79236f19b44fb4fcb8a0a2f0b2dce9a3d856d41a196aaaaa5f0d37704 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8dcc1d89d955bf58c80a8c30a37960f0cf95719953951a92626cc332cc75ec60 -R b9c18dd1095e3f7b028bdaea88f9fcb0 +P f95f0f02ab6c6cf45f25b613c7ab57f68249689d0a9eddf4c9518ddf0edad365 +R 0e7ee563b8599ed64c48e30038f5f99f U dan -Z edabfc3afbc2656b0ce1a8fb6b1e492b +Z f093e1d7a99537670d8620da5d0b299b diff --git a/manifest.uuid b/manifest.uuid index fdcc68f9a9..3f61303065 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f95f0f02ab6c6cf45f25b613c7ab57f68249689d0a9eddf4c9518ddf0edad365 \ No newline at end of file +a50768314d10d743a0cc013b434b516f0763e0a6c5b79655d8fefde7de53e869 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 7989c9dfde..05ce02d605 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6372,6 +6372,8 @@ static RecoverTable *recoverNewTable( static int recoverDatabaseCmd(ShellState *pState){ int rc = SQLITE_OK; sqlite3_stmt *pLoop = 0; /* Loop through all root pages */ + sqlite3_stmt *pPages = 0; /* Loop through all pages in a group */ + sqlite3_stmt *pCells = 0; /* Loop through all cells in a page */ shellExec(pState->db, &rc, /* Attach an in-memory database named 'recovery'. Create an indexed @@ -6460,6 +6462,15 @@ static int recoverDatabaseCmd(ShellState *pState){ shellFinalize(&rc, pStmt); } + shellPrepare(pState->db, &rc, + "SELECT pgno FROM recovery.map WHERE root=?", &pPages + ); + shellPrepare(pState->db, &rc, + "SELECT max(field), group_concat(shell_escape_crnl(quote(value)), ', ')" + "FROM sqlite_dbdata WHERE pgno = ? AND field != ?" + "GROUP BY cell", &pCells + ); + /* Loop through each root page. */ shellPrepare(pState->db, &rc, "SELECT root, intkey, max(maxlen) FROM recovery.map" @@ -6475,30 +6486,29 @@ static int recoverDatabaseCmd(ShellState *pState){ pTab = recoverNewTable(pState, &rc, iRoot, bIntkey, nCol); if( pTab ){ - sqlite3_stmt *pData = 0; if( 0==sqlite3_stricmp(pTab->zName, "sqlite_sequence") ){ raw_printf(pState->out, "DELETE FROM sqlite_sequence;\n"); } - shellPreparePrintf(pState->db, &rc, &pData, - "SELECT max(field), group_concat(shell_escape_crnl(quote(value)),', ')" - "FROM sqlite_dbdata WHERE pgno IN (" - " SELECT pgno FROM recovery.map WHERE root=%d" - ")" - " AND field!=%d " - "GROUP BY pgno, cell;", iRoot, pTab->iPk - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pData) ){ - int iMax = sqlite3_column_int(pData, 0); - const char *zVal = (const char*)sqlite3_column_text(pData, 1); - raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", - pTab->zQuoted, pTab->azlCol[iMax>0?iMax:0], zVal - ); + sqlite3_bind_int(pPages, 1, iRoot); + sqlite3_bind_int(pCells, 2, pTab->iPk); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPages) ){ + sqlite3_bind_int(pCells, 1, sqlite3_column_int(pPages, 0)); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pCells) ){ + int iMax = sqlite3_column_int(pCells, 0); + const char *zVal = (const char*)sqlite3_column_text(pCells, 1); + raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", + pTab->zQuoted, pTab->azlCol[iMax>0?iMax:0], zVal + ); + } + shellReset(&rc, pCells); } - shellFinalize(&rc, pData); + shellReset(&rc, pPages); } recoverFreeTable(pTab); } shellFinalize(&rc, pLoop); + shellFinalize(&rc, pPages); + shellFinalize(&rc, pCells); /* The rest of the schema */ if( rc==SQLITE_OK ){ From b9b71dbfd4066df5d62fd59c1f5350faca74fcd4 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 25 Apr 2019 16:20:40 +0000 Subject: [PATCH 08/38] Fix a bug preventing .recover from working on databases where the final page of the db is corrupt. FossilOrigin-Name: 959bbd11e92cc789973daf20dfcb8a6d8dc724dd603b286cbdd59e5d1fdb2909 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 9 +++++---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 039fa63685..2701068663 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\sperformance\sof\sthe\s.recover\scommand. -D 2019-04-24T20:48:55.802 +C Fix\sa\sbug\spreventing\s.recover\sfrom\sworking\son\sdatabases\swhere\sthe\sfinal\spage\sof\sthe\sdb\sis\scorrupt. +D 2019-04-25T16:20:40.499 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 9263f5c30dd44c7ac2eb29f40a7ec64322a96885b71c00de6bc30b756c2e1c49 -F src/shell.c.in 569ee1f79236f19b44fb4fcb8a0a2f0b2dce9a3d856d41a196aaaaa5f0d37704 +F src/shell.c.in 2e9b6b05fd202499bbe233632f9e5a913d2afcd645b59a114bfeb440c3e5cd38 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f95f0f02ab6c6cf45f25b613c7ab57f68249689d0a9eddf4c9518ddf0edad365 -R 0e7ee563b8599ed64c48e30038f5f99f +P a50768314d10d743a0cc013b434b516f0763e0a6c5b79655d8fefde7de53e869 +R 9da7af39576825c9e597192b24527e0d U dan -Z f093e1d7a99537670d8620da5d0b299b +Z 0de6a10b348dda1bb213a1119e4cc094 diff --git a/manifest.uuid b/manifest.uuid index 3f61303065..e18c3e1cac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a50768314d10d743a0cc013b434b516f0763e0a6c5b79655d8fefde7de53e869 \ No newline at end of file +959bbd11e92cc789973daf20dfcb8a6d8dc724dd603b286cbdd59e5d1fdb2909 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 05ce02d605..73bf9dff97 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6369,7 +6369,7 @@ static RecoverTable *recoverNewTable( ** to construct a new database containing all recovered data is output ** on stream pState->out. */ -static int recoverDatabaseCmd(ShellState *pState){ +static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ int rc = SQLITE_OK; sqlite3_stmt *pLoop = 0; /* Loop through all root pages */ sqlite3_stmt *pPages = 0; /* Loop through all pages in a group */ @@ -6406,8 +6406,9 @@ static int recoverDatabaseCmd(ShellState *pState){ ** database, the following adds all pages in such a loop to the map ** as individual root pages. This could be handled better. */ "WITH pages(i, maxlen) AS (" - " SELECT page_count, max(field+1) " - " FROM pragma_page_count, sqlite_dbdata WHERE pgno=page_count" + " SELECT page_count, (" + " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=page_count" + " ) FROM pragma_page_count" " UNION ALL" " SELECT i-1, (" " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=i-1" @@ -6831,7 +6832,7 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='r' && strncmp(azArg[0], "recover", n)==0 ){ open_db(p, 0); - rc = recoverDatabaseCmd(p); + rc = recoverDatabaseCmd(p, nArg, azArg); }else if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){ From 9c014f8b0c63b3b8c8c2ae30dd9c9cbe1b883952 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 25 Apr 2019 19:23:15 +0000 Subject: [PATCH 09/38] Unless the "--freelist-corrupt" option is specified, do not have the .recover command attempt to recover data from pages that are on the database free-list. FossilOrigin-Name: 8d2f52bb640d6d0f84b18d746043e56f45a73ace93239be1d036701f7f4018fd --- manifest | 12 ++++---- manifest.uuid | 2 +- src/shell.c.in | 78 +++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 84 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2701068663..f7a1f6f81c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\spreventing\s.recover\sfrom\sworking\son\sdatabases\swhere\sthe\sfinal\spage\sof\sthe\sdb\sis\scorrupt. -D 2019-04-25T16:20:40.499 +C Unless\sthe\s"--freelist-corrupt"\soption\sis\sspecified,\sdo\snot\shave\sthe\s.recover\scommand\sattempt\sto\srecover\sdata\sfrom\spages\sthat\sare\son\sthe\sdatabase\sfree-list. +D 2019-04-25T19:23:15.197 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c 9263f5c30dd44c7ac2eb29f40a7ec64322a96885b71c00de6bc30b756c2e1c49 -F src/shell.c.in 2e9b6b05fd202499bbe233632f9e5a913d2afcd645b59a114bfeb440c3e5cd38 +F src/shell.c.in a3b1bb23c3c3dde9aab1823dc37cb0f3e81debfd3937e40ee8fa69fe1f0e4c16 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a50768314d10d743a0cc013b434b516f0763e0a6c5b79655d8fefde7de53e869 -R 9da7af39576825c9e597192b24527e0d +P 959bbd11e92cc789973daf20dfcb8a6d8dc724dd603b286cbdd59e5d1fdb2909 +R d86f5b916c733bd261b32efe2b58444b U dan -Z 0de6a10b348dda1bb213a1119e4cc094 +Z 3b2dfcc08d011cfeb383f50fbff24d3e diff --git a/manifest.uuid b/manifest.uuid index e18c3e1cac..37c6cf3c7d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -959bbd11e92cc789973daf20dfcb8a6d8dc724dd603b286cbdd59e5d1fdb2909 \ No newline at end of file +8d2f52bb640d6d0f84b18d746043e56f45a73ace93239be1d036701f7f4018fd \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 73bf9dff97..95c18ae4d1 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -3934,6 +3934,35 @@ readHexDb_error: } #endif /* SQLITE_ENABLE_DESERIALIZE */ +/* +** Scalar function "shell_int32". The first argument to this function +** must be a blob. The second a non-negative integer. This function +** reads and returns a 32-bit big-endian integer from byte +** offset (4*) of the blob. +*/ +static void shellInt32( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *pBlob; + int nBlob; + int iInt; + + nBlob = sqlite3_value_bytes(argv[0]); + pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]); + iInt = sqlite3_value_int(argv[1]); + + if( iInt>=0 && (iInt+1)*4<=nBlob ){ + const unsigned char *a = &pBlob[iInt*4]; + sqlite3_int64 iVal = ((sqlite3_int64)a[0]<<24) + + ((sqlite3_int64)a[1]<<16) + + ((sqlite3_int64)a[2]<< 8) + + ((sqlite3_int64)a[3]<< 0); + sqlite3_result_int64(context, iVal); + } +} + /* ** Scalar function "shell_escape_crnl" used by the .recover command. ** The argument passed to this function is the output of built-in @@ -4105,6 +4134,8 @@ static void open_db(ShellState *p, int openFlags){ shellPutsFunc, 0, 0); sqlite3_create_function(p->db, "shell_escape_crnl", 1, SQLITE_UTF8, 0, shellEscapeCrnl, 0, 0); + sqlite3_create_function(p->db, "shell_int32", 2, SQLITE_UTF8, 0, + shellInt32, 0, 0); #ifndef SQLITE_NOHAVE_SYSTEM sqlite3_create_function(p->db, "edit", 1, SQLITE_UTF8, 0, editFunc, 0, 0); @@ -6374,6 +6405,25 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ sqlite3_stmt *pLoop = 0; /* Loop through all root pages */ sqlite3_stmt *pPages = 0; /* Loop through all pages in a group */ sqlite3_stmt *pCells = 0; /* Loop through all cells in a page */ + int i; + + int bFreelist = 1; /* 0 if --freelist-corrupt is specified */ + for(i=1; idb, &rc, /* Attach an in-memory database named 'recovery'. Create an indexed @@ -6395,6 +6445,32 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ " SELECT child FROM recovery.dbptr GROUP BY child HAVING count(*)>1" ");" + "CREATE TABLE recovery.freelist(pgno INTEGER PRIMARY KEY);" + ); + + if( bFreelist ){ + shellExec(pState->db, &rc, + "WITH trunk(pgno) AS (" + " SELECT shell_int32(" + " (SELECT data FROM sqlite_dbpage WHERE pgno=1), 8) AS x " + " WHERE x>0" + " UNION" + " SELECT shell_int32(" + " (SELECT data FROM sqlite_dbpage WHERE pgno=trunk.pgno), 0) AS x " + " FROM trunk WHERE x>0" + ")," + "freelist(data, n, freepgno) AS (" + " SELECT data, shell_int32(data, 1)-1, t.pgno " + " FROM trunk t, sqlite_dbpage s WHERE s.pgno=t.pgno" + " UNION ALL" + " SELECT data, n-1, shell_int32(data, 2+n) " + " FROM freelist WHERE n>=0" + ")" + "REPLACE INTO recovery.freelist SELECT freepgno FROM freelist;" + ); + } + + shellExec(pState->db, &rc, /* Create the "map" table that will (eventually) contain instructions ** for dealing with each page in the db that contains one or more ** records. */ @@ -6424,7 +6500,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ " )" " SELECT pgno FROM p WHERE (parent IS NULL OR pgno = orig)" ") " - "FROM pages WHERE maxlen > 0;" + "FROM pages WHERE maxlen > 0 AND i NOT IN freelist;" "UPDATE recovery.map AS o SET intkey = (" " SELECT substr(data, 1, 1)==X'0D' FROM sqlite_dbpage WHERE pgno=o.pgno" ");" From c0b42437ab69d01daa61bd77f74badb95c82bd40 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 26 Apr 2019 15:14:53 +0000 Subject: [PATCH 10/38] Fix a locking-page related problem with the ".recover" command. FossilOrigin-Name: afdae10424f0f3d0f10a4b73e9732aa55c5ee664814d8ca0edd372cfb17c2445 --- ext/misc/dbdata.c | 66 +++++++++++++++++++++++++++++++++++------------ manifest | 14 +++++----- manifest.uuid | 2 +- src/shell.c.in | 42 ++++++++++++++++++++++++------ 4 files changed, 91 insertions(+), 33 deletions(-) diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c index 13bb51f08b..a27f19dc7a 100644 --- a/ext/misc/dbdata.c +++ b/ext/misc/dbdata.c @@ -89,6 +89,7 @@ struct DbdataCursor { int nCell; /* Number of cells on aPage[] */ int iCell; /* Current cell number */ int bOnePage; /* True to stop after one page */ + int szDb; sqlite3_int64 iRowid; /* Only for the sqlite_dbdata table */ @@ -303,18 +304,21 @@ static int dbdataLoadPage( sqlite3_bind_int64(pStmt, 2, pgno); if( SQLITE_ROW==sqlite3_step(pStmt) ){ int nCopy = sqlite3_column_bytes(pStmt, 0); - u8 *pPage = (u8*)sqlite3_malloc64(nCopy); - if( pPage==0 ){ - rc = SQLITE_NOMEM; - }else{ - const u8 *pCopy = sqlite3_column_blob(pStmt, 0); - memcpy(pPage, pCopy, nCopy); + if( nCopy>0 ){ + u8 *pPage; + pPage = (u8*)sqlite3_malloc64(nCopy); + if( pPage==0 ){ + rc = SQLITE_NOMEM; + }else{ + const u8 *pCopy = sqlite3_column_blob(pStmt, 0); + memcpy(pPage, pCopy, nCopy); + } *ppPage = pPage; *pnPage = nCopy; } } rc2 = sqlite3_reset(pStmt); - if( *ppPage==0 ) rc = rc2; + if( rc==SQLITE_OK ) rc = rc2; return rc; } @@ -419,8 +423,13 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ int iOff = (pCsr->iPgno==1 ? 100 : 0); if( pCsr->aPage==0 ){ - rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage); - if( rc!=SQLITE_OK || pCsr->aPage==0 ) return rc; + while( 1 ){ + if( pCsr->bOnePage==0 && pCsr->iPgno>pCsr->szDb ) return SQLITE_OK; + rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage); + if( rc!=SQLITE_OK ) return rc; + if( pCsr->aPage ) break; + pCsr->iPgno++; + } pCsr->iCell = pTab->bPtr ? -2 : 0; pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]); } @@ -574,6 +583,24 @@ static int dbdataEof(sqlite3_vtab_cursor *pCursor){ return pCsr->aPage==0; } +static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){ + DbdataTable *pTab = (DbdataTable*)pCsr->base.pVtab; + char *zSql = 0; + int rc, rc2; + sqlite3_stmt *pStmt = 0; + + zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema); + if( zSql==0 ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + pCsr->szDb = sqlite3_column_int(pStmt, 0); + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + return rc; +} + /* Position a cursor back to the beginning. */ static int dbdataFilter( @@ -594,16 +621,21 @@ static int dbdataFilter( if( idxNum & 0x02 ){ pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]); pCsr->bOnePage = 1; + }else{ + pCsr->nPage = dbdataDbsize(pCsr, zSchema); + rc = dbdataDbsize(pCsr, zSchema); } - if( pTab->pStmt ){ - pCsr->pStmt = pTab->pStmt; - pTab->pStmt = 0; - }else{ - rc = sqlite3_prepare_v2(pTab->db, - "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1, - &pCsr->pStmt, 0 - ); + if( rc==SQLITE_OK ){ + if( pTab->pStmt ){ + pCsr->pStmt = pTab->pStmt; + pTab->pStmt = 0; + }else{ + rc = sqlite3_prepare_v2(pTab->db, + "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1, + &pCsr->pStmt, 0 + ); + } } if( rc==SQLITE_OK ){ rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT); diff --git a/manifest b/manifest index c6b5a05076..d361dc7ebc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\sinto\sthis\sbranch. -D 2019-04-25T20:06:34.069 +C Fix\sa\slocking-page\srelated\sproblem\swith\sthe\s".recover"\scommand. +D 2019-04-26T15:14:53.514 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -284,7 +284,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c cec672d40604075bb341a7f11ac48393efdcd90a979269b8fe7977ea62d0547f F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189 F ext/misc/csv.c 7f047aeb68f5802e7ce6639292095d622a488bb43526ed04810e0649faa71ceb -F ext/misc/dbdata.c 0c80f0757c3a1b5c57027328ab0ccfe30fcef1a4875b41dd9ddc0f53c7087e97 +F ext/misc/dbdata.c b7547f43906f9296e43be807ca78e2ef3f335ca26d6e91d178df30cd2fd46572 F ext/misc/dbdump.c baf6e37447c9d6968417b1cd34cbedb0b0ab3f91b5329501d8a8d5be3287c336 F ext/misc/eval.c 4b4757592d00fd32e44c7a067e6a0e4839c81a4d57abc4131ee7806d1be3104e F ext/misc/explain.c d5c12962d79913ef774b297006872af1fccda388f61a11d37758f9179a09551f @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c b7304d2f491c11a03a7fbdf34bc218282ac54052377809d4dc3b4b1e7f4bfc93 -F src/shell.c.in 78004b100ed486695f4ff8bbc40b99c2e4d7b6d06278ce64b80ac1900bf356a3 +F src/shell.c.in 63b1075817997806d7f09c2a0433ffac854b6dee47836ef938d51531cdb8042f F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 8d2f52bb640d6d0f84b18d746043e56f45a73ace93239be1d036701f7f4018fd 7be6222c9ec44596e4eddd906c831eb1272b90fbdf68641d791f216264feb7cf -R af342ccd780cc18bf46aaaf7890063fa +P 1da302d85d7ad4ba54f877117a45d667439fd2ef31dc70ea1d54dc1fba196e68 +R 981714fd52a70e51cf4586f924a49fdc U dan -Z 147b326af09d7b23ccf3f83204577057 +Z 303a0ea3c8fcc86a61370f3e12d83ec5 diff --git a/manifest.uuid b/manifest.uuid index 0d49487589..b5f67ee780 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1da302d85d7ad4ba54f877117a45d667439fd2ef31dc70ea1d54dc1fba196e68 \ No newline at end of file +afdae10424f0f3d0f10a4b73e9732aa55c5ee664814d8ca0edd372cfb17c2445 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 55f7964319..1884e7bfe2 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6165,6 +6165,22 @@ static void shellExec(sqlite3 *db, int *pRc, const char *zSql){ } } +static void shellExecPrintf(sqlite3 *db, int *pRc, const char *zFmt, ...){ + char *z = 0; + if( *pRc==SQLITE_OK ){ + va_list ap; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( z==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + shellExec(db, pRc, z); + } + sqlite3_free(z); + } +} + static void *shellMalloc(int *pRc, sqlite3_int64 nByte){ void *pRet = 0; if( *pRc==SQLITE_OK ){ @@ -6405,6 +6421,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ sqlite3_stmt *pLoop = 0; /* Loop through all root pages */ sqlite3_stmt *pPages = 0; /* Loop through all pages in a group */ sqlite3_stmt *pCells = 0; /* Loop through all cells in a page */ + const char *zRecoveryDb = ""; /* Name of "recovery" database */ int i; int bFreelist = 1; /* 0 if --freelist-corrupt is specified */ @@ -6416,23 +6433,32 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ if( n<=17 && memcmp("-freelist-corrupt", z, n)==0 ){ bFreelist = 0; } + if( n<=12 && memcmp("-recovery-db", z, n)==0 && i<(nArg-1) ){ + i++; + zRecoveryDb = azArg[i]; + } else{ - raw_printf(stderr, - "unexpected option: %s - expected \"--freelist-corrupt\"\n", - azArg[i] - ); + raw_printf(stderr, "unexpected option: %s\n", azArg[i]); + raw_printf(stderr, "options are:\n"); + raw_printf(stderr, " --freelist-corrupt\n"); + raw_printf(stderr, " --recovery-db DATABASE\n"); return 1; } } - shellExec(pState->db, &rc, + shellExecPrintf(pState->db, &rc, /* Attach an in-memory database named 'recovery'. Create an indexed ** cache of the sqlite_dbptr virtual table. */ - "ATTACH '' AS recovery;" + "ATTACH %Q AS recovery;" + "DROP TABLE IF EXISTS recovery.dbptr;" + "DROP TABLE IF EXISTS recovery.freelist;" + "DROP TABLE IF EXISTS recovery.map;" + "DROP TABLE IF EXISTS recovery.schema;" "CREATE TABLE recovery.dbptr(" " pgno, child, PRIMARY KEY(child, pgno)" ") WITHOUT ROWID;" - "INSERT OR IGNORE INTO dbptr(pgno, child) SELECT * FROM sqlite_dbptr;" + "INSERT OR IGNORE INTO recovery.dbptr(pgno, child) " + " SELECT * FROM sqlite_dbptr;" /* Delete any pointer to page 1. This ensures that page 1 is considered ** a root page, regardless of how corrupt the db is. */ @@ -6445,7 +6471,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ " SELECT child FROM recovery.dbptr GROUP BY child HAVING count(*)>1" ");" - "CREATE TABLE recovery.freelist(pgno INTEGER PRIMARY KEY);" + "CREATE TABLE recovery.freelist(pgno INTEGER PRIMARY KEY);", zRecoveryDb ); if( bFreelist ){ From ca424382613ed62e80b54734231a1d1d8dd916bd Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 26 Apr 2019 15:40:27 +0000 Subject: [PATCH 11/38] Fix another problem with database freelist handling in the ".recover" command. FossilOrigin-Name: bee2652ac26370e612a8c81dd7554befc2d523442a2fbbc77dc73479e6a0d7fd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 35 ++++++++++++++++++----------------- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index d361dc7ebc..2ea5b39962 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\slocking-page\srelated\sproblem\swith\sthe\s".recover"\scommand. -D 2019-04-26T15:14:53.514 +C Fix\sanother\sproblem\swith\sdatabase\sfreelist\shandling\sin\sthe\s".recover"\scommand. +D 2019-04-26T15:40:27.902 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c b7304d2f491c11a03a7fbdf34bc218282ac54052377809d4dc3b4b1e7f4bfc93 -F src/shell.c.in 63b1075817997806d7f09c2a0433ffac854b6dee47836ef938d51531cdb8042f +F src/shell.c.in 44dce9f5b3c68c07e30db740aa4e635819ab6fa01a229a21356ddd4fa8f1e47e F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1da302d85d7ad4ba54f877117a45d667439fd2ef31dc70ea1d54dc1fba196e68 -R 981714fd52a70e51cf4586f924a49fdc +P afdae10424f0f3d0f10a4b73e9732aa55c5ee664814d8ca0edd372cfb17c2445 +R 5b76306d65317b1b0fee3a74fc0359d7 U dan -Z 303a0ea3c8fcc86a61370f3e12d83ec5 +Z 068a6ba7f1f2eaa523c5f432cd2b8c0c diff --git a/manifest.uuid b/manifest.uuid index b5f67ee780..4c5a229461 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -afdae10424f0f3d0f10a4b73e9732aa55c5ee664814d8ca0edd372cfb17c2445 \ No newline at end of file +bee2652ac26370e612a8c81dd7554befc2d523442a2fbbc77dc73479e6a0d7fd \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 1884e7bfe2..c417160d52 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6454,23 +6454,6 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ "DROP TABLE IF EXISTS recovery.freelist;" "DROP TABLE IF EXISTS recovery.map;" "DROP TABLE IF EXISTS recovery.schema;" - "CREATE TABLE recovery.dbptr(" - " pgno, child, PRIMARY KEY(child, pgno)" - ") WITHOUT ROWID;" - "INSERT OR IGNORE INTO recovery.dbptr(pgno, child) " - " SELECT * FROM sqlite_dbptr;" - - /* Delete any pointer to page 1. This ensures that page 1 is considered - ** a root page, regardless of how corrupt the db is. */ - "DELETE FROM recovery.dbptr WHERE child = 1;" - - /* Delete all pointers to any pages that have more than one pointer - ** to them. Such pages will be treated as root pages when recovering - ** data. */ - "DELETE FROM recovery.dbptr WHERE child IN (" - " SELECT child FROM recovery.dbptr GROUP BY child HAVING count(*)>1" - ");" - "CREATE TABLE recovery.freelist(pgno INTEGER PRIMARY KEY);", zRecoveryDb ); @@ -6497,6 +6480,24 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ } shellExec(pState->db, &rc, + "CREATE TABLE recovery.dbptr(" + " pgno, child, PRIMARY KEY(child, pgno)" + ") WITHOUT ROWID;" + "INSERT OR IGNORE INTO recovery.dbptr(pgno, child) " + " SELECT * FROM sqlite_dbptr" + " WHERE pgno NOT IN freelist AND child NOT IN freelist;" + + /* Delete any pointer to page 1. This ensures that page 1 is considered + ** a root page, regardless of how corrupt the db is. */ + "DELETE FROM recovery.dbptr WHERE child = 1;" + + /* Delete all pointers to any pages that have more than one pointer + ** to them. Such pages will be treated as root pages when recovering + ** data. */ + "DELETE FROM recovery.dbptr WHERE child IN (" + " SELECT child FROM recovery.dbptr GROUP BY child HAVING count(*)>1" + ");" + /* Create the "map" table that will (eventually) contain instructions ** for dealing with each page in the db that contains one or more ** records. */ From db5ba5c385496191770a418e27607522474e611c Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 26 Apr 2019 17:08:50 +0000 Subject: [PATCH 12/38] Omit tests of the LIKE optimization in like3.test when SQLITE_ENABLE_ICU is defined. FossilOrigin-Name: af53c41a127c314c0608f3fd016d3a26896783745e46cd180976a188400cdb75 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/like3.test | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 261ca06463..158d60af72 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\stest\scases\sin\stest/fuzzdata8.db. -D 2019-04-24T17:04:02.548 +C Omit\stests\sof\sthe\sLIKE\soptimization\sin\slike3.test\swhen\sSQLITE_ENABLE_ICU\sis\sdefined. +D 2019-04-26T17:08:50.839 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1085,7 +1085,7 @@ F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/lemon-test01.y 58b764610fd934e189ffbb0bbfa33d171b9cb06019b55bdc04d090d6767e11d7 F test/like.test 11cfd7d4ef8625389df9efc46735ff0b0b41d5e62047ef0f3bc24c380d28a7a6 F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da -F test/like3.test 0ce2630e39e32e42ce02d171f0a315189ca71fec37c5ddfb0191eecc3fe9d4da +F test/like3.test d3684b5c60dd2d1f9574f9f296b31c995187c8c10c6ca573f9918c96af9ba642 F test/limit.test 0c99a27a87b14c646a9d583c7c89fd06c352663e F test/limit2.test 9409b033284642a859fafc95f29a5a6a557bd57c1f0d7c3f554bd64ed69df77e F test/loadext.test faa4f6eed07a5aac35d57fdd7bc07f8fc82464cfd327567c10cf0ba3c86cde04 @@ -1818,7 +1818,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P e1724f1d618cfbcfd1e495d8965a395656cfc1114e1bffd4bc3be0bd5cdb6550 -R 3e1175ca2cd3d699a0989ce182ed2c32 -U drh -Z b7a4bb53a6677ebeb4f1be668326c373 +P 7be6222c9ec44596e4eddd906c831eb1272b90fbdf68641d791f216264feb7cf +R 1ba2bc865b250121afcb0af52a507e48 +U dan +Z 432d74fc77b640bcf830ac2298b92cb0 diff --git a/manifest.uuid b/manifest.uuid index 7b9871a25b..0646b8052e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7be6222c9ec44596e4eddd906c831eb1272b90fbdf68641d791f216264feb7cf \ No newline at end of file +af53c41a127c314c0608f3fd016d3a26896783745e46cd180976a188400cdb75 \ No newline at end of file diff --git a/test/like3.test b/test/like3.test index 622f8335eb..89c9efa792 100644 --- a/test/like3.test +++ b/test/like3.test @@ -182,6 +182,7 @@ do_eqp_test like3-5.211 { # Verify that the LIKE optimization works with an ESCAPE clause when # using PRAGMA case_sensitive_like=ON. # +ifcapable !icu { do_execsql_test like3-6.100 { DROP TABLE IF EXISTS t1; CREATE TABLE t1(path TEXT COLLATE nocase PRIMARY KEY,a,b,c) WITHOUT ROWID; @@ -229,5 +230,6 @@ do_eqp_test like3-6.240 { QUERY PLAN `--SEARCH TABLE t2 USING INDEX t2path2 (path>? AND path Date: Fri, 26 Apr 2019 21:11:37 +0000 Subject: [PATCH 13/38] Have .recover store all orphaned rows in a single table, with extra columns to indicate the orphaned page and sub-tree they were discovered within. FossilOrigin-Name: 7221f6e33ed6a5a96ec61e25f2a1f70b84aae66e503d897eb7b7ff1aec42355d --- manifest | 12 +-- manifest.uuid | 2 +- src/shell.c.in | 209 +++++++++++++++++++++++++++++-------------------- 3 files changed, 133 insertions(+), 90 deletions(-) diff --git a/manifest b/manifest index 2ea5b39962..8404fd782b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sanother\sproblem\swith\sdatabase\sfreelist\shandling\sin\sthe\s".recover"\scommand. -D 2019-04-26T15:40:27.902 +C Have\s.recover\sstore\sall\sorphaned\srows\sin\sa\ssingle\stable,\swith\sextra\scolumns\sto\sindicate\sthe\sorphaned\spage\sand\ssub-tree\sthey\swere\sdiscovered\swithin. +D 2019-04-26T21:11:37.615 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c b7304d2f491c11a03a7fbdf34bc218282ac54052377809d4dc3b4b1e7f4bfc93 -F src/shell.c.in 44dce9f5b3c68c07e30db740aa4e635819ab6fa01a229a21356ddd4fa8f1e47e +F src/shell.c.in 146dbc2708a314556710f8e8b13226d647fbb70778c973bb6fd655186811ce63 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P afdae10424f0f3d0f10a4b73e9732aa55c5ee664814d8ca0edd372cfb17c2445 -R 5b76306d65317b1b0fee3a74fc0359d7 +P bee2652ac26370e612a8c81dd7554befc2d523442a2fbbc77dc73479e6a0d7fd +R 52eb405413df1bbf1e7b369ddce62695 U dan -Z 068a6ba7f1f2eaa523c5f432cd2b8c0c +Z 87ebd735a2a8d2115b37082d2935a00e diff --git a/manifest.uuid b/manifest.uuid index 4c5a229461..3cfd1fa139 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bee2652ac26370e612a8c81dd7554befc2d523442a2fbbc77dc73479e6a0d7fd \ No newline at end of file +7221f6e33ed6a5a96ec61e25f2a1f70b84aae66e503d897eb7b7ff1aec42355d \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index c417160d52..0737a347a0 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6226,7 +6226,7 @@ static void recoverFreeTable(RecoverTable *pTab){ sqlite3_free(pTab->zQuoted); if( pTab->azlCol ){ int i; - for(i=0; inCol; i++){ + for(i=0; i<=pTab->nCol; i++){ sqlite3_free(pTab->azlCol[i]); } sqlite3_free(pTab->azlCol); @@ -6235,9 +6235,8 @@ static void recoverFreeTable(RecoverTable *pTab){ } } -static void recoverOldTable( +static RecoverTable *recoverOldTable( int *pRc, /* IN/OUT: Error code */ - RecoverTable *pTab, const char *zName, /* Name of table */ const char *zSql, /* CREATE TABLE statement */ int bIntkey, @@ -6245,12 +6244,14 @@ static void recoverOldTable( ){ sqlite3 *dbtmp = 0; /* sqlite3 handle for testing CREATE TABLE */ int rc = *pRc; + RecoverTable *pTab = 0; + pTab = (RecoverTable*)shellMalloc(&rc, sizeof(RecoverTable)); if( rc==SQLITE_OK ){ int nSqlCol = 0; int bSqlIntkey = 0; sqlite3_stmt *pStmt = 0; - + rc = sqlite3_open("", &dbtmp); if( rc==SQLITE_OK ){ rc = sqlite3_exec(dbtmp, "PRAGMA writable_schema = on", 0, 0, 0); @@ -6285,43 +6286,48 @@ static void recoverOldTable( shellFinalize(&rc, pStmt); if( bIntkey==bSqlIntkey ){ + int i; const char *zPk = "_rowid_"; sqlite3_stmt *pPkFinder = 0; - shellPreparePrintf(dbtmp, &rc, &pPkFinder, + pTab->iPk = -2; + if( bIntkey ){ + shellPreparePrintf(dbtmp, &rc, &pPkFinder, "SELECT cid, name FROM pragma_table_info(%Q) " " WHERE pk=1 AND type='integer' COLLATE nocase" - " AND NOT EXISTS (SELECT cid FROM pragma_table_info(%Q) WHERE pk=2)", - zName, zName - ); - if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPkFinder) ){ - pTab->iPk = sqlite3_column_int(pPkFinder, 0); - zPk = (const char*)sqlite3_column_text(pPkFinder, 1); + " AND NOT EXISTS (SELECT cid FROM pragma_table_info(%Q) WHERE pk=2)" + , zName, zName + ); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPkFinder) ){ + pTab->iPk = sqlite3_column_int(pPkFinder, 0); + zPk = (const char*)sqlite3_column_text(pPkFinder, 1); + } } pTab->zName = shellMPrintf(&rc, "%s", zName); pTab->zQuoted = shellMPrintf(&rc, "%Q", pTab->zName); - pTab->azlCol = (char**)shellMalloc(&rc, sizeof(char*) * nSqlCol); + pTab->azlCol = (char**)shellMalloc(&rc, sizeof(char*) * (nSqlCol+1)); pTab->nCol = nSqlCol; - if( nSqlCol==1 && pTab->iPk==0 ){ + if( bIntkey ){ pTab->azlCol[0] = shellMPrintf(&rc, "%Q", zPk); }else{ - shellPreparePrintf(dbtmp, &rc, &pStmt, - "SELECT -1+row_number() OVER (ORDER BY cid)," - " %Q||%Q||group_concat(name, ', ') FILTER (WHERE cid!=%d) " - " OVER (ORDER BY cid) " - "FROM pragma_table_info(%Q)", - (bIntkey ? zPk : ""), (bIntkey ? ", " : ""), - pTab->iPk, zName - ); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - int idx = sqlite3_column_int(pStmt, 0); - const char *zText = (const char*)sqlite3_column_text(pStmt, 1); - pTab->azlCol[idx] = shellMPrintf(&rc, "%s", zText); - } - shellFinalize(&rc, pStmt); + pTab->azlCol[0] = shellMPrintf(&rc, ""); } + i = 1; + shellPreparePrintf(dbtmp, &rc, &pStmt, + "SELECT %Q || group_concat(name, ', ') " + " FILTER (WHERE cid!=%d) OVER (ORDER BY cid) " + "FROM pragma_table_info(%Q)", + bIntkey ? ", " : "", pTab->iPk, zName + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zText = (const char*)sqlite3_column_text(pStmt, 0); + pTab->azlCol[i] = shellMPrintf(&rc, "%s%s", pTab->azlCol[0], zText); + i++; + } + shellFinalize(&rc, pStmt); + shellFinalize(&rc, pPkFinder); } } @@ -6329,6 +6335,11 @@ static void recoverOldTable( finished: sqlite3_close(dbtmp); *pRc = rc; + if( rc!=SQLITE_OK ){ + recoverFreeTable(pTab); + pTab = 0; + } + return pTab; } static RecoverTable *recoverNewTable( @@ -6336,7 +6347,8 @@ static RecoverTable *recoverNewTable( int *pRc, int iRoot, int bIntkey, - int nCol + int nCol, + int *pbNoop ){ sqlite3_stmt *pStmt = 0; RecoverTable *pRet = 0; @@ -6344,8 +6356,6 @@ static RecoverTable *recoverNewTable( const char *zSql = 0; const char *zName = 0; - pRet = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable)); - if( pRet ) pRet->iPk = -2; /* Search the recovered schema for an object with root page iRoot. */ shellPreparePrintf(pState->db, pRc, &pStmt, @@ -6360,55 +6370,56 @@ static RecoverTable *recoverNewTable( if( sqlite3_stricmp(zType, "table")==0 ){ zName = (const char*)sqlite3_column_text(pStmt, 1); zSql = (const char*)sqlite3_column_text(pStmt, 2); - recoverOldTable(pRc, pRet, zName, zSql, bIntkey, nCol); + pRet = recoverOldTable(pRc, zName, zSql, bIntkey, nCol); break; } } + shellFinalize(pRc, pStmt); - if( bNoop ){ - sqlite3_free(pRet); - return 0; - } + *pbNoop = bNoop; + return pRet; +} - if( pRet && pRet->zName==0 ){ - sqlite3_stmt *pStmt = 0; - - pRet->zName = shellMPrintf(pRc, "orphan_%d_%d", nCol, iRoot); - pRet->zQuoted = shellMPrintf(pRc, "%Q", pRet->zName); - pRet->azlCol = (char**)shellMalloc(pRc, sizeof(char*) * nCol); - pRet->nCol = nCol; - - shellPreparePrintf(pState->db, pRc, &pStmt, - "WITH s(i) AS (" - " SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<%d" - ")" - "SELECT i-1, %Q || group_concat('c' || i, ', ') OVER (ORDER BY i) FROM s", - nCol, (bIntkey ? "id, " : "") +static RecoverTable *recoverOrphanTable( + ShellState *pState, + int *pRc, + int nCol +){ + RecoverTable *pTab = 0; + if( nCol>=0 && *pRc==SQLITE_OK ){ + int i; + raw_printf(pState->out, + "CREATE TABLE recover_orphan(rootpgno INTEGER, " + "pgno INTEGER, nfield INTEGER, id INTEGER" ); - while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ - int idx = sqlite3_column_int(pStmt, 0); - const char *zText = (const char*)sqlite3_column_text(pStmt, 1); - pRet->azlCol[idx] = shellMPrintf(pRc, "%s", zText); + for(i=0; iout, ", c%d", i); } - shellFinalize(pRc, pStmt); + raw_printf(pState->out, ");\n"); - if( *pRc==SQLITE_OK ){ - char *zCreate = shellMPrintf(pRc, "CREATE TABLE %Q (%s)", - pRet->zName, pRet->azlCol[nCol-1] - ); - if( zCreate ){ - raw_printf(pState->out, "%s;\n", zCreate); - sqlite3_free(zCreate); + pTab = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable)); + if( pTab ){ + pTab->zName = shellMPrintf(pRc, "%s", "recover_orphan"); + pTab->zQuoted = shellMPrintf(pRc, "%Q", pTab->zName); + pTab->nCol = nCol; + pTab->iPk = -2; + if( nCol>0 ){ + pTab->azlCol = (char**)shellMalloc(pRc, sizeof(char*) * (nCol+1)); + if( pTab->azlCol ){ + pTab->azlCol[nCol] = shellMPrintf(pRc, ""); + for(i=nCol-1; i>=0; i--){ + pTab->azlCol[i] = shellMPrintf(pRc, "%s, NULL", pTab->azlCol[i+1]); + } + } } } - } - if( *pRc!=SQLITE_OK ){ - recoverFreeTable(pRet); - pRet = 0; + if( *pRc!=SQLITE_OK ){ + recoverFreeTable(pTab); + pTab = 0; + } } - - return pRet; + return pTab; } /* @@ -6423,6 +6434,8 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ sqlite3_stmt *pCells = 0; /* Loop through all cells in a page */ const char *zRecoveryDb = ""; /* Name of "recovery" database */ int i; + int nOrphan = -1; + RecoverTable *pOrphan = 0; int bFreelist = 1; /* 0 if --freelist-corrupt is specified */ for(i=1; idb, &rc, + "SELECT coalesce(max(maxlen), -2) FROM recovery.map" + " WHERE root>1 AND root NOT IN (SELECT rootpage FROM recovery.schema)" + , &pLoop + ); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ + nOrphan = sqlite3_column_int(pLoop, 0); + } + shellFinalize(&rc, pLoop); + pLoop = 0; + pOrphan = recoverOrphanTable(pState, &rc, nOrphan); + shellPrepare(pState->db, &rc, "SELECT pgno FROM recovery.map WHERE root=?", &pPages ); @@ -6586,33 +6614,48 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ int iRoot = sqlite3_column_int(pLoop, 0); int bIntkey = sqlite3_column_int(pLoop, 1); int nCol = sqlite3_column_int(pLoop, 2); + int bNoop = 0; RecoverTable *pTab; - pTab = recoverNewTable(pState, &rc, iRoot, bIntkey, nCol); - if( pTab ){ - if( 0==sqlite3_stricmp(pTab->zName, "sqlite_sequence") ){ - raw_printf(pState->out, "DELETE FROM sqlite_sequence;\n"); - } - sqlite3_bind_int(pPages, 1, iRoot); - sqlite3_bind_int(pCells, 2, pTab->iPk); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPages) ){ - sqlite3_bind_int(pCells, 1, sqlite3_column_int(pPages, 0)); - while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pCells) ){ - int iMax = sqlite3_column_int(pCells, 0); - const char *zVal = (const char*)sqlite3_column_text(pCells, 1); + pTab = recoverNewTable(pState, &rc, iRoot, bIntkey, nCol, &bNoop); + if( bNoop || rc ) continue; + if( pTab==0 ) pTab = pOrphan; + + if( 0==sqlite3_stricmp(pTab->zName, "sqlite_sequence") ){ + raw_printf(pState->out, "DELETE FROM sqlite_sequence;\n"); + } + sqlite3_bind_int(pPages, 1, iRoot); + sqlite3_bind_int(pCells, 2, pTab->iPk); + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPages) ){ + int iPgno = sqlite3_column_int(pPages, 0); + sqlite3_bind_int(pCells, 1, iPgno); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pCells) ){ + int nField = sqlite3_column_int(pCells, 0); + const char *zVal = (const char*)sqlite3_column_text(pCells, 1); + + nField = nField+1; + if( pTab==pOrphan ){ + raw_printf(pState->out, + "INSERT INTO %s VALUES(%d, %d, %d, %s%s%s);\n", + pTab->zQuoted, iRoot, iPgno, nField, + bIntkey ? "" : "NULL, ", zVal, pTab->azlCol[nField] + ); + }else{ raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", - pTab->zQuoted, pTab->azlCol[iMax>0?iMax:0], zVal + pTab->zQuoted, pTab->azlCol[nField], zVal ); } - shellReset(&rc, pCells); } - shellReset(&rc, pPages); + shellReset(&rc, pCells); } - recoverFreeTable(pTab); + shellReset(&rc, pPages); + if( pTab!=pOrphan ) recoverFreeTable(pTab); } shellFinalize(&rc, pLoop); shellFinalize(&rc, pPages); shellFinalize(&rc, pCells); + recoverFreeTable(pOrphan); /* The rest of the schema */ if( rc==SQLITE_OK ){ From f57bea31bad6081613945a4a344e6859cd189de1 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 27 Apr 2019 15:35:45 +0000 Subject: [PATCH 14/38] Fix a problem in the .recover command with recovering WITHOUT ROWID tables where the PK columns are not the leftmost in the CREATE TABLE statement. FossilOrigin-Name: 91df4b8e0386105d01614921e8410994b621404a3d46ec4af8687b8743c52d52 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 11 +++++++++-- test/recover.test | 11 +++++++++++ 4 files changed, 28 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 8404fd782b..94b2e05ed0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\s.recover\sstore\sall\sorphaned\srows\sin\sa\ssingle\stable,\swith\sextra\scolumns\sto\sindicate\sthe\sorphaned\spage\sand\ssub-tree\sthey\swere\sdiscovered\swithin. -D 2019-04-26T21:11:37.615 +C Fix\sa\sproblem\sin\sthe\s.recover\scommand\swith\srecovering\sWITHOUT\sROWID\stables\swhere\sthe\sPK\scolumns\sare\snot\sthe\sleftmost\sin\sthe\sCREATE\sTABLE\sstatement. +D 2019-04-27T15:35:45.828 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c b7304d2f491c11a03a7fbdf34bc218282ac54052377809d4dc3b4b1e7f4bfc93 -F src/shell.c.in 146dbc2708a314556710f8e8b13226d647fbb70778c973bb6fd655186811ce63 +F src/shell.c.in 3701177f3821330c8eb2af96f60123245cf42273abdae472bcb96bb120dcba8f F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1226,7 +1226,7 @@ F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 -F test/recover.test a8fed5acf2742268e366abb76a01de6ddce278aae01658488b950c968ebef638 +F test/recover.test bfeb5ab4574f9a264b3893ce0e41f04c2052b72b174e5dd9d847b6e7b8f4d15c F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/regexp2.test 40e894223b3d6672655481493f1be12012f2b33c F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P bee2652ac26370e612a8c81dd7554befc2d523442a2fbbc77dc73479e6a0d7fd -R 52eb405413df1bbf1e7b369ddce62695 +P 7221f6e33ed6a5a96ec61e25f2a1f70b84aae66e503d897eb7b7ff1aec42355d +R e71b5bc56d773e53b87820c68d2be836 U dan -Z 87ebd735a2a8d2115b37082d2935a00e +Z d098c243e647f537ccde95cbf714168f diff --git a/manifest.uuid b/manifest.uuid index 3cfd1fa139..6b5834e3bb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7221f6e33ed6a5a96ec61e25f2a1f70b84aae66e503d897eb7b7ff1aec42355d \ No newline at end of file +91df4b8e0386105d01614921e8410994b621404a3d46ec4af8687b8743c52d52 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 0737a347a0..f906205bff 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6290,6 +6290,11 @@ static RecoverTable *recoverOldTable( const char *zPk = "_rowid_"; sqlite3_stmt *pPkFinder = 0; + /* If this is an intkey table and there is an INTEGER PRIMARY KEY, + ** set zPk to the name of the PK column, and pTab->iPk to the index + ** of the column, where columns are 0-numbered from left to right. + ** Or, if this is a WITHOUT ROWID table or if there is no IPK column, + ** leave zPk as "_rowid_" and pTab->iPk at -2. */ pTab->iPk = -2; if( bIntkey ){ shellPreparePrintf(dbtmp, &rc, &pPkFinder, @@ -6317,9 +6322,11 @@ static RecoverTable *recoverOldTable( i = 1; shellPreparePrintf(dbtmp, &rc, &pStmt, "SELECT %Q || group_concat(name, ', ') " - " FILTER (WHERE cid!=%d) OVER (ORDER BY cid) " + " FILTER (WHERE cid!=%d) OVER (ORDER BY %s cid) " "FROM pragma_table_info(%Q)", - bIntkey ? ", " : "", pTab->iPk, zName + bIntkey ? ", " : "", pTab->iPk, + bIntkey ? "" : "(CASE WHEN pk=0 THEN 1000000 ELSE pk END), ", + zName ); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ const char *zText = (const char*)sqlite3_column_text(pStmt, 0); diff --git a/test/recover.test b/test/recover.test index 6ee35054f8..4de7f503a5 100644 --- a/test/recover.test +++ b/test/recover.test @@ -85,4 +85,15 @@ do_execsql_test 1.3.1 " " do_recover_test 1.3.2 +#------------------------------------------------------------------------- +reset_db +do_execsql_test 2.1.0 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(b, c)) WITHOUT ROWID; + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1 VALUES(7, 8, 9); +} + +do_recover_test 2.1.1 + finish_test From 42ebb01e9fd94c94d4f90b0a59b46194aea28400 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 27 Apr 2019 18:47:03 +0000 Subject: [PATCH 15/38] Add the "--lost-and-found" option to the ".recover" command. For setting the name of the orphaned rows table. FossilOrigin-Name: 67bb88e24c74d02ae0c4ac6ff2f873f6b0035ccefe5cccfc71c5686cbc76b4c3 --- ext/misc/dbdata.c | 131 ++++++++++++++++++++++++++-------------- manifest | 16 ++--- manifest.uuid | 2 +- src/shell.c.in | 150 ++++++++++++++++++++++++++++++++-------------- test/recover.test | 36 ++++++++++- 5 files changed, 234 insertions(+), 101 deletions(-) diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c index a27f19dc7a..786369ce0a 100644 --- a/ext/misc/dbdata.c +++ b/ext/misc/dbdata.c @@ -10,10 +10,14 @@ ** ****************************************************************************** ** -** This file contains an implementation of the eponymous "sqlite_dbdata" -** virtual table. sqlite_dbdata is used to extract data directly from a -** database b-tree page and its associated overflow pages, bypassing the b-tree -** layer. The table schema is equivalent to: +** This file contains an implementation of two eponymous virtual tables, +** "sqlite_dbdata" and "sqlite_dbptr". Both modules require that the +** "sqlite_dbpage" eponymous virtual table be available. +** +** SQLITE_DBDATA: +** sqlite_dbdata is used to extract data directly from a database b-tree +** page and its associated overflow pages, bypassing the b-tree layer. +** The table schema is equivalent to: ** ** CREATE TABLE sqlite_dbdata( ** pgno INTEGER, @@ -23,23 +27,25 @@ ** schema TEXT HIDDEN ** ); ** -** IMPORTANT: THE VIRTUAL TABLE SCHEMA ABOVE IS SUBJECT TO CHANGE. IN THE -** FUTURE NEW NON-HIDDEN COLUMNS MAY BE ADDED BETWEEN "value" AND "schema". +** IMPORTANT: THE VIRTUAL TABLE SCHEMA ABOVE IS SUBJECT TO CHANGE. IN THE +** FUTURE NEW NON-HIDDEN COLUMNS MAY BE ADDED BETWEEN "value" AND +** "schema". ** -** Each page of the database is inspected. If it cannot be interpreted as a -** b-tree page, or if it is a b-tree page containing 0 entries, the -** sqlite_dbdata table contains no rows for that page. Otherwise, the table -** contains one row for each field in the record associated with each -** cell on the page. For intkey b-trees, the key value is stored in field -1. +** Each page of the database is inspected. If it cannot be interpreted as +** a b-tree page, or if it is a b-tree page containing 0 entries, the +** sqlite_dbdata table contains no rows for that page. Otherwise, the +** table contains one row for each field in the record associated with +** each cell on the page. For intkey b-trees, the key value is stored in +** field -1. ** -** For example, for the database: +** For example, for the database: ** ** CREATE TABLE t1(a, b); -- root page is page 2 ** INSERT INTO t1(rowid, a, b) VALUES(5, 'v', 'five'); ** INSERT INTO t1(rowid, a, b) VALUES(10, 'x', 'ten'); ** -** the sqlite_dbdata table contains, as well as from entries related to -** page 1, content equivalent to: +** the sqlite_dbdata table contains, as well as from entries related to +** page 1, content equivalent to: ** ** INSERT INTO sqlite_dbdata(pgno, cell, field, value) VALUES ** (2, 0, -1, 5 ), @@ -49,19 +55,21 @@ ** (2, 1, 0, 'x' ), ** (2, 1, 1, 'ten' ); ** -** If database corruption is encountered, this module does not report an -** error. Instead, it attempts to extract as much data as possible and -** ignores the corruption. -** -** This module requires that the "sqlite_dbpage" eponymous virtual table be -** available. +** If database corruption is encountered, this module does not report an +** error. Instead, it attempts to extract as much data as possible and +** ignores the corruption. ** +** SQLITE_DBPTR: +** The sqlite_dbptr table has the following schema: ** ** CREATE TABLE sqlite_dbptr( ** pgno INTEGER, ** child INTEGER, ** schema TEXT HIDDEN ** ); +** +** It contains one entry for each b-tree pointer between a parent and +** child page in the database. */ #if !defined(SQLITEINT_H) #include "sqlite3ext.h" @@ -77,8 +85,7 @@ SQLITE_EXTENSION_INIT1 typedef struct DbdataTable DbdataTable; typedef struct DbdataCursor DbdataCursor; - -/* A cursor for the sqlite_dbdata table */ +/* Cursor object */ struct DbdataCursor { sqlite3_vtab_cursor base; /* Base class. Must be first */ sqlite3_stmt *pStmt; /* For fetching database pages */ @@ -103,7 +110,7 @@ struct DbdataCursor { sqlite3_int64 iIntkey; /* Integer key value */ }; -/* The sqlite_dbdata table */ +/* Table object */ struct DbdataTable { sqlite3_vtab base; /* Base class. Must be first */ sqlite3 *db; /* The database connection */ @@ -111,16 +118,12 @@ struct DbdataTable { int bPtr; /* True for sqlite3_dbptr table */ }; +/* Column and schema definitions for sqlite_dbdata */ #define DBDATA_COLUMN_PGNO 0 #define DBDATA_COLUMN_CELL 1 #define DBDATA_COLUMN_FIELD 2 #define DBDATA_COLUMN_VALUE 3 #define DBDATA_COLUMN_SCHEMA 4 - -#define DBPTR_COLUMN_PGNO 0 -#define DBPTR_COLUMN_CHILD 1 -#define DBPTR_COLUMN_SCHEMA 2 - #define DBDATA_SCHEMA \ "CREATE TABLE x(" \ " pgno INTEGER," \ @@ -130,6 +133,10 @@ struct DbdataTable { " schema TEXT HIDDEN" \ ")" +/* Column and schema definitions for sqlite_dbptr */ +#define DBPTR_COLUMN_PGNO 0 +#define DBPTR_COLUMN_CHILD 1 +#define DBPTR_COLUMN_SCHEMA 2 #define DBPTR_SCHEMA \ "CREATE TABLE x(" \ " pgno INTEGER," \ @@ -138,7 +145,8 @@ struct DbdataTable { ")" /* -** Connect to the sqlite_dbdata virtual table. +** Connect to an sqlite_dbdata (pAux==0) or sqlite_dbptr (pAux!=0) virtual +** table. */ static int dbdataConnect( sqlite3 *db, @@ -166,7 +174,7 @@ static int dbdataConnect( } /* -** Disconnect from or destroy a dbdata virtual table. +** Disconnect from or destroy a sqlite_dbdata or sqlite_dbptr virtual table. */ static int dbdataDisconnect(sqlite3_vtab *pVtab){ DbdataTable *pTab = (DbdataTable*)pVtab; @@ -178,7 +186,6 @@ static int dbdataDisconnect(sqlite3_vtab *pVtab){ } /* -** ** This function interprets two types of constraints: ** ** schema=? @@ -239,7 +246,7 @@ static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdx){ } /* -** Open a new dbdata cursor. +** Open a new sqlite_dbdata or sqlite_dbptr cursor. */ static int dbdataOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ DbdataCursor *pCsr; @@ -256,6 +263,10 @@ static int dbdataOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ return SQLITE_OK; } +/* +** Restore a cursor object to the state it was in when first allocated +** by dbdataOpen(). +*/ static void dbdataResetCursor(DbdataCursor *pCsr){ DbdataTable *pTab = (DbdataTable*)(pCsr->base.pVtab); if( pTab->pStmt==0 ){ @@ -271,7 +282,7 @@ static void dbdataResetCursor(DbdataCursor *pCsr){ } /* -** Close a dbdata cursor. +** Close an sqlite_dbdata or sqlite_dbptr cursor. */ static int dbdataClose(sqlite3_vtab_cursor *pCursor){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; @@ -280,8 +291,9 @@ static int dbdataClose(sqlite3_vtab_cursor *pCursor){ return SQLITE_OK; } - -/* Decode big-endian integers */ +/* +** Utility methods to decode 16 and 32-bit big-endian unsigned integers. +*/ static unsigned int get_uint16(unsigned char *a){ return (a[0]<<8)|a[1]; } @@ -289,11 +301,21 @@ static unsigned int get_uint32(unsigned char *a){ return (a[0]<<24)|(a[1]<<16)|(a[2]<<8)|a[3]; } +/* +** Load page pgno from the database via the sqlite_dbpage virtual table. +** If successful, set (*ppPage) to point to a buffer containing the page +** data, (*pnPage) to the size of that buffer in bytes and return +** SQLITE_OK. In this case it is the responsibility of the caller to +** eventually free the buffer using sqlite3_free(). +** +** Or, if an error occurs, set both (*ppPage) and (*pnPage) to 0 and +** return an SQLite error code. +*/ static int dbdataLoadPage( - DbdataCursor *pCsr, - u32 pgno, - u8 **ppPage, - int *pnPage + DbdataCursor *pCsr, /* Cursor object */ + u32 pgno, /* Page number of page to load */ + u8 **ppPage, /* OUT: pointer to page buffer */ + int *pnPage /* OUT: Size of (*ppPage) in bytes */ ){ int rc2; int rc = SQLITE_OK; @@ -338,6 +360,10 @@ static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){ return 9; } +/* +** Return the number of bytes of space used by an SQLite value of type +** eType. +*/ static int dbdataValueBytes(int eType){ switch( eType ){ case 0: case 8: case 9: @@ -361,6 +387,10 @@ static int dbdataValueBytes(int eType){ } } +/* +** Load a value of type eType from buffer pData and use it to set the +** result of context object pCtx. +*/ static void dbdataValue(sqlite3_context *pCtx, int eType, u8 *pData){ switch( eType ){ case 0: @@ -411,7 +441,7 @@ static void dbdataValue(sqlite3_context *pCtx, int eType, u8 *pData){ /* -** Move a dbdata cursor to the next entry in the file. +** Move an sqlite_dbdata or sqlite_dbptr cursor to the next entry. */ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; @@ -575,14 +605,20 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ return SQLITE_OK; } -/* We have reached EOF if previous sqlite3_step() returned -** anything other than SQLITE_ROW; +/* +** Return true if the cursor is at EOF. */ static int dbdataEof(sqlite3_vtab_cursor *pCursor){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; return pCsr->aPage==0; } +/* +** Determine the size in pages of database zSchema (where zSchema is +** "main", "temp" or the name of an attached database) and set +** pCsr->szDb accordingly. If successful, return SQLITE_OK. Otherwise, +** an SQLite error code. +*/ static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){ DbdataTable *pTab = (DbdataTable*)pCsr->base.pVtab; char *zSql = 0; @@ -601,7 +637,8 @@ static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){ return rc; } -/* Position a cursor back to the beginning. +/* +** xFilter method for sqlite_dbdata and sqlite_dbptr. */ static int dbdataFilter( sqlite3_vtab_cursor *pCursor, @@ -648,7 +685,9 @@ static int dbdataFilter( return rc; } -/* Return a column for the sqlite_dbdata table */ +/* +** Return a column for the sqlite_dbdata or sqlite_dbptr table. +*/ static int dbdataColumn( sqlite3_vtab_cursor *pCursor, sqlite3_context *ctx, @@ -699,7 +738,9 @@ static int dbdataColumn( return SQLITE_OK; } -/* Return the ROWID for the sqlite_dbdata table */ +/* +** Return the rowid for an sqlite_dbdata or sqlite_dptr table. +*/ static int dbdataRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ DbdataCursor *pCsr = (DbdataCursor*)pCursor; *pRowid = pCsr->iRowid; diff --git a/manifest b/manifest index 94b2e05ed0..2f2f86a8fc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sthe\s.recover\scommand\swith\srecovering\sWITHOUT\sROWID\stables\swhere\sthe\sPK\scolumns\sare\snot\sthe\sleftmost\sin\sthe\sCREATE\sTABLE\sstatement. -D 2019-04-27T15:35:45.828 +C Add\sthe\s"--lost-and-found"\soption\sto\sthe\s".recover"\scommand.\sFor\ssetting\sthe\sname\sof\sthe\sorphaned\srows\stable. +D 2019-04-27T18:47:03.466 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -284,7 +284,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c cec672d40604075bb341a7f11ac48393efdcd90a979269b8fe7977ea62d0547f F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189 F ext/misc/csv.c 7f047aeb68f5802e7ce6639292095d622a488bb43526ed04810e0649faa71ceb -F ext/misc/dbdata.c b7547f43906f9296e43be807ca78e2ef3f335ca26d6e91d178df30cd2fd46572 +F ext/misc/dbdata.c fe978dad2df13dd4b377b5d38f4883282801b18711220a229d0fd266a5deab26 F ext/misc/dbdump.c baf6e37447c9d6968417b1cd34cbedb0b0ab3f91b5329501d8a8d5be3287c336 F ext/misc/eval.c 4b4757592d00fd32e44c7a067e6a0e4839c81a4d57abc4131ee7806d1be3104e F ext/misc/explain.c d5c12962d79913ef774b297006872af1fccda388f61a11d37758f9179a09551f @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c b7304d2f491c11a03a7fbdf34bc218282ac54052377809d4dc3b4b1e7f4bfc93 -F src/shell.c.in 3701177f3821330c8eb2af96f60123245cf42273abdae472bcb96bb120dcba8f +F src/shell.c.in 51f027f6b48fab39e0745915d979747880b7c35827798535726ac44366a291f1 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1226,7 +1226,7 @@ F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 -F test/recover.test bfeb5ab4574f9a264b3893ce0e41f04c2052b72b174e5dd9d847b6e7b8f4d15c +F test/recover.test 52609c8cc24e72d3d8a20fb8bc32ba2ce8ca2093a7f4573bd4f2969f78f6d2b4 F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/regexp2.test 40e894223b3d6672655481493f1be12012f2b33c F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7221f6e33ed6a5a96ec61e25f2a1f70b84aae66e503d897eb7b7ff1aec42355d -R e71b5bc56d773e53b87820c68d2be836 +P 91df4b8e0386105d01614921e8410994b621404a3d46ec4af8687b8743c52d52 +R a9542f620433fc5d37a56caa32ea75c3 U dan -Z d098c243e647f537ccde95cbf714168f +Z 07c2469e7eeced685ce45289494bc94e diff --git a/manifest.uuid b/manifest.uuid index 6b5834e3bb..e6d0f63ea7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -91df4b8e0386105d01614921e8410994b621404a3d46ec4af8687b8743c52d52 \ No newline at end of file +67bb88e24c74d02ae0c4ac6ff2f873f6b0035ccefe5cccfc71c5686cbc76b4c3 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index f906205bff..8c967a83d8 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -1104,7 +1104,6 @@ struct ShellState { #define SHFLG_Newlines 0x00000010 /* .dump --newline flag */ #define SHFLG_CountChanges 0x00000020 /* .changes setting */ #define SHFLG_Echo 0x00000040 /* .echo or --echo setting */ -#define SHFLG_Recover 0x00000080 /* .dump is --recover */ /* ** Macros for testing and setting shellFlgs @@ -3577,6 +3576,7 @@ static const char *(azHelp[]) = { ".prompt MAIN CONTINUE Replace the standard prompts", ".quit Exit this program", ".read FILE Read input from FILE", + ".recover Recover as much data as possible from corrupt db.", ".restore ?DB? FILE Restore content of DB (default \"main\") from FILE", ".save FILE Write in-memory database into FILE", ".scanstats on|off Turn sqlite3_stmt_scanstatus() metrics on or off", @@ -6153,6 +6153,12 @@ end_ar_command: **********************************************************************************/ #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */ +/* +** If (*pRc) is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, the SQL statement or statements in zSql are executed using +** database connection db and the error code written to *pRc before +** this function returns. +*/ static void shellExec(sqlite3 *db, int *pRc, const char *zSql){ int rc = *pRc; if( rc==SQLITE_OK ){ @@ -6165,6 +6171,9 @@ static void shellExec(sqlite3 *db, int *pRc, const char *zSql){ } } +/* +** Like shellExec(), except that zFmt is a printf() style format string. +*/ static void shellExecPrintf(sqlite3 *db, int *pRc, const char *zFmt, ...){ char *z = 0; if( *pRc==SQLITE_OK ){ @@ -6181,6 +6190,12 @@ static void shellExecPrintf(sqlite3 *db, int *pRc, const char *zFmt, ...){ } } +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, an attempt is made to allocate, zero and return a pointer +** to a buffer nByte bytes in size. If an OOM error occurs, *pRc is set +** to SQLITE_NOMEM and NULL returned. +*/ static void *shellMalloc(int *pRc, sqlite3_int64 nByte){ void *pRet = 0; if( *pRc==SQLITE_OK ){ @@ -6194,6 +6209,17 @@ static void *shellMalloc(int *pRc, sqlite3_int64 nByte){ return pRet; } +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, zFmt is treated as a printf() style string. The result of +** formatting it along with any trailing arguments is written into a +** buffer obtained from sqlite3_malloc(), and pointer to which is returned. +** It is the responsibility of the caller to eventually free this buffer +** using a call to sqlite3_free(). +** +** If an OOM error occurs, (*pRc) is set to SQLITE_NOMEM and a NULL +** pointer returned. +*/ static char *shellMPrintf(int *pRc, const char *zFmt, ...){ char *z = 0; if( *pRc==SQLITE_OK ){ @@ -6208,21 +6234,25 @@ static char *shellMPrintf(int *pRc, const char *zFmt, ...){ return z; } +/* +** When running the ".recover" command, each output table, and the special +** orphaned row table if it is required, is represented by an instance +** of the following struct. +*/ typedef struct RecoverTable RecoverTable; struct RecoverTable { - char *zName; /* Name of table */ - char *zQuoted; /* Quoted version of zName */ + char *zQuoted; /* Quoted version of table name */ int nCol; /* Number of columns in table */ char **azlCol; /* Array of column lists */ - int iPk; + int iPk; /* Index of IPK column */ }; /* -** Free a RecoverTable object allocated by recoverNewTable() +** Free a RecoverTable object allocated by recoverFindTable() or +** recoverOrphanTable(). */ static void recoverFreeTable(RecoverTable *pTab){ if( pTab ){ - sqlite3_free(pTab->zName); sqlite3_free(pTab->zQuoted); if( pTab->azlCol ){ int i; @@ -6235,7 +6265,14 @@ static void recoverFreeTable(RecoverTable *pTab){ } } -static RecoverTable *recoverOldTable( +/* +** This function is a no-op if (*pRc) is not SQLITE_OK when it is called. +** Otherwise, it allocates and returns a RecoverTable object based on the +** final four arguments passed to this function. It is the responsibility +** of the caller to eventually free the returned object using +** recoverFreeTable(). +*/ +static RecoverTable *recoverNewTable( int *pRc, /* IN/OUT: Error code */ const char *zName, /* Name of table */ const char *zSql, /* CREATE TABLE statement */ @@ -6309,8 +6346,7 @@ static RecoverTable *recoverOldTable( } } - pTab->zName = shellMPrintf(&rc, "%s", zName); - pTab->zQuoted = shellMPrintf(&rc, "%Q", pTab->zName); + pTab->zQuoted = shellMPrintf(&rc, "%Q", zName); pTab->azlCol = (char**)shellMalloc(&rc, sizeof(char*) * (nSqlCol+1)); pTab->nCol = nSqlCol; @@ -6349,7 +6385,7 @@ static RecoverTable *recoverOldTable( return pTab; } -static RecoverTable *recoverNewTable( +static RecoverTable *recoverFindTable( ShellState *pState, int *pRc, int iRoot, @@ -6363,7 +6399,6 @@ static RecoverTable *recoverNewTable( const char *zSql = 0; const char *zName = 0; - /* Search the recovered schema for an object with root page iRoot. */ shellPreparePrintf(pState->db, pRc, &pStmt, "SELECT type, name, sql FROM recovery.schema WHERE rootpage=%d", iRoot @@ -6377,7 +6412,7 @@ static RecoverTable *recoverNewTable( if( sqlite3_stricmp(zType, "table")==0 ){ zName = (const char*)sqlite3_column_text(pStmt, 1); zSql = (const char*)sqlite3_column_text(pStmt, 2); - pRet = recoverOldTable(pRc, zName, zSql, bIntkey, nCol); + pRet = recoverNewTable(pRc, zName, zSql, bIntkey, nCol); break; } } @@ -6390,24 +6425,35 @@ static RecoverTable *recoverNewTable( static RecoverTable *recoverOrphanTable( ShellState *pState, int *pRc, + const char *zLostAndFound, int nCol ){ RecoverTable *pTab = 0; if( nCol>=0 && *pRc==SQLITE_OK ){ int i; - raw_printf(pState->out, - "CREATE TABLE recover_orphan(rootpgno INTEGER, " - "pgno INTEGER, nfield INTEGER, id INTEGER" + + /* This block determines the name of the orphan table. The prefered + ** name is zLostAndFound. But if that clashes with another name + ** in the recovered schema, try zLostAndFound_0, zLostAndFound_1 + ** and so on until a non-clashing name is found. */ + int iTab = 0; + char *zTab = shellMPrintf(pRc, "%s", zLostAndFound); + sqlite3_stmt *pTest = 0; + shellPrepare(pState->db, pRc, + "SELECT 1 FROM recovery.schema WHERE name=?", &pTest ); - for(i=0; iout, ", c%d", i); + if( pTest ) sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT); + while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pTest) ){ + shellReset(pRc, pTest); + sqlite3_free(zTab); + zTab = shellMPrintf(pRc, "%s_%d", zLostAndFound, iTab++); + sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT); } - raw_printf(pState->out, ");\n"); + shellFinalize(pRc, pTest); pTab = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable)); if( pTab ){ - pTab->zName = shellMPrintf(pRc, "%s", "recover_orphan"); - pTab->zQuoted = shellMPrintf(pRc, "%Q", pTab->zName); + pTab->zQuoted = shellMPrintf(pRc, "%Q", zTab); pTab->nCol = nCol; pTab->iPk = -2; if( nCol>0 ){ @@ -6419,12 +6465,22 @@ static RecoverTable *recoverOrphanTable( } } } - } - if( *pRc!=SQLITE_OK ){ - recoverFreeTable(pTab); - pTab = 0; + if( *pRc!=SQLITE_OK ){ + recoverFreeTable(pTab); + pTab = 0; + }else{ + raw_printf(pState->out, + "CREATE TABLE %s(rootpgno INTEGER, " + "pgno INTEGER, nfield INTEGER, id INTEGER", pTab->zQuoted + ); + for(i=0; iout, ", c%d", i); + } + raw_printf(pState->out, ");\n"); + } } + sqlite3_free(zTab); } return pTab; } @@ -6440,6 +6496,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ sqlite3_stmt *pPages = 0; /* Loop through all pages in a group */ sqlite3_stmt *pCells = 0; /* Loop through all cells in a page */ const char *zRecoveryDb = ""; /* Name of "recovery" database */ + const char *zLostAndFound = "lost_and_found"; int i; int nOrphan = -1; RecoverTable *pOrphan = 0; @@ -6452,16 +6509,21 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ n = strlen(z); if( n<=17 && memcmp("-freelist-corrupt", z, n)==0 ){ bFreelist = 0; - } + }else if( n<=12 && memcmp("-recovery-db", z, n)==0 && i<(nArg-1) ){ i++; zRecoveryDb = azArg[i]; + }else + if( n<=15 && memcmp("-lost-and-found", z, n)==0 && i<(nArg-1) ){ + i++; + zLostAndFound = azArg[i]; } else{ raw_printf(stderr, "unexpected option: %s\n", azArg[i]); raw_printf(stderr, "options are:\n"); raw_printf(stderr, " --freelist-corrupt\n"); raw_printf(stderr, " --recovery-db DATABASE\n"); + raw_printf(stderr, " --lost-and-found TABLE-NAME\n"); return 1; } } @@ -6599,7 +6661,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ } shellFinalize(&rc, pLoop); pLoop = 0; - pOrphan = recoverOrphanTable(pState, &rc, nOrphan); + pOrphan = recoverOrphanTable(pState, &rc, zLostAndFound, nOrphan); shellPrepare(pState->db, &rc, "SELECT pgno FROM recovery.map WHERE root=?", &pPages @@ -6624,11 +6686,11 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ int bNoop = 0; RecoverTable *pTab; - pTab = recoverNewTable(pState, &rc, iRoot, bIntkey, nCol, &bNoop); + pTab = recoverFindTable(pState, &rc, iRoot, bIntkey, nCol, &bNoop); if( bNoop || rc ) continue; if( pTab==0 ) pTab = pOrphan; - if( 0==sqlite3_stricmp(pTab->zName, "sqlite_sequence") ){ + if( 0==sqlite3_stricmp(pTab->zQuoted, "'sqlite_sequence'") ){ raw_printf(pState->out, "DELETE FROM sqlite_sequence;\n"); } sqlite3_bind_int(pPages, 1, iRoot); @@ -7042,30 +7104,30 @@ static int do_meta_command(char *zLine, ShellState *p){ p->nErr = 0; if( zLike==0 ){ run_schema_dump_query(p, - "SELECT name, type, sql FROM sqlite_master " - "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'" - ); + "SELECT name, type, sql FROM sqlite_master " + "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'" + ); run_schema_dump_query(p, - "SELECT name, type, sql FROM sqlite_master " - "WHERE name=='sqlite_sequence'" - ); + "SELECT name, type, sql FROM sqlite_master " + "WHERE name=='sqlite_sequence'" + ); run_table_dump_query(p, - "SELECT sql FROM sqlite_master " - "WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0 - ); + "SELECT sql FROM sqlite_master " + "WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0 + ); }else{ char *zSql; zSql = sqlite3_mprintf( - "SELECT name, type, sql FROM sqlite_master " - "WHERE tbl_name LIKE %Q AND type=='table'" - " AND sql NOT NULL", zLike); + "SELECT name, type, sql FROM sqlite_master " + "WHERE tbl_name LIKE %Q AND type=='table'" + " AND sql NOT NULL", zLike); run_schema_dump_query(p,zSql); sqlite3_free(zSql); zSql = sqlite3_mprintf( - "SELECT sql FROM sqlite_master " - "WHERE sql NOT NULL" - " AND type IN ('index','trigger','view')" - " AND tbl_name LIKE %Q", zLike); + "SELECT sql FROM sqlite_master " + "WHERE sql NOT NULL" + " AND type IN ('index','trigger','view')" + " AND tbl_name LIKE %Q", zLike); run_table_dump_query(p, zSql, 0); sqlite3_free(zSql); } diff --git a/test/recover.test b/test/recover.test index 4de7f503a5..4dad0d646d 100644 --- a/test/recover.test +++ b/test/recover.test @@ -39,7 +39,7 @@ proc compare_dbs {db1 db2} { } } -proc do_recover_test {tn} { +proc do_recover_test {tn {tsql {}} {res {}}} { set fd [open "|$::CLI test.db .recover"] fconfigure $fd -encoding binary fconfigure $fd -translation binary @@ -48,9 +48,12 @@ proc do_recover_test {tn} { forcedelete test.db2 sqlite3 db2 test.db2 - breakpoint execsql $sql db2 - uplevel [list do_test $tn [list compare_dbs db db2] {}] + if {$tsql==""} { + uplevel [list do_test $tn [list compare_dbs db db2] {}] + } else { + uplevel [list do_execsql_test -db db2 $tn $tsql $res] + } db2 close } @@ -96,4 +99,31 @@ do_execsql_test 2.1.0 { do_recover_test 2.1.1 +do_execsql_test 2.2.0 { + PRAGMA writable_schema = 1; + DELETE FROM sqlite_master WHERE name='t1'; +} +do_recover_test 2.2.1 { + SELECT name FROM sqlite_master +} {lost_and_found} + +do_execsql_test 2.3.0 { + CREATE TABLE lost_and_found(a, b, c); +} +do_recover_test 2.3.1 { + SELECT name FROM sqlite_master +} {lost_and_found lost_and_found_0} + +do_execsql_test 2.4.0 { + CREATE TABLE lost_and_found_0(a, b, c); +} +do_recover_test 2.4.1 { + SELECT name FROM sqlite_master; + SELECT * FROM lost_and_found_1; +} {lost_and_found lost_and_found_0 lost_and_found_1 + 2 2 3 {} 2 3 1 + 2 2 3 {} 5 6 4 + 2 2 3 {} 8 9 7 +} + finish_test From 0aa01ee42cd9616fdca75f97b88e861651481d3c Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 27 Apr 2019 19:36:49 +0000 Subject: [PATCH 16/38] Add comments and fix formatting issues in new code in shell.c.in. FossilOrigin-Name: b91d819bd16de43fc99e379da0ba9c915b0c5afc68e804a50c3c1662c1f9a740 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 38 ++++++++++++++++++++++++++++---------- 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 2f2f86a8fc..c4c8fd8087 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"--lost-and-found"\soption\sto\sthe\s".recover"\scommand.\sFor\ssetting\sthe\sname\sof\sthe\sorphaned\srows\stable. -D 2019-04-27T18:47:03.466 +C Add\scomments\sand\sfix\sformatting\sissues\sin\snew\scode\sin\sshell.c.in. +D 2019-04-27T19:36:49.564 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c b7304d2f491c11a03a7fbdf34bc218282ac54052377809d4dc3b4b1e7f4bfc93 -F src/shell.c.in 51f027f6b48fab39e0745915d979747880b7c35827798535726ac44366a291f1 +F src/shell.c.in 2316b9ee7a43e6f127e5abe8d74d69b92f6bbae1be38826ffb58384aab832013 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 91df4b8e0386105d01614921e8410994b621404a3d46ec4af8687b8743c52d52 -R a9542f620433fc5d37a56caa32ea75c3 +P 67bb88e24c74d02ae0c4ac6ff2f873f6b0035ccefe5cccfc71c5686cbc76b4c3 +R 1520b643c98e19ff415724ed970cd8b1 U dan -Z 07c2469e7eeced685ce45289494bc94e +Z 3d67e0e6129b0deb0a86f6c6a9eda8ce diff --git a/manifest.uuid b/manifest.uuid index e6d0f63ea7..73bfbb6ae3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -67bb88e24c74d02ae0c4ac6ff2f873f6b0035ccefe5cccfc71c5686cbc76b4c3 \ No newline at end of file +b91d819bd16de43fc99e379da0ba9c915b0c5afc68e804a50c3c1662c1f9a740 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 8c967a83d8..1fc08662db 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -6385,13 +6385,28 @@ static RecoverTable *recoverNewTable( return pTab; } +/* +** This function is called to search the schema recovered from the +** sqlite_master table of the (possibly) corrupt database as part +** of a ".recover" command. Specifically, for a table with root page +** iRoot and at least nCol columns. Additionally, if bIntkey is 0, the +** table must be a WITHOUT ROWID table, or if non-zero, not one of +** those. +** +** If a table is found, a (RecoverTable*) object is returned. Or, if +** no such table is found, but bIntkey is false and iRoot is the +** root page of an index in the recovered schema, then (*pbNoop) is +** set to true and NULL returned. Or, if there is no such table or +** index, NULL is returned and (*pbNoop) set to 0, indicating that +** the caller should write data to the orphans table. +*/ static RecoverTable *recoverFindTable( - ShellState *pState, - int *pRc, - int iRoot, - int bIntkey, - int nCol, - int *pbNoop + ShellState *pState, /* Shell state object */ + int *pRc, /* IN/OUT: Error code */ + int iRoot, /* Root page of table */ + int bIntkey, /* True for an intkey table */ + int nCol, /* Number of columns in table */ + int *pbNoop /* OUT: True if iRoot is root of index */ ){ sqlite3_stmt *pStmt = 0; RecoverTable *pRet = 0; @@ -6422,11 +6437,14 @@ static RecoverTable *recoverFindTable( return pRet; } +/* +** Return a RecoverTable object representing the orphans table. +*/ static RecoverTable *recoverOrphanTable( - ShellState *pState, - int *pRc, - const char *zLostAndFound, - int nCol + ShellState *pState, /* Shell state object */ + int *pRc, /* IN/OUT: Error code */ + const char *zLostAndFound, /* Base name for orphans table */ + int nCol /* Number of user data columns */ ){ RecoverTable *pTab = 0; if( nCol>=0 && *pRc==SQLITE_OK ){ From 1b16216f9eb297ddebc36fd83d3c3816baad0163 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 27 Apr 2019 20:15:15 +0000 Subject: [PATCH 17/38] Fix building the shell with SQLITE_OMIT_VIRTUAL_TABLE. And without SQLITE_ENABLE_DBPAGE_VTAB. FossilOrigin-Name: 425d708c3908fe74f69b62e6dd1722a0018088977e12f14b312dad1df0fbb804 --- ext/misc/dbdata.c | 5 ++--- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c.in | 21 +++++++++++++++++---- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c index 786369ce0a..0e7f7d59cc 100644 --- a/ext/misc/dbdata.c +++ b/ext/misc/dbdata.c @@ -75,7 +75,6 @@ #include "sqlite3ext.h" typedef unsigned char u8; -typedef unsigned long u32; #endif SQLITE_EXTENSION_INIT1 @@ -313,7 +312,7 @@ static unsigned int get_uint32(unsigned char *a){ */ static int dbdataLoadPage( DbdataCursor *pCsr, /* Cursor object */ - u32 pgno, /* Page number of page to load */ + unsigned int pgno, /* Page number of page to load */ u8 **ppPage, /* OUT: pointer to page buffer */ int *pnPage /* OUT: Size of (*ppPage) in bytes */ ){ @@ -556,7 +555,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){ /* Load content from overflow pages */ if( nPayload>nLocal ){ sqlite3_int64 nRem = nPayload - nLocal; - u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); + unsigned int pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); while( nRem>0 ){ u8 *aOvfl = 0; int nOvfl = 0; diff --git a/manifest b/manifest index c4c8fd8087..1abc30e6f6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\scomments\sand\sfix\sformatting\sissues\sin\snew\scode\sin\sshell.c.in. -D 2019-04-27T19:36:49.564 +C Fix\sbuilding\sthe\sshell\swith\sSQLITE_OMIT_VIRTUAL_TABLE.\sAnd\swithout\sSQLITE_ENABLE_DBPAGE_VTAB. +D 2019-04-27T20:15:15.393 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -284,7 +284,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c cec672d40604075bb341a7f11ac48393efdcd90a979269b8fe7977ea62d0547f F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189 F ext/misc/csv.c 7f047aeb68f5802e7ce6639292095d622a488bb43526ed04810e0649faa71ceb -F ext/misc/dbdata.c fe978dad2df13dd4b377b5d38f4883282801b18711220a229d0fd266a5deab26 +F ext/misc/dbdata.c 1b3751b02d8f575d25c6bda358670d2e39ace368a0d05595989c308a10c615f6 F ext/misc/dbdump.c baf6e37447c9d6968417b1cd34cbedb0b0ab3f91b5329501d8a8d5be3287c336 F ext/misc/eval.c 4b4757592d00fd32e44c7a067e6a0e4839c81a4d57abc4131ee7806d1be3104e F ext/misc/explain.c d5c12962d79913ef774b297006872af1fccda388f61a11d37758f9179a09551f @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c b7304d2f491c11a03a7fbdf34bc218282ac54052377809d4dc3b4b1e7f4bfc93 -F src/shell.c.in 2316b9ee7a43e6f127e5abe8d74d69b92f6bbae1be38826ffb58384aab832013 +F src/shell.c.in 104bbae904a2b67bc6c0c95337447544d15d0594dc46468608aae769d5f51da9 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 67bb88e24c74d02ae0c4ac6ff2f873f6b0035ccefe5cccfc71c5686cbc76b4c3 -R 1520b643c98e19ff415724ed970cd8b1 +P b91d819bd16de43fc99e379da0ba9c915b0c5afc68e804a50c3c1662c1f9a740 +R dcbe5105dfc58103275280fa56f0f458 U dan -Z 3d67e0e6129b0deb0a86f6c6a9eda8ce +Z a7824dfb7c5c59a3dfb17eb398274a3e diff --git a/manifest.uuid b/manifest.uuid index 73bfbb6ae3..3985d129a5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b91d819bd16de43fc99e379da0ba9c915b0c5afc68e804a50c3c1662c1f9a740 \ No newline at end of file +425d708c3908fe74f69b62e6dd1722a0018088977e12f14b312dad1df0fbb804 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 1fc08662db..2f9a2456a1 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -948,7 +948,9 @@ INCLUDE ../ext/misc/sqlar.c INCLUDE ../ext/expert/sqlite3expert.h INCLUDE ../ext/expert/sqlite3expert.c +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) INCLUDE ../ext/misc/dbdata.c +#endif #if defined(SQLITE_ENABLE_SESSION) /* @@ -3576,7 +3578,9 @@ static const char *(azHelp[]) = { ".prompt MAIN CONTINUE Replace the standard prompts", ".quit Exit this program", ".read FILE Read input from FILE", +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) ".recover Recover as much data as possible from corrupt db.", +#endif ".restore ?DB? FILE Restore content of DB (default \"main\") from FILE", ".save FILE Write in-memory database into FILE", ".scanstats on|off Turn sqlite3_stmt_scanstatus() metrics on or off", @@ -4121,7 +4125,9 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_fileio_init(p->db, 0, 0); sqlite3_shathree_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) sqlite3_dbdata_init(p->db, 0, 0); +#endif #ifdef SQLITE_HAVE_ZLIB sqlite3_zipfile_init(p->db, 0, 0); sqlite3_sqlar_init(p->db, 0, 0); @@ -5390,10 +5396,7 @@ static int lintDotCommand( return SQLITE_ERROR; } -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) -/********************************************************************************* -** The ".archive" or ".ar" command. -*/ +#if !defined SQLITE_OMIT_VIRTUALTABLE static void shellPrepare( sqlite3 *db, int *pRc, @@ -5464,6 +5467,12 @@ static void shellReset( *pRc = rc; } } +#endif /* !defined SQLITE_OMIT_VIRTUALTABLE */ + +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) +/********************************************************************************* +** The ".archive" or ".ar" command. +*/ /* ** Structure representing a single ".ar" command. */ @@ -6153,6 +6162,7 @@ end_ar_command: **********************************************************************************/ #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */ +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) /* ** If (*pRc) is not SQLITE_OK when this function is called, it is a no-op. ** Otherwise, the SQL statement or statements in zSql are executed using @@ -6775,6 +6785,7 @@ static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ sqlite3_exec(pState->db, "DETACH recovery", 0, 0, 0); return rc; } +#endif /* !(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) */ /* @@ -7063,10 +7074,12 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = shell_dbinfo_command(p, nArg, azArg); }else +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) if( c=='r' && strncmp(azArg[0], "recover", n)==0 ){ open_db(p, 0); rc = recoverDatabaseCmd(p, nArg, azArg); }else +#endif /* !(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) */ if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){ const char *zLike = 0; From 6c59136498547fefb69e3edaff4d949b0bf1113e Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 27 Apr 2019 20:16:42 +0000 Subject: [PATCH 18/38] Fix a minor typo in a comment. No changes to code. FossilOrigin-Name: 95209072176ff21a91e96d5bd014b35ef100da2b0b93958baf6df4294a8daa85 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 158d60af72..1b2109530a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\stests\sof\sthe\sLIKE\soptimization\sin\slike3.test\swhen\sSQLITE_ENABLE_ICU\sis\sdefined. -D 2019-04-26T17:08:50.839 +C Fix\sa\sminor\stypo\sin\sa\scomment.\s\sNo\schanges\sto\scode. +D 2019-04-27T20:16:42.497 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -462,7 +462,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 F src/btree.c ffe7101006aee2ab9e9dec2fc001998e57a8e59419c6ea4072d6c3935d3d50fb F src/btree.h c11446f07ec0e9dc85af8041cb0855c52f5359c8b2a43e47e02a685282504d89 F src/btreeInt.h 6111c15868b90669f79081039d19e7ea8674013f907710baa3c814dc3f8bfd3f -F src/build.c 61655dad911a967a69fb49df57268fd15ce8f1af3fe0a1bd90c128ef2cacfb7a +F src/build.c e9d560fdc39e0f037b1ebd78fde0ff616963646c8d85a7afa18db064c52c5b75 F src/callback.c 25dda5e1c2334a367b94a64077b1d06b2553369f616261ca6783c48bcb6bda73 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 109e58d00f62e8e71ee1eb5944ac18b90171c928ab2e082e058056e1137cc20b @@ -1818,7 +1818,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7be6222c9ec44596e4eddd906c831eb1272b90fbdf68641d791f216264feb7cf -R 1ba2bc865b250121afcb0af52a507e48 -U dan -Z 432d74fc77b640bcf830ac2298b92cb0 +P af53c41a127c314c0608f3fd016d3a26896783745e46cd180976a188400cdb75 +R 5d8e72521d268ff442af4eda367a86ab +U drh +Z c27e4c4c6167f0761ffa85b745f3dbd7 diff --git a/manifest.uuid b/manifest.uuid index 0646b8052e..9bb2a9c04f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -af53c41a127c314c0608f3fd016d3a26896783745e46cd180976a188400cdb75 \ No newline at end of file +95209072176ff21a91e96d5bd014b35ef100da2b0b93958baf6df4294a8daa85 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 9ecd31b8e8..3e4101df92 100644 --- a/src/build.c +++ b/src/build.c @@ -1329,7 +1329,7 @@ void sqlite3AddDefaultValue( ** accept it. This routine does the necessary conversion. It converts ** the expression given in its argument from a TK_STRING into a TK_ID ** if the expression is just a TK_STRING with an optional COLLATE clause. -** If the epxression is anything other than TK_STRING, the expression is +** If the expression is anything other than TK_STRING, the expression is ** unchanged. */ static void sqlite3StringToId(Expr *p){ From f78d0f426c2ce84904823a087e3950e343c8e6ec Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 28 Apr 2019 19:27:02 +0000 Subject: [PATCH 19/38] Take collating sequence into account when removing redundant columns from indexes on WITHOUT ROWID tables. This is the first proof-of-concept fix for ticket [3182d3879020ef3]. More testing needed. FossilOrigin-Name: b34fa5bff40d3d364bd8c80e7de55c606ef3caac47b14b5265ebcb38857eb85e --- manifest | 18 +++++++++-------- manifest.uuid | 2 +- src/build.c | 54 +++++++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 59 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index ffbce1300f..dec5885b95 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s".recover"\scommand\sto\sthe\sshell\stool.\sFor\srecovering\sas\smuch\sdata\sas\spossible\sfrom\scorrupt\sdatabases. -D 2019-04-27T20:30:19.423 +C Take\scollating\ssequence\sinto\saccount\swhen\sremoving\sredundant\scolumns\sfrom\nindexes\son\sWITHOUT\sROWID\stables.\s\sThis\sis\sthe\sfirst\sproof-of-concept\sfix\nfor\sticket\s[3182d3879020ef3].\sMore\stesting\sneeded. +D 2019-04-28T19:27:02.781 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -463,7 +463,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 F src/btree.c ffe7101006aee2ab9e9dec2fc001998e57a8e59419c6ea4072d6c3935d3d50fb F src/btree.h c11446f07ec0e9dc85af8041cb0855c52f5359c8b2a43e47e02a685282504d89 F src/btreeInt.h 6111c15868b90669f79081039d19e7ea8674013f907710baa3c814dc3f8bfd3f -F src/build.c e9d560fdc39e0f037b1ebd78fde0ff616963646c8d85a7afa18db064c52c5b75 +F src/build.c 3fedab95ff7a9b3ed51eceb12aaa61de6bbd063dabd276767494cc448a151b7b F src/callback.c 25dda5e1c2334a367b94a64077b1d06b2553369f616261ca6783c48bcb6bda73 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 109e58d00f62e8e71ee1eb5944ac18b90171c928ab2e082e058056e1137cc20b @@ -1821,8 +1821,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 95209072176ff21a91e96d5bd014b35ef100da2b0b93958baf6df4294a8daa85 425d708c3908fe74f69b62e6dd1722a0018088977e12f14b312dad1df0fbb804 -R f359cf293ea45304e0ddadd16c7a1206 -T +closed 425d708c3908fe74f69b62e6dd1722a0018088977e12f14b312dad1df0fbb804 -U dan -Z ee7f2d083d1fca64ba3bb33b40a19208 +P 50fe48458942fa7a6bcc76316c6321f95b23dc34f2f8e0a483826483b2fb16f6 +R fa5b9567c2bc0a92e3c8181fab56d7f8 +T *branch * tkt-3182d38790 +T *sym-tkt-3182d38790 * +T -sym-trunk * +U drh +Z dd82847aa38c87860cc4ec05cc3ab230 diff --git a/manifest.uuid b/manifest.uuid index 55700b6967..6c45322202 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -50fe48458942fa7a6bcc76316c6321f95b23dc34f2f8e0a483826483b2fb16f6 \ No newline at end of file +b34fa5bff40d3d364bd8c80e7de55c606ef3caac47b14b5265ebcb38857eb85e \ No newline at end of file diff --git a/src/build.c b/src/build.c index 3e4101df92..fac3163eba 100644 --- a/src/build.c +++ b/src/build.c @@ -1726,10 +1726,46 @@ static void estimateIndexWidth(Index *pIdx){ pIdx->szIdxRow = sqlite3LogEst(wIndex*4); } -/* Return true if value x is found any of the first nCol entries of aiCol[] +/* Return true if column number x is any of the first nCol entries of aiCol[]. +** This is used to determine if the column number x appears in any of the +** first nCol entries of an index. */ static int hasColumn(const i16 *aiCol, int nCol, int x){ - while( nCol-- > 0 ) if( x==*(aiCol++) ) return 1; + while( nCol-- > 0 ){ + assert( aiCol[0]>=0 ); + if( x==*(aiCol++) ){ + return 1; + } + } + return 0; +} + +/* +** Return true if any of the first nKey entries of index pIdx1 exactly +** match the iCol-th entry of pIdx2. +** +** The first nKey entries of pIdx1 are guaranteed to be ordinary columns, +** not a rowid or expression. +** +** This routine differs from hasColumn() in that both the column and the +** collating sequence must match for this routine, but for hasColumn() only +** the column name must match. +*/ +static int isDupColumn(Index *pIdx1, int nKey, Index *pIdx2, int iCol){ + int i, j; + assert( nKey<=pIdx1->nColumn ); + assert( iColnColumn,pIdx2->nKeyCol) ); + j = pIdx2->aiColumn[iCol]; + testcase( j==XN_EXPR ); + assert( j!=XN_ROWID ); + for(i=0; iaiColumn[i]>=0 || j>=0 ); + if( pIdx1->aiColumn[i]==j + && sqlite3StrICmp(pIdx1->azColl[i],pIdx2->azColl[iCol])==0 + ){ + return 1; + } + } return 0; } @@ -1835,9 +1871,10 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ ** code assumes the PRIMARY KEY contains no repeated columns. */ for(i=j=1; inKeyCol; i++){ - if( hasColumn(pPk->aiColumn, j, pPk->aiColumn[i]) ){ + if( isDupColumn(pPk, j, pPk, i) ){ pPk->nColumn--; }else{ + testcase( hasColumn(pPk->aiColumn, j, pPk->aiColumn[i]) ); pPk->aiColumn[j++] = pPk->aiColumn[i]; } } @@ -1867,7 +1904,10 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ int n; if( IsPrimaryKeyIndex(pIdx) ) continue; for(i=n=0; iaiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ) n++; + if( !isDupColumn(pIdx, pIdx->nKeyCol, pPk, i) ){ + testcase( hasColumn(pIdx->aiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ); + n++; + } } if( n==0 ){ /* This index is a superset of the primary key */ @@ -1876,7 +1916,8 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ } if( resizeIndexObject(db, pIdx, pIdx->nKeyCol+n) ) return; for(i=0, j=pIdx->nKeyCol; iaiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ){ + if( !isDupColumn(pIdx, pIdx->nKeyCol, pPk, i) ){ + testcase( hasColumn(pIdx->aiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ); pIdx->aiColumn[j] = pPk->aiColumn[i]; pIdx->azColl[j] = pPk->azColl[i]; j++; @@ -3392,9 +3433,10 @@ void sqlite3CreateIndex( for(j=0; jnKeyCol; j++){ int x = pPk->aiColumn[j]; assert( x>=0 ); - if( hasColumn(pIndex->aiColumn, pIndex->nKeyCol, x) ){ + if( isDupColumn(pIndex, pIndex->nKeyCol, pPk, j) ){ pIndex->nColumn--; }else{ + testcase( hasColumn(pIndex->aiColumn,pIndex->nKeyCol,x) ); pIndex->aiColumn[i] = x; pIndex->azColl[i] = pPk->azColl[j]; pIndex->aSortOrder[i] = pPk->aSortOrder[j]; From 490e6f2506a1ebd37c2acdddcd2484d7b59d8071 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 29 Apr 2019 11:27:58 +0000 Subject: [PATCH 20/38] Fix a stack overflow that could occur when renaming a table that has a trigger containing a window function invocation that itself contains a specific syntax error. FossilOrigin-Name: c621fc668c6538f9f5bdac204f012c64998679a61aa8e224d212503820224c09 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/resolve.c | 4 +++- test/altertab3.test | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index ffbce1300f..58229ddb68 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s".recover"\scommand\sto\sthe\sshell\stool.\sFor\srecovering\sas\smuch\sdata\sas\spossible\sfrom\scorrupt\sdatabases. -D 2019-04-27T20:30:19.423 +C Fix\sa\sstack\soverflow\sthat\scould\soccur\swhen\srenaming\sa\stable\sthat\shas\sa\strigger\scontaining\sa\swindow\sfunction\sinvocation\sthat\sitself\scontains\sa\sspecific\ssyntax\serror. +D 2019-04-29T11:27:58.967 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -517,7 +517,7 @@ F src/pragma.h a776bb9c915207e9d1117b5754743ddf1bf6a39cc092a4a44e74e6cb5fab1177 F src/prepare.c 78027c6231fbb19ca186a5f5f0c0a1375d9c2cec0655273f9bd90d9ff74a34b3 F src/printf.c 67f79227273a9009d86a017619717c3f554f50b371294526da59faa6014ed2cd F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 +F src/resolve.c 408632d9531ca8f1df8591f00530797daaa7bde3fe0d3211de4d431cbb99347e F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c b7304d2f491c11a03a7fbdf34bc218282ac54052377809d4dc3b4b1e7f4bfc93 F src/shell.c.in 104bbae904a2b67bc6c0c95337447544d15d0594dc46468608aae769d5f51da9 @@ -629,7 +629,7 @@ F test/altermalloc.test 167a47de41b5c638f5f5c6efb59784002b196fff70f98d9b4ed3cd74 F test/altermalloc2.test fa7b1c1139ea39b8dec407cf1feb032ca8e0076bd429574969b619175ad0174b F test/altertab.test 372df7d8f09e1ee22d23551677cedff3b048b0059c1f1b9a01a6401b94a2367c F test/altertab2.test 5d423a2d1006085b05cc1b788863d5a860ea2da21c4f892d15e2f2a34c78348a -F test/altertab3.test 40f2ce9be675e354d3e55c72f8baf38813be975ff4dd9e6b3144493c3c5bc033 +F test/altertab3.test 2433d0cc6cb9cffe087f9138cd36818c7abd5c396804aa6e6dc8c2b80e2cd406 F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 7168c8bffa5d5cbc53c05b7e9c7fcdd24b365a1bc5046ce80c45efa3c02e6b7c F test/analyze3.test ff62d9029e6deb2c914490c6b00caf7fae47cc85cdc046e4a0d0a4d4b87c71d8 @@ -1821,8 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 95209072176ff21a91e96d5bd014b35ef100da2b0b93958baf6df4294a8daa85 425d708c3908fe74f69b62e6dd1722a0018088977e12f14b312dad1df0fbb804 -R f359cf293ea45304e0ddadd16c7a1206 -T +closed 425d708c3908fe74f69b62e6dd1722a0018088977e12f14b312dad1df0fbb804 +P 50fe48458942fa7a6bcc76316c6321f95b23dc34f2f8e0a483826483b2fb16f6 +R 50f0438c4e5efd5bb6070c75e8184302 U dan -Z ee7f2d083d1fca64ba3bb33b40a19208 +Z dfacee40ac6434bcc0aa927bae743b38 diff --git a/manifest.uuid b/manifest.uuid index 55700b6967..16446b273a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -50fe48458942fa7a6bcc76316c6321f95b23dc34f2f8e0a483826483b2fb16f6 \ No newline at end of file +c621fc668c6538f9f5bdac204f012c64998679a61aa8e224d212503820224c09 \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index 50755e59f3..f6b6af1dfa 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -866,7 +866,9 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ #ifndef SQLITE_OMIT_WINDOWFUNC if( pExpr->y.pWin ){ Select *pSel = pNC->pWinSelect; - sqlite3WindowUpdate(pParse, pSel->pWinDefn, pExpr->y.pWin, pDef); + if( IN_RENAME_OBJECT==0 ){ + sqlite3WindowUpdate(pParse, pSel->pWinDefn, pExpr->y.pWin, pDef); + } sqlite3WalkExprList(pWalker, pExpr->y.pWin->pPartition); sqlite3WalkExprList(pWalker, pExpr->y.pWin->pOrderBy); sqlite3WalkExpr(pWalker, pExpr->y.pWin->pFilter); diff --git a/test/altertab3.test b/test/altertab3.test index 2bf6a9ddc1..b37d9ed1f9 100644 --- a/test/altertab3.test +++ b/test/altertab3.test @@ -142,6 +142,39 @@ do_execsql_test 6.1 { ALTER TABLE Table0 RENAME Col0 TO Col0; } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 7.1.0 { + CREATE TABLE t1(a,b,c); + CREATE TRIGGER AFTER INSERT ON t1 BEGIN + SELECT a, rank() OVER w1 FROM t1 + WINDOW w1 AS (PARTITION BY b, percent_rank() OVER w1); + END; +} + +do_execsql_test 7.1.2 { + ALTER TABLE t1 RENAME TO t1x; + SELECT sql FROM sqlite_master; +} { + {CREATE TABLE "t1x"(a,b,c)} + {CREATE TRIGGER AFTER INSERT ON "t1x" BEGIN + SELECT a, rank() OVER w1 FROM "t1x" + WINDOW w1 AS (PARTITION BY b, percent_rank() OVER w1); + END} +} + +do_execsql_test 7.2.1 { + DROP TRIGGER after; + CREATE TRIGGER AFTER INSERT ON t1x BEGIN + SELECT a, rank() OVER w1 FROM t1x + WINDOW w1 AS (PARTITION BY b, percent_rank() OVER w1 ORDER BY d); + END; +} + +do_catchsql_test 7.2.2 { + ALTER TABLE t1x RENAME TO t1; +} {1 {error in trigger AFTER: no such column: d}} + finish_test From 51f5ffa1a429bfdfe177e417482f1fbbf6685e76 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 29 Apr 2019 11:41:46 +0000 Subject: [PATCH 21/38] Fix a buffer overwrite in shell.c.in (part of the new .recover code). FossilOrigin-Name: 92facbc73a940d2844ac88fafd2d2dadb10886fb0b7c53e23f346d18fa6d6327 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c.in | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 58229ddb68..cf4308824d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sstack\soverflow\sthat\scould\soccur\swhen\srenaming\sa\stable\sthat\shas\sa\strigger\scontaining\sa\swindow\sfunction\sinvocation\sthat\sitself\scontains\sa\sspecific\ssyntax\serror. -D 2019-04-29T11:27:58.967 +C Fix\sa\sbuffer\soverwrite\sin\sshell.c.in\s(part\sof\sthe\snew\s.recover\scode). +D 2019-04-29T11:41:46.359 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -520,7 +520,7 @@ F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 F src/resolve.c 408632d9531ca8f1df8591f00530797daaa7bde3fe0d3211de4d431cbb99347e F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c b7304d2f491c11a03a7fbdf34bc218282ac54052377809d4dc3b4b1e7f4bfc93 -F src/shell.c.in 104bbae904a2b67bc6c0c95337447544d15d0594dc46468608aae769d5f51da9 +F src/shell.c.in 567236da9ee68b1dfa363426858ee5e310976ffe422a7b7ae220c0315d7e8c53 F src/sqlite.h.in 38390767acc1914d58930e03149595ee4710afa4e3c43ab6c3a8aea3f1a6b8cd F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 50fe48458942fa7a6bcc76316c6321f95b23dc34f2f8e0a483826483b2fb16f6 -R 50f0438c4e5efd5bb6070c75e8184302 +P c621fc668c6538f9f5bdac204f012c64998679a61aa8e224d212503820224c09 +R fab7a84bb2576871c69ae5af2474d189 U dan -Z dfacee40ac6434bcc0aa927bae743b38 +Z 238bfca672a6077522e49009580b426a diff --git a/manifest.uuid b/manifest.uuid index 16446b273a..491b1c92fc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c621fc668c6538f9f5bdac204f012c64998679a61aa8e224d212503820224c09 \ No newline at end of file +92facbc73a940d2844ac88fafd2d2dadb10886fb0b7c53e23f346d18fa6d6327 \ No newline at end of file diff --git a/src/shell.c.in b/src/shell.c.in index 2f9a2456a1..e145a1547c 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -4010,7 +4010,7 @@ static void shellEscapeCrnl( if( zNL || zCR ){ int iOut = 0; i64 nMax = (nNL > nCR) ? nNL : nCR; - i64 nAlloc = nMax * nText + (nMax+12)*2; + i64 nAlloc = nMax * nText + (nMax+64)*2; char *zOut = (char*)sqlite3_malloc64(nAlloc); if( zOut==0 ){ sqlite3_result_error_nomem(context); From c19b63c9a32f9de0830eb43f50c03aad3b351b51 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 29 Apr 2019 13:30:16 +0000 Subject: [PATCH 22/38] Improved header comment and precondition checking for the new isDupColumn() function. FossilOrigin-Name: 740d5ff6cc9bf7b151dfb8b27409e5923cfb2789b5398fe13d89563aff8ffc07 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/build.c | 29 +++++++++++++++++------------ 3 files changed, 24 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index dec5885b95..688c836776 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Take\scollating\ssequence\sinto\saccount\swhen\sremoving\sredundant\scolumns\sfrom\nindexes\son\sWITHOUT\sROWID\stables.\s\sThis\sis\sthe\sfirst\sproof-of-concept\sfix\nfor\sticket\s[3182d3879020ef3].\sMore\stesting\sneeded. -D 2019-04-28T19:27:02.781 +C Improved\sheader\scomment\sand\sprecondition\schecking\sfor\sthe\snew\sisDupColumn()\nfunction. +D 2019-04-29T13:30:16.066 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -463,7 +463,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 F src/btree.c ffe7101006aee2ab9e9dec2fc001998e57a8e59419c6ea4072d6c3935d3d50fb F src/btree.h c11446f07ec0e9dc85af8041cb0855c52f5359c8b2a43e47e02a685282504d89 F src/btreeInt.h 6111c15868b90669f79081039d19e7ea8674013f907710baa3c814dc3f8bfd3f -F src/build.c 3fedab95ff7a9b3ed51eceb12aaa61de6bbd063dabd276767494cc448a151b7b +F src/build.c 2d9ddfeaf8e1dafc7e1fcc8a84e7a8b455199dac3b69037fc73af6279aa8447b F src/callback.c 25dda5e1c2334a367b94a64077b1d06b2553369f616261ca6783c48bcb6bda73 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 109e58d00f62e8e71ee1eb5944ac18b90171c928ab2e082e058056e1137cc20b @@ -1821,10 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 50fe48458942fa7a6bcc76316c6321f95b23dc34f2f8e0a483826483b2fb16f6 -R fa5b9567c2bc0a92e3c8181fab56d7f8 -T *branch * tkt-3182d38790 -T *sym-tkt-3182d38790 * -T -sym-trunk * +P b34fa5bff40d3d364bd8c80e7de55c606ef3caac47b14b5265ebcb38857eb85e +R 4500a529c2eaff26172712aaa84d5151 U drh -Z dd82847aa38c87860cc4ec05cc3ab230 +Z 5d2313cdcb9e1838679491aff8b0ed48 diff --git a/manifest.uuid b/manifest.uuid index 6c45322202..e70eba18c7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b34fa5bff40d3d364bd8c80e7de55c606ef3caac47b14b5265ebcb38857eb85e \ No newline at end of file +740d5ff6cc9bf7b151dfb8b27409e5923cfb2789b5398fe13d89563aff8ffc07 \ No newline at end of file diff --git a/src/build.c b/src/build.c index fac3163eba..8684e1ad69 100644 --- a/src/build.c +++ b/src/build.c @@ -1741,27 +1741,32 @@ static int hasColumn(const i16 *aiCol, int nCol, int x){ } /* -** Return true if any of the first nKey entries of index pIdx1 exactly -** match the iCol-th entry of pIdx2. +** Return true if any of the first nKey entries of index pIdx exactly +** match the iCol-th entry of pPk. pPk is always a WITHOUT ROWID +** PRIMARY KEY index. pIdx is an index on the same table. pIdx may +** or may not be the same index as pPk. ** -** The first nKey entries of pIdx1 are guaranteed to be ordinary columns, +** The first nKey entries of pIdx are guaranteed to be ordinary columns, ** not a rowid or expression. ** ** This routine differs from hasColumn() in that both the column and the ** collating sequence must match for this routine, but for hasColumn() only ** the column name must match. */ -static int isDupColumn(Index *pIdx1, int nKey, Index *pIdx2, int iCol){ +static int isDupColumn(Index *pIdx, int nKey, Index *pPk, int iCol){ int i, j; - assert( nKey<=pIdx1->nColumn ); - assert( iColnColumn,pIdx2->nKeyCol) ); - j = pIdx2->aiColumn[iCol]; - testcase( j==XN_EXPR ); - assert( j!=XN_ROWID ); + assert( nKey<=pIdx->nColumn ); + assert( iColnColumn,pPk->nKeyCol) ); + assert( pPk->idxType==SQLITE_IDXTYPE_PRIMARYKEY ); + assert( pPk->pTable->tabFlags & TF_WithoutRowid ); + assert( pPk->pTable==pIdx->pTable ); + testcase( pPk==pIdx ); + j = pPk->aiColumn[iCol]; + assert( j!=XN_ROWID && j!=XN_EXPR ); for(i=0; iaiColumn[i]>=0 || j>=0 ); - if( pIdx1->aiColumn[i]==j - && sqlite3StrICmp(pIdx1->azColl[i],pIdx2->azColl[iCol])==0 + assert( pIdx->aiColumn[i]>=0 || j>=0 ); + if( pIdx->aiColumn[i]==j + && sqlite3StrICmp(pIdx->azColl[i], pPk->azColl[iCol])==0 ){ return 1; } From 3af1b60e3a3963b70399320cdf59286537004f69 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 29 Apr 2019 16:44:11 +0000 Subject: [PATCH 23/38] Changes to oserror.test so that it works even on systems that allow an unusually large number of file descriptors. FossilOrigin-Name: a27b0b880d76c6838c0365f66bcd69b1b49b7594470993b608f4e490cbdc4882 --- manifest | 13 ++++++------- manifest.uuid | 2 +- test/oserror.test | 32 +++++++++++++++++++++++--------- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 0576b14f7c..7678e35d69 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sde-duplicate\scolumns\sindex\scolumns\sassociated\swith\sa\sWITHOUT\sROWID\stable\nif\sthe\scolumns\shave\sdifferent\scollating\ssequences.\s\sThis\sis\sthe\sfix\sfor\nticket\s[3182d3879020ef3b2].\s\sThere\sis\sone\stest\scase\sadded,\sbut\smost\sof\sthe\ntests\sare\sdone\sin\sTH3. -D 2019-04-29T13:48:45.899 +C Changes\sto\soserror.test\sso\sthat\sit\sworks\seven\son\ssystems\sthat\sallow\nan\sunusually\slarge\snumber\sof\sfile\sdescriptors. +D 2019-04-29T16:44:11.554 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1185,7 +1185,7 @@ F test/orderby6.test 8b38138ab0972588240b3fca0985d2e400432859 F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da F test/orderby8.test 23ef1a5d72bd3adcc2f65561c654295d1b8047bd F test/orderby9.test 87fb9548debcc2cd141c5299002dd94672fa76a3 -F test/oserror.test e7b3416be4b9d5dd2fe0b42dd394daaddbb6c83eeec1f0e47b120b53e0ad3ace +F test/oserror.test 1fc9746b83d778e70d115049747ba19c7fba154afce7cc165b09feb6ca6abbc5 F test/ossfuzz.c 18af635fa73d12a109b305faca727a734c1fa28a421b161d9d15c5a84a4998a2 F test/ossshell.c f125c5bd16e537a2549aa579b328dd1c59905e7ab1338dfc210e755bb7b69f17 F test/ovfl.test 199c482696defceacee8c8e0e0ef36da62726b2f @@ -1821,8 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 92facbc73a940d2844ac88fafd2d2dadb10886fb0b7c53e23f346d18fa6d6327 740d5ff6cc9bf7b151dfb8b27409e5923cfb2789b5398fe13d89563aff8ffc07 -R 202811e2e95855247d322c66d5116799 -T +closed 740d5ff6cc9bf7b151dfb8b27409e5923cfb2789b5398fe13d89563aff8ffc07 +P 1b1dd4d48cd79a585e1fa7ee79128e9f2a9ee9846339dc56bbd67b75112dcad5 +R 0bcdc9eac356966e584a16ee79a622df U drh -Z b55be1ccaf3a50c1a3ba7b999ad660d8 +Z e312b5e28707c934ce76aba346ace36f diff --git a/manifest.uuid b/manifest.uuid index e85adcff39..6158031446 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1b1dd4d48cd79a585e1fa7ee79128e9f2a9ee9846339dc56bbd67b75112dcad5 \ No newline at end of file +a27b0b880d76c6838c0365f66bcd69b1b49b7594470993b608f4e490cbdc4882 \ No newline at end of file diff --git a/test/oserror.test b/test/oserror.test index 271163aaad..a51301cc52 100644 --- a/test/oserror.test +++ b/test/oserror.test @@ -52,18 +52,32 @@ proc do_re_test {tn script expression} { # an error may be reported for either open() or getcwd() here. # if {![clang_sanitize_address]} { + unset -nocomplain rc + unset -nocomplain nOpen + set nOpen 20000 do_test 1.1.1 { set ::log [list] - list [catch { - for {set i 0} {$i < 20000} {incr i} { sqlite3 dbh_$i test.db -readonly 1 } - } msg] $msg - } {1 {unable to open database file}} + set ::rc [catch { + for {set i 0} {$i < $::nOpen} {incr i} { sqlite3 dbh_$i test.db -readonly 1 } + } msg] + if {$::rc==0} { + # Some system (ex: Debian) are able to create 20000+ file descriptiors + # such systems will not fail here + set x ok + } elseif {$::rc==1 && $msg=="unable to open database file"} { + set x ok + } else { + set x [list $::rc $msg] + } + } {ok} do_test 1.1.2 { - catch { for {set i 0} {$i < 20000} {incr i} { dbh_$i close } } - } {1} - do_re_test 1.1.3 { - lindex $::log 0 - } {^os_unix.c:\d+: \(\d+\) (open|getcwd)\(.*test.db\) - } + catch { for {set i 0} {$i < $::nOpen} {incr i} { dbh_$i close } } + } $::rc + if {$rc} { + do_re_test 1.1.3 { + lindex $::log 0 + } {^os_unix.c:\d+: \(\d+\) (open|getcwd)\(.*test.db\) - } + } } From b70b0df8eb8cea79074289826079699a00a0aef4 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 30 Apr 2019 01:08:42 +0000 Subject: [PATCH 24/38] Slightly smaller and faster implementation of the OP_MakeRecord opcode. FossilOrigin-Name: 3bdce7ef1a6bb03affe978243fec603d5a55c071aa6d87c469a3c199d23f3b5e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 47 +++++++++++++++++++++++------------------------ 3 files changed, 30 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index 7678e35d69..6efd90d506 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\soserror.test\sso\sthat\sit\sworks\seven\son\ssystems\sthat\sallow\nan\sunusually\slarge\snumber\sof\sfile\sdescriptors. -D 2019-04-29T16:44:11.554 +C Slightly\ssmaller\sand\sfaster\simplementation\sof\sthe\sOP_MakeRecord\sopcode. +D 2019-04-30T01:08:42.471 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/upsert.c 0dd81b40206841814d46942a7337786932475f085716042d0cb2fc7791bf8ca4 F src/utf.c 2f0fac345c7660d5c5bd3df9e9d8d33d4c27f366bcfb09e07443064d751a0507 F src/util.c 5061987401c2e8003177fa30d73196aa036727c8f04bf36a2df0c82b1904a236 F src/vacuum.c 82dcec9e7b1afa980288718ad11bc499651c722d7b9f32933c4d694d91cb6ebf -F src/vdbe.c 711ef421b3bb3db3b2476067b2dc3c71ef5844d9b1a723026578f89f6da621e8 +F src/vdbe.c 2782da511932d17e45e128f04f73314e5f5d765f7a499f5ac4bb1e0a82e082cc F src/vdbe.h 712bca562eaed1c25506b9faf9680bdc75fc42e2f4a1cd518d883fa79c7a4237 F src/vdbeInt.h 2c12704db9740c8e899786ecfc7a5797a9d067563496eb1b6ed03c592d7b8d90 F src/vdbeapi.c 2ddd60f4a351f15ee98d841e346af16111ad59dfa4d25d2dd4012e9875bf7d92 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1b1dd4d48cd79a585e1fa7ee79128e9f2a9ee9846339dc56bbd67b75112dcad5 -R 0bcdc9eac356966e584a16ee79a622df +P a27b0b880d76c6838c0365f66bcd69b1b49b7594470993b608f4e490cbdc4882 +R b12c0740e7e730a41b03261e6f060f1c U drh -Z e312b5e28707c934ce76aba346ace36f +Z b866775507213d2cd189ff81277fbb67 diff --git a/manifest.uuid b/manifest.uuid index 6158031446..cfd74f4be8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a27b0b880d76c6838c0365f66bcd69b1b49b7594470993b608f4e490cbdc4882 \ No newline at end of file +3bdce7ef1a6bb03affe978243fec603d5a55c071aa6d87c469a3c199d23f3b5e \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index c004137c47..faaf947308 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2791,7 +2791,6 @@ case OP_Affinity: { ** If P4 is NULL then all index fields have the affinity BLOB. */ case OP_MakeRecord: { - u8 *zNewRecord; /* A buffer to hold the data for the new record */ Mem *pRec; /* The new record */ u64 nData; /* Number of bytes of data space */ int nHdr; /* Number of bytes of header space */ @@ -2804,9 +2803,9 @@ case OP_MakeRecord: { int nField; /* Number of fields in the record */ char *zAffinity; /* The affinity string for the record */ int file_format; /* File format to use for encoding */ - int i; /* Space used in zNewRecord[] header */ - int j; /* Space used in zNewRecord[] content */ u32 len; /* Length of a field */ + u8 *zHdr; /* Where to write next byte of the header */ + u8 *zPayload; /* Where to write next byte of the payload */ /* Assuming the record contains N fields, the record format looks ** like this: @@ -2933,34 +2932,34 @@ case OP_MakeRecord: { goto no_mem; } } - zNewRecord = (u8 *)pOut->z; - - /* Write the record */ - i = putVarint32(zNewRecord, nHdr); - j = nHdr; - assert( pData0<=pLast ); - pRec = pData0; - do{ - serial_type = pRec->uTemp; - /* EVIDENCE-OF: R-06529-47362 Following the size varint are one or more - ** additional varints, one per column. */ - i += putVarint32(&zNewRecord[i], serial_type); /* serial type */ - /* EVIDENCE-OF: R-64536-51728 The values for each column in the record - ** immediately follow the header. */ - j += sqlite3VdbeSerialPut(&zNewRecord[j], pRec, serial_type); /* content */ - }while( (++pRec)<=pLast ); - assert( i==nHdr ); - assert( j==nByte ); - - assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); pOut->n = (int)nByte; pOut->flags = MEM_Blob; if( nZero ){ pOut->u.nZero = nZero; pOut->flags |= MEM_Zero; } - REGISTER_TRACE(pOp->p3, pOut); UPDATE_MAX_BLOBSIZE(pOut); + zHdr = (u8 *)pOut->z; + zPayload = zHdr + nHdr; + + /* Write the record */ + zHdr += putVarint32(zHdr, nHdr); + assert( pData0<=pLast ); + pRec = pData0; + do{ + serial_type = pRec->uTemp; + /* EVIDENCE-OF: R-06529-47362 Following the size varint are one or more + ** additional varints, one per column. */ + zHdr += putVarint32(zHdr, serial_type); /* serial type */ + /* EVIDENCE-OF: R-64536-51728 The values for each column in the record + ** immediately follow the header. */ + zPayload += sqlite3VdbeSerialPut(zPayload, pRec, serial_type); /* content */ + }while( (++pRec)<=pLast ); + assert( nHdr==(int)(zHdr - (u8*)pOut->z) ); + assert( nByte==(int)(zPayload - (u8*)pOut->z) ); + + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); + REGISTER_TRACE(pOp->p3, pOut); break; } From cf83323936dccae4ccafa8c6d17d4a9d7c439e5a Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 30 Apr 2019 11:54:36 +0000 Subject: [PATCH 25/38] Small performance increase and size reduction in the implementation of the LIKE and GLOB operators. FossilOrigin-Name: f97626f921dafe596b615a168ef31987f4a1c0b52956443e1a5c1148b49cab74 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/func.c | 6 ++---- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 6efd90d506..17f6e743ba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Slightly\ssmaller\sand\sfaster\simplementation\sof\sthe\sOP_MakeRecord\sopcode. -D 2019-04-30T01:08:42.471 +C Small\sperformance\sincrease\sand\ssize\sreduction\sin\sthe\simplementation\sof\sthe\nLIKE\sand\sGLOB\soperators. +D 2019-04-30T11:54:36.560 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -474,7 +474,7 @@ F src/delete.c d08c9e01a2664afd12edcfa3a9c6578517e8ff8735f35509582693adbe0edeaf F src/expr.c f65db06a0fcff760cadfb79d579a41e3eb7eff38848d5d6359137822f4fa2ec9 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 0e14d4bef8eac2d87bbd517e492d9084c65008d117823f8922c5e7b2b599bd33 -F src/func.c 2ccf4ae12430b1ae7096be5f0675887e1bd0732828af0ac0f7496339b7c6edee +F src/func.c ac05ea6b47b407586ad2c0878c4c81c3acb08b67ecf86648830f91f40325ae37 F src/global.c 0dea3065ea72a65ae941559b6686aad6516d4913e76fa4f79a95ff7787f624ec F src/hash.c 8d7dda241d0ebdafb6ffdeda3149a412d7df75102cecfc1021c98d6219823b19 F src/hash.h 9d56a9079d523b648774c1784b74b89bd93fac7b365210157482e4319a468f38 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a27b0b880d76c6838c0365f66bcd69b1b49b7594470993b608f4e490cbdc4882 -R b12c0740e7e730a41b03261e6f060f1c +P 3bdce7ef1a6bb03affe978243fec603d5a55c071aa6d87c469a3c199d23f3b5e +R 9a779d76ef4cdc16bf512841a59b2ab4 U drh -Z b866775507213d2cd189ff81277fbb67 +Z 487e34d77ec73a391deebf3b3507c69b diff --git a/manifest.uuid b/manifest.uuid index cfd74f4be8..c0c3e4bb99 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3bdce7ef1a6bb03affe978243fec603d5a55c071aa6d87c469a3c199d23f3b5e \ No newline at end of file +f97626f921dafe596b615a168ef31987f4a1c0b52956443e1a5c1148b49cab74 \ No newline at end of file diff --git a/src/func.c b/src/func.c index 03595b70d2..dae85d60a8 100644 --- a/src/func.c +++ b/src/func.c @@ -843,8 +843,6 @@ static void likeFunc( return; } #endif - zB = sqlite3_value_text(argv[0]); - zA = sqlite3_value_text(argv[1]); /* Limit the length of the LIKE or GLOB pattern to avoid problems ** of deep recursion and N*N behavior in patternCompare(). @@ -856,8 +854,6 @@ static void likeFunc( sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); return; } - assert( zB==sqlite3_value_text(argv[0]) ); /* Encoding did not change */ - if( argc==3 ){ /* The escape character string must consist of a single UTF-8 character. ** Otherwise, return an error. @@ -873,6 +869,8 @@ static void likeFunc( }else{ escape = pInfo->matchSet; } + zB = sqlite3_value_text(argv[0]); + zA = sqlite3_value_text(argv[1]); if( zA && zB ){ #ifdef SQLITE_TEST sqlite3_like_count++; From f135cb7d7f123a42940ddbb4dadc91e86f84cb35 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 30 Apr 2019 14:26:31 +0000 Subject: [PATCH 26/38] Fix an error message in the Lemon parser generator. FossilOrigin-Name: b6d7d42b7426622a26b67809cd1f21285fea120aa1897377b9946840463b41f1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/lemon.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 17f6e743ba..2b7a987399 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\sincrease\sand\ssize\sreduction\sin\sthe\simplementation\sof\sthe\nLIKE\sand\sGLOB\soperators. -D 2019-04-30T11:54:36.560 +C Fix\san\serror\smessage\sin\sthe\sLemon\sparser\sgenerator. +D 2019-04-30T14:26:31.199 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1740,7 +1740,7 @@ F tool/genfkey.test b6afd7b825d797a1e1274f519ab5695373552ecad5cd373530c63533638a F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/index_usage.c 9ec344d29cbeb03fdc0fce668eedfb7495792170de933adf95cf8d6904a166ad F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f -F tool/lemon.c 900a15b9efba9890d10e7959914db94c4ad5162912127f061c4328add122d6fb +F tool/lemon.c d02a276728c507a7007333944eeabafab1668033794af348389b1166075869ee F tool/lempar.c 61af95b8fac2bfd59c09d55330e78f3f5e352d7aa80bf37404b96ef795be3fdc F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 @@ -1821,7 +1821,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 3bdce7ef1a6bb03affe978243fec603d5a55c071aa6d87c469a3c199d23f3b5e -R 9a779d76ef4cdc16bf512841a59b2ab4 +P f97626f921dafe596b615a168ef31987f4a1c0b52956443e1a5c1148b49cab74 +R 716d456c280daf5c4e086f08a5ceb68c U drh -Z 487e34d77ec73a391deebf3b3507c69b +Z 897d4bc356296ccb4d7eda6e238b1690 diff --git a/manifest.uuid b/manifest.uuid index c0c3e4bb99..ed0e54318a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f97626f921dafe596b615a168ef31987f4a1c0b52956443e1a5c1148b49cab74 \ No newline at end of file +b6d7d42b7426622a26b67809cd1f21285fea120aa1897377b9946840463b41f1 \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index 7ef99fd525..eb4adbbc69 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -3848,7 +3848,7 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){ ErrorMsg(lemp->filename,rp->ruleline, "%s(%s) has the same label as the LHS but is not the left-most " "symbol on the RHS.", - rp->rhs[i]->name, rp->rhsalias); + rp->rhs[i]->name, rp->rhsalias[i]); lemp->errorcnt++; } for(j=0; j Date: Tue, 30 Apr 2019 15:36:32 +0000 Subject: [PATCH 27/38] Fix a problem allowing a Table object to be deleted from within a call to the xDestroy method of the associated virtual table, causing a use-after-free error. FossilOrigin-Name: 1dbbb0101e8213b92b9a4c78c0fd2f9d0240a8ea3b40dff1033d1b8d71fb04ef --- manifest | 15 ++++++++------- manifest.uuid | 2 +- src/vtab.c | 2 ++ test/fts4rename.test | 44 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 8 deletions(-) create mode 100644 test/fts4rename.test diff --git a/manifest b/manifest index 2b7a987399..18cfa0b238 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\serror\smessage\sin\sthe\sLemon\sparser\sgenerator. -D 2019-04-30T14:26:31.199 +C Fix\sa\sproblem\sallowing\sa\sTable\sobject\sto\sbe\sdeleted\sfrom\swithin\sa\scall\sto\sthe\sxDestroy\smethod\sof\sthe\sassociated\svirtual\stable,\scausing\sa\suse-after-free\serror. +D 2019-04-30T15:36:32.034 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -600,7 +600,7 @@ F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c9419 F src/vdbemem.c dd2ee49255c4c5450f2b0887ef44cea8faa1cd7a46501b39a1a82b113ae418e3 F src/vdbesort.c 66592d478dbb46f19aed0b42222325eadb84deb40a90eebe25c6e7c1d8468f47 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 -F src/vtab.c 4c5959e00b7a142198d178e3a822f4e05f36f2d1a3c57657373f9487154fc06b +F src/vtab.c 1fa256c6ddad7a81e2a4dc080d015d4b0a7135767717d311298e47f6fca64bb3 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b09a2a9cab50efa08451a8c81d47052120ad5da174048c6d0b08d405384abdf2 F src/wal.h 606292549f5a7be50b6227bd685fa76e3a4affad71bb8ac5ce4cb5c79f6a176a @@ -975,6 +975,7 @@ F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b F test/fts4noti.test 5553d7bb2e20bf4a06b23e849352efc022ce6309 F test/fts4onepass.test d69ddc4ee3415e40b0c5d1d0408488a87614d4f63ba9c44f3e52db541d6b7cc7 F test/fts4opt.test 0fd0cc84000743ff2a883b9b84b4a5be07249f0ba790c8848a757164cdd46b2a +F test/fts4rename.test 6015a355ec3a11a51eb5b88802b3b2c1788786c54b77b17f3e077b7c93ff8611 F test/fts4umlaut.test fcaca4471de7e78c9d1f7e8976e3e8704d7d8ad979d57a739d00f3f757380429 F test/fts4unicode.test ceca76422abc251818cb25dabe33d3c3970da5f7c90e1540f190824e6b3a7c95 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d @@ -1821,7 +1822,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P f97626f921dafe596b615a168ef31987f4a1c0b52956443e1a5c1148b49cab74 -R 716d456c280daf5c4e086f08a5ceb68c -U drh -Z 897d4bc356296ccb4d7eda6e238b1690 +P b6d7d42b7426622a26b67809cd1f21285fea120aa1897377b9946840463b41f1 +R 6adb84036e04ae37c244138c75843359 +U dan +Z bb2c7686d9426e0e670050982240668c diff --git a/manifest.uuid b/manifest.uuid index ed0e54318a..3189f67ccc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b6d7d42b7426622a26b67809cd1f21285fea120aa1897377b9946840463b41f1 \ No newline at end of file +1dbbb0101e8213b92b9a4c78c0fd2f9d0240a8ea3b40dff1033d1b8d71fb04ef \ No newline at end of file diff --git a/src/vtab.c b/src/vtab.c index 7806eb946f..41e26ef62f 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -841,6 +841,7 @@ int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab){ p = vtabDisconnectAll(db, pTab); xDestroy = p->pMod->pModule->xDestroy; assert( xDestroy!=0 ); /* Checked before the virtual table is created */ + pTab->nTabRef++; rc = xDestroy(p->pVtab); /* Remove the sqlite3_vtab* from the aVTrans[] array, if applicable */ if( rc==SQLITE_OK ){ @@ -849,6 +850,7 @@ int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab){ pTab->pVTable = 0; sqlite3VtabUnlock(p); } + sqlite3DeleteTable(db, pTab); } return rc; diff --git a/test/fts4rename.test b/test/fts4rename.test new file mode 100644 index 0000000000..1c711e74e6 --- /dev/null +++ b/test/fts4rename.test @@ -0,0 +1,44 @@ +# 2019 April 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/fts3_common.tcl +set ::testprefix fts4rename + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE temp.t1 USING fts3(a); + BEGIN; + CREATE TABLE t2(x); +} {} + +do_catchsql_test 1.1 { + ALTER TABLE t1_content RENAME c0a TO docid; +} {1 {duplicate column name: docid}} + +do_catchsql_test 1.2 { + UPDATE t1 SET Col0 = 1 ; +} {1 {no such column: Col0}} + +do_catchsql_test 1.3 { + ROLLBACK; + DROP TABLE t1; +} {0 {}} + +finish_test + From 919458923dd9ea81db155fce5976a42207be11c7 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 30 Apr 2019 20:43:10 +0000 Subject: [PATCH 28/38] Update wapptest.tcl to use a simpler slave script. And to leave scripts wapptest_configure.sh and wapptest_make.sh in each test directory. FossilOrigin-Name: 07e527d781838412b2a434e64baaa49cbf7410a51c7393f54adc7b8eaffd5229 --- manifest | 13 ++--- manifest.uuid | 2 +- test/wapptest.tcl | 122 +++++++++++++++++++++++++++++++++++----------- 3 files changed, 102 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index 18cfa0b238..52011015d3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sallowing\sa\sTable\sobject\sto\sbe\sdeleted\sfrom\swithin\sa\scall\sto\sthe\sxDestroy\smethod\sof\sthe\sassociated\svirtual\stable,\scausing\sa\suse-after-free\serror. -D 2019-04-30T15:36:32.034 +C Update\swapptest.tcl\sto\suse\sa\ssimpler\sslave\sscript.\sAnd\sto\sleave\sscripts\swapptest_configure.sh\sand\swapptest_make.sh\sin\seach\stest\sdirectory. +D 2019-04-30T20:43:10.531 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1656,7 +1656,7 @@ F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test 14b20fcfa6ae152f5d8e12f5dc8a8a724b7ef189f5d8ef1e2ceab79f2af51747 F test/walvfs.test c0faffda13d045a96dfc541347886bb1a3d6f3205857fc98e683edfab766ea88 F test/wapp.tcl b440cd8cf57953d3a49e7ee81e6a18f18efdaf113b69f7d8482b0710a64566ec -F test/wapptest.tcl 78aff97afe76fd9728cf5f84710a772412735bc68a612b4789279072177a424e x +F test/wapptest.tcl 7cdac27ab7945b96719aea1afdeac5922b38657d488c676b91fff173552087e0 x F test/where.test 0607caa5a1fbfe7b93b95705981b463a3a0408038f22ae6e9dc11b36902b0e95 F test/where2.test 478d2170637b9211f593120648858593bf2445a1 F test/where3.test 2341a294e17193a6b1699ea7f192124a5286ca6acfcc3f4b06d16c931fbcda2c @@ -1822,7 +1822,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b6d7d42b7426622a26b67809cd1f21285fea120aa1897377b9946840463b41f1 -R 6adb84036e04ae37c244138c75843359 +P 1dbbb0101e8213b92b9a4c78c0fd2f9d0240a8ea3b40dff1033d1b8d71fb04ef +R f8f2ef19afa4c11d5a1036e92efd9ad2 +T +closed 9e849f25029b05b44220acd881356bcade9ba76f9d1308556850cd4d6b6a94d7 U dan -Z bb2c7686d9426e0e670050982240668c +Z 8ade123def7ea1f2c0a2e92932d92c90 diff --git a/manifest.uuid b/manifest.uuid index 3189f67ccc..0f16a1f690 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1dbbb0101e8213b92b9a4c78c0fd2f9d0240a8ea3b40dff1033d1b8d71fb04ef \ No newline at end of file +07e527d781838412b2a434e64baaa49cbf7410a51c7393f54adc7b8eaffd5229 \ No newline at end of file diff --git a/test/wapptest.tcl b/test/wapptest.tcl index d12d8e7222..be6271a9c0 100755 --- a/test/wapptest.tcl +++ b/test/wapptest.tcl @@ -37,9 +37,6 @@ proc wapptest_init {} { # The root of the SQLite source tree. set G(srcdir) [file dirname [file dirname [info script]]] - # releasetest.tcl script - set G(releaseTest) [file join [file dirname [info script]] releasetest.tcl] - set G(sqlite_version) "unknown" # Either "config", "running" or "stopped": @@ -52,28 +49,20 @@ proc wapptest_init {} { append G(host) " $::tcl_platform(machine) $::tcl_platform(byteOrder)" } -# Check to see if there are uncommitted changes in the SQLite source -# directory. Return true if there are, or false otherwise. +# Generate the text for the box at the top of the UI. The current SQLite +# version, according to fossil, along with a warning if there are +# uncommitted changes in the checkout. # -proc check_uncommitted {} { - global G - set ret 0 - set pwd [pwd] - cd $G(srcdir) - if {[catch {exec fossil changes} res]==0 && [string trim $res]!=""} { - set ret 1 - } - cd $pwd - return $ret -} - proc generate_fossil_info {} { global G set pwd [pwd] cd $G(srcdir) - if {[catch {exec fossil info} r1]} return - if {[catch {exec fossil changes} r2]} return + set rc [catch { + set r1 [exec fossil info] + set r2 [exec fossil changes] + }] cd $pwd + if {$rc} return foreach line [split $r1 "\n"] { if {[regexp {^checkout: *(.*)$} $line -> co]} { @@ -239,6 +228,88 @@ proc slave_fileevent {name} { do_some_stuff } +proc wapptest_slave_script {} { + global G + set res { + proc readfile {filename} { + set fd [open $filename] + set data [read $fd] + close $fd + return $data + } + } + + if {$G(msvc)==0} { + append res { + set cfg [readfile wapptest_configure.sh] + set rc [catch { exec {*}$cfg >& test.log } msg] + if {$rc==0} { + set make [readfile wapptest_make.sh] + catch { exec {*}$make >>& test.log } + } + } + } else { + append res { + set make [readfile wapptest_make.sh] + catch { exec {*}$make >>& test.log } + } + } + + set res +} + + +# Launch a slave process to run a test. +# +proc slave_launch { + name wtcl title dir configOpts testtarget makeOpts cflags opts +} { + global G + + catch { file mkdir $dir } msg + foreach f [glob -nocomplain [file join $dir *]] { + catch { file delete -force $f } + } + + # Write the configure command to wapptest_configure.sh. This file + # is empty if using MSVC - MSVC does not use configure. + # + set fd1 [open [file join $dir wapptest_configure.sh] w] + if {$G(msvc)==0} { + puts $fd1 "[file join .. $G(srcdir) configure] $wtcl $configOpts" + } + close $fd1 + + # Write the make command to wapptest_make.sh. Using nmake for MSVC and + # make for all other systems. + # + set makecmd "make" + if {$G(msvc)} { + set nativedir [file nativename $G(srcdir)] + set nativedir [string map [list "\\" "\\\\"] $nativedir] + set makecmd "nmake /f [file join $nativedir Makefile.msc] TOP=$nativedir" + } + set fd2 [open [file join $dir wapptest_make.sh] w] + puts $fd2 "$makecmd $makeOpts $testtarget \"CFLAGS=$cflags\" \"OPTS=$opts\"" + close $fd2 + + # Write the wapptest_run.tcl script to the test directory. To run the + # commands in the other two files. + # + set fd3 [open [file join $dir wapptest_run.tcl] w] + puts $fd3 [wapptest_slave_script] + close $fd3 + + set pwd [pwd] + cd $dir + set fd [open "|[info nameofexecutable] wapptest_run.tcl" r+] + cd $pwd + + set G(test.$name.channel) $fd + fconfigure $fd -blocking 0 + fileevent $fd readable [list slave_fileevent $name] +} + proc do_some_stuff {} { global G @@ -275,15 +346,9 @@ proc do_some_stuff {} { if { ![info exists G(test.$name.channel)] && ![info exists G(test.$name.done)] } { + set target [dict get $j target] set G(test.$name.start) [clock seconds] - set fd [open "|[info nameofexecutable] $G(releaseTest) --slave" r+] - set G(test.$name.channel) $fd - fconfigure $fd -blocking 0 - fileevent $fd readable [list slave_fileevent $name] - - puts $fd [list 0 $G(msvc) 0 $G(keep)] - set wtcl "" if {$G(tcl)!=""} { set wtcl "--with-tcl=$G(tcl)" } @@ -303,8 +368,9 @@ proc do_some_stuff {} { } set L [make_test_suite $G(msvc) $wtcl $name $target $opts] - puts $fd $L - flush $fd + set G(test.$name.log) [file join [lindex $L 1] test.log] + slave_launch $name $wtcl {*}$L + set G(test.$name.log) [file join [lindex $L 1] test.log] incr nLaunch -1 } From a3e086d80362607fcee57c3e9d9a5c9eef9fbcfc Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 1 May 2019 08:48:44 +0000 Subject: [PATCH 29/38] Fix an incompatibility with auto-vacuum mode in new test script recover.test. FossilOrigin-Name: 36dd5b0804797a35d0dc596b6ca4f71813a155c5a470237ab6e3d1bcd9ccc6be --- manifest | 13 ++++++------- manifest.uuid | 2 +- test/recover.test | 1 + 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 52011015d3..30142ae07f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\swapptest.tcl\sto\suse\sa\ssimpler\sslave\sscript.\sAnd\sto\sleave\sscripts\swapptest_configure.sh\sand\swapptest_make.sh\sin\seach\stest\sdirectory. -D 2019-04-30T20:43:10.531 +C Fix\san\sincompatibility\swith\sauto-vacuum\smode\sin\snew\stest\sscript\srecover.test. +D 2019-05-01T08:48:44.061 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1227,7 +1227,7 @@ F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 -F test/recover.test 52609c8cc24e72d3d8a20fb8bc32ba2ce8ca2093a7f4573bd4f2969f78f6d2b4 +F test/recover.test 4c45b1519c11c6d10cc28bced260e2a18f3967fc6abccaace44af6c22b30280c F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/regexp2.test 40e894223b3d6672655481493f1be12012f2b33c F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 @@ -1822,8 +1822,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 1dbbb0101e8213b92b9a4c78c0fd2f9d0240a8ea3b40dff1033d1b8d71fb04ef -R f8f2ef19afa4c11d5a1036e92efd9ad2 -T +closed 9e849f25029b05b44220acd881356bcade9ba76f9d1308556850cd4d6b6a94d7 +P 07e527d781838412b2a434e64baaa49cbf7410a51c7393f54adc7b8eaffd5229 +R 5ef0ce02daac1f5b87f34d32f98b67b4 U dan -Z 8ade123def7ea1f2c0a2e92932d92c90 +Z 076daaf71d4826b61a79c8ff95610b0a diff --git a/manifest.uuid b/manifest.uuid index 0f16a1f690..795204fd4c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07e527d781838412b2a434e64baaa49cbf7410a51c7393f54adc7b8eaffd5229 \ No newline at end of file +36dd5b0804797a35d0dc596b6ca4f71813a155c5a470237ab6e3d1bcd9ccc6be \ No newline at end of file diff --git a/test/recover.test b/test/recover.test index 4dad0d646d..75360bc86b 100644 --- a/test/recover.test +++ b/test/recover.test @@ -91,6 +91,7 @@ do_recover_test 1.3.2 #------------------------------------------------------------------------- reset_db do_execsql_test 2.1.0 { + PRAGMA auto_vacuum = 0; CREATE TABLE t1(a, b, c, PRIMARY KEY(b, c)) WITHOUT ROWID; INSERT INTO t1 VALUES(1, 2, 3); INSERT INTO t1 VALUES(4, 5, 6); From 6fcc1ecc99dbf222394bbfe207496b1148ece800 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 May 2019 14:41:47 +0000 Subject: [PATCH 30/38] In "PRAGMA vdbe_trace" output, show the results of OP_Affinity opcodes. FossilOrigin-Name: 56604bb60a8ebac8d2854628d1b052d594d7effe14be8333977995dc07b65114 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 30142ae07f..9b557e68a4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sincompatibility\swith\sauto-vacuum\smode\sin\snew\stest\sscript\srecover.test. -D 2019-05-01T08:48:44.061 +C In\s"PRAGMA\svdbe_trace"\soutput,\sshow\sthe\sresults\sof\sOP_Affinity\sopcodes. +D 2019-05-01T14:41:47.678 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/upsert.c 0dd81b40206841814d46942a7337786932475f085716042d0cb2fc7791bf8ca4 F src/utf.c 2f0fac345c7660d5c5bd3df9e9d8d33d4c27f366bcfb09e07443064d751a0507 F src/util.c 5061987401c2e8003177fa30d73196aa036727c8f04bf36a2df0c82b1904a236 F src/vacuum.c 82dcec9e7b1afa980288718ad11bc499651c722d7b9f32933c4d694d91cb6ebf -F src/vdbe.c 2782da511932d17e45e128f04f73314e5f5d765f7a499f5ac4bb1e0a82e082cc +F src/vdbe.c 74ee707ef31b74edc05923cddc04657d165c111fa9e6fe957df8fa7d5b63b8cf F src/vdbe.h 712bca562eaed1c25506b9faf9680bdc75fc42e2f4a1cd518d883fa79c7a4237 F src/vdbeInt.h 2c12704db9740c8e899786ecfc7a5797a9d067563496eb1b6ed03c592d7b8d90 F src/vdbeapi.c 2ddd60f4a351f15ee98d841e346af16111ad59dfa4d25d2dd4012e9875bf7d92 @@ -1822,7 +1822,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 07e527d781838412b2a434e64baaa49cbf7410a51c7393f54adc7b8eaffd5229 -R 5ef0ce02daac1f5b87f34d32f98b67b4 -U dan -Z 076daaf71d4826b61a79c8ff95610b0a +P 36dd5b0804797a35d0dc596b6ca4f71813a155c5a470237ab6e3d1bcd9ccc6be +R ea64878fb21eadae4dca5f57318c8810 +U drh +Z 04e7e3cb0fe5e91830c8ba41df015203 diff --git a/manifest.uuid b/manifest.uuid index 795204fd4c..b8c0cbeaad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -36dd5b0804797a35d0dc596b6ca4f71813a155c5a470237ab6e3d1bcd9ccc6be \ No newline at end of file +56604bb60a8ebac8d2854628d1b052d594d7effe14be8333977995dc07b65114 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index faaf947308..a5c339bdf5 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2769,6 +2769,7 @@ case OP_Affinity: { assert( pIn1 <= &p->aMem[(p->nMem+1 - p->nCursor)] ); assert( memIsValid(pIn1) ); applyAffinity(pIn1, *(zAffinity++), encoding); + REGISTER_TRACE((int)(pIn1-aMem), pIn1); pIn1++; }while( zAffinity[0] ); break; From c97001fd55d61b5214fe91b764795d8f04360dae Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 1 May 2019 15:25:38 +0000 Subject: [PATCH 31/38] Update wapptest.tcl so that it deletes extra files if the "Keep files:" checkbox is clear. Set it by default. FossilOrigin-Name: 09623cc4cc82e3c123d1fd5d88b2f4b50ec5f2cc7e579a7203258bf0c246a74f --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/wapptest.tcl | 37 +++++++++++++++++++++++++++++++++++-- 3 files changed, 43 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 9b557e68a4..a50a5799d9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\s"PRAGMA\svdbe_trace"\soutput,\sshow\sthe\sresults\sof\sOP_Affinity\sopcodes. -D 2019-05-01T14:41:47.678 +C Update\swapptest.tcl\sso\sthat\sit\sdeletes\sextra\sfiles\sif\sthe\s"Keep\sfiles:"\scheckbox\sis\sclear.\sSet\sit\sby\sdefault. +D 2019-05-01T15:25:38.538 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1656,7 +1656,7 @@ F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test 14b20fcfa6ae152f5d8e12f5dc8a8a724b7ef189f5d8ef1e2ceab79f2af51747 F test/walvfs.test c0faffda13d045a96dfc541347886bb1a3d6f3205857fc98e683edfab766ea88 F test/wapp.tcl b440cd8cf57953d3a49e7ee81e6a18f18efdaf113b69f7d8482b0710a64566ec -F test/wapptest.tcl 7cdac27ab7945b96719aea1afdeac5922b38657d488c676b91fff173552087e0 x +F test/wapptest.tcl c4c32b89607f970500568cad83ed67a869beaf8c2f07bbc030a335ea6c7c750c x F test/where.test 0607caa5a1fbfe7b93b95705981b463a3a0408038f22ae6e9dc11b36902b0e95 F test/where2.test 478d2170637b9211f593120648858593bf2445a1 F test/where3.test 2341a294e17193a6b1699ea7f192124a5286ca6acfcc3f4b06d16c931fbcda2c @@ -1822,7 +1822,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 36dd5b0804797a35d0dc596b6ca4f71813a155c5a470237ab6e3d1bcd9ccc6be -R ea64878fb21eadae4dca5f57318c8810 -U drh -Z 04e7e3cb0fe5e91830c8ba41df015203 +P 56604bb60a8ebac8d2854628d1b052d594d7effe14be8333977995dc07b65114 +R 927c208fc5b5ee3c4c01e2595fed96f3 +U dan +Z 6871de4eaeba6ddd2436f9afc805bd1a diff --git a/manifest.uuid b/manifest.uuid index b8c0cbeaad..7af2ec6456 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -56604bb60a8ebac8d2854628d1b052d594d7effe14be8333977995dc07b65114 \ No newline at end of file +09623cc4cc82e3c123d1fd5d88b2f4b50ec5f2cc7e579a7203258bf0c246a74f \ No newline at end of file diff --git a/test/wapptest.tcl b/test/wapptest.tcl index be6271a9c0..fe7ea202e8 100755 --- a/test/wapptest.tcl +++ b/test/wapptest.tcl @@ -20,8 +20,8 @@ source [file join [file dirname [info script]] releasetest_data.tcl] # set G(platform) $::tcl_platform(os)-$::tcl_platform(machine) set G(test) Normal -set G(keep) 0 -set G(msvc) 0 +set G(keep) 1 +set G(msvc) [expr {$::tcl_platform(platform)=="windows"}] set G(tcl) [::tcl::pkgconfig get libdir,install] set G(jobs) 3 set G(debug) 0 @@ -197,6 +197,10 @@ proc count_tests_and_errors {name logfile} { } } +# This command is invoked once a slave process has finished running its +# tests, successfully or otherwise. Parameter $name is the name of the +# test, $rc the exit code returned by the slave process. +# proc slave_test_done {name rc} { global G set G(test.$name.done) [clock seconds] @@ -209,8 +213,28 @@ proc slave_test_done {name rc} { if {[file exists $G(test.$name.log)]} { count_tests_and_errors $name $G(test.$name.log) } + + # If the "keep files" checkbox is clear, delete all files except for + # the executables and test logs. And any core file that is present. + if {$G(keep)==0} { + set keeplist { + testfixture testfixture.exe + sqlite3 sqlite3.exe + test.log test-out.txt + core + } + foreach f [glob -nocomplain [file join $G(test.$name.dir) *]] { + set t [file tail $f] + if {[lsearch $keeplist $t]<0} { + catch { file delete -force $f } + } + } + } } +# This is a fileevent callback invoked each time a file-descriptor that +# connects this process to a slave process is readable. +# proc slave_fileevent {name} { global G set fd $G(test.$name.channel) @@ -228,6 +252,14 @@ proc slave_fileevent {name} { do_some_stuff } +# Return the contents of the "slave script" - the script run by slave +# processes to actually perform the test. It does two things: +# +# 1. Reads and [exec]s the contents of file wapptest_configure.sh. +# 2. Reads and [exec]s the contents of file wapptest_make.sh. +# +# Step 1 is omitted if the test uses MSVC (which does not use configure). +# proc wapptest_slave_script {} { global G set res { @@ -270,6 +302,7 @@ proc slave_launch { foreach f [glob -nocomplain [file join $dir *]] { catch { file delete -force $f } } + set G(test.$name.dir) $dir # Write the configure command to wapptest_configure.sh. This file # is empty if using MSVC - MSVC does not use configure. From 8a7e11fb7894e286ac9f66ff93a6c8b0668e954a Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 May 2019 15:32:40 +0000 Subject: [PATCH 32/38] Avoid unwelcomed side effects on the input operands in the OP_Concat operator. Fix for ticket [3be1295b264be2fac49b681] FossilOrigin-Name: 713caa382cf7ddef872e510a76a5fca40be1a8d8876ce2f91b632bb0810a6630 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbe.c | 37 ++++++++++++++++++++++++------------- test/index.test | 9 +++++++++ 4 files changed, 42 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index a50a5799d9..12c2ec4d32 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\swapptest.tcl\sso\sthat\sit\sdeletes\sextra\sfiles\sif\sthe\s"Keep\sfiles:"\scheckbox\sis\sclear.\sSet\sit\sby\sdefault. -D 2019-05-01T15:25:38.538 +C Avoid\sunwelcomed\sside\seffects\son\sthe\sinput\soperands\sin\sthe\sOP_Concat\noperator.\s\sFix\sfor\sticket\s[3be1295b264be2fac49b681] +D 2019-05-01T15:32:40.553 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/upsert.c 0dd81b40206841814d46942a7337786932475f085716042d0cb2fc7791bf8ca4 F src/utf.c 2f0fac345c7660d5c5bd3df9e9d8d33d4c27f366bcfb09e07443064d751a0507 F src/util.c 5061987401c2e8003177fa30d73196aa036727c8f04bf36a2df0c82b1904a236 F src/vacuum.c 82dcec9e7b1afa980288718ad11bc499651c722d7b9f32933c4d694d91cb6ebf -F src/vdbe.c 74ee707ef31b74edc05923cddc04657d165c111fa9e6fe957df8fa7d5b63b8cf +F src/vdbe.c 57b0b697d349876716499e073fb5e2d20ebc6cc0f752327a4e54031ed7e062f3 F src/vdbe.h 712bca562eaed1c25506b9faf9680bdc75fc42e2f4a1cd518d883fa79c7a4237 F src/vdbeInt.h 2c12704db9740c8e899786ecfc7a5797a9d067563496eb1b6ed03c592d7b8d90 F src/vdbeapi.c 2ddd60f4a351f15ee98d841e346af16111ad59dfa4d25d2dd4012e9875bf7d92 @@ -1031,7 +1031,7 @@ F test/incrvacuum.test 2aaee202b1f230e55779f70d155f6ba67bbdff8481d650214d256ab0f F test/incrvacuum2.test 7d26cfda66c7e55898d196de54ac4ec7d86a4e3d F test/incrvacuum3.test 75256fb1377e7c39ef2de62bfc42bbff67be295a F test/incrvacuum_ioerr.test 6ae2f783424e47a0033304808fe27789cf93e635 -F test/index.test df4cddf4435314a948237fdfa9acee67de21f7bebc789beab4b89b575b4f6a70 +F test/index.test 58d6fba7748b7c545080759b334e759b22a06b728aa68a2abd106b8065184bdd F test/index2.test f835d5e13ca163bd78c4459ca15fd2e4ed487407 F test/index3.test 51685f39345462b84fcf77eb8537af847fdf438cc96b05c45d6aaca4e473ade0 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 @@ -1822,7 +1822,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 56604bb60a8ebac8d2854628d1b052d594d7effe14be8333977995dc07b65114 -R 927c208fc5b5ee3c4c01e2595fed96f3 -U dan -Z 6871de4eaeba6ddd2436f9afc805bd1a +P 09623cc4cc82e3c123d1fd5d88b2f4b50ec5f2cc7e579a7203258bf0c246a74f +R 36135537650a3222bb6001d7e31d8024 +U drh +Z 34e12fa35e79c97a2bb14481b73be0be diff --git a/manifest.uuid b/manifest.uuid index 7af2ec6456..566972896c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -09623cc4cc82e3c123d1fd5d88b2f4b50ec5f2cc7e579a7203258bf0c246a74f \ No newline at end of file +713caa382cf7ddef872e510a76a5fca40be1a8d8876ce2f91b632bb0810a6630 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index a5c339bdf5..455d2b618a 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -195,14 +195,6 @@ int sqlite3_found_count = 0; } #endif -/* -** Convert the given register into a string if it isn't one -** already. Return non-zero if a malloc() fails. -*/ -#define Stringify(P, enc) \ - if(((P)->flags&(MEM_Str|MEM_Blob))==0 && sqlite3VdbeMemStringify(P,enc,0)) \ - { goto no_mem; } - /* ** An ephemeral string value (signified by the MEM_Ephem flag) contains ** a pointer to a dynamically allocated string where some other entity @@ -1463,19 +1455,34 @@ case OP_ResultRow: { ** to avoid a memcpy(). */ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ - i64 nByte; + i64 nByte; /* Total size of the output string or blob */ + u16 flags1; /* Initial flags for P1 */ + u16 flags2; /* Initial flags for P2 */ pIn1 = &aMem[pOp->p1]; pIn2 = &aMem[pOp->p2]; pOut = &aMem[pOp->p3]; + testcase( pIn1==pIn2 ); + testcase( pOut==pIn2 ); assert( pIn1!=pOut ); - if( (pIn1->flags | pIn2->flags) & MEM_Null ){ + flags1 = pIn1->flags; + testcase( flags1 & MEM_Null ); + testcase( pIn2->flags & MEM_Null ); + if( (flags1 | pIn2->flags) & MEM_Null ){ sqlite3VdbeMemSetNull(pOut); break; } - if( ExpandBlob(pIn1) || ExpandBlob(pIn2) ) goto no_mem; - Stringify(pIn1, encoding); - Stringify(pIn2, encoding); + if( (flags1 & (MEM_Str|MEM_Blob))==0 ){ + if( sqlite3VdbeMemStringify(pIn1,encoding,0) ) goto no_mem; + }else if( (flags1 & MEM_Zero)!=0 ){ + if( sqlite3VdbeMemExpandBlob(pIn1) ) goto no_mem; + } + flags2 = pIn2->flags; + if( (flags2 & (MEM_Str|MEM_Blob))==0 ){ + if( sqlite3VdbeMemStringify(pIn2,encoding,0) ) goto no_mem; + }else if( (flags2 & MEM_Zero)!=0 ){ + if( sqlite3VdbeMemExpandBlob(pIn2) ) goto no_mem; + } nByte = pIn1->n + pIn2->n; if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; @@ -1486,8 +1493,12 @@ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ MemSetTypeFlag(pOut, MEM_Str); if( pOut!=pIn2 ){ memcpy(pOut->z, pIn2->z, pIn2->n); + assert( (pIn2->flags & MEM_Dyn) == (flags2 & MEM_Dyn) ); + pIn2->flags = flags2; } memcpy(&pOut->z[pIn2->n], pIn1->z, pIn1->n); + assert( (pIn1->flags & MEM_Dyn) == (flags1 & MEM_Dyn) ); + pIn1->flags = flags1; pOut->z[nByte]=0; pOut->z[nByte+1] = 0; pOut->flags |= MEM_Term; diff --git a/test/index.test b/test/index.test index ae16470570..e6c3d94eb5 100644 --- a/test/index.test +++ b/test/index.test @@ -738,6 +738,15 @@ do_test index-21.2 { } } {0 {9 5 1}} +# 2019-05-01 ticket https://www.sqlite.org/src/info/3be1295b264be2fa +do_execsql_test index-22.0 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a, b TEXT); + CREATE UNIQUE INDEX IF NOT EXISTS x1 ON t1(b==0); + CREATE INDEX IF NOT EXISTS x2 ON t1(a || 0) WHERE b; + INSERT INTO t1(a,b) VALUES('a',1),('a',0); + SELECT a, b, '|' FROM t1; +} {a 1 | a 0 |} finish_test From 0de0ab82074f1898fe1b7374683aeea6afae0675 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 1 May 2019 17:32:36 +0000 Subject: [PATCH 33/38] Fix a case in wapptest.tcl where a failed test might report 0 errors. FossilOrigin-Name: 2be1ed70df605663822d1afdde757f426ccf2ee38add8dc6b6bb4fc4d90a31dc --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/wapptest.tcl | 9 +++++++-- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 12c2ec4d32..d503d5998c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sunwelcomed\sside\seffects\son\sthe\sinput\soperands\sin\sthe\sOP_Concat\noperator.\s\sFix\sfor\sticket\s[3be1295b264be2fac49b681] -D 2019-05-01T15:32:40.553 +C Fix\sa\scase\sin\swapptest.tcl\swhere\sa\sfailed\stest\smight\sreport\s0\serrors. +D 2019-05-01T17:32:36.167 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1656,7 +1656,7 @@ F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test 14b20fcfa6ae152f5d8e12f5dc8a8a724b7ef189f5d8ef1e2ceab79f2af51747 F test/walvfs.test c0faffda13d045a96dfc541347886bb1a3d6f3205857fc98e683edfab766ea88 F test/wapp.tcl b440cd8cf57953d3a49e7ee81e6a18f18efdaf113b69f7d8482b0710a64566ec -F test/wapptest.tcl c4c32b89607f970500568cad83ed67a869beaf8c2f07bbc030a335ea6c7c750c x +F test/wapptest.tcl 32a23f9b4c9fa1126d29250368ba6d5689b7503aa0694df7edf9253f1d56f1d7 x F test/where.test 0607caa5a1fbfe7b93b95705981b463a3a0408038f22ae6e9dc11b36902b0e95 F test/where2.test 478d2170637b9211f593120648858593bf2445a1 F test/where3.test 2341a294e17193a6b1699ea7f192124a5286ca6acfcc3f4b06d16c931fbcda2c @@ -1822,7 +1822,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 09623cc4cc82e3c123d1fd5d88b2f4b50ec5f2cc7e579a7203258bf0c246a74f -R 36135537650a3222bb6001d7e31d8024 -U drh -Z 34e12fa35e79c97a2bb14481b73be0be +P 713caa382cf7ddef872e510a76a5fca40be1a8d8876ce2f91b632bb0810a6630 +R dc354a168f1bf70c2fd5f402743058b3 +U dan +Z 22ec1d29c68aeb8e171bdc1d1c0ccaa3 diff --git a/manifest.uuid b/manifest.uuid index 566972896c..fa703b3e9a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -713caa382cf7ddef872e510a76a5fca40be1a8d8876ce2f91b632bb0810a6630 \ No newline at end of file +2be1ed70df605663822d1afdde757f426ccf2ee38add8dc6b6bb4fc4d90a31dc \ No newline at end of file diff --git a/test/wapptest.tcl b/test/wapptest.tcl index fe7ea202e8..88f0f074bf 100755 --- a/test/wapptest.tcl +++ b/test/wapptest.tcl @@ -222,6 +222,9 @@ proc slave_test_done {name rc} { sqlite3 sqlite3.exe test.log test-out.txt core + wapptest_make.sh + wapptest_configure.sh + wapptest_run.tcl } foreach f [glob -nocomplain [file join $G(test.$name.dir) *]] { set t [file tail $f] @@ -277,16 +280,18 @@ proc wapptest_slave_script {} { set rc [catch { exec {*}$cfg >& test.log } msg] if {$rc==0} { set make [readfile wapptest_make.sh] - catch { exec {*}$make >>& test.log } + set rc [catch { exec {*}$make >>& test.log }] } } } else { append res { set make [readfile wapptest_make.sh] - catch { exec {*}$make >>& test.log } + set rc [catch { exec {*}$make >>& test.log }] } } + append res { exit $rc } + set res } From f78408c775274e7b01ed9f836ef4c5c52d9463e8 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 1 May 2019 17:36:56 +0000 Subject: [PATCH 34/38] Fix an incompatibility with -DSQLITE_OMIT_LOAD_EXTENSION=1 in dbdata.test. FossilOrigin-Name: a77cd85b1a8b86e71b511f05f8c67faa046d24a48684139d2f64e51249203411 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/dbdata.test | 5 +++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index d503d5998c..16fb7d5c5c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scase\sin\swapptest.tcl\swhere\sa\sfailed\stest\smight\sreport\s0\serrors. -D 2019-05-01T17:32:36.167 +C Fix\san\sincompatibility\swith\s-DSQLITE_OMIT_LOAD_EXTENSION=1\sin\sdbdata.test. +D 2019-05-01T17:36:56.822 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -787,7 +787,7 @@ F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee F test/dataversion1.test 6e5e86ac681f0782e766ebcb56c019ae001522d114e0e111e5ebf68ccf2a7bb8 F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10 -F test/dbdata.test c8d97bafd1b2efb1e445871c4641208dcd91e686d2dfbb6463d83934adbd1ac5 +F test/dbdata.test 042f49acff3438f940eeba5868d3af080ae64ddf26ae78f80c92bec3ca7d8603 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e F test/dbfuzz001.test e32d14465f1c77712896fda6a1ccc0f037b481c191c1696a9c44f6c9e4964faf F test/dbfuzz2-seed1.db e6225c6f3d7b63f9c5b6867146a5f329d997ab105bee64644dc2b3a2f2aebaee @@ -1822,7 +1822,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 713caa382cf7ddef872e510a76a5fca40be1a8d8876ce2f91b632bb0810a6630 -R dc354a168f1bf70c2fd5f402743058b3 +P 2be1ed70df605663822d1afdde757f426ccf2ee38add8dc6b6bb4fc4d90a31dc +R ae90daf9628470675bc89644e7eb21cd U dan -Z 22ec1d29c68aeb8e171bdc1d1c0ccaa3 +Z 1604ae08830baf499595804bc5cb5239 diff --git a/manifest.uuid b/manifest.uuid index fa703b3e9a..96fada382a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2be1ed70df605663822d1afdde757f426ccf2ee38add8dc6b6bb4fc4d90a31dc \ No newline at end of file +a77cd85b1a8b86e71b511f05f8c67faa046d24a48684139d2f64e51249203411 \ No newline at end of file diff --git a/test/dbdata.test b/test/dbdata.test index 391b278025..5b1150c78f 100644 --- a/test/dbdata.test +++ b/test/dbdata.test @@ -20,8 +20,9 @@ ifcapable !vtab||!compound { finish_test return } -db enable_load_extension 1 -if { [catch { db eval { SELECT load_extension('../dbdata') } }] } { +if { [catch { db enable_load_extension 1 }] + || [catch { db eval { SELECT load_extension('../dbdata') } }] +} { finish_test return } From 83a1dafb03af90f40c75e4839c81cf74b5cbfbc0 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 May 2019 18:59:33 +0000 Subject: [PATCH 35/38] When values have real affinity and are converted into strings for CHECK constraints or index expressions, do the conversions into a real-number format even if the values are stored as integers for efficiency. This appears to fix ticket [ae0f637bddc5290b446]. FossilOrigin-Name: 5997d075665faca6b70fa647e877ebc84c473b32887b96235865d59ce80247f8 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/vdbe.c | 15 ++++++++++++--- src/vdbeInt.h | 2 +- src/vdbemem.c | 51 +++++++++++++++++++++++++++------------------------ 5 files changed, 50 insertions(+), 38 deletions(-) diff --git a/manifest b/manifest index 16fb7d5c5c..fdae1f543e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sincompatibility\swith\s-DSQLITE_OMIT_LOAD_EXTENSION=1\sin\sdbdata.test. -D 2019-05-01T17:36:56.822 +C When\svalues\shave\sreal\saffinity\sand\sare\sconverted\sinto\sstrings\sfor\sCHECK\nconstraints\sor\sindex\sexpressions,\sdo\sthe\sconversions\sinto\sa\sreal-number\sformat\neven\sif\sthe\svalues\sare\sstored\sas\sintegers\sfor\sefficiency.\nThis\sappears\sto\sfix\sticket\s[ae0f637bddc5290b446]. +D 2019-05-01T18:59:33.898 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,13 +591,13 @@ F src/upsert.c 0dd81b40206841814d46942a7337786932475f085716042d0cb2fc7791bf8ca4 F src/utf.c 2f0fac345c7660d5c5bd3df9e9d8d33d4c27f366bcfb09e07443064d751a0507 F src/util.c 5061987401c2e8003177fa30d73196aa036727c8f04bf36a2df0c82b1904a236 F src/vacuum.c 82dcec9e7b1afa980288718ad11bc499651c722d7b9f32933c4d694d91cb6ebf -F src/vdbe.c 57b0b697d349876716499e073fb5e2d20ebc6cc0f752327a4e54031ed7e062f3 +F src/vdbe.c c15d6a105c41db6a166b0aa9650829bdc0d92918a8926a92332ea1feb27c33ba F src/vdbe.h 712bca562eaed1c25506b9faf9680bdc75fc42e2f4a1cd518d883fa79c7a4237 -F src/vdbeInt.h 2c12704db9740c8e899786ecfc7a5797a9d067563496eb1b6ed03c592d7b8d90 +F src/vdbeInt.h 0e2c44958fb42d90a4eacb122d77e2d5b89b82f5e2b4b047b422962dc0346357 F src/vdbeapi.c 2ddd60f4a351f15ee98d841e346af16111ad59dfa4d25d2dd4012e9875bf7d92 F src/vdbeaux.c f873b5c2efcf8a4d6ecfc5b1a5b06fd810419198f3bd882175d371cc03801873 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 -F src/vdbemem.c dd2ee49255c4c5450f2b0887ef44cea8faa1cd7a46501b39a1a82b113ae418e3 +F src/vdbemem.c df36fd36c7585e42869f3a44f5da5dc70e13306bc97ba52eebe069e174ba55db F src/vdbesort.c 66592d478dbb46f19aed0b42222325eadb84deb40a90eebe25c6e7c1d8468f47 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 F src/vtab.c 1fa256c6ddad7a81e2a4dc080d015d4b0a7135767717d311298e47f6fca64bb3 @@ -1822,7 +1822,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 2be1ed70df605663822d1afdde757f426ccf2ee38add8dc6b6bb4fc4d90a31dc -R ae90daf9628470675bc89644e7eb21cd -U dan -Z 1604ae08830baf499595804bc5cb5239 +P a77cd85b1a8b86e71b511f05f8c67faa046d24a48684139d2f64e51249203411 +R 24b657a2152d8497e843307e630b65a0 +U drh +Z 532f184e0e66c8f742c69b40ffa2c03e diff --git a/manifest.uuid b/manifest.uuid index 96fada382a..5fa0ea51e0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a77cd85b1a8b86e71b511f05f8c67faa046d24a48684139d2f64e51249203411 \ No newline at end of file +5997d075665faca6b70fa647e877ebc84c473b32887b96235865d59ce80247f8 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 455d2b618a..7fa7bc2a71 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -514,6 +514,8 @@ static void memTracePrint(Mem *p){ printf(p->flags & MEM_Zero ? " NULL-nochng" : " NULL"); }else if( (p->flags & (MEM_Int|MEM_Str))==(MEM_Int|MEM_Str) ){ printf(" si:%lld", p->u.i); + }else if( (p->flags & (MEM_Int|MEM_IntReal))==(MEM_Int|MEM_IntReal) ){ + printf(" ir:%lld", p->u.i); }else if( p->flags & MEM_Int ){ printf(" i:%lld", p->u.i); #ifndef SQLITE_OMIT_FLOATING_POINT @@ -2776,13 +2778,20 @@ case OP_Affinity: { assert( pOp->p2>0 ); assert( zAffinity[pOp->p2]==0 ); pIn1 = &aMem[pOp->p1]; - do{ + while( 1 /*edit-by-break*/ ){ assert( pIn1 <= &p->aMem[(p->nMem+1 - p->nCursor)] ); assert( memIsValid(pIn1) ); - applyAffinity(pIn1, *(zAffinity++), encoding); + applyAffinity(pIn1, zAffinity[0], encoding); + if( zAffinity[0]==SQLITE_AFF_REAL && (pIn1->flags & MEM_Int)!=0 ){ + /* When applying REAL affinity, if the result is still MEM_Int, + ** indicate that REAL is actually desired */ + pIn1->flags |= MEM_IntReal; + } REGISTER_TRACE((int)(pIn1-aMem), pIn1); + zAffinity++; + if( zAffinity[0]==0 ) break; pIn1++; - }while( zAffinity[0] ); + } break; } diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 15a371d550..c84e4e4390 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -247,7 +247,7 @@ struct sqlite3_value { #define MEM_Blob 0x0010 /* Value is a BLOB */ #define MEM_AffMask 0x001f /* Mask of affinity bits */ #define MEM_FromBind 0x0020 /* Value originates from sqlite3_bind() */ -/* Available 0x0040 */ +#define MEM_IntReal 0x0040 /* MEM_Int that stringifies like MEM_Real */ #define MEM_Undefined 0x0080 /* Value is undefined */ #define MEM_Cleared 0x0100 /* NULL set by OP_Null, not from data */ #define MEM_TypeMask 0xc1df /* Mask of type bits */ diff --git a/src/vdbemem.c b/src/vdbemem.c index 80d494867c..82f7f3e102 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -92,6 +92,25 @@ int sqlite3VdbeCheckMemInvariants(Mem *p){ } #endif +/* +** Render a Mem object which is either MEM_Int or MEM_Real into a +** buffer. +*/ +static void vdbeMemRenderNum(int sz, char *zBuf, Mem *p){ + StrAccum acc; + assert( p->flags & (MEM_Int|MEM_Real) ); + sqlite3StrAccumInit(&acc, 0, zBuf, sz, 0); + if( p->flags & MEM_IntReal ){ + sqlite3_str_appendf(&acc, "%!.15g", (double)p->u.i); + }else if( p->flags & MEM_Int ){ + sqlite3_str_appendf(&acc, "%lld", p->u.i); + }else{ + sqlite3_str_appendf(&acc, "%!.15g", p->u.r); + } + assert( acc.zText==zBuf && acc.mxAlloc<=0 ); + zBuf[acc.nChar] = 0; /* Fast version of sqlite3StrAccumFinish(&acc) */ +} + #ifdef SQLITE_DEBUG /* ** Check that string value of pMem agrees with its integer or real value. @@ -118,11 +137,7 @@ int sqlite3VdbeMemConsistentDualRep(Mem *p){ int i, j, incr; if( (p->flags & MEM_Str)==0 ) return 1; if( (p->flags & (MEM_Int|MEM_Real))==0 ) return 1; - if( p->flags & MEM_Int ){ - sqlite3_snprintf(sizeof(zBuf),zBuf,"%lld",p->u.i); - }else{ - sqlite3_snprintf(sizeof(zBuf),zBuf,"%!.15g",p->u.r); - } + vdbeMemRenderNum(sizeof(zBuf), zBuf, p); z = p->z; i = j = 0; incr = 1; @@ -248,7 +263,7 @@ int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ } assert( (pMem->flags & MEM_Dyn)==0 ); pMem->z = pMem->zMalloc; - pMem->flags &= (MEM_Null|MEM_Int|MEM_Real); + pMem->flags &= (MEM_Null|MEM_Int|MEM_Real|MEM_IntReal); return SQLITE_OK; } @@ -349,13 +364,12 @@ int sqlite3VdbeMemNulTerminate(Mem *pMem){ ** user and the latter is an internal programming error. */ int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ - int fg = pMem->flags; const int nByte = 32; assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( !(fg&MEM_Zero) ); - assert( !(fg&(MEM_Str|MEM_Blob)) ); - assert( fg&(MEM_Int|MEM_Real) ); + assert( !(pMem->flags&MEM_Zero) ); + assert( !(pMem->flags&(MEM_Str|MEM_Blob)) ); + assert( pMem->flags&(MEM_Int|MEM_Real) ); assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); @@ -365,23 +379,12 @@ int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ return SQLITE_NOMEM_BKPT; } - /* For a Real or Integer, use sqlite3_snprintf() to produce the UTF-8 - ** string representation of the value. Then, if the required encoding - ** is UTF-16le or UTF-16be do a translation. - ** - ** FIX ME: It would be better if sqlite3_snprintf() could do UTF-16. - */ - if( fg & MEM_Int ){ - sqlite3_snprintf(nByte, pMem->z, "%lld", pMem->u.i); - }else{ - assert( fg & MEM_Real ); - sqlite3_snprintf(nByte, pMem->z, "%!.15g", pMem->u.r); - } + vdbeMemRenderNum(nByte, pMem->z, pMem); assert( pMem->z!=0 ); pMem->n = sqlite3Strlen30NN(pMem->z); pMem->enc = SQLITE_UTF8; pMem->flags |= MEM_Str|MEM_Term; - if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real); + if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); sqlite3VdbeChangeEncoding(pMem, enc); return SQLITE_OK; } @@ -741,7 +744,7 @@ void sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ pMem->flags |= (pMem->flags&MEM_Blob)>>3; sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); - pMem->flags &= ~(MEM_Int|MEM_Real|MEM_Blob|MEM_Zero); + pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal|MEM_Blob|MEM_Zero); break; } } From 7d0a3fd340b3a74d3811ccd70d9df58b4f772aac Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 May 2019 19:01:27 +0000 Subject: [PATCH 36/38] Add a test case for ticket [ae0f637bddc5290b44669e066a]. FossilOrigin-Name: ece481695fc3c959c3eba0fb485cdda43a10b06d17259b0121e15bfc5e8e8b9f --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/index.test | 11 ++++++++++- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index fdae1f543e..6979d5436c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\svalues\shave\sreal\saffinity\sand\sare\sconverted\sinto\sstrings\sfor\sCHECK\nconstraints\sor\sindex\sexpressions,\sdo\sthe\sconversions\sinto\sa\sreal-number\sformat\neven\sif\sthe\svalues\sare\sstored\sas\sintegers\sfor\sefficiency.\nThis\sappears\sto\sfix\sticket\s[ae0f637bddc5290b446]. -D 2019-05-01T18:59:33.898 +C Add\sa\stest\scase\sfor\sticket\s[ae0f637bddc5290b44669e066a]. +D 2019-05-01T19:01:27.714 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -1031,7 +1031,7 @@ F test/incrvacuum.test 2aaee202b1f230e55779f70d155f6ba67bbdff8481d650214d256ab0f F test/incrvacuum2.test 7d26cfda66c7e55898d196de54ac4ec7d86a4e3d F test/incrvacuum3.test 75256fb1377e7c39ef2de62bfc42bbff67be295a F test/incrvacuum_ioerr.test 6ae2f783424e47a0033304808fe27789cf93e635 -F test/index.test 58d6fba7748b7c545080759b334e759b22a06b728aa68a2abd106b8065184bdd +F test/index.test 05414fc7e1e128c327e089c2216d041ae7fb02232571f708f009a79a482cf1a3 F test/index2.test f835d5e13ca163bd78c4459ca15fd2e4ed487407 F test/index3.test 51685f39345462b84fcf77eb8537af847fdf438cc96b05c45d6aaca4e473ade0 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 @@ -1822,7 +1822,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P a77cd85b1a8b86e71b511f05f8c67faa046d24a48684139d2f64e51249203411 -R 24b657a2152d8497e843307e630b65a0 +P 5997d075665faca6b70fa647e877ebc84c473b32887b96235865d59ce80247f8 +R c6735d94da423c75a5fbf44f945ad851 U drh -Z 532f184e0e66c8f742c69b40ffa2c03e +Z bc6fd770927f38308301380ce5d36731 diff --git a/manifest.uuid b/manifest.uuid index 5fa0ea51e0..a535d6515f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5997d075665faca6b70fa647e877ebc84c473b32887b96235865d59ce80247f8 \ No newline at end of file +ece481695fc3c959c3eba0fb485cdda43a10b06d17259b0121e15bfc5e8e8b9f \ No newline at end of file diff --git a/test/index.test b/test/index.test index e6c3d94eb5..4602041cd8 100644 --- a/test/index.test +++ b/test/index.test @@ -747,6 +747,15 @@ do_execsql_test index-22.0 { INSERT INTO t1(a,b) VALUES('a',1),('a',0); SELECT a, b, '|' FROM t1; } {a 1 | a 0 |} - + +# 2019-05-10 ticket https://www.sqlite.org/src/info/ae0f637bddc5290b +do_execsql_test index-23.0 { + DROP TABLE t1; + CREATE TABLE t1(a TEXT, b REAL); + CREATE UNIQUE INDEX t1x1 ON t1(a GLOB b); + INSERT INTO t1(a,b) VALUES('0.0','1'),('1.0','1'); + SELECT * FROM t1; + REINDEX; +} {0.0 1.0 1.0 1.0} finish_test From 01325a3f756501a8929c0642eb69734667f24289 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 2 May 2019 00:52:50 +0000 Subject: [PATCH 37/38] Fix an issue (discovered by OSSFuzz) in the enhanced OP_Concat operator from check-in [713caa382cf7dd] earlier today. FossilOrigin-Name: 3e897702f8f789fe5119b9042fb93eca3fbfcc44564fbfa66c65628725b1157d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 6979d5436c..6e3dac55c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\scase\sfor\sticket\s[ae0f637bddc5290b44669e066a]. -D 2019-05-01T19:01:27.714 +C Fix\san\sissue\s(discovered\sby\sOSSFuzz)\sin\sthe\senhanced\sOP_Concat\soperator\nfrom\scheck-in\s[713caa382cf7dd]\searlier\stoday. +D 2019-05-02T00:52:50.915 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -591,7 +591,7 @@ F src/upsert.c 0dd81b40206841814d46942a7337786932475f085716042d0cb2fc7791bf8ca4 F src/utf.c 2f0fac345c7660d5c5bd3df9e9d8d33d4c27f366bcfb09e07443064d751a0507 F src/util.c 5061987401c2e8003177fa30d73196aa036727c8f04bf36a2df0c82b1904a236 F src/vacuum.c 82dcec9e7b1afa980288718ad11bc499651c722d7b9f32933c4d694d91cb6ebf -F src/vdbe.c c15d6a105c41db6a166b0aa9650829bdc0d92918a8926a92332ea1feb27c33ba +F src/vdbe.c 36993059b87e7c2adf671aaa4ef5e0f826b6f4d95be15b019aee14308f0047b5 F src/vdbe.h 712bca562eaed1c25506b9faf9680bdc75fc42e2f4a1cd518d883fa79c7a4237 F src/vdbeInt.h 0e2c44958fb42d90a4eacb122d77e2d5b89b82f5e2b4b047b422962dc0346357 F src/vdbeapi.c 2ddd60f4a351f15ee98d841e346af16111ad59dfa4d25d2dd4012e9875bf7d92 @@ -1822,7 +1822,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 5997d075665faca6b70fa647e877ebc84c473b32887b96235865d59ce80247f8 -R c6735d94da423c75a5fbf44f945ad851 +P ece481695fc3c959c3eba0fb485cdda43a10b06d17259b0121e15bfc5e8e8b9f +R e11be1adfb0c3f0eaf3587a8d84226ea U drh -Z bc6fd770927f38308301380ce5d36731 +Z 373f392215a854f1484c07e1e5f7b6f0 diff --git a/manifest.uuid b/manifest.uuid index a535d6515f..27e888c142 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ece481695fc3c959c3eba0fb485cdda43a10b06d17259b0121e15bfc5e8e8b9f \ No newline at end of file +3e897702f8f789fe5119b9042fb93eca3fbfcc44564fbfa66c65628725b1157d \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 7fa7bc2a71..9bc5e4ee56 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1476,14 +1476,18 @@ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ } if( (flags1 & (MEM_Str|MEM_Blob))==0 ){ if( sqlite3VdbeMemStringify(pIn1,encoding,0) ) goto no_mem; + flags1 = pIn1->flags & ~MEM_Str; }else if( (flags1 & MEM_Zero)!=0 ){ if( sqlite3VdbeMemExpandBlob(pIn1) ) goto no_mem; + flags1 = pIn1->flags & ~MEM_Str; } flags2 = pIn2->flags; if( (flags2 & (MEM_Str|MEM_Blob))==0 ){ if( sqlite3VdbeMemStringify(pIn2,encoding,0) ) goto no_mem; + flags2 = pIn2->flags & ~MEM_Str; }else if( (flags2 & MEM_Zero)!=0 ){ if( sqlite3VdbeMemExpandBlob(pIn2) ) goto no_mem; + flags2 = pIn2->flags & ~MEM_Str; } nByte = pIn1->n + pIn2->n; if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ From cf1747b78259462caad907eecb10750993ee7e4e Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 2 May 2019 01:41:53 +0000 Subject: [PATCH 38/38] The collating sequence of the column must be TEXT if the LIKE or GLOB pattern starts with a "+" sign. This is another case of ticket [c94369cae9b561b1f996d005] that was discovered by Manuel Rigger. FossilOrigin-Name: b043a54c3de54b286c4eae564eab6b99118a410d99bdb63480faba3123d2ca11 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/whereexpr.c | 5 +++-- test/like3.test | 10 ++++++++++ 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 6e3dac55c2..c5d7ac5c43 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sissue\s(discovered\sby\sOSSFuzz)\sin\sthe\senhanced\sOP_Concat\soperator\nfrom\scheck-in\s[713caa382cf7dd]\searlier\stoday. -D 2019-05-02T00:52:50.915 +C The\scollating\ssequence\sof\sthe\scolumn\smust\sbe\sTEXT\sif\sthe\sLIKE\sor\sGLOB\spattern\nstarts\swith\sa\s"+"\ssign.\s\sThis\sis\sanother\scase\sof\sticket\n[c94369cae9b561b1f996d005]\sthat\swas\sdiscovered\sby\sManuel\sRigger. +D 2019-05-02T01:41:53.006 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -608,7 +608,7 @@ F src/walker.c 7607f1a68130c028255d8d56094ea602fc402c79e1e35a46e6282849d90d5fe4 F src/where.c 99c7b718ef846ac952016083aaf4e22ede2290beceaf4730a2df55c023251369 F src/whereInt.h 5f14db426ca46a83eabab1ae9aa6d4b8f27504ad35b64c290916289b1ddb2e88 F src/wherecode.c 0e76672930bea322eb3606d891a4744be55c09bcd3a995bfd501af62a46e0625 -F src/whereexpr.c 90859652920f153d2c03f075488744be2926625ebd36911bcbcb17d0d29c891c +F src/whereexpr.c 7fedf990999722dafda5ab8040feac93937a6f95f4671d8d629f2baf014b4b80 F src/window.c 038c248267e74ff70a2bb9b1884d40fd145c5183b017823ecb6cbb14bc781478 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd @@ -1088,7 +1088,7 @@ F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/lemon-test01.y 58b764610fd934e189ffbb0bbfa33d171b9cb06019b55bdc04d090d6767e11d7 F test/like.test 11cfd7d4ef8625389df9efc46735ff0b0b41d5e62047ef0f3bc24c380d28a7a6 F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da -F test/like3.test d3684b5c60dd2d1f9574f9f296b31c995187c8c10c6ca573f9918c96af9ba642 +F test/like3.test b065d1ca38c03dd76caae1d4cc84ed3d6eb3f64b3ff6b0dfad6413a7b406cca4 F test/limit.test 0c99a27a87b14c646a9d583c7c89fd06c352663e F test/limit2.test 9409b033284642a859fafc95f29a5a6a557bd57c1f0d7c3f554bd64ed69df77e F test/loadext.test faa4f6eed07a5aac35d57fdd7bc07f8fc82464cfd327567c10cf0ba3c86cde04 @@ -1822,7 +1822,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P ece481695fc3c959c3eba0fb485cdda43a10b06d17259b0121e15bfc5e8e8b9f -R e11be1adfb0c3f0eaf3587a8d84226ea +P 3e897702f8f789fe5119b9042fb93eca3fbfcc44564fbfa66c65628725b1157d +R 9786e5fb7c74febd927aa84f3b72acbe U drh -Z 373f392215a854f1484c07e1e5f7b6f0 +Z a4cc8a25e8e445f4572d088b41336647 diff --git a/manifest.uuid b/manifest.uuid index 27e888c142..bbae2efd09 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3e897702f8f789fe5119b9042fb93eca3fbfcc44564fbfa66c65628725b1157d \ No newline at end of file +b043a54c3de54b286c4eae564eab6b99118a410d99bdb63480faba3123d2ca11 \ No newline at end of file diff --git a/src/whereexpr.c b/src/whereexpr.c index 961495c584..90e1b5be65 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -263,11 +263,11 @@ static int isLikeOrGlob( } zNew[iTo] = 0; - /* If the RHS begins with a digit or a minus sign, then the LHS must be + /* If the RHS begins with a digit or a +/- sign, then the LHS must be ** an ordinary column (not a virtual table column) with TEXT affinity. ** Otherwise the LHS might be numeric and "lhs >= rhs" would be false ** even though "lhs LIKE rhs" is true. But if the RHS does not start - ** with a digit or '-', then "lhs LIKE rhs" will always be false if + ** with a digit or +/-, then "lhs LIKE rhs" will always be false if ** the LHS is numeric and so the optimization still works. ** ** 2018-09-10 ticket c94369cae9b561b1f996d0054bfab11389f9d033 @@ -277,6 +277,7 @@ static int isLikeOrGlob( */ if( sqlite3Isdigit(zNew[0]) || zNew[0]=='-' + || zNew[0]=='+' || (zNew[0]+1=='0' && iTo==1) ){ if( pLeft->op!=TK_COLUMN diff --git a/test/like3.test b/test/like3.test index 89c9efa792..f9120e7384 100644 --- a/test/like3.test +++ b/test/like3.test @@ -178,6 +178,16 @@ do_eqp_test like3-5.211 { `--SEARCH TABLE t5b USING COVERING INDEX sqlite_autoindex_t5b_1 (x>? AND x