diff --git a/ext/misc/dbdata.c b/ext/misc/dbdata.c new file mode 100644 index 0000000000..0e7f7d59cc --- /dev/null +++ b/ext/misc/dbdata.c @@ -0,0 +1,798 @@ +/* +** 2019-04-17 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains an implementation of two eponymous virtual tables, +** "sqlite_dbdata" and "sqlite_dbptr". Both modules require that the +** "sqlite_dbpage" eponymous virtual table be available. +** +** SQLITE_DBDATA: +** sqlite_dbdata is used to extract data directly from a database b-tree +** page and its associated overflow pages, bypassing the b-tree layer. +** The table schema is equivalent to: +** +** CREATE TABLE sqlite_dbdata( +** pgno INTEGER, +** cell INTEGER, +** field INTEGER, +** value ANY, +** schema TEXT HIDDEN +** ); +** +** IMPORTANT: THE VIRTUAL TABLE SCHEMA ABOVE IS SUBJECT TO CHANGE. IN THE +** FUTURE NEW NON-HIDDEN COLUMNS MAY BE ADDED BETWEEN "value" AND +** "schema". +** +** Each page of the database is inspected. If it cannot be interpreted as +** a b-tree page, or if it is a b-tree page containing 0 entries, the +** sqlite_dbdata table contains no rows for that page. Otherwise, the +** table contains one row for each field in the record associated with +** each cell on the page. For intkey b-trees, the key value is stored in +** field -1. +** +** For example, for the database: +** +** CREATE TABLE t1(a, b); -- root page is page 2 +** INSERT INTO t1(rowid, a, b) VALUES(5, 'v', 'five'); +** INSERT INTO t1(rowid, a, b) VALUES(10, 'x', 'ten'); +** +** the sqlite_dbdata table contains, as well as from entries related to +** page 1, content equivalent to: +** +** INSERT INTO sqlite_dbdata(pgno, cell, field, value) VALUES +** (2, 0, -1, 5 ), +** (2, 0, 0, 'v' ), +** (2, 0, 1, 'five'), +** (2, 1, -1, 10 ), +** (2, 1, 0, 'x' ), +** (2, 1, 1, 'ten' ); +** +** If database corruption is encountered, this module does not report an +** error. Instead, it attempts to extract as much data as possible and +** ignores the corruption. +** +** SQLITE_DBPTR: +** The sqlite_dbptr table has the following schema: +** +** CREATE TABLE sqlite_dbptr( +** pgno INTEGER, +** child INTEGER, +** schema TEXT HIDDEN +** ); +** +** It contains one entry for each b-tree pointer between a parent and +** child page in the database. +*/ +#if !defined(SQLITEINT_H) +#include "sqlite3ext.h" + +typedef unsigned char u8; + +#endif +SQLITE_EXTENSION_INIT1 +#include +#include + +typedef struct DbdataTable DbdataTable; +typedef struct DbdataCursor DbdataCursor; + +/* Cursor object */ +struct DbdataCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + sqlite3_stmt *pStmt; /* For fetching database pages */ + + int iPgno; /* Current page number */ + u8 *aPage; /* Buffer containing page */ + int nPage; /* Size of aPage[] in bytes */ + int nCell; /* Number of cells on aPage[] */ + int iCell; /* Current cell number */ + int bOnePage; /* True to stop after one page */ + int szDb; + sqlite3_int64 iRowid; + + /* Only for the sqlite_dbdata table */ + u8 *pRec; /* Buffer containing current record */ + int nRec; /* Size of pRec[] in bytes */ + int nHdr; /* Size of header in bytes */ + int iField; /* Current field number */ + u8 *pHdrPtr; + u8 *pPtr; + + sqlite3_int64 iIntkey; /* Integer key value */ +}; + +/* Table object */ +struct DbdataTable { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; /* The database connection */ + sqlite3_stmt *pStmt; /* For fetching database pages */ + int bPtr; /* True for sqlite3_dbptr table */ +}; + +/* Column and schema definitions for sqlite_dbdata */ +#define DBDATA_COLUMN_PGNO 0 +#define DBDATA_COLUMN_CELL 1 +#define DBDATA_COLUMN_FIELD 2 +#define DBDATA_COLUMN_VALUE 3 +#define DBDATA_COLUMN_SCHEMA 4 +#define DBDATA_SCHEMA \ + "CREATE TABLE x(" \ + " pgno INTEGER," \ + " cell INTEGER," \ + " field INTEGER," \ + " value ANY," \ + " schema TEXT HIDDEN" \ + ")" + +/* Column and schema definitions for sqlite_dbptr */ +#define DBPTR_COLUMN_PGNO 0 +#define DBPTR_COLUMN_CHILD 1 +#define DBPTR_COLUMN_SCHEMA 2 +#define DBPTR_SCHEMA \ + "CREATE TABLE x(" \ + " pgno INTEGER," \ + " child INTEGER," \ + " schema TEXT HIDDEN" \ + ")" + +/* +** Connect to an sqlite_dbdata (pAux==0) or sqlite_dbptr (pAux!=0) virtual +** table. +*/ +static int dbdataConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + DbdataTable *pTab = 0; + int rc = sqlite3_declare_vtab(db, pAux ? DBPTR_SCHEMA : DBDATA_SCHEMA); + + if( rc==SQLITE_OK ){ + pTab = (DbdataTable*)sqlite3_malloc64(sizeof(DbdataTable)); + if( pTab==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pTab, 0, sizeof(DbdataTable)); + pTab->db = db; + pTab->bPtr = (pAux!=0); + } + } + + *ppVtab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** Disconnect from or destroy a sqlite_dbdata or sqlite_dbptr virtual table. +*/ +static int dbdataDisconnect(sqlite3_vtab *pVtab){ + DbdataTable *pTab = (DbdataTable*)pVtab; + if( pTab ){ + sqlite3_finalize(pTab->pStmt); + sqlite3_free(pVtab); + } + return SQLITE_OK; +} + +/* +** This function interprets two types of constraints: +** +** schema=? +** pgno=? +** +** If neither are present, idxNum is set to 0. If schema=? is present, +** the 0x01 bit in idxNum is set. If pgno=? is present, the 0x02 bit +** in idxNum is set. +** +** If both parameters are present, schema is in position 0 and pgno in +** position 1. +*/ +static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdx){ + DbdataTable *pTab = (DbdataTable*)tab; + int i; + int iSchema = -1; + int iPgno = -1; + int colSchema = (pTab->bPtr ? DBPTR_COLUMN_SCHEMA : DBDATA_COLUMN_SCHEMA); + + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pIdx->aConstraint[i]; + if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + if( p->iColumn==colSchema ){ + if( p->usable==0 ) return SQLITE_CONSTRAINT; + iSchema = i; + } + if( p->iColumn==DBDATA_COLUMN_PGNO && p->usable ){ + iPgno = i; + } + } + } + + if( iSchema>=0 ){ + pIdx->aConstraintUsage[iSchema].argvIndex = 1; + pIdx->aConstraintUsage[iSchema].omit = 1; + } + if( iPgno>=0 ){ + pIdx->aConstraintUsage[iPgno].argvIndex = 1 + (iSchema>=0); + pIdx->aConstraintUsage[iPgno].omit = 1; + pIdx->estimatedCost = 100; + pIdx->estimatedRows = 50; + + if( pTab->bPtr==0 && pIdx->nOrderBy && pIdx->aOrderBy[0].desc==0 ){ + int iCol = pIdx->aOrderBy[0].iColumn; + if( pIdx->nOrderBy==1 ){ + pIdx->orderByConsumed = (iCol==0 || iCol==1); + }else if( pIdx->nOrderBy==2 && pIdx->aOrderBy[1].desc==0 && iCol==0 ){ + pIdx->orderByConsumed = (pIdx->aOrderBy[1].iColumn==1); + } + } + + }else{ + pIdx->estimatedCost = 100000000; + pIdx->estimatedRows = 1000000000; + } + pIdx->idxNum = (iSchema>=0 ? 0x01 : 0x00) | (iPgno>=0 ? 0x02 : 0x00); + return SQLITE_OK; +} + +/* +** Open a new sqlite_dbdata or sqlite_dbptr cursor. +*/ +static int dbdataOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + DbdataCursor *pCsr; + + pCsr = (DbdataCursor*)sqlite3_malloc64(sizeof(DbdataCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + }else{ + memset(pCsr, 0, sizeof(DbdataCursor)); + pCsr->base.pVtab = pVTab; + } + + *ppCursor = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Restore a cursor object to the state it was in when first allocated +** by dbdataOpen(). +*/ +static void dbdataResetCursor(DbdataCursor *pCsr){ + DbdataTable *pTab = (DbdataTable*)(pCsr->base.pVtab); + if( pTab->pStmt==0 ){ + pTab->pStmt = pCsr->pStmt; + }else{ + sqlite3_finalize(pCsr->pStmt); + } + pCsr->pStmt = 0; + pCsr->iPgno = 1; + pCsr->iCell = 0; + pCsr->iField = 0; + pCsr->bOnePage = 0; +} + +/* +** Close an sqlite_dbdata or sqlite_dbptr cursor. +*/ +static int dbdataClose(sqlite3_vtab_cursor *pCursor){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + dbdataResetCursor(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Utility methods to decode 16 and 32-bit big-endian unsigned integers. +*/ +static unsigned int get_uint16(unsigned char *a){ + return (a[0]<<8)|a[1]; +} +static unsigned int get_uint32(unsigned char *a){ + return (a[0]<<24)|(a[1]<<16)|(a[2]<<8)|a[3]; +} + +/* +** Load page pgno from the database via the sqlite_dbpage virtual table. +** If successful, set (*ppPage) to point to a buffer containing the page +** data, (*pnPage) to the size of that buffer in bytes and return +** SQLITE_OK. In this case it is the responsibility of the caller to +** eventually free the buffer using sqlite3_free(). +** +** Or, if an error occurs, set both (*ppPage) and (*pnPage) to 0 and +** return an SQLite error code. +*/ +static int dbdataLoadPage( + DbdataCursor *pCsr, /* Cursor object */ + unsigned int pgno, /* Page number of page to load */ + u8 **ppPage, /* OUT: pointer to page buffer */ + int *pnPage /* OUT: Size of (*ppPage) in bytes */ +){ + int rc2; + int rc = SQLITE_OK; + sqlite3_stmt *pStmt = pCsr->pStmt; + + *ppPage = 0; + *pnPage = 0; + sqlite3_bind_int64(pStmt, 2, pgno); + if( SQLITE_ROW==sqlite3_step(pStmt) ){ + int nCopy = sqlite3_column_bytes(pStmt, 0); + if( nCopy>0 ){ + u8 *pPage; + pPage = (u8*)sqlite3_malloc64(nCopy); + if( pPage==0 ){ + rc = SQLITE_NOMEM; + }else{ + const u8 *pCopy = sqlite3_column_blob(pStmt, 0); + memcpy(pPage, pCopy, nCopy); + } + *ppPage = pPage; + *pnPage = nCopy; + } + } + rc2 = sqlite3_reset(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + + return rc; +} + +/* +** Read a varint. Put the value in *pVal and return the number of bytes. +*/ +static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){ + sqlite3_int64 v = 0; + int i; + for(i=0; i<8; i++){ + v = (v<<7) + (z[i]&0x7f); + if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; } + } + v = (v<<8) + (z[i]&0xff); + *pVal = v; + return 9; +} + +/* +** Return the number of bytes of space used by an SQLite value of type +** eType. +*/ +static int dbdataValueBytes(int eType){ + switch( eType ){ + case 0: case 8: case 9: + case 10: case 11: + return 0; + case 1: + return 1; + case 2: + return 2; + case 3: + return 3; + case 4: + return 4; + case 5: + return 6; + case 6: + case 7: + return 8; + default: + return ((eType-12) / 2); + } +} + +/* +** Load a value of type eType from buffer pData and use it to set the +** result of context object pCtx. +*/ +static void dbdataValue(sqlite3_context *pCtx, int eType, u8 *pData){ + switch( eType ){ + case 0: + case 10: + case 11: + sqlite3_result_null(pCtx); + break; + + case 8: + sqlite3_result_int(pCtx, 0); + break; + case 9: + sqlite3_result_int(pCtx, 1); + break; + + case 1: case 2: case 3: case 4: case 5: case 6: case 7: { + sqlite3_uint64 v = (signed char)pData[0]; + pData++; + switch( eType ){ + case 7: + case 6: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 5: v = (v<<16) + (pData[0]<<8) + pData[1]; pData += 2; + case 4: v = (v<<8) + pData[0]; pData++; + case 3: v = (v<<8) + pData[0]; pData++; + case 2: v = (v<<8) + pData[0]; pData++; + } + + if( eType==7 ){ + double r; + memcpy(&r, &v, sizeof(r)); + sqlite3_result_double(pCtx, r); + }else{ + sqlite3_result_int64(pCtx, (sqlite3_int64)v); + } + break; + } + + default: { + int n = ((eType-12) / 2); + if( eType % 2 ){ + sqlite3_result_text(pCtx, (const char*)pData, n, SQLITE_TRANSIENT); + }else{ + sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT); + } + } + } +} + + +/* +** Move an sqlite_dbdata or sqlite_dbptr cursor to the next entry. +*/ +static int dbdataNext(sqlite3_vtab_cursor *pCursor){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; + + pCsr->iRowid++; + while( 1 ){ + int rc; + int iOff = (pCsr->iPgno==1 ? 100 : 0); + + if( pCsr->aPage==0 ){ + while( 1 ){ + if( pCsr->bOnePage==0 && pCsr->iPgno>pCsr->szDb ) return SQLITE_OK; + rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage); + if( rc!=SQLITE_OK ) return rc; + if( pCsr->aPage ) break; + pCsr->iPgno++; + } + pCsr->iCell = pTab->bPtr ? -2 : 0; + pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]); + } + + if( pTab->bPtr ){ + if( pCsr->aPage[iOff]!=0x02 && pCsr->aPage[iOff]!=0x05 ){ + pCsr->iCell = pCsr->nCell; + } + pCsr->iCell++; + if( pCsr->iCell>=pCsr->nCell ){ + sqlite3_free(pCsr->aPage); + pCsr->aPage = 0; + if( pCsr->bOnePage ) return SQLITE_OK; + pCsr->iPgno++; + }else{ + return SQLITE_OK; + } + }else{ + /* If there is no record loaded, load it now. */ + if( pCsr->pRec==0 ){ + int bHasRowid = 0; + int nPointer = 0; + sqlite3_int64 nPayload = 0; + sqlite3_int64 nHdr = 0; + int iHdr; + int U, X; + int nLocal; + + switch( pCsr->aPage[iOff] ){ + case 0x02: + nPointer = 4; + break; + case 0x0a: + break; + case 0x0d: + bHasRowid = 1; + break; + default: + /* This is not a b-tree page with records on it. Continue. */ + pCsr->iCell = pCsr->nCell; + break; + } + + if( pCsr->iCell>=pCsr->nCell ){ + sqlite3_free(pCsr->aPage); + pCsr->aPage = 0; + if( pCsr->bOnePage ) return SQLITE_OK; + pCsr->iPgno++; + continue; + } + + iOff += 8 + nPointer + pCsr->iCell*2; + iOff = get_uint16(&pCsr->aPage[iOff]); + + /* For an interior node cell, skip past the child-page number */ + iOff += nPointer; + + /* Load the "byte of payload including overflow" field */ + iOff += dbdataGetVarint(&pCsr->aPage[iOff], &nPayload); + + /* If this is a leaf intkey cell, load the rowid */ + if( bHasRowid ){ + iOff += dbdataGetVarint(&pCsr->aPage[iOff], &pCsr->iIntkey); + } + + /* Allocate space for payload */ + pCsr->pRec = (u8*)sqlite3_malloc64(nPayload); + if( pCsr->pRec==0 ) return SQLITE_NOMEM; + pCsr->nRec = nPayload; + + U = pCsr->nPage; + if( bHasRowid ){ + X = U-35; + }else{ + X = ((U-12)*64/255)-23; + } + if( nPayload<=X ){ + nLocal = nPayload; + }else{ + int M, K; + M = ((U-12)*32/255)-23; + K = M+((nPayload-M)%(U-4)); + if( K<=X ){ + nLocal = K; + }else{ + nLocal = M; + } + } + + /* Load the nLocal bytes of payload */ + memcpy(pCsr->pRec, &pCsr->aPage[iOff], nLocal); + iOff += nLocal; + + /* Load content from overflow pages */ + if( nPayload>nLocal ){ + sqlite3_int64 nRem = nPayload - nLocal; + unsigned int pgnoOvfl = get_uint32(&pCsr->aPage[iOff]); + while( nRem>0 ){ + u8 *aOvfl = 0; + int nOvfl = 0; + int nCopy; + rc = dbdataLoadPage(pCsr, pgnoOvfl, &aOvfl, &nOvfl); + assert( rc!=SQLITE_OK || nOvfl==pCsr->nPage ); + if( rc!=SQLITE_OK ) return rc; + + nCopy = U-4; + if( nCopy>nRem ) nCopy = nRem; + memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy); + nRem -= nCopy; + + pgnoOvfl = get_uint32(aOvfl); + sqlite3_free(aOvfl); + } + } + + iHdr = dbdataGetVarint(pCsr->pRec, &nHdr); + pCsr->nHdr = nHdr; + pCsr->pHdrPtr = &pCsr->pRec[iHdr]; + pCsr->pPtr = &pCsr->pRec[pCsr->nHdr]; + pCsr->iField = (bHasRowid ? -1 : 0); + }else{ + pCsr->iField++; + if( pCsr->iField>0 ){ + sqlite3_int64 iType; + pCsr->pHdrPtr += dbdataGetVarint(pCsr->pHdrPtr, &iType); + pCsr->pPtr += dbdataValueBytes(iType); + } + } + + if( pCsr->iField<0 || pCsr->pHdrPtr<&pCsr->pRec[pCsr->nHdr] ){ + return SQLITE_OK; + } + + /* Advance to the next cell. The next iteration of the loop will load + ** the record and so on. */ + sqlite3_free(pCsr->pRec); + pCsr->pRec = 0; + pCsr->iCell++; + } + } + + assert( !"can't get here" ); + return SQLITE_OK; +} + +/* +** Return true if the cursor is at EOF. +*/ +static int dbdataEof(sqlite3_vtab_cursor *pCursor){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + return pCsr->aPage==0; +} + +/* +** Determine the size in pages of database zSchema (where zSchema is +** "main", "temp" or the name of an attached database) and set +** pCsr->szDb accordingly. If successful, return SQLITE_OK. Otherwise, +** an SQLite error code. +*/ +static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){ + DbdataTable *pTab = (DbdataTable*)pCsr->base.pVtab; + char *zSql = 0; + int rc, rc2; + sqlite3_stmt *pStmt = 0; + + zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema); + if( zSql==0 ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + pCsr->szDb = sqlite3_column_int(pStmt, 0); + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + return rc; +} + +/* +** xFilter method for sqlite_dbdata and sqlite_dbptr. +*/ +static int dbdataFilter( + sqlite3_vtab_cursor *pCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; + int rc = SQLITE_OK; + const char *zSchema = "main"; + + dbdataResetCursor(pCsr); + assert( pCsr->iPgno==1 ); + if( idxNum & 0x01 ){ + zSchema = (const char*)sqlite3_value_text(argv[0]); + } + if( idxNum & 0x02 ){ + pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]); + pCsr->bOnePage = 1; + }else{ + pCsr->nPage = dbdataDbsize(pCsr, zSchema); + rc = dbdataDbsize(pCsr, zSchema); + } + + if( rc==SQLITE_OK ){ + if( pTab->pStmt ){ + pCsr->pStmt = pTab->pStmt; + pTab->pStmt = 0; + }else{ + rc = sqlite3_prepare_v2(pTab->db, + "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1, + &pCsr->pStmt, 0 + ); + } + } + if( rc==SQLITE_OK ){ + rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT); + }else{ + pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); + } + if( rc==SQLITE_OK ){ + rc = dbdataNext(pCursor); + } + return rc; +} + +/* +** Return a column for the sqlite_dbdata or sqlite_dbptr table. +*/ +static int dbdataColumn( + sqlite3_vtab_cursor *pCursor, + sqlite3_context *ctx, + int i +){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + DbdataTable *pTab = (DbdataTable*)pCursor->pVtab; + if( pTab->bPtr ){ + switch( i ){ + case DBPTR_COLUMN_PGNO: + sqlite3_result_int64(ctx, pCsr->iPgno); + break; + case DBPTR_COLUMN_CHILD: { + int iOff = pCsr->iPgno==1 ? 100 : 0; + if( pCsr->iCell<0 ){ + iOff += 8; + }else{ + iOff += 12 + pCsr->iCell*2; + iOff = get_uint16(&pCsr->aPage[iOff]); + } + sqlite3_result_int64(ctx, get_uint32(&pCsr->aPage[iOff])); + break; + } + } + }else{ + switch( i ){ + case DBDATA_COLUMN_PGNO: + sqlite3_result_int64(ctx, pCsr->iPgno); + break; + case DBDATA_COLUMN_CELL: + sqlite3_result_int(ctx, pCsr->iCell); + break; + case DBDATA_COLUMN_FIELD: + sqlite3_result_int(ctx, pCsr->iField); + break; + case DBDATA_COLUMN_VALUE: { + if( pCsr->iField<0 ){ + sqlite3_result_int64(ctx, pCsr->iIntkey); + }else{ + sqlite3_int64 iType; + dbdataGetVarint(pCsr->pHdrPtr, &iType); + dbdataValue(ctx, iType, pCsr->pPtr); + } + break; + } + } + } + return SQLITE_OK; +} + +/* +** Return the rowid for an sqlite_dbdata or sqlite_dptr table. +*/ +static int dbdataRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + DbdataCursor *pCsr = (DbdataCursor*)pCursor; + *pRowid = pCsr->iRowid; + return SQLITE_OK; +} + + +/* +** Invoke this routine to register the "sqlite_dbdata" virtual table module +*/ +static int sqlite3DbdataRegister(sqlite3 *db){ + static sqlite3_module dbdata_module = { + 0, /* iVersion */ + 0, /* xCreate */ + dbdataConnect, /* xConnect */ + dbdataBestIndex, /* xBestIndex */ + dbdataDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + dbdataOpen, /* xOpen - open a cursor */ + dbdataClose, /* xClose - close a cursor */ + dbdataFilter, /* xFilter - configure scan constraints */ + dbdataNext, /* xNext - advance a cursor */ + dbdataEof, /* xEof - check for end of scan */ + dbdataColumn, /* xColumn - read data */ + dbdataRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0 /* xShadowName */ + }; + + int rc = sqlite3_create_module(db, "sqlite_dbdata", &dbdata_module, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "sqlite_dbptr", &dbdata_module, (void*)1); + } + return rc; +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_dbdata_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3DbdataRegister(db); +} diff --git a/main.mk b/main.mk index f418eec68a..508554d733 100644 --- a/main.mk +++ b/main.mk @@ -738,6 +738,7 @@ SHELL_SRC = \ $(TOP)/ext/expert/sqlite3expert.h \ $(TOP)/ext/misc/zipfile.c \ $(TOP)/ext/misc/memtrace.c \ + $(TOP)/ext/misc/dbdata.c \ $(TOP)/src/test_windirent.c shell.c: $(SHELL_SRC) $(TOP)/tool/mkshellc.tcl diff --git a/manifest b/manifest index db88510e70..04c0aa028c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssqlite3_hard_heap_limit64()\sinterface\sand\sthe\scorresponding\n"PRAGMA\shard_heap_limit=N"\scommand. -D 2019-04-25T18:15:38.825 +C Merge\senhancements\sfrom\strunk +D 2019-05-02T14:15:12.561 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -284,6 +284,7 @@ F ext/misc/closure.c dbfd8543b2a017ae6b1a5843986b22ddf99ff126ec9634a2f4047cd14c8 F ext/misc/completion.c cec672d40604075bb341a7f11ac48393efdcd90a979269b8fe7977ea62d0547f F ext/misc/compress.c dd4f8a6d0baccff3c694757db5b430f3bbd821d8686d1fc24df55cf9f035b189 F ext/misc/csv.c 7f047aeb68f5802e7ce6639292095d622a488bb43526ed04810e0649faa71ceb +F ext/misc/dbdata.c 1b3751b02d8f575d25c6bda358670d2e39ace368a0d05595989c308a10c615f6 F ext/misc/dbdump.c baf6e37447c9d6968417b1cd34cbedb0b0ab3f91b5329501d8a8d5be3287c336 F ext/misc/eval.c 4b4757592d00fd32e44c7a067e6a0e4839c81a4d57abc4131ee7806d1be3104e F ext/misc/explain.c d5c12962d79913ef774b297006872af1fccda388f61a11d37758f9179a09551f @@ -440,7 +441,7 @@ F ext/userauth/userauth.c f81aa5a3ecacf406f170c62a144405858f6f6de51dbdc0920134e6 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 23d3660f7053d196aef76938bf78b10fc3ce1831a85d96bd71565758788f34d4 +F main.mk 125adda36bb32c99dc3a11340bd029ef373b9523eac2b2af76087bfe82d4fdf8 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 @@ -462,7 +463,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6 F src/btree.c ffe7101006aee2ab9e9dec2fc001998e57a8e59419c6ea4072d6c3935d3d50fb F src/btree.h c11446f07ec0e9dc85af8041cb0855c52f5359c8b2a43e47e02a685282504d89 F src/btreeInt.h 6111c15868b90669f79081039d19e7ea8674013f907710baa3c814dc3f8bfd3f -F src/build.c 61655dad911a967a69fb49df57268fd15ce8f1af3fe0a1bd90c128ef2cacfb7a +F src/build.c 2d9ddfeaf8e1dafc7e1fcc8a84e7a8b455199dac3b69037fc73af6279aa8447b F src/callback.c 25dda5e1c2334a367b94a64077b1d06b2553369f616261ca6783c48bcb6bda73 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e F src/ctime.c 109e58d00f62e8e71ee1eb5944ac18b90171c928ab2e082e058056e1137cc20b @@ -473,7 +474,7 @@ F src/delete.c d08c9e01a2664afd12edcfa3a9c6578517e8ff8735f35509582693adbe0edeaf F src/expr.c f65db06a0fcff760cadfb79d579a41e3eb7eff38848d5d6359137822f4fa2ec9 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007 F src/fkey.c 0e14d4bef8eac2d87bbd517e492d9084c65008d117823f8922c5e7b2b599bd33 -F src/func.c 2ccf4ae12430b1ae7096be5f0675887e1bd0732828af0ac0f7496339b7c6edee +F src/func.c ac05ea6b47b407586ad2c0878c4c81c3acb08b67ecf86648830f91f40325ae37 F src/global.c 0dea3065ea72a65ae941559b6686aad6516d4913e76fa4f79a95ff7787f624ec F src/hash.c 8d7dda241d0ebdafb6ffdeda3149a412d7df75102cecfc1021c98d6219823b19 F src/hash.h 9d56a9079d523b648774c1784b74b89bd93fac7b365210157482e4319a468f38 @@ -516,10 +517,10 @@ F src/pragma.h 9af5ddde96902a3f318e0100feea3a455a6f87cd9930c9183773f1e362055070 F src/prepare.c 78027c6231fbb19ca186a5f5f0c0a1375d9c2cec0655273f9bd90d9ff74a34b3 F src/printf.c 67f79227273a9009d86a017619717c3f554f50b371294526da59faa6014ed2cd F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384 -F src/resolve.c 567888ee3faec14dae06519b4306201771058364a37560186a3e0e755ebc4cb8 +F src/resolve.c 408632d9531ca8f1df8591f00530797daaa7bde3fe0d3211de4d431cbb99347e F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93 F src/select.c b7304d2f491c11a03a7fbdf34bc218282ac54052377809d4dc3b4b1e7f4bfc93 -F src/shell.c.in bcfa17eb257bf8dc2359e99ba7e6bdfab7901705db013bc47a5be6d7fa7a037e +F src/shell.c.in 567236da9ee68b1dfa363426858ee5e310976ffe422a7b7ae220c0315d7e8c53 F src/sqlite.h.in 7593b6df09ca8f4b9f22005a2164413704e5a4e7cb82697efae4315c8e12e0a3 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8 F src/sqlite3ext.h aa8c3f601d8a6e8efdc485e4bbfd6cdbb18ac4e53b1a71329a07f2d6ded6b1c5 @@ -590,16 +591,16 @@ F src/upsert.c 0dd81b40206841814d46942a7337786932475f085716042d0cb2fc7791bf8ca4 F src/utf.c 2f0fac345c7660d5c5bd3df9e9d8d33d4c27f366bcfb09e07443064d751a0507 F src/util.c 5061987401c2e8003177fa30d73196aa036727c8f04bf36a2df0c82b1904a236 F src/vacuum.c 82dcec9e7b1afa980288718ad11bc499651c722d7b9f32933c4d694d91cb6ebf -F src/vdbe.c 711ef421b3bb3db3b2476067b2dc3c71ef5844d9b1a723026578f89f6da621e8 +F src/vdbe.c 36993059b87e7c2adf671aaa4ef5e0f826b6f4d95be15b019aee14308f0047b5 F src/vdbe.h 712bca562eaed1c25506b9faf9680bdc75fc42e2f4a1cd518d883fa79c7a4237 -F src/vdbeInt.h 2c12704db9740c8e899786ecfc7a5797a9d067563496eb1b6ed03c592d7b8d90 +F src/vdbeInt.h 0e2c44958fb42d90a4eacb122d77e2d5b89b82f5e2b4b047b422962dc0346357 F src/vdbeapi.c 2ddd60f4a351f15ee98d841e346af16111ad59dfa4d25d2dd4012e9875bf7d92 F src/vdbeaux.c f873b5c2efcf8a4d6ecfc5b1a5b06fd810419198f3bd882175d371cc03801873 F src/vdbeblob.c f5c70f973ea3a9e915d1693278a5f890dc78594300cf4d54e64f2b0917c94191 -F src/vdbemem.c dd2ee49255c4c5450f2b0887ef44cea8faa1cd7a46501b39a1a82b113ae418e3 +F src/vdbemem.c df36fd36c7585e42869f3a44f5da5dc70e13306bc97ba52eebe069e174ba55db F src/vdbesort.c 66592d478dbb46f19aed0b42222325eadb84deb40a90eebe25c6e7c1d8468f47 F src/vdbetrace.c 79d6dbbc479267b255a7de8080eee6e729928a0ef93ed9b0bfa5618875b48392 -F src/vtab.c 4c5959e00b7a142198d178e3a822f4e05f36f2d1a3c57657373f9487154fc06b +F src/vtab.c 1fa256c6ddad7a81e2a4dc080d015d4b0a7135767717d311298e47f6fca64bb3 F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9 F src/wal.c b09a2a9cab50efa08451a8c81d47052120ad5da174048c6d0b08d405384abdf2 F src/wal.h 606292549f5a7be50b6227bd685fa76e3a4affad71bb8ac5ce4cb5c79f6a176a @@ -607,7 +608,7 @@ F src/walker.c 7607f1a68130c028255d8d56094ea602fc402c79e1e35a46e6282849d90d5fe4 F src/where.c 99c7b718ef846ac952016083aaf4e22ede2290beceaf4730a2df55c023251369 F src/whereInt.h 5f14db426ca46a83eabab1ae9aa6d4b8f27504ad35b64c290916289b1ddb2e88 F src/wherecode.c 0e76672930bea322eb3606d891a4744be55c09bcd3a995bfd501af62a46e0625 -F src/whereexpr.c 90859652920f153d2c03f075488744be2926625ebd36911bcbcb17d0d29c891c +F src/whereexpr.c 7fedf990999722dafda5ab8040feac93937a6f95f4671d8d629f2baf014b4b80 F src/window.c 038c248267e74ff70a2bb9b1884d40fd145c5183b017823ecb6cbb14bc781478 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd @@ -628,7 +629,7 @@ F test/altermalloc.test 167a47de41b5c638f5f5c6efb59784002b196fff70f98d9b4ed3cd74 F test/altermalloc2.test fa7b1c1139ea39b8dec407cf1feb032ca8e0076bd429574969b619175ad0174b F test/altertab.test 372df7d8f09e1ee22d23551677cedff3b048b0059c1f1b9a01a6401b94a2367c F test/altertab2.test 5d423a2d1006085b05cc1b788863d5a860ea2da21c4f892d15e2f2a34c78348a -F test/altertab3.test 40f2ce9be675e354d3e55c72f8baf38813be975ff4dd9e6b3144493c3c5bc033 +F test/altertab3.test 2433d0cc6cb9cffe087f9138cd36818c7abd5c396804aa6e6dc8c2b80e2cd406 F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 7168c8bffa5d5cbc53c05b7e9c7fcdd24b365a1bc5046ce80c45efa3c02e6b7c F test/analyze3.test ff62d9029e6deb2c914490c6b00caf7fae47cc85cdc046e4a0d0a4d4b87c71d8 @@ -786,6 +787,7 @@ F test/cursorhint2.test 6f3aa9cb19e7418967a10ec6905209bcbb5968054da855fc36c8beee F test/dataversion1.test 6e5e86ac681f0782e766ebcb56c019ae001522d114e0e111e5ebf68ccf2a7bb8 F test/date.test 9b73bbeb1b82d9c1f44dec5cf563bf7da58d2373 F test/date2.test 74c234bece1b016e94dd4ef9c8cc7a199a8806c0e2291cab7ba64bace6350b10 +F test/dbdata.test 042f49acff3438f940eeba5868d3af080ae64ddf26ae78f80c92bec3ca7d8603 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e F test/dbfuzz001.test e32d14465f1c77712896fda6a1ccc0f037b481c191c1696a9c44f6c9e4964faf F test/dbfuzz2-seed1.db e6225c6f3d7b63f9c5b6867146a5f329d997ab105bee64644dc2b3a2f2aebaee @@ -973,6 +975,7 @@ F test/fts4merge4.test d895b1057a7798b67e03455d0fa50e9ea836c47b F test/fts4noti.test 5553d7bb2e20bf4a06b23e849352efc022ce6309 F test/fts4onepass.test d69ddc4ee3415e40b0c5d1d0408488a87614d4f63ba9c44f3e52db541d6b7cc7 F test/fts4opt.test 0fd0cc84000743ff2a883b9b84b4a5be07249f0ba790c8848a757164cdd46b2a +F test/fts4rename.test 6015a355ec3a11a51eb5b88802b3b2c1788786c54b77b17f3e077b7c93ff8611 F test/fts4umlaut.test fcaca4471de7e78c9d1f7e8976e3e8704d7d8ad979d57a739d00f3f757380429 F test/fts4unicode.test ceca76422abc251818cb25dabe33d3c3970da5f7c90e1540f190824e6b3a7c95 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d @@ -1028,7 +1031,7 @@ F test/incrvacuum.test 2aaee202b1f230e55779f70d155f6ba67bbdff8481d650214d256ab0f F test/incrvacuum2.test 7d26cfda66c7e55898d196de54ac4ec7d86a4e3d F test/incrvacuum3.test 75256fb1377e7c39ef2de62bfc42bbff67be295a F test/incrvacuum_ioerr.test 6ae2f783424e47a0033304808fe27789cf93e635 -F test/index.test df4cddf4435314a948237fdfa9acee67de21f7bebc789beab4b89b575b4f6a70 +F test/index.test 05414fc7e1e128c327e089c2216d041ae7fb02232571f708f009a79a482cf1a3 F test/index2.test f835d5e13ca163bd78c4459ca15fd2e4ed487407 F test/index3.test 51685f39345462b84fcf77eb8537af847fdf438cc96b05c45d6aaca4e473ade0 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 @@ -1085,7 +1088,7 @@ F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/lemon-test01.y 58b764610fd934e189ffbb0bbfa33d171b9cb06019b55bdc04d090d6767e11d7 F test/like.test 11cfd7d4ef8625389df9efc46735ff0b0b41d5e62047ef0f3bc24c380d28a7a6 F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da -F test/like3.test 0ce2630e39e32e42ce02d171f0a315189ca71fec37c5ddfb0191eecc3fe9d4da +F test/like3.test b065d1ca38c03dd76caae1d4cc84ed3d6eb3f64b3ff6b0dfad6413a7b406cca4 F test/limit.test 0c99a27a87b14c646a9d583c7c89fd06c352663e F test/limit2.test 9409b033284642a859fafc95f29a5a6a557bd57c1f0d7c3f554bd64ed69df77e F test/loadext.test faa4f6eed07a5aac35d57fdd7bc07f8fc82464cfd327567c10cf0ba3c86cde04 @@ -1183,7 +1186,7 @@ F test/orderby6.test 8b38138ab0972588240b3fca0985d2e400432859 F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da F test/orderby8.test 23ef1a5d72bd3adcc2f65561c654295d1b8047bd F test/orderby9.test 87fb9548debcc2cd141c5299002dd94672fa76a3 -F test/oserror.test e7b3416be4b9d5dd2fe0b42dd394daaddbb6c83eeec1f0e47b120b53e0ad3ace +F test/oserror.test 1fc9746b83d778e70d115049747ba19c7fba154afce7cc165b09feb6ca6abbc5 F test/ossfuzz.c 18af635fa73d12a109b305faca727a734c1fa28a421b161d9d15c5a84a4998a2 F test/ossshell.c f125c5bd16e537a2549aa579b328dd1c59905e7ab1338dfc210e755bb7b69f17 F test/ovfl.test 199c482696defceacee8c8e0e0ef36da62726b2f @@ -1224,6 +1227,7 @@ F test/randexpr1.tcl 40dec52119ed3a2b8b2a773bce24b63a3a746459 F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rbu.test 168573d353cd0fd10196b87b0caa322c144ef736 F test/rdonly.test 64e2696c322e3538df0b1ed624e21f9a23ed9ff8 +F test/recover.test 4c45b1519c11c6d10cc28bced260e2a18f3967fc6abccaace44af6c22b30280c F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/regexp2.test 40e894223b3d6672655481493f1be12012f2b33c F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 @@ -1652,7 +1656,7 @@ F test/walslow.test c05c68d4dc2700a982f89133ce103a1a84cc285f F test/walthread.test 14b20fcfa6ae152f5d8e12f5dc8a8a724b7ef189f5d8ef1e2ceab79f2af51747 F test/walvfs.test c0faffda13d045a96dfc541347886bb1a3d6f3205857fc98e683edfab766ea88 F test/wapp.tcl b440cd8cf57953d3a49e7ee81e6a18f18efdaf113b69f7d8482b0710a64566ec -F test/wapptest.tcl 78aff97afe76fd9728cf5f84710a772412735bc68a612b4789279072177a424e x +F test/wapptest.tcl 32a23f9b4c9fa1126d29250368ba6d5689b7503aa0694df7edf9253f1d56f1d7 x F test/where.test 0607caa5a1fbfe7b93b95705981b463a3a0408038f22ae6e9dc11b36902b0e95 F test/where2.test 478d2170637b9211f593120648858593bf2445a1 F test/where3.test 2341a294e17193a6b1699ea7f192124a5286ca6acfcc3f4b06d16c931fbcda2c @@ -1704,7 +1708,7 @@ F test/with2.test e0030e2f0267a910d6c0e4f46f2dfe941c1cc0d4f659ba69b3597728e7e8f1 F test/with3.test 8d26920c88283e0a473ceebd3451554922108ce7b2a6a1157c47eb0a7011212c F test/with4.test 257be66c0c67fee1defbbac0f685c3465e2cad037f21ce65f23f86084f198205 F test/withM.test 693b61765f2b387b5e3e24a4536e2e82de15ff64 -F test/without_rowid1.test b5ec93f7df2c1d684e0923247dac6aca8888e088bf50a9f244c3933e0e813a72 +F test/without_rowid1.test 7ac016d20317e36a2f142e960679e558e74f6809ce5f27bde668af01782500df F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 F test/without_rowid3.test ea4b59dd1b0d7f5f5e4b7cca978cdb905752a9d7c57dc4344a591dba765a3691 F test/without_rowid4.test 4e08bcbaee0399f35d58b5581881e7a6243d458a @@ -1737,7 +1741,7 @@ F tool/genfkey.test b6afd7b825d797a1e1274f519ab5695373552ecad5cd373530c63533638a F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/index_usage.c 9ec344d29cbeb03fdc0fce668eedfb7495792170de933adf95cf8d6904a166ad F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f -F tool/lemon.c 900a15b9efba9890d10e7959914db94c4ad5162912127f061c4328add122d6fb +F tool/lemon.c d02a276728c507a7007333944eeabafab1668033794af348389b1166075869ee F tool/lempar.c 61af95b8fac2bfd59c09d55330e78f3f5e352d7aa80bf37404b96ef795be3fdc F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862 @@ -1752,7 +1756,7 @@ F tool/mkopcodec.tcl d1b6362bd3aa80d5520d4d6f3765badf01f6c43c F tool/mkopcodeh.tcl 352a4319c0ad869eb26442bf7c3b015aa15594c21f1cce5a6420dbe999367c21 F tool/mkopts.tcl 680f785fdb09729fd9ac50632413da4eadbdf9071535e3f26d03795828ab07fa F tool/mkpragmatab.tcl 0b0d2500ca37ae0f21abe19440ecc1abcde64e8ccc955f670ac69098beaf0b0d -F tool/mkshellc.tcl 1f45770aea226ac093a9c72f718efbb88a2a2833409ec2e1c4cecae4202626f5 +F tool/mkshellc.tcl 70a9978e363b0f3280ca9ce1c46d72563ff479c1930a12a7375e3881b7325712 F tool/mksourceid.c d458f9004c837bee87a6382228ac20d3eae3c49ea3b0a5aace936f8b60748d3b F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 4f7cfef5152b0c91920355cbfc1d608a4ad242cb819f1aea07f6d0274f584a7f @@ -1818,10 +1822,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 7be6222c9ec44596e4eddd906c831eb1272b90fbdf68641d791f216264feb7cf -R 5060ced5c6b6fe418072d1c42b893584 -T *branch * hard-heap-limit -T *sym-hard-heap-limit * -T -sym-trunk * +P b0ccef61a7f92d20228becbf4f997bf0f4e46dad2deaf0896dc63b976ad1dd11 b043a54c3de54b286c4eae564eab6b99118a410d99bdb63480faba3123d2ca11 +R a89d7e67760240a4adc746cc054ccd07 U drh -Z b301fb8fbe07a4e2955f0dbf52997652 +Z 04e5ccc16cb84aeda9f5596141eed90d diff --git a/manifest.uuid b/manifest.uuid index 130e7c1c67..3261ee26a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b0ccef61a7f92d20228becbf4f997bf0f4e46dad2deaf0896dc63b976ad1dd11 \ No newline at end of file +3a4751a9f2784131f81071305b838caa63410a76533fb879627e1849d626f893 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 9ecd31b8e8..8684e1ad69 100644 --- a/src/build.c +++ b/src/build.c @@ -1329,7 +1329,7 @@ void sqlite3AddDefaultValue( ** accept it. This routine does the necessary conversion. It converts ** the expression given in its argument from a TK_STRING into a TK_ID ** if the expression is just a TK_STRING with an optional COLLATE clause. -** If the epxression is anything other than TK_STRING, the expression is +** If the expression is anything other than TK_STRING, the expression is ** unchanged. */ static void sqlite3StringToId(Expr *p){ @@ -1726,10 +1726,51 @@ static void estimateIndexWidth(Index *pIdx){ pIdx->szIdxRow = sqlite3LogEst(wIndex*4); } -/* Return true if value x is found any of the first nCol entries of aiCol[] +/* Return true if column number x is any of the first nCol entries of aiCol[]. +** This is used to determine if the column number x appears in any of the +** first nCol entries of an index. */ static int hasColumn(const i16 *aiCol, int nCol, int x){ - while( nCol-- > 0 ) if( x==*(aiCol++) ) return 1; + while( nCol-- > 0 ){ + assert( aiCol[0]>=0 ); + if( x==*(aiCol++) ){ + return 1; + } + } + return 0; +} + +/* +** Return true if any of the first nKey entries of index pIdx exactly +** match the iCol-th entry of pPk. pPk is always a WITHOUT ROWID +** PRIMARY KEY index. pIdx is an index on the same table. pIdx may +** or may not be the same index as pPk. +** +** The first nKey entries of pIdx are guaranteed to be ordinary columns, +** not a rowid or expression. +** +** This routine differs from hasColumn() in that both the column and the +** collating sequence must match for this routine, but for hasColumn() only +** the column name must match. +*/ +static int isDupColumn(Index *pIdx, int nKey, Index *pPk, int iCol){ + int i, j; + assert( nKey<=pIdx->nColumn ); + assert( iColnColumn,pPk->nKeyCol) ); + assert( pPk->idxType==SQLITE_IDXTYPE_PRIMARYKEY ); + assert( pPk->pTable->tabFlags & TF_WithoutRowid ); + assert( pPk->pTable==pIdx->pTable ); + testcase( pPk==pIdx ); + j = pPk->aiColumn[iCol]; + assert( j!=XN_ROWID && j!=XN_EXPR ); + for(i=0; iaiColumn[i]>=0 || j>=0 ); + if( pIdx->aiColumn[i]==j + && sqlite3StrICmp(pIdx->azColl[i], pPk->azColl[iCol])==0 + ){ + return 1; + } + } return 0; } @@ -1835,9 +1876,10 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ ** code assumes the PRIMARY KEY contains no repeated columns. */ for(i=j=1; inKeyCol; i++){ - if( hasColumn(pPk->aiColumn, j, pPk->aiColumn[i]) ){ + if( isDupColumn(pPk, j, pPk, i) ){ pPk->nColumn--; }else{ + testcase( hasColumn(pPk->aiColumn, j, pPk->aiColumn[i]) ); pPk->aiColumn[j++] = pPk->aiColumn[i]; } } @@ -1867,7 +1909,10 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ int n; if( IsPrimaryKeyIndex(pIdx) ) continue; for(i=n=0; iaiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ) n++; + if( !isDupColumn(pIdx, pIdx->nKeyCol, pPk, i) ){ + testcase( hasColumn(pIdx->aiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ); + n++; + } } if( n==0 ){ /* This index is a superset of the primary key */ @@ -1876,7 +1921,8 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ } if( resizeIndexObject(db, pIdx, pIdx->nKeyCol+n) ) return; for(i=0, j=pIdx->nKeyCol; iaiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ){ + if( !isDupColumn(pIdx, pIdx->nKeyCol, pPk, i) ){ + testcase( hasColumn(pIdx->aiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ); pIdx->aiColumn[j] = pPk->aiColumn[i]; pIdx->azColl[j] = pPk->azColl[i]; j++; @@ -3392,9 +3438,10 @@ void sqlite3CreateIndex( for(j=0; jnKeyCol; j++){ int x = pPk->aiColumn[j]; assert( x>=0 ); - if( hasColumn(pIndex->aiColumn, pIndex->nKeyCol, x) ){ + if( isDupColumn(pIndex, pIndex->nKeyCol, pPk, j) ){ pIndex->nColumn--; }else{ + testcase( hasColumn(pIndex->aiColumn,pIndex->nKeyCol,x) ); pIndex->aiColumn[i] = x; pIndex->azColl[i] = pPk->azColl[j]; pIndex->aSortOrder[i] = pPk->aSortOrder[j]; diff --git a/src/func.c b/src/func.c index 03595b70d2..dae85d60a8 100644 --- a/src/func.c +++ b/src/func.c @@ -843,8 +843,6 @@ static void likeFunc( return; } #endif - zB = sqlite3_value_text(argv[0]); - zA = sqlite3_value_text(argv[1]); /* Limit the length of the LIKE or GLOB pattern to avoid problems ** of deep recursion and N*N behavior in patternCompare(). @@ -856,8 +854,6 @@ static void likeFunc( sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); return; } - assert( zB==sqlite3_value_text(argv[0]) ); /* Encoding did not change */ - if( argc==3 ){ /* The escape character string must consist of a single UTF-8 character. ** Otherwise, return an error. @@ -873,6 +869,8 @@ static void likeFunc( }else{ escape = pInfo->matchSet; } + zB = sqlite3_value_text(argv[0]); + zA = sqlite3_value_text(argv[1]); if( zA && zB ){ #ifdef SQLITE_TEST sqlite3_like_count++; diff --git a/src/resolve.c b/src/resolve.c index 50755e59f3..f6b6af1dfa 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -866,7 +866,9 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ #ifndef SQLITE_OMIT_WINDOWFUNC if( pExpr->y.pWin ){ Select *pSel = pNC->pWinSelect; - sqlite3WindowUpdate(pParse, pSel->pWinDefn, pExpr->y.pWin, pDef); + if( IN_RENAME_OBJECT==0 ){ + sqlite3WindowUpdate(pParse, pSel->pWinDefn, pExpr->y.pWin, pDef); + } sqlite3WalkExprList(pWalker, pExpr->y.pWin->pPartition); sqlite3WalkExprList(pWalker, pExpr->y.pWin->pOrderBy); sqlite3WalkExpr(pWalker, pExpr->y.pWin->pFilter); diff --git a/src/shell.c.in b/src/shell.c.in index 1f57d1ec3d..e145a1547c 100644 --- a/src/shell.c.in +++ b/src/shell.c.in @@ -948,6 +948,10 @@ INCLUDE ../ext/misc/sqlar.c INCLUDE ../ext/expert/sqlite3expert.h INCLUDE ../ext/expert/sqlite3expert.c +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) +INCLUDE ../ext/misc/dbdata.c +#endif + #if defined(SQLITE_ENABLE_SESSION) /* ** State information for a single open session @@ -3574,6 +3578,9 @@ static const char *(azHelp[]) = { ".prompt MAIN CONTINUE Replace the standard prompts", ".quit Exit this program", ".read FILE Read input from FILE", +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) + ".recover Recover as much data as possible from corrupt db.", +#endif ".restore ?DB? FILE Restore content of DB (default \"main\") from FILE", ".save FILE Write in-memory database into FILE", ".scanstats on|off Turn sqlite3_stmt_scanstatus() metrics on or off", @@ -3931,6 +3938,125 @@ readHexDb_error: } #endif /* SQLITE_ENABLE_DESERIALIZE */ +/* +** Scalar function "shell_int32". The first argument to this function +** must be a blob. The second a non-negative integer. This function +** reads and returns a 32-bit big-endian integer from byte +** offset (4*) of the blob. +*/ +static void shellInt32( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *pBlob; + int nBlob; + int iInt; + + nBlob = sqlite3_value_bytes(argv[0]); + pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]); + iInt = sqlite3_value_int(argv[1]); + + if( iInt>=0 && (iInt+1)*4<=nBlob ){ + const unsigned char *a = &pBlob[iInt*4]; + sqlite3_int64 iVal = ((sqlite3_int64)a[0]<<24) + + ((sqlite3_int64)a[1]<<16) + + ((sqlite3_int64)a[2]<< 8) + + ((sqlite3_int64)a[3]<< 0); + sqlite3_result_int64(context, iVal); + } +} + +/* +** Scalar function "shell_escape_crnl" used by the .recover command. +** The argument passed to this function is the output of built-in +** function quote(). If the first character of the input is "'", +** indicating that the value passed to quote() was a text value, +** then this function searches the input for "\n" and "\r" characters +** and adds a wrapper similar to the following: +** +** replace(replace(, '\n', char(10), '\r', char(13)); +** +** Or, if the first character of the input is not "'", then a copy +** of the input is returned. +*/ +static void shellEscapeCrnl( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zText = (const char*)sqlite3_value_text(argv[0]); + if( zText[0]=='\'' ){ + int nText = sqlite3_value_bytes(argv[0]); + int i; + char zBuf1[20]; + char zBuf2[20]; + const char *zNL = 0; + const char *zCR = 0; + int nCR = 0; + int nNL = 0; + + for(i=0; zText[i]; i++){ + if( zNL==0 && zText[i]=='\n' ){ + zNL = unused_string(zText, "\\n", "\\012", zBuf1); + nNL = (int)strlen(zNL); + } + if( zCR==0 && zText[i]=='\r' ){ + zCR = unused_string(zText, "\\r", "\\015", zBuf2); + nCR = (int)strlen(zCR); + } + } + + if( zNL || zCR ){ + int iOut = 0; + i64 nMax = (nNL > nCR) ? nNL : nCR; + i64 nAlloc = nMax * nText + (nMax+64)*2; + char *zOut = (char*)sqlite3_malloc64(nAlloc); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + if( zNL && zCR ){ + memcpy(&zOut[iOut], "replace(replace(", 16); + iOut += 16; + }else{ + memcpy(&zOut[iOut], "replace(", 8); + iOut += 8; + } + for(i=0; zText[i]; i++){ + if( zText[i]=='\n' ){ + memcpy(&zOut[iOut], zNL, nNL); + iOut += nNL; + }else if( zText[i]=='\r' ){ + memcpy(&zOut[iOut], zCR, nCR); + iOut += nCR; + }else{ + zOut[iOut] = zText[i]; + iOut++; + } + } + + if( zNL ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zNL, nNL); iOut += nNL; + memcpy(&zOut[iOut], "', char(10))", 12); iOut += 12; + } + if( zCR ){ + memcpy(&zOut[iOut], ",'", 2); iOut += 2; + memcpy(&zOut[iOut], zCR, nCR); iOut += nCR; + memcpy(&zOut[iOut], "', char(13))", 12); iOut += 12; + } + + sqlite3_result_text(context, zOut, iOut, SQLITE_TRANSIENT); + sqlite3_free(zOut); + return; + } + } + + sqlite3_result_value(context, argv[0]); +} + /* Flags for open_db(). ** ** The default behavior of open_db() is to exit(1) if the database fails to @@ -3999,6 +4125,9 @@ static void open_db(ShellState *p, int openFlags){ sqlite3_fileio_init(p->db, 0, 0); sqlite3_shathree_init(p->db, 0, 0); sqlite3_completion_init(p->db, 0, 0); +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) + sqlite3_dbdata_init(p->db, 0, 0); +#endif #ifdef SQLITE_HAVE_ZLIB sqlite3_zipfile_init(p->db, 0, 0); sqlite3_sqlar_init(p->db, 0, 0); @@ -4009,6 +4138,10 @@ static void open_db(ShellState *p, int openFlags){ shellModuleSchema, 0, 0); sqlite3_create_function(p->db, "shell_putsnl", 1, SQLITE_UTF8, p, shellPutsFunc, 0, 0); + sqlite3_create_function(p->db, "shell_escape_crnl", 1, SQLITE_UTF8, 0, + shellEscapeCrnl, 0, 0); + sqlite3_create_function(p->db, "shell_int32", 2, SQLITE_UTF8, 0, + shellInt32, 0, 0); #ifndef SQLITE_NOHAVE_SYSTEM sqlite3_create_function(p->db, "edit", 1, SQLITE_UTF8, 0, editFunc, 0, 0); @@ -5263,10 +5396,7 @@ static int lintDotCommand( return SQLITE_ERROR; } -#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) -/********************************************************************************* -** The ".archive" or ".ar" command. -*/ +#if !defined SQLITE_OMIT_VIRTUALTABLE static void shellPrepare( sqlite3 *db, int *pRc, @@ -5337,6 +5467,12 @@ static void shellReset( *pRc = rc; } } +#endif /* !defined SQLITE_OMIT_VIRTUALTABLE */ + +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) +/********************************************************************************* +** The ".archive" or ".ar" command. +*/ /* ** Structure representing a single ".ar" command. */ @@ -6026,6 +6162,631 @@ end_ar_command: **********************************************************************************/ #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_HAVE_ZLIB) */ +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) +/* +** If (*pRc) is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, the SQL statement or statements in zSql are executed using +** database connection db and the error code written to *pRc before +** this function returns. +*/ +static void shellExec(sqlite3 *db, int *pRc, const char *zSql){ + int rc = *pRc; + if( rc==SQLITE_OK ){ + char *zErr = 0; + rc = sqlite3_exec(db, zSql, 0, 0, &zErr); + if( rc!=SQLITE_OK ){ + raw_printf(stderr, "SQL error: %s\n", zErr); + } + *pRc = rc; + } +} + +/* +** Like shellExec(), except that zFmt is a printf() style format string. +*/ +static void shellExecPrintf(sqlite3 *db, int *pRc, const char *zFmt, ...){ + char *z = 0; + if( *pRc==SQLITE_OK ){ + va_list ap; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( z==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + shellExec(db, pRc, z); + } + sqlite3_free(z); + } +} + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, an attempt is made to allocate, zero and return a pointer +** to a buffer nByte bytes in size. If an OOM error occurs, *pRc is set +** to SQLITE_NOMEM and NULL returned. +*/ +static void *shellMalloc(int *pRc, sqlite3_int64 nByte){ + void *pRet = 0; + if( *pRc==SQLITE_OK ){ + pRet = sqlite3_malloc64(nByte); + if( pRet==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, nByte); + } + } + return pRet; +} + +/* +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, zFmt is treated as a printf() style string. The result of +** formatting it along with any trailing arguments is written into a +** buffer obtained from sqlite3_malloc(), and pointer to which is returned. +** It is the responsibility of the caller to eventually free this buffer +** using a call to sqlite3_free(). +** +** If an OOM error occurs, (*pRc) is set to SQLITE_NOMEM and a NULL +** pointer returned. +*/ +static char *shellMPrintf(int *pRc, const char *zFmt, ...){ + char *z = 0; + if( *pRc==SQLITE_OK ){ + va_list ap; + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + va_end(ap); + if( z==0 ){ + *pRc = SQLITE_NOMEM; + } + } + return z; +} + +/* +** When running the ".recover" command, each output table, and the special +** orphaned row table if it is required, is represented by an instance +** of the following struct. +*/ +typedef struct RecoverTable RecoverTable; +struct RecoverTable { + char *zQuoted; /* Quoted version of table name */ + int nCol; /* Number of columns in table */ + char **azlCol; /* Array of column lists */ + int iPk; /* Index of IPK column */ +}; + +/* +** Free a RecoverTable object allocated by recoverFindTable() or +** recoverOrphanTable(). +*/ +static void recoverFreeTable(RecoverTable *pTab){ + if( pTab ){ + sqlite3_free(pTab->zQuoted); + if( pTab->azlCol ){ + int i; + for(i=0; i<=pTab->nCol; i++){ + sqlite3_free(pTab->azlCol[i]); + } + sqlite3_free(pTab->azlCol); + } + sqlite3_free(pTab); + } +} + +/* +** This function is a no-op if (*pRc) is not SQLITE_OK when it is called. +** Otherwise, it allocates and returns a RecoverTable object based on the +** final four arguments passed to this function. It is the responsibility +** of the caller to eventually free the returned object using +** recoverFreeTable(). +*/ +static RecoverTable *recoverNewTable( + int *pRc, /* IN/OUT: Error code */ + const char *zName, /* Name of table */ + const char *zSql, /* CREATE TABLE statement */ + int bIntkey, + int nCol +){ + sqlite3 *dbtmp = 0; /* sqlite3 handle for testing CREATE TABLE */ + int rc = *pRc; + RecoverTable *pTab = 0; + + pTab = (RecoverTable*)shellMalloc(&rc, sizeof(RecoverTable)); + if( rc==SQLITE_OK ){ + int nSqlCol = 0; + int bSqlIntkey = 0; + sqlite3_stmt *pStmt = 0; + + rc = sqlite3_open("", &dbtmp); + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(dbtmp, "PRAGMA writable_schema = on", 0, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(dbtmp, zSql, 0, 0, 0); + if( rc==SQLITE_ERROR ){ + rc = SQLITE_OK; + goto finished; + } + } + shellPreparePrintf(dbtmp, &rc, &pStmt, + "SELECT count(*) FROM pragma_table_info(%Q)", zName + ); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + nSqlCol = sqlite3_column_int(pStmt, 0); + } + shellFinalize(&rc, pStmt); + + if( rc!=SQLITE_OK || nSqlColiPk to the index + ** of the column, where columns are 0-numbered from left to right. + ** Or, if this is a WITHOUT ROWID table or if there is no IPK column, + ** leave zPk as "_rowid_" and pTab->iPk at -2. */ + pTab->iPk = -2; + if( bIntkey ){ + shellPreparePrintf(dbtmp, &rc, &pPkFinder, + "SELECT cid, name FROM pragma_table_info(%Q) " + " WHERE pk=1 AND type='integer' COLLATE nocase" + " AND NOT EXISTS (SELECT cid FROM pragma_table_info(%Q) WHERE pk=2)" + , zName, zName + ); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPkFinder) ){ + pTab->iPk = sqlite3_column_int(pPkFinder, 0); + zPk = (const char*)sqlite3_column_text(pPkFinder, 1); + } + } + + pTab->zQuoted = shellMPrintf(&rc, "%Q", zName); + pTab->azlCol = (char**)shellMalloc(&rc, sizeof(char*) * (nSqlCol+1)); + pTab->nCol = nSqlCol; + + if( bIntkey ){ + pTab->azlCol[0] = shellMPrintf(&rc, "%Q", zPk); + }else{ + pTab->azlCol[0] = shellMPrintf(&rc, ""); + } + i = 1; + shellPreparePrintf(dbtmp, &rc, &pStmt, + "SELECT %Q || group_concat(name, ', ') " + " FILTER (WHERE cid!=%d) OVER (ORDER BY %s cid) " + "FROM pragma_table_info(%Q)", + bIntkey ? ", " : "", pTab->iPk, + bIntkey ? "" : "(CASE WHEN pk=0 THEN 1000000 ELSE pk END), ", + zName + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zText = (const char*)sqlite3_column_text(pStmt, 0); + pTab->azlCol[i] = shellMPrintf(&rc, "%s%s", pTab->azlCol[0], zText); + i++; + } + shellFinalize(&rc, pStmt); + + shellFinalize(&rc, pPkFinder); + } + } + + finished: + sqlite3_close(dbtmp); + *pRc = rc; + if( rc!=SQLITE_OK ){ + recoverFreeTable(pTab); + pTab = 0; + } + return pTab; +} + +/* +** This function is called to search the schema recovered from the +** sqlite_master table of the (possibly) corrupt database as part +** of a ".recover" command. Specifically, for a table with root page +** iRoot and at least nCol columns. Additionally, if bIntkey is 0, the +** table must be a WITHOUT ROWID table, or if non-zero, not one of +** those. +** +** If a table is found, a (RecoverTable*) object is returned. Or, if +** no such table is found, but bIntkey is false and iRoot is the +** root page of an index in the recovered schema, then (*pbNoop) is +** set to true and NULL returned. Or, if there is no such table or +** index, NULL is returned and (*pbNoop) set to 0, indicating that +** the caller should write data to the orphans table. +*/ +static RecoverTable *recoverFindTable( + ShellState *pState, /* Shell state object */ + int *pRc, /* IN/OUT: Error code */ + int iRoot, /* Root page of table */ + int bIntkey, /* True for an intkey table */ + int nCol, /* Number of columns in table */ + int *pbNoop /* OUT: True if iRoot is root of index */ +){ + sqlite3_stmt *pStmt = 0; + RecoverTable *pRet = 0; + int bNoop = 0; + const char *zSql = 0; + const char *zName = 0; + + /* Search the recovered schema for an object with root page iRoot. */ + shellPreparePrintf(pState->db, pRc, &pStmt, + "SELECT type, name, sql FROM recovery.schema WHERE rootpage=%d", iRoot + ); + while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zType = (const char*)sqlite3_column_text(pStmt, 0); + if( bIntkey==0 && sqlite3_stricmp(zType, "index")==0 ){ + bNoop = 1; + break; + } + if( sqlite3_stricmp(zType, "table")==0 ){ + zName = (const char*)sqlite3_column_text(pStmt, 1); + zSql = (const char*)sqlite3_column_text(pStmt, 2); + pRet = recoverNewTable(pRc, zName, zSql, bIntkey, nCol); + break; + } + } + + shellFinalize(pRc, pStmt); + *pbNoop = bNoop; + return pRet; +} + +/* +** Return a RecoverTable object representing the orphans table. +*/ +static RecoverTable *recoverOrphanTable( + ShellState *pState, /* Shell state object */ + int *pRc, /* IN/OUT: Error code */ + const char *zLostAndFound, /* Base name for orphans table */ + int nCol /* Number of user data columns */ +){ + RecoverTable *pTab = 0; + if( nCol>=0 && *pRc==SQLITE_OK ){ + int i; + + /* This block determines the name of the orphan table. The prefered + ** name is zLostAndFound. But if that clashes with another name + ** in the recovered schema, try zLostAndFound_0, zLostAndFound_1 + ** and so on until a non-clashing name is found. */ + int iTab = 0; + char *zTab = shellMPrintf(pRc, "%s", zLostAndFound); + sqlite3_stmt *pTest = 0; + shellPrepare(pState->db, pRc, + "SELECT 1 FROM recovery.schema WHERE name=?", &pTest + ); + if( pTest ) sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT); + while( *pRc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pTest) ){ + shellReset(pRc, pTest); + sqlite3_free(zTab); + zTab = shellMPrintf(pRc, "%s_%d", zLostAndFound, iTab++); + sqlite3_bind_text(pTest, 1, zTab, -1, SQLITE_TRANSIENT); + } + shellFinalize(pRc, pTest); + + pTab = (RecoverTable*)shellMalloc(pRc, sizeof(RecoverTable)); + if( pTab ){ + pTab->zQuoted = shellMPrintf(pRc, "%Q", zTab); + pTab->nCol = nCol; + pTab->iPk = -2; + if( nCol>0 ){ + pTab->azlCol = (char**)shellMalloc(pRc, sizeof(char*) * (nCol+1)); + if( pTab->azlCol ){ + pTab->azlCol[nCol] = shellMPrintf(pRc, ""); + for(i=nCol-1; i>=0; i--){ + pTab->azlCol[i] = shellMPrintf(pRc, "%s, NULL", pTab->azlCol[i+1]); + } + } + } + + if( *pRc!=SQLITE_OK ){ + recoverFreeTable(pTab); + pTab = 0; + }else{ + raw_printf(pState->out, + "CREATE TABLE %s(rootpgno INTEGER, " + "pgno INTEGER, nfield INTEGER, id INTEGER", pTab->zQuoted + ); + for(i=0; iout, ", c%d", i); + } + raw_printf(pState->out, ");\n"); + } + } + sqlite3_free(zTab); + } + return pTab; +} + +/* +** This function is called to recover data from the database. A script +** to construct a new database containing all recovered data is output +** on stream pState->out. +*/ +static int recoverDatabaseCmd(ShellState *pState, int nArg, char **azArg){ + int rc = SQLITE_OK; + sqlite3_stmt *pLoop = 0; /* Loop through all root pages */ + sqlite3_stmt *pPages = 0; /* Loop through all pages in a group */ + sqlite3_stmt *pCells = 0; /* Loop through all cells in a page */ + const char *zRecoveryDb = ""; /* Name of "recovery" database */ + const char *zLostAndFound = "lost_and_found"; + int i; + int nOrphan = -1; + RecoverTable *pOrphan = 0; + + int bFreelist = 1; /* 0 if --freelist-corrupt is specified */ + for(i=1; idb, &rc, + /* Attach an in-memory database named 'recovery'. Create an indexed + ** cache of the sqlite_dbptr virtual table. */ + "ATTACH %Q AS recovery;" + "DROP TABLE IF EXISTS recovery.dbptr;" + "DROP TABLE IF EXISTS recovery.freelist;" + "DROP TABLE IF EXISTS recovery.map;" + "DROP TABLE IF EXISTS recovery.schema;" + "CREATE TABLE recovery.freelist(pgno INTEGER PRIMARY KEY);", zRecoveryDb + ); + + if( bFreelist ){ + shellExec(pState->db, &rc, + "WITH trunk(pgno) AS (" + " SELECT shell_int32(" + " (SELECT data FROM sqlite_dbpage WHERE pgno=1), 8) AS x " + " WHERE x>0" + " UNION" + " SELECT shell_int32(" + " (SELECT data FROM sqlite_dbpage WHERE pgno=trunk.pgno), 0) AS x " + " FROM trunk WHERE x>0" + ")," + "freelist(data, n, freepgno) AS (" + " SELECT data, shell_int32(data, 1)-1, t.pgno " + " FROM trunk t, sqlite_dbpage s WHERE s.pgno=t.pgno" + " UNION ALL" + " SELECT data, n-1, shell_int32(data, 2+n) " + " FROM freelist WHERE n>=0" + ")" + "REPLACE INTO recovery.freelist SELECT freepgno FROM freelist;" + ); + } + + shellExec(pState->db, &rc, + "CREATE TABLE recovery.dbptr(" + " pgno, child, PRIMARY KEY(child, pgno)" + ") WITHOUT ROWID;" + "INSERT OR IGNORE INTO recovery.dbptr(pgno, child) " + " SELECT * FROM sqlite_dbptr" + " WHERE pgno NOT IN freelist AND child NOT IN freelist;" + + /* Delete any pointer to page 1. This ensures that page 1 is considered + ** a root page, regardless of how corrupt the db is. */ + "DELETE FROM recovery.dbptr WHERE child = 1;" + + /* Delete all pointers to any pages that have more than one pointer + ** to them. Such pages will be treated as root pages when recovering + ** data. */ + "DELETE FROM recovery.dbptr WHERE child IN (" + " SELECT child FROM recovery.dbptr GROUP BY child HAVING count(*)>1" + ");" + + /* Create the "map" table that will (eventually) contain instructions + ** for dealing with each page in the db that contains one or more + ** records. */ + "CREATE TABLE recovery.map(" + "pgno INTEGER PRIMARY KEY, maxlen INT, intkey, root INT" + ");" + + /* Populate table [map]. If there are circular loops of pages in the + ** database, the following adds all pages in such a loop to the map + ** as individual root pages. This could be handled better. */ + "WITH pages(i, maxlen) AS (" + " SELECT page_count, (" + " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=page_count" + " ) FROM pragma_page_count" + " UNION ALL" + " SELECT i-1, (" + " SELECT max(field+1) FROM sqlite_dbdata WHERE pgno=i-1" + " ) FROM pages WHERE i>=2" + ")" + "INSERT INTO recovery.map(pgno, maxlen, intkey, root) " + " SELECT i, maxlen, NULL, (" + " WITH p(orig, pgno, parent) AS (" + " SELECT 0, i, (SELECT pgno FROM recovery.dbptr WHERE child=i)" + " UNION ALL" + " SELECT i, p.parent, " + " (SELECT pgno FROM recovery.dbptr WHERE child=p.parent) FROM p" + " )" + " SELECT pgno FROM p WHERE (parent IS NULL OR pgno = orig)" + ") " + "FROM pages WHERE maxlen > 0 AND i NOT IN freelist;" + "UPDATE recovery.map AS o SET intkey = (" + " SELECT substr(data, 1, 1)==X'0D' FROM sqlite_dbpage WHERE pgno=o.pgno" + ");" + + /* Extract data from page 1 and any linked pages into table + ** recovery.schema. With the same schema as an sqlite_master table. */ + "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);" + "INSERT INTO recovery.schema SELECT " + " max(CASE WHEN field=0 THEN value ELSE NULL END)," + " max(CASE WHEN field=1 THEN value ELSE NULL END)," + " max(CASE WHEN field=2 THEN value ELSE NULL END)," + " max(CASE WHEN field=3 THEN value ELSE NULL END)," + " max(CASE WHEN field=4 THEN value ELSE NULL END)" + "FROM sqlite_dbdata WHERE pgno IN (" + " SELECT pgno FROM recovery.map WHERE root=1" + ")" + "GROUP BY pgno, cell;" + "CREATE INDEX recovery.schema_rootpage ON schema(rootpage);" + ); + + /* Open a transaction, then print out all non-virtual, non-"sqlite_%" + ** CREATE TABLE statements that extracted from the existing schema. */ + if( rc==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + raw_printf(pState->out, "BEGIN;\n"); + raw_printf(pState->out, "PRAGMA writable_schema = on;\n"); + shellPrepare(pState->db, &rc, + "SELECT sql FROM recovery.schema " + "WHERE type='table' AND sql LIKE 'create table%'", &pStmt + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zCreateTable = (const char*)sqlite3_column_text(pStmt, 0); + raw_printf(pState->out, "CREATE TABLE IF NOT EXISTS %s;\n", + &zCreateTable[12] + ); + } + shellFinalize(&rc, pStmt); + } + + /* Figure out if an orphan table will be required. And if so, how many + ** user columns it should contain */ + shellPrepare(pState->db, &rc, + "SELECT coalesce(max(maxlen), -2) FROM recovery.map" + " WHERE root>1 AND root NOT IN (SELECT rootpage FROM recovery.schema)" + , &pLoop + ); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ + nOrphan = sqlite3_column_int(pLoop, 0); + } + shellFinalize(&rc, pLoop); + pLoop = 0; + pOrphan = recoverOrphanTable(pState, &rc, zLostAndFound, nOrphan); + + shellPrepare(pState->db, &rc, + "SELECT pgno FROM recovery.map WHERE root=?", &pPages + ); + shellPrepare(pState->db, &rc, + "SELECT max(field), group_concat(shell_escape_crnl(quote(value)), ', ')" + "FROM sqlite_dbdata WHERE pgno = ? AND field != ?" + "GROUP BY cell", &pCells + ); + + /* Loop through each root page. */ + shellPrepare(pState->db, &rc, + "SELECT root, intkey, max(maxlen) FROM recovery.map" + " WHERE root>1 GROUP BY root, intkey ORDER BY root=(" + " SELECT rootpage FROM recovery.schema WHERE name='sqlite_sequence'" + ")", &pLoop + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pLoop) ){ + int iRoot = sqlite3_column_int(pLoop, 0); + int bIntkey = sqlite3_column_int(pLoop, 1); + int nCol = sqlite3_column_int(pLoop, 2); + int bNoop = 0; + RecoverTable *pTab; + + pTab = recoverFindTable(pState, &rc, iRoot, bIntkey, nCol, &bNoop); + if( bNoop || rc ) continue; + if( pTab==0 ) pTab = pOrphan; + + if( 0==sqlite3_stricmp(pTab->zQuoted, "'sqlite_sequence'") ){ + raw_printf(pState->out, "DELETE FROM sqlite_sequence;\n"); + } + sqlite3_bind_int(pPages, 1, iRoot); + sqlite3_bind_int(pCells, 2, pTab->iPk); + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPages) ){ + int iPgno = sqlite3_column_int(pPages, 0); + sqlite3_bind_int(pCells, 1, iPgno); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pCells) ){ + int nField = sqlite3_column_int(pCells, 0); + const char *zVal = (const char*)sqlite3_column_text(pCells, 1); + + nField = nField+1; + if( pTab==pOrphan ){ + raw_printf(pState->out, + "INSERT INTO %s VALUES(%d, %d, %d, %s%s%s);\n", + pTab->zQuoted, iRoot, iPgno, nField, + bIntkey ? "" : "NULL, ", zVal, pTab->azlCol[nField] + ); + }else{ + raw_printf(pState->out, "INSERT INTO %s(%s) VALUES( %s );\n", + pTab->zQuoted, pTab->azlCol[nField], zVal + ); + } + } + shellReset(&rc, pCells); + } + shellReset(&rc, pPages); + if( pTab!=pOrphan ) recoverFreeTable(pTab); + } + shellFinalize(&rc, pLoop); + shellFinalize(&rc, pPages); + shellFinalize(&rc, pCells); + recoverFreeTable(pOrphan); + + /* The rest of the schema */ + if( rc==SQLITE_OK ){ + sqlite3_stmt *pStmt = 0; + shellPrepare(pState->db, &rc, + "SELECT sql, name FROM recovery.schema " + "WHERE sql NOT LIKE 'create table%'", &pStmt + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zSql = (const char*)sqlite3_column_text(pStmt, 0); + if( sqlite3_strnicmp(zSql, "create virt", 11)==0 ){ + const char *zName = (const char*)sqlite3_column_text(pStmt, 1); + char *zPrint = shellMPrintf(&rc, + "INSERT INTO sqlite_master VALUES('table', %Q, %Q, 0, %Q)", + zName, zName, zSql + ); + raw_printf(pState->out, "%s;\n", zPrint); + sqlite3_free(zPrint); + }else{ + raw_printf(pState->out, "%s;\n", zSql); + } + } + shellFinalize(&rc, pStmt); + } + + if( rc==SQLITE_OK ){ + raw_printf(pState->out, "PRAGMA writable_schema = off;\n"); + raw_printf(pState->out, "COMMIT;\n"); + } + sqlite3_exec(pState->db, "DETACH recovery", 0, 0, 0); + return rc; +} +#endif /* !(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) */ + /* ** If an input line begins with "." then invoke this routine to @@ -6313,6 +7074,13 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = shell_dbinfo_command(p, nArg, azArg); }else +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) + if( c=='r' && strncmp(azArg[0], "recover", n)==0 ){ + open_db(p, 0); + rc = recoverDatabaseCmd(p, nArg, azArg); + }else +#endif /* !(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_ENABLE_DBPAGE_VTAB) */ + if( c=='d' && strncmp(azArg[0], "dump", n)==0 ){ const char *zLike = 0; int i; @@ -6350,7 +7118,9 @@ static int do_meta_command(char *zLine, ShellState *p){ zLike = azArg[i]; } } + open_db(p, 0); + /* When playing back a "dump", the content might appear in an order ** which causes immediate foreign key constraints to be violated. ** So disable foreign-key constraint enforcement to prevent problems. */ @@ -6398,7 +7168,7 @@ static int do_meta_command(char *zLine, ShellState *p){ } sqlite3_exec(p->db, "PRAGMA writable_schema=OFF;", 0, 0, 0); sqlite3_exec(p->db, "RELEASE dump;", 0, 0, 0); - raw_printf(p->out, p->nErr ? "ROLLBACK; -- due to errors\n" : "COMMIT;\n"); + raw_printf(p->out, p->nErr?"ROLLBACK; -- due to errors\n":"COMMIT;\n"); p->showHeader = savedShowHeader; p->shellFlgs = savedShellFlags; }else diff --git a/src/vdbe.c b/src/vdbe.c index c004137c47..9bc5e4ee56 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -195,14 +195,6 @@ int sqlite3_found_count = 0; } #endif -/* -** Convert the given register into a string if it isn't one -** already. Return non-zero if a malloc() fails. -*/ -#define Stringify(P, enc) \ - if(((P)->flags&(MEM_Str|MEM_Blob))==0 && sqlite3VdbeMemStringify(P,enc,0)) \ - { goto no_mem; } - /* ** An ephemeral string value (signified by the MEM_Ephem flag) contains ** a pointer to a dynamically allocated string where some other entity @@ -522,6 +514,8 @@ static void memTracePrint(Mem *p){ printf(p->flags & MEM_Zero ? " NULL-nochng" : " NULL"); }else if( (p->flags & (MEM_Int|MEM_Str))==(MEM_Int|MEM_Str) ){ printf(" si:%lld", p->u.i); + }else if( (p->flags & (MEM_Int|MEM_IntReal))==(MEM_Int|MEM_IntReal) ){ + printf(" ir:%lld", p->u.i); }else if( p->flags & MEM_Int ){ printf(" i:%lld", p->u.i); #ifndef SQLITE_OMIT_FLOATING_POINT @@ -1463,19 +1457,38 @@ case OP_ResultRow: { ** to avoid a memcpy(). */ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ - i64 nByte; + i64 nByte; /* Total size of the output string or blob */ + u16 flags1; /* Initial flags for P1 */ + u16 flags2; /* Initial flags for P2 */ pIn1 = &aMem[pOp->p1]; pIn2 = &aMem[pOp->p2]; pOut = &aMem[pOp->p3]; + testcase( pIn1==pIn2 ); + testcase( pOut==pIn2 ); assert( pIn1!=pOut ); - if( (pIn1->flags | pIn2->flags) & MEM_Null ){ + flags1 = pIn1->flags; + testcase( flags1 & MEM_Null ); + testcase( pIn2->flags & MEM_Null ); + if( (flags1 | pIn2->flags) & MEM_Null ){ sqlite3VdbeMemSetNull(pOut); break; } - if( ExpandBlob(pIn1) || ExpandBlob(pIn2) ) goto no_mem; - Stringify(pIn1, encoding); - Stringify(pIn2, encoding); + if( (flags1 & (MEM_Str|MEM_Blob))==0 ){ + if( sqlite3VdbeMemStringify(pIn1,encoding,0) ) goto no_mem; + flags1 = pIn1->flags & ~MEM_Str; + }else if( (flags1 & MEM_Zero)!=0 ){ + if( sqlite3VdbeMemExpandBlob(pIn1) ) goto no_mem; + flags1 = pIn1->flags & ~MEM_Str; + } + flags2 = pIn2->flags; + if( (flags2 & (MEM_Str|MEM_Blob))==0 ){ + if( sqlite3VdbeMemStringify(pIn2,encoding,0) ) goto no_mem; + flags2 = pIn2->flags & ~MEM_Str; + }else if( (flags2 & MEM_Zero)!=0 ){ + if( sqlite3VdbeMemExpandBlob(pIn2) ) goto no_mem; + flags2 = pIn2->flags & ~MEM_Str; + } nByte = pIn1->n + pIn2->n; if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; @@ -1486,8 +1499,12 @@ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ MemSetTypeFlag(pOut, MEM_Str); if( pOut!=pIn2 ){ memcpy(pOut->z, pIn2->z, pIn2->n); + assert( (pIn2->flags & MEM_Dyn) == (flags2 & MEM_Dyn) ); + pIn2->flags = flags2; } memcpy(&pOut->z[pIn2->n], pIn1->z, pIn1->n); + assert( (pIn1->flags & MEM_Dyn) == (flags1 & MEM_Dyn) ); + pIn1->flags = flags1; pOut->z[nByte]=0; pOut->z[nByte+1] = 0; pOut->flags |= MEM_Term; @@ -2765,12 +2782,20 @@ case OP_Affinity: { assert( pOp->p2>0 ); assert( zAffinity[pOp->p2]==0 ); pIn1 = &aMem[pOp->p1]; - do{ + while( 1 /*edit-by-break*/ ){ assert( pIn1 <= &p->aMem[(p->nMem+1 - p->nCursor)] ); assert( memIsValid(pIn1) ); - applyAffinity(pIn1, *(zAffinity++), encoding); + applyAffinity(pIn1, zAffinity[0], encoding); + if( zAffinity[0]==SQLITE_AFF_REAL && (pIn1->flags & MEM_Int)!=0 ){ + /* When applying REAL affinity, if the result is still MEM_Int, + ** indicate that REAL is actually desired */ + pIn1->flags |= MEM_IntReal; + } + REGISTER_TRACE((int)(pIn1-aMem), pIn1); + zAffinity++; + if( zAffinity[0]==0 ) break; pIn1++; - }while( zAffinity[0] ); + } break; } @@ -2791,7 +2816,6 @@ case OP_Affinity: { ** If P4 is NULL then all index fields have the affinity BLOB. */ case OP_MakeRecord: { - u8 *zNewRecord; /* A buffer to hold the data for the new record */ Mem *pRec; /* The new record */ u64 nData; /* Number of bytes of data space */ int nHdr; /* Number of bytes of header space */ @@ -2804,9 +2828,9 @@ case OP_MakeRecord: { int nField; /* Number of fields in the record */ char *zAffinity; /* The affinity string for the record */ int file_format; /* File format to use for encoding */ - int i; /* Space used in zNewRecord[] header */ - int j; /* Space used in zNewRecord[] content */ u32 len; /* Length of a field */ + u8 *zHdr; /* Where to write next byte of the header */ + u8 *zPayload; /* Where to write next byte of the payload */ /* Assuming the record contains N fields, the record format looks ** like this: @@ -2933,34 +2957,34 @@ case OP_MakeRecord: { goto no_mem; } } - zNewRecord = (u8 *)pOut->z; - - /* Write the record */ - i = putVarint32(zNewRecord, nHdr); - j = nHdr; - assert( pData0<=pLast ); - pRec = pData0; - do{ - serial_type = pRec->uTemp; - /* EVIDENCE-OF: R-06529-47362 Following the size varint are one or more - ** additional varints, one per column. */ - i += putVarint32(&zNewRecord[i], serial_type); /* serial type */ - /* EVIDENCE-OF: R-64536-51728 The values for each column in the record - ** immediately follow the header. */ - j += sqlite3VdbeSerialPut(&zNewRecord[j], pRec, serial_type); /* content */ - }while( (++pRec)<=pLast ); - assert( i==nHdr ); - assert( j==nByte ); - - assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); pOut->n = (int)nByte; pOut->flags = MEM_Blob; if( nZero ){ pOut->u.nZero = nZero; pOut->flags |= MEM_Zero; } - REGISTER_TRACE(pOp->p3, pOut); UPDATE_MAX_BLOBSIZE(pOut); + zHdr = (u8 *)pOut->z; + zPayload = zHdr + nHdr; + + /* Write the record */ + zHdr += putVarint32(zHdr, nHdr); + assert( pData0<=pLast ); + pRec = pData0; + do{ + serial_type = pRec->uTemp; + /* EVIDENCE-OF: R-06529-47362 Following the size varint are one or more + ** additional varints, one per column. */ + zHdr += putVarint32(zHdr, serial_type); /* serial type */ + /* EVIDENCE-OF: R-64536-51728 The values for each column in the record + ** immediately follow the header. */ + zPayload += sqlite3VdbeSerialPut(zPayload, pRec, serial_type); /* content */ + }while( (++pRec)<=pLast ); + assert( nHdr==(int)(zHdr - (u8*)pOut->z) ); + assert( nByte==(int)(zPayload - (u8*)pOut->z) ); + + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); + REGISTER_TRACE(pOp->p3, pOut); break; } diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 15a371d550..c84e4e4390 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -247,7 +247,7 @@ struct sqlite3_value { #define MEM_Blob 0x0010 /* Value is a BLOB */ #define MEM_AffMask 0x001f /* Mask of affinity bits */ #define MEM_FromBind 0x0020 /* Value originates from sqlite3_bind() */ -/* Available 0x0040 */ +#define MEM_IntReal 0x0040 /* MEM_Int that stringifies like MEM_Real */ #define MEM_Undefined 0x0080 /* Value is undefined */ #define MEM_Cleared 0x0100 /* NULL set by OP_Null, not from data */ #define MEM_TypeMask 0xc1df /* Mask of type bits */ diff --git a/src/vdbemem.c b/src/vdbemem.c index 80d494867c..82f7f3e102 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -92,6 +92,25 @@ int sqlite3VdbeCheckMemInvariants(Mem *p){ } #endif +/* +** Render a Mem object which is either MEM_Int or MEM_Real into a +** buffer. +*/ +static void vdbeMemRenderNum(int sz, char *zBuf, Mem *p){ + StrAccum acc; + assert( p->flags & (MEM_Int|MEM_Real) ); + sqlite3StrAccumInit(&acc, 0, zBuf, sz, 0); + if( p->flags & MEM_IntReal ){ + sqlite3_str_appendf(&acc, "%!.15g", (double)p->u.i); + }else if( p->flags & MEM_Int ){ + sqlite3_str_appendf(&acc, "%lld", p->u.i); + }else{ + sqlite3_str_appendf(&acc, "%!.15g", p->u.r); + } + assert( acc.zText==zBuf && acc.mxAlloc<=0 ); + zBuf[acc.nChar] = 0; /* Fast version of sqlite3StrAccumFinish(&acc) */ +} + #ifdef SQLITE_DEBUG /* ** Check that string value of pMem agrees with its integer or real value. @@ -118,11 +137,7 @@ int sqlite3VdbeMemConsistentDualRep(Mem *p){ int i, j, incr; if( (p->flags & MEM_Str)==0 ) return 1; if( (p->flags & (MEM_Int|MEM_Real))==0 ) return 1; - if( p->flags & MEM_Int ){ - sqlite3_snprintf(sizeof(zBuf),zBuf,"%lld",p->u.i); - }else{ - sqlite3_snprintf(sizeof(zBuf),zBuf,"%!.15g",p->u.r); - } + vdbeMemRenderNum(sizeof(zBuf), zBuf, p); z = p->z; i = j = 0; incr = 1; @@ -248,7 +263,7 @@ int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ } assert( (pMem->flags & MEM_Dyn)==0 ); pMem->z = pMem->zMalloc; - pMem->flags &= (MEM_Null|MEM_Int|MEM_Real); + pMem->flags &= (MEM_Null|MEM_Int|MEM_Real|MEM_IntReal); return SQLITE_OK; } @@ -349,13 +364,12 @@ int sqlite3VdbeMemNulTerminate(Mem *pMem){ ** user and the latter is an internal programming error. */ int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ - int fg = pMem->flags; const int nByte = 32; assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( !(fg&MEM_Zero) ); - assert( !(fg&(MEM_Str|MEM_Blob)) ); - assert( fg&(MEM_Int|MEM_Real) ); + assert( !(pMem->flags&MEM_Zero) ); + assert( !(pMem->flags&(MEM_Str|MEM_Blob)) ); + assert( pMem->flags&(MEM_Int|MEM_Real) ); assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); @@ -365,23 +379,12 @@ int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ return SQLITE_NOMEM_BKPT; } - /* For a Real or Integer, use sqlite3_snprintf() to produce the UTF-8 - ** string representation of the value. Then, if the required encoding - ** is UTF-16le or UTF-16be do a translation. - ** - ** FIX ME: It would be better if sqlite3_snprintf() could do UTF-16. - */ - if( fg & MEM_Int ){ - sqlite3_snprintf(nByte, pMem->z, "%lld", pMem->u.i); - }else{ - assert( fg & MEM_Real ); - sqlite3_snprintf(nByte, pMem->z, "%!.15g", pMem->u.r); - } + vdbeMemRenderNum(nByte, pMem->z, pMem); assert( pMem->z!=0 ); pMem->n = sqlite3Strlen30NN(pMem->z); pMem->enc = SQLITE_UTF8; pMem->flags |= MEM_Str|MEM_Term; - if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real); + if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); sqlite3VdbeChangeEncoding(pMem, enc); return SQLITE_OK; } @@ -741,7 +744,7 @@ void sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ pMem->flags |= (pMem->flags&MEM_Blob)>>3; sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); - pMem->flags &= ~(MEM_Int|MEM_Real|MEM_Blob|MEM_Zero); + pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal|MEM_Blob|MEM_Zero); break; } } diff --git a/src/vtab.c b/src/vtab.c index 7806eb946f..41e26ef62f 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -841,6 +841,7 @@ int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab){ p = vtabDisconnectAll(db, pTab); xDestroy = p->pMod->pModule->xDestroy; assert( xDestroy!=0 ); /* Checked before the virtual table is created */ + pTab->nTabRef++; rc = xDestroy(p->pVtab); /* Remove the sqlite3_vtab* from the aVTrans[] array, if applicable */ if( rc==SQLITE_OK ){ @@ -849,6 +850,7 @@ int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab){ pTab->pVTable = 0; sqlite3VtabUnlock(p); } + sqlite3DeleteTable(db, pTab); } return rc; diff --git a/src/whereexpr.c b/src/whereexpr.c index 961495c584..90e1b5be65 100644 --- a/src/whereexpr.c +++ b/src/whereexpr.c @@ -263,11 +263,11 @@ static int isLikeOrGlob( } zNew[iTo] = 0; - /* If the RHS begins with a digit or a minus sign, then the LHS must be + /* If the RHS begins with a digit or a +/- sign, then the LHS must be ** an ordinary column (not a virtual table column) with TEXT affinity. ** Otherwise the LHS might be numeric and "lhs >= rhs" would be false ** even though "lhs LIKE rhs" is true. But if the RHS does not start - ** with a digit or '-', then "lhs LIKE rhs" will always be false if + ** with a digit or +/-, then "lhs LIKE rhs" will always be false if ** the LHS is numeric and so the optimization still works. ** ** 2018-09-10 ticket c94369cae9b561b1f996d0054bfab11389f9d033 @@ -277,6 +277,7 @@ static int isLikeOrGlob( */ if( sqlite3Isdigit(zNew[0]) || zNew[0]=='-' + || zNew[0]=='+' || (zNew[0]+1=='0' && iTo==1) ){ if( pLeft->op!=TK_COLUMN diff --git a/test/altertab3.test b/test/altertab3.test index 2bf6a9ddc1..b37d9ed1f9 100644 --- a/test/altertab3.test +++ b/test/altertab3.test @@ -142,6 +142,39 @@ do_execsql_test 6.1 { ALTER TABLE Table0 RENAME Col0 TO Col0; } +#------------------------------------------------------------------------- +reset_db +do_execsql_test 7.1.0 { + CREATE TABLE t1(a,b,c); + CREATE TRIGGER AFTER INSERT ON t1 BEGIN + SELECT a, rank() OVER w1 FROM t1 + WINDOW w1 AS (PARTITION BY b, percent_rank() OVER w1); + END; +} + +do_execsql_test 7.1.2 { + ALTER TABLE t1 RENAME TO t1x; + SELECT sql FROM sqlite_master; +} { + {CREATE TABLE "t1x"(a,b,c)} + {CREATE TRIGGER AFTER INSERT ON "t1x" BEGIN + SELECT a, rank() OVER w1 FROM "t1x" + WINDOW w1 AS (PARTITION BY b, percent_rank() OVER w1); + END} +} + +do_execsql_test 7.2.1 { + DROP TRIGGER after; + CREATE TRIGGER AFTER INSERT ON t1x BEGIN + SELECT a, rank() OVER w1 FROM t1x + WINDOW w1 AS (PARTITION BY b, percent_rank() OVER w1 ORDER BY d); + END; +} + +do_catchsql_test 7.2.2 { + ALTER TABLE t1x RENAME TO t1; +} {1 {error in trigger AFTER: no such column: d}} + finish_test diff --git a/test/dbdata.test b/test/dbdata.test new file mode 100644 index 0000000000..5b1150c78f --- /dev/null +++ b/test/dbdata.test @@ -0,0 +1,115 @@ +# 2019-04-11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the sqlite_dbpage virtual table. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix dbdata + +ifcapable !vtab||!compound { + finish_test + return +} +if { [catch { db enable_load_extension 1 }] + || [catch { db eval { SELECT load_extension('../dbdata') } }] +} { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE TABLE T1(a, b); + INSERT INTO t1(rowid, a ,b) VALUES(5, 'v', 'five'); + INSERT INTO t1(rowid, a, b) VALUES(10, 'x', 'ten'); +} + +do_execsql_test 1.1 { + SELECT pgno, cell, field, quote(value) FROM sqlite_dbdata WHERE pgno=2; +} { + 2 0 -1 5 + 2 0 0 'v' + 2 0 1 'five' + 2 1 -1 10 + 2 1 0 'x' + 2 1 1 'ten' +} + +breakpoint +do_execsql_test 1.2 { + SELECT pgno, cell, field, quote(value) FROM sqlite_dbdata; +} { + 1 0 -1 1 + 1 0 0 'table' + 1 0 1 'T1' + 1 0 2 'T1' + 1 0 3 2 + 1 0 4 {'CREATE TABLE T1(a, b)'} + 2 0 -1 5 + 2 0 0 'v' + 2 0 1 'five' + 2 1 -1 10 + 2 1 0 'x' + 2 1 1 'ten' +} + +set big [string repeat big 2000] +do_execsql_test 1.3 { + INSERT INTO t1 VALUES(NULL, $big); + SELECT value FROM sqlite_dbdata WHERE pgno=2 AND cell=2 AND field=1; +} $big + +do_execsql_test 1.4 { + DELETE FROM t1; + INSERT INTO t1 VALUES(NULL, randomblob(5050)); +} +do_test 1.5 { + execsql { + SELECT quote(value) FROM sqlite_dbdata WHERE pgno=2 AND cell=0 AND field=1; + } +} [db one {SELECT quote(b) FROM t1}] + +#------------------------------------------------------------------------- +reset_db +db enable_load_extension 1 +db eval { SELECT load_extension('../dbdata') } + +do_execsql_test 2.0 { + CREATE TABLE t1(a); + CREATE INDEX i1 ON t1(a); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<10 + ) + INSERT INTO t1 SELECT randomblob(900) FROM s; +} + +do_execsql_test 2.1 { + SELECT * FROM sqlite_dbptr WHERE pgno=2; +} { + 2 25 2 6 2 7 2 9 2 11 2 13 2 15 2 17 2 19 2 21 +} + +do_execsql_test 2.2 { + SELECT * FROM sqlite_dbptr WHERE pgno=3; +} { + 3 24 3 23 +} + +do_execsql_test 2.3 { + SELECT * FROM sqlite_dbptr +} { + 2 25 2 6 2 7 2 9 2 11 2 13 2 15 2 17 2 19 2 21 + 3 24 3 23 +} + + +finish_test diff --git a/test/fts4rename.test b/test/fts4rename.test new file mode 100644 index 0000000000..1c711e74e6 --- /dev/null +++ b/test/fts4rename.test @@ -0,0 +1,44 @@ +# 2019 April 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/fts3_common.tcl +set ::testprefix fts4rename + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE temp.t1 USING fts3(a); + BEGIN; + CREATE TABLE t2(x); +} {} + +do_catchsql_test 1.1 { + ALTER TABLE t1_content RENAME c0a TO docid; +} {1 {duplicate column name: docid}} + +do_catchsql_test 1.2 { + UPDATE t1 SET Col0 = 1 ; +} {1 {no such column: Col0}} + +do_catchsql_test 1.3 { + ROLLBACK; + DROP TABLE t1; +} {0 {}} + +finish_test + diff --git a/test/index.test b/test/index.test index ae16470570..4602041cd8 100644 --- a/test/index.test +++ b/test/index.test @@ -738,6 +738,24 @@ do_test index-21.2 { } } {0 {9 5 1}} - +# 2019-05-01 ticket https://www.sqlite.org/src/info/3be1295b264be2fa +do_execsql_test index-22.0 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a, b TEXT); + CREATE UNIQUE INDEX IF NOT EXISTS x1 ON t1(b==0); + CREATE INDEX IF NOT EXISTS x2 ON t1(a || 0) WHERE b; + INSERT INTO t1(a,b) VALUES('a',1),('a',0); + SELECT a, b, '|' FROM t1; +} {a 1 | a 0 |} + +# 2019-05-10 ticket https://www.sqlite.org/src/info/ae0f637bddc5290b +do_execsql_test index-23.0 { + DROP TABLE t1; + CREATE TABLE t1(a TEXT, b REAL); + CREATE UNIQUE INDEX t1x1 ON t1(a GLOB b); + INSERT INTO t1(a,b) VALUES('0.0','1'),('1.0','1'); + SELECT * FROM t1; + REINDEX; +} {0.0 1.0 1.0 1.0} finish_test diff --git a/test/like3.test b/test/like3.test index 622f8335eb..f9120e7384 100644 --- a/test/like3.test +++ b/test/like3.test @@ -178,10 +178,21 @@ do_eqp_test like3-5.211 { `--SEARCH TABLE t5b USING COVERING INDEX sqlite_autoindex_t5b_1 (x>? AND x? AND path co]} { @@ -208,6 +197,10 @@ proc count_tests_and_errors {name logfile} { } } +# This command is invoked once a slave process has finished running its +# tests, successfully or otherwise. Parameter $name is the name of the +# test, $rc the exit code returned by the slave process. +# proc slave_test_done {name rc} { global G set G(test.$name.done) [clock seconds] @@ -220,8 +213,31 @@ proc slave_test_done {name rc} { if {[file exists $G(test.$name.log)]} { count_tests_and_errors $name $G(test.$name.log) } + + # If the "keep files" checkbox is clear, delete all files except for + # the executables and test logs. And any core file that is present. + if {$G(keep)==0} { + set keeplist { + testfixture testfixture.exe + sqlite3 sqlite3.exe + test.log test-out.txt + core + wapptest_make.sh + wapptest_configure.sh + wapptest_run.tcl + } + foreach f [glob -nocomplain [file join $G(test.$name.dir) *]] { + set t [file tail $f] + if {[lsearch $keeplist $t]<0} { + catch { file delete -force $f } + } + } + } } +# This is a fileevent callback invoked each time a file-descriptor that +# connects this process to a slave process is readable. +# proc slave_fileevent {name} { global G set fd $G(test.$name.channel) @@ -239,6 +255,99 @@ proc slave_fileevent {name} { do_some_stuff } +# Return the contents of the "slave script" - the script run by slave +# processes to actually perform the test. It does two things: +# +# 1. Reads and [exec]s the contents of file wapptest_configure.sh. +# 2. Reads and [exec]s the contents of file wapptest_make.sh. +# +# Step 1 is omitted if the test uses MSVC (which does not use configure). +# +proc wapptest_slave_script {} { + global G + set res { + proc readfile {filename} { + set fd [open $filename] + set data [read $fd] + close $fd + return $data + } + } + + if {$G(msvc)==0} { + append res { + set cfg [readfile wapptest_configure.sh] + set rc [catch { exec {*}$cfg >& test.log } msg] + if {$rc==0} { + set make [readfile wapptest_make.sh] + set rc [catch { exec {*}$make >>& test.log }] + } + } + } else { + append res { + set make [readfile wapptest_make.sh] + set rc [catch { exec {*}$make >>& test.log }] + } + } + + append res { exit $rc } + + set res +} + + +# Launch a slave process to run a test. +# +proc slave_launch { + name wtcl title dir configOpts testtarget makeOpts cflags opts +} { + global G + + catch { file mkdir $dir } msg + foreach f [glob -nocomplain [file join $dir *]] { + catch { file delete -force $f } + } + set G(test.$name.dir) $dir + + # Write the configure command to wapptest_configure.sh. This file + # is empty if using MSVC - MSVC does not use configure. + # + set fd1 [open [file join $dir wapptest_configure.sh] w] + if {$G(msvc)==0} { + puts $fd1 "[file join .. $G(srcdir) configure] $wtcl $configOpts" + } + close $fd1 + + # Write the make command to wapptest_make.sh. Using nmake for MSVC and + # make for all other systems. + # + set makecmd "make" + if {$G(msvc)} { + set nativedir [file nativename $G(srcdir)] + set nativedir [string map [list "\\" "\\\\"] $nativedir] + set makecmd "nmake /f [file join $nativedir Makefile.msc] TOP=$nativedir" + } + set fd2 [open [file join $dir wapptest_make.sh] w] + puts $fd2 "$makecmd $makeOpts $testtarget \"CFLAGS=$cflags\" \"OPTS=$opts\"" + close $fd2 + + # Write the wapptest_run.tcl script to the test directory. To run the + # commands in the other two files. + # + set fd3 [open [file join $dir wapptest_run.tcl] w] + puts $fd3 [wapptest_slave_script] + close $fd3 + + set pwd [pwd] + cd $dir + set fd [open "|[info nameofexecutable] wapptest_run.tcl" r+] + cd $pwd + + set G(test.$name.channel) $fd + fconfigure $fd -blocking 0 + fileevent $fd readable [list slave_fileevent $name] +} + proc do_some_stuff {} { global G @@ -275,15 +384,9 @@ proc do_some_stuff {} { if { ![info exists G(test.$name.channel)] && ![info exists G(test.$name.done)] } { + set target [dict get $j target] set G(test.$name.start) [clock seconds] - set fd [open "|[info nameofexecutable] $G(releaseTest) --slave" r+] - set G(test.$name.channel) $fd - fconfigure $fd -blocking 0 - fileevent $fd readable [list slave_fileevent $name] - - puts $fd [list 0 $G(msvc) 0 $G(keep)] - set wtcl "" if {$G(tcl)!=""} { set wtcl "--with-tcl=$G(tcl)" } @@ -303,8 +406,9 @@ proc do_some_stuff {} { } set L [make_test_suite $G(msvc) $wtcl $name $target $opts] - puts $fd $L - flush $fd + set G(test.$name.log) [file join [lindex $L 1] test.log] + slave_launch $name $wtcl {*}$L + set G(test.$name.log) [file join [lindex $L 1] test.log] incr nLaunch -1 } diff --git a/test/without_rowid1.test b/test/without_rowid1.test index c8f018737b..4dd967234c 100644 --- a/test/without_rowid1.test +++ b/test/without_rowid1.test @@ -391,5 +391,13 @@ do_execsql_test 10.6 { SELECT * FROM t1; } {b a 3 b b 4} +# 2019-04-29 ticket https://www.sqlite.org/src/info/3182d3879020ef3 +do_execsql_test 11.1 { + CREATE TABLE t11(a TEXT PRIMARY KEY, b INT) WITHOUT ROWID; + CREATE INDEX t11a ON t11(a COLLATE NOCASE); + INSERT INTO t11(a,b) VALUES ('A',1),('a',2); + PRAGMA integrity_check; + SELECT a FROM t11 ORDER BY a COLLATE binary; +} {ok A a} finish_test diff --git a/tool/lemon.c b/tool/lemon.c index 7ef99fd525..eb4adbbc69 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -3848,7 +3848,7 @@ PRIVATE int translate_code(struct lemon *lemp, struct rule *rp){ ErrorMsg(lemp->filename,rp->ruleline, "%s(%s) has the same label as the LHS but is not the left-most " "symbol on the RHS.", - rp->rhs[i]->name, rp->rhsalias); + rp->rhs[i]->name, rp->rhsalias[i]); lemp->errorcnt++; } for(j=0; j