From 17fb042578f7c2333be771ac6a5f1a4e103283df Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 23 Jun 2011 17:09:51 +0000 Subject: [PATCH 01/52] Fix some of the code issues (missing comments etc.) in the new FTS code. FossilOrigin-Name: 8230d83120e0f4d217fde56e22c6f05aa5adee09 --- ext/fts3/fts3.c | 1021 +++++++++++++++++++++++++++++--------------- ext/fts3/fts3Int.h | 3 - manifest | 17 +- manifest.uuid | 2 +- 4 files changed, 693 insertions(+), 350 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 67a0ea2096..40a0297add 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -312,6 +312,11 @@ SQLITE_EXTENSION_INIT1 #endif +static int fts3EvalNext(Fts3Cursor *pCsr); +static int fts3EvalStart(Fts3Cursor *pCsr); +static int fts3TermSegReaderCursor( + Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **); + /* ** Write a 64-bit variable-length integer to memory starting at p[0]. ** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. @@ -820,9 +825,23 @@ static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ return zRet; } +/* +** This function interprets the string at (*pp) as a non-negative integer +** value. It reads the integer and sets *pnOut to the value read, then +** sets *pp to point to the byte immediately following the last byte of +** the integer value. +** +** Only decimal digits ('0'..'9') may be part of an integer value. +** +** If *pp does not being with a decimal digit SQLITE_ERROR is returned and +** the output value undefined. Otherwise SQLITE_OK is returned. +** +** This function is used when parsing the "prefix=" FTS4 parameter. +*/ static int fts3GobbleInt(const char **pp, int *pnOut){ - const char *p = *pp; - int nInt = 0; + const char *p = *pp; /* Iterator pointer */ + int nInt = 0; /* Output value */ + for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ nInt = nInt * 10 + (p[0] - '0'); } @@ -832,15 +851,30 @@ static int fts3GobbleInt(const char **pp, int *pnOut){ return SQLITE_OK; } - +/* +** This function is called to allocate an array of Fts3Index structures +** representing the indexes maintained by the current FTS table. FTS tables +** always maintain the main "terms" index, but may also maintain one or +** more "prefix" indexes, depending on the value of the "prefix=" parameter +** (if any) specified as part of the CREATE VIRTUAL TABLE statement. +** +** Argument zParam is passed the value of the "prefix=" option if one was +** specified, or NULL otherwise. +** +** If no error occurs, SQLITE_OK is returned and *apIndex set to point to +** the allocated array. *pnIndex is set to the number of elements in the +** array. If an error does occur, an SQLite error code is returned. +** +** Regardless of whether or not an error is returned, it is the responsibility +** of the caller to call sqlite3_free() on the output array to free it. +*/ static int fts3PrefixParameter( const char *zParam, /* ABC in prefix=ABC parameter to parse */ int *pnIndex, /* OUT: size of *apIndex[] array */ - struct Fts3Index **apIndex, /* OUT: Array of indexes for this table */ - struct Fts3Index **apFree /* OUT: Free this with sqlite3_free() */ + struct Fts3Index **apIndex /* OUT: Array of indexes for this table */ ){ - struct Fts3Index *aIndex; - int nIndex = 1; + struct Fts3Index *aIndex; /* Allocated array */ + int nIndex = 1; /* Number of entries in array */ if( zParam && zParam[0] ){ const char *p; @@ -851,7 +885,7 @@ static int fts3PrefixParameter( } aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); - *apIndex = *apFree = aIndex; + *apIndex = aIndex; *pnIndex = nIndex; if( !aIndex ){ return SQLITE_NOMEM; @@ -908,8 +942,7 @@ static int fts3InitVtab( sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ int nIndex; /* Size of aIndex[] array */ - struct Fts3Index *aIndex; /* Array of indexes for this table */ - struct Fts3Index *aFree = 0; /* Free this before returning */ + struct Fts3Index *aIndex = 0; /* Array of indexes for this table */ /* The results of parsing supported FTS4 key=value options: */ int bNoDocsize = 0; /* True to omit %_docsize table */ @@ -1046,7 +1079,7 @@ static int fts3InitVtab( } assert( pTokenizer ); - rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex, &aFree); + rc = fts3PrefixParameter(zPrefix, &nIndex, &aIndex); if( rc==SQLITE_ERROR ){ assert( zPrefix ); *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix); @@ -1133,7 +1166,7 @@ static int fts3InitVtab( fts3_init_out: sqlite3_free(zPrefix); - sqlite3_free(aFree); + sqlite3_free(aIndex); sqlite3_free(zCompress); sqlite3_free(zUncompress); sqlite3_free((void *)aCol); @@ -1724,8 +1757,6 @@ static void fts3PoslistMerge( } /* -** nToken==1 searches for adjacent positions. -** ** This function is used to merge two position lists into one. When it is ** called, *pp1 and *pp2 must both point to position lists. A position-list is ** the part of a doclist that follows each document id. For example, if a row @@ -1745,6 +1776,8 @@ static void fts3PoslistMerge( ** *pp1 so that (pos(*pp2)>pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. ** when the *pp1 token appears before the *pp2 token, but not more than nToken ** slots before it. +** +** e.g. nToken==1 searches for adjacent positions. */ static int fts3PoslistPhraseMerge( char **pp, /* IN/OUT: Preallocated output buffer */ @@ -1911,22 +1944,34 @@ static int fts3PoslistNearMerge( } /* -** A pointer to an instance of this structure is used as the context -** argument to sqlite3Fts3SegReaderIterate() +** An instance of this function is used to merge together the (potentially +** large number of) doclists for each term that matches a prefix query. +** See function fts3TermSelectMerge() for details. */ typedef struct TermSelect TermSelect; struct TermSelect { - int isReqPos; - char *aaOutput[16]; /* Malloc'd output buffer */ - int anOutput[16]; /* Size of output in bytes */ + char *aaOutput[16]; /* Malloc'd output buffers */ + int anOutput[16]; /* Size each output buffer in bytes */ }; - +/* +** This function is used to read a single varint from a buffer. Parameter +** pEnd points 1 byte past the end of the buffer. When this function is +** called, if *pp points to pEnd or greater, then the end of the buffer +** has been reached. In this case *pp is set to 0 and the function returns. +** +** If *pp does not point to or past pEnd, then a single varint is read +** from *pp. *pp is then set to point 1 byte past the end of the read varint. +** +** If bDescIdx is false, the value read is added to *pVal before returning. +** If it is true, the value read is subtracted from *pVal before this +** function returns. +*/ static void fts3GetDeltaVarint3( - char **pp, - char *pEnd, - int bDescIdx, - sqlite3_int64 *pVal + char **pp, /* IN/OUT: Point to read varint from */ + char *pEnd, /* End of buffer */ + int bDescIdx, /* True if docids are descending */ + sqlite3_int64 *pVal /* IN/OUT: Integer value */ ){ if( *pp>=pEnd ){ *pp = 0; @@ -1941,6 +1986,21 @@ static void fts3GetDeltaVarint3( } } +/* +** This function is used to write a single varint to a buffer. The varint +** is written to *pp. Before returning, *pp is set to point 1 byte past the +** end of the value written. +** +** If *pbFirst is zero when this function is called, the value written to +** the buffer is that of parameter iVal. +** +** If *pbFirst is non-zero when this function is called, then the value +** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal) +** (if bDescIdx is non-zero). +** +** Before returning, this function always sets *pbFirst to 1 and *piPrev +** to the value of parameter iVal. +*/ static void fts3PutDeltaVarint3( char **pp, /* IN/OUT: Output pointer */ int bDescIdx, /* True for descending docids */ @@ -1961,10 +2021,34 @@ static void fts3PutDeltaVarint3( *pbFirst = 1; } -#define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2)) +/* +** This macro is used by various functions that merge doclists. The two +** arguments are 64-bit docid values. If the value of the stack variable +** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). +** Otherwise, (i2-i1). +** +** Using this makes it easier to write code that can merge doclists that are +** sorted in either ascending or descending order. +*/ +#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2)) + +/* +** This function does an "OR" merge of two doclists (output contains all +** positions contained in either argument doclist). If the docids in the +** input doclists are sorted in ascending order, parameter bDescDoclist +** should be false. If they are sorted in ascending order, it should be +** passed a non-zero value. +** +** If no error occurs, *paOut is set to point at an sqlite3_malloc'd buffer +** containing the output doclist and SQLITE_OK is returned. In this case +** *pnOut is set to the number of bytes in the output doclist. +** +** If an error occurs, an SQLite error code is returned. The output values +** are undefined in this case. +*/ static int fts3DoclistOrMerge( - int bDescIdx, /* True if arguments are desc */ + int bDescDoclist, /* True if arguments are desc */ char *a1, int n1, /* First doclist */ char *a2, int n2, /* Second doclist */ char **paOut, int *pnOut /* OUT: Malloc'd doclist */ @@ -1989,21 +2073,21 @@ static int fts3DoclistOrMerge( fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); while( p1 || p2 ){ - sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); if( p2 && p1 && iDiff==0 ){ - fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); fts3PoslistMerge(&p, &p1, &p2); - fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); }else if( !p2 || (p1 && iDiff<0) ){ - fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); fts3PoslistCopy(&p, &p1); - fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); }else{ - fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2); + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i2); fts3PoslistCopy(&p, &p2); - fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); } } @@ -2012,8 +2096,20 @@ static int fts3DoclistOrMerge( return SQLITE_OK; } +/* +** This function does a "phrase" merge of two doclists. In a phrase merge, +** the output contains a copy of each position from the right-hand input +** doclist for which there is a position in the left-hand input doclist +** exactly nDist tokens before it. +** +** If the docids in the input doclists are sorted in ascending order, +** parameter bDescDoclist should be false. If they are sorted in ascending +** order, it should be passed a non-zero value. +** +** The right-hand input doclist is overwritten by this function. +*/ static void fts3DoclistPhraseMerge( - int bDescIdx, /* True if arguments are desc */ + int bDescDoclist, /* True if arguments are desc */ int nDist, /* Distance from left to right (1=adjacent) */ char *aLeft, int nLeft, /* Left doclist */ char *aRight, int *pnRight /* IN/OUT: Right/output doclist */ @@ -2036,26 +2132,26 @@ static void fts3DoclistPhraseMerge( fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); while( p1 && p2 ){ - sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + sqlite3_int64 iDiff = DOCID_CMP(i1, i2); if( iDiff==0 ){ char *pSave = p; sqlite3_int64 iPrevSave = iPrev; int bFirstOutSave = bFirstOut; - fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ p = pSave; iPrev = iPrevSave; bFirstOut = bFirstOutSave; } - fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); }else if( iDiff<0 ){ fts3PoslistCopy(0, &p1); - fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); }else{ fts3PoslistCopy(0, &p2); - fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); } } @@ -2072,7 +2168,7 @@ static void fts3DoclistPhraseMerge( ** the responsibility of the caller to free any doclists left in the ** TermSelect.aaOutput[] array. */ -static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){ +static int fts3TermSelectFinishMerge(Fts3Table *p, TermSelect *pTS){ char *aOut = 0; int nOut = 0; int i; @@ -2113,24 +2209,25 @@ static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){ } /* -** This function is used as the sqlite3Fts3SegReaderIterate() callback when -** querying the full-text index for a doclist associated with a term or -** term-prefix. +** Merge the doclist aDoclist/nDoclist into the TermSelect object passed +** as the first argument. The merge is an "OR" merge (see function +** fts3DoclistOrMerge() for details). +** +** This function is called with the doclist for each term that matches +** a queried prefix. It merges all these doclists into one, the doclist +** for the specified prefix. Since there can be a very large number of +** doclists to merge, the merging is done pair-wise using the TermSelect +** object. +** +** This function returns SQLITE_OK if the merge is successful, or an +** SQLite error code (SQLITE_NOMEM) if an error occurs. */ -static int fts3TermSelectCb( - Fts3Table *p, /* Virtual table object */ - void *pContext, /* Pointer to TermSelect structure */ - char *zTerm, - int nTerm, - char *aDoclist, - int nDoclist +static int fts3TermSelectMerge( + Fts3Table *p, /* FTS table handle */ + TermSelect *pTS, /* TermSelect object to merge into */ + char *aDoclist, /* Pointer to doclist */ + int nDoclist /* Size of aDoclist in bytes */ ){ - TermSelect *pTS = (TermSelect *)pContext; - - UNUSED_PARAMETER(p); - UNUSED_PARAMETER(zTerm); - UNUSED_PARAMETER(nTerm); - if( pTS->aaOutput[0]==0 ){ /* If this is the first term selected, copy the doclist to the output ** buffer using memcpy(). */ @@ -2201,6 +2298,13 @@ static int fts3SegReaderCursorAppend( return SQLITE_OK; } +/* +** Add seg-reader objects to the Fts3MultiSegReader object passed as the +** 8th argument. +** +** This function returns SQLITE_OK if successful, or an SQLite error code +** otherwise. +*/ static int fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ int iIndex, /* Index to search (from 0 to p->nIndex-1) */ @@ -2209,11 +2313,11 @@ static int fts3SegReaderCursor( int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ int isScan, /* True to scan from zTerm to EOF */ - Fts3MultiSegReader *pCsr /* Cursor object to populate */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ ){ - int rc = SQLITE_OK; - int rc2; - sqlite3_stmt *pStmt = 0; + int rc = SQLITE_OK; /* Error code */ + sqlite3_stmt *pStmt = 0; /* Statement to iterate through segments */ + int rc2; /* Result of sqlite3_reset() */ /* If iLevel is less than 0 and this is not a scan, include a seg-reader ** for the pending-terms. If this is a scan, then this call must be being @@ -2302,24 +2406,42 @@ int sqlite3Fts3SegReaderCursor( ); } +/* +** In addition to its current configuration, have the Fts3MultiSegReader +** passed as the 4th argument also scan the doclist for term zTerm/nTerm. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ static int fts3SegReaderCursorAddZero( - Fts3Table *p, - const char *zTerm, - int nTerm, - Fts3MultiSegReader *pCsr + Fts3Table *p, /* FTS virtual table handle */ + const char *zTerm, /* Term to scan doclist of */ + int nTerm, /* Number of bytes in zTerm */ + Fts3MultiSegReader *pCsr /* Fts3MultiSegReader to modify */ ){ return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr); } - -int sqlite3Fts3TermSegReaderCursor( +/* +** Open an Fts3MultiSegReader to scan the doclist for term zTerm/nTerm. Or, +** if isPrefix is true, to scan the doclist for all terms for which +** zTerm/nTerm is a prefix. If successful, return SQLITE_OK and write +** a pointer to the new Fts3MultiSegReader to *ppSegcsr. Otherwise, return +** an SQLite error code. +** +** It is the responsibility of the caller to free this object by eventually +** passing it to fts3SegReaderCursorFree() +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +** Output parameter *ppSegcsr is set to 0 if an error occurs. +*/ +static int fts3TermSegReaderCursor( Fts3Cursor *pCsr, /* Virtual table cursor handle */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ ){ - Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */ + Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */ int rc = SQLITE_NOMEM; /* Return code */ pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader)); @@ -2363,6 +2485,9 @@ int sqlite3Fts3TermSegReaderCursor( return rc; } +/* +** Free an Fts3MultiSegReader allocated by fts3TermSegReaderCursor(). +*/ static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ sqlite3Fts3SegReaderFinish(pSegcsr); sqlite3_free(pSegcsr); @@ -2370,35 +2495,25 @@ static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ /* ** This function retreives the doclist for the specified term (or term -** prefix) from the database. -** -** The returned doclist may be in one of two formats, depending on the -** value of parameter isReqPos. If isReqPos is zero, then the doclist is -** a sorted list of delta-compressed docids (a bare doclist). If isReqPos -** is non-zero, then the returned list is in the same format as is stored -** in the database without the found length specifier at the start of on-disk -** doclists. +** prefix) from the database. */ static int fts3TermSelect( Fts3Table *p, /* Virtual table handle */ Fts3PhraseToken *pTok, /* Token to query for */ int iColumn, /* Column to query (or -ve for all columns) */ - int isReqPos, /* True to include position lists in output */ int *pnOut, /* OUT: Size of buffer at *ppOut */ char **ppOut /* OUT: Malloced result buffer */ ){ int rc; /* Return code */ - Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ - TermSelect tsc; /* Context object for fts3TermSelectCb() */ + Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ + TermSelect tsc; /* Object for pair-wise doclist merging */ Fts3SegFilter filter; /* Segment term filter configuration */ pSegcsr = pTok->pSegcsr; memset(&tsc, 0, sizeof(TermSelect)); - tsc.isReqPos = isReqPos; - filter.flags = FTS3_SEGMENT_IGNORE_EMPTY + filter.flags = FTS3_SEGMENT_IGNORE_EMPTY | FTS3_SEGMENT_REQUIRE_POS | (pTok->isPrefix ? FTS3_SEGMENT_PREFIX : 0) - | (isReqPos ? FTS3_SEGMENT_REQUIRE_POS : 0) | (iColumnnColumn ? FTS3_SEGMENT_COLUMN_FILTER : 0); filter.iCol = iColumn; filter.zTerm = pTok->z; @@ -2408,13 +2523,11 @@ static int fts3TermSelect( while( SQLITE_OK==rc && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) ){ - rc = fts3TermSelectCb(p, (void *)&tsc, - pSegcsr->zTerm, pSegcsr->nTerm, pSegcsr->aDoclist, pSegcsr->nDoclist - ); + rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist); } if( rc==SQLITE_OK ){ - rc = fts3TermSelectMerge(p, &tsc); + rc = fts3TermSelectFinishMerge(p, &tsc); } if( rc==SQLITE_OK ){ *ppOut = tsc.aaOutput[0]; @@ -2487,7 +2600,7 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ rc = SQLITE_OK; } }else{ - rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor); + rc = fts3EvalNext((Fts3Cursor *)pCursor); } assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); return rc; @@ -2564,7 +2677,7 @@ static int fts3FilterMethod( rc = sqlite3Fts3ReadLock(p); if( rc!=SQLITE_OK ) return rc; - rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1); + rc = fts3EvalStart(pCsr); sqlite3Fts3SegmentsClose(p); if( rc!=SQLITE_OK ) return rc; @@ -2971,6 +3084,11 @@ static int fts3RenameMethod( return rc; } +/* +** The xSavepoint() method. +** +** Flush the contents of the pending-terms table to disk. +*/ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ UNUSED_PARAMETER(iSavepoint); assert( ((Fts3Table *)pVtab)->inTransaction ); @@ -2978,6 +3096,12 @@ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); return fts3SyncMethod(pVtab); } + +/* +** The xRelease() method. +** +** This is a no-op. +*/ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); UNUSED_PARAMETER(iSavepoint); @@ -2987,6 +3111,12 @@ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ TESTONLY( p->mxSavepoint = iSavepoint-1 ); return SQLITE_OK; } + +/* +** The xRollbackTo() method. +** +** Discard the contents of the pending terms table. +*/ static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ Fts3Table *p = (Fts3Table*)pVtab; UNUSED_PARAMETER(iSavepoint); @@ -3136,18 +3266,6 @@ int sqlite3Fts3Init(sqlite3 *db){ return rc; } -#if !SQLITE_CORE -int sqlite3_extension_init( - sqlite3 *db, - char **pzErrMsg, - const sqlite3_api_routines *pApi -){ - SQLITE_EXTENSION_INIT2(pApi) - return sqlite3Fts3Init(db); -} -#endif - - /* ** Allocate an Fts3MultiSegReader for each token in the expression headed ** by pExpr. @@ -3164,11 +3282,11 @@ int sqlite3_extension_init( ** doclist and then traversed. */ static void fts3EvalAllocateReaders( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Allocate readers for this expression */ int *pnToken, /* OUT: Total number of tokens in phrase. */ int *pnOr, /* OUT: Total number of OR nodes in expr. */ - int *pRc + int *pRc /* IN/OUT: Error code */ ){ if( pExpr && SQLITE_OK==*pRc ){ if( pExpr->eType==FTSQUERY_PHRASE ){ @@ -3177,7 +3295,7 @@ static void fts3EvalAllocateReaders( *pnToken += nToken; for(i=0; ipPhrase->aToken[i]; - int rc = sqlite3Fts3TermSegReaderCursor(pCsr, + int rc = fts3TermSegReaderCursor(pCsr, pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr ); if( rc!=SQLITE_OK ){ @@ -3195,12 +3313,20 @@ static void fts3EvalAllocateReaders( } } +/* +** Arguments pList/nList contain the doclist for token iToken of phrase p. +** It is merged into the main doclist stored in p->doclist.aAll/nAll. +** +** This function assumes that pList points to a buffer allocated using +** sqlite3_malloc(). This function takes responsibility for eventually +** freeing the buffer. +*/ static void fts3EvalPhraseMergeToken( - Fts3Table *pTab, - Fts3Phrase *p, - int iToken, - char *pList, - int nList + Fts3Table *pTab, /* FTS Table pointer */ + Fts3Phrase *p, /* Phrase to merge pList/nList into */ + int iToken, /* Token pList/nList corresponds to */ + char *pList, /* Pointer to doclist */ + int nList /* Number of bytes in pList */ ){ assert( iToken!=p->iDoclistToken ); @@ -3249,9 +3375,15 @@ static void fts3EvalPhraseMergeToken( if( iToken>p->iDoclistToken ) p->iDoclistToken = iToken; } +/* +** Load the doclist for phrase p into p->doclist.aAll/nAll. The loaded doclist +** does not take deferred tokens into account. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ static int fts3EvalPhraseLoad( - Fts3Cursor *pCsr, - Fts3Phrase *p + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p /* Phrase object */ ){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int iToken; @@ -3264,7 +3396,7 @@ static int fts3EvalPhraseLoad( if( pToken->pSegcsr ){ int nThis = 0; char *pThis = 0; - rc = fts3TermSelect(pTab, pToken, p->iColumn, 1, &nThis, &pThis); + rc = fts3TermSelect(pTab, pToken, p->iColumn, &nThis, &pThis); if( rc==SQLITE_OK ){ fts3EvalPhraseMergeToken(pTab, p, iToken, pThis, nThis); } @@ -3275,14 +3407,22 @@ static int fts3EvalPhraseLoad( return rc; } +/* +** This function is called on each phrase after the position lists for +** any deferred tokens have been loaded into memory. It updates the phrases +** current position list to include only those positions that are really +** instances of the phrase (after considering deferred tokens). If this +** means that the phrase does not appear in the current row, doclist.pList +** and doclist.nList are both zeroed. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. +*/ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ - int iToken; - int rc = SQLITE_OK; - - int nMaxUndeferred = pPhrase->iDoclistToken; - char *aPoslist = 0; - int nPoslist = 0; - int iPrev = -1; + int iToken; /* Used to iterate through phrase tokens */ + int rc = SQLITE_OK; /* Return code */ + char *aPoslist = 0; /* Position list for deferred tokens */ + int nPoslist = 0; /* Number of bytes in aPoslist */ + int iPrev = -1; /* Token number of previous deferred token */ assert( pPhrase->doclist.bFreeList==0 ); @@ -3328,6 +3468,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ } if( iPrev>=0 ){ + int nMaxUndeferred = pPhrase->iDoclistToken; if( nMaxUndeferred<0 ){ pPhrase->doclist.pList = aPoslist; pPhrase->doclist.nList = nPoslist; @@ -3376,9 +3517,15 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ ** expression to initialize the mechanism for returning rows. Once this ** function has been called successfully on an Fts3Phrase, it may be ** used with fts3EvalPhraseNext() to iterate through the matching docids. +** +** If parameter bOptOk is true, then the phrase may (or may not) use the +** incremental loading strategy. Otherwise, the entire doclist is loaded into +** memory within this call. +** +** SQLITE_OK is returned if no error occurs, otherwise an SQLite error code. */ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ - int rc; + int rc; /* Error code */ Fts3PhraseToken *pFirst = &p->aToken[0]; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; @@ -3406,7 +3553,13 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ /* ** This function is used to iterate backwards (from the end to start) -** through doclists. +** through doclists. It is used by this module to iterate through phrase +** doclists in reverse and by the fts3_write.c module to iterate through +** pending-terms lists when writing to databases with "order=desc". +** +** The doclist may be sorted in ascending (parameter bDescIdx==0) or +** descending (parameter bDescIdx==1) order of docid. Regardless, this +** function iterates from the end of the doclist to the beginning. */ void sqlite3Fts3DoclistPrev( int bDescIdx, /* True if the doclist is desc */ @@ -3471,9 +3624,9 @@ void sqlite3Fts3DoclistPrev( ** successfully advanced, *pbEof is set to 0. */ static int fts3EvalPhraseNext( - Fts3Cursor *pCsr, - Fts3Phrase *p, - u8 *pbEof + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Phrase *p, /* Phrase object to advance to next docid */ + u8 *pbEof /* OUT: Set to 1 if EOF */ ){ int rc = SQLITE_OK; Fts3Doclist *pDL = &p->doclist; @@ -3519,10 +3672,10 @@ static int fts3EvalPhraseNext( /* pIter now points just past the 0x00 that terminates the position- ** list for document pDL->iDocid. However, if this position-list was - ** edited in place by fts3EvalNearTrim2(), then pIter may not actually + ** edited in place by fts3EvalNearTrim(), then pIter may not actually ** point to the start of the next docid value. The following line deals ** with this case by advancing pIter past the zero-padding added by - ** fts3EvalNearTrim2(). */ + ** fts3EvalNearTrim(). */ while( pIterpNextDocid = pIter; @@ -3534,11 +3687,27 @@ static int fts3EvalPhraseNext( return rc; } +/* +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, fts3EvalPhraseStart() is called on all phrases within the +** expression. Also the Fts3Expr.bDeferred variable is set to true for any +** expressions for which all descendent tokens are deferred. +** +** If parameter bOptOk is zero, then it is guaranteed that the +** Fts3Phrase.doclist.aAll/nAll variables contain the entire doclist for +** each phrase in the expression (subject to deferred token processing). +** Or, if bOptOk is non-zero, then one or more tokens within the expression +** may be loaded incrementally, meaning doclist.aAll/nAll is not available. +** +** If an error occurs within this function, *pRc is set to an SQLite error +** code before returning. +*/ static void fts3EvalStartReaders( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, - int bOptOk, - int *pRc + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expression to initialize phrases in */ + int bOptOk, /* True to enable incremental loading */ + int *pRc /* IN/OUT: Error code */ ){ if( pExpr && SQLITE_OK==*pRc ){ if( pExpr->eType==FTSQUERY_PHRASE ){ @@ -3557,23 +3726,42 @@ static void fts3EvalStartReaders( } } +/* +** An array of the following structures is assembled as part of the process +** of selecting tokens to defer before the query starts executing (as part +** of the xFilter() method). There is one element in the array for each +** token in the FTS expression. +** +** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong +** to phrases that are connected only by AND and NEAR operators (not OR or +** NOT). When determining tokens to defer, each AND/NEAR cluster is considered +** separately. The root of a tokens AND/NEAR cluster is stored in +** Fts3TokenAndCost.pRoot. +*/ typedef struct Fts3TokenAndCost Fts3TokenAndCost; struct Fts3TokenAndCost { Fts3Phrase *pPhrase; /* The phrase the token belongs to */ int iToken; /* Position of token in phrase */ Fts3PhraseToken *pToken; /* The token itself */ - Fts3Expr *pRoot; - int nOvfl; + Fts3Expr *pRoot; /* Root of NEAR/AND cluster */ + int nOvfl; /* Number of overflow pages to load doclist */ int iCol; /* The column the token must match */ }; +/* +** This function is used to populate an allocated Fts3TokenAndCost array. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** Otherwise, if an error occurs during execution, *pRc is set to an +** SQLite error code. +*/ static void fts3EvalTokenCosts( - Fts3Cursor *pCsr, - Fts3Expr *pRoot, - Fts3Expr *pExpr, - Fts3TokenAndCost **ppTC, - Fts3Expr ***ppOr, - int *pRc + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Root of current AND/NEAR cluster */ + Fts3Expr *pExpr, /* Expression to consider */ + Fts3TokenAndCost **ppTC, /* Write new entries to *(*ppTC)++ */ + Fts3Expr ***ppOr, /* Write new OR root to *(*ppOr)++ */ + int *pRc /* IN/OUT: Error code */ ){ if( *pRc==SQLITE_OK && pExpr ){ if( pExpr->eType==FTSQUERY_PHRASE ){ @@ -3605,19 +3793,30 @@ static void fts3EvalTokenCosts( } } +/* +** Determine the average document (row) size in pages. If successful, +** write this value to *pnPage and return SQLITE_OK. Otherwise, return +** an SQLite error code. +** +** The average document size in pages is calculated by first calculating +** determining the average size in bytes, B. If B is less than the amount +** of data that will fit on a single leaf page of an intkey table in +** this database, then the average docsize is 1. Otherwise, it is 1 plus +** the number of overflow pages consumed by a record B bytes in size. +*/ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ if( pCsr->nRowAvg==0 ){ /* The average document size, which is required to calculate the cost - ** of each doclist, has not yet been determined. Read the required - ** data from the %_stat table to calculate it. - ** - ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 - ** varints, where nCol is the number of columns in the FTS3 table. - ** The first varint is the number of documents currently stored in - ** the table. The following nCol varints contain the total amount of - ** data stored in all rows of each column of the table, from left - ** to right. - */ + ** of each doclist, has not yet been determined. Read the required + ** data from the %_stat table to calculate it. + ** + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 + ** varints, where nCol is the number of columns in the FTS3 table. + ** The first varint is the number of documents currently stored in + ** the table. The following nCol varints contain the total amount of + ** data stored in all rows of each column of the table, from left + ** to right. + */ int rc; Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; sqlite3_stmt *pStmt; @@ -3652,45 +3851,69 @@ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ return SQLITE_OK; } +/* +** This function is called to select the tokens (if any) that will be +** deferred. The array aTC[] has already been populated when this is +** called. +** +** This function is called once for each AND/NEAR cluster in the +** expression. Each invocation determines which tokens to defer within +** the cluster with root node pRoot. See comments above the definition +** of struct Fts3TokenAndCost for more details. +** +** If no error occurs, SQLITE_OK is returned and sqlite3Fts3DeferToken() +** called on each token to defer. Otherwise, an SQLite error code is +** returned. +*/ static int fts3EvalSelectDeferred( - Fts3Cursor *pCsr, - Fts3Expr *pRoot, - Fts3TokenAndCost *aTC, - int nTC + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pRoot, /* Consider tokens with this root node */ + Fts3TokenAndCost *aTC, /* Array of expression tokens and costs */ + int nTC /* Number of entries in aTC[] */ ){ - int nDocSize = 0; - int nDocEst = 0; - int rc = SQLITE_OK; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - int ii; - - int nOvfl = 0; - int nTerm = 0; + int nDocSize = 0; /* Number of pages per doc loaded */ + int nDocEst = 0; /* Est. docs if all other tokens deferred */ + int rc = SQLITE_OK; /* Return code */ + int ii; /* Iterator variable for various purposes */ + int nOvfl = 0; /* Total overflow pages used by doclists */ + int nToken = 0; /* Total number of tokens in cluster */ + /* Count the tokens in this AND/NEAR cluster. If none of the doclists + ** associated with the tokens spill onto overflow pages, or if there is + ** only 1 token, exit early. No tokens to defer in this case. */ for(ii=0; ii0 ); - for(ii=0; iinOvfl) + /* Set pTC to point to the cheapest remaining token. */ + for(iTC=0; iTCnOvfl) ){ - pTC = &aTC[jj]; + pTC = &aTC[iTC]; } } assert( pTC ); - /* At this point pTC points to the cheapest remaining token. */ + /* Determine if token pTC should be deferred. If not, update nDocEst. + ** + ** TODO: If there are performance regressions involving deferred tokens, + ** this (the logic that selects the tokens to be deferred) is probably + ** the bit that needs to change. + */ if( ii==0 ){ if( pTC->nOvfl ){ nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10; @@ -3698,9 +3921,8 @@ static int fts3EvalSelectDeferred( Fts3PhraseToken *pToken = pTC->pToken; int nList = 0; char *pList = 0; - rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList); + rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); assert( rc==SQLITE_OK || pList==0 ); - if( rc==SQLITE_OK ){ nDocEst = fts3DoclistCountDocids(1, pList, nList); fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList); @@ -3721,36 +3943,29 @@ static int fts3EvalSelectDeferred( return rc; } -int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ +/* +** This function is called from within the xFilter method. It initializes +** the full-text query currently stored in pCsr->pExpr. To iterate through +** the results of a query, the caller does: +** +** fts3EvalStart(pCsr); +** while( 1 ){ +** fts3EvalNext(pCsr); +** if( pCsr->bEof ) break; +** ... return row pCsr->iPrevId to the caller ... +** } +*/ +static int fts3EvalStart(Fts3Cursor *pCsr){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int rc = SQLITE_OK; int nToken = 0; int nOr = 0; /* Allocate a MultiSegReader for each token in the expression. */ - fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &nOr, &rc); + fts3EvalAllocateReaders(pCsr, pCsr->pExpr, &nToken, &nOr, &rc); - /* Call fts3EvalPhraseStart() on all phrases in the expression. TODO: - ** This call will eventually also be responsible for determining which - ** tokens are 'deferred' until the document text is loaded into memory. - ** - ** Each token in each phrase is dealt with using one of the following - ** three strategies: - ** - ** 1. Entire doclist loaded into memory as part of the - ** fts3EvalStartReaders() call. - ** - ** 2. Doclist loaded into memory incrementally, as part of each - ** sqlite3Fts3EvalNext() call. - ** - ** 3. Token doclist is never loaded. Instead, documents are loaded into - ** memory and scanned for the token as part of the sqlite3Fts3EvalNext() - ** call. This is known as a "deferred" token. - */ - - /* If bOptOk is true, check if there are any tokens that should be deferred. - */ - if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){ + /* Determine which, if any, tokens in the expression should be deferred. */ + if( rc==SQLITE_OK && nToken>1 && pTab->bHasStat ){ Fts3TokenAndCost *aTC; Fts3Expr **apOr; aTC = (Fts3TokenAndCost *)sqlite3_malloc( @@ -3766,7 +3981,7 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ Fts3TokenAndCost *pTC = aTC; Fts3Expr **ppOr = apOr; - fts3EvalTokenCosts(pCsr, 0, pExpr, &pTC, &ppOr, &rc); + fts3EvalTokenCosts(pCsr, 0, pCsr->pExpr, &pTC, &ppOr, &rc); nToken = pTC-aTC; nOr = ppOr-apOr; @@ -3781,11 +3996,14 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ } } - fts3EvalStartReaders(pCsr, pExpr, bOptOk, &rc); + fts3EvalStartReaders(pCsr, pCsr->pExpr, 1, &rc); return rc; } -static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){ +/* +** Invalidate the current position list for phrase pPhrase. +*/ +static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){ if( pPhrase->doclist.bFreeList ){ sqlite3_free(pPhrase->doclist.pList); } @@ -3794,8 +4012,30 @@ static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){ pPhrase->doclist.bFreeList = 0; } -static int fts3EvalNearTrim2( - int nNear, +/* +** This function is called to edit the position list associated with +** the phrase object passed as the fifth argument according to a NEAR +** condition. For example: +** +** abc NEAR/5 "def ghi" +** +** Parameter nNear is passed the NEAR distance of the expression (5 in +** the example above). When this function is called, *paPoslist points to +** the position list, and *pnToken is the number of phrase tokens in, the +** phrase on the other side of the NEAR operator to pPhrase. For example, +** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to +** the position list associated with phrase "abc". +** +** All positions in the pPhrase position list that are not sufficiently +** close to a position in the *paPoslist position list are removed. If this +** leaves 0 positions, zero is returned. Otherwise, non-zero. +** +** Before returning, *paPoslist is set to point to the position lsit +** associated with pPhrase. And *pnToken is set to the number of tokens in +** pPhrase. +*/ +static int fts3EvalNearTrim( + int nNear, /* NEAR distance. As in "NEAR/nNear". */ char *aTmp, /* Temporary space to use */ char **paPoslist, /* IN/OUT: Position list */ int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ @@ -3827,6 +4067,172 @@ static int fts3EvalNearTrim2( return res; } +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is called. +** Otherwise, it advances the expression passed as the second argument to +** point to the next matching row in the database. Expressions iterate through +** matching rows in docid order. Ascending order if Fts3Cursor.bDesc is zero, +** or descending if it is non-zero. +** +** If an error occurs, *pRc is set to an SQLite error code. Otherwise, if +** successful, the following variables in pExpr are set: +** +** Fts3Expr.bEof (non-zero if EOF - there is no next row) +** Fts3Expr.iDocid (valid if bEof==0. The docid of the next row) +** +** If the expression is of type FTSQUERY_PHRASE, and the expression is not +** at EOF, then the following variables are populated with the position list +** for the phrase for the visited row: +** +** FTs3Expr.pPhrase->doclist.nList (length of pList in bytes) +** FTs3Expr.pPhrase->doclist.pList (pointer to position list) +** +** It says above that this function advances the expression to the next +** matching row. This is usually true, but there are the following exceptions: +** +** 1. Deferred tokens are not taken into account. If a phrase consists +** entirely of deferred tokens, it is assumed to match every row in +** the db. In this case the position-list is not populated at all. +** +** Or, if a phrase contains one or more deferred tokens and one or +** more non-deferred tokens, then the expression is advanced to the +** next possible match, considering only non-deferred tokens. In other +** words, if the phrase is "A B C", and "B" is deferred, the expression +** is advanced to the next row that contains an instance of "A * C", +** where "*" may match any single token. The position list in this case +** is populated as for "A * C" before returning. +** +** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is +** advanced to point to the next row that matches "x AND y". +** +** See fts3EvalTestDeferredAndNear() for details on testing if a row is +** really a match, taking into account deferred tokens and NEAR operators. +*/ +static void fts3EvalNextRow( + Fts3Cursor *pCsr, /* FTS Cursor handle */ + Fts3Expr *pExpr, /* Expr. to advance to next matching row */ + int *pRc /* IN/OUT: Error code */ +){ + if( *pRc==SQLITE_OK ){ + int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */ + assert( pExpr->bEof==0 ); + pExpr->bStart = 1; + + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + assert( !pLeft->bDeferred || !pRight->bDeferred ); + + if( pLeft->bDeferred ){ + /* LHS is entirely deferred. So we assume it matches every row. + ** Advance the RHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pRight, pRc); + pExpr->iDocid = pRight->iDocid; + pExpr->bEof = pRight->bEof; + }else if( pRight->bDeferred ){ + /* RHS is entirely deferred. So we assume it matches every row. + ** Advance the LHS iterator to find the next row visited. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + }else{ + /* Neither the RHS or LHS are deferred. */ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ + sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( iDiff==0 ) break; + if( iDiff<0 ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else{ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = (pLeft->bEof || pRight->bEof); + } + break; + } + + case FTSQUERY_OR: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + + assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); + assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); + + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + fts3EvalNextRow(pCsr, pLeft, pRc); + }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ + fts3EvalNextRow(pCsr, pRight, pRc); + }else{ + fts3EvalNextRow(pCsr, pLeft, pRc); + fts3EvalNextRow(pCsr, pRight, pRc); + } + + pExpr->bEof = (pLeft->bEof && pRight->bEof); + iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ + pExpr->iDocid = pLeft->iDocid; + }else{ + pExpr->iDocid = pRight->iDocid; + } + + break; + } + + case FTSQUERY_NOT: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + + if( pRight->bStart==0 ){ + fts3EvalNextRow(pCsr, pRight, pRc); + assert( *pRc!=SQLITE_OK || pRight->bStart ); + } + + fts3EvalNextRow(pCsr, pLeft, pRc); + if( pLeft->bEof==0 ){ + while( !*pRc + && !pRight->bEof + && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 + ){ + fts3EvalNextRow(pCsr, pRight, pRc); + } + } + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + break; + } + + default: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + fts3EvalInvalidatePoslist(pPhrase); + *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); + pExpr->iDocid = pPhrase->doclist.iDocid; + break; + } + } + } +} + +/* +** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR +** cluster, then this function returns 1 immediately. +** +** Otherwise, it checks if the current row really does match the NEAR +** expression, using the data currently stored in the position lists +** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression. +** +** If the current row is a match, the position list associated with each +** phrase in the NEAR expression is edited in place to contain only those +** phrase instances sufficiently close to their peers to satisfy all NEAR +** constraints. In this case it returns 1. If the NEAR expression does not +** match the current row, 0 is returned. The position lists may or may not +** be edited if 0 is returned. +*/ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ int res = 1; @@ -3848,7 +4254,7 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ ** ** The right-hand child of a NEAR node is always a phrase. The ** left-hand child may be either a phrase or a NEAR node. There are - ** no exceptions to this. + ** no exceptions to this - it's the way the parser in fts3_expr.c works. */ if( *pRc==SQLITE_OK && pExpr->eType==FTSQUERY_NEAR @@ -3875,7 +4281,7 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ Fts3Phrase *pPhrase = p->pRight->pPhrase; int nNear = p->nNear; - res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); } aPoslist = pExpr->pRight->pPhrase->doclist.pList; @@ -3885,7 +4291,7 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ Fts3Phrase *pPhrase = ( p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase ); - res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); } } @@ -3896,128 +4302,29 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ } /* -** This macro is used by the fts3EvalNext() function. The two arguments are -** 64-bit docid values. If the current query is "ORDER BY docid ASC", then -** the macro returns (i1 - i2). Or if it is "ORDER BY docid DESC", then -** it returns (i2 - i1). This allows the same code to be used for merging -** doclists in ascending or descending order. +** This function is a helper function for fts3EvalTestDeferredAndNear(). +** Assuming no error occurs or has occurred, It returns non-zero if the +** expression passed as the second argument matches the row that pCsr +** currently points to, or zero if it does not. +** +** If *pRc is not SQLITE_OK when this function is called, it is a no-op. +** If an error occurs during execution of this function, *pRc is set to +** the appropriate SQLite error code. In this case the returned value is +** undefined. */ -#define DOCID_CMP(i1, i2) ((pCsr->bDesc?-1:1) * (i1-i2)) - -static void fts3EvalNext( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, - int *pRc +static int fts3EvalTestExpr( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Expr to test. May or may not be root. */ + int *pRc /* IN/OUT: Error code */ ){ - if( *pRc==SQLITE_OK ){ - assert( pExpr->bEof==0 ); - pExpr->bStart = 1; - - switch( pExpr->eType ){ - case FTSQUERY_NEAR: - case FTSQUERY_AND: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - assert( !pLeft->bDeferred || !pRight->bDeferred ); - if( pLeft->bDeferred ){ - fts3EvalNext(pCsr, pRight, pRc); - pExpr->iDocid = pRight->iDocid; - pExpr->bEof = pRight->bEof; - }else if( pRight->bDeferred ){ - fts3EvalNext(pCsr, pLeft, pRc); - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = pLeft->bEof; - }else{ - fts3EvalNext(pCsr, pLeft, pRc); - fts3EvalNext(pCsr, pRight, pRc); - - while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ - sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - if( iDiff==0 ) break; - if( iDiff<0 ){ - fts3EvalNext(pCsr, pLeft, pRc); - }else{ - fts3EvalNext(pCsr, pRight, pRc); - } - } - - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = (pLeft->bEof || pRight->bEof); - } - break; - } - - case FTSQUERY_OR: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - - assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); - assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); - - if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ - fts3EvalNext(pCsr, pLeft, pRc); - }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ - fts3EvalNext(pCsr, pRight, pRc); - }else{ - fts3EvalNext(pCsr, pLeft, pRc); - fts3EvalNext(pCsr, pRight, pRc); - } - - pExpr->bEof = (pLeft->bEof && pRight->bEof); - iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ - pExpr->iDocid = pLeft->iDocid; - }else{ - pExpr->iDocid = pRight->iDocid; - } - - break; - } - - case FTSQUERY_NOT: { - Fts3Expr *pLeft = pExpr->pLeft; - Fts3Expr *pRight = pExpr->pRight; - - if( pRight->bStart==0 ){ - fts3EvalNext(pCsr, pRight, pRc); - assert( *pRc!=SQLITE_OK || pRight->bStart ); - } - - fts3EvalNext(pCsr, pLeft, pRc); - if( pLeft->bEof==0 ){ - while( !*pRc - && !pRight->bEof - && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 - ){ - fts3EvalNext(pCsr, pRight, pRc); - } - } - pExpr->iDocid = pLeft->iDocid; - pExpr->bEof = pLeft->bEof; - break; - } - - default: { - Fts3Phrase *pPhrase = pExpr->pPhrase; - fts3EvalZeroPoslist(pPhrase); - *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); - pExpr->iDocid = pPhrase->doclist.iDocid; - break; - } - } - } -} - -static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ - int bHit = 1; + int bHit = 1; /* Return value */ if( *pRc==SQLITE_OK ){ switch( pExpr->eType ){ case FTSQUERY_NEAR: case FTSQUERY_AND: bHit = ( - fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) - && fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) && fts3EvalNearTest(pExpr, pRc) ); @@ -4043,27 +4350,27 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ Fts3Expr *p; for(p=pExpr; p->pPhrase==0; p=p->pLeft){ if( p->pRight->iDocid==pCsr->iPrevId ){ - fts3EvalZeroPoslist(p->pRight->pPhrase); + fts3EvalInvalidatePoslist(p->pRight->pPhrase); } } if( p->iDocid==pCsr->iPrevId ){ - fts3EvalZeroPoslist(p->pPhrase); + fts3EvalInvalidatePoslist(p->pPhrase); } } break; case FTSQUERY_OR: { - int bHit1 = fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc); - int bHit2 = fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc); + int bHit1 = fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc); + int bHit2 = fts3EvalTestExpr(pCsr, pExpr->pRight, pRc); bHit = bHit1 || bHit2; break; } case FTSQUERY_NOT: bHit = ( - fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) - && !fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + fts3EvalTestExpr(pCsr, pExpr->pLeft, pRc) + && !fts3EvalTestExpr(pCsr, pExpr->pRight, pRc) ); break; @@ -4074,7 +4381,7 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ Fts3Phrase *pPhrase = pExpr->pPhrase; assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); if( pExpr->bDeferred ){ - fts3EvalZeroPoslist(pPhrase); + fts3EvalInvalidatePoslist(pPhrase); } *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); bHit = (pPhrase->doclist.pList!=0); @@ -4090,27 +4397,49 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ } /* -** Return 1 if both of the following are true: +** This function is called as the second part of each xNext operation when +** iterating through the results of a full-text query. At this point the +** cursor points to a row that matches the query expression, with the +** following caveats: +** +** * Up until this point, "NEAR" operators in the expression have been +** treated as "AND". +** +** * Deferred tokens have not yet been considered. +** +** If *pRc is not SQLITE_OK when this function is called, it immediately +** returns 0. Otherwise, it tests whether or not after considering NEAR +** operators and deferred tokens the current row is still a match for the +** expression. It returns 1 if both of the following are true: ** ** 1. *pRc is SQLITE_OK when this function returns, and ** ** 2. After scanning the current FTS table row for the deferred tokens, -** it is determined that the row does not match the query. +** it is determined that the row does *not* match the query. ** ** Or, if no error occurs and it seems the current row does match the FTS ** query, return 0. */ -static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){ +static int fts3EvalTestDeferredAndNear(Fts3Cursor *pCsr, int *pRc){ int rc = *pRc; int bMiss = 0; if( rc==SQLITE_OK ){ + + /* If there are one or more deferred tokens, load the current row into + ** memory and scan it to determine the position list for each deferred + ** token. Then, see if this row is really a match, considering deferred + ** tokens and NEAR operators (neither of which were taken into account + ** earlier, by fts3EvalNextRow()). + */ if( pCsr->pDeferred ){ rc = fts3CursorSeek(0, pCsr); if( rc==SQLITE_OK ){ rc = sqlite3Fts3CacheDeferredDoclists(pCsr); } } - bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc)); + bMiss = (0==fts3EvalTestExpr(pCsr, pCsr->pExpr, &rc)); + + /* Free the position-lists accumulated for each deferred token above. */ sqlite3Fts3FreeDeferredDoclists(pCsr); *pRc = rc; } @@ -4121,7 +4450,7 @@ static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){ ** Advance to the next document that matches the FTS expression in ** Fts3Cursor.pExpr. */ -int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){ +static int fts3EvalNext(Fts3Cursor *pCsr){ int rc = SQLITE_OK; /* Return Code */ Fts3Expr *pExpr = pCsr->pExpr; assert( pCsr->isEof==0 ); @@ -4133,19 +4462,19 @@ int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){ sqlite3_reset(pCsr->pStmt); } assert( sqlite3_data_count(pCsr->pStmt)==0 ); - fts3EvalNext(pCsr, pExpr, &rc); + fts3EvalNextRow(pCsr, pExpr, &rc); pCsr->isEof = pExpr->bEof; pCsr->isRequireSeek = 1; pCsr->isMatchinfoNeeded = 1; pCsr->iPrevId = pExpr->iDocid; - }while( pCsr->isEof==0 && fts3EvalLoadDeferred(pCsr, &rc) ); + }while( pCsr->isEof==0 && fts3EvalTestDeferredAndNear(pCsr, &rc) ); } return rc; } /* ** Restart interation for expression pExpr so that the next call to -** sqlite3Fts3EvalNext() visits the first row. Do not allow incremental +** fts3EvalNext() visits the first row. Do not allow incremental ** loading or merging of phrase doclists for this iteration. ** ** If *pRc is other than SQLITE_OK when this function is called, it is @@ -4161,7 +4490,7 @@ static void fts3EvalRestart( Fts3Phrase *pPhrase = pExpr->pPhrase; if( pPhrase ){ - fts3EvalZeroPoslist(pPhrase); + fts3EvalInvalidatePoslist(pPhrase); if( pPhrase->bIncr ){ assert( pPhrase->nToken==1 ); assert( pPhrase->aToken[0].pSegcsr ); @@ -4277,14 +4606,14 @@ static int fts3EvalGatherStats( assert( sqlite3_data_count(pCsr->pStmt)==0 ); /* Advance to the next document */ - fts3EvalNext(pCsr, pRoot, &rc); + fts3EvalNextRow(pCsr, pRoot, &rc); pCsr->isEof = pRoot->bEof; pCsr->isRequireSeek = 1; pCsr->isMatchinfoNeeded = 1; pCsr->iPrevId = pRoot->iDocid; }while( pCsr->isEof==0 && pRoot->eType==FTSQUERY_NEAR - && fts3EvalLoadDeferred(pCsr, &rc) + && fts3EvalTestDeferredAndNear(pCsr, &rc) ); if( rc==SQLITE_OK && pCsr->isEof==0 ){ @@ -4306,10 +4635,10 @@ static int fts3EvalGatherStats( */ fts3EvalRestart(pCsr, pRoot, &rc); do { - fts3EvalNext(pCsr, pRoot, &rc); + fts3EvalNextRow(pCsr, pRoot, &rc); assert( pRoot->bEof==0 ); }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); - fts3EvalLoadDeferred(pCsr, &rc); + fts3EvalTestDeferredAndNear(pCsr, &rc); } } return rc; @@ -4440,7 +4769,7 @@ void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ if( pPhrase ){ int i; sqlite3_free(pPhrase->doclist.aAll); - fts3EvalZeroPoslist(pPhrase); + fts3EvalInvalidatePoslist(pPhrase); memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); for(i=0; inToken; i++){ fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); @@ -4449,4 +4778,18 @@ void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ } } +#if !SQLITE_CORE +/* +** Initialize API pointer table, if required. +*/ +int sqlite3_extension_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3Fts3Init(db); +} +#endif + #endif diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 1664379609..694108b5c1 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -500,9 +500,6 @@ int sqlite3Fts3TermSegReaderCursor( void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); -int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int); -int sqlite3Fts3EvalNext(Fts3Cursor *pCsr); - int sqlite3Fts3MsrIncrStart( Fts3Table*, Fts3MultiSegReader*, int, const char*, int); int sqlite3Fts3MsrIncrNext( diff --git a/manifest b/manifest index 674d6dedb7..5f55a7ba07 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\sfor\sticket\s[91e2e8ba6f].\sNo\schanges\sto\scode. -D 2011-06-23T16:40:26.224 +C Fix\ssome\sof\sthe\scode\sissues\s(missing\scomments\setc.)\sin\sthe\snew\sFTS\scode. +D 2011-06-23T17:09:51.936 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -62,9 +62,9 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c ca776037493d0081da70a6afc0df80f5ce347cb7 +F ext/fts3/fts3.c c25723c7e96763fc88652a8716564e4b22c8a327 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h fa493ccbad78a2c99ad1c984f651c0c202e68536 +F ext/fts3/fts3Int.h 7b452eeb592134c7adf87720c9e56d6060d1ed5a F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 F ext/fts3/fts3_expr.c 23791de01b3a5d313d76e02befd2601d4096bc2b F ext/fts3/fts3_hash.c aad95afa01cf2a5ffaa448e4b0ab043880cd1efb @@ -949,7 +949,10 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 0b3174e0b1364ccc31853dce02bce5f7d3d431db -R f6237300c8e94b60ef9cb6b4fc0d6ad4 +P c271f7e88fc081a460dd3f4afb24aa9fb7fa2917 +R b9f14b373682048157818c9e1116d00f +T *branch * fts3-changes +T *sym-fts3-changes * +T -sym-trunk * U dan -Z 66a01206844bd880ddebccebd0a5e317 +Z a7e048a2a0bdbbc293d8f131a4bc43fc diff --git a/manifest.uuid b/manifest.uuid index bffb816668..f18b078c6a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c271f7e88fc081a460dd3f4afb24aa9fb7fa2917 \ No newline at end of file +8230d83120e0f4d217fde56e22c6f05aa5adee09 \ No newline at end of file From 15d31e1ee1e504e3a0a83ece6e27e6983c0da07f Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 23 Jun 2011 19:49:22 +0000 Subject: [PATCH 02/52] Version 3.7.7 release candidate #1. FossilOrigin-Name: 4374b7e83ea0a3fbc3691f9c0c936272862f32f2 --- manifest | 8 ++++---- manifest.uuid | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/manifest b/manifest index 5971790c88..6afb8c5324 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sbit\sto\sthe\sSQLITE_TESTCTRL_OPTIMIZATIONS\ssetting\sthat\swill\sdisable\naffinity\swhen\swriting\sto\sany\sindex,\sregardless\sof\swhether\sor\snot\sthe\sindex\nis\son\sa\smanifestation\sof\sa\sview.\s\sThis\sallows\sbetter\stesting\sof\sthe\sfix\nfor\sticket\s[91e2e8ba6ff2e2]. -D 2011-06-23T17:29:33.244 +C Version\s3.7.7\srelease\scandidate\s#1. +D 2011-06-23T19:49:22.376 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -949,7 +949,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P c271f7e88fc081a460dd3f4afb24aa9fb7fa2917 +P b61a76a53af04f731fe7617f7b6b4fb2aef6587b R 5805b6f3882ce897a61c4ce87076bc70 U drh -Z 83d974f3e6b84b4810ac6511db96428d +Z a111ec09c232c3c09811b275ae45bf17 diff --git a/manifest.uuid b/manifest.uuid index b421fc183b..1781623b8d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b61a76a53af04f731fe7617f7b6b4fb2aef6587b \ No newline at end of file +4374b7e83ea0a3fbc3691f9c0c936272862f32f2 \ No newline at end of file From df3aa16e2bd0543ec80be0703a429cbcf7923403 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Jun 2011 11:29:51 +0000 Subject: [PATCH 03/52] Fix some harmless compiler warnings that were occurring with THREADSAFE=0. FossilOrigin-Name: 9b191bb4c7c1e1b12b188c0b3eee1f8f587887c8 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/date.c | 2 ++ src/os_unix.c | 2 +- tool/warnings.sh | 6 +++++- 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 6afb8c5324..703d2960ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.7.7\srelease\scandidate\s#1. -D 2011-06-23T19:49:22.376 +C Fix\ssome\sharmless\scompiler\swarnings\sthat\swere\soccurring\swith\sTHREADSAFE=0. +D 2011-06-24T11:29:51.234 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -131,7 +131,7 @@ F src/build.c 5a428625d21ad409514afb40ad083bee25dd957a F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 7deec4534f3b5a0c3b4a4cbadf809d321f64f9c4 -F src/date.c d3c11de76392ea62637bfac0f4655889fc2f5a85 +F src/date.c a3c6842bad7ae632281811de112a8ba63ff08ab3 F src/delete.c 4925f9121525fc871f5d8d13c1f7dcc91abb38bb F src/expr.c ab46ab0f0c44979a8164ca31728d7d10ae5e8106 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb @@ -165,7 +165,7 @@ F src/os.c 22ac61d06e72a0dac900400147333b07b13d8e1d F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c 07acbb3e074e52b48a4248c06f66c9a91db1a0ce +F src/os_unix.c d3e7b17100704ee0fe2ef71a98c478b947480f4d F src/os_win.c eafcd6b91cf204a7ef29ac1ef2a1b7132e132e58 F src/pager.c 120550e7ef01dafaa2cbb4a0528c0d87c8f12b41 F src/pager.h 3f8c783de1d4706b40b1ac15b64f5f896bcc78d1 @@ -948,8 +948,8 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P b61a76a53af04f731fe7617f7b6b4fb2aef6587b -R 5805b6f3882ce897a61c4ce87076bc70 +F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 +P 4374b7e83ea0a3fbc3691f9c0c936272862f32f2 +R 1f317b87075f75e0f868cf711b7bb21e U drh -Z a111ec09c232c3c09811b275ae45bf17 +Z 5efaa6b46127bcc37e3b2f98e5f441cd diff --git a/manifest.uuid b/manifest.uuid index 1781623b8d..b9b8f7e7c3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4374b7e83ea0a3fbc3691f9c0c936272862f32f2 \ No newline at end of file +9b191bb4c7c1e1b12b188c0b3eee1f8f587887c8 \ No newline at end of file diff --git a/src/date.c b/src/date.c index c0744e8c72..f9411ea5b3 100644 --- a/src/date.c +++ b/src/date.c @@ -427,7 +427,9 @@ static int osLocaltime(time_t *t, struct tm *pTm){ #if (!defined(HAVE_LOCALTIME_R) || !HAVE_LOCALTIME_R) \ && (!defined(HAVE_LOCALTIME_S) || !HAVE_LOCALTIME_S) struct tm *pX; +#if SQLITE_THREADSAFE>0 sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); +#endif sqlite3_mutex_enter(mutex); pX = localtime(t); #ifndef SQLITE_OMIT_BUILTIN_TEST diff --git a/src/os_unix.c b/src/os_unix.c index 682e74c958..f9a12a2fb9 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3672,7 +3672,7 @@ static void unixShmPurge(unixFile *pFd){ if( p && p->nRef==0 ){ int i; assert( p->pInode==pFd->pInode ); - if( p->mutex ) sqlite3_mutex_free(p->mutex); + sqlite3_mutex_free(p->mutex); for(i=0; inRegion; i++){ if( p->h>=0 ){ munmap(p->apRegion[i], p->szRegion); diff --git a/tool/warnings.sh b/tool/warnings.sh index e1fa2b2d91..2eb3992f09 100644 --- a/tool/warnings.sh +++ b/tool/warnings.sh @@ -8,7 +8,11 @@ echo '********** No optimizations. Includes FTS4 and RTREE *********' gcc -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \ -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \ sqlite3.c -echo '********** Optimized -O3. Includes FTS4 and RTREE *********' +echo '********** No optimizations. ENABLE_STAT2. THREADSAFE=0 *******' +gcc -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \ + -ansi -DSQLITE_ENABLE_STAT2 -DSQLITE_THREADSAFE=0 \ + sqlite3.c +echo '********** Optimized -O3. Includes FTS4 and RTREE ************' gcc -O3 -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \ -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \ sqlite3.c From 600e62fdd5b8bb2de14b779ddca87671e36876db Mon Sep 17 00:00:00 2001 From: shaneh Date: Fri, 24 Jun 2011 13:50:13 +0000 Subject: [PATCH 04/52] Fix a Windows line ending issue in a test case that was causing an incrblob test to fail. FossilOrigin-Name: afbe9023a706375be53bce389ef4e806f17fc7ec --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/incrblob_err.test | 6 ++++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 703d2960ab..8a766d0e4b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sharmless\scompiler\swarnings\sthat\swere\soccurring\swith\sTHREADSAFE=0. -D 2011-06-24T11:29:51.234 +C Fix\sa\sWindows\sline\sending\sissue\sin\sa\stest\scase\sthat\swas\scausing\san\sincrblob\stest\sto\sfail. +D 2011-06-24T13:50:13.840 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -501,7 +501,7 @@ F test/in4.test 64f3cc1acde1b9161ccdd8e5bde3daefdb5b2617 F test/incrblob.test 76e787ca3301d9bfa6906031c626d26f8dd707de F test/incrblob2.test edc3a96e557bd61fb39acc8d2edd43371fbbaa19 F test/incrblob3.test aedbb35ea1b6450c33b98f2b6ed98e5020be8dc7 -F test/incrblob_err.test c577c91d4ed9e8336cdb188b15d6ee2a6fe9604e +F test/incrblob_err.test d2562d2771ebffd4b3af89ef64c140dd44371597 F test/incrblobfault.test 917c0292224c64a56ef7215fd633a3a82f805be0 F test/incrvacuum.test 453d1e490d8f5ad2c9b3a54282a0690d6ae56462 F test/incrvacuum2.test ae04573b73ad52179f56e194fff0fbe43b509d23 @@ -949,7 +949,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 4374b7e83ea0a3fbc3691f9c0c936272862f32f2 -R 1f317b87075f75e0f868cf711b7bb21e -U drh -Z 5efaa6b46127bcc37e3b2f98e5f441cd +P 9b191bb4c7c1e1b12b188c0b3eee1f8f587887c8 +R 32b8d769c1bc3fb3a1b9110c79256b1a +U shaneh +Z 84e18ea858841ffe13a9371ad8c31731 diff --git a/manifest.uuid b/manifest.uuid index b9b8f7e7c3..d375e114ae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b191bb4c7c1e1b12b188c0b3eee1f8f587887c8 \ No newline at end of file +afbe9023a706375be53bce389ef4e806f17fc7ec \ No newline at end of file diff --git a/test/incrblob_err.test b/test/incrblob_err.test index b4c258772a..35d37fee92 100644 --- a/test/incrblob_err.test +++ b/test/incrblob_err.test @@ -35,6 +35,7 @@ do_malloc_test 1 -tclprep { } } -tclbody { set ::blob [db incrblob blobs v 1] + fconfigure $::blob -translation binary set rc [catch {puts -nonewline $::blob $::data}] if {$rc} { error "out of memory" } } @@ -71,8 +72,7 @@ do_malloc_test 3 -tclprep { if {$rc} { error "out of memory" } -} - +} do_ioerr_test incrblob_err-4 -cksum 1 -sqlprep { CREATE TABLE blobs(k, v BLOB); @@ -87,6 +87,7 @@ do_ioerr_test incrblob_err-5 -cksum 1 -sqlprep { INSERT INTO blobs VALUES(1, zeroblob(length(CAST($::data AS BLOB)))); } -tclbody { set ::blob [db incrblob blobs v 1] + fconfigure $::blob -translation binary puts -nonewline $::blob $::data close $::blob } @@ -96,6 +97,7 @@ do_ioerr_test incrblob_err-6 -cksum 1 -sqlprep { INSERT INTO blobs VALUES(1, $::data || $::data || $::data); } -tclbody { set ::blob [db incrblob blobs v 1] + fconfigure $::blob -translation binary seek $::blob -20 end puts -nonewline $::blob "12345678900987654321" close $::blob From 28e4e355861916ff548d5ca171a44325dec67296 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 24 Jun 2011 18:43:23 +0000 Subject: [PATCH 05/52] Explicitly cast an argument to printf() to (int), in case the result of the pointer arithmetic expression is a 64-bit integer on some platforms. FossilOrigin-Name: 90cfeaf7b68eebc2cd75bc60b1bb86823646fe30 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/build.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 8a766d0e4b..369f2971cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sWindows\sline\sending\sissue\sin\sa\stest\scase\sthat\swas\scausing\san\sincrblob\stest\sto\sfail. -D 2011-06-24T13:50:13.840 +C Explicitly\scast\san\sargument\sto\sprintf()\sto\s(int),\sin\scase\sthe\sresult\sof\sthe\spointer\sarithmetic\sexpression\sis\sa\s64-bit\sinteger\son\ssome\splatforms. +D 2011-06-24T18:43:23.777 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -127,7 +127,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 8c46f0ab69ad9549c75a3a91fed87abdaa743e2f F src/btree.h f5d775cd6cfc7ac32a2535b70e8d2af48ef5f2ce F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3 -F src/build.c 5a428625d21ad409514afb40ad083bee25dd957a +F src/build.c 5e614e586d9f8a81c16c80b545b9e1747f96c1bb F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 7deec4534f3b5a0c3b4a4cbadf809d321f64f9c4 @@ -949,7 +949,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 9b191bb4c7c1e1b12b188c0b3eee1f8f587887c8 -R 32b8d769c1bc3fb3a1b9110c79256b1a -U shaneh -Z 84e18ea858841ffe13a9371ad8c31731 +P afbe9023a706375be53bce389ef4e806f17fc7ec +R 39e40cd4e043bc5ee4d5e0c0bd1144c1 +U dan +Z 0fb26f42eab966013b31af43c43a2eeb diff --git a/manifest.uuid b/manifest.uuid index d375e114ae..150eec39e9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -afbe9023a706375be53bce389ef4e806f17fc7ec \ No newline at end of file +90cfeaf7b68eebc2cd75bc60b1bb86823646fe30 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 97758cbcbd..455b35b56e 100644 --- a/src/build.c +++ b/src/build.c @@ -2791,7 +2791,7 @@ Index *sqlite3CreateIndex( /* A named index with an explicit CREATE INDEX statement */ zStmt = sqlite3MPrintf(db, "CREATE%s INDEX %.*s", onError==OE_None ? "" : " UNIQUE", - pEnd->z - pName->z + 1, + (int)(pEnd->z - pName->z) + 1, pName->z); }else{ /* An automatic index created by a PRIMARY KEY or UNIQUE constraint */ From 5b0b6fded766b43dad5df8c12bb2423b85a8e467 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 25 Jun 2011 01:14:36 +0000 Subject: [PATCH 06/52] Support using nmake via cmd.exe, allow Tcl related variables to be overridden via the environment, and add 'dll' to the 'all' target. FossilOrigin-Name: 0cb0f30603b1658ae24d1b74d6ab59d05ef4e448 --- Makefile.msc | 44 ++++++++++++++++++++++++++++++-------------- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index a6d1bf4c2f..718464d6ec 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -43,16 +43,32 @@ TCC = $(TCC) -I$(TOP)\ext\rtree # TCC = $(TCC) -DNDEBUG -# The library that programs using TCL must link against. +# The locations of the Tcl header and library files. Also, the library that +# non-stubs enabled programs using Tcl must link against. These variables +# (TCLINCDIR, TCLLIBDIR, and LIBTCL) may be overridden via the environment +# prior to running nmake in order to match the actual installed location and +# version on this machine. # -LIBTCL = tcl85.lib +!if "$(TCLINCDIR)" == "" TCLINCDIR = c:\tcl\include +!endif + +!if "$(TCLLIBDIR)" == "" TCLLIBDIR = c:\tcl\lib +!endif + +!if "$(LIBTCL)" == "" +LIBTCL = tcl85.lib +!endif # This is the command to use for tclsh - normally just "tclsh", but we may -# know the specific version we want to use +# know the specific version we want to use. This variable (TCLSH_CMD) may be +# overridden via the environment prior to running nmake in order to select a +# specific Tcl shell to use. # +!if "$(TCLSH_CMD)" == "" TCLSH_CMD = tclsh85 +!endif # Compiler options needed for programs that use the readline() library. # @@ -118,7 +134,7 @@ LTLIBOPTS = /MACHINE:$(PLATFORM) !ENDIF # nawk compatible awk. -NAWK = .\gawk.exe +NAWK = gawk.exe # You should not have to change anything below this line ############################################################################### @@ -439,7 +455,7 @@ EXTHDR = $(EXTHDR) \ # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. # -all: libsqlite3.lib sqlite3.exe libtclsqlite3.lib +all: dll libsqlite3.lib sqlite3.exe libtclsqlite3.lib libsqlite3.lib: $(LIBOBJ) $(LTLIB) $(LTLIBOPTS) /OUT:$@ $(LIBOBJ) $(TLIBS) @@ -463,7 +479,7 @@ sqlite3.exe: $(TOP)\src\shell.c libsqlite3.lib sqlite3.h -mkdir tsrc for %i in ($(SRC)) do copy /Y %i tsrc del /Q tsrc\sqlite.h.in tsrc\parse.y - $(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl vdbe.new + $(TCLSH_CMD) $(TOP)\tool\vdbe-compress.tcl < tsrc\vdbe.c > vdbe.new move vdbe.new tsrc\vdbe.c echo > .target_source @@ -720,10 +736,10 @@ tclsqlite3.exe: tclsqlite-shell.lo libsqlite3.lib # Rules to build opcodes.c and opcodes.h # opcodes.c: opcodes.h $(TOP)\mkopcodec.awk - $(NAWK) "/#define OP_/ { print }" opcodes.h | sort /+45 | $(NAWK) -f $(TOP)\mkopcodec.awk >opcodes.c + $(NAWK) "/#define OP_/ { print }" opcodes.h | sort /+45 | $(NAWK) -f $(TOP)\mkopcodec.awk > opcodes.c opcodes.h: parse.h $(TOP)\src\vdbe.c $(TOP)\mkopcodeh.awk - type parse.h $(TOP)\src\vdbe.c | $(NAWK) -f $(TOP)\mkopcodeh.awk >opcodes.h + type parse.h $(TOP)\src\vdbe.c | $(NAWK) -f $(TOP)\mkopcodeh.awk > opcodes.h # Rules to build parse.c and parse.h - the outputs of lemon. # @@ -734,16 +750,16 @@ parse.c: $(TOP)\src\parse.y lemon.exe $(TOP)\addopcodes.awk copy $(TOP)\src\parse.y . .\lemon.exe $(OPT_FEATURE_FLAGS) $(OPTS) parse.y move parse.h parse.h.temp - $(NAWK) -f $(TOP)\addopcodes.awk parse.h.temp >parse.h + $(NAWK) -f $(TOP)\addopcodes.awk parse.h.temp > parse.h sqlite3.h: $(TOP)\src\sqlite.h.in $(TOP)\manifest.uuid $(TOP)\VERSION - $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP) >sqlite3.h + $(TCLSH_CMD) $(TOP)\tool\mksqlite3h.tcl $(TOP) > sqlite3.h mkkeywordhash.exe: $(TOP)\tool\mkkeywordhash.c $(BCC) -Femkkeywordhash.exe $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)\tool\mkkeywordhash.c keywordhash.h: $(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe - .\mkkeywordhash.exe >keywordhash.h + .\mkkeywordhash.exe > keywordhash.h @@ -839,7 +855,7 @@ test: testfixture.exe sqlite3.exe spaceanal_tcl.h: $(TOP)\tool\spaceanal.tcl $(NAWK) -f $(TOP)/tool/tostr.awk \ - $(TOP)\tool\spaceanal.tcl >spaceanal_tcl.h + $(TOP)\tool\spaceanal.tcl > spaceanal_tcl.h sqlite3_analyzer.exe: $(TESTFIXTURE_SRC) spaceanal_tcl.h $(LTLINK) -DTCLSH=2 -DSQLITE_TEST=1 -DSQLITE_CRASH_TEST=1 \ @@ -866,10 +882,10 @@ clean: dll: sqlite3.dll sqlite3.def: libsqlite3.lib - echo EXPORTS >sqlite3.def + echo EXPORTS > sqlite3.def dumpbin /all libsqlite3.lib \ | $(NAWK) "/ 1 _sqlite3_/ { sub(/^.* _/,\"\");print }" \ - | sort >>sqlite3.def + | sort >> sqlite3.def sqlite3.dll: $(LIBOBJ) sqlite3.def link $(LTLINKOPTS) /DLL /DEF:sqlite3.def /OUT:$@ $(LIBOBJ) diff --git a/manifest b/manifest index 369f2971cc..495f9da122 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Explicitly\scast\san\sargument\sto\sprintf()\sto\s(int),\sin\scase\sthe\sresult\sof\sthe\spointer\sarithmetic\sexpression\sis\sa\s64-bit\sinteger\son\ssome\splatforms. -D 2011-06-24T18:43:23.777 +C Support\susing\snmake\svia\scmd.exe,\sallow\sTcl\srelated\svariables\sto\sbe\soverridden\svia\sthe\senvironment,\sand\sadd\s'dll'\sto\sthe\s'all'\starget. +D 2011-06-25T01:14:36.767 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 11082f65b452b908d93013292c17850378c39284 +F Makefile.msc 9c76731c20561a33ce1257c70be3f029cedb62a6 F Makefile.vxworks c85ec1d8597fe2f7bc225af12ac1666e21379151 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 3fcdd7fbe3eb282df3978fe77288544543767961 @@ -949,7 +949,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P afbe9023a706375be53bce389ef4e806f17fc7ec -R 39e40cd4e043bc5ee4d5e0c0bd1144c1 -U dan -Z 0fb26f42eab966013b31af43c43a2eeb +P 90cfeaf7b68eebc2cd75bc60b1bb86823646fe30 +R 9d6679872042f0704eb46a854652b6b2 +U mistachkin +Z 10a97d70fb2ecf74c56670a3fe99eddd diff --git a/manifest.uuid b/manifest.uuid index 150eec39e9..1bf5d940d2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -90cfeaf7b68eebc2cd75bc60b1bb86823646fe30 \ No newline at end of file +0cb0f30603b1658ae24d1b74d6ab59d05ef4e448 \ No newline at end of file From c7060c4dc5cf0c9bae1202513ae7a181c7646a60 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 26 Jun 2011 23:44:35 +0000 Subject: [PATCH 07/52] Modifications to the "like.test" script in order to expose the problem reported by ticket [25ee81271091ec27a8c5]. FossilOrigin-Name: c4db5b640638a211b096454ba90b58faf595392b --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/like.test | 7 +++++-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 495f9da122..552a042538 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Support\susing\snmake\svia\scmd.exe,\sallow\sTcl\srelated\svariables\sto\sbe\soverridden\svia\sthe\senvironment,\sand\sadd\s'dll'\sto\sthe\s'all'\starget. -D 2011-06-25T01:14:36.767 +C Modifications\sto\sthe\s"like.test"\sscript\sin\sorder\sto\sexpose\sthe\sproblem\nreported\sby\sticket\s[25ee81271091ec27a8c5]. +D 2011-06-26T23:44:35.143 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -540,7 +540,7 @@ F test/jrnlmode3.test c6522b276ba315fd1416198de6fc1da9e72409fb F test/keyword1.test a2400977a2e4fde43bf33754c2929fda34dbca05 F test/lastinsert.test 474d519c68cb79d07ecae56a763aa7f322c72f51 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 -F test/like.test a47f52692aac96ba82508efba74819214cdebc17 +F test/like.test 9cc5261d22f2108a27cedff8a972aa3284a4ba52 F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da F test/limit.test 2db7b3b34fb925b8e847d583d2eb67531d0ce67e F test/loadext.test 0393ce12d9616aa87597dd0ec88181de181f6db0 @@ -949,7 +949,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 90cfeaf7b68eebc2cd75bc60b1bb86823646fe30 -R 9d6679872042f0704eb46a854652b6b2 -U mistachkin -Z 10a97d70fb2ecf74c56670a3fe99eddd +P 0cb0f30603b1658ae24d1b74d6ab59d05ef4e448 +R b4c0c5b0c346256e14626d78568a95ff +U drh +Z 547c3ad3eebb979ac90e817759786a1e diff --git a/manifest.uuid b/manifest.uuid index 1bf5d940d2..76ea188c40 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0cb0f30603b1658ae24d1b74d6ab59d05ef4e448 \ No newline at end of file +c4db5b640638a211b096454ba90b58faf595392b \ No newline at end of file diff --git a/test/like.test b/test/like.test index bd9a6c39c1..45d5a97c5b 100644 --- a/test/like.test +++ b/test/like.test @@ -70,12 +70,15 @@ do_test like-1.4 { } } {ABC abc} do_test like-1.5.1 { + # Use sqlite3_exec() to verify fix for ticket [25ee81271091] 2011-06-26 + sqlite3_exec db {PRAGMA case_sensitive_like=on} +} {0 {}} +do_test like-1.5.2 { execsql { - PRAGMA case_sensitive_like=on; SELECT x FROM t1 WHERE x LIKE 'abc' ORDER BY 1; } } {abc} -do_test like-1.5.2 { +do_test like-1.5.3 { execsql { PRAGMA case_sensitive_like; -- no argument; does not change setting SELECT x FROM t1 WHERE x LIKE 'abc' ORDER BY 1; From aab910c459eafdc68606ff916f4ecbc60ee06a38 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 27 Jun 2011 00:01:22 +0000 Subject: [PATCH 08/52] Make sure all new statements begin life unexpired, even if they registered functions or did other actions during preparation that would have expired all statements. Fix for ticket [25ee81271091] FossilOrigin-Name: faa38c8724d3eebad97197bfa44d11e75cfe6c69 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 552a042538..1a8f207d3b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modifications\sto\sthe\s"like.test"\sscript\sin\sorder\sto\sexpose\sthe\sproblem\nreported\sby\sticket\s[25ee81271091ec27a8c5]. -D 2011-06-26T23:44:35.143 +C Make\ssure\sall\snew\sstatements\sbegin\slife\sunexpired,\seven\sif\sthey\sregistered\nfunctions\sor\sdid\sother\sactions\sduring\spreparation\sthat\swould\shave\sexpired\nall\sstatements.\s\sFix\sfor\sticket\s[25ee81271091] +D 2011-06-27T00:01:22.248 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -242,7 +242,7 @@ F src/vdbe.c 50b5b4ef29d928a4f60905f820ee1ab2f8ced29f F src/vdbe.h 5cf09e7ee8a3f7d93bc51f196a96550786afe7a1 F src/vdbeInt.h ad84226cc0adcb1185c22b70696b235a1678bb45 F src/vdbeapi.c 2f908c65337b16c6249ab378d477e74492cf759b -F src/vdbeaux.c 0b2e2880f13af400a27c92a7673287c3eaec8b21 +F src/vdbeaux.c 4d100407e3c72e163854aff8903d19d5ecdf46c0 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114 @@ -949,7 +949,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 0cb0f30603b1658ae24d1b74d6ab59d05ef4e448 -R b4c0c5b0c346256e14626d78568a95ff +P c4db5b640638a211b096454ba90b58faf595392b +R 1d2c6c9972e41fe17af0cc7482f056ad U drh -Z 547c3ad3eebb979ac90e817759786a1e +Z 50eb339a1b9fd382164ccbf3dfada87e diff --git a/manifest.uuid b/manifest.uuid index 76ea188c40..6f48ead043 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c4db5b640638a211b096454ba90b58faf595392b \ No newline at end of file +faa38c8724d3eebad97197bfa44d11e75cfe6c69 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 447ba57a1e..989a8003d3 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1505,6 +1505,7 @@ void sqlite3VdbeMakeReady( memset(zCsr, 0, zEnd-zCsr); zCsr += (zCsr - (u8*)0)&7; assert( EIGHT_BYTE_ALIGNMENT(zCsr) ); + p->expired = 0; /* Memory for registers, parameters, cursor, etc, is allocated in two ** passes. On the first pass, we try to reuse unused space at the From 5b217c308ffe74c1f9d12cd3664cccc0d5f4286b Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 27 Jun 2011 11:15:53 +0000 Subject: [PATCH 09/52] Changes to improve the selection of deferred tokens within phrases. FossilOrigin-Name: 2c4bbd90e2fca593c186bf412b608aff8c9f9061 --- ext/fts3/fts3.c | 67 ++++++++++++++++++++++++------------------------- manifest | 15 +++++------ manifest.uuid | 2 +- 3 files changed, 40 insertions(+), 44 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 40a0297add..16067aacc7 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -2553,24 +2553,15 @@ static int fts3TermSelect( ** that the doclist is simply a list of docids stored as delta encoded ** varints. */ -static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){ +static int fts3DoclistCountDocids(char *aList, int nList){ int nDoc = 0; /* Return value */ if( aList ){ char *aEnd = &aList[nList]; /* Pointer to one byte after EOF */ char *p = aList; /* Cursor */ - if( !isPoslist ){ - /* The number of docids in the list is the same as the number of - ** varints. In FTS3 a varint consists of a single byte with the 0x80 - ** bit cleared and zero or more bytes with the 0x80 bit set. So to - ** count the varints in the buffer, just count the number of bytes - ** with the 0x80 bit clear. */ - while( pnOvfl ){ - nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10; - }else{ - Fts3PhraseToken *pToken = pTC->pToken; - int nList = 0; - char *pList = 0; - rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); - assert( rc==SQLITE_OK || pList==0 ); - if( rc==SQLITE_OK ){ - nDocEst = fts3DoclistCountDocids(1, pList, nList); - fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList); - } - } - }else{ - if( pTC->nOvfl>=(nDocEst*nDocSize) ){ - Fts3PhraseToken *pToken = pTC->pToken; - rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); - fts3SegReaderCursorFree(pToken->pSegcsr); - pToken->pSegcsr = 0; + + if( ii && pTC->nOvfl>=(nDocEst*nDocSize) ){ + /* The number of overflow pages to load for this (and therefore all + ** subsequent) tokens is greater than the estimated number of pages + ** that will be loaded if all subsequent tokens are deferred. + */ + Fts3PhraseToken *pToken = pTC->pToken; + rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); + fts3SegReaderCursorFree(pToken->pSegcsr); + pToken->pSegcsr = 0; + }else if( ii==0 || pTC->pPhrase->nToken>1 ){ + /* Either this is the cheapest token in the entire query, or it is + ** part of a multi-token phrase. Either way, the entire doclist will + ** (eventually) be loaded into memory. It may as well be now. */ + Fts3PhraseToken *pToken = pTC->pToken; + int nList = 0; + char *pList = 0; + rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); + assert( rc==SQLITE_OK || pList==0 ); + if( rc==SQLITE_OK ){ + fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList); + nDocEst = fts3DoclistCountDocids( + pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll + ); } + }else { + /* This token will not be deferred. And it will not be loaded into + ** memory at this point either. So assume that it filters out 75% of + ** the currently estimated number of documents. */ nDocEst = 1 + (nDocEst/4); } pTC->pToken = 0; diff --git a/manifest b/manifest index 5f55a7ba07..c54d53e491 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sof\sthe\scode\sissues\s(missing\scomments\setc.)\sin\sthe\snew\sFTS\scode. -D 2011-06-23T17:09:51.936 +C Changes\sto\simprove\sthe\sselection\sof\sdeferred\stokens\swithin\sphrases. +D 2011-06-27T11:15:53.752 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -62,7 +62,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c c25723c7e96763fc88652a8716564e4b22c8a327 +F ext/fts3/fts3.c 70127bf57c54c45545a4ddd73161caf2d9bd1abe F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 7b452eeb592134c7adf87720c9e56d6060d1ed5a F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 @@ -949,10 +949,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P c271f7e88fc081a460dd3f4afb24aa9fb7fa2917 -R b9f14b373682048157818c9e1116d00f -T *branch * fts3-changes -T *sym-fts3-changes * -T -sym-trunk * +P 8230d83120e0f4d217fde56e22c6f05aa5adee09 +R a3718a7f2e746e02b745366a3c6c0fd0 U dan -Z a7e048a2a0bdbbc293d8f131a4bc43fc +Z deeb565ce480cc0dfb49de44342bb4c6 diff --git a/manifest.uuid b/manifest.uuid index f18b078c6a..b01b761298 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8230d83120e0f4d217fde56e22c6f05aa5adee09 \ No newline at end of file +2c4bbd90e2fca593c186bf412b608aff8c9f9061 \ No newline at end of file From c431fd558068005380f8acace49b48895305e779 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 27 Jun 2011 16:55:50 +0000 Subject: [PATCH 10/52] Add a case to permutations.test to run the veryquick test suite using sqlite3_prepare() instead of sqlite3_prepare_v2(). This helps to test the fix for bug [25ee812710]. FossilOrigin-Name: d9f7993bfa2ecb30c44455de538eb6f8a4a59087 --- manifest | 24 ++++---- manifest.uuid | 2 +- src/tclsqlite.c | 137 ++++++++++++++++++++++++++++++++++++----- test/all.test | 1 + test/alter2.test | 1 + test/exists.test | 2 +- test/permutations.test | 9 +++ test/temptable.test | 1 + 8 files changed, 147 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index 1a8f207d3b..c734903830 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sall\snew\sstatements\sbegin\slife\sunexpired,\seven\sif\sthey\sregistered\nfunctions\sor\sdid\sother\sactions\sduring\spreparation\sthat\swould\shave\sexpired\nall\sstatements.\s\sFix\sfor\sticket\s[25ee81271091] -D 2011-06-27T00:01:22.248 +C Add\sa\scase\sto\spermutations.test\sto\srun\sthe\sveryquick\stest\ssuite\susing\ssqlite3_prepare()\sinstead\sof\ssqlite3_prepare_v2().\sThis\shelps\sto\stest\sthe\sfix\sfor\sbug\s[25ee812710]. +D 2011-06-27T16:55:50.495 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -187,7 +187,7 @@ F src/sqliteInt.h 7b72f7c6929757c0678d94524dbc7e7e3d6dc398 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e -F src/tclsqlite.c 5db825be61708b1a2b3f8f6e185e9b753829acef +F src/tclsqlite.c c355b33803fb756dc09297ec84d19604301fa622 F src/test1.c 8eef932f1024f1076a8adb8a6f5d9c5267c9e969 F src/test2.c 80d323d11e909cf0eb1b6fbb4ac22276483bcf31 F src/test3.c 124ff9735fb6bb7d41de180d6bac90e7b1509432 @@ -254,9 +254,9 @@ F src/where.c 55403ce19c506be6a321c7f129aff693d6103db5 F test/8_3_names.test b93687beebd17f6ebf812405a6833bae5d1f4199 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 -F test/all.test 51756962d522e474338e9b2ebb26e7364d4aa125 +F test/all.test 52fc8dee494092031a556911d404ca30a749a30b F test/alter.test a3f570072b53d7c0fe463bab3f5affa8e113c487 -F test/alter2.test 75f731508f1bf27ba09a6075c66cd02216ba464b +F test/alter2.test e0c09d630d650ea32333781a4ed3c45eb02c4289 F test/alter3.test 8677e48d95536f7a6ed86a1a774744dadcc22b07 F test/alter4.test 1e5dd6b951e9f65ca66422edff02e56df82dd403 F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc @@ -393,7 +393,7 @@ F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 F test/exclusive.test 53e1841b422e554cecf0160f937c473d6d0e3062 F test/exclusive2.test 343d55130c12c67b8bf10407acec043a6c26c86b F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 -F test/exists.test 5e2d64b4eb5a9d08876599bdae2e1213d2d12e2a +F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30 F test/expr.test 67c9fd6f8f829e239dc8b0f4a08a73c08b09196d F test/fallocate.test 43dc34b8c24be6baffadc3b4401ee15710ce83c6 F test/filectrl.test 97003734290887566e01dded09dc9e99cb937e9e @@ -615,7 +615,7 @@ F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806 F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16 -F test/permutations.test 1e8892ebf1bd6e9e8036f4841c72a91bf72da74a +F test/permutations.test aae89941ca0968b53633c97b5dd0d6f9f9899d27 F test/pragma.test fdfc09067ea104a0c247a1a79d8093b56656f850 F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea @@ -693,7 +693,7 @@ F test/table.test 04ba066432430657712d167ebf28080fe878d305 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tclsqlite.test 8c154101e704170c2be10f137a5499ac2c6da8d3 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c -F test/temptable.test f42121a0d29a62f00f93274464164177ab1cc24a +F test/temptable.test 1a21a597055dcf6002b6f1ee867632dccd6e925c F test/temptrigger.test b0273db072ce5f37cf19140ceb1f0d524bbe9f05 F test/tester.tcl 7d3d17914ca5397a82f801db733b81d4a82f50c3 F test/thread001.test a3e6a7254d1cb057836cb3145b60c10bf5b7e60f @@ -949,7 +949,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P c4db5b640638a211b096454ba90b58faf595392b -R 1d2c6c9972e41fe17af0cc7482f056ad -U drh -Z 50eb339a1b9fd382164ccbf3dfada87e +P faa38c8724d3eebad97197bfa44d11e75cfe6c69 +R ea65f158609dffa4f41c8ce1eb564414 +U dan +Z 2df66e3ffb41532c6826925a7e3446a4 diff --git a/manifest.uuid b/manifest.uuid index 6f48ead043..4cd588aa86 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -faa38c8724d3eebad97197bfa44d11e75cfe6c69 \ No newline at end of file +d9f7993bfa2ecb30c44455de538eb6f8a4a59087 \ No newline at end of file diff --git a/src/tclsqlite.c b/src/tclsqlite.c index ad4d27a126..339b8967df 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -107,6 +107,11 @@ typedef struct IncrblobChannel IncrblobChannel; /* ** There is one instance of this structure for each SQLite database ** that has been opened by the SQLite TCL interface. +** +** If this module is built with SQLITE_TEST defined (to create the SQLite +** testfixture executable), then it may be configured to use either +** sqlite3_prepare_v2() or sqlite3_prepare() to prepare SQL statements. +** If SqliteDb.bLegacyPrepare is true, sqlite3_prepare() is used. */ typedef struct SqliteDb SqliteDb; struct SqliteDb { @@ -135,6 +140,9 @@ struct SqliteDb { IncrblobChannel *pIncrblob;/* Linked list of open incrblob channels */ int nStep, nSort, nIndex; /* Statistics for most recent operation */ int nTransaction; /* Number of nested [transaction] methods */ +#ifdef SQLITE_TEST + int bLegacyPrepare; /* True to use sqlite3_prepare() */ +#endif }; struct IncrblobChannel { @@ -429,20 +437,33 @@ static SqlFunc *findSqlFunc(SqliteDb *pDb, const char *zName){ return pNew; } +/* +** Free a single SqlPreparedStmt object. +*/ +static void dbFreeStmt(SqlPreparedStmt *pStmt){ +#ifdef SQLITE_TEST + if( sqlite3_sql(pStmt->pStmt)==0 ){ + Tcl_Free((char *)pStmt->zSql); + } +#endif + sqlite3_finalize(pStmt->pStmt); + Tcl_Free((char *)pStmt); +} + /* ** Finalize and free a list of prepared statements */ -static void flushStmtCache( SqliteDb *pDb ){ +static void flushStmtCache(SqliteDb *pDb){ SqlPreparedStmt *pPreStmt; + SqlPreparedStmt *pNext; - while( pDb->stmtList ){ - sqlite3_finalize( pDb->stmtList->pStmt ); - pPreStmt = pDb->stmtList; - pDb->stmtList = pDb->stmtList->pNext; - Tcl_Free( (char*)pPreStmt ); + for(pPreStmt = pDb->stmtList; pPreStmt; pPreStmt=pNext){ + pNext = pPreStmt->pNext; + dbFreeStmt(pPreStmt); } pDb->nStmt = 0; pDb->stmtLast = 0; + pDb->stmtList = 0; } /* @@ -1030,6 +1051,27 @@ static int DbTransPostCmd( return rc; } +/* +** Unless SQLITE_TEST is defined, this function is a simple wrapper around +** sqlite3_prepare_v2(). If SQLITE_TEST is defined, then it uses either +** sqlite3_prepare_v2() or legacy interface sqlite3_prepare(), depending +** on whether or not the [db_use_legacy_prepare] command has been used to +** configure the connection. +*/ +static int dbPrepare( + SqliteDb *pDb, /* Database object */ + const char *zSql, /* SQL to compile */ + sqlite3_stmt **ppStmt, /* OUT: Prepared statement */ + const char **pzOut /* OUT: Pointer to next SQL statement */ +){ +#ifdef SQLITE_TEST + if( pDb->bLegacyPrepare ){ + return sqlite3_prepare(pDb->db, zSql, -1, ppStmt, pzOut); + } +#endif + return sqlite3_prepare_v2(pDb->db, zSql, -1, ppStmt, pzOut); +} + /* ** Search the cache for a prepared-statement object that implements the ** first SQL statement in the buffer pointed to by parameter zIn. If @@ -1100,7 +1142,7 @@ static int dbPrepareAndBind( if( pPreStmt==0 ){ int nByte; - if( SQLITE_OK!=sqlite3_prepare_v2(pDb->db, zSql, -1, &pStmt, pzOut) ){ + if( SQLITE_OK!=dbPrepare(pDb, zSql, &pStmt, pzOut) ){ Tcl_SetObjResult(interp, dbTextToObj(sqlite3_errmsg(pDb->db))); return TCL_ERROR; } @@ -1127,6 +1169,14 @@ static int dbPrepareAndBind( pPreStmt->nSql = (*pzOut - zSql); pPreStmt->zSql = sqlite3_sql(pStmt); pPreStmt->apParm = (Tcl_Obj **)&pPreStmt[1]; +#ifdef SQLITE_TEST + if( pPreStmt->zSql==0 ){ + char *zCopy = Tcl_Alloc(pPreStmt->nSql + 1); + memcpy(zCopy, zSql, pPreStmt->nSql); + zCopy[pPreStmt->nSql] = '\0'; + pPreStmt->zSql = zCopy; + } +#endif } assert( pPreStmt ); assert( strlen30(pPreStmt->zSql)==pPreStmt->nSql ); @@ -1180,7 +1230,6 @@ static int dbPrepareAndBind( return TCL_OK; } - /* ** Release a statement reference obtained by calling dbPrepareAndBind(). ** There should be exactly one call to this function for each call to @@ -1205,8 +1254,7 @@ static void dbReleaseStmt( if( pDb->maxStmt<=0 || discard ){ /* If the cache is turned off, deallocated the statement */ - sqlite3_finalize(pPreStmt->pStmt); - Tcl_Free((char *)pPreStmt); + dbFreeStmt(pPreStmt); }else{ /* Add the prepared statement to the beginning of the cache list. */ pPreStmt->pNext = pDb->stmtList; @@ -1226,11 +1274,11 @@ static void dbReleaseStmt( /* If we have too many statement in cache, remove the surplus from ** the end of the cache list. */ while( pDb->nStmt>pDb->maxStmt ){ - sqlite3_finalize(pDb->stmtLast->pStmt); - pDb->stmtLast = pDb->stmtLast->pPrev; - Tcl_Free((char*)pDb->stmtLast->pNext); + SqlPreparedStmt *pLast = pDb->stmtLast; + pDb->stmtLast = pLast->pPrev; pDb->stmtLast->pNext = 0; pDb->nStmt--; + dbFreeStmt(pLast); } } } @@ -1363,9 +1411,12 @@ static void dbEvalRowInfo( ** no further rows available. This is similar to SQLITE_DONE. */ static int dbEvalStep(DbEvalContext *p){ + const char *zPrevSql = 0; /* Previous value of p->zSql */ + while( p->zSql[0] || p->pPreStmt ){ int rc; if( p->pPreStmt==0 ){ + zPrevSql = (p->zSql==zPrevSql ? 0 : p->zSql); rc = dbPrepareAndBind(p->pDb, p->zSql, &p->zSql, &p->pPreStmt); if( rc!=TCL_OK ) return rc; }else{ @@ -1392,8 +1443,19 @@ static int dbEvalStep(DbEvalContext *p){ if( rcs!=SQLITE_OK ){ /* If a run-time error occurs, report the error and stop reading ** the SQL. */ - Tcl_SetObjResult(pDb->interp, dbTextToObj(sqlite3_errmsg(pDb->db))); dbReleaseStmt(pDb, pPreStmt, 1); +#if SQLITE_TEST + if( p->pDb->bLegacyPrepare && rcs==SQLITE_SCHEMA && zPrevSql ){ + /* If the runtime error was an SQLITE_SCHEMA, and the database + ** handle is configured to use the legacy sqlite3_prepare() + ** interface, retry prepare()/step() on the same SQL statement. + ** This only happens once. If there is a second SQLITE_SCHEMA + ** error, the error will be returned to the caller. */ + p->zSql = zPrevSql; + continue; + } +#endif + Tcl_SetObjResult(pDb->interp, dbTextToObj(sqlite3_errmsg(pDb->db))); return TCL_ERROR; }else{ dbReleaseStmt(pDb, pPreStmt, 0); @@ -2926,7 +2988,7 @@ static int DbMain(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ }else{ flags &= ~SQLITE_OPEN_NOMUTEX; } - }else if( strcmp(zArg, "-fullmutex")==0 ){ + }else if( strcmp(zArg, "-fullmutex")==0 ){ int b; if( Tcl_GetBooleanFromObj(interp, objv[i+1], &b) ) return TCL_ERROR; if( b ){ @@ -3527,6 +3589,44 @@ static int init_all_cmd( init_all(slave); return TCL_OK; } + +/* +** Tclcmd: db_use_legacy_prepare DB BOOLEAN +** +** The first argument to this command must be a database command created by +** [sqlite3]. If the second argument is true, then the handle is configured +** to use the sqlite3_prepare_v2() function to prepare statements. If it +** is false, sqlite3_prepare(). +*/ +static int db_use_legacy_prepare_cmd( + ClientData cd, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + Tcl_CmdInfo cmdInfo; + SqliteDb *pDb; + int bPrepare; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB BOOLEAN"); + return TCL_ERROR; + } + + if( !Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &cmdInfo) ){ + Tcl_AppendResult(interp, "no such db: ", Tcl_GetString(objv[1]), (char*)0); + return TCL_ERROR; + } + pDb = (SqliteDb*)cmdInfo.objClientData; + if( Tcl_GetBooleanFromObj(interp, objv[2], &bPrepare) ){ + return TCL_ERROR; + } + + pDb->bLegacyPrepare = bPrepare; + + Tcl_ResetResult(interp); + return TCL_OK; +} #endif /* @@ -3633,7 +3733,12 @@ static void init_all(Tcl_Interp *interp){ Sqlitetestfts3_Init(interp); #endif - Tcl_CreateObjCommand(interp,"load_testfixture_extensions",init_all_cmd,0,0); + Tcl_CreateObjCommand( + interp, "load_testfixture_extensions", init_all_cmd, 0, 0 + ); + Tcl_CreateObjCommand( + interp, "db_use_legacy_prepare", db_use_legacy_prepare_cmd, 0, 0 + ); #ifdef SQLITE_SSE Sqlitetestsse_Init(interp); diff --git a/test/all.test b/test/all.test index 9671246681..54302d02e9 100644 --- a/test/all.test +++ b/test/all.test @@ -38,6 +38,7 @@ run_test_suite pcache10 run_test_suite pcache50 run_test_suite pcache90 run_test_suite pcache100 +run_test_suite prepare if {$::tcl_platform(platform)=="unix"} { ifcapable !default_autovacuum { diff --git a/test/alter2.test b/test/alter2.test index 712960c0ea..66b1950945 100644 --- a/test/alter2.test +++ b/test/alter2.test @@ -137,6 +137,7 @@ do_test alter2-1.8 { do_test alter2-1.9 { # ALTER TABLE abc ADD COLUMN d; alter_table abc {CREATE TABLE abc(a, b, c, d);} + if {[permutation] == "prepare"} { db cache flush } execsql { SELECT * FROM abc; } execsql { UPDATE abc SET d = 11 WHERE c IS NULL AND a<4; diff --git a/test/exists.test b/test/exists.test index c7c14a2402..fb73797d29 100644 --- a/test/exists.test +++ b/test/exists.test @@ -159,7 +159,7 @@ foreach jm {rollback wal} { sql1 { DROP INDEX IF EXISTS i2 } sql2 { CREATE INDEX aux.i2 ON t2(x) } sql1 { DROP INDEX IF EXISTS i2 } - sql2 { SELECT name FROM aux.sqlite_master WHERE type = 'index' } + sql2 { SELECT * FROM aux.sqlite_master WHERE type = 'index' } } {} # VIEW objects. diff --git a/test/permutations.test b/test/permutations.test index 22e2189d98..57f69462a5 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -776,6 +776,15 @@ test_suite "no_optimization" -description { optimization_control $::dbhandle all 0 } +test_suite "prepare" -description { + Run tests with the db connection using sqlite3_prepare() instead of _v2(). +} -dbconfig { + db_use_legacy_prepare $::dbhandle 1 + #$::dbhandle cache size 0 +} -files [ + test_set $allquicktests -exclude *malloc* *ioerr* *fault* +] + # End of tests ############################################################################# diff --git a/test/temptable.test b/test/temptable.test index ad85f7e9f6..d82ea36444 100644 --- a/test/temptable.test +++ b/test/temptable.test @@ -292,6 +292,7 @@ do_test temptable-5.1 { execsql { CREATE TEMP TABLE mask(a,b,c) } db2 + if {[permutation]=="prepare"} { db2 cache flush } execsql { CREATE INDEX mask ON t2(x); SELECT * FROM t2; From 2af0b7e0f455cadf7c222084da670d50b7526201 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 27 Jun 2011 19:12:57 +0000 Subject: [PATCH 11/52] Remove an unnecessary assignment from vdbeapi.c. FossilOrigin-Name: 25e5b7686a06e7f4ee993d8559d23143e2e8c7a8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeapi.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c54d53e491..ed26c9da13 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\simprove\sthe\sselection\sof\sdeferred\stokens\swithin\sphrases. -D 2011-06-27T11:15:53.752 +C Remove\san\sunnecessary\sassignment\sfrom\svdbeapi.c. +D 2011-06-27T19:12:57.742 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -241,7 +241,7 @@ F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e F src/vdbe.c 50b5b4ef29d928a4f60905f820ee1ab2f8ced29f F src/vdbe.h 5cf09e7ee8a3f7d93bc51f196a96550786afe7a1 F src/vdbeInt.h ad84226cc0adcb1185c22b70696b235a1678bb45 -F src/vdbeapi.c 2f908c65337b16c6249ab378d477e74492cf759b +F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c 0b2e2880f13af400a27c92a7673287c3eaec8b21 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b @@ -949,7 +949,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 8230d83120e0f4d217fde56e22c6f05aa5adee09 -R a3718a7f2e746e02b745366a3c6c0fd0 +P 2c4bbd90e2fca593c186bf412b608aff8c9f9061 +R 2d15c61ad82393e3871d39f558fa91b1 U dan -Z deeb565ce480cc0dfb49de44342bb4c6 +Z 368095b2328022a768848f2679ec1079 diff --git a/manifest.uuid b/manifest.uuid index b01b761298..0475acf58b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2c4bbd90e2fca593c186bf412b608aff8c9f9061 \ No newline at end of file +25e5b7686a06e7f4ee993d8559d23143e2e8c7a8 \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index a87b5c9acd..678f0d91e7 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -488,7 +488,7 @@ int sqlite3_step(sqlite3_stmt *pStmt){ && cnt++ < SQLITE_MAX_SCHEMA_RETRY && (rc2 = rc = sqlite3Reprepare(v))==SQLITE_OK ){ sqlite3_reset(pStmt); - v->expired = 0; + assert( v->expired==0 ); } if( rc2!=SQLITE_OK && ALWAYS(v->isPrepareV2) && ALWAYS(db->pErr) ){ /* This case occurs after failing to recompile an sql statement. From 08f5f4c57da24a9e4a468d20d17acc6ed802a666 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 27 Jun 2011 19:37:23 +0000 Subject: [PATCH 12/52] Remove an unnecessary assignment from vdbeapi.c. FossilOrigin-Name: 6c871ac1fa3c79baa0053dc49ab106cb3db4cfa0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeapi.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c734903830..5d2ec3b45d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\scase\sto\spermutations.test\sto\srun\sthe\sveryquick\stest\ssuite\susing\ssqlite3_prepare()\sinstead\sof\ssqlite3_prepare_v2().\sThis\shelps\sto\stest\sthe\sfix\sfor\sbug\s[25ee812710]. -D 2011-06-27T16:55:50.495 +C Remove\san\sunnecessary\sassignment\sfrom\svdbeapi.c. +D 2011-06-27T19:37:23.089 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -241,7 +241,7 @@ F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e F src/vdbe.c 50b5b4ef29d928a4f60905f820ee1ab2f8ced29f F src/vdbe.h 5cf09e7ee8a3f7d93bc51f196a96550786afe7a1 F src/vdbeInt.h ad84226cc0adcb1185c22b70696b235a1678bb45 -F src/vdbeapi.c 2f908c65337b16c6249ab378d477e74492cf759b +F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 F src/vdbeaux.c 4d100407e3c72e163854aff8903d19d5ecdf46c0 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3 F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b @@ -949,7 +949,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P faa38c8724d3eebad97197bfa44d11e75cfe6c69 -R ea65f158609dffa4f41c8ce1eb564414 +P d9f7993bfa2ecb30c44455de538eb6f8a4a59087 +R 883da57d7e06eb3695abd9be14247d70 U dan -Z 2df66e3ffb41532c6826925a7e3446a4 +Z c99a55a7e4b26c5d9d0bfe655fb88e62 diff --git a/manifest.uuid b/manifest.uuid index 4cd588aa86..f017ba332a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d9f7993bfa2ecb30c44455de538eb6f8a4a59087 \ No newline at end of file +6c871ac1fa3c79baa0053dc49ab106cb3db4cfa0 \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index a87b5c9acd..678f0d91e7 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -488,7 +488,7 @@ int sqlite3_step(sqlite3_stmt *pStmt){ && cnt++ < SQLITE_MAX_SCHEMA_RETRY && (rc2 = rc = sqlite3Reprepare(v))==SQLITE_OK ){ sqlite3_reset(pStmt); - v->expired = 0; + assert( v->expired==0 ); } if( rc2!=SQLITE_OK && ALWAYS(v->isPrepareV2) && ALWAYS(db->pErr) ){ /* This case occurs after failing to recompile an sql statement. From 0ea2517c4ed876406c665a91e33db3fcc26a9f13 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 28 Jun 2011 07:15:43 +0000 Subject: [PATCH 13/52] Changes to allow FTS to be compiled as a loadable module again. FossilOrigin-Name: 29e69f389cb0078e125ba5814a68b3fe8cf634c6 --- ext/fts3/fts3Int.h | 11 +++++++++-- ext/fts3/fts3_hash.c | 1 - ext/fts3/fts3_test.c | 3 +++ ext/fts3/fts3_tokenizer.c | 5 ----- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/loadext.c | 6 ++++++ src/sqlite3ext.h | 6 ++++++ 8 files changed, 36 insertions(+), 20 deletions(-) diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 1664379609..7e033a7e9f 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -27,7 +27,14 @@ # define SQLITE_ENABLE_FTS3 #endif -#ifdef SQLITE_ENABLE_FTS3 +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) + +/* If not building as part of the core, include sqlite3ext.h. */ +#ifndef SQLITE_CORE +# include "sqlite3ext.h" +extern const sqlite3_api_routines *sqlite3_api; +#endif + #include "sqlite3.h" #include "fts3_tokenizer.h" #include "fts3_hash.h" @@ -513,5 +520,5 @@ int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); -#endif /* SQLITE_ENABLE_FTS3 */ +#endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ #endif /* _FTSINT_H */ diff --git a/ext/fts3/fts3_hash.c b/ext/fts3/fts3_hash.c index b7c3e8b5a7..57c59b587a 100644 --- a/ext/fts3/fts3_hash.c +++ b/ext/fts3/fts3_hash.c @@ -30,7 +30,6 @@ #include #include -#include "sqlite3.h" #include "fts3_hash.h" /* diff --git a/ext/fts3/fts3_test.c b/ext/fts3/fts3_test.c index c5f72d84f4..72735f3d12 100644 --- a/ext/fts3/fts3_test.c +++ b/ext/fts3/fts3_test.c @@ -19,6 +19,8 @@ #include #include +#ifdef SQLITE_TEST + /* Required so that the "ifdef SQLITE_ENABLE_FTS3" below works */ #include "fts3Int.h" @@ -319,3 +321,4 @@ int Sqlitetestfts3_Init(Tcl_Interp *interp){ ); return TCL_OK; } +#endif /* ifdef SQLITE_TEST */ diff --git a/ext/fts3/fts3_tokenizer.c b/ext/fts3/fts3_tokenizer.c index c4ad6a5c75..6494bb96d8 100644 --- a/ext/fts3/fts3_tokenizer.c +++ b/ext/fts3/fts3_tokenizer.c @@ -23,12 +23,7 @@ ** * The FTS3 module is being built into the core of ** SQLite (in which case SQLITE_ENABLE_FTS3 is defined). */ -#include "sqlite3ext.h" -#ifndef SQLITE_CORE - SQLITE_EXTENSION_INIT1 -#endif #include "fts3Int.h" - #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) #include diff --git a/manifest b/manifest index 5d2ec3b45d..52cfcaf6e7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunnecessary\sassignment\sfrom\svdbeapi.c. -D 2011-06-27T19:37:23.089 +C Changes\sto\sallow\sFTS\sto\sbe\scompiled\sas\sa\sloadable\smodule\sagain. +D 2011-06-28T07:15:43.990 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -64,17 +64,17 @@ F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d F ext/fts3/fts3.c ca776037493d0081da70a6afc0df80f5ce347cb7 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h fa493ccbad78a2c99ad1c984f651c0c202e68536 +F ext/fts3/fts3Int.h 61c90f512b563345f9aa99c48c7afd0928de055c F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 F ext/fts3/fts3_expr.c 23791de01b3a5d313d76e02befd2601d4096bc2b -F ext/fts3/fts3_hash.c aad95afa01cf2a5ffaa448e4b0ab043880cd1efb +F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c 6c8f395cdf9e1e3afa7fadb7e523dbbf381c6dfa F ext/fts3/fts3_porter.c 8d946908f4812c005d3d33fcbe78418b1f4eb70c F ext/fts3/fts3_snippet.c 58b2ba2b934c1e2a2f6ac857d7f3c7e1a14b4532 F ext/fts3/fts3_term.c a5457992723455a58804cb75c8cbd8978db5c2ef -F ext/fts3/fts3_test.c c0aae3219df989d3e827d07846097adf8cb09945 -F ext/fts3/fts3_tokenizer.c 6089986ebcfc19d4b7aabd2b92773368efa4354f +F ext/fts3/fts3_test.c 24fa13f330db011500acb95590da9eee24951894 +F ext/fts3/fts3_tokenizer.c 9ff7ec66ae3c5c0340fa081958e64f395c71a106 F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 0dde8f307b8045565cf63797ba9acfaff1c50c68 F ext/fts3/fts3_write.c 194829c8fd024a448fc899e5ff02a8ed06595529 @@ -145,7 +145,7 @@ F src/insert.c 3eea5a53d2644116fb865afaa4699fabe62b441c F src/journal.c 552839e54d1bf76fb8f7abe51868b66acacf6a0e F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e -F src/loadext.c 3ae0d52da013a6326310655be6473fd472347b85 +F src/loadext.c 99a161b27a499fc8ad40745b7b1900a26f0a5f51 F src/main.c fa654e1802e14a5f6c09c37971c3b69c1001d9c8 F src/malloc.c 591aedb20ae40813f1045f2ef253438a334775d9 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -182,7 +182,7 @@ F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c d9d440809025a58547e39f4f268c2a296bfb56ff F src/shell.c 0e0173b3e79d956368013e759f084caa7995ecb1 F src/sqlite.h.in 4b7255c10d39c5faf089dbd29cde7c367ff39f1f -F src/sqlite3ext.h c90bd5507099f62043832d73f6425d8d5c5da754 +F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 F src/sqliteInt.h 7b72f7c6929757c0678d94524dbc7e7e3d6dc398 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf @@ -949,7 +949,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P d9f7993bfa2ecb30c44455de538eb6f8a4a59087 -R 883da57d7e06eb3695abd9be14247d70 +P 6c871ac1fa3c79baa0053dc49ab106cb3db4cfa0 +R 4e5d4c059d7e31bcf35d2d245a2997b7 U dan -Z c99a55a7e4b26c5d9d0bfe655fb88e62 +Z c2844ce2a2f8548d3356adfa633cbd5b diff --git a/manifest.uuid b/manifest.uuid index f017ba332a..b82e4229ca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6c871ac1fa3c79baa0053dc49ab106cb3db4cfa0 \ No newline at end of file +29e69f389cb0078e125ba5814a68b3fe8cf634c6 \ No newline at end of file diff --git a/src/loadext.c b/src/loadext.c index c832e684ef..079458d143 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -84,6 +84,8 @@ # define sqlite3_create_module 0 # define sqlite3_create_module_v2 0 # define sqlite3_declare_vtab 0 +# define sqlite3_vtab_config 0 +# define sqlite3_vtab_on_conflict 0 #endif #ifdef SQLITE_OMIT_SHARED_CACHE @@ -107,6 +109,7 @@ #define sqlite3_blob_open 0 #define sqlite3_blob_read 0 #define sqlite3_blob_write 0 +#define sqlite3_blob_reopen 0 #endif /* @@ -372,6 +375,9 @@ static const sqlite3_api_routines sqlite3Apis = { 0, 0, #endif + sqlite3_blob_reopen, + sqlite3_vtab_config, + sqlite3_vtab_on_conflict, }; /* diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index e45e691fd6..50dd5b6055 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -212,6 +212,9 @@ struct sqlite3_api_routines { int (*wal_autocheckpoint)(sqlite3*,int); int (*wal_checkpoint)(sqlite3*,const char*); void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*); + int (*blob_reopen)(sqlite3_blob*,sqlite3_int64); + int (*vtab_config)(sqlite3*,int op,...); + int (*vtab_on_conflict)(sqlite3*); }; /* @@ -412,6 +415,9 @@ struct sqlite3_api_routines { #define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint #define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint #define sqlite3_wal_hook sqlite3_api->wal_hook +#define sqlite3_blob_reopen sqlite3_api->blob_reopen +#define sqlite3_vtab_config sqlite3_api->vtab_config +#define sqlite3_vtab_on_conflict sqlite3_api->vtab_on_conflict #endif /* SQLITE_CORE */ #define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0; From da7b45615d2ddd4b528f18fdeabab1d263dba228 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 28 Jun 2011 11:58:09 +0000 Subject: [PATCH 14/52] Add a fix and tests for the FTS deferred token logic. FossilOrigin-Name: 91daea7d2ec41f014fb30c6371aae447cc07f287 --- ext/fts3/fts3.c | 74 ++++++++++++++++++++++++++++------------------ manifest | 14 ++++----- manifest.uuid | 2 +- test/fts3auto.test | 52 ++++++++++++++++++++++++++++++-- 4 files changed, 103 insertions(+), 39 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 16067aacc7..448a5a403a 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3864,12 +3864,14 @@ static int fts3EvalSelectDeferred( ){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int nDocSize = 0; /* Number of pages per doc loaded */ - int nDocEst = 0; /* Est. docs if all other tokens deferred */ int rc = SQLITE_OK; /* Return code */ int ii; /* Iterator variable for various purposes */ int nOvfl = 0; /* Total overflow pages used by doclists */ int nToken = 0; /* Total number of tokens in cluster */ + int nMinEst = 0; /* The minimum count for any phrase so far. */ + int nLoad4 = 1; /* (Phrases that will be loaded)^4. */ + /* Count the tokens in this AND/NEAR cluster. If none of the doclists ** associated with the tokens spill onto overflow pages, or if there is ** only 1 token, exit early. No tokens to defer in this case. */ @@ -3885,6 +3887,29 @@ static int fts3EvalSelectDeferred( rc = fts3EvalAverageDocsize(pCsr, &nDocSize); assert( rc!=SQLITE_OK || nDocSize>0 ); + + /* Iterate through all tokens in this AND/NEAR cluster, in ascending order + ** of the number of overflow pages that will be loaded by the pager layer + ** to retrieve the entire doclist for the token from the full-text index. + ** Load the doclists for tokens that are either: + ** + ** a. The cheapest token in the entire query (i.e. the one visited by the + ** first iteration of this loop), or + ** + ** b. Part of a multi-token phrase. + ** + ** After each token doclist is loaded, merge it with the others from the + ** same phrase and count the number of documents that the merged doclist + ** contains. Set variable "nMinEst" to the smallest number of documents in + ** any phrase doclist for which 1 or more token doclists have been loaded. + ** Let nOther be the number of other phrases for which it is certain that + ** one or more tokens will not be deferred. + ** + ** Then, for each token, defer it if loading the doclist would result in + ** loading N or more overflow pages into memory, where N is computed as: + ** + ** (nMinEst + 4^nOther - 1) / (4^nOther) + */ for(ii=0; iinOvfl>=(nDocEst*nDocSize) ){ + if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){ /* The number of overflow pages to load for this (and therefore all ** subsequent) tokens is greater than the estimated number of pages ** that will be loaded if all subsequent tokens are deferred. @@ -3915,26 +3933,26 @@ static int fts3EvalSelectDeferred( rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); fts3SegReaderCursorFree(pToken->pSegcsr); pToken->pSegcsr = 0; - }else if( ii==0 || pTC->pPhrase->nToken>1 ){ - /* Either this is the cheapest token in the entire query, or it is - ** part of a multi-token phrase. Either way, the entire doclist will - ** (eventually) be loaded into memory. It may as well be now. */ - Fts3PhraseToken *pToken = pTC->pToken; - int nList = 0; - char *pList = 0; - rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); - assert( rc==SQLITE_OK || pList==0 ); - if( rc==SQLITE_OK ){ - fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList); - nDocEst = fts3DoclistCountDocids( - pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll - ); + }else{ + nLoad4 = nLoad4*4; + if( ii==0 || pTC->pPhrase->nToken>1 ){ + /* Either this is the cheapest token in the entire query, or it is + ** part of a multi-token phrase. Either way, the entire doclist will + ** (eventually) be loaded into memory. It may as well be now. */ + Fts3PhraseToken *pToken = pTC->pToken; + int nList = 0; + char *pList = 0; + rc = fts3TermSelect(pTab, pToken, pTC->iCol, &nList, &pList); + assert( rc==SQLITE_OK || pList==0 ); + if( rc==SQLITE_OK ){ + int nCount; + fts3EvalPhraseMergeToken(pTab, pTC->pPhrase, pTC->iToken,pList,nList); + nCount = fts3DoclistCountDocids( + pTC->pPhrase->doclist.aAll, pTC->pPhrase->doclist.nAll + ); + if( ii==0 || nCountpToken = 0; } diff --git a/manifest b/manifest index 57c9073015..f8bc9380fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\swith\sfts3-changes\sbranch. -D 2011-06-28T09:51:22.803 +C Add\sa\sfix\sand\stests\sfor\sthe\sFTS\sdeferred\stoken\slogic. +D 2011-06-28T11:58:09.194 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -62,7 +62,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 70127bf57c54c45545a4ddd73161caf2d9bd1abe +F ext/fts3/fts3.c f45ad45053a587ad1c005459b704b7ade8bd504e F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h e670e2c58b3dca76e44f7e7d35e61113d079db58 F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 @@ -456,7 +456,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3 -F test/fts3auto.test f1cb0a55130897013ca5850dbee2945c2908a45a +F test/fts3auto.test c1a30b37002b7c764a96937fbc71065b73d69494 F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 @@ -949,7 +949,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 11ea98e454ad35b2606e81dec9ac866e4dd9f7e1 29e69f389cb0078e125ba5814a68b3fe8cf634c6 -R 99f54668f5b612c34e36e4280019aae3 +P 226686475c28cdc87e86d2d3f10014436a9a2b8f +R 26c661c69e309ad434e75bfd241a096d U dan -Z c4e55cbac13868c8cb421abab69674f2 +Z 1b1eba0343676ae88643e1d9f2f985a9 diff --git a/manifest.uuid b/manifest.uuid index 8ca57f2fa7..0e9f22faf1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -226686475c28cdc87e86d2d3f10014436a9a2b8f \ No newline at end of file +91daea7d2ec41f014fb30c6371aae447cc07f287 \ No newline at end of file diff --git a/test/fts3auto.test b/test/fts3auto.test index 484fd6ace2..1c58a17204 100644 --- a/test/fts3auto.test +++ b/test/fts3auto.test @@ -107,15 +107,17 @@ proc do_fts3query_test {tn args} { " $matchinfo_asc } -# fts3_make_deferrable TABLE TOKEN +# fts3_make_deferrable TABLE TOKEN ?NROW? # -proc fts3_make_deferrable {tbl token} { +proc fts3_make_deferrable {tbl token {nRow 0}} { set stmt [sqlite3_prepare db "SELECT * FROM $tbl" -1 dummy] set name [sqlite3_column_name $stmt 0] sqlite3_finalize $stmt - set nRow [db one "SELECT count(*) FROM $tbl"] + if {$nRow==0} { + set nRow [db one "SELECT count(*) FROM $tbl"] + } set pgsz [db one "PRAGMA page_size"] execsql BEGIN for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} { @@ -653,6 +655,50 @@ foreach {tn pending create} { catchsql { COMMIT } } +foreach {tn create} { + 1 "fts4(x)" + 2 "fts4(x, order=DESC)" +} { + execsql [subst { + DROP TABLE IF EXISTS t1; + CREATE VIRTUAL TABLE t1 USING $create; + }] + + foreach {x} { + "F E N O T K X V A X I E X A P G Q V H U" + "R V A E T C V Q N I E L O N U G J K L U" + "U Y I G W M V F J L X I D C H F P J Q B" + "S G D Z X R P G S S Y B K A S G A I L L" + "L S I C H T Z S R Q P R N K J X L F M J" + "C C C D P X B Z C M A D A C X S B T X V" + "W Y J M D R G V R K B X S A W R I T N C" + "P K L W T M S P O Y Y V V O E H Q A I R" + "C D Y I C Z F H J C O Y A Q F L S B D K" + "P G S C Y C Y V I M B D S Z D D Y W I E" + "Z K Z U E E S F Y X T U A L W O U J C Q" + "P A T Z S W L P L Q V Y Y I P W U X S S" + "I U I H U O F Z F R H R F T N D X A G M" + "N A B M S H K X S O Y D T X S B R Y H Z" + "L U D A S K I L S V Z J P U B E B Y H M" + } { + execsql { INSERT INTO t1 VALUES($x) } + } + + # Add extra documents to the database such that token "B" will be considered + # deferrable if considering the other tokens means that 2 or fewer documents + # will be loaded into memory. + # + fts3_make_deferrable t1 B 2 + + # B is not deferred in either of the first two tests below, since filtering + # on "M" or "D" returns 10 documents or so. But filtering on "M * D" only + # returns 2, so B is deferred in this case. + # + do_fts3query_test 7.$tn.1 t1 {"M B"} + do_fts3query_test 7.$tn.2 t1 {"B D"} + do_fts3query_test 7.$tn.3 -deferred B t1 {"M B D"} +} + set sqlite_fts3_enable_parentheses $sfep finish_test From e1b4f0f177406df827af3ae30b2209cb7e682ef7 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 29 Jun 2011 17:11:39 +0000 Subject: [PATCH 15/52] Pass the BTREE_UNORDERED hint into both sqlite3BtreeOpen() and into sqlite3BtreeCreateTable(). FossilOrigin-Name: 591de898f41630156cc0fc6ef17dd3ee5e7c479f --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 2109443aae..fe85fe4d38 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sfts3-changes\sbranch\sback\sinto\sthe\strunk. -D 2011-06-28T14:16:42.198 +C Pass\sthe\sBTREE_UNORDERED\shint\sinto\sboth\ssqlite3BtreeOpen()\sand\sinto\nsqlite3BtreeCreateTable(). +D 2011-06-29T17:11:39.745 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -238,7 +238,7 @@ F src/update.c 80d77311d91ebc06b27149e75701f1b3e9356622 F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 F src/util.c 0f33bbbdfcc4a2d8cf20c3b2a16ffc3b57c58a70 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e -F src/vdbe.c 50b5b4ef29d928a4f60905f820ee1ab2f8ced29f +F src/vdbe.c a9ced64f380bbd8b04da3a1c3a9602d3942704b5 F src/vdbe.h 5cf09e7ee8a3f7d93bc51f196a96550786afe7a1 F src/vdbeInt.h ad84226cc0adcb1185c22b70696b235a1678bb45 F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98 @@ -949,7 +949,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 29e69f389cb0078e125ba5814a68b3fe8cf634c6 91daea7d2ec41f014fb30c6371aae447cc07f287 -R 26c661c69e309ad434e75bfd241a096d -U dan -Z 7f90c6a2312621252bb79173b43d611c +P b9477eb056d120826ed82b0d65e6f27b5d0c087a +R 04bb3d810ba550780b3b6fda2c65d8d3 +U drh +Z 9a15d45fcfccc0dc2281a762dda887ad diff --git a/manifest.uuid b/manifest.uuid index cfd8608c10..e1c5a804ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b9477eb056d120826ed82b0d65e6f27b5d0c087a \ No newline at end of file +591de898f41630156cc0fc6ef17dd3ee5e7c479f \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index d965b967a3..bec422a988 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3172,7 +3172,7 @@ case OP_OpenEphemeral: { if( pOp->p4.pKeyInfo ){ int pgno; assert( pOp->p4type==P4_KEYINFO ); - rc = sqlite3BtreeCreateTable(pCx->pBt, &pgno, BTREE_BLOBKEY); + rc = sqlite3BtreeCreateTable(pCx->pBt, &pgno, BTREE_BLOBKEY | pOp->p5); if( rc==SQLITE_OK ){ assert( pgno==MASTER_ROOT+1 ); rc = sqlite3BtreeCursor(pCx->pBt, pgno, 1, From 38cc40c216f080dd309cb47f9e7fc976ba4fc2dd Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 30 Jun 2011 20:17:15 +0000 Subject: [PATCH 16/52] Experimental changes to improve optimization of DISTINCT queries. FossilOrigin-Name: f7ba0219ef2f235543c258be736955d91ca5ecce --- manifest | 39 +++++++------ manifest.uuid | 2 +- src/delete.c | 4 +- src/fkey.c | 2 +- src/select.c | 65 ++++++++++++++++----- src/sqliteInt.h | 6 +- src/update.c | 4 +- src/where.c | 137 +++++++++++++++++++++++++++++++++++++++------ test/collate5.test | 6 +- test/e_select.test | 12 ++-- test/fuzzer1.test | 2 +- test/insert4.test | 2 +- test/misc5.test | 2 +- test/selectB.test | 2 +- 14 files changed, 219 insertions(+), 66 deletions(-) diff --git a/manifest b/manifest index fe85fe4d38..4197d1e43f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Pass\sthe\sBTREE_UNORDERED\shint\sinto\sboth\ssqlite3BtreeOpen()\sand\sinto\nsqlite3BtreeCreateTable(). -D 2011-06-29T17:11:39.745 +C Experimental\schanges\sto\simprove\soptimization\sof\sDISTINCT\squeries. +D 2011-06-30T20:17:15.042 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -132,10 +132,10 @@ F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 7deec4534f3b5a0c3b4a4cbadf809d321f64f9c4 F src/date.c a3c6842bad7ae632281811de112a8ba63ff08ab3 -F src/delete.c 4925f9121525fc871f5d8d13c1f7dcc91abb38bb +F src/delete.c ff68e5ef23aee08c0ff528f699a19397ed8bbed8 F src/expr.c ab46ab0f0c44979a8164ca31728d7d10ae5e8106 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb -F src/fkey.c 9fabba17a4d4778dc660f0cb9d781fc86d7b9d41 +F src/fkey.c c8492fed772af1ed61251582707266227612b45b F src/func.c 59bb046d7e3df1ab512ac339ccb0a6f996a17cb7 F src/global.c c70a46f28680f8d7c097dbc0430ccf3b932e90b0 F src/hash.c 458488dcc159c301b8e7686280ab209f1fb915af @@ -179,11 +179,11 @@ F src/printf.c 585a36b6a963df832cfb69505afa3a34ed5ef8a1 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 1c0f32b64f8e3f555fe1f732f9d6f501a7f05706 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 -F src/select.c d9d440809025a58547e39f4f268c2a296bfb56ff +F src/select.c cf259606f91f53f71bafd7a2da47f30cda6efc58 F src/shell.c 0e0173b3e79d956368013e759f084caa7995ecb1 F src/sqlite.h.in 4b7255c10d39c5faf089dbd29cde7c367ff39f1f F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 -F src/sqliteInt.h 7b72f7c6929757c0678d94524dbc7e7e3d6dc398 +F src/sqliteInt.h 58df570a42d605f8e53d6eb920a16ca291d5822e F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -234,7 +234,7 @@ F src/test_wholenumber.c 6129adfbe7c7444f2e60cc785927f3aa74e12290 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c c819d9f72168a035d545a5bdafe9b085b20df705 F src/trigger.c c836a6caac16ba96611558922106858f6ca3d6bf -F src/update.c 80d77311d91ebc06b27149e75701f1b3e9356622 +F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 F src/util.c 0f33bbbdfcc4a2d8cf20c3b2a16ffc3b57c58a70 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e @@ -250,7 +250,7 @@ F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 0c70ad7b1cac6005fa5e2cbefd23ee05e391c290 F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f -F src/where.c 55403ce19c506be6a321c7f129aff693d6103db5 +F src/where.c 0bdcf6704dd0d8471052712e1d63a02990b7d0bf F test/8_3_names.test b93687beebd17f6ebf812405a6833bae5d1f4199 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 @@ -322,7 +322,7 @@ F test/collate1.test e3eaa48c21e150814be1a7b852d2a8af24458d04 F test/collate2.test 04cebe4a033be319d6ddbb3bbc69464e01700b49 F test/collate3.test d28d2cfab2c3a3d4628ae4b2b7afc9965daa3b4c F test/collate4.test 3d3f123f83fd8ccda6f48d617e44e661b9870c7d -F test/collate5.test fe0f43c4740d7b71b959cac668d19e42f2e06e4d +F test/collate5.test 67f1d3e848e230ff4802815a79acb0a8b5e69bd7 F test/collate6.test 8be65a182abaac8011a622131486dafb8076e907 F test/collate7.test fac8db7aac3978466c04ae892cc74dcf2bc031aa F test/collate8.test df26649cfcbddf109c04122b340301616d3a88f6 @@ -379,7 +379,7 @@ F test/e_fts3.test 75bb0aee26384ef586165e21018a17f7cd843469 F test/e_insert.test 76d4bb5da9b28014d515d91ffe29a79a1e99f2bc F test/e_reindex.test a064f0878b8f848fbca38f1f61f82f15a3000c64 F test/e_resolve.test dcce9308fb13b934ce29591105d031d3e14fbba6 -F test/e_select.test 7ac53674e822d4d77bbb4a9a4aaefa5fdc9e493f +F test/e_select.test 8d7fac7a268eaeb80b9a7ba7964505b9d30f5458 F test/e_select2.test 5c3d3da19c7b3e90ae444579db2b70098599ab92 F test/e_update.test b926341a65955d69a6375c9eb4fd82e7089bc83a F test/e_uri.test 6f35b491f80dac005c8144f38b2dfb4d96483596 @@ -491,7 +491,7 @@ F test/fuzz2.test 207d0f9d06db3eaf47a6b7bfc835b8e2fc397167 F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test dd7001ac86d09c154a7dff064f4739c60e2b312c -F test/fuzzer1.test 3105b5a89a6cb0d475f0877debec942fe4143462 +F test/fuzzer1.test ddfb04f3bd5cfdda3b1aa15b78d3ad055c9cc50f F test/hook.test f2277c309e4ee8067d95d6b9b315568e9d5329b2 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 19b642bb134308980a92249750ea4ce3f6c75c2d @@ -514,7 +514,7 @@ F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 F test/insert.test aef273dd1cee84cc92407469e6bd1b3cdcb76908 F test/insert2.test 4f3a04d168c728ed5ec2c88842e772606c7ce435 F test/insert3.test 1b7db95a03ad9c5013fdf7d6722b6cd66ee55e30 -F test/insert4.test b3e02648a5fc3075c29e13c369b5127bf859b5a2 +F test/insert4.test 63ea672b0fc6d3a9a0ccee774a771510b1e684c4 F test/insert5.test 1f93cbe9742110119133d7e8e3ccfe6d7c249766 F test/intarray.test 066b7d7ac38d25bf96f87f1b017bfc687551cdd4 F test/interrupt.test 42e7cf98646fd9cb4a3b131a93ed3c50b9e149f1 @@ -590,7 +590,7 @@ F test/misc1.test e56baf44656dd68d6475a4b44521045a60241e9b F test/misc2.test a628db7b03e18973e5d446c67696b03de718c9fd F test/misc3.test 72c5dc87a78e7865c5ec7a969fc572913dbe96b6 F test/misc4.test 9c078510fbfff05a9869a0b6d8b86a623ad2c4f6 -F test/misc5.test 45b2e3ed5f79af2b4f38ae362eaf4c49674575bd +F test/misc5.test 9f9338f8211c7f5d1cbe16331fa65d019501aa50 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test 29032efcd3d826fbd409e2a7af873e7939f4a4e3 F test/misuse.test 30b3a458e5a70c31e74c291937b6c82204c59f33 @@ -655,7 +655,7 @@ F test/select7.test dad6f00f0d49728a879d6eb6451d4752db0b0abe F test/select8.test 391de11bdd52339c30580dabbbbe97e3e9a3c79d F test/select9.test 74c0fb2c6eecb0219cbed0cbe3df136f8fbf9343 F test/selectA.test 06d1032fa9009314c95394f2ca2e60d9f7ae8532 -F test/selectB.test f305cc6660804cb239aab4e2f26b0e288b59958b +F test/selectB.test 0d072c5846071b569766e6cd7f923f646a8b2bfa F test/selectC.test f9bf1bc4581b5b8158caa6e4e4f682acb379fb25 F test/server1.test f5b790d4c0498179151ca8a7715a65a7802c859c F test/shared.test b9114eaea7e748a3a4c8ff7b9ca806c8f95cef3e @@ -949,7 +949,10 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P b9477eb056d120826ed82b0d65e6f27b5d0c087a -R 04bb3d810ba550780b3b6fda2c65d8d3 -U drh -Z 9a15d45fcfccc0dc2281a762dda887ad +P 591de898f41630156cc0fc6ef17dd3ee5e7c479f +R 5673acf06c190a75a743da9daebc8ac9 +T *branch * experimental +T *sym-experimental * +T -sym-trunk * +U dan +Z ab52fc5a81a2f6dd6b96daa429077988 diff --git a/manifest.uuid b/manifest.uuid index e1c5a804ad..5aa8e13950 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -591de898f41630156cc0fc6ef17dd3ee5e7c479f \ No newline at end of file +f7ba0219ef2f235543c258be736955d91ca5ecce \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index ec0ecec7e2..147a5ca898 100644 --- a/src/delete.c +++ b/src/delete.c @@ -371,7 +371,9 @@ void sqlite3DeleteFrom( /* Collect rowids of every row to be deleted. */ sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0,WHERE_DUPLICATES_OK); + pWInfo = sqlite3WhereBegin( + pParse, pTabList, pWhere, 0, 0, WHERE_DUPLICATES_OK + ); if( pWInfo==0 ) goto delete_from_cleanup; regRowid = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, iRowid); sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, regRowid); diff --git a/src/fkey.c b/src/fkey.c index dda8a50d44..37d4744dda 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -560,7 +560,7 @@ static void fkScanChildren( ** clause. If the constraint is not deferred, throw an exception for ** each row found. Otherwise, for deferred constraints, increment the ** deferred constraint counter by nIncr for each row selected. */ - pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0); + pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0); if( nIncr>0 && pFKey->isDeferred==0 ){ sqlite3ParseToplevel(pParse)->mayAbort = 1; } diff --git a/src/select.c b/src/select.c index 8dcfb84b8a..00107e1de6 100644 --- a/src/select.c +++ b/src/select.c @@ -3721,6 +3721,7 @@ int sqlite3Select( int distinct; /* Table to use for the distinct set */ int rc = 1; /* Value to return from this function */ int addrSortIndex; /* Address of an OP_OpenEphemeral instruction */ + int addrDistinctIndex; /* Address of an OP_OpenEphemeral instruction */ AggInfo sAggInfo; /* Information used by aggregate queries */ int iEnd; /* Address of the end of the query */ sqlite3 *db; /* The database connection */ @@ -3850,12 +3851,14 @@ int sqlite3Select( /* If possible, rewrite the query to use GROUP BY instead of DISTINCT. ** GROUP BY might use an index, DISTINCT never does. */ +#if 0 assert( p->pGroupBy==0 || (p->selFlags & SF_Aggregate)!=0 ); if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ){ p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); pGroupBy = p->pGroupBy; p->selFlags &= ~SF_Distinct; } +#endif /* If there is both a GROUP BY and an ORDER BY clause and they are ** identical, then disable the ORDER BY clause since the GROUP BY @@ -3904,11 +3907,10 @@ int sqlite3Select( */ if( p->selFlags & SF_Distinct ){ KeyInfo *pKeyInfo; - assert( isAgg || pGroupBy ); distinct = pParse->nTab++; pKeyInfo = keyInfoFromExprList(pParse, p->pEList); - sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0, - (char*)pKeyInfo, P4_KEYINFO_HANDOFF); + addrDistinctIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, distinct, 0, 0, + (char*)pKeyInfo, P4_KEYINFO_HANDOFF); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); }else{ distinct = -1; @@ -3916,10 +3918,10 @@ int sqlite3Select( /* Aggregate and non-aggregate queries are handled differently */ if( !isAgg && pGroupBy==0 ){ - /* This case is for non-aggregate queries - ** Begin the database scan - */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, 0); + ExprList *pDist = (isDistinct ? p->pEList : 0); + + /* Begin the database scan. */ + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pOrderBy, pDist, 0); if( pWInfo==0 ) goto select_end; if( pWInfo->nRowOut < p->nSelectRow ) p->nSelectRow = pWInfo->nRowOut; @@ -3932,10 +3934,47 @@ int sqlite3Select( p->addrOpenEphm[2] = -1; } - /* Use the standard inner loop - */ - assert(!isDistinct); - selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, -1, pDest, + if( pWInfo->eDistinct ){ + assert( isDistinct ); + assert( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED + || pWInfo->eDistinct==WHERE_DISTINCT_UNIQUE + ); + distinct = -1; + if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED ){ + int iJump; + int iExpr; + int iFlag = ++pParse->nMem; + int iBase = pParse->nMem+1; + int iBase2 = iBase + pEList->nExpr; + pParse->nMem += (pEList->nExpr*2); + VdbeOp *pOp; + + /* Change the OP_OpenEphemeral coded earlier to an OP_Integer. The + ** OP_Integer initializes the "first row" flag. */ + pOp = sqlite3VdbeGetOp(v, addrDistinctIndex); + pOp->opcode = OP_Integer; + pOp->p1 = 1; + pOp->p2 = iFlag; + + sqlite3ExprCodeExprList(pParse, pEList, iBase, 1); + iJump = sqlite3VdbeCurrentAddr(v) + 1 + pEList->nExpr + 1 + 1; + sqlite3VdbeAddOp2(v, OP_If, iFlag, iJump-1); + for(iExpr=0; iExprnExpr; iExpr++){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pEList->a[iExpr].pExpr); + sqlite3VdbeAddOp3(v, OP_Ne, iBase+iExpr, iJump, iBase2+iExpr); + sqlite3VdbeChangeP4(v, -1, (const char *)pColl, P4_COLLSEQ); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + } + sqlite3VdbeAddOp2(v, OP_Goto, 0, pWInfo->iContinue); + + sqlite3VdbeAddOp2(v, OP_Integer, 0, iFlag); + assert( sqlite3VdbeCurrentAddr(v)==iJump ); + sqlite3VdbeAddOp3(v, OP_Move, iBase, iBase2, pEList->nExpr); + } + } + + /* Use the standard inner loop. */ + selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, distinct, pDest, pWInfo->iContinue, pWInfo->iBreak); /* End the database scan loop. @@ -4045,7 +4084,7 @@ int sqlite3Select( ** in the right order to begin with. */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pGroupBy, 0, 0); if( pWInfo==0 ) goto select_end; if( pGroupBy==0 ){ /* The optimizer is able to deliver rows in group by order so @@ -4307,7 +4346,7 @@ int sqlite3Select( ** of output. */ resetAccumulator(pParse, &sAggInfo); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, flag); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, &pMinMax, 0, flag); if( pWInfo==0 ){ sqlite3ExprListDelete(db, pDel); goto select_end; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 09f64b7939..c1e055b267 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1966,6 +1966,7 @@ struct WhereInfo { u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE or DELETE */ u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ + u8 eDistinct; SrcList *pTabList; /* List of tables in the join */ int iTop; /* The very beginning of the WHERE loop */ int iContinue; /* Jump here to continue with next record */ @@ -1977,6 +1978,9 @@ struct WhereInfo { WhereLevel a[1]; /* Information about each nest loop in WHERE */ }; +#define WHERE_DISTINCT_UNIQUE 1 +#define WHERE_DISTINCT_ORDERED 2 + /* ** A NameContext defines a context in which to resolve table and column ** names. The context consists of a list of tables (the pSrcList) field and @@ -2738,7 +2742,7 @@ Expr *sqlite3LimitWhere(Parse *, SrcList *, Expr *, ExprList *, Expr *, Expr *, #endif void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); -WhereInfo *sqlite3WhereBegin(Parse*, SrcList*, Expr*, ExprList**, u16); +WhereInfo *sqlite3WhereBegin(Parse*, SrcList*, Expr*, ExprList**,ExprList*,u16); void sqlite3WhereEnd(WhereInfo*); int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int); void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); diff --git a/src/update.c b/src/update.c index d445f01f4e..2b5efff37e 100644 --- a/src/update.c +++ b/src/update.c @@ -311,7 +311,9 @@ void sqlite3Update( /* Begin the database scan */ sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0, WHERE_ONEPASS_DESIRED); + pWInfo = sqlite3WhereBegin( + pParse, pTabList, pWhere, 0, 0, WHERE_ONEPASS_DESIRED + ); if( pWInfo==0 ) goto update_cleanup; okOnePass = pWInfo->okOnePass; diff --git a/src/where.c b/src/where.c index cf30d94d67..8aecf9788d 100644 --- a/src/where.c +++ b/src/where.c @@ -253,6 +253,7 @@ struct WhereCost { #define WHERE_VIRTUALTABLE 0x08000000 /* Use virtual-table processing */ #define WHERE_MULTI_OR 0x10000000 /* OR using multiple indices */ #define WHERE_TEMP_INDEX 0x20000000 /* Uses an ephemeral index */ +#define WHERE_DISTINCT 0x40000000 /* Correct order for DISTINCT */ /* ** Initialize a preallocated WhereClause structure. @@ -1433,7 +1434,10 @@ static int isSortingIndex( struct ExprList_item *pTerm; /* A term of the ORDER BY clause */ sqlite3 *db = pParse->db; - assert( pOrderBy!=0 ); + if( !pOrderBy ) return 0; + if( wsFlags & WHERE_COLUMN_IN ) return 0; + if( pIdx->bUnordered ) return 0; + nTerm = pOrderBy->nExpr; assert( nTerm>0 ); @@ -1523,7 +1527,7 @@ static int isSortingIndex( } } - *pbRev = sortOrder!=0; + if( pbRev ) *pbRev = sortOrder!=0; if( j>=nTerm ){ /* All terms of the ORDER BY clause are covered by this index so ** this index can be used for sorting. */ @@ -2689,6 +2693,7 @@ static void bestBtreeIndex( Bitmask notReady, /* Mask of cursors not available for indexing */ Bitmask notValid, /* Cursors not available for any purpose */ ExprList *pOrderBy, /* The ORDER BY clause */ + ExprList *pDistinct, /* The select-list if query is DISTINCT */ WhereCost *pCost /* Lowest cost query plan */ ){ int iCur = pSrc->iCursor; /* The cursor of the table to be accessed */ @@ -2829,7 +2834,8 @@ static void bestBtreeIndex( int nInMul = 1; /* Number of distinct equalities to lookup */ int estBound = 100; /* Estimated reduction in search space */ int nBound = 0; /* Number of range constraints seen */ - int bSort = 0; /* True if external sort required */ + int bSort = !!pOrderBy; /* True if external sort required */ + int bDist = !!pDistinct; /* True if index cannot help with DISTINCT */ int bLookup = 0; /* True if not a covering index */ WhereTerm *pTerm; /* A single term of the WHERE clause */ #ifdef SQLITE_ENABLE_STAT2 @@ -2893,17 +2899,22 @@ static void bestBtreeIndex( ** naturally scan rows in the required order, set the appropriate flags ** in wsFlags. Otherwise, if there is an ORDER BY clause but the index ** will scan rows in a different order, set the bSort variable. */ - if( pOrderBy ){ - if( (wsFlags & WHERE_COLUMN_IN)==0 - && pProbe->bUnordered==0 - && isSortingIndex(pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy, - nEq, wsFlags, &rev) - ){ - wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY; - wsFlags |= (rev ? WHERE_REVERSE : 0); - }else{ - bSort = 1; - } + if( isSortingIndex( + pParse, pWC->pMaskSet, pProbe, iCur, pOrderBy, nEq, wsFlags, &rev) + ){ + bSort = 0; + wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_ORDERBY; + wsFlags |= (rev ? WHERE_REVERSE : 0); + } + + /* If there is a DISTINCT qualifier and this index will scan rows in + ** order of the DISTINCT expressions, clear bDist and set the appropriate + ** flags in wsFlags. */ + if( isSortingIndex( + pParse, pWC->pMaskSet, pProbe, iCur, pDistinct, nEq, wsFlags, 0) + ){ + bDist = 0; + wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_DISTINCT; } /* If currently calculating the cost of using an index (not the IPK @@ -3020,6 +3031,9 @@ static void bestBtreeIndex( if( bSort ){ cost += nRow*estLog(nRow)*3; } + if( bDist ){ + cost += nRow*estLog(nRow)*3; + } /**** Cost of using this index has now been computed ****/ @@ -3165,7 +3179,7 @@ static void bestIndex( }else #endif { - bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, pCost); + bestBtreeIndex(pParse, pWC, pSrc, notReady, notValid, pOrderBy, 0, pCost); } } @@ -4127,7 +4141,7 @@ static Bitmask codeOneLoopStart( if( pOrTerm->leftCursor==iCur || pOrTerm->eOperator==WO_AND ){ WhereInfo *pSubWInfo; /* Info for single OR-term scan */ /* Loop through table entries that match term pOrTerm. */ - pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0, + pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrTerm->pExpr, 0, 0, WHERE_OMIT_OPEN | WHERE_OMIT_CLOSE | WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY); if( pSubWInfo ){ @@ -4274,6 +4288,79 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ } } +/* +** Return true if the DISTINCT expression-list passed as the third argument +** is redundant. A DISTINCT list is redundant if the database contains a +** UNIQUE index that guarantees that the result of the query will be distinct +** anyway. +*/ +static int whereDistinctRedundant( + Parse *pParse, + SrcList *pTabList, + WhereClause *pWC, + ExprList *pDistinct +){ + Table *pTab; + Index *pIdx; + int i; + int iBase; + + /* If there is more than one table or sub-select in the FROM clause of + ** this query, then it will not be possible to show that the DISTINCT + ** clause is redundant. */ + if( pTabList->nSrc!=1 ) return 0; + iBase = pTabList->a[0].iCursor; + pTab = pTabList->a[0].pTab; + + /* Check if all the expressions in the ExprList are of type TK_COLUMN and + ** on the same table. If this is not the case, return early, since it will + ** not be possible to prove that the DISTINCT qualifier is redundant. + ** If any of the expressions is an IPK column, then return true. + */ + for(i=0; inExpr; i++){ + Expr *p = pDistinct->a[i].pExpr; + if( p->op!=TK_COLUMN || p->iTable!=iBase ) return 0; + if( p->iColumn<0 ) return 1; + } + + /* Loop through all indices on the table, checking each to see if it makes + ** the DISTINCT qualifier redundant. It does so if: + ** + ** 1. The index is itself UNIQUE, and + ** + ** 2. All of the columns in the index are either part of the pDistinct + ** list, or else the WHERE clause contains a term of the form "col=X", + ** where X is a constant value. The collation sequences of the + ** comparison and select-list expressions must match those of the index. + */ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->onError==OE_None ) continue; + for(i=0; inColumn; i++){ + int iCol = pIdx->aiColumn[i]; + const char *zColl = pIdx->azColl[i]; + + if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) ){ + int j; + for(j=0; jnExpr; j++){ + Expr *p = pDistinct->a[j].pExpr; + if( p->iColumn==iCol ){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, p); + const char *zEColl = (pColl ? pColl : pParse->db->pDfltColl)->zName; + if( 0==sqlite3StrICmp(zColl, zEColl) ) break; + } + } + if( j==pDistinct->nExpr ) break; + } + } + if( i==pIdx->nColumn ){ + /* This index implies that the DISTINCT qualifier is redundant. */ + return 1; + } + } + + return 0; +} + /* ** Generate the beginning of the loop used for WHERE clause processing. @@ -4368,6 +4455,7 @@ WhereInfo *sqlite3WhereBegin( SrcList *pTabList, /* A list of all tables to be scanned */ Expr *pWhere, /* The WHERE clause */ ExprList **ppOrderBy, /* An ORDER BY clause, or NULL */ + ExprList *pDistinct, /* The select-list for DISTINCT queries - or NULL */ u16 wctrlFlags /* One of the WHERE_* flags defined in sqliteInt.h */ ){ int i; /* Loop counter */ @@ -4495,6 +4583,15 @@ WhereInfo *sqlite3WhereBegin( goto whereBeginError; } + /* Check if the DISTINCT qualifier, if there is one, is redundant. + ** If it is, then set pDistinct to NULL and WhereInfo.eDistinct to + ** WHERE_DISTINCT_UNIQUE to tell the caller to ignore the DISTINCT. + */ + if( pDistinct && whereDistinctRedundant(pParse, pTabList, pWC, pDistinct) ){ + pDistinct = 0; + pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; + } + /* Chose the best index to use for each table in the FROM clause. ** ** This loop fills in the following fields: @@ -4578,6 +4675,7 @@ WhereInfo *sqlite3WhereBegin( int doNotReorder; /* True if this table should not be reordered */ WhereCost sCost; /* Cost information from best[Virtual]Index() */ ExprList *pOrderBy; /* ORDER BY clause for index to optimize */ + ExprList *pDist; /* DISTINCT clause for index to optimize */ doNotReorder = (pTabItem->jointype & (JT_LEFT|JT_CROSS))!=0; if( j!=iFrom && doNotReorder ) break; @@ -4588,6 +4686,7 @@ WhereInfo *sqlite3WhereBegin( } mask = (isOptimal ? m : notReady); pOrderBy = ((i==0 && ppOrderBy )?*ppOrderBy:0); + pDist = (i==0 ? pDistinct : 0); if( pTabItem->pIndex==0 ) nUnconstrained++; WHERETRACE(("=== trying table %d with isOptimal=%d ===\n", @@ -4602,7 +4701,7 @@ WhereInfo *sqlite3WhereBegin( #endif { bestBtreeIndex(pParse, pWC, pTabItem, mask, notReady, pOrderBy, - &sCost); + pDist, &sCost); } assert( isOptimal || (sCost.used¬Ready)==0 ); @@ -4663,6 +4762,10 @@ WhereInfo *sqlite3WhereBegin( if( (bestPlan.plan.wsFlags & WHERE_ORDERBY)!=0 ){ *ppOrderBy = 0; } + if( (bestPlan.plan.wsFlags & WHERE_DISTINCT)!=0 ){ + assert( pWInfo->eDistinct==0 ); + pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + } andFlags &= bestPlan.plan.wsFlags; pLevel->plan = bestPlan.plan; testcase( bestPlan.plan.wsFlags & WHERE_INDEXED ); diff --git a/test/collate5.test b/test/collate5.test index 2e4b89d5ca..e5bea7a5a5 100644 --- a/test/collate5.test +++ b/test/collate5.test @@ -57,17 +57,17 @@ do_test collate5-1.1 { execsql { SELECT DISTINCT a FROM collate5t1; } -} {A B N} +} {a b n} do_test collate5-1.2 { execsql { SELECT DISTINCT b FROM collate5t1; } -} {{} Apple apple banana} +} {apple Apple banana {}} do_test collate5-1.3 { execsql { SELECT DISTINCT a, b FROM collate5t1; } -} {A Apple a apple B banana N {}} +} {a apple A Apple b banana n {}} # Ticket #3376 # diff --git a/test/e_select.test b/test/e_select.test index 6194872fa5..e0f5f0fb91 100644 --- a/test/e_select.test +++ b/test/e_select.test @@ -1238,8 +1238,8 @@ do_select_tests e_select-5 { 3.1 "SELECT x FROM h2" {One Two Three Four one two three four} 3.2 "SELECT x FROM h1, h2 ON (x=b)" {One one Four four} - 4.1 "SELECT DISTINCT x FROM h2" {four one three two} - 4.2 "SELECT DISTINCT x FROM h1, h2 ON (x=b)" {four one} + 4.1 "SELECT DISTINCT x FROM h2" {One Two Three Four} + 4.2 "SELECT DISTINCT x FROM h1, h2 ON (x=b)" {One Four} } # EVIDENCE-OF: R-02054-15343 For the purposes of detecting duplicate @@ -1253,11 +1253,11 @@ do_select_tests e_select-5.5 { # sequence to compare text values with apply. # do_select_tests e_select-5.6 { - 1 "SELECT DISTINCT b FROM h1" {I IV four i iv one} - 2 "SELECT DISTINCT b COLLATE nocase FROM h1" {four i iv one} - 3 "SELECT DISTINCT x FROM h2" {four one three two} + 1 "SELECT DISTINCT b FROM h1" {one I i four IV iv} + 2 "SELECT DISTINCT b COLLATE nocase FROM h1" {one I four IV} + 3 "SELECT DISTINCT x FROM h2" {One Two Three Four} 4 "SELECT DISTINCT x COLLATE binary FROM h2" { - Four One Three Two four one three two + One Two Three Four one two three four } } diff --git a/test/fuzzer1.test b/test/fuzzer1.test index d0575d2d00..6c23211859 100644 --- a/test/fuzzer1.test +++ b/test/fuzzer1.test @@ -1376,7 +1376,7 @@ do_test fuzzer1-2.3 { AND f2.distance<=200 AND streetname.n>=f2.word AND streetname.n<=(f2.word || x'F7BFBFBF') } -} {steelewood tallia tallu talwyn taymouth thelema trailer {tyler finley}} +} {{tyler finley} trailer taymouth steelewood tallia tallu talwyn thelema} finish_test diff --git a/test/insert4.test b/test/insert4.test index 44b428a1c7..cb02b9dccb 100644 --- a/test/insert4.test +++ b/test/insert4.test @@ -112,7 +112,7 @@ do_test insert4-2.4.1 { INSERT INTO t3 SELECT DISTINCT * FROM t2; SELECT * FROM t3; } -} {1 9 9 1} +} {9 1 1 9} xferopt_test insert4-2.4.2 0 do_test insert4-2.4.3 { catchsql { diff --git a/test/misc5.test b/test/misc5.test index 34b2284bc1..b3832f18ae 100644 --- a/test/misc5.test +++ b/test/misc5.test @@ -505,7 +505,7 @@ ifcapable subquery { ) ORDER BY LOWER(artist) ASC; } - } {one} + } {two} } # Ticket #1370. Do not overwrite small files (less than 1024 bytes) diff --git a/test/selectB.test b/test/selectB.test index 3fdf85c0f9..b9d979acb7 100644 --- a/test/selectB.test +++ b/test/selectB.test @@ -355,7 +355,7 @@ for {set ii 3} {$ii <= 4} {incr ii} { SELECT DISTINCT (a/10) FROM t1 UNION ALL SELECT DISTINCT(d%2) FROM t2 ) } - } {0 1 0 1} + } {0 1 1 0} do_test selectB-$ii.20 { execsql { From 6fea83e52c23c41989c6bdb4da326fe7041a3b8e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 1 Jul 2011 13:50:44 +0000 Subject: [PATCH 17/52] Ignore the database name on the target table when parsing a CREATE TABLE statement out of the sqlite_master table. This is a fix for ticket [d6ddba6706353] that preserves backwards compatibility. FossilOrigin-Name: 009c96ea7836edf75d65b150055e920079bb45e3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/trigger.c | 19 ++++++++++++++++--- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index fe85fe4d38..7a5f32f68d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Pass\sthe\sBTREE_UNORDERED\shint\sinto\sboth\ssqlite3BtreeOpen()\sand\sinto\nsqlite3BtreeCreateTable(). -D 2011-06-29T17:11:39.745 +C Ignore\sthe\sdatabase\sname\son\sthe\starget\stable\swhen\sparsing\sa\sCREATE\sTABLE\nstatement\sout\sof\sthe\ssqlite_master\stable.\s\sThis\sis\sa\sfix\sfor\nticket\s[d6ddba6706353]\sthat\spreserves\sbackwards\scompatibility. +D 2011-07-01T13:50:44.953 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -233,7 +233,7 @@ F src/test_vfstrace.c 0b884e06094a746da729119a2cabdc7aa790063d F src/test_wholenumber.c 6129adfbe7c7444f2e60cc785927f3aa74e12290 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c c819d9f72168a035d545a5bdafe9b085b20df705 -F src/trigger.c c836a6caac16ba96611558922106858f6ca3d6bf +F src/trigger.c 1cfb80e2290ef66ea89cb4e821caae65a02c0d56 F src/update.c 80d77311d91ebc06b27149e75701f1b3e9356622 F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 F src/util.c 0f33bbbdfcc4a2d8cf20c3b2a16ffc3b57c58a70 @@ -949,7 +949,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P b9477eb056d120826ed82b0d65e6f27b5d0c087a -R 04bb3d810ba550780b3b6fda2c65d8d3 +P 591de898f41630156cc0fc6ef17dd3ee5e7c479f +R 0219e5330d7b506bd713afee90f76519 U drh -Z 9a15d45fcfccc0dc2281a762dda887ad +Z 5f346c2968ae8ad6ce9a9eb04ceee5d9 diff --git a/manifest.uuid b/manifest.uuid index e1c5a804ad..ae85bf4c8d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -591de898f41630156cc0fc6ef17dd3ee5e7c479f \ No newline at end of file +009c96ea7836edf75d65b150055e920079bb45e3 \ No newline at end of file diff --git a/src/trigger.c b/src/trigger.c index 6242de5e6c..22c4877b6a 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -117,15 +117,28 @@ void sqlite3BeginTrigger( goto trigger_cleanup; } } + if( !pTableName || db->mallocFailed ){ + goto trigger_cleanup; + } + + /* A long-standing parser bug is that this syntax was allowed: + ** + ** CREATE TRIGGER attached.demo AFTER INSERT ON attached.tab .... + ** ^^^^^^^^ + ** + ** To maintain backwards compatibility, ignore the database + ** name on pTableName if we are reparsing our of SQLITE_MASTER. + */ + if( db->init.busy && iDb!=1 ){ + sqlite3DbFree(db, pTableName->a[0].zDatabase); + pTableName->a[0].zDatabase = 0; + } /* If the trigger name was unqualified, and the table is a temp table, ** then set iDb to 1 to create the trigger in the temporary database. ** If sqlite3SrcListLookup() returns 0, indicating the table does not ** exist, the error is caught by the block below. */ - if( !pTableName || db->mallocFailed ){ - goto trigger_cleanup; - } pTab = sqlite3SrcListLookup(pParse, pTableName); if( db->init.busy==0 && pName2->n==0 && pTab && pTab->pSchema==db->aDb[1].pSchema ){ From 50118cdfdb27180fdf6913ae0310ef189cc07148 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 1 Jul 2011 14:21:38 +0000 Subject: [PATCH 18/52] Improvements and tests for detection of redundant DISTINCT qualifiers. FossilOrigin-Name: 7337293c87fb563604dd6ad284f2d1e30c938b4c --- manifest | 18 ++++--- manifest.uuid | 2 +- src/select.c | 44 +++++++++++------ src/where.c | 18 +++---- test/distinct.test | 115 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 161 insertions(+), 36 deletions(-) create mode 100644 test/distinct.test diff --git a/manifest b/manifest index 4197d1e43f..7cf6c1516e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\schanges\sto\simprove\soptimization\sof\sDISTINCT\squeries. -D 2011-06-30T20:17:15.042 +C Improvements\sand\stests\sfor\sdetection\sof\sredundant\sDISTINCT\squalifiers. +D 2011-07-01T14:21:38.743 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -179,7 +179,7 @@ F src/printf.c 585a36b6a963df832cfb69505afa3a34ed5ef8a1 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 1c0f32b64f8e3f555fe1f732f9d6f501a7f05706 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 -F src/select.c cf259606f91f53f71bafd7a2da47f30cda6efc58 +F src/select.c e9d74f943c195f2673990febd3a455b421964ca0 F src/shell.c 0e0173b3e79d956368013e759f084caa7995ecb1 F src/sqlite.h.in 4b7255c10d39c5faf089dbd29cde7c367ff39f1f F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 @@ -250,7 +250,7 @@ F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 0c70ad7b1cac6005fa5e2cbefd23ee05e391c290 F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f -F src/where.c 0bdcf6704dd0d8471052712e1d63a02990b7d0bf +F src/where.c 207cf2c12b391cfdfae89ca3c8afe2adbf46f712 F test/8_3_names.test b93687beebd17f6ebf812405a6833bae5d1f4199 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 @@ -368,6 +368,7 @@ F test/descidx1.test b1353c1a15cfbee97b13a1dcedaf0fe78163ba6a F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test fe720e8b37d59f4cef808b0bf4e1b391c2e56b6f F test/diskfull.test 0cede7ef9d8f415d9d3944005c76be7589bb5ebb +F test/distinct.test 17985484790bbb6d098af682f808f11940ca6375 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 F test/e_createtable.test 4771686a586b6ae414f927c389b2c101cc05c028 F test/e_delete.test e2ae0d3fce5efd70fef99025e932afffc5616fab @@ -949,10 +950,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 591de898f41630156cc0fc6ef17dd3ee5e7c479f -R 5673acf06c190a75a743da9daebc8ac9 -T *branch * experimental -T *sym-experimental * -T -sym-trunk * +P f7ba0219ef2f235543c258be736955d91ca5ecce +R 778c4d57ca4312ecfa96b1ac67177d07 U dan -Z ab52fc5a81a2f6dd6b96daa429077988 +Z 94d823140247a8802d583b432d46d049 diff --git a/manifest.uuid b/manifest.uuid index 5aa8e13950..a582d766d2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f7ba0219ef2f235543c258be736955d91ca5ecce \ No newline at end of file +7337293c87fb563604dd6ad284f2d1e30c938b4c \ No newline at end of file diff --git a/src/select.c b/src/select.c index 00107e1de6..45f1c7545d 100644 --- a/src/select.c +++ b/src/select.c @@ -3848,18 +3848,6 @@ int sqlite3Select( } #endif - /* If possible, rewrite the query to use GROUP BY instead of DISTINCT. - ** GROUP BY might use an index, DISTINCT never does. - */ -#if 0 - assert( p->pGroupBy==0 || (p->selFlags & SF_Aggregate)!=0 ); - if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ){ - p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); - pGroupBy = p->pGroupBy; - p->selFlags &= ~SF_Distinct; - } -#endif - /* If there is both a GROUP BY and an ORDER BY clause and they are ** identical, then disable the ORDER BY clause since the GROUP BY ** will cause elements to come out in the correct order. This is @@ -3872,6 +3860,30 @@ int sqlite3Select( pOrderBy = 0; } + /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and + ** if the select-list is the same as the ORDER BY list, then this query + ** can be rewritten as a GROUP BY. In other words, this: + ** + ** SELECT DISTINCT xyz FROM ... ORDER BY xyz + ** + ** is transformed to: + ** + ** SELECT xyz FROM ... GROUP BY xyz + ** + ** The second form is preferred as a single index (or temp-table) may be + ** used for both the ORDER BY and DISTINCT processing. As originally + ** written the query must use a temp-table for at least one of the ORDER + ** BY and DISTINCT, and an index or separate temp-table for the other. + */ + if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct + && sqlite3ExprListCompare(pOrderBy, p->pEList)==0 + ){ + p->selFlags &= ~SF_Distinct; + p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); + pGroupBy = p->pGroupBy; + pOrderBy = 0; + } + /* If there is an ORDER BY clause, then this sorting ** index might end up being unused if the data can be ** extracted in pre-sorted order. If that is the case, then the @@ -3935,6 +3947,10 @@ int sqlite3Select( } if( pWInfo->eDistinct ){ + VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ + + pOp = sqlite3VdbeGetOp(v, addrDistinctIndex); + assert( isDistinct ); assert( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED || pWInfo->eDistinct==WHERE_DISTINCT_UNIQUE @@ -3947,11 +3963,9 @@ int sqlite3Select( int iBase = pParse->nMem+1; int iBase2 = iBase + pEList->nExpr; pParse->nMem += (pEList->nExpr*2); - VdbeOp *pOp; /* Change the OP_OpenEphemeral coded earlier to an OP_Integer. The ** OP_Integer initializes the "first row" flag. */ - pOp = sqlite3VdbeGetOp(v, addrDistinctIndex); pOp->opcode = OP_Integer; pOp->p1 = 1; pOp->p2 = iFlag; @@ -3970,6 +3984,8 @@ int sqlite3Select( sqlite3VdbeAddOp2(v, OP_Integer, 0, iFlag); assert( sqlite3VdbeCurrentAddr(v)==iJump ); sqlite3VdbeAddOp3(v, OP_Move, iBase, iBase2, pEList->nExpr); + }else{ + pOp->opcode = OP_Noop; } } diff --git a/src/where.c b/src/where.c index 8aecf9788d..70e9722d8c 100644 --- a/src/where.c +++ b/src/where.c @@ -4302,7 +4302,7 @@ static int whereDistinctRedundant( ){ Table *pTab; Index *pIdx; - int i; + int i; int iBase; /* If there is more than one table or sub-select in the FROM clause of @@ -4312,15 +4312,11 @@ static int whereDistinctRedundant( iBase = pTabList->a[0].iCursor; pTab = pTabList->a[0].pTab; - /* Check if all the expressions in the ExprList are of type TK_COLUMN and - ** on the same table. If this is not the case, return early, since it will - ** not be possible to prove that the DISTINCT qualifier is redundant. - ** If any of the expressions is an IPK column, then return true. - */ + /* If any of the expressions is an IPK column, then return true. */ for(i=0; inExpr; i++){ Expr *p = pDistinct->a[i].pExpr; - if( p->op!=TK_COLUMN || p->iTable!=iBase ) return 0; - if( p->iColumn<0 ) return 1; + assert( p->op!=TK_COLUMN || p->iTable==iBase ); + if( p->op==TK_COLUMN && p->iColumn<0 ) return 1; } /* Loop through all indices on the table, checking each to see if it makes @@ -4343,10 +4339,10 @@ static int whereDistinctRedundant( int j; for(j=0; jnExpr; j++){ Expr *p = pDistinct->a[j].pExpr; - if( p->iColumn==iCol ){ + assert( p->op!=TK_COLUMN || p->iTable==iBase ); + if( p->op==TK_COLUMN && p->iColumn==iCol ){ CollSeq *pColl = sqlite3ExprCollSeq(pParse, p); - const char *zEColl = (pColl ? pColl : pParse->db->pDfltColl)->zName; - if( 0==sqlite3StrICmp(zColl, zEColl) ) break; + if( pColl && 0==sqlite3StrICmp(zColl, pColl->zName) ) break; } } if( j==pDistinct->nExpr ) break; diff --git a/test/distinct.test b/test/distinct.test new file mode 100644 index 0000000000..f855caa833 --- /dev/null +++ b/test/distinct.test @@ -0,0 +1,115 @@ +# 2011 July 1 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this script is the DISTINCT modifier. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +set testprefix distinct + + +proc is_distinct_noop {sql} { + set sql1 $sql + set sql2 [string map {DISTINCT ""} $sql] + + set program1 [list] + set program2 [list] + db eval "EXPLAIN $sql1" { + if {$opcode != "Noop"} { lappend program1 $opcode } + } + db eval "EXPLAIN $sql2" { + if {$opcode != "Noop"} { lappend program2 $opcode } + } + + return [expr {$program1==$program2}] +} + +proc do_distinct_noop_test {tn sql} { + uplevel [list do_test $tn [list is_distinct_noop $sql] 1] +} +proc do_distinct_not_noop_test {tn sql} { + uplevel [list do_test $tn [list is_distinct_noop $sql] 0] +} + + +#------------------------------------------------------------------------- +# The following tests - distinct-1.* - check that the planner correctly +# detects cases where a UNIQUE index means that a DISTINCT clause is +# redundant. Currently the planner only detects such cases when there +# is a single table in the FROM clause. +# +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c, d); + CREATE UNIQUE INDEX i1 ON t1(b, c); + CREATE UNIQUE INDEX i2 ON t1(d COLLATE nocase); + + CREATE TABLE t2(x INTEGER PRIMARY KEY, y); + + CREATE TABLE t3(c1 PRIMARY KEY, c2); + CREATE INDEX i3 ON t3(c2); +} +foreach {tn noop sql} { + + 1 1 "SELECT DISTINCT b, c FROM t1" + 2 1 "SELECT DISTINCT c FROM t1 WHERE b = ?" + 3 1 "SELECT DISTINCT rowid FROM t1" + 4 1 "SELECT DISTINCT rowid, a FROM t1" + 5 1 "SELECT DISTINCT x FROM t2" + 6 1 "SELECT DISTINCT * FROM t2" + 7 1 "SELECT DISTINCT * FROM (SELECT * FROM t2)" + + 8 1 "SELECT DISTINCT * FROM t1" + + 8 0 "SELECT DISTINCT a, b FROM t1" + + 9 0 "SELECT DISTINCT c FROM t1 WHERE b IN (1,2)" + 10 0 "SELECT DISTINCT c FROM t1" + 11 0 "SELECT DISTINCT b FROM t1" + + 12 0 "SELECT DISTINCT a, d FROM t1" + 13 0 "SELECT DISTINCT a, b, c COLLATE nocase FROM t1" + 14 1 "SELECT DISTINCT a, d COLLATE nocase FROM t1" + 15 0 "SELECT DISTINCT a, d COLLATE binary FROM t1" + 16 1 "SELECT DISTINCT a, b, c COLLATE binary FROM t1" + + 16 0 "SELECT DISTINCT t1.rowid FROM t1, t2" + 17 0 { /* Technically, it would be possible to detect that DISTINCT + ** is a no-op in cases like the following. But SQLite does not + ** do so. */ + SELECT DISTINCT t1.rowid FROM t1, t2 WHERE t1.rowid=t2.rowid } + + 18 1 "SELECT DISTINCT c1, c2 FROM t3" + 19 1 "SELECT DISTINCT c1 FROM t3" + 20 1 "SELECT DISTINCT * FROM t3" + 21 0 "SELECT DISTINCT c2 FROM t3" + + 22 0 "SELECT DISTINCT * FROM (SELECT 1, 2, 3 UNION SELECT 4, 5, 6)" + 23 1 "SELECT DISTINCT rowid FROM (SELECT 1, 2, 3 UNION SELECT 4, 5, 6)" + + 24 0 "SELECT DISTINCT rowid/2 FROM t1" + 25 1 "SELECT DISTINCT rowid/2, rowid FROM t1" + 26 1 "SELECT DISTINCT rowid/2, b FROM t1 WHERE c = ?" +} { + if {$noop} { + do_distinct_noop_test 1.$tn $sql + } else { + do_distinct_not_noop_test 1.$tn $sql + } +} + + + + + + +finish_test From f475e5ae9894f9f406603b2fa831293185fd8fd7 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 1 Jul 2011 14:22:04 +0000 Subject: [PATCH 19/52] Test case for ticket [d6ddba6706353915ceed] FossilOrigin-Name: 953e169e8a7dac05a0b56b4ef5d500ec8399d37f --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/triggerD.test | 45 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 7a5f32f68d..52a2aaa76b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ignore\sthe\sdatabase\sname\son\sthe\starget\stable\swhen\sparsing\sa\sCREATE\sTABLE\nstatement\sout\sof\sthe\ssqlite_master\stable.\s\sThis\sis\sa\sfix\sfor\nticket\s[d6ddba6706353]\sthat\spreserves\sbackwards\scompatibility. -D 2011-07-01T13:50:44.953 +C Test\scase\sfor\sticket\s[d6ddba6706353915ceed] +D 2011-07-01T14:22:04.191 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -840,7 +840,7 @@ F test/trigger9.test 5b0789f1c5c4600961f8e68511b825b87be53e31 F test/triggerA.test eaf11a29db2a11967d2d4b49d37f92bce598194e F test/triggerB.test 56780c031b454abac2340dbb3b71ac5c56c3d7fe F test/triggerC.test 02c690febf608ae20b9af86184a9867f79855b1d -F test/triggerD.test c6add3817351451e419f6ff9e9a259b02b6e2de7 +F test/triggerD.test bfdac1143deee8fb12b6a3640d76e5669a567ff6 F test/tt3_checkpoint.c 415eccce672d681b297485fc20f44cdf0eac93af F test/types.test bf816ce73c7dfcfe26b700c19f97ef4050d194ff F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84 @@ -949,7 +949,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 591de898f41630156cc0fc6ef17dd3ee5e7c479f -R 0219e5330d7b506bd713afee90f76519 +P 009c96ea7836edf75d65b150055e920079bb45e3 +R c79f1fc1420004e852fc4dd4b2df1b0e U drh -Z 5f346c2968ae8ad6ce9a9eb04ceee5d9 +Z fee6ccae1aad0120ae0a15394efb76ef diff --git a/manifest.uuid b/manifest.uuid index ae85bf4c8d..f76287d586 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -009c96ea7836edf75d65b150055e920079bb45e3 \ No newline at end of file +953e169e8a7dac05a0b56b4ef5d500ec8399d37f \ No newline at end of file diff --git a/test/triggerD.test b/test/triggerD.test index 08945081da..0cce4f3cde 100644 --- a/test/triggerD.test +++ b/test/triggerD.test @@ -14,6 +14,12 @@ # the use of these columns in triggers will refer to the column and not # to the actual ROWID. Ticket [34d2ae1c6d08b5271ba5e5592936d4a1d913ffe3] # +# Also, verify that triggers created like this: +# +# CREATE TRIGGER attached.trig AFTER INSERT ON attached.tab ... +# +# can be reparsed as a main database. Ticket [d6ddba6706353915ceedc56b4e3] +# set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -171,4 +177,43 @@ do_test triggerD-3.2 { } {10003 20004} +############################################################################# +# +# Ticket [d6ddba6706353915ceedc56b4e3e72ecb4d77ba4] +# +# The following syntax really should not be allowed: +# +# CREATE TRIGGER xyz.trig BEFORE UPDATE ON xyz.tab BEGIN ... +# +# But a long-standing bug does allow it. And the "xyz.tab" slips into +# the sqlite_master table. We cannot fix the bug simply by disallowing +# "xyz.tab" since that could break legacy applications. We have to +# fix the system so that the "xyz." on "xyz.tab" is ignored. +# Verify that this is the case. +# +do_test triggerD-4.1 { + db close + file delete -force test.db test2.db + sqlite3 db test.db + db eval { + CREATE TABLE t1(x); + ATTACH 'test2.db' AS db2; + CREATE TABLE db2.t2(y); + CREATE TABLE db2.log(z); + CREATE TRIGGER db2.trig AFTER INSERT ON db2.t2 BEGIN + INSERT INTO log(z) VALUES(new.y); + END; + INSERT INTO t2 VALUES(123); + SELECT * FROM log; + } +} {123} +do_test triggerD-4.2 { + sqlite3 db2 test2.db + db2 eval { + INSERT INTO t2 VALUES(234); + SELECT * FROM log; + } +} {123 234} +db2 close + finish_test From 6f3439697e4025bc9209ae0256b1f2928ca9a073 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 1 Jul 2011 18:26:40 +0000 Subject: [PATCH 20/52] Improve use of indexes to optimize DISTINCT queries. FossilOrigin-Name: 6c202ea0247ff509f696eee3839975a88ed26cf2 --- manifest | 16 ++-- manifest.uuid | 2 +- src/where.c | 225 ++++++++++++++++++++++++++++++--------------- test/distinct.test | 48 ++++++++++ test/tester.tcl | 2 +- 5 files changed, 210 insertions(+), 83 deletions(-) diff --git a/manifest b/manifest index 7cf6c1516e..2ac880f628 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sand\stests\sfor\sdetection\sof\sredundant\sDISTINCT\squalifiers. -D 2011-07-01T14:21:38.743 +C Improve\suse\sof\sindexes\sto\soptimize\sDISTINCT\squeries. +D 2011-07-01T18:26:40.769 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -250,7 +250,7 @@ F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 0c70ad7b1cac6005fa5e2cbefd23ee05e391c290 F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f -F src/where.c 207cf2c12b391cfdfae89ca3c8afe2adbf46f712 +F src/where.c 83cf8e8637826a311e598980bf5c920069847a77 F test/8_3_names.test b93687beebd17f6ebf812405a6833bae5d1f4199 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 @@ -368,7 +368,7 @@ F test/descidx1.test b1353c1a15cfbee97b13a1dcedaf0fe78163ba6a F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test fe720e8b37d59f4cef808b0bf4e1b391c2e56b6f F test/diskfull.test 0cede7ef9d8f415d9d3944005c76be7589bb5ebb -F test/distinct.test 17985484790bbb6d098af682f808f11940ca6375 +F test/distinct.test dbedaddb15c7a96729d39e5c3d7c2993ac905a1d F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 F test/e_createtable.test 4771686a586b6ae414f927c389b2c101cc05c028 F test/e_delete.test e2ae0d3fce5efd70fef99025e932afffc5616fab @@ -696,7 +696,7 @@ F test/tclsqlite.test 8c154101e704170c2be10f137a5499ac2c6da8d3 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test 1a21a597055dcf6002b6f1ee867632dccd6e925c F test/temptrigger.test b0273db072ce5f37cf19140ceb1f0d524bbe9f05 -F test/tester.tcl 7d3d17914ca5397a82f801db733b81d4a82f50c3 +F test/tester.tcl aad52c7644513e5650f6613d371b0af3193c3885 F test/thread001.test a3e6a7254d1cb057836cb3145b60c10bf5b7e60f F test/thread002.test afd20095e6e845b405df4f2c920cb93301ca69db F test/thread003.test b824d4f52b870ae39fc5bae4d8070eca73085dca @@ -950,7 +950,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P f7ba0219ef2f235543c258be736955d91ca5ecce -R 778c4d57ca4312ecfa96b1ac67177d07 +P 7337293c87fb563604dd6ad284f2d1e30c938b4c +R 0a65cc1234bf3efb15e20cfbd6948f45 U dan -Z 94d823140247a8802d583b432d46d049 +Z be26c18c91323ba4302e319914f9eafb diff --git a/manifest.uuid b/manifest.uuid index a582d766d2..f1ce564ac1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7337293c87fb563604dd6ad284f2d1e30c938b4c \ No newline at end of file +6c202ea0247ff509f696eee3839975a88ed26cf2 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 70e9722d8c..5cbf5833c6 100644 --- a/src/where.c +++ b/src/where.c @@ -1398,6 +1398,156 @@ static int referencesOtherTables( return 0; } +/* +** This function searches the expression list passed as the second argument +** for an expression of type TK_COLUMN that refers to the same column and +** uses the same collation sequence as the iCol'th column of index pIdx. +** Argument iBase is the cursor number used for the table that pIdx refers +** to. +** +** If such an expression is found, its index in pList->a[] is returned. If +** no expression is found, -1 is returned. +*/ +static int findIndexCol( + Parse *pParse, /* Parse context */ + ExprList *pList, /* Expression list to search */ + int iBase, /* Cursor for table associated with pIdx */ + Index *pIdx, /* Index to match column of */ + int iCol /* Column of index to match */ +){ + int i; + const char *zColl = pIdx->azColl[iCol]; + + for(i=0; inExpr; i++){ + Expr *p = pList->a[i].pExpr; + if( pIdx->aiColumn[iCol]==p->iColumn && iBase==p->iTable ){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, p); + if( pColl && 0==sqlite3StrICmp(pColl->zName, zColl) ){ + return i; + } + } + } + + return -1; +} + +/* +** This routine determines if pIdx can be used to assist in processing a +** DISTINCT qualifier. In other words, it tests whether or not using this +** index for the outer loop guarantees that rows with equal values for +** all expressions in the pDistinct list are delivered grouped together. +** +** For example, the query +** +** SELECT DISTINCT a, b, c FROM tbl WHERE a = ? +** +** can benefit from any index on columns "b" and "c". +*/ +static int isDistinctIndex( + Parse *pParse, /* Parsing context */ + WhereClause *pWC, /* The WHERE clause */ + Index *pIdx, /* The index being considered */ + int base, /* Cursor number for the table pIdx is on */ + ExprList *pDistinct, /* The DISTINCT expressions */ + int nEqCol /* Number of index columns with == */ +){ + Bitmask mask = 0; /* Mask of unaccounted for pDistinct exprs */ + int i; /* Iterator variable */ + + if( pIdx->zName==0 || pDistinct==0 || pDistinct->nExpr>=BMS ) return 0; + + /* Loop through all the expressions in the distinct list. If any of them + ** are not simple column references, return early. Otherwise, test if the + ** WHERE clause contains a "col=X" clause. If it does, the expression + ** can be ignored. If it does not, and the column does not belong to the + ** same table as index pIdx, return early. Finally, if there is no + ** matching "col=X" expression and the column is on the same table as pIdx, + ** set the corresponding bit in variable mask. + */ + for(i=0; inExpr; i++){ + WhereTerm *pTerm; + Expr *p = pDistinct->a[i].pExpr; + if( p->op!=TK_COLUMN ) return 0; + pTerm = findTerm(pWC, p->iTable, p->iColumn, ~(Bitmask)0, WO_EQ, 0); + if( pTerm ){ + Expr *pX = pTerm->pExpr; + CollSeq *p1 = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); + CollSeq *p2 = sqlite3ExprCollSeq(pParse, p); + if( p1==p2 ) continue; + } + if( p->iTable!=base ) return 0; + mask |= (((Bitmask)1) << i); + } + + for(i=nEqCol; mask && inColumn; i++){ + int iExpr = findIndexCol(pParse, pDistinct, base, pIdx, i); + if( iExpr<0 ) break; + mask &= ~(((Bitmask)1) << iExpr); + } + + return (mask==0); +} + + +/* +** Return true if the DISTINCT expression-list passed as the third argument +** is redundant. A DISTINCT list is redundant if the database contains a +** UNIQUE index that guarantees that the result of the query will be distinct +** anyway. +*/ +static int isDistinctRedundant( + Parse *pParse, + SrcList *pTabList, + WhereClause *pWC, + ExprList *pDistinct +){ + Table *pTab; + Index *pIdx; + int i; + int iBase; + + /* If there is more than one table or sub-select in the FROM clause of + ** this query, then it will not be possible to show that the DISTINCT + ** clause is redundant. */ + if( pTabList->nSrc!=1 ) return 0; + iBase = pTabList->a[0].iCursor; + pTab = pTabList->a[0].pTab; + + /* If any of the expressions is an IPK column, then return true. */ + for(i=0; inExpr; i++){ + Expr *p = pDistinct->a[i].pExpr; + assert( p->op!=TK_COLUMN || p->iTable==iBase ); + if( p->op==TK_COLUMN && p->iColumn<0 ) return 1; + } + + /* Loop through all indices on the table, checking each to see if it makes + ** the DISTINCT qualifier redundant. It does so if: + ** + ** 1. The index is itself UNIQUE, and + ** + ** 2. All of the columns in the index are either part of the pDistinct + ** list, or else the WHERE clause contains a term of the form "col=X", + ** where X is a constant value. The collation sequences of the + ** comparison and select-list expressions must match those of the index. + */ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->onError==OE_None ) continue; + for(i=0; inColumn; i++){ + int iCol = pIdx->aiColumn[i]; + if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) + && 0>findIndexCol(pParse, pDistinct, iBase, pIdx, i) + ){ + break; + } + } + if( i==pIdx->nColumn ){ + /* This index implies that the DISTINCT qualifier is redundant. */ + return 1; + } + } + + return 0; +} /* ** This routine decides if pIdx can be used to satisfy the ORDER BY @@ -2910,9 +3060,7 @@ static void bestBtreeIndex( /* If there is a DISTINCT qualifier and this index will scan rows in ** order of the DISTINCT expressions, clear bDist and set the appropriate ** flags in wsFlags. */ - if( isSortingIndex( - pParse, pWC->pMaskSet, pProbe, iCur, pDistinct, nEq, wsFlags, 0) - ){ + if( isDistinctIndex(pParse, pWC, pProbe, iCur, pDistinct, nEq) ){ bDist = 0; wsFlags |= WHERE_ROWID_RANGE|WHERE_COLUMN_RANGE|WHERE_DISTINCT; } @@ -4288,75 +4436,6 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ } } -/* -** Return true if the DISTINCT expression-list passed as the third argument -** is redundant. A DISTINCT list is redundant if the database contains a -** UNIQUE index that guarantees that the result of the query will be distinct -** anyway. -*/ -static int whereDistinctRedundant( - Parse *pParse, - SrcList *pTabList, - WhereClause *pWC, - ExprList *pDistinct -){ - Table *pTab; - Index *pIdx; - int i; - int iBase; - - /* If there is more than one table or sub-select in the FROM clause of - ** this query, then it will not be possible to show that the DISTINCT - ** clause is redundant. */ - if( pTabList->nSrc!=1 ) return 0; - iBase = pTabList->a[0].iCursor; - pTab = pTabList->a[0].pTab; - - /* If any of the expressions is an IPK column, then return true. */ - for(i=0; inExpr; i++){ - Expr *p = pDistinct->a[i].pExpr; - assert( p->op!=TK_COLUMN || p->iTable==iBase ); - if( p->op==TK_COLUMN && p->iColumn<0 ) return 1; - } - - /* Loop through all indices on the table, checking each to see if it makes - ** the DISTINCT qualifier redundant. It does so if: - ** - ** 1. The index is itself UNIQUE, and - ** - ** 2. All of the columns in the index are either part of the pDistinct - ** list, or else the WHERE clause contains a term of the form "col=X", - ** where X is a constant value. The collation sequences of the - ** comparison and select-list expressions must match those of the index. - */ - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->onError==OE_None ) continue; - for(i=0; inColumn; i++){ - int iCol = pIdx->aiColumn[i]; - const char *zColl = pIdx->azColl[i]; - - if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) ){ - int j; - for(j=0; jnExpr; j++){ - Expr *p = pDistinct->a[j].pExpr; - assert( p->op!=TK_COLUMN || p->iTable==iBase ); - if( p->op==TK_COLUMN && p->iColumn==iCol ){ - CollSeq *pColl = sqlite3ExprCollSeq(pParse, p); - if( pColl && 0==sqlite3StrICmp(zColl, pColl->zName) ) break; - } - } - if( j==pDistinct->nExpr ) break; - } - } - if( i==pIdx->nColumn ){ - /* This index implies that the DISTINCT qualifier is redundant. */ - return 1; - } - } - - return 0; -} - /* ** Generate the beginning of the loop used for WHERE clause processing. @@ -4583,7 +4662,7 @@ WhereInfo *sqlite3WhereBegin( ** If it is, then set pDistinct to NULL and WhereInfo.eDistinct to ** WHERE_DISTINCT_UNIQUE to tell the caller to ignore the DISTINCT. */ - if( pDistinct && whereDistinctRedundant(pParse, pTabList, pWC, pDistinct) ){ + if( pDistinct && isDistinctRedundant(pParse, pTabList, pWC, pDistinct) ){ pDistinct = 0; pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; } diff --git a/test/distinct.test b/test/distinct.test index f855caa833..14252b23fe 100644 --- a/test/distinct.test +++ b/test/distinct.test @@ -41,6 +41,23 @@ proc do_distinct_not_noop_test {tn sql} { uplevel [list do_test $tn [list is_distinct_noop $sql] 0] } +proc do_temptables_test {tn sql temptables} { + uplevel [list do_test $tn [subst -novar { + set ret "" + db eval "EXPLAIN [set sql]" { + if {$opcode == "OpenEphemeral"} { + if {$p5 != "10" && $p5!="00"} { error "p5 = $p5" } + if {$p5 == "10"} { + lappend ret hash + } else { + lappend ret btree + } + } + } + set ret + }] $temptables] +} + #------------------------------------------------------------------------- # The following tests - distinct-1.* - check that the planner correctly @@ -107,6 +124,37 @@ foreach {tn noop sql} { } } +#------------------------------------------------------------------------- +# The following tests - distinct-2.* - test cases where an index is +# used to deliver results in order of the DISTINCT expressions. +# +drop_all_tables +do_execsql_test 2.0 { + CREATE TABLE t1(a, b, c); + + CREATE INDEX i1 ON t1(a, b); + CREATE INDEX i2 ON t1(b COLLATE nocase, c COLLATE nocase); + + INSERT INTO t1 VALUES('a', 'b', 'c'); + INSERT INTO t1 VALUES('A', 'B', 'C'); + INSERT INTO t1 VALUES('a', 'b', 'c'); + INSERT INTO t1 VALUES('A', 'B', 'C'); +} + +foreach {tn sql temptables res} { + 1 "a, b FROM t1" {} {A B a b} + 2 "b, a FROM t1" {} {B A b a} + 3 "a, b, c FROM t1" {hash} {a b c A B C} + 4 "a, b, c FROM t1 ORDER BY a, b, c" {btree} {A B C a b c} + 5 "b FROM t1 WHERE a = 'a'" {} {b} + 6 "b FROM t1" {hash} {b B} + 7 "a FROM t1" {} {A a} + 8 "b COLLATE nocase FROM t1" {} {b} + 9 "b COLLATE nocase FROM t1 ORDER BY b COLLATE nocase" {} {B} +} { + do_execsql_test 2.$tn.1 "SELECT DISTINCT $sql" $res + do_temptables_test 2.$tn.2 "SELECT DISTINCT $sql" $temptables +} diff --git a/test/tester.tcl b/test/tester.tcl index 8624821a5e..fabb32af52 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -20,7 +20,7 @@ # Commands to manipulate the db and the file-system at a high level: # # copy_file FROM TO -# drop_all_table ?DB? +# drop_all_tables ?DB? # forcedelete FILENAME # # Test the capability of the SQLite version built into the interpreter to From 94e08d92f93f8ae3504e0902107fff6cc35d6521 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 2 Jul 2011 06:44:05 +0000 Subject: [PATCH 21/52] Fix a broken assert() in where.c. FossilOrigin-Name: 090b29177fbc9faa83b560d92ddc3710d291776e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 8 +++++--- test/distinct.test | 3 +++ 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 2dd09e925a..5d0c137486 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\swith\sexperimental\sbranch. -D 2011-07-01T18:43:13.469 +C Fix\sa\sbroken\sassert()\sin\swhere.c. +D 2011-07-02T06:44:05.968 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -250,7 +250,7 @@ F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 0c70ad7b1cac6005fa5e2cbefd23ee05e391c290 F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f -F src/where.c 83cf8e8637826a311e598980bf5c920069847a77 +F src/where.c dcc0d91cc4b843adf79476b721c10ca6f382cb85 F test/8_3_names.test b93687beebd17f6ebf812405a6833bae5d1f4199 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 @@ -368,7 +368,7 @@ F test/descidx1.test b1353c1a15cfbee97b13a1dcedaf0fe78163ba6a F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test fe720e8b37d59f4cef808b0bf4e1b391c2e56b6f F test/diskfull.test 0cede7ef9d8f415d9d3944005c76be7589bb5ebb -F test/distinct.test dbedaddb15c7a96729d39e5c3d7c2993ac905a1d +F test/distinct.test 8c4d951fc40aba84421060e07b16099d2f4c2fdf F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 F test/e_createtable.test 4771686a586b6ae414f927c389b2c101cc05c028 F test/e_delete.test e2ae0d3fce5efd70fef99025e932afffc5616fab @@ -950,7 +950,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 6c202ea0247ff509f696eee3839975a88ed26cf2 953e169e8a7dac05a0b56b4ef5d500ec8399d37f -R 0920e7efb36d6d861c0873d40a58b5e8 +P e56be74eabc03d1a524d96aa4e87102dea4e348e +R a0416d3328a66ac3932394627191f7dc U dan -Z 076f69de0e54769c5ccba2bae7e8f9fd +Z 6cc3cca355617f23d46e4dc8112cd183 diff --git a/manifest.uuid b/manifest.uuid index 40e2339bd7..c0528273a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e56be74eabc03d1a524d96aa4e87102dea4e348e \ No newline at end of file +090b29177fbc9faa83b560d92ddc3710d291776e \ No newline at end of file diff --git a/src/where.c b/src/where.c index 5cbf5833c6..61361bd95d 100644 --- a/src/where.c +++ b/src/where.c @@ -1513,11 +1513,13 @@ static int isDistinctRedundant( iBase = pTabList->a[0].iCursor; pTab = pTabList->a[0].pTab; - /* If any of the expressions is an IPK column, then return true. */ + /* If any of the expressions is an IPK column on table iBase, then return + ** true. Note: The (p->iTable==iBase) part of this test may be false if the + ** current SELECT is a correlated sub-query. + */ for(i=0; inExpr; i++){ Expr *p = pDistinct->a[i].pExpr; - assert( p->op!=TK_COLUMN || p->iTable==iBase ); - if( p->op==TK_COLUMN && p->iColumn<0 ) return 1; + if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1; } /* Loop through all indices on the table, checking each to see if it makes diff --git a/test/distinct.test b/test/distinct.test index 14252b23fe..87456de876 100644 --- a/test/distinct.test +++ b/test/distinct.test @@ -156,6 +156,9 @@ foreach {tn sql temptables res} { do_temptables_test 2.$tn.2 "SELECT DISTINCT $sql" $temptables } +do_execsql_test 2.A { + SELECT (SELECT DISTINCT o.a FROM t1 AS i) FROM t1 AS o; +} {a A a A} From 969e5595e823dd58d4ccef42bead3f229121e708 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 2 Jul 2011 15:32:57 +0000 Subject: [PATCH 22/52] Ensure that automatic indexes are only created in scenarios where they may be used more than once. FossilOrigin-Name: 27c65d4d9c58bfc4ea8f9337fa15090459fb26c5 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 4 ++++ test/autoindex1.test | 10 ++++++++++ 4 files changed, 22 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ea458a6ada..b933d5723e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sexperimental\schanges\simproving\soptimization\sof\sDISTINCT\squeries\swith\sthe\strunk. -D 2011-07-02T09:46:52.496 +C Ensure\sthat\sautomatic\sindexes\sare\sonly\screated\sin\sscenarios\swhere\sthey\smay\sbe\sused\smore\sthan\sonce. +D 2011-07-02T15:32:57.635 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -250,7 +250,7 @@ F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 0c70ad7b1cac6005fa5e2cbefd23ee05e391c290 F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f -F src/where.c dcc0d91cc4b843adf79476b721c10ca6f382cb85 +F src/where.c 30dc117ecd9406b7e442426cb5368199035c27ff F test/8_3_names.test b93687beebd17f6ebf812405a6833bae5d1f4199 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 @@ -281,7 +281,7 @@ F test/auth.test b047105c32da7db70b842fd24056723125ecc2ff F test/auth2.test 270baddc8b9c273682760cffba6739d907bd2882 F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5 F test/autoinc.test 85ef3180a737e6580086a018c09c6f1a52759b46 -F test/autoindex1.test 860fc83f4fefb0c68ad062afc3ff43faa1534fc4 +F test/autoindex1.test 058d0b331ae6840a61bbee910d8cbae27bfd5991 F test/autovacuum.test bb7c0885e6f8f1d633045de48f2b66082162766d F test/autovacuum_ioerr2.test 598b0663074d3673a9c1bc9a16e80971313bafe6 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 @@ -950,7 +950,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 953e169e8a7dac05a0b56b4ef5d500ec8399d37f 090b29177fbc9faa83b560d92ddc3710d291776e -R a0416d3328a66ac3932394627191f7dc +P 45e581bff7a75db6c9a2c45b73d034d0b8a166d1 +R 6e4dc8fc887e24292adb3ae0b641a038 U dan -Z a35e180a4755a506b249eeb5b9af1ec2 +Z d167de182be16055ea09a1a9326ef0ca diff --git a/manifest.uuid b/manifest.uuid index e885436314..670b072715 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -45e581bff7a75db6c9a2c45b73d034d0b8a166d1 \ No newline at end of file +27c65d4d9c58bfc4ea8f9337fa15090459fb26c5 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 61361bd95d..f843e10740 100644 --- a/src/where.c +++ b/src/where.c @@ -1902,6 +1902,10 @@ static void bestAutomaticIndex( WhereTerm *pWCEnd; /* End of pWC->a[] */ Table *pTable; /* Table tht might be indexed */ + if( pParse->nQueryLoop<=(double)1 ){ + /* There is no point in building an automatic index for a single scan */ + return; + } if( (pParse->db->flags & SQLITE_AutoIndex)==0 ){ /* Automatic indices are disabled at run-time */ return; diff --git a/test/autoindex1.test b/test/autoindex1.test index bc628dc256..6c8d54d125 100644 --- a/test/autoindex1.test +++ b/test/autoindex1.test @@ -248,4 +248,14 @@ do_execsql_test autoindex1-600 { 0 1 1 {SEARCH SUBQUERY 1 AS y USING AUTOMATIC COVERING INDEX (sheep_no=?) (~8 rows)} } + +do_execsql_test autoindex1-700 { + CREATE TABLE t5(a, b, c); + EXPLAIN QUERY PLAN SELECT a FROM t5 WHERE b=10 ORDER BY c; +} { + 0 0 0 {SCAN TABLE t5 (~100000 rows)} + 0 0 0 {USE TEMP B-TREE FOR ORDER BY} +} + + finish_test From b24d83fb3562552416ae8b5ba74b8c9398da8fa0 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 2 Jul 2011 19:12:05 +0000 Subject: [PATCH 23/52] Add a testcase macro to ensure testing a boundary case in DISTINCT processing. FossilOrigin-Name: ff9fc722dc481df1ce6a9733db1f033b88141d5d --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index b933d5723e..291bcdee80 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sautomatic\sindexes\sare\sonly\screated\sin\sscenarios\swhere\sthey\smay\sbe\sused\smore\sthan\sonce. -D 2011-07-02T15:32:57.635 +C Add\sa\stestcase\smacro\sto\sensure\stesting\sa\sboundary\scase\sin\sDISTINCT\sprocessing. +D 2011-07-02T19:12:05.917 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -250,7 +250,7 @@ F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 0c70ad7b1cac6005fa5e2cbefd23ee05e391c290 F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f -F src/where.c 30dc117ecd9406b7e442426cb5368199035c27ff +F src/where.c c80c154db79ddbaff56ceb8879d96e71b090917d F test/8_3_names.test b93687beebd17f6ebf812405a6833bae5d1f4199 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 @@ -950,7 +950,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 45e581bff7a75db6c9a2c45b73d034d0b8a166d1 -R 6e4dc8fc887e24292adb3ae0b641a038 -U dan -Z d167de182be16055ea09a1a9326ef0ca +P 27c65d4d9c58bfc4ea8f9337fa15090459fb26c5 +R 31108ab0f55c59747082de8c46c3b2b9 +U drh +Z d273315218f9229b31cebba00fe83934 diff --git a/manifest.uuid b/manifest.uuid index 670b072715..e10d1d6e88 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27c65d4d9c58bfc4ea8f9337fa15090459fb26c5 \ No newline at end of file +ff9fc722dc481df1ce6a9733db1f033b88141d5d \ No newline at end of file diff --git a/src/where.c b/src/where.c index f843e10740..85a52fadff 100644 --- a/src/where.c +++ b/src/where.c @@ -1455,6 +1455,7 @@ static int isDistinctIndex( int i; /* Iterator variable */ if( pIdx->zName==0 || pDistinct==0 || pDistinct->nExpr>=BMS ) return 0; + testcase( pDistinct->nExpr==BMS-1 ); /* Loop through all the expressions in the distinct list. If any of them ** are not simple column references, return early. Otherwise, test if the From da6e42ab32ab32078d069e55e0e98c0a0391c9ef Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 4 Jul 2011 06:52:15 +0000 Subject: [PATCH 24/52] Adjust a couple of test scripts so that they work with OMIT_UTF16 builds. FossilOrigin-Name: 6c51bad0a306558d2725463b5675c3799fabcb9d --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- test/func3.test | 38 ++++++++++++++++++++------------------ test/sqllimits1.test | 20 ++++++++++++-------- 4 files changed, 41 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index 291bcdee80..15ccd8161f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stestcase\smacro\sto\sensure\stesting\sa\sboundary\scase\sin\sDISTINCT\sprocessing. -D 2011-07-02T19:12:05.917 +C Adjust\sa\scouple\sof\stest\sscripts\sso\sthat\sthey\swork\swith\sOMIT_UTF16\sbuilds. +D 2011-07-04T06:52:15.309 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -486,7 +486,7 @@ F test/fts3sort.test 63d52c1812904b751f9e1ff487472e44833f5402 F test/fts4aa.test 148d9eb54901af23b5d402b1f388f43e559e1728 F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f -F test/func3.test 7ba2ca5a1e9bca900ba2c230cf04bd67184bc1bc +F test/func3.test 001021e5b88bd02a3b365a5c5fd8f6f49d39744a F test/fuzz.test 77fd50afc12847af50fcf1941679d90adebadde6 F test/fuzz2.test 207d0f9d06db3eaf47a6b7bfc835b8e2fc397167 F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 @@ -680,7 +680,7 @@ F test/speed3.test 5a419039e9da95d906adb2298af2849600c81c11 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/sqllimits1.test e90a0ed94452076f6a10209d378e06b5f75ef0a0 +F test/sqllimits1.test 0ebf7bed0b99c96f24e0b7fa5e59dbc42359c421 F test/stat.test c7b20ea43003dc2dc33335e231c27be8284c4a2a F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9 F test/subquery.test b524f57c9574b2c0347045b4510ef795d4686796 @@ -950,7 +950,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 27c65d4d9c58bfc4ea8f9337fa15090459fb26c5 -R 31108ab0f55c59747082de8c46c3b2b9 -U drh -Z d273315218f9229b31cebba00fe83934 +P ff9fc722dc481df1ce6a9733db1f033b88141d5d +R f023392f9f22f7fc5dcd77586a5f9e95 +U dan +Z 70ce870889abf3a5e434fe4697977568 diff --git a/manifest.uuid b/manifest.uuid index e10d1d6e88..6a5de2af47 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ff9fc722dc481df1ce6a9733db1f033b88141d5d \ No newline at end of file +6c51bad0a306558d2725463b5675c3799fabcb9d \ No newline at end of file diff --git a/test/func3.test b/test/func3.test index e58d730f3a..d5a462fe33 100644 --- a/test/func3.test +++ b/test/func3.test @@ -17,24 +17,26 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -do_test func3-1.1 { - set destroyed 0 - proc destroy {} { set ::destroyed 1 } - sqlite3_create_function_v2 db f2 -1 any -func f2 -destroy destroy - set destroyed -} 0 -do_test func3-1.2 { - sqlite3_create_function_v2 db f2 -1 utf8 -func f2 - set destroyed -} 0 -do_test func3-1.3 { - sqlite3_create_function_v2 db f2 -1 utf16le -func f2 - set destroyed -} 0 -do_test func3-1.4 { - sqlite3_create_function_v2 db f2 -1 utf16be -func f2 - set destroyed -} 1 +ifcapable utf16 { + do_test func3-1.1 { + set destroyed 0 + proc destroy {} { set ::destroyed 1 } + sqlite3_create_function_v2 db f2 -1 any -func f2 -destroy destroy + set destroyed + } 0 + do_test func3-1.2 { + sqlite3_create_function_v2 db f2 -1 utf8 -func f2 + set destroyed + } 0 + do_test func3-1.3 { + sqlite3_create_function_v2 db f2 -1 utf16le -func f2 + set destroyed + } 0 + do_test func3-1.4 { + sqlite3_create_function_v2 db f2 -1 utf16be -func f2 + set destroyed + } 1 +} do_test func3-2.1 { set destroyed 0 diff --git a/test/sqllimits1.test b/test/sqllimits1.test index bf4a262678..8f2521353f 100644 --- a/test/sqllimits1.test +++ b/test/sqllimits1.test @@ -319,18 +319,22 @@ do_test sqllimits1-5.14.4 { catch {sqlite3_bind_text $::STMT 1 $::str1 -1} res set res } {SQLITE_TOOBIG} -do_test sqllimits1-5.14.5 { - catch {sqlite3_bind_text16 $::STMT 1 $::str1 -1} res - set res -} {SQLITE_TOOBIG} +ifcapable utf16 { + do_test sqllimits1-5.14.5 { + catch {sqlite3_bind_text16 $::STMT 1 $::str1 -1} res + set res + } {SQLITE_TOOBIG} +} do_test sqllimits1-5.14.6 { catch {sqlite3_bind_text $::STMT 1 $::str1 $np1} res set res } {SQLITE_TOOBIG} -do_test sqllimits1-5.14.7 { - catch {sqlite3_bind_text16 $::STMT 1 $::str1 $np1} res - set res -} {SQLITE_TOOBIG} +ifcapable utf16 { + do_test sqllimits1-5.14.7 { + catch {sqlite3_bind_text16 $::STMT 1 $::str1 $np1} res + set res + } {SQLITE_TOOBIG} +} do_test sqllimits1-5.14.8 { set n [expr {$np1-1}] catch {sqlite3_bind_text $::STMT 1 $::str1 $n} res From b0a43057fb23592e9564a398c258e7f258c4ae1a Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 6 Jul 2011 09:36:59 +0000 Subject: [PATCH 25/52] Expose the value of compile time symbol SQLITE_MAX_DEFAULT_PAGE_SIZE to test scripts. FossilOrigin-Name: 335736637c4ac3fe56a44fe01d4c037aaefa916a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/test_config.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 15ccd8161f..d689a8f657 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjust\sa\scouple\sof\stest\sscripts\sso\sthat\sthey\swork\swith\sOMIT_UTF16\sbuilds. -D 2011-07-04T06:52:15.309 +C Expose\sthe\svalue\sof\scompile\stime\ssymbol\sSQLITE_MAX_DEFAULT_PAGE_SIZE\sto\stest\sscripts. +D 2011-07-06T09:36:59.031 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -201,7 +201,7 @@ F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de F src/test_btree.c 47cd771250f09cdc6e12dda5bc71bc0b3abc96e2 -F src/test_config.c 791a9acbf9a695cf9c12ff6bbe874dfde9a8c5f3 +F src/test_config.c b4648b103586d2ae863056080c657680f6fa4825 F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_func.c cbdec5cededa0761daedde5baf06004a9bf416b5 @@ -950,7 +950,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P ff9fc722dc481df1ce6a9733db1f033b88141d5d -R f023392f9f22f7fc5dcd77586a5f9e95 +P 6c51bad0a306558d2725463b5675c3799fabcb9d +R d622adfd2b101711a0c600209c82490a U dan -Z 70ce870889abf3a5e434fe4697977568 +Z ba49e9f4aa222a31146e61a57307661a diff --git a/manifest.uuid b/manifest.uuid index 6a5de2af47..64b88f77b8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6c51bad0a306558d2725463b5675c3799fabcb9d \ No newline at end of file +335736637c4ac3fe56a44fe01d4c037aaefa916a \ No newline at end of file diff --git a/src/test_config.c b/src/test_config.c index a2796c772a..e8d6f88f62 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -571,6 +571,7 @@ Tcl_SetVar2(interp, "sqlite_options", "long_double", LINKVAR( DEFAULT_PAGE_SIZE ); LINKVAR( DEFAULT_FILE_FORMAT ); LINKVAR( MAX_ATTACHED ); + LINKVAR( MAX_DEFAULT_PAGE_SIZE ); { static const int cv_TEMP_STORE = SQLITE_TEMP_STORE; From f443e317f7828e922bb588aa0ca0dc829b9c39a7 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 7 Jul 2011 07:37:53 +0000 Subject: [PATCH 26/52] Remove accidental global variable "doclist" from fts3Int.h. FossilOrigin-Name: f244b2f3959e86fac555b91438cfe90528ab1648 --- ext/fts3/fts3Int.h | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 13434953e5..c85ef5b789 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -297,7 +297,7 @@ struct Fts3Doclist { int bFreeList; /* True if pList should be sqlite3_free()d */ char *pList; /* Pointer to position list following iDocid */ int nList; /* Length of position list */ -} doclist; +}; /* ** A "phrase" is a sequence of one or more tokens that must match in diff --git a/manifest b/manifest index d689a8f657..89d2fa15b5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Expose\sthe\svalue\sof\scompile\stime\ssymbol\sSQLITE_MAX_DEFAULT_PAGE_SIZE\sto\stest\sscripts. -D 2011-07-06T09:36:59.031 +C Remove\saccidental\sglobal\svariable\s"doclist"\sfrom\sfts3Int.h. +D 2011-07-07T07:37:53.958 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -64,7 +64,7 @@ F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d F ext/fts3/fts3.c f45ad45053a587ad1c005459b704b7ade8bd504e F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h e670e2c58b3dca76e44f7e7d35e61113d079db58 +F ext/fts3/fts3Int.h d18022edfa82fd33d8fee35c1b1990f3cf937a1e F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 F ext/fts3/fts3_expr.c 23791de01b3a5d313d76e02befd2601d4096bc2b F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 @@ -950,7 +950,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 6c51bad0a306558d2725463b5675c3799fabcb9d -R d622adfd2b101711a0c600209c82490a +P 335736637c4ac3fe56a44fe01d4c037aaefa916a +R b695993b25d82559c7810189a37bcfab U dan -Z ba49e9f4aa222a31146e61a57307661a +Z 2063226294bddc8bcb5afad858fcd8e0 diff --git a/manifest.uuid b/manifest.uuid index 64b88f77b8..a846e9ae62 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -335736637c4ac3fe56a44fe01d4c037aaefa916a \ No newline at end of file +f244b2f3959e86fac555b91438cfe90528ab1648 \ No newline at end of file From e7152dc76b3c6ee3868c9e8741a1ca9625c17852 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 7 Jul 2011 08:19:16 +0000 Subject: [PATCH 27/52] Add a target to main.mk that will fail if the amalgamation contains any exported symbols that do not begin with "sqlite3_". Run this target from within releasetest.tcl. Add "static" to a couple of private functions in mem3.c. FossilOrigin-Name: a68b6580c7d436c896a644a475cdba33c2c8d6e6 --- main.mk | 7 +++++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/mem3.c | 6 +++--- test/releasetest.tcl | 3 ++- 5 files changed, 21 insertions(+), 13 deletions(-) diff --git a/main.mk b/main.mk index 33b2e6ec14..16bb7115c9 100644 --- a/main.mk +++ b/main.mk @@ -565,6 +565,13 @@ $(TEST_EXTENSION): $(TOP)/src/test_loadext.c extensiontest: testfixture$(EXE) $(TEST_EXTENSION) ./testfixture$(EXE) $(TOP)/test/loadext.test +# This target will fail if the SQLite amalgamation contains any exported +# symbols that do not begin with "sqlite3_". It is run as part of the +# releasetest.tcl script. +# +checksymbols: sqlite3.o + nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0 + # Standard install and cleanup targets # diff --git a/manifest b/manifest index 89d2fa15b5..89d0ab3824 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\saccidental\sglobal\svariable\s"doclist"\sfrom\sfts3Int.h. -D 2011-07-07T07:37:53.958 +C Add\sa\starget\sto\smain.mk\sthat\swill\sfail\sif\sthe\samalgamation\scontains\sany\sexported\ssymbols\sthat\sdo\snot\sbegin\swith\s"sqlite3_".\sRun\sthis\starget\sfrom\swithin\sreleasetest.tcl.\sAdd\s"static"\sto\sa\scouple\sof\sprivate\sfunctions\sin\smem3.c. +D 2011-07-07T08:19:16.876 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -104,7 +104,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk d799fb600383cee11fed58876a7cccb749dd2614 +F main.mk d81d86f0f70444f3abc241eccf5ace4a79ff9b69 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -151,7 +151,7 @@ F src/malloc.c 591aedb20ae40813f1045f2ef253438a334775d9 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 00bd8265c81abb665c48fea1e0c234eb3b922206 F src/mem2.c e307323e86b5da1853d7111b68fd6b84ad6f09cf -F src/mem3.c 9b237d911ba9904142a804be727cc6664873f8a3 +F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 F src/mem5.c c2c63b7067570b00bf33d751c39af24182316f7f F src/memjournal.c 0ebce851677a7ac035ba1512a7e65851b34530c6 F src/mutex.c 6949180803ff05a7d0e2b9334a95b4fb5a00e23f @@ -630,7 +630,7 @@ F test/randexpr1.test 1084050991e9ba22c1c10edd8d84673b501cc25a F test/rdonly.test c267d050a1d9a6a321de502b737daf28821a518d F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a -F test/releasetest.tcl c0c0865f1dff08dde08a964ef49e83217ebedbf8 +F test/releasetest.tcl fa302d03fd9acfce6d910553a33473bfcf561958 F test/rollback.test 1a83118ea6db4e7d8c10eaa63871b5e90502ffdc F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test e58e0acef38b527ed1b0b70d3ada588f804af287 @@ -950,7 +950,7 @@ F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 335736637c4ac3fe56a44fe01d4c037aaefa916a -R b695993b25d82559c7810189a37bcfab +P f244b2f3959e86fac555b91438cfe90528ab1648 +R 6592aa0ef1e7319ccd9410585affa7e2 U dan -Z 2063226294bddc8bcb5afad858fcd8e0 +Z ed79b421c9d7255d4a5ce5800ff3667b diff --git a/manifest.uuid b/manifest.uuid index a846e9ae62..a1121262cc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f244b2f3959e86fac555b91438cfe90528ab1648 \ No newline at end of file +a68b6580c7d436c896a644a475cdba33c2c8d6e6 \ No newline at end of file diff --git a/src/mem3.c b/src/mem3.c index e2d66815b3..1a1b791f28 100644 --- a/src/mem3.c +++ b/src/mem3.c @@ -433,7 +433,7 @@ static void *memsys3MallocUnsafe(int nByte){ ** This function assumes that the necessary mutexes, if any, are ** already held by the caller. Hence "Unsafe". */ -void memsys3FreeUnsafe(void *pOld){ +static void memsys3FreeUnsafe(void *pOld){ Mem3Block *p = (Mem3Block*)pOld; int i; u32 size, x; @@ -508,7 +508,7 @@ static void *memsys3Malloc(int nBytes){ /* ** Free memory. */ -void memsys3Free(void *pPrior){ +static void memsys3Free(void *pPrior){ assert( pPrior ); memsys3Enter(); memsys3FreeUnsafe(pPrior); @@ -518,7 +518,7 @@ void memsys3Free(void *pPrior){ /* ** Change the size of an existing memory allocation */ -void *memsys3Realloc(void *pPrior, int nBytes){ +static void *memsys3Realloc(void *pPrior, int nBytes){ int nOld; void *p; if( pPrior==0 ){ diff --git a/test/releasetest.tcl b/test/releasetest.tcl index 84542bb8a5..7725630187 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -155,10 +155,10 @@ array set ::Configs { array set ::Platforms { Linux-x86_64 { + "Debug-One" "checksymbols test" "Secure-Delete" test "Unlock-Notify" "QUICKTEST_INCLUDE=notify2.test test" "Update-Delete-Limit" test - "Debug-One" test "Extra-Robustness" test "Device-Two" test "Ftrapv" test @@ -177,6 +177,7 @@ array set ::Platforms { } } + # End of configuration section. ######################################################################### ######################################################################### From e363c1d1879ebb14d41d2b6bf83a42d68f2bf253 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 7 Jul 2011 08:52:24 +0000 Subject: [PATCH 28/52] Update the tool/symbols.sh script to check for accidentally exported global variables. FossilOrigin-Name: 71b749a9dca953cdf671d94afec16e2c98f3aee5 --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/symbols.sh | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 89d0ab3824..d7349dfb9a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\starget\sto\smain.mk\sthat\swill\sfail\sif\sthe\samalgamation\scontains\sany\sexported\ssymbols\sthat\sdo\snot\sbegin\swith\s"sqlite3_".\sRun\sthis\starget\sfrom\swithin\sreleasetest.tcl.\sAdd\s"static"\sto\sa\scouple\sof\sprivate\sfunctions\sin\smem3.c. -D 2011-07-07T08:19:16.876 +C Update\sthe\stool/symbols.sh\sscript\sto\scheck\sfor\saccidentally\sexported\nglobal\svariables. +D 2011-07-07T08:52:24.692 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -946,11 +946,11 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c -F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 +F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P f244b2f3959e86fac555b91438cfe90528ab1648 -R 6592aa0ef1e7319ccd9410585affa7e2 -U dan -Z ed79b421c9d7255d4a5ce5800ff3667b +P a68b6580c7d436c896a644a475cdba33c2c8d6e6 +R 20857ee0c494bf7ca6cccf72c0173c3f +U drh +Z ead2bbb7b49847c65c37c7bbdac703c9 diff --git a/manifest.uuid b/manifest.uuid index a1121262cc..bff787855f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a68b6580c7d436c896a644a475cdba33c2c8d6e6 \ No newline at end of file +71b749a9dca953cdf671d94afec16e2c98f3aee5 \ No newline at end of file diff --git a/tool/symbols.sh b/tool/symbols.sh index 8aec00569a..74b1243e6e 100644 --- a/tool/symbols.sh +++ b/tool/symbols.sh @@ -12,10 +12,10 @@ gcc -c -DSQLITE_ENABLE_FTS3 -DSQLITE_ENABLE_RTREE \ -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_ENABLE_ATOMIC_WRITE \ -DSQLITE_ENABLE_ICU \ sqlite3.c -nm sqlite3.o | grep ' T ' | sort -k 3 +nm sqlite3.o | grep ' [TD] ' | sort -k 3 echo '****** Surplus symbols from a build including RTREE, FTS4 & ICU ******' -nm sqlite3.o | grep ' T ' | grep -v ' sqlite3_' +nm sqlite3.o | grep ' [TD] ' | grep -v ' .*sqlite3_' echo '****** Dependencies of the core. No extensions. No OS interface *******' gcc -c -DSQLITE_ENABLE_MEMORY_MANAGEMENT -DSQLITE_ENABLE_STAT2 \ From a9b1b914362d21ab6367294f28a1a625f082b100 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 8 Jul 2011 13:07:02 +0000 Subject: [PATCH 29/52] Extend the SQLITE_TESTCTRL_OPTIMIZATIONS option to disable DISTINCT optimizations. FossilOrigin-Name: 18501dd1a8aa8299c091c36e35b9a10a29d4467b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqliteInt.h | 1 + src/where.c | 4 ++++ 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d7349dfb9a..9f98ca8728 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\stool/symbols.sh\sscript\sto\scheck\sfor\saccidentally\sexported\nglobal\svariables. -D 2011-07-07T08:52:24.692 +C Extend\sthe\sSQLITE_TESTCTRL_OPTIMIZATIONS\soption\sto\sdisable\sDISTINCT\noptimizations. +D 2011-07-08T13:07:02.960 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -183,7 +183,7 @@ F src/select.c e9d74f943c195f2673990febd3a455b421964ca0 F src/shell.c 0e0173b3e79d956368013e759f084caa7995ecb1 F src/sqlite.h.in 4b7255c10d39c5faf089dbd29cde7c367ff39f1f F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 -F src/sqliteInt.h 58df570a42d605f8e53d6eb920a16ca291d5822e +F src/sqliteInt.h a7143a30c5817e5ed6cde42430255face266a710 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -250,7 +250,7 @@ F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 0c70ad7b1cac6005fa5e2cbefd23ee05e391c290 F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f -F src/where.c c80c154db79ddbaff56ceb8879d96e71b090917d +F src/where.c c814c35a2731ff1f636bce7a628be91b0d77d734 F test/8_3_names.test b93687beebd17f6ebf812405a6833bae5d1f4199 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 @@ -950,7 +950,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P a68b6580c7d436c896a644a475cdba33c2c8d6e6 -R 20857ee0c494bf7ca6cccf72c0173c3f +P 71b749a9dca953cdf671d94afec16e2c98f3aee5 +R 26528fdf903671e1d2a91c9d7db991cc U drh -Z ead2bbb7b49847c65c37c7bbdac703c9 +Z 1d6ed3edda2aa263aea05f58878a859f diff --git a/manifest.uuid b/manifest.uuid index bff787855f..d4fcb0a135 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -71b749a9dca953cdf671d94afec16e2c98f3aee5 \ No newline at end of file +18501dd1a8aa8299c091c36e35b9a10a29d4467b \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index c1e055b267..9f47ffddec 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -956,6 +956,7 @@ struct sqlite3 { #define SQLITE_GroupByOrder 0x20 /* Disable GROUPBY cover of ORDERBY */ #define SQLITE_FactorOutConst 0x40 /* Disable factoring out constants */ #define SQLITE_IdxRealAsInt 0x80 /* Store REAL as INT in indices */ +#define SQLITE_DistinctOpt 0x80 /* DISTINCT using indexes */ #define SQLITE_OptMask 0xff /* Mask of all disablable opts */ /* diff --git a/src/where.c b/src/where.c index 85a52fadff..05d955bada 100644 --- a/src/where.c +++ b/src/where.c @@ -4598,6 +4598,10 @@ WhereInfo *sqlite3WhereBegin( pWInfo->savedNQueryLoop = pParse->nQueryLoop; pMaskSet = (WhereMaskSet*)&pWC[1]; + /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via + ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ + if( db->flags & SQLITE_DistinctOpt ) pDistinct = 0; + /* Split the WHERE clause into separate subexpressions where each ** subexpression is separated by an AND operator. */ From da79cf0cc9cc00588782f771b76b087460507586 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 8 Jul 2011 16:10:54 +0000 Subject: [PATCH 30/52] Fix for [54844eea3f]: Do not create automatic indexes on correlated sub-queries. FossilOrigin-Name: 9f14fa56ba31afe3de8b0cf26ed09573a2cc2283 --- manifest | 21 +++++++------ manifest.uuid | 2 +- src/expr.c | 1 + src/resolve.c | 14 +++++++++ src/sqliteInt.h | 1 + src/where.c | 4 +++ test/tkt-54844eea3f.test | 67 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 99 insertions(+), 11 deletions(-) create mode 100644 test/tkt-54844eea3f.test diff --git a/manifest b/manifest index 9f98ca8728..7bc4892cad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Extend\sthe\sSQLITE_TESTCTRL_OPTIMIZATIONS\soption\sto\sdisable\sDISTINCT\noptimizations. -D 2011-07-08T13:07:02.960 +C Fix\sfor\s[54844eea3f]:\sDo\snot\screate\sautomatic\sindexes\son\scorrelated\ssub-queries. +D 2011-07-08T16:10:54.482 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -133,7 +133,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 7deec4534f3b5a0c3b4a4cbadf809d321f64f9c4 F src/date.c a3c6842bad7ae632281811de112a8ba63ff08ab3 F src/delete.c ff68e5ef23aee08c0ff528f699a19397ed8bbed8 -F src/expr.c ab46ab0f0c44979a8164ca31728d7d10ae5e8106 +F src/expr.c 4bbdfaf66bc614be9254ce0c26a17429067a3e07 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c c8492fed772af1ed61251582707266227612b45b F src/func.c 59bb046d7e3df1ab512ac339ccb0a6f996a17cb7 @@ -177,13 +177,13 @@ F src/pragma.c ebcd20f1e654f5cb3aeef864ed69c4697719fbaa F src/prepare.c e64261559a3187698a3e7e6c8b001a4f4f98dab4 F src/printf.c 585a36b6a963df832cfb69505afa3a34ed5ef8a1 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 -F src/resolve.c 1c0f32b64f8e3f555fe1f732f9d6f501a7f05706 +F src/resolve.c 36368f44569208fa074e61f4dd0b6c4fb60ca2b4 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c e9d74f943c195f2673990febd3a455b421964ca0 F src/shell.c 0e0173b3e79d956368013e759f084caa7995ecb1 F src/sqlite.h.in 4b7255c10d39c5faf089dbd29cde7c367ff39f1f F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 -F src/sqliteInt.h a7143a30c5817e5ed6cde42430255face266a710 +F src/sqliteInt.h ba4a6d6288efb25b84bc0d7d0aaf80f9b42523ba F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -250,7 +250,7 @@ F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 0c70ad7b1cac6005fa5e2cbefd23ee05e391c290 F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f -F src/where.c c814c35a2731ff1f636bce7a628be91b0d77d734 +F src/where.c defae982c6a5ced7c9a972a5a4850ad140a6d631 F test/8_3_names.test b93687beebd17f6ebf812405a6833bae5d1f4199 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 @@ -718,6 +718,7 @@ F test/tkt-38cb5df375.test 9e9b19857dba0896a8efdaf334d405ba423492f2 F test/tkt-3998683a16.test 6d1d04d551ed1704eb3396ca87bb9ccc8c5c1eb7 F test/tkt-3fe897352e.test 10de1a67bd5c66b238a4c96abe55531b37bb4f00 F test/tkt-4a03edc4c8.test 2865e4edbc075b954daa82f8da7cc973033ec76e +F test/tkt-54844eea3f.test a12b851128f46a695e4e378cca67409b9b8f5894 F test/tkt-5d863f876e.test 884072c2de496ddbb90c387c9ebc0d4f44a91b8e F test/tkt-5e10420e8d.test 904d1687b3c06d43e5b3555bbcf6802e7c0ffd84 F test/tkt-5ee23731f.test 3581260f2a71e51db94e1506ba6b0f7311d002a9 @@ -950,7 +951,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 71b749a9dca953cdf671d94afec16e2c98f3aee5 -R 26528fdf903671e1d2a91c9d7db991cc -U drh -Z 1d6ed3edda2aa263aea05f58878a859f +P 18501dd1a8aa8299c091c36e35b9a10a29d4467b +R f7d91718025095a30a469ac579a4e46b +U dan +Z fc4423411c04e2aaef6760d45d2bd142 diff --git a/manifest.uuid b/manifest.uuid index d4fcb0a135..16b8760bdc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -18501dd1a8aa8299c091c36e35b9a10a29d4467b \ No newline at end of file +9f14fa56ba31afe3de8b0cf26ed09573a2cc2283 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index be2f4d7c66..ab4547db9c 100644 --- a/src/expr.c +++ b/src/expr.c @@ -902,6 +902,7 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ pNewItem->jointype = pOldItem->jointype; pNewItem->iCursor = pOldItem->iCursor; pNewItem->isPopulated = pOldItem->isPopulated; + pNewItem->isCorrelated = pOldItem->isCorrelated; pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex); pNewItem->notIndexed = pOldItem->notIndexed; pNewItem->pIndex = pOldItem->pIndex; diff --git a/src/resolve.c b/src/resolve.c index 74d6aaef93..d29d2a8344 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -996,11 +996,25 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ for(i=0; ipSrc->nSrc; i++){ struct SrcList_item *pItem = &p->pSrc->a[i]; if( pItem->pSelect ){ + NameContext *pNC; /* Used to iterate name contexts */ + int nRef = 0; /* Refcount for pOuterNC and outer contexts */ const char *zSavedContext = pParse->zAuthContext; + + /* Count the total number of references to pOuterNC and all of its + ** parent contexts. After resolving references to expressions in + ** pItem->pSelect, check if this value has changed. If so, then + ** SELECT statement pItem->pSelect must be correlated. Set the + ** pItem->isCorrelated flag if this is the case. */ + for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef += pNC->nRef; + if( pItem->zName ) pParse->zAuthContext = pItem->zName; sqlite3ResolveSelectNames(pParse, pItem->pSelect, pOuterNC); pParse->zAuthContext = zSavedContext; if( pParse->nErr || db->mallocFailed ) return WRC_Abort; + + for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef -= pNC->nRef; + assert( pItem->isCorrelated==0 && nRef<=0 ); + pItem->isCorrelated = (nRef!=0); } } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9f47ffddec..bcf6a591af 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1848,6 +1848,7 @@ struct SrcList { u8 isPopulated; /* Temporary table associated with SELECT is populated */ u8 jointype; /* Type of join between this able and the previous */ u8 notIndexed; /* True if there is a NOT INDEXED clause */ + u8 isCorrelated; /* True if sub-query is correlated */ #ifndef SQLITE_OMIT_EXPLAIN u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */ #endif diff --git a/src/where.c b/src/where.c index 05d955bada..918a3df5d5 100644 --- a/src/where.c +++ b/src/where.c @@ -1919,6 +1919,10 @@ static void bestAutomaticIndex( /* The NOT INDEXED clause appears in the SQL. */ return; } + if( pSrc->isCorrelated ){ + /* The source is a correlated sub-query. No point in indexing it. */ + return; + } assert( pParse->nQueryLoop >= (double)1 ); pTable = pSrc->pTab; diff --git a/test/tkt-54844eea3f.test b/test/tkt-54844eea3f.test new file mode 100644 index 0000000000..7b04f6935d --- /dev/null +++ b/test/tkt-54844eea3f.test @@ -0,0 +1,67 @@ +# 2011 July 8 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing that bug [54844eea3f] has been fixed. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +set ::testprefix tkt-54844eea3f + +do_test 1.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY); + INSERT INTO t1 VALUES(1); + INSERT INTO t1 VALUES(4); + + CREATE TABLE t2(b INTEGER PRIMARY KEY); + INSERT INTO t2 VALUES(1); + INSERT INTO t2 VALUES(2); + INSERT INTO t2 SELECT b+2 FROM t2; + INSERT INTO t2 SELECT b+4 FROM t2; + INSERT INTO t2 SELECT b+8 FROM t2; + INSERT INTO t2 SELECT b+16 FROM t2; + + CREATE TABLE t3(c INTEGER PRIMARY KEY); + INSERT INTO t3 VALUES(1); + INSERT INTO t3 VALUES(2); + INSERT INTO t3 VALUES(3); + } +} {} + +do_test 1.1 { + execsql { + SELECT 'test-2', t3.c, ( + SELECT count(*) + FROM t1 JOIN (SELECT DISTINCT t3.c AS p FROM t2) AS x ON t1.a=x.p + ) + FROM t3; + } +} {test-2 1 1 test-2 2 0 test-2 3 0} + +do_test 1.2 { + execsql { + CREATE TABLE t4(a, b, c); + INSERT INTO t4 VALUES('a', 1, 'one'); + INSERT INTO t4 VALUES('a', 2, 'two'); + INSERT INTO t4 VALUES('b', 1, 'three'); + INSERT INTO t4 VALUES('b', 2, 'four'); + SELECT ( + SELECT c FROM ( + SELECT * FROM t4 WHERE a=out.a ORDER BY b LIMIT 10 OFFSET 1 + ) WHERE b=out.b + ) FROM t4 AS out; + } +} {{} two {} four} + + +finish_test From 6f25e89d27072d8abcd481e8e58176ebfd8ff63c Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 8 Jul 2011 17:02:57 +0000 Subject: [PATCH 31/52] Change the default chunk size on test_multiplex.c to 2147418112 bytes (formerly 1073741824 bytes) and make the default configurable at compile-time using SQLITE_MULTIPLEX_CHUNK_SIZE and at run-time using the "chunksize" URI query parameter. Add support fo test_multiplex to the shell. FossilOrigin-Name: e305b5a931374e2a1d2e66ea0a1248b9d4aecb19 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/shell.c | 14 ++++++++++++++ src/test_multiplex.c | 21 +++++++++++++++++---- 4 files changed, 40 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 7bc4892cad..5bea541a60 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sfor\s[54844eea3f]:\sDo\snot\screate\sautomatic\sindexes\son\scorrelated\ssub-queries. -D 2011-07-08T16:10:54.482 +C Change\sthe\sdefault\schunk\ssize\son\stest_multiplex.c\sto\s2147418112\sbytes\n(formerly\s1073741824\sbytes)\sand\smake\sthe\sdefault\sconfigurable\sat\scompile-time\nusing\sSQLITE_MULTIPLEX_CHUNK_SIZE\sand\sat\srun-time\susing\sthe\s"chunksize"\nURI\squery\sparameter.\s\sAdd\ssupport\sfo\stest_multiplex\sto\sthe\sshell. +D 2011-07-08T17:02:57.508 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -180,7 +180,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 36368f44569208fa074e61f4dd0b6c4fb60ca2b4 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c e9d74f943c195f2673990febd3a455b421964ca0 -F src/shell.c 0e0173b3e79d956368013e759f084caa7995ecb1 +F src/shell.c bbe7818ff5bc8614105ceb81ad67b8bdc0b671dd F src/sqlite.h.in 4b7255c10d39c5faf089dbd29cde7c367ff39f1f F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 F src/sqliteInt.h ba4a6d6288efb25b84bc0d7d0aaf80f9b42523ba @@ -213,7 +213,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c 03313c693cca72959dcaaf79f8d76f21c01e19ff F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 7ca7be34e0e09ef0ed6619544552ed95732e41f6 -F src/test_multiplex.c a7457a1ac46964b7f897d75ecfd447410ec067e6 +F src/test_multiplex.c 0585bb12a98521795344f67d17d77e258ea7b57f F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec @@ -951,7 +951,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 18501dd1a8aa8299c091c36e35b9a10a29d4467b -R f7d91718025095a30a469ac579a4e46b -U dan -Z fc4423411c04e2aaef6760d45d2bd142 +P 9f14fa56ba31afe3de8b0cf26ed09573a2cc2283 +R e83eed2583035b436dc963e6112fe3df +U drh +Z 8b9b6b1bb1eb76d53c04bd9c864821f0 diff --git a/manifest.uuid b/manifest.uuid index 16b8760bdc..2800aec7ee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9f14fa56ba31afe3de8b0cf26ed09573a2cc2283 \ No newline at end of file +e305b5a931374e2a1d2e66ea0a1248b9d4aecb19 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index a54c922e87..05272b2dfa 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2632,6 +2632,9 @@ static const char zOptions[] = #ifdef SQLITE_ENABLE_VFSTRACE " -vfstrace enable tracing of all VFS calls\n" #endif +#ifdef SQLITE_ENABLE_MULTIPLEX + " -multiplex enable the multiplexor VFS\n" +#endif ; static void usage(int showDetail){ fprintf(stderr, @@ -2732,6 +2735,11 @@ int main(int argc, char **argv){ int makeDefault ); vfstrace_register("trace",0,(int(*)(const char*,void*))fputs,stderr,1); +#endif +#ifdef SQLITE_ENABLE_MULTIPLEX + }else if( strcmp(argv[i],"-multiplex")==0 ){ + extern int sqlite3_multiple_initialize(const char*,int); + sqlite3_multiplex_initialize(0, 1); #endif }else if( strcmp(argv[i],"-vfs")==0 ){ sqlite3_vfs *pVfs = sqlite3_vfs_find(argv[++i]); @@ -2851,8 +2859,14 @@ int main(int argc, char **argv){ i++; }else if( strcmp(z,"-vfs")==0 ){ i++; +#ifdef SQLITE_ENABLE_VFSTRACE }else if( strcmp(z,"-vfstrace")==0 ){ i++; +#endif +#ifdef SQLITE_ENABLE_MULTIPLEX + }else if( strcmp(z,"-multiplex")==0 ){ + i++; +#endif }else if( strcmp(z,"-help")==0 || strcmp(z, "--help")==0 ){ usage(1); }else{ diff --git a/src/test_multiplex.c b/src/test_multiplex.c index f709c9a99f..60ccbcc35f 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -45,6 +45,7 @@ #include "sqlite3.h" #include #include +#include #include "test_multiplex.h" #ifndef SQLITE_CORE @@ -78,20 +79,26 @@ /************************ Shim Definitions ******************************/ -#define SQLITE_MULTIPLEX_VFS_NAME "multiplex" +#ifndef SQLITE_MULTIPLEX_VFS_NAME +# define SQLITE_MULTIPLEX_VFS_NAME "multiplex" +#endif /* This is the limit on the chunk size. It may be changed by calling ** the xFileControl() interface. It will be rounded up to a -** multiple of MAX_PAGE_SIZE. We default it here to 1GB. +** multiple of MAX_PAGE_SIZE. We default it here to 2GiB less 64KiB. */ -#define SQLITE_MULTIPLEX_CHUNK_SIZE (MAX_PAGE_SIZE*16384) +#ifndef SQLITE_MULTIPLEX_CHUNK_SIZE +# define SQLITE_MULTIPLEX_CHUNK_SIZE 2147418112 +#endif /* Default limit on number of chunks. Care should be taken ** so that values for chunks numbers fit in the SQLITE_MULTIPLEX_EXT_FMT ** format specifier. It may be changed by calling ** the xFileControl() interface. */ -#define SQLITE_MULTIPLEX_MAX_CHUNKS 32 +#ifndef SQLITE_MULTIPLEX_MAX_CHUNKS +# define SQLITE_MULTIPLEX_MAX_CHUNKS 32 +#endif /* If SQLITE_MULTIPLEX_EXT_OVWR is defined, the ** last SQLITE_MULTIPLEX_EXT_SZ characters of the @@ -431,12 +438,18 @@ static int multiplexOpen( } if( rc==SQLITE_OK ){ + const char *zChunkSize; /* assign pointers to extra space allocated */ char *p = (char *)&pGroup[1]; pMultiplexOpen->pGroup = pGroup; memset(pGroup, 0, sz); pGroup->bEnabled = -1; pGroup->nChunkSize = SQLITE_MULTIPLEX_CHUNK_SIZE; + zChunkSize = sqlite3_uri_parameter(zName, "chunksize"); + if( zChunkSize ){ + int n = atoi(zChunkSize); + if( n>0 ) pGroup->nChunkSize = (n+0xffff)&~0xffff; + } pGroup->nMaxChunks = SQLITE_MULTIPLEX_MAX_CHUNKS; pGroup->pReal = (sqlite3_file **)p; p += (sizeof(sqlite3_file *)*pGroup->nMaxChunks); From f1d3e329091ae39048728620337a4827c37b0eb1 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 9 Jul 2011 13:00:41 +0000 Subject: [PATCH 32/52] In where.c::findIndexCol - make sure that the Expr.op is TK_COLUMN before accessing the Expr.iColumn and Expr.iTable fields. Also fix a couple of unreachable branches. FossilOrigin-Name: 418a4da2a96cf33055f18c9a667754fad2111cf3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 9 ++++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 5bea541a60..e0235e287e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sdefault\schunk\ssize\son\stest_multiplex.c\sto\s2147418112\sbytes\n(formerly\s1073741824\sbytes)\sand\smake\sthe\sdefault\sconfigurable\sat\scompile-time\nusing\sSQLITE_MULTIPLEX_CHUNK_SIZE\sand\sat\srun-time\susing\sthe\s"chunksize"\nURI\squery\sparameter.\s\sAdd\ssupport\sfo\stest_multiplex\sto\sthe\sshell. -D 2011-07-08T17:02:57.508 +C In\swhere.c::findIndexCol\s-\smake\ssure\sthat\sthe\sExpr.op\sis\sTK_COLUMN\sbefore\naccessing\sthe\sExpr.iColumn\sand\sExpr.iTable\sfields.\s\sAlso\sfix\sa\scouple\sof\nunreachable\sbranches. +D 2011-07-09T13:00:41.795 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -250,7 +250,7 @@ F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 0c70ad7b1cac6005fa5e2cbefd23ee05e391c290 F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f -F src/where.c defae982c6a5ced7c9a972a5a4850ad140a6d631 +F src/where.c ce7cce80c5cb07ba40e9bf38a33ef806c61f55f2 F test/8_3_names.test b93687beebd17f6ebf812405a6833bae5d1f4199 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 @@ -951,7 +951,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 9f14fa56ba31afe3de8b0cf26ed09573a2cc2283 -R e83eed2583035b436dc963e6112fe3df +P e305b5a931374e2a1d2e66ea0a1248b9d4aecb19 +R 9f2366d63a9041cf0166a322890b593c U drh -Z 8b9b6b1bb1eb76d53c04bd9c864821f0 +Z 1af89a162ab75877f31c3bef94a3cd45 diff --git a/manifest.uuid b/manifest.uuid index 2800aec7ee..ca3a56f8b2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e305b5a931374e2a1d2e66ea0a1248b9d4aecb19 \ No newline at end of file +418a4da2a96cf33055f18c9a667754fad2111cf3 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 918a3df5d5..bb3bee357c 100644 --- a/src/where.c +++ b/src/where.c @@ -1420,9 +1420,12 @@ static int findIndexCol( for(i=0; inExpr; i++){ Expr *p = pList->a[i].pExpr; - if( pIdx->aiColumn[iCol]==p->iColumn && iBase==p->iTable ){ + if( p->op==TK_COLUMN + && p->iColumn==pIdx->aiColumn[iCol] + && p->iTable==iBase + ){ CollSeq *pColl = sqlite3ExprCollSeq(pParse, p); - if( pColl && 0==sqlite3StrICmp(pColl->zName, zColl) ){ + if( ALWAYS(pColl) && 0==sqlite3StrICmp(pColl->zName, zColl) ){ return i; } } @@ -1680,7 +1683,7 @@ static int isSortingIndex( } } - if( pbRev ) *pbRev = sortOrder!=0; + *pbRev = sortOrder!=0; if( j>=nTerm ){ /* All terms of the ORDER BY clause are covered by this index so ** this index can be used for sorting. */ From ec661058ddb96df5bcb3db107a32fd8af064434d Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 9 Jul 2011 16:17:18 +0000 Subject: [PATCH 33/52] Fix harmless compiler warnings on unix. FossilOrigin-Name: 90b1aea17400bbda5ebc8ae4eb4e12127519e42e --- ext/fts3/fts3Int.h | 8 -------- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 3 ++- 4 files changed, 10 insertions(+), 17 deletions(-) diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index c85ef5b789..ed8043adf6 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -497,14 +497,6 @@ int sqlite3Fts3InitTerm(sqlite3 *db); /* fts3_aux.c */ int sqlite3Fts3InitAux(sqlite3 *db); -int sqlite3Fts3TermSegReaderCursor( - Fts3Cursor *pCsr, /* Virtual table cursor handle */ - const char *zTerm, /* Term to query for */ - int nTerm, /* Size of zTerm in bytes */ - int isPrefix, /* True for a prefix search */ - Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ -); - void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); int sqlite3Fts3MsrIncrStart( diff --git a/manifest b/manifest index e0235e287e..4f1bbb1b0b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\swhere.c::findIndexCol\s-\smake\ssure\sthat\sthe\sExpr.op\sis\sTK_COLUMN\sbefore\naccessing\sthe\sExpr.iColumn\sand\sExpr.iTable\sfields.\s\sAlso\sfix\sa\scouple\sof\nunreachable\sbranches. -D 2011-07-09T13:00:41.795 +C Fix\sharmless\scompiler\swarnings\son\sunix. +D 2011-07-09T16:17:18.236 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -64,7 +64,7 @@ F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d F ext/fts3/fts3.c f45ad45053a587ad1c005459b704b7ade8bd504e F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h d18022edfa82fd33d8fee35c1b1990f3cf937a1e +F ext/fts3/fts3Int.h 30063fdd0bc433b5db1532e3a363cb0f2f7e8eb3 F ext/fts3/fts3_aux.c 0ebfa7b86cf8ff6a0861605fcc63b83ec1b70691 F ext/fts3/fts3_expr.c 23791de01b3a5d313d76e02befd2601d4096bc2b F ext/fts3/fts3_hash.c 8dd2d06b66c72c628c2732555a32bc0943114914 @@ -179,7 +179,7 @@ F src/printf.c 585a36b6a963df832cfb69505afa3a34ed5ef8a1 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 36368f44569208fa074e61f4dd0b6c4fb60ca2b4 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 -F src/select.c e9d74f943c195f2673990febd3a455b421964ca0 +F src/select.c d219c4b68d603cc734b6f9b1e2780fee12a1fa0d F src/shell.c bbe7818ff5bc8614105ceb81ad67b8bdc0b671dd F src/sqlite.h.in 4b7255c10d39c5faf089dbd29cde7c367ff39f1f F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 @@ -951,7 +951,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P e305b5a931374e2a1d2e66ea0a1248b9d4aecb19 -R 9f2366d63a9041cf0166a322890b593c +P 418a4da2a96cf33055f18c9a667754fad2111cf3 +R bd1403a9e638958833ef8e7e7178479a U drh -Z 1af89a162ab75877f31c3bef94a3cd45 +Z 18ede22d9ea411d29db6ab17e40995c3 diff --git a/manifest.uuid b/manifest.uuid index ca3a56f8b2..0650cd683d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -418a4da2a96cf33055f18c9a667754fad2111cf3 \ No newline at end of file +90b1aea17400bbda5ebc8ae4eb4e12127519e42e \ No newline at end of file diff --git a/src/select.c b/src/select.c index 45f1c7545d..bd5e964e7c 100644 --- a/src/select.c +++ b/src/select.c @@ -3925,7 +3925,7 @@ int sqlite3Select( (char*)pKeyInfo, P4_KEYINFO_HANDOFF); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); }else{ - distinct = -1; + distinct = addrDistinctIndex = -1; } /* Aggregate and non-aggregate queries are handled differently */ @@ -3949,6 +3949,7 @@ int sqlite3Select( if( pWInfo->eDistinct ){ VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ + assert( addrDistinctIndex>0 ); pOp = sqlite3VdbeGetOp(v, addrDistinctIndex); assert( isDistinct ); From 5d9ef0a612105c623790418d46967b420fc530ad Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 11 Jul 2011 18:17:56 +0000 Subject: [PATCH 34/52] Change the windows backend to retry read and write requests if the encounter ERROR_LOCK_VIOLATION and ERROR_SHARING_VIOLATION errors - which we think sometimes happens due to agressive anti-virus software. FossilOrigin-Name: c20aca06610407c197ea50ea77c2591aacf2252a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 39 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 4f1bbb1b0b..bf84f0f2ec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\son\sunix. -D 2011-07-09T16:17:18.236 +C Change\sthe\swindows\sbackend\sto\sretry\sread\sand\swrite\srequests\sif\sthe\sencounter\nERROR_LOCK_VIOLATION\sand\sERROR_SHARING_VIOLATION\serrors\s-\swhich\swe\sthink\nsometimes\shappens\sdue\sto\sagressive\santi-virus\ssoftware. +D 2011-07-11T18:17:56.144 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 F src/os_unix.c d3e7b17100704ee0fe2ef71a98c478b947480f4d -F src/os_win.c eafcd6b91cf204a7ef29ac1ef2a1b7132e132e58 +F src/os_win.c 6ba8a531bdc739b23143e8f1d26222ead4d0bdb0 F src/pager.c 120550e7ef01dafaa2cbb4a0528c0d87c8f12b41 F src/pager.h 3f8c783de1d4706b40b1ac15b64f5f896bcc78d1 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 @@ -951,7 +951,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 418a4da2a96cf33055f18c9a667754fad2111cf3 -R bd1403a9e638958833ef8e7e7178479a +P 90b1aea17400bbda5ebc8ae4eb4e12127519e42e +R 349c7f009f94630db798c4428c847d95 U drh -Z 18ede22d9ea411d29db6ab17e40995c3 +Z 6ec74c1e47a6a671533b542ae146a0f8 diff --git a/manifest.uuid b/manifest.uuid index 0650cd683d..2020e584ff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -90b1aea17400bbda5ebc8ae4eb4e12127519e42e \ No newline at end of file +c20aca06610407c197ea50ea77c2591aacf2252a \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index bd0f2f216a..59baeebb5f 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -402,6 +402,33 @@ static int winLogErrorAtLine( return errcode; } +/* +** The number of times that a ReadFile() or WriteFile() will be retried +** following a locking error. +*/ +#ifndef SQLITE_WIN32_IOERR_RETRY +# define SQLITE_WIN32_IOERR_RETRY 5 +#endif + +/* +** If a ReadFile() or WriteFile() error occurs, invoke this routine +** to see if it should be retried. Return TRUE to retry. Return FALSE +** to give up with an error. +*/ +static int retryIoerr(int *pnRetry){ + DWORD e; + if( *pnRetry>=SQLITE_WIN32_IOERR_RETRY ){ + return 0; + } + e = GetLastError(); + if( e==ERROR_LOCK_VIOLATION || e==ERROR_SHARING_VIOLATION ){ + Sleep(50 + 50*(*pnRetry)); + ++*pnRetry; + return 1; + } + return 0; +} + #if SQLITE_OS_WINCE /************************************************************************* ** This section contains code for WinCE only. @@ -820,6 +847,7 @@ static int winRead( ){ winFile *pFile = (winFile*)id; /* file handle */ DWORD nRead; /* Number of bytes actually read from file */ + int nRetry = 0; /* Number of retrys */ assert( id!=0 ); SimulateIOError(return SQLITE_IOERR_READ); @@ -828,7 +856,8 @@ static int winRead( if( seekWinFile(pFile, offset) ){ return SQLITE_FULL; } - if( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ + while( !ReadFile(pFile->h, pBuf, amt, &nRead, 0) ){ + if( retryIoerr(&nRetry) ) continue; pFile->lastErrno = GetLastError(); return winLogError(SQLITE_IOERR_READ, "winRead", pFile->zPath); } @@ -866,8 +895,14 @@ static int winWrite( u8 *aRem = (u8 *)pBuf; /* Data yet to be written */ int nRem = amt; /* Number of bytes yet to be written */ DWORD nWrite; /* Bytes written by each WriteFile() call */ + int nRetry = 0; /* Number of retries */ - while( nRem>0 && WriteFile(pFile->h, aRem, nRem, &nWrite, 0) && nWrite>0 ){ + while( nRem>0 ){ + if( !WriteFile(pFile->h, aRem, nRem, &nWrite, 0) ){ + if( retryIoerr(&nRetry) ) continue; + break; + } + if( nWrite<=0 ) break; aRem += nWrite; nRem -= nWrite; } From 80084ca8e803214078a1fd94816ecddbd48c08e1 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 11 Jul 2011 23:45:44 +0000 Subject: [PATCH 35/52] Update the TCL commands for setting windows manditory locks. Add test cases for manditory lock delays under windows. FossilOrigin-Name: 03af4c175c6ba303ec0a5be25fd42771e38f7347 --- manifest | 13 ++++--- manifest.uuid | 2 +- src/test1.c | 90 +++++++++++++++++++++++++++++++++++++++++++++ test/win32lock.test | 57 ++++++++++++++++++++++++++++ 4 files changed, 155 insertions(+), 7 deletions(-) create mode 100644 test/win32lock.test diff --git a/manifest b/manifest index bf84f0f2ec..e631fa3a74 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\swindows\sbackend\sto\sretry\sread\sand\swrite\srequests\sif\sthe\sencounter\nERROR_LOCK_VIOLATION\sand\sERROR_SHARING_VIOLATION\serrors\s-\swhich\swe\sthink\nsometimes\shappens\sdue\sto\sagressive\santi-virus\ssoftware. -D 2011-07-11T18:17:56.144 +C Update\sthe\sTCL\scommands\sfor\ssetting\swindows\smanditory\slocks.\nAdd\stest\scases\sfor\smanditory\slock\sdelays\sunder\swindows. +D 2011-07-11T23:45:44.051 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -188,7 +188,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c c355b33803fb756dc09297ec84d19604301fa622 -F src/test1.c 8eef932f1024f1076a8adb8a6f5d9c5267c9e969 +F src/test1.c 6623e003ce840ed4a303918dbbc124f9e222974c F src/test2.c 80d323d11e909cf0eb1b6fbb4ac22276483bcf31 F src/test3.c 124ff9735fb6bb7d41de180d6bac90e7b1509432 F src/test4.c d1e5a5e904d4b444cf572391fdcb017638e36ff7 @@ -911,6 +911,7 @@ F test/where9.test 24f19ad14bb1b831564ced5273e681e495662848 F test/whereA.test 24c234263c8fe358f079d5e57d884fb569d2da0a F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 +F test/win32lock.test 7fa3287dccc836fcaa8e0d1e77aa017de5889231 F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F tool/build-shell.sh 12aa4391073a777fcb6dcc490b219a018ae98bac F tool/diffdb.c 7524b1b5df217c20cd0431f6789851a4e0cb191b @@ -951,7 +952,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 90b1aea17400bbda5ebc8ae4eb4e12127519e42e -R 349c7f009f94630db798c4428c847d95 +P c20aca06610407c197ea50ea77c2591aacf2252a +R 4c1b031b9c10734fc2cbb78ed1f104dd U drh -Z 6ec74c1e47a6a671533b542ae146a0f8 +Z 4f5517f8772eb0c19d9bc59edbaff4cc diff --git a/manifest.uuid b/manifest.uuid index 2020e584ff..688cc99f9e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c20aca06610407c197ea50ea77c2591aacf2252a \ No newline at end of file +03af4c175c6ba303ec0a5be25fd42771e38f7347 \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 7fdf54b128..91bb1b4998 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5574,6 +5574,93 @@ static int test_test_control( return TCL_OK; } +#if SQLITE_OS_WIN +/* +** Information passed from the main thread into the windows file locker +** background thread. +*/ +struct win32FileLocker { + HANDLE h; /* Handle of the file to be locked */ + int delay1; /* Delay before locking */ + int delay2; /* Delay before unlocking */ + int ok; /* Finished ok */ + int err; /* True if an error occurs */ +}; +#endif + + +#if SQLITE_OS_WIN +/* +** The background thread that does file locking. +*/ +static void win32_file_locker(void *pAppData){ + struct win32FileLocker *p = (struct win32FileLocker*)pAppData; + if( p->delay1 ) Sleep(p->delay1); + if( LockFile(p->h, 0, 0, 100000000, 0) ){ + Sleep(p->delay2); + UnlockFile(p->h, 0, 0, 100000000, 0); + p->ok = 1; + }else{ + p->err = 1; + } + CloseHandle(p->h); + p->h = 0; + p->delay1 = 0; + p->delay2 = 0; +} +#endif + +#if SQLITE_OS_WIN +/* +** lock_win32_file FILENAME DELAY1 DELAY2 +** +** Get an exclusive manditory lock on file for DELAY2 milliseconds. +** Wait DELAY1 milliseconds before acquiring the lock. +*/ +static int win32_file_lock( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + static struct win32FileLocker x = { 0, 0, 0 }; + const char *zFilename; + int retry = 0; + + if( objc!=4 && objc!=1 ){ + Tcl_WrongNumArgs(interp, 1, objv, "FILENAME DELAY1 DELAY2"); + return TCL_ERROR; + } + if( objc==1 ){ + char zBuf[200]; + sqlite3_snprintf(sizeof(zBuf), zBuf, "%d %d %d %d %d", + x.ok, x.err, x.delay1, x.delay2, x.h); + Tcl_AppendResult(interp, zBuf, (char*)0); + return TCL_OK; + } + while( x.h && retry<10 ){ + retry++; + Sleep(100); + } + if( x.h ){ + Tcl_AppendResult(interp, "busy", (char*)0); + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[2], &x.delay1) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[3], &x.delay2) ) return TCL_ERROR; + zFilename = Tcl_GetString(objv[1]); + x.h = CreateFile(zFilename, GENERIC_READ|GENERIC_WRITE, + FILE_SHARE_READ|FILE_SHARE_WRITE, 0, OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, 0); + if( !x.h ){ + Tcl_AppendResult(interp, "cannot open file: ", zFilename, (char*)0); + return TCL_ERROR; + } + _beginthread(win32_file_locker, 0, (void*)&x); + Sleep(0); + return TCL_OK; +} +#endif /* ** optimization_control DB OPT BOOLEAN @@ -5754,6 +5841,9 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "restore_prng_state", restore_prng_state, 0 }, { "reset_prng_state", reset_prng_state, 0 }, { "optimization_control", optimization_control,0}, +#if SQLITE_OS_WIN + { "lock_win32_file", win32_file_lock, 0 }, +#endif { "tcl_objproc", runAsObjProc, 0 }, /* sqlite3_column_*() API */ diff --git a/test/win32lock.test b/test/win32lock.test new file mode 100644 index 0000000000..a45712d173 --- /dev/null +++ b/test/win32lock.test @@ -0,0 +1,57 @@ +# 2011 July 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this script is recovery from transient manditory locks +# that sometimes appear on database files due to anti-virus software. +# + +if {$tcl_platform(platform)!="windows"} return + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +set testprefix win32lock + +do_test win32lock-1.1 { + db eval { + PRAGMA cache_size=10; + CREATE TABLE t1(x,y); + INSERT INTO t1 VALUES(1,randomblob(100000)); + INSERT INTO t1 VALUES(2,randomblob(50000)); + INSERT INTO t1 VALUES(3,randomblob(25000)); + INSERT INTO t1 VALUES(4,randomblob(12500)); + SELECT x, length(y) FROM t1 ORDER BY rowid; + } +} {1 100000 2 50000 3 25000 4 12500} + +unset -nocomplain delay1 rc msg +set delay1 50 +set rc 0 +set old_pending_byte [sqlite3_test_control_pending_byte 0x40000000] +while {1} { + sqlite3_sleep 10 + lock_win32_file test.db 0 $::delay1 + set rc [catch {db eval {SELECT x, length(y) FROM t1 ORDER BY rowid}} msg] + if {$rc} { + do_test win32lock-1.2-$delay1-fin { + set ::msg + } {disk I/O error} + break + } else { + do_test win32lock-1.2-$delay1 { + set ::msg + } {1 100000 2 50000 3 25000 4 12500} + incr delay1 50 + } +} +sqlite3_test_control_pending_byte $old_pending_byte + +finish_test From 52564d7002c4f2f64735dfc2d93cdfa827145fb1 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 12 Jul 2011 11:04:18 +0000 Subject: [PATCH 36/52] Update the anti-virus retry logic for DeleteFile(). Invoke sqlite3_log() for each anti-virus retry. Make the retry delay configurable at compile-time. FossilOrigin-Name: 89f1848d7f7d98b4f7da9218f99ed90d22dd43a8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 44 ++++++++++++++++++-------------------------- test/wal6.test | 13 ++----------- 4 files changed, 28 insertions(+), 45 deletions(-) diff --git a/manifest b/manifest index e631fa3a74..7953e79169 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sTCL\scommands\sfor\ssetting\swindows\smanditory\slocks.\nAdd\stest\scases\sfor\smanditory\slock\sdelays\sunder\swindows. -D 2011-07-11T23:45:44.051 +C Update\sthe\santi-virus\sretry\slogic\sfor\sDeleteFile().\s\sInvoke\ssqlite3_log()\nfor\seach\santi-virus\sretry.\s\sMake\sthe\sretry\sdelay\sconfigurable\sat\ncompile-time. +D 2011-07-12T11:04:18.838 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 F src/os_unix.c d3e7b17100704ee0fe2ef71a98c478b947480f4d -F src/os_win.c 6ba8a531bdc739b23143e8f1d26222ead4d0bdb0 +F src/os_win.c 4cea89f0a6c98b6a7f4fb3895544af8a07d7807d F src/pager.c 120550e7ef01dafaa2cbb4a0528c0d87c8f12b41 F src/pager.h 3f8c783de1d4706b40b1ac15b64f5f896bcc78d1 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 @@ -882,7 +882,7 @@ F test/wal2.test aa0fb2314b3235be4503c06873e41ebfc0757782 F test/wal3.test 5c396cc22497244d627306f4c1d360167353f8dd F test/wal4.test 3404b048fa5e10605facaf70384e6d2943412e30 F test/wal5.test f06a0427e06db00347e32eb9fa99d6a5c0f2d088 -F test/wal6.test 07aa31ca8892d0527f2c5c5a9a2a87aa421dfaa8 +F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe F test/walbak.test 4df1c7369da0301caeb9a48fa45997fd592380e4 @@ -952,7 +952,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P c20aca06610407c197ea50ea77c2591aacf2252a -R 4c1b031b9c10734fc2cbb78ed1f104dd +P 03af4c175c6ba303ec0a5be25fd42771e38f7347 +R b94301e8087eadceda58922babc3618f U drh -Z 4f5517f8772eb0c19d9bc59edbaff4cc +Z 71facc3c10c90f54a5e699820f127261 diff --git a/manifest.uuid b/manifest.uuid index 688cc99f9e..ccdb128bfa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -03af4c175c6ba303ec0a5be25fd42771e38f7347 \ No newline at end of file +89f1848d7f7d98b4f7da9218f99ed90d22dd43a8 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 59baeebb5f..f0cef15359 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -403,11 +403,16 @@ static int winLogErrorAtLine( } /* -** The number of times that a ReadFile() or WriteFile() will be retried -** following a locking error. +** The number of times that a ReadFile(), WriteFile(), and DeleteFile() +** will be retried following a locking error - probably caused by +** antivirus software. Also the initial delay before the first retry. +** The delay increases linearly with each retry. */ #ifndef SQLITE_WIN32_IOERR_RETRY -# define SQLITE_WIN32_IOERR_RETRY 5 +# define SQLITE_WIN32_IOERR_RETRY 10 +#endif +#ifndef SQLITE_WIN32_IOERR_RETRY_DELAY +# define SQLITE_WIN32_IOERR_RETRY_DELAY 25 #endif /* @@ -422,7 +427,10 @@ static int retryIoerr(int *pnRetry){ } e = GetLastError(); if( e==ERROR_LOCK_VIOLATION || e==ERROR_SHARING_VIOLATION ){ - Sleep(50 + 50*(*pnRetry)); + int delay = SQLITE_WIN32_IOERR_RETRY_DELAY*(1+*pnRetry); + sqlite3_log(SQLITE_IOERR, "delay %dms for lock/sharing violation - " + "probably due to antivirus software", delay); + Sleep(delay); ++*pnRetry; return 1; } @@ -2351,15 +2359,13 @@ static int winOpen( ** to MX_DELETION_ATTEMPTs deletion attempts are run before giving ** up and returning an error. */ -#define MX_DELETION_ATTEMPTS 5 static int winDelete( sqlite3_vfs *pVfs, /* Not used on win32 */ const char *zFilename, /* Name of file to delete */ int syncDir /* Not used on win32 */ ){ int cnt = 0; - DWORD rc; - DWORD error = 0; + int rc; void *zConverted; UNUSED_PARAMETER(pVfs); UNUSED_PARAMETER(syncDir); @@ -2370,34 +2376,20 @@ static int winDelete( return SQLITE_NOMEM; } if( isNT() ){ - do{ - DeleteFileW(zConverted); - }while( ( ((rc = GetFileAttributesW(zConverted)) != INVALID_FILE_ATTRIBUTES) - || ((error = GetLastError()) == ERROR_ACCESS_DENIED)) - && (++cnt < MX_DELETION_ATTEMPTS) - && (Sleep(100), 1) ); + while( (rc = DeleteFileW(zConverted))!=0 || retryIoerr(&cnt) ){} /* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. ** Since the ASCII version of these Windows API do not exist for WINCE, ** it's important to not reference them for WINCE builds. */ #if SQLITE_OS_WINCE==0 }else{ - do{ - DeleteFileA(zConverted); - }while( ( ((rc = GetFileAttributesA(zConverted)) != INVALID_FILE_ATTRIBUTES) - || ((error = GetLastError()) == ERROR_ACCESS_DENIED)) - && (++cnt < MX_DELETION_ATTEMPTS) - && (Sleep(100), 1) ); + while( (rc = DeleteFileW(zConverted))!=0 || retryIoerr(&cnt) ){} #endif } + if( rc ) rc = winLogError(SQLITE_IOERR_DELETE, "winDelete", zFilename); free(zConverted); - OSTRACE(("DELETE \"%s\" %s\n", zFilename, - ( (rc==INVALID_FILE_ATTRIBUTES) && (error==ERROR_FILE_NOT_FOUND)) ? - "ok" : "failed" )); - - return ( (rc == INVALID_FILE_ATTRIBUTES) - && (error == ERROR_FILE_NOT_FOUND)) ? SQLITE_OK : - winLogError(SQLITE_IOERR_DELETE, "winDelete", zFilename); + OSTRACE(("DELETE \"%s\" %s\n", zFilename, (rc ? "failed" : "ok" ))); + return rc; } /* diff --git a/test/wal6.test b/test/wal6.test index 6fae48e9ea..ec31bb8b0f 100644 --- a/test/wal6.test +++ b/test/wal6.test @@ -43,19 +43,11 @@ foreach jmode $all_journal_modes { } {1 2} # Under Windows, you'll get an error trying to delete -# a file this is already opened. For now, make sure -# we get that error, then close the first connection +# a file this is already opened. Close the first connection # so the other tests work. if {$tcl_platform(platform)=="windows"} { if {$jmode=="persist" || $jmode=="truncate"} { - do_test wal6-1.2.$jmode.win { - sqlite3 db2 test.db - catchsql { - PRAGMA journal_mode=WAL; - } db2 - } {1 {disk I/O error}} - db2 close - db close + db close } } @@ -87,4 +79,3 @@ if {$tcl_platform(platform)=="windows"} { } finish_test - From a32ad8434e99b1412b2df9a45b7f5b6b7122bdf8 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 12 Jul 2011 13:51:05 +0000 Subject: [PATCH 37/52] Improvements to the logging that occurs on an antivirus I/O retry. FossilOrigin-Name: ff0ff75c3559f5bbe29c73204cc8ff1cb80f42cb --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 28 ++++++++++++++++++++++------ test/win32lock.test | 19 ++++++++++++++++++- 4 files changed, 48 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 7953e79169..8789521b05 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\santi-virus\sretry\slogic\sfor\sDeleteFile().\s\sInvoke\ssqlite3_log()\nfor\seach\santi-virus\sretry.\s\sMake\sthe\sretry\sdelay\sconfigurable\sat\ncompile-time. -D 2011-07-12T11:04:18.838 +C Improvements\sto\sthe\slogging\sthat\soccurs\son\san\santivirus\sI/O\sretry. +D 2011-07-12T13:51:05.210 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 F src/os_unix.c d3e7b17100704ee0fe2ef71a98c478b947480f4d -F src/os_win.c 4cea89f0a6c98b6a7f4fb3895544af8a07d7807d +F src/os_win.c 07050df9e4956f8a8bb8983788b2523fd6e7db87 F src/pager.c 120550e7ef01dafaa2cbb4a0528c0d87c8f12b41 F src/pager.h 3f8c783de1d4706b40b1ac15b64f5f896bcc78d1 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 @@ -911,7 +911,7 @@ F test/where9.test 24f19ad14bb1b831564ced5273e681e495662848 F test/whereA.test 24c234263c8fe358f079d5e57d884fb569d2da0a F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 -F test/win32lock.test 7fa3287dccc836fcaa8e0d1e77aa017de5889231 +F test/win32lock.test 58d2505e035a8175c2db524e58e7c5c9bf1a926c F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F tool/build-shell.sh 12aa4391073a777fcb6dcc490b219a018ae98bac F tool/diffdb.c 7524b1b5df217c20cd0431f6789851a4e0cb191b @@ -952,7 +952,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 03af4c175c6ba303ec0a5be25fd42771e38f7347 -R b94301e8087eadceda58922babc3618f +P 89f1848d7f7d98b4f7da9218f99ed90d22dd43a8 +R 060b096970846ad68ec6f99f0023cc33 U drh -Z 71facc3c10c90f54a5e699820f127261 +Z 9da4b869d45a82433907baf19d8c1435 diff --git a/manifest.uuid b/manifest.uuid index ccdb128bfa..2d27522bef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -89f1848d7f7d98b4f7da9218f99ed90d22dd43a8 \ No newline at end of file +ff0ff75c3559f5bbe29c73204cc8ff1cb80f42cb \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index f0cef15359..cf2d572682 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -427,16 +427,25 @@ static int retryIoerr(int *pnRetry){ } e = GetLastError(); if( e==ERROR_LOCK_VIOLATION || e==ERROR_SHARING_VIOLATION ){ - int delay = SQLITE_WIN32_IOERR_RETRY_DELAY*(1+*pnRetry); - sqlite3_log(SQLITE_IOERR, "delay %dms for lock/sharing violation - " - "probably due to antivirus software", delay); - Sleep(delay); + Sleep(SQLITE_WIN32_IOERR_RETRY_DELAY*(1+*pnRetry)); ++*pnRetry; return 1; } return 0; } +/* +** Log a I/O error retry episode. +*/ +static void logIoerr(int nRetry){ + if( nRetry ){ + sqlite3_log(SQLITE_IOERR, + "delayed %dms for lock/sharing conflict", + SQLITE_WIN32_IOERR_RETRY_DELAY*nRetry*(nRetry+1)/2 + ); + } +} + #if SQLITE_OS_WINCE /************************************************************************* ** This section contains code for WinCE only. @@ -869,6 +878,7 @@ static int winRead( pFile->lastErrno = GetLastError(); return winLogError(SQLITE_IOERR_READ, "winRead", pFile->zPath); } + logIoerr(nRetry); if( nRead<(DWORD)amt ){ /* Unread parts of the buffer must be zero-filled */ memset(&((char*)pBuf)[nRead], 0, amt-nRead); @@ -890,6 +900,7 @@ static int winWrite( ){ int rc; /* True if error has occured, else false */ winFile *pFile = (winFile*)id; /* File handle */ + int nRetry = 0; /* Number of retries */ assert( amt>0 ); assert( pFile ); @@ -903,7 +914,6 @@ static int winWrite( u8 *aRem = (u8 *)pBuf; /* Data yet to be written */ int nRem = amt; /* Number of bytes yet to be written */ DWORD nWrite; /* Bytes written by each WriteFile() call */ - int nRetry = 0; /* Number of retries */ while( nRem>0 ){ if( !WriteFile(pFile->h, aRem, nRem, &nWrite, 0) ){ @@ -926,6 +936,8 @@ static int winWrite( return SQLITE_FULL; } return winLogError(SQLITE_IOERR_WRITE, "winWrite", pFile->zPath); + }else{ + logIoerr(nRetry); } return SQLITE_OK; } @@ -2386,7 +2398,11 @@ static int winDelete( while( (rc = DeleteFileW(zConverted))!=0 || retryIoerr(&cnt) ){} #endif } - if( rc ) rc = winLogError(SQLITE_IOERR_DELETE, "winDelete", zFilename); + if( rc ){ + rc = winLogError(SQLITE_IOERR_DELETE, "winDelete", zFilename); + }else{ + logIoerr(cnt); + } free(zConverted); OSTRACE(("DELETE \"%s\" %s\n", zFilename, (rc ? "failed" : "ok" ))); return rc; diff --git a/test/win32lock.test b/test/win32lock.test index a45712d173..0fc88eb3f7 100644 --- a/test/win32lock.test +++ b/test/win32lock.test @@ -20,6 +20,14 @@ source $testdir/tester.tcl set testprefix win32lock +db close +sqlite3_shutdown +test_sqlite3_log xLog +proc xLog {error_code msg} { + lappend ::log $msg +} +sqlite3 db test.db + do_test win32lock-1.1 { db eval { PRAGMA cache_size=10; @@ -49,9 +57,18 @@ while {1} { do_test win32lock-1.2-$delay1 { set ::msg } {1 100000 2 50000 3 25000 4 12500} + if {$::log!=""} { + do_test win32lock-1.2-$delay1-log1 { + regsub {\d+} $::log # x + set x + } {{delayed #ms for lock/sharing conflict}} + } incr delay1 50 } + set ::log {} } sqlite3_test_control_pending_byte $old_pending_byte - +sqlite3_shutdown +test_sqlite3_log +sqlite3_initialize finish_test From dc9e9587c6ca4fb22256e7855bc23216b09c3dec Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 12 Jul 2011 14:02:47 +0000 Subject: [PATCH 38/52] Revise logic in winDelete to check the file prior to attempting to delete it. FossilOrigin-Name: 36f11acc531a524407e03c797a6a1bcf88bad809 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 14 +++++++++++--- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 8789521b05..12792f6e0a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\slogging\sthat\soccurs\son\san\santivirus\sI/O\sretry. -D 2011-07-12T13:51:05.210 +C Revise\slogic\sin\swinDelete\sto\scheck\sthe\sfile\sprior\sto\sattempting\sto\sdelete\sit. +D 2011-07-12T14:02:47.638 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 F src/os_unix.c d3e7b17100704ee0fe2ef71a98c478b947480f4d -F src/os_win.c 07050df9e4956f8a8bb8983788b2523fd6e7db87 +F src/os_win.c 57778b70d209f30070dc70e0e9f8e4adba0de5bc F src/pager.c 120550e7ef01dafaa2cbb4a0528c0d87c8f12b41 F src/pager.h 3f8c783de1d4706b40b1ac15b64f5f896bcc78d1 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 @@ -952,7 +952,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 89f1848d7f7d98b4f7da9218f99ed90d22dd43a8 -R 060b096970846ad68ec6f99f0023cc33 -U drh -Z 9da4b869d45a82433907baf19d8c1435 +P ff0ff75c3559f5bbe29c73204cc8ff1cb80f42cb +R 15ec97d6baeea0684b5deea8be1be88e +U mistachkin +Z 94bbfee187493f0fa246b9c95f6c61dc diff --git a/manifest.uuid b/manifest.uuid index 2d27522bef..1b53d3ad80 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ff0ff75c3559f5bbe29c73204cc8ff1cb80f42cb \ No newline at end of file +36f11acc531a524407e03c797a6a1bcf88bad809 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index cf2d572682..7a1713a610 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -426,7 +426,9 @@ static int retryIoerr(int *pnRetry){ return 0; } e = GetLastError(); - if( e==ERROR_LOCK_VIOLATION || e==ERROR_SHARING_VIOLATION ){ + if( e==ERROR_ACCESS_DENIED || + e==ERROR_LOCK_VIOLATION || + e==ERROR_SHARING_VIOLATION ){ Sleep(SQLITE_WIN32_IOERR_RETRY_DELAY*(1+*pnRetry)); ++*pnRetry; return 1; @@ -2388,14 +2390,20 @@ static int winDelete( return SQLITE_NOMEM; } if( isNT() ){ - while( (rc = DeleteFileW(zConverted))!=0 || retryIoerr(&cnt) ){} + rc = 1; + while( GetFileAttributesW(zConverted)!=INVALID_FILE_ATTRIBUTES && + (rc = DeleteFileW(zConverted))==0 && retryIoerr(&cnt) ){} + rc = rc ? SQLITE_OK : SQLITE_ERROR; /* isNT() is 1 if SQLITE_OS_WINCE==1, so this else is never executed. ** Since the ASCII version of these Windows API do not exist for WINCE, ** it's important to not reference them for WINCE builds. */ #if SQLITE_OS_WINCE==0 }else{ - while( (rc = DeleteFileW(zConverted))!=0 || retryIoerr(&cnt) ){} + rc = 1; + while( GetFileAttributesA(zConverted)!=INVALID_FILE_ATTRIBUTES && + (rc = DeleteFileA(zConverted))==0 && retryIoerr(&cnt) ){} + rc = rc ? SQLITE_OK : SQLITE_ERROR; #endif } if( rc ){ From d0cdf012a2cfb309a57c1d3bf26f7045c7ce853f Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 13 Jul 2011 16:03:46 +0000 Subject: [PATCH 39/52] Added the SQLITE_FCNTL_WIN32_AV_RETRY file control for configuring the retry counts and delays in the windows VFS. FossilOrigin-Name: 7aaf0a6ae1238129e07eb191ca3f043df445e27a --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/os_win.c | 22 +++++++++++++++++++--- src/sqlite.h.in | 19 ++++++++++++++++++- src/test1.c | 39 +++++++++++++++++++++++++++++++++++++-- test/win32lock.test | 34 ++++++++++++++++++++++++++++++++++ 6 files changed, 118 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 00a73b3bc9..8568405a64 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\simproved\santi-virus\sdefenses\sinto\sthe\strunk. -D 2011-07-12T14:38:25.733 +C Added\sthe\sSQLITE_FCNTL_WIN32_AV_RETRY\sfile\scontrol\sfor\sconfiguring\sthe\nretry\scounts\sand\sdelays\sin\sthe\swindows\sVFS. +D 2011-07-13T16:03:46.096 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 F src/os_unix.c d3e7b17100704ee0fe2ef71a98c478b947480f4d -F src/os_win.c 57778b70d209f30070dc70e0e9f8e4adba0de5bc +F src/os_win.c c5eadb2c0fc11347296a660f77b9844090265c0c F src/pager.c 120550e7ef01dafaa2cbb4a0528c0d87c8f12b41 F src/pager.h 3f8c783de1d4706b40b1ac15b64f5f896bcc78d1 F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58 @@ -181,14 +181,14 @@ F src/resolve.c 36368f44569208fa074e61f4dd0b6c4fb60ca2b4 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697 F src/select.c d219c4b68d603cc734b6f9b1e2780fee12a1fa0d F src/shell.c bbe7818ff5bc8614105ceb81ad67b8bdc0b671dd -F src/sqlite.h.in 4b7255c10d39c5faf089dbd29cde7c367ff39f1f +F src/sqlite.h.in badc4f56b9b57a8731475c2ddbccd8198ce33b9b F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93 F src/sqliteInt.h ba4a6d6288efb25b84bc0d7d0aaf80f9b42523ba F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c c355b33803fb756dc09297ec84d19604301fa622 -F src/test1.c 6623e003ce840ed4a303918dbbc124f9e222974c +F src/test1.c 693d9a63dfe2c68b167080c99cab82f267f5a38e F src/test2.c 80d323d11e909cf0eb1b6fbb4ac22276483bcf31 F src/test3.c 124ff9735fb6bb7d41de180d6bac90e7b1509432 F src/test4.c d1e5a5e904d4b444cf572391fdcb017638e36ff7 @@ -911,7 +911,7 @@ F test/where9.test 24f19ad14bb1b831564ced5273e681e495662848 F test/whereA.test 24c234263c8fe358f079d5e57d884fb569d2da0a F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 -F test/win32lock.test 58d2505e035a8175c2db524e58e7c5c9bf1a926c +F test/win32lock.test 0a16a7df4a51575bda27529ac992a5a94e4a38bd F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F tool/build-shell.sh 12aa4391073a777fcb6dcc490b219a018ae98bac F tool/diffdb.c 7524b1b5df217c20cd0431f6789851a4e0cb191b @@ -952,7 +952,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 03af4c175c6ba303ec0a5be25fd42771e38f7347 36f11acc531a524407e03c797a6a1bcf88bad809 -R 15ec97d6baeea0684b5deea8be1be88e +P 0207fd9b0c0f6baa3c2cb8ba588ad585507848b3 +R c10af965bd436aa363363314cb7d7b89 U drh -Z 2176be5b1a19aaddd13f35903c55d94b +Z 2357d1ef0784887fd664a2ba2dbc9797 diff --git a/manifest.uuid b/manifest.uuid index 21872bd981..b78982fe75 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0207fd9b0c0f6baa3c2cb8ba588ad585507848b3 \ No newline at end of file +7aaf0a6ae1238129e07eb191ca3f043df445e27a \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 7a1713a610..2d4f0adad2 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -414,6 +414,8 @@ static int winLogErrorAtLine( #ifndef SQLITE_WIN32_IOERR_RETRY_DELAY # define SQLITE_WIN32_IOERR_RETRY_DELAY 25 #endif +static int win32IoerrRetry = SQLITE_WIN32_IOERR_RETRY; +static int win32IoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; /* ** If a ReadFile() or WriteFile() error occurs, invoke this routine @@ -422,14 +424,14 @@ static int winLogErrorAtLine( */ static int retryIoerr(int *pnRetry){ DWORD e; - if( *pnRetry>=SQLITE_WIN32_IOERR_RETRY ){ + if( *pnRetry>=win32IoerrRetry ){ return 0; } e = GetLastError(); if( e==ERROR_ACCESS_DENIED || e==ERROR_LOCK_VIOLATION || e==ERROR_SHARING_VIOLATION ){ - Sleep(SQLITE_WIN32_IOERR_RETRY_DELAY*(1+*pnRetry)); + Sleep(win32IoerrRetryDelay*(1+*pnRetry)); ++*pnRetry; return 1; } @@ -443,7 +445,7 @@ static void logIoerr(int nRetry){ if( nRetry ){ sqlite3_log(SQLITE_IOERR, "delayed %dms for lock/sharing conflict", - SQLITE_WIN32_IOERR_RETRY_DELAY*nRetry*(nRetry+1)/2 + win32IoerrRetryDelay*nRetry*(nRetry+1)/2 ); } } @@ -1356,6 +1358,20 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ case SQLITE_FCNTL_SYNC_OMITTED: { return SQLITE_OK; } + case SQLITE_FCNTL_WIN32_AV_RETRY: { + int *a = (int*)pArg; + if( a[0]>0 ){ + win32IoerrRetry = a[0]; + }else{ + a[0] = win32IoerrRetry; + } + if( a[1]>0 ){ + win32IoerrRetryDelay = a[1]; + }else{ + a[1] = win32IoerrRetryDelay; + } + return SQLITE_OK; + } } return SQLITE_NOTFOUND; } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 2e6ca2b74a..b8e8bfbc46 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -736,6 +736,23 @@ struct sqlite3_io_methods { ** Applications should not call [sqlite3_file_control()] with this ** opcode as doing so may disrupt the operation of the specialized VFSes ** that do require it. +** +** ^The [SQLITE_FCNTL_WIN32_AV_RETRY] opcode is used to configure automatic +** retry counts and intervals for certain disk I/O operations for the +** windows [VFS] in order to work to provide robustness against +** anti-virus programs. By default, the windows VFS will retry file read, +** file write, and file delete opertions up to 10 times, with a delay +** of 25 milliseconds before the first retry and with the delay increasing +** by an additional 25 milliseconds with each subsequent retry. This +** opcode allows those to values (10 retries and 25 milliseconds of delay) +** to be adjusted. The values are changed for all database connections +** within the same process. The argument is a pointer to an array of two +** integers where the first integer i the new retry count and the second +** integer is the delay. If either integer is negative, then the setting +** is not changed but instead the prior value of that setting is written +** into the array entry, allowing the current retry settings to be +** interrogated. The zDbName parameter is ignored. +** */ #define SQLITE_FCNTL_LOCKSTATE 1 #define SQLITE_GET_LOCKPROXYFILE 2 @@ -745,7 +762,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_CHUNK_SIZE 6 #define SQLITE_FCNTL_FILE_POINTER 7 #define SQLITE_FCNTL_SYNC_OMITTED 8 - +#define SQLITE_FCNTL_WIN32_AV_RETRY 9 /* ** CAPI3REF: Mutex Handle diff --git a/src/test1.c b/src/test1.c index 91bb1b4998..3301ab7403 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5096,6 +5096,39 @@ static int file_control_lockproxy_test( return TCL_OK; } +/* +** tclcmd: file_control_win32_av_retry DB NRETRY DELAY +** +** This TCL command runs the sqlite3_file_control interface with +** the SQLITE_FCNTL_WIN32_AV_RETRY opcode. +*/ +static int file_control_win32_av_retry( + ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3 *db; + int rc; + int a[2]; + char z[100]; + + if( objc!=4 ){ + Tcl_AppendResult(interp, "wrong # args: should be \"", + Tcl_GetStringFromObj(objv[0], 0), " DB NRETRY DELAY", 0); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[2], &a[0]) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[3], &a[1]) ) return TCL_ERROR; + rc = sqlite3_file_control(db, NULL, SQLITE_FCNTL_WIN32_AV_RETRY, (void*)a); + sqlite3_snprintf(sizeof(z), z, "%d %d %d", rc, a[0], a[1]); + Tcl_AppendResult(interp, z, (char*)0); + return TCL_OK; +} + /* ** tclcmd: sqlite3_vfs_list @@ -5638,7 +5671,7 @@ static int win32_file_lock( Tcl_AppendResult(interp, zBuf, (char*)0); return TCL_OK; } - while( x.h && retry<10 ){ + while( x.h && retry<30 ){ retry++; Sleep(100); } @@ -5662,6 +5695,7 @@ static int win32_file_lock( } #endif + /* ** optimization_control DB OPT BOOLEAN ** @@ -5892,7 +5926,8 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "file_control_lasterrno_test", file_control_lasterrno_test, 0 }, { "file_control_lockproxy_test", file_control_lockproxy_test, 0 }, { "file_control_chunksize_test", file_control_chunksize_test, 0 }, - { "file_control_sizehint_test", file_control_sizehint_test, 0 }, + { "file_control_sizehint_test", file_control_sizehint_test, 0 }, + { "file_control_win32_av_retry", file_control_win32_av_retry, 0 }, { "sqlite3_vfs_list", vfs_list, 0 }, { "sqlite3_create_function_v2", test_create_function_v2, 0 }, diff --git a/test/win32lock.test b/test/win32lock.test index 0fc88eb3f7..802ea27d16 100644 --- a/test/win32lock.test +++ b/test/win32lock.test @@ -67,6 +67,40 @@ while {1} { } set ::log {} } + +do_test win32lock-2.0 { + file_control_win32_av_retry db -1 -1 +} {0 10 25} +do_test win32lock-2.1 { + file_control_win32_av_retry db 1 1 +} {0 1 1} + +set delay1 50 +while {1} { + sqlite3_sleep 10 + lock_win32_file test.db 0 $::delay1 + set rc [catch {db eval {SELECT x, length(y) FROM t1 ORDER BY rowid}} msg] + if {$rc} { + do_test win32lock-2.2-$delay1-fin { + set ::msg + } {disk I/O error} + break + } else { + do_test win32lock-2.2-$delay1 { + set ::msg + } {1 100000 2 50000 3 25000 4 12500} + if {$::log!=""} { + do_test win32lock-2.2-$delay1-log1 { + regsub {\d+} $::log # x + set x + } {{delayed #ms for lock/sharing conflict}} + } + incr delay1 50 + } + set ::log {} +} + +file_control_win32_av_retry db 10 25 sqlite3_test_control_pending_byte $old_pending_byte sqlite3_shutdown test_sqlite3_log From bf4ec555fa99ce2968f8a309dab38e239bab6abc Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 13 Jul 2011 18:31:10 +0000 Subject: [PATCH 40/52] Do not try to use STAT2 to refine the row estimate of a query that uses a unique or nearly-unique index. FossilOrigin-Name: efffc49baf38698ed3de05d16b4261bf405d42d7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 5 +++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 8568405a64..fab7ca4d78 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Added\sthe\sSQLITE_FCNTL_WIN32_AV_RETRY\sfile\scontrol\sfor\sconfiguring\sthe\nretry\scounts\sand\sdelays\sin\sthe\swindows\sVFS. -D 2011-07-13T16:03:46.096 +C Do\snot\stry\sto\suse\sSTAT2\sto\srefine\sthe\srow\sestimate\sof\sa\squery\sthat\nuses\sa\sunique\sor\snearly-unique\sindex. +D 2011-07-13T18:31:10.999 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -250,7 +250,7 @@ F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582 F src/wal.c 0c70ad7b1cac6005fa5e2cbefd23ee05e391c290 F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f -F src/where.c ce7cce80c5cb07ba40e9bf38a33ef806c61f55f2 +F src/where.c 106cd9ab3eb410dfa7d0598194c277664bb2e9a3 F test/8_3_names.test b93687beebd17f6ebf812405a6833bae5d1f4199 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 @@ -952,7 +952,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 0207fd9b0c0f6baa3c2cb8ba588ad585507848b3 -R c10af965bd436aa363363314cb7d7b89 +P 7aaf0a6ae1238129e07eb191ca3f043df445e27a +R 86cf02b90f1b613c4a5037d43837eb94 U drh -Z 2357d1ef0784887fd664a2ba2dbc9797 +Z 59e834e53ef42b78107149fbc0ea32b5 diff --git a/manifest.uuid b/manifest.uuid index b78982fe75..307dd6ff67 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7aaf0a6ae1238129e07eb191ca3f043df445e27a \ No newline at end of file +efffc49baf38698ed3de05d16b4261bf405d42d7 \ No newline at end of file diff --git a/src/where.c b/src/where.c index bb3bee357c..d312232868 100644 --- a/src/where.c +++ b/src/where.c @@ -3111,12 +3111,13 @@ static void bestBtreeIndex( } #ifdef SQLITE_ENABLE_STAT2 - /* If the constraint is of the form x=VALUE and histogram + /* If the constraint is of the form x=VALUE or x IN (E1,E2,...) + ** and we do not think that values of x are unique and if histogram ** data is available for column x, then it might be possible ** to get a better estimate on the number of rows based on ** VALUE and how common that value is according to the histogram. */ - if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 ){ + if( nRow>(double)1 && nEq==1 && pFirstTerm!=0 && aiRowEst[1]>1 ){ if( pFirstTerm->eOperator & (WO_EQ|WO_ISNULL) ){ testcase( pFirstTerm->eOperator==WO_EQ ); testcase( pFirstTerm->eOperator==WO_ISNULL ); From 33067e7393cd2272e1afb2fea2a8d7f8ae4dea89 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 15 Jul 2011 13:43:34 +0000 Subject: [PATCH 41/52] In os_unix.c, check if the ESTALE macro is defined before using it. FossilOrigin-Name: 87017410f2fc3f2ae15ef06714563ba9ad4350c7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index fab7ca4d78..461b5baf23 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\stry\sto\suse\sSTAT2\sto\srefine\sthe\srow\sestimate\sof\sa\squery\sthat\nuses\sa\sunique\sor\snearly-unique\sindex. -D 2011-07-13T18:31:10.999 +C In\sos_unix.c,\scheck\sif\sthe\sESTALE\smacro\sis\sdefined\sbefore\susing\sit. +D 2011-07-15T13:43:34.523 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -165,7 +165,7 @@ F src/os.c 22ac61d06e72a0dac900400147333b07b13d8e1d F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 -F src/os_unix.c d3e7b17100704ee0fe2ef71a98c478b947480f4d +F src/os_unix.c dcd6d5782dd30e918dc3d111cdcb1883bfb95345 F src/os_win.c c5eadb2c0fc11347296a660f77b9844090265c0c F src/pager.c 120550e7ef01dafaa2cbb4a0528c0d87c8f12b41 F src/pager.h 3f8c783de1d4706b40b1ac15b64f5f896bcc78d1 @@ -952,7 +952,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 7aaf0a6ae1238129e07eb191ca3f043df445e27a -R 86cf02b90f1b613c4a5037d43837eb94 -U drh -Z 59e834e53ef42b78107149fbc0ea32b5 +P efffc49baf38698ed3de05d16b4261bf405d42d7 +R a0a001d9a4f1e78d5a5b15dec2791e15 +U dan +Z e14888b321d90cd480e1dbf5240f97b2 diff --git a/manifest.uuid b/manifest.uuid index 307dd6ff67..a687671df4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -efffc49baf38698ed3de05d16b4261bf405d42d7 \ No newline at end of file +87017410f2fc3f2ae15ef06714563ba9ad4350c7 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index f9a12a2fb9..b2956c1647 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -677,7 +677,9 @@ static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { case ENODEV: case ENXIO: case ENOENT: +#ifdef ESTALE /* ESTALE is not defined on Interix systems */ case ESTALE: +#endif case ENOSYS: /* these should force the client to close the file and reconnect */ From c4e384beb86742cc5f3f662809726cb2c1f8ae0f Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 19 Jul 2011 15:27:59 +0000 Subject: [PATCH 42/52] Increase the version number to 3.7.8. FossilOrigin-Name: 4355bf2908b923e844907c1e3c09be3bf469f833 --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/VERSION b/VERSION index d2577d9756..a0fc9e07cb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.7.7 +3.7.8 diff --git a/configure b/configure index e0985011af..14b71c67b1 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.7.7. +# Generated by GNU Autoconf 2.62 for sqlite 3.7.8. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.7.7' -PACKAGE_STRING='sqlite 3.7.7' +PACKAGE_VERSION='3.7.8' +PACKAGE_STRING='sqlite 3.7.8' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1485,7 +1485,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.7.7 to adapt to many kinds of systems. +\`configure' configures sqlite 3.7.8 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1550,7 +1550,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.7.7:";; + short | recursive ) echo "Configuration of sqlite 3.7.8:";; esac cat <<\_ACEOF @@ -1666,7 +1666,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.7.7 +sqlite configure 3.7.8 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1680,7 +1680,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.7.7, which was +It was created by sqlite $as_me 3.7.8, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -14030,7 +14030,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.7.7, which was +This file was extended by sqlite $as_me 3.7.8, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14083,7 +14083,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.7.7 +sqlite config.status 3.7.8 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/manifest b/manifest index 461b5baf23..cb88bfa144 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C In\sos_unix.c,\scheck\sif\sthe\sESTALE\smacro\sis\sdefined\sbefore\susing\sit. -D 2011-07-15T13:43:34.523 +C Increase\sthe\sversion\snumber\sto\s3.7.8. +D 2011-07-19T15:27:59.774 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc 9c76731c20561a33ce1257c70be3f029cedb62a6 F Makefile.vxworks c85ec1d8597fe2f7bc225af12ac1666e21379151 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 -F VERSION 3fcdd7fbe3eb282df3978fe77288544543767961 +F VERSION f724de7326e87b7f3b0a55f16ef4b4d993680d54 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 17dc593f791f874d2c23a0f9360850ded0286531 F art/2005osaward.gif 0d1851b2a7c1c9d0ccce545f3e14bca42d7fd248 @@ -23,7 +23,7 @@ F art/src_logo.gif 9341ef09f0e53cd44c0c9b6fc3c16f7f3d6c2ad9 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 405a958bdb3af382a809dccb08a44694923ddd61 F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure f9e97ee7cdc9848e2f3f5ef015fdf861f46fb1bf x +F configure 93e7e695581fa7bef4949161453d9845c5592ad0 x F configure.ac 298a759c086e72c013da459c2aec02a104f4224f F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html f0f682f50210928c07e562621c3b7e8ab912a538 @@ -952,7 +952,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P efffc49baf38698ed3de05d16b4261bf405d42d7 -R a0a001d9a4f1e78d5a5b15dec2791e15 -U dan -Z e14888b321d90cd480e1dbf5240f97b2 +P 87017410f2fc3f2ae15ef06714563ba9ad4350c7 +R 6dbb7226fc6df92a6d1fc7b30a91d1b6 +U drh +Z 6ea6097405e5ac181fe822d84336e434 diff --git a/manifest.uuid b/manifest.uuid index a687671df4..ee31564b14 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -87017410f2fc3f2ae15ef06714563ba9ad4350c7 \ No newline at end of file +4355bf2908b923e844907c1e3c09be3bf469f833 \ No newline at end of file From 8efdb732569807ea0995a08079cbbfdf3005aa45 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 19 Jul 2011 18:29:00 +0000 Subject: [PATCH 43/52] Formatting changes on the multiplexor code - make sure no lines exceed 80 characters. No logical changes. FossilOrigin-Name: ed5f0aad6b21066bacd01521e82c22e96991f400 --- manifest | 12 +++---- manifest.uuid | 2 +- src/test_multiplex.c | 74 +++++++++++++++++++++++++++++++------------- 3 files changed, 60 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index cb88bfa144..dc0a0982a2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sversion\snumber\sto\s3.7.8. -D 2011-07-19T15:27:59.774 +C Formatting\schanges\son\sthe\smultiplexor\scode\s-\smake\ssure\sno\slines\sexceed\s80\ncharacters.\s\sNo\slogical\schanges. +D 2011-07-19T18:29:00.027 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -213,7 +213,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c 03313c693cca72959dcaaf79f8d76f21c01e19ff F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 7ca7be34e0e09ef0ed6619544552ed95732e41f6 -F src/test_multiplex.c 0585bb12a98521795344f67d17d77e258ea7b57f +F src/test_multiplex.c d1b43de1209bd1bc207b694a738fdb84a162f65b F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec @@ -952,7 +952,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 87017410f2fc3f2ae15ef06714563ba9ad4350c7 -R 6dbb7226fc6df92a6d1fc7b30a91d1b6 +P 4355bf2908b923e844907c1e3c09be3bf469f833 +R 33ae39ed3129fdb3c8dbf2c830ddcfd5 U drh -Z 6ea6097405e5ac181fe822d84336e434 +Z 60f009445613def050e13415b0c93faf diff --git a/manifest.uuid b/manifest.uuid index ee31564b14..a91085ca7c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4355bf2908b923e844907c1e3c09be3bf469f833 \ No newline at end of file +ed5f0aad6b21066bacd01521e82c22e96991f400 \ No newline at end of file diff --git a/src/test_multiplex.c b/src/test_multiplex.c index 60ccbcc35f..2203c60b54 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -272,7 +272,8 @@ static int multiplexGetTempname(sqlite3_vfs *pOrigVfs, int nBuf, char *zBuf){ attempts++; sqlite3_randomness(8, &zBuf[j]); for(i=0; i<8; i++){ - zBuf[j+i] = (char)zChars[ ((unsigned char)zBuf[j+i])%(sizeof(zChars)-1) ]; + unsigned char uc = (unsigned char)zBuf[j+i]; + zBuf[j+i] = (char)zChars[uc%(sizeof(zChars)-1)]; } memcpy(&zBuf[j+i], ".tmp", 5); rc = pOrigVfs->xAccess(pOrigVfs, zBuf, SQLITE_ACCESS_EXISTS, &exists); @@ -289,21 +290,31 @@ static int multiplexGetTempname(sqlite3_vfs *pOrigVfs, int nBuf, char *zBuf){ /* Translate an sqlite3_file* that is really a multiplexGroup* into ** the sqlite3_file* for the underlying original VFS. */ -static sqlite3_file *multiplexSubOpen(multiplexConn *pConn, int iChunk, int *rc, int *pOutFlags){ +static sqlite3_file *multiplexSubOpen( + multiplexConn *pConn, + int iChunk, + int *rc, + int *pOutFlags +){ multiplexGroup *pGroup = pConn->pGroup; sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ if( iChunknMaxChunks ){ - sqlite3_file *pSubOpen = pGroup->pReal[iChunk]; /* Real file descriptor */ + sqlite3_file *pSubOpen = pGroup->pReal[iChunk]; /* Real file descriptor */ if( !pGroup->bOpen[iChunk] ){ memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1); if( iChunk ){ #ifdef SQLITE_MULTIPLEX_EXT_OVWR - sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, SQLITE_MULTIPLEX_EXT_FMT, iChunk); + sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, + gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, + SQLITE_MULTIPLEX_EXT_FMT, iChunk); #else - sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, gMultiplex.zName+pGroup->nName, SQLITE_MULTIPLEX_EXT_FMT, iChunk); + sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, + gMultiplex.zName+pGroup->nName, + SQLITE_MULTIPLEX_EXT_FMT, iChunk); #endif } - *rc = pOrigVfs->xOpen(pOrigVfs, gMultiplex.zName, pSubOpen, pGroup->flags, pOutFlags); + *rc = pOrigVfs->xOpen(pOrigVfs, gMultiplex.zName, pSubOpen, + pGroup->flags, pOutFlags); if( *rc==SQLITE_OK ){ pGroup->bOpen[iChunk] = -1; return pSubOpen; @@ -389,9 +400,9 @@ static int multiplexOpen( int flags, /* Flags to control the opening */ int *pOutFlags /* Flags showing results of opening */ ){ - int rc = SQLITE_OK; /* Result code */ - multiplexConn *pMultiplexOpen; /* The new multiplex file descriptor */ - multiplexGroup *pGroup; /* Corresponding multiplexGroup object */ + int rc = SQLITE_OK; /* Result code */ + multiplexConn *pMultiplexOpen; /* The new multiplex file descriptor */ + multiplexGroup *pGroup; /* Corresponding multiplexGroup object */ sqlite3_file *pSubOpen; /* Real file descriptor */ sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ int nName; @@ -419,7 +430,7 @@ static int multiplexOpen( if( rc==SQLITE_OK ){ /* allocate space for group */ nName = multiplexStrlen30(zName); - sz = sizeof(multiplexGroup) /* multiplexGroup */ + sz = sizeof(multiplexGroup) /* multiplexGroup */ + (sizeof(sqlite3_file *)*SQLITE_MULTIPLEX_MAX_CHUNKS) /* pReal[] */ + (pOrigVfs->szOsFile*SQLITE_MULTIPLEX_MAX_CHUNKS) /* *pReal */ + SQLITE_MULTIPLEX_MAX_CHUNKS /* bOpen[] */ @@ -624,16 +635,22 @@ static int multiplexRead( multiplexEnter(); if( !pGroup->bEnabled ){ sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); - rc = ( !pSubOpen ) ? SQLITE_IOERR_READ : pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, iOfst); + if( !pSubOpen ){ + rc = SQLITE_IOERR_READ; + }else{ + rc = pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, iOfst); + } }else{ while( iAmt > 0 ){ int i = (int)(iOfst / pGroup->nChunkSize); sqlite3_file *pSubOpen = multiplexSubOpen(p, i, &rc, NULL); if( pSubOpen ){ - int extra = ((int)(iOfst % pGroup->nChunkSize) + iAmt) - pGroup->nChunkSize; + int extra = ((int)(iOfst % pGroup->nChunkSize) + iAmt) - + pGroup->nChunkSize; if( extra<0 ) extra = 0; iAmt -= extra; - rc = pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, iOfst % pGroup->nChunkSize); + rc = pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, + iOfst % pGroup->nChunkSize); if( rc!=SQLITE_OK ) break; pBuf = (char *)pBuf + iAmt; iOfst += iAmt; @@ -664,16 +681,22 @@ static int multiplexWrite( multiplexEnter(); if( !pGroup->bEnabled ){ sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); - rc = ( !pSubOpen ) ? SQLITE_IOERR_WRITE : pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, iOfst); + if( pSubOpen==0 ){ + rc = SQLITE_IOERR_WRITE; + }else{ + rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, iOfst); + } }else{ while( iAmt > 0 ){ int i = (int)(iOfst / pGroup->nChunkSize); sqlite3_file *pSubOpen = multiplexSubOpen(p, i, &rc, NULL); if( pSubOpen ){ - int extra = ((int)(iOfst % pGroup->nChunkSize) + iAmt) - pGroup->nChunkSize; + int extra = ((int)(iOfst % pGroup->nChunkSize) + iAmt) - + pGroup->nChunkSize; if( extra<0 ) extra = 0; iAmt -= extra; - rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, iOfst % pGroup->nChunkSize); + rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, + iOfst % pGroup->nChunkSize); if( rc!=SQLITE_OK ) break; pBuf = (char *)pBuf + iAmt; iOfst += iAmt; @@ -699,7 +722,11 @@ static int multiplexTruncate(sqlite3_file *pConn, sqlite3_int64 size){ multiplexEnter(); if( !pGroup->bEnabled ){ sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); - rc = ( !pSubOpen ) ? SQLITE_IOERR_TRUNCATE : pSubOpen->pMethods->xTruncate(pSubOpen, size); + if( pSubOpen==0 ){ + rc = SQLITE_IOERR_TRUNCATE; + }else{ + rc = pSubOpen->pMethods->xTruncate(pSubOpen, size); + } }else{ int rc2; int i; @@ -727,7 +754,7 @@ static int multiplexTruncate(sqlite3_file *pConn, sqlite3_int64 size){ rc2 = pOrigVfs->xDelete(pOrigVfs, gMultiplex.zName, 0); if( rc2!=SQLITE_OK ) rc = SQLITE_IOERR_TRUNCATE; } - pSubOpen = multiplexSubOpen(p, (int)(size / pGroup->nChunkSize), &rc2, NULL); + pSubOpen = multiplexSubOpen(p, (int)(size/pGroup->nChunkSize), &rc2, NULL); if( pSubOpen ){ rc2 = pSubOpen->pMethods->xTruncate(pSubOpen, size % pGroup->nChunkSize); if( rc2!=SQLITE_OK ) rc = rc2; @@ -771,7 +798,11 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){ multiplexEnter(); if( !pGroup->bEnabled ){ sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); - rc = ( !pSubOpen ) ? SQLITE_IOERR_FSTAT : pSubOpen->pMethods->xFileSize(pSubOpen, pSize); + if( pSubOpen==0 ){ + rc = SQLITE_IOERR_FSTAT; + }else{ + rc = pSubOpen->pMethods->xFileSize(pSubOpen, pSize); + } }else{ *pSize = 0; for(i=0; inMaxChunks; i++){ @@ -955,7 +986,7 @@ static int multiplexShmMap( int rc; sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); if( pSubOpen ){ - return pSubOpen->pMethods->xShmMap(pSubOpen, iRegion, szRegion, bExtend, pp); + return pSubOpen->pMethods->xShmMap(pSubOpen, iRegion, szRegion, bExtend,pp); } return SQLITE_IOERR; } @@ -1060,7 +1091,8 @@ int sqlite3_multiplex_initialize(const char *zOrigVfsName, int makeDefault){ gMultiplex.sIoMethodsV1.xCheckReservedLock = multiplexCheckReservedLock; gMultiplex.sIoMethodsV1.xFileControl = multiplexFileControl; gMultiplex.sIoMethodsV1.xSectorSize = multiplexSectorSize; - gMultiplex.sIoMethodsV1.xDeviceCharacteristics = multiplexDeviceCharacteristics; + gMultiplex.sIoMethodsV1.xDeviceCharacteristics = + multiplexDeviceCharacteristics; gMultiplex.sIoMethodsV2 = gMultiplex.sIoMethodsV1; gMultiplex.sIoMethodsV2.iVersion = 2; gMultiplex.sIoMethodsV2.xShmMap = multiplexShmMap; From f3717af48d856fc95c8ef34646300d7567d17eaf Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 20 Jul 2011 16:35:31 +0000 Subject: [PATCH 44/52] When the multiplexor opens an auxiliary file, it now persists the name of that file until it is closed, as it should. Remove the limit on the number of auxiliary files used by the multiplexor. FossilOrigin-Name: 1ffa542bf913200a18ef77447aec4fc3ca1ed618 --- manifest | 14 +- manifest.uuid | 2 +- src/test_multiplex.c | 351 ++++++++++++++++++------------------------- test/multiplex.test | 81 +++------- 4 files changed, 177 insertions(+), 271 deletions(-) diff --git a/manifest b/manifest index dc0a0982a2..c0583658f0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Formatting\schanges\son\sthe\smultiplexor\scode\s-\smake\ssure\sno\slines\sexceed\s80\ncharacters.\s\sNo\slogical\schanges. -D 2011-07-19T18:29:00.027 +C When\sthe\smultiplexor\sopens\san\sauxiliary\sfile,\sit\snow\spersists\sthe\sname\sof\sthat\nfile\suntil\sit\sis\sclosed,\sas\sit\sshould.\s\sRemove\sthe\slimit\son\sthe\snumber\sof\nauxiliary\sfiles\sused\sby\sthe\smultiplexor. +D 2011-07-20T16:35:31.624 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -213,7 +213,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c 03313c693cca72959dcaaf79f8d76f21c01e19ff F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 7ca7be34e0e09ef0ed6619544552ed95732e41f6 -F src/test_multiplex.c d1b43de1209bd1bc207b694a738fdb84a162f65b +F src/test_multiplex.c 282f315fd035c8d8342a4cc59748c60216b61fef F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec @@ -595,7 +595,7 @@ F test/misc5.test 9f9338f8211c7f5d1cbe16331fa65d019501aa50 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test 29032efcd3d826fbd409e2a7af873e7939f4a4e3 F test/misuse.test 30b3a458e5a70c31e74c291937b6c82204c59f33 -F test/multiplex.test 555080c87abfc72ba68e2f3df01d4a9a7a4fdf58 +F test/multiplex.test 2256e90e8780de6eeafc07d4084427c3a97d3bcb F test/mutex1.test 78b2b9bb320e51d156c4efdb71b99b051e7a4b41 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test dc212a22b36109fd1ae37154292444ef249c5ec2 @@ -952,7 +952,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 4355bf2908b923e844907c1e3c09be3bf469f833 -R 33ae39ed3129fdb3c8dbf2c830ddcfd5 +P ed5f0aad6b21066bacd01521e82c22e96991f400 +R e63c3d9c3ff7ac85b18419c4f45127d4 U drh -Z 60f009445613def050e13415b0c93faf +Z 1541751b97b37082064c8e76500e9b76 diff --git a/manifest.uuid b/manifest.uuid index a91085ca7c..04e86b35f3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ed5f0aad6b21066bacd01521e82c22e96991f400 \ No newline at end of file +1ffa542bf913200a18ef77447aec4fc3ca1ed618 \ No newline at end of file diff --git a/src/test_multiplex.c b/src/test_multiplex.c index 2203c60b54..b6fe9e8b63 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -126,13 +126,15 @@ typedef struct multiplexConn multiplexConn; ** group. */ struct multiplexGroup { - sqlite3_file **pReal; /* Handles to each chunk */ - char *bOpen; /* array of bools - 0 if chunk not opened */ + struct multiplexReal { /* For each chunk */ + sqlite3_file *p; /* Handle for the chunk */ + char *z; /* Name of this chunk */ + } *aReal; /* list of all chunks */ + int nReal; /* Number of chunks */ char *zName; /* Base filename of this group */ int nName; /* Length of base filename */ int flags; /* Flags used for original opening */ int nChunkSize; /* Chunk size used for this group */ - int nMaxChunks; /* Max number of chunks for this group */ int bEnabled; /* TRUE to use Multiplex VFS for this file */ multiplexGroup *pNext, *pPrev; /* Doubly linked list of all group objects */ }; @@ -191,12 +193,6 @@ static struct { /* List of multiplexGroup objects. */ multiplexGroup *pGroups; - - /* Storage for temp file names. Allocated during - ** initialization to the max pathname of the underlying VFS. - */ - char *zName; - } gMultiplex; /************************* Utility Routines *********************************/ @@ -287,45 +283,69 @@ static int multiplexGetTempname(sqlite3_vfs *pOrigVfs, int nBuf, char *zBuf){ return rc; } +/* Compute the filename for the iChunk-th chunk +*/ +static int multiplexSubFilename(multiplexGroup *pGroup, int iChunk){ + if( iChunk>=pGroup->nReal ){ + struct multiplexReal *p; + p = sqlite3_realloc(pGroup->aReal, (iChunk+1)*sizeof(*p)); + if( p==0 ){ + return SQLITE_NOMEM; + } + memset(&p[pGroup->nReal], 0, sizeof(p[0])*(iChunk+1-pGroup->nReal)); + pGroup->aReal = p; + pGroup->nReal = iChunk+1; + } + if( pGroup->aReal[iChunk].z==0 ){ + char *z; + int n = pGroup->nName; + pGroup->aReal[iChunk].z = z = sqlite3_malloc( n+3 ); + if( z==0 ){ + return SQLITE_NOMEM; + } + memcpy(z, pGroup->zName, n+1); + if( iChunk>0 ){ +#ifdef SQLITE_ENABLE_8_3_NAMES + if( n>3 && z[n-3]=='.' ){ + n--; + }else if( n>4 && z[n-4]=='.' ){ + n -= 2; + } +#endif + sqlite3_snprintf(3,&z[n],"%02d",iChunk); + } + } + return SQLITE_OK; +} + /* Translate an sqlite3_file* that is really a multiplexGroup* into ** the sqlite3_file* for the underlying original VFS. */ static sqlite3_file *multiplexSubOpen( - multiplexConn *pConn, + multiplexGroup *pGroup, int iChunk, int *rc, int *pOutFlags ){ - multiplexGroup *pGroup = pConn->pGroup; + sqlite3_file *pSubOpen = 0; sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ - if( iChunknMaxChunks ){ - sqlite3_file *pSubOpen = pGroup->pReal[iChunk]; /* Real file descriptor */ - if( !pGroup->bOpen[iChunk] ){ - memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1); - if( iChunk ){ -#ifdef SQLITE_MULTIPLEX_EXT_OVWR - sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, - gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, - SQLITE_MULTIPLEX_EXT_FMT, iChunk); -#else - sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, - gMultiplex.zName+pGroup->nName, - SQLITE_MULTIPLEX_EXT_FMT, iChunk); -#endif - } - *rc = pOrigVfs->xOpen(pOrigVfs, gMultiplex.zName, pSubOpen, - pGroup->flags, pOutFlags); - if( *rc==SQLITE_OK ){ - pGroup->bOpen[iChunk] = -1; - return pSubOpen; - } - return NULL; + *rc = multiplexSubFilename(pGroup, iChunk); + if( (*rc)==SQLITE_OK && (pSubOpen = pGroup->aReal[iChunk].p)==0 ){ + pSubOpen = sqlite3_malloc( pOrigVfs->szOsFile ); + if( pSubOpen==0 ){ + *rc = SQLITE_NOMEM; + return 0; + } + pGroup->aReal[iChunk].p = pSubOpen; + *rc = pOrigVfs->xOpen(pOrigVfs, pGroup->aReal[iChunk].z, pSubOpen, + pGroup->flags, pOutFlags); + if( *rc!=SQLITE_OK ){ + sqlite3_free(pSubOpen); + pGroup->aReal[iChunk].p = 0; + return 0; } - *rc = SQLITE_OK; - return pSubOpen; } - *rc = SQLITE_FULL; - return NULL; + return pSubOpen; } /* @@ -384,6 +404,36 @@ static int multiplexFuncInit( return rc; } +/* +** Close a single sub-file in the connection group. +*/ +static void multiplexSubClose( + multiplexGroup *pGroup, + int iChunk, + sqlite3_vfs *pOrigVfs +){ + sqlite3_file *pSubOpen = pGroup->aReal[iChunk].p; + if( pSubOpen ){ + if( pOrigVfs ) pOrigVfs->xDelete(pOrigVfs, pGroup->aReal[iChunk].z, 0); + pSubOpen->pMethods->xClose(pSubOpen); + sqlite3_free(pGroup->aReal[iChunk].p); + } + sqlite3_free(pGroup->aReal[iChunk].z); + memset(&pGroup->aReal[iChunk], 0, sizeof(pGroup->aReal[iChunk])); +} + +/* +** Deallocate memory held by a multiplexGroup +*/ +static void multiplexFreeComponents(multiplexGroup *pGroup){ + int i; + for(i=0; inReal; i++){ multiplexSubClose(pGroup, i, 0); } + sqlite3_free(pGroup->aReal); + pGroup->aReal = 0; + pGroup->nReal = 0; +} + + /************************* VFS Method Wrappers *****************************/ /* @@ -406,10 +456,11 @@ static int multiplexOpen( sqlite3_file *pSubOpen; /* Real file descriptor */ sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ int nName; - int i; int sz; + char *zToFree = 0; UNUSED_PARAMETER(pVfs); + memset(pConn, 0, pVfs->szOsFile); /* We need to create a group structure and manage ** access to this group of files. @@ -423,28 +474,22 @@ static int multiplexOpen( ** it. */ if( !zName ){ - rc = multiplexGetTempname(pOrigVfs, pOrigVfs->mxPathname, gMultiplex.zName); - zName = gMultiplex.zName; + zName = zToFree = sqlite3_malloc( pOrigVfs->mxPathname + 10 ); + if( zName==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = multiplexGetTempname(pOrigVfs, pOrigVfs->mxPathname, zToFree); + } } if( rc==SQLITE_OK ){ /* allocate space for group */ nName = multiplexStrlen30(zName); sz = sizeof(multiplexGroup) /* multiplexGroup */ - + (sizeof(sqlite3_file *)*SQLITE_MULTIPLEX_MAX_CHUNKS) /* pReal[] */ - + (pOrigVfs->szOsFile*SQLITE_MULTIPLEX_MAX_CHUNKS) /* *pReal */ - + SQLITE_MULTIPLEX_MAX_CHUNKS /* bOpen[] */ - + nName + 1; /* zName */ -#ifndef SQLITE_MULTIPLEX_EXT_OVWR - sz += SQLITE_MULTIPLEX_EXT_SZ; - assert(nName+SQLITE_MULTIPLEX_EXT_SZ < pOrigVfs->mxPathname); -#else - assert(nName >= SQLITE_MULTIPLEX_EXT_SZ); - assert(nName < pOrigVfs->mxPathname); -#endif + + nName + 1; /* zName */ pGroup = sqlite3_malloc( sz ); if( pGroup==0 ){ - rc=SQLITE_NOMEM; + rc = SQLITE_NOMEM; } } @@ -456,27 +501,19 @@ static int multiplexOpen( memset(pGroup, 0, sz); pGroup->bEnabled = -1; pGroup->nChunkSize = SQLITE_MULTIPLEX_CHUNK_SIZE; - zChunkSize = sqlite3_uri_parameter(zName, "chunksize"); - if( zChunkSize ){ - int n = atoi(zChunkSize); - if( n>0 ) pGroup->nChunkSize = (n+0xffff)&~0xffff; + if( flags & SQLITE_OPEN_URI ){ + zChunkSize = sqlite3_uri_parameter(zName, "chunksize"); + if( zChunkSize ){ + int n = atoi(zChunkSize); + if( n>0 ) pGroup->nChunkSize = (n+0xffff)&~0xffff; + } } - pGroup->nMaxChunks = SQLITE_MULTIPLEX_MAX_CHUNKS; - pGroup->pReal = (sqlite3_file **)p; - p += (sizeof(sqlite3_file *)*pGroup->nMaxChunks); - for(i=0; inMaxChunks; i++){ - pGroup->pReal[i] = (sqlite3_file *)p; - p += pOrigVfs->szOsFile; - } - /* bOpen[] vals should all be zero from memset above */ - pGroup->bOpen = p; - p += pGroup->nMaxChunks; pGroup->zName = p; /* save off base filename, name length, and original open flags */ memcpy(pGroup->zName, zName, nName+1); pGroup->nName = nName; pGroup->flags = flags; - pSubOpen = multiplexSubOpen(pMultiplexOpen, 0, &rc, pOutFlags); + pSubOpen = multiplexSubOpen(pGroup, 0, &rc, pOutFlags); if( pSubOpen ){ /* if this file is already larger than chunk size, disable ** the multiplex feature. @@ -496,17 +533,18 @@ static int multiplexOpen( if( gMultiplex.pGroups ) gMultiplex.pGroups->pPrev = pGroup; gMultiplex.pGroups = pGroup; }else{ + multiplexFreeComponents(pGroup); sqlite3_free(pGroup); } } multiplexLeave(); + sqlite3_free(zToFree); return rc; } /* ** This is the xDelete method used for the "multiplex" VFS. -** It attempts to delete the filename specified, as well -** as additional files with the SQLITE_MULTIPLEX_EXT_FMT extension. +** It attempts to delete the filename specified. */ static int multiplexDelete( sqlite3_vfs *pVfs, /* The multiplex VFS */ @@ -514,41 +552,7 @@ static int multiplexDelete( int syncDir ){ sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ - int rc = SQLITE_OK; - int nName = multiplexStrlen30(zName); - int i; - - UNUSED_PARAMETER(pVfs); - - multiplexEnter(); - memcpy(gMultiplex.zName, zName, nName+1); - for(i=0; ixAccess(pOrigVfs, gMultiplex.zName, - SQLITE_ACCESS_EXISTS, &exists); - if( rc2==SQLITE_OK && exists ){ - /* if it exists, delete it */ - rc2 = pOrigVfs->xDelete(pOrigVfs, gMultiplex.zName, syncDir); - if( rc2!=SQLITE_OK ) rc = rc2; - }else{ - /* stop at first "gap" */ - break; - } - } - multiplexLeave(); - return rc; + return pOrigVfs->xDelete(pOrigVfs, zName, syncDir); } static int multiplexAccess(sqlite3_vfs *a, const char *b, int c, int *d){ @@ -596,17 +600,8 @@ static int multiplexClose(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; multiplexGroup *pGroup = p->pGroup; int rc = SQLITE_OK; - int i; multiplexEnter(); - /* close any open handles */ - for(i=0; inMaxChunks; i++){ - if( pGroup->bOpen[i] ){ - sqlite3_file *pSubOpen = pGroup->pReal[i]; - int rc2 = pSubOpen->pMethods->xClose(pSubOpen); - if( rc2!=SQLITE_OK ) rc = rc2; - pGroup->bOpen[i] = 0; - } - } + multiplexFreeComponents(pGroup); /* remove from linked list */ if( pGroup->pNext ) pGroup->pNext->pPrev = pGroup->pPrev; if( pGroup->pPrev ){ @@ -634,8 +629,8 @@ static int multiplexRead( int rc = SQLITE_OK; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); - if( !pSubOpen ){ + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); + if( pSubOpen==0 ){ rc = SQLITE_IOERR_READ; }else{ rc = pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, iOfst); @@ -643,7 +638,7 @@ static int multiplexRead( }else{ while( iAmt > 0 ){ int i = (int)(iOfst / pGroup->nChunkSize); - sqlite3_file *pSubOpen = multiplexSubOpen(p, i, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL); if( pSubOpen ){ int extra = ((int)(iOfst % pGroup->nChunkSize) + iAmt) - pGroup->nChunkSize; @@ -680,7 +675,7 @@ static int multiplexWrite( int rc = SQLITE_OK; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); if( pSubOpen==0 ){ rc = SQLITE_IOERR_WRITE; }else{ @@ -689,7 +684,7 @@ static int multiplexWrite( }else{ while( iAmt > 0 ){ int i = (int)(iOfst / pGroup->nChunkSize); - sqlite3_file *pSubOpen = multiplexSubOpen(p, i, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL); if( pSubOpen ){ int extra = ((int)(iOfst % pGroup->nChunkSize) + iAmt) - pGroup->nChunkSize; @@ -721,7 +716,7 @@ static int multiplexTruncate(sqlite3_file *pConn, sqlite3_int64 size){ int rc = SQLITE_OK; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); if( pSubOpen==0 ){ rc = SQLITE_IOERR_TRUNCATE; }else{ @@ -732,29 +727,11 @@ static int multiplexTruncate(sqlite3_file *pConn, sqlite3_int64 size){ int i; sqlite3_file *pSubOpen; sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ - memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1); /* delete the chunks above the truncate limit */ - for(i=(int)(size / pGroup->nChunkSize)+1; inMaxChunks; i++){ - /* close any open chunks before deleting them */ - if( pGroup->bOpen[i] ){ - pSubOpen = pGroup->pReal[i]; - rc2 = pSubOpen->pMethods->xClose(pSubOpen); - if( rc2!=SQLITE_OK ) rc = SQLITE_IOERR_TRUNCATE; - pGroup->bOpen[i] = 0; - } -#ifdef SQLITE_MULTIPLEX_EXT_OVWR - sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, - gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, - SQLITE_MULTIPLEX_EXT_FMT, i); -#else - sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, - gMultiplex.zName+pGroup->nName, - SQLITE_MULTIPLEX_EXT_FMT, i); -#endif - rc2 = pOrigVfs->xDelete(pOrigVfs, gMultiplex.zName, 0); - if( rc2!=SQLITE_OK ) rc = SQLITE_IOERR_TRUNCATE; + for(i=(int)(size / pGroup->nChunkSize)+1; inReal; i++){ + multiplexSubClose(pGroup, i, pOrigVfs); } - pSubOpen = multiplexSubOpen(p, (int)(size/pGroup->nChunkSize), &rc2, NULL); + pSubOpen = multiplexSubOpen(pGroup, (int)(size/pGroup->nChunkSize), &rc2,0); if( pSubOpen ){ rc2 = pSubOpen->pMethods->xTruncate(pSubOpen, size % pGroup->nChunkSize); if( rc2!=SQLITE_OK ) rc = rc2; @@ -774,10 +751,9 @@ static int multiplexSync(sqlite3_file *pConn, int flags){ int rc = SQLITE_OK; int i; multiplexEnter(); - for(i=0; inMaxChunks; i++){ - /* if we don't have it open, we don't need to sync it */ - if( pGroup->bOpen[i] ){ - sqlite3_file *pSubOpen = pGroup->pReal[i]; + for(i=0; inReal; i++){ + sqlite3_file *pSubOpen = pGroup->aReal[i].p; + if( pSubOpen ){ int rc2 = pSubOpen->pMethods->xSync(pSubOpen, flags); if( rc2!=SQLITE_OK ) rc = rc2; } @@ -797,7 +773,7 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){ int i; multiplexEnter(); if( !pGroup->bEnabled ){ - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); if( pSubOpen==0 ){ rc = SQLITE_IOERR_FSTAT; }else{ @@ -805,35 +781,20 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){ } }else{ *pSize = 0; - for(i=0; inMaxChunks; i++){ - sqlite3_file *pSubOpen = NULL; - /* if not opened already, check to see if the chunk exists */ - if( pGroup->bOpen[i] ){ - pSubOpen = pGroup->pReal[i]; + for(i=0; 1; i++){ + sqlite3_file *pSubOpen = 0; + sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; + int exists = 0; + rc = multiplexSubFilename(pGroup, i); + if( rc ) break; + rc2 = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[i].z, + SQLITE_ACCESS_EXISTS, &exists); + if( rc2==SQLITE_OK && exists){ + /* if it exists, open it */ + pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL); }else{ - sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ - int exists = 0; - memcpy(gMultiplex.zName, pGroup->zName, pGroup->nName+1); - if( i ){ -#ifdef SQLITE_MULTIPLEX_EXT_OVWR - sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, - gMultiplex.zName+pGroup->nName-SQLITE_MULTIPLEX_EXT_SZ, - SQLITE_MULTIPLEX_EXT_FMT, i); -#else - sqlite3_snprintf(SQLITE_MULTIPLEX_EXT_SZ+1, - gMultiplex.zName+pGroup->nName, - SQLITE_MULTIPLEX_EXT_FMT, i); -#endif - } - rc2 = pOrigVfs->xAccess(pOrigVfs, gMultiplex.zName, - SQLITE_ACCESS_EXISTS, &exists); - if( rc2==SQLITE_OK && exists){ - /* if it exists, open it */ - pSubOpen = multiplexSubOpen(p, i, &rc, NULL); - }else{ - /* stop at first "gap" */ - break; - } + /* stop at first "gap" */ + break; } if( pSubOpen ){ sqlite3_int64 sz; @@ -860,7 +821,7 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){ static int multiplexLock(sqlite3_file *pConn, int lock){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xLock(pSubOpen, lock); } @@ -872,7 +833,7 @@ static int multiplexLock(sqlite3_file *pConn, int lock){ static int multiplexUnlock(sqlite3_file *pConn, int lock){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xUnlock(pSubOpen, lock); } @@ -884,7 +845,7 @@ static int multiplexUnlock(sqlite3_file *pConn, int lock){ static int multiplexCheckReservedLock(sqlite3_file *pConn, int *pResOut){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xCheckReservedLock(pSubOpen, pResOut); } @@ -924,15 +885,7 @@ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){ } break; case MULTIPLEX_CTRL_SET_MAX_CHUNKS: - if( pArg ) { - int nMaxChunks = *(int *)pArg; - if(( nMaxChunks<1 ) || ( nMaxChunks>SQLITE_MULTIPLEX_MAX_CHUNKS )){ - rc = SQLITE_MISUSE; - }else{ - pGroup->nMaxChunks = nMaxChunks; - rc = SQLITE_OK; - } - } + rc = SQLITE_OK; break; case SQLITE_FCNTL_SIZE_HINT: case SQLITE_FCNTL_CHUNK_SIZE: @@ -940,7 +893,7 @@ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){ rc = SQLITE_OK; break; default: - pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL); if( pSubOpen ){ rc = pSubOpen->pMethods->xFileControl(pSubOpen, op, pArg); } @@ -954,7 +907,7 @@ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){ static int multiplexSectorSize(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xSectorSize(pSubOpen); } @@ -966,7 +919,7 @@ static int multiplexSectorSize(sqlite3_file *pConn){ static int multiplexDeviceCharacteristics(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xDeviceCharacteristics(pSubOpen); } @@ -984,7 +937,7 @@ static int multiplexShmMap( ){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xShmMap(pSubOpen, iRegion, szRegion, bExtend,pp); } @@ -1001,7 +954,7 @@ static int multiplexShmLock( ){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xShmLock(pSubOpen, ofst, n, flags); } @@ -1013,7 +966,7 @@ static int multiplexShmLock( static void multiplexShmBarrier(sqlite3_file *pConn){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ pSubOpen->pMethods->xShmBarrier(pSubOpen); } @@ -1024,7 +977,7 @@ static void multiplexShmBarrier(sqlite3_file *pConn){ static int multiplexShmUnmap(sqlite3_file *pConn, int deleteFlag){ multiplexConn *p = (multiplexConn*)pConn; int rc; - sqlite3_file *pSubOpen = multiplexSubOpen(p, 0, &rc, NULL); + sqlite3_file *pSubOpen = multiplexSubOpen(p->pGroup, 0, &rc, NULL); if( pSubOpen ){ return pSubOpen->pMethods->xShmUnmap(pSubOpen, deleteFlag); } @@ -1054,11 +1007,6 @@ int sqlite3_multiplex_initialize(const char *zOrigVfsName, int makeDefault){ if( !gMultiplex.pMutex ){ return SQLITE_NOMEM; } - gMultiplex.zName = sqlite3_malloc(pOrigVfs->mxPathname); - if( !gMultiplex.zName ){ - sqlite3_mutex_free(gMultiplex.pMutex); - return SQLITE_NOMEM; - } gMultiplex.pGroups = NULL; gMultiplex.isInitialized = 1; gMultiplex.pOrigVfs = pOrigVfs; @@ -1119,7 +1067,6 @@ int sqlite3_multiplex_shutdown(void){ if( gMultiplex.isInitialized==0 ) return SQLITE_MISUSE; if( gMultiplex.pGroups ) return SQLITE_MISUSE; gMultiplex.isInitialized = 0; - sqlite3_free(gMultiplex.zName); sqlite3_mutex_free(gMultiplex.pMutex); sqlite3_vfs_unregister(&gMultiplex.sThisVfs); memset(&gMultiplex, 0, sizeof(gMultiplex)); @@ -1221,8 +1168,8 @@ static int test_multiplex_dump( Tcl_NewIntObj(pGroup->flags)); /* count number of chunks with open handles */ - for(i=0; inMaxChunks; i++){ - if( pGroup->bOpen[i] ) nChunks++; + for(i=0; inReal; i++){ + if( pGroup->aReal[i].p!=0 ) nChunks++; } Tcl_ListObjAppendElement(interp, pGroupTerm, Tcl_NewIntObj(nChunks)); @@ -1230,7 +1177,7 @@ static int test_multiplex_dump( Tcl_ListObjAppendElement(interp, pGroupTerm, Tcl_NewIntObj(pGroup->nChunkSize)); Tcl_ListObjAppendElement(interp, pGroupTerm, - Tcl_NewIntObj(pGroup->nMaxChunks)); + Tcl_NewIntObj(pGroup->nReal)); Tcl_ListObjAppendElement(interp, pResult, pGroupTerm); } diff --git a/test/multiplex.test b/test/multiplex.test index 9278e84274..93d4aafc16 100644 --- a/test/multiplex.test +++ b/test/multiplex.test @@ -49,6 +49,7 @@ proc multiplex_set {db name chunk_size max_chunks} { # and files with the chunk extension. proc multiplex_delete {name} { global g_max_chunks + forcedelete $name for {set i 0} {$i<$g_max_chunks} {incr i} { forcedelete [multiplex_name $name $i] forcedelete [multiplex_name $name-journal $i] @@ -78,10 +79,10 @@ do_test multiplex-1.8 { sqlite3_multiplex_shutdown } {SQLITE_OK} do_test multiplex-1.9.1 { sqlite3_multiplex_initialize "" 1 } {SQLITE_OK} do_test multiplex-1.9.2 { sqlite3 db test.db } {} do_test multiplex-1.9.3 { multiplex_set db main 32768 16 } {SQLITE_OK} -do_test multiplex-1.9.4 { multiplex_set db main 32768 -1 } {SQLITE_MISUSE} +do_test multiplex-1.9.4 { multiplex_set db main 32768 -1 } {SQLITE_OK} do_test multiplex-1.9.5 { multiplex_set db main -1 16 } {SQLITE_MISUSE} do_test multiplex-1.9.6 { multiplex_set db main 31 16 } {SQLITE_OK} -do_test multiplex-1.9.7 { multiplex_set db main 32768 100 } {SQLITE_MISUSE} +do_test multiplex-1.9.7 { multiplex_set db main 32768 100 } {SQLITE_OK} do_test multiplex-1.9.8 { multiplex_set db main 1073741824 1 } {SQLITE_OK} do_test multiplex-1.9.9 { db close } {} do_test multiplex-1.9.10 { sqlite3_multiplex_shutdown } {SQLITE_OK} @@ -89,10 +90,10 @@ do_test multiplex-1.9.10 { sqlite3_multiplex_shutdown } {SQLITE_OK} do_test multiplex-1.10.1 { sqlite3_multiplex_initialize "" 1 } {SQLITE_OK} do_test multiplex-1.10.2 { sqlite3 db test.db } {} do_test multiplex-1.10.3 { lindex [ catchsql { SELECT multiplex_control(2, 32768); } ] 0 } {0} -do_test multiplex-1.10.4 { lindex [ catchsql { SELECT multiplex_control(3, -1); } ] 0 } {1} +do_test multiplex-1.10.4 { lindex [ catchsql { SELECT multiplex_control(3, -1); } ] 0 } {0} do_test multiplex-1.10.5 { lindex [ catchsql { SELECT multiplex_control(2, -1); } ] 0 } {1} do_test multiplex-1.10.6 { lindex [ catchsql { SELECT multiplex_control(2, 31); } ] 0 } {0} -do_test multiplex-1.10.7 { lindex [ catchsql { SELECT multiplex_control(3, 100); } ] 0 } {1} +do_test multiplex-1.10.7 { lindex [ catchsql { SELECT multiplex_control(3, 100); } ] 0 } {0} do_test multiplex-1.10.8 { lindex [ catchsql { SELECT multiplex_control(2, 1073741824); } ] 0 } {0} do_test multiplex-1.10.9 { db close } {} do_test multiplex-1.10.10 { sqlite3_multiplex_shutdown } {SQLITE_OK} @@ -146,8 +147,9 @@ do_test multiplex-1.13.7 { sqlite3_multiplex_shutdown } sqlite3_multiplex_initialize "" 1 multiplex_set db main 32768 16 +file delete -force test.x do_test multiplex-2.1.2 { - sqlite3 db test.db + sqlite3 db test.x execsql { PRAGMA page_size=1024; PRAGMA auto_vacuum=OFF; @@ -159,7 +161,7 @@ do_test multiplex-2.1.2 { INSERT INTO t1 VALUES(2, randomblob(1100)); } } {} -do_test multiplex-2.1.3 { file size [multiplex_name test.db 0] } {4096} +do_test multiplex-2.1.3 { file size [multiplex_name test.x 0] } {4096} do_test multiplex-2.1.4 { execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) } } {} @@ -167,10 +169,10 @@ do_test multiplex-2.1.4 { do_test multiplex-2.2.1 { execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) } } {} -do_test multiplex-2.2.3 { file size [multiplex_name test.db 0] } {6144} +do_test multiplex-2.2.3 { file size [multiplex_name test.x 0] } {6144} do_test multiplex-2.3.1 { - sqlite3 db2 test2.db + sqlite3 db2 test2.x db2 close } {} @@ -181,7 +183,7 @@ do_test multiplex-2.4.1 { do_test multiplex-2.4.2 { execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) } } {} -do_test multiplex-2.4.4 { file size [multiplex_name test.db 0] } {7168} +do_test multiplex-2.4.4 { file size [multiplex_name test.x 0] } {7168} do_test multiplex-2.4.99 { db close sqlite3_multiplex_shutdown @@ -189,9 +191,9 @@ do_test multiplex-2.4.99 { do_test multiplex-2.5.1 { - multiplex_delete test.db + multiplex_delete test.x sqlite3_multiplex_initialize "" 1 - sqlite3 db test.db + sqlite3 db test.x multiplex_set db main 4096 16 } {SQLITE_OK} @@ -236,8 +238,8 @@ do_test multiplex-2.5.8 { db eval {SELECT a,length(b) FROM t1 WHERE a=4} } {4 4000} -do_test multiplex-2.5.9 { file size [multiplex_name test.db 0] } [list $g_chunk_size] -do_test multiplex-2.5.10 { file size [multiplex_name test.db 1] } [list $g_chunk_size] +do_test multiplex-2.5.9 { file size [multiplex_name test.x 0] } [list $g_chunk_size] +do_test multiplex-2.5.10 { file size [multiplex_name test.x 1] } [list $g_chunk_size] do_test multiplex-2.5.99 { db close @@ -523,50 +525,6 @@ do_faultsim_test multiplex-5.5 -prep { multiplex_set db main 32768 16 } -# test that mismatch filesize is detected -# -# Do not run this test if $::G(perm:presql) is set. If it is set, then the -# expected IO error will occur within the Tcl [sqlite3] wrapper, not within -# the first SQL statement executed below. This breaks the test case. -# -if {0==[info exists ::G(perm:presql)] || $::G(perm:presql) == ""} { - set all_journal_modes {delete persist truncate memory off} - foreach jmode $all_journal_modes { - do_test multiplex-5.6.1.$jmode { - sqlite3_multiplex_shutdown - multiplex_delete test.db - sqlite3 db test.db - db eval { - PRAGMA page_size = 1024; - PRAGMA auto_vacuum = off; - } - db eval "PRAGMA journal_mode = $jmode;" - } $jmode - do_test multiplex-5.6.2.$jmode { - execsql { - CREATE TABLE t1(a, b); - INSERT INTO t1 VALUES(1, randomblob(15000)); - INSERT INTO t1 VALUES(2, randomblob(15000)); - INSERT INTO t1 VALUES(3, randomblob(15000)); - INSERT INTO t1 VALUES(4, randomblob(15000)); - INSERT INTO t1 VALUES(5, randomblob(15000)); - } - db close - sqlite3_multiplex_initialize "" 1 - sqlite3 db test.db - multiplex_set db main 4096 16 - } {SQLITE_OK} - do_test multiplex-5.6.3.$jmode { - catchsql { - INSERT INTO t1 VALUES(6, randomblob(15000)); - } - } {1 {disk I/O error}} - do_test multiplex-5.6.4.$jmode { - db close - } {} - } -} - #------------------------------------------------------------------------- # Test that you can vacuum a multiplex'ed DB. @@ -575,8 +533,9 @@ ifcapable vacuum { sqlite3_multiplex_shutdown do_test multiplex-6.0.0 { multiplex_delete test.db + multiplex_delete test.x sqlite3_multiplex_initialize "" 1 - sqlite3 db test.db + sqlite3 db test.x multiplex_set db main 4096 16 } {SQLITE_OK} @@ -592,8 +551,8 @@ do_test multiplex-6.1.0 { INSERT INTO t1 VALUES(2, randomblob($g_chunk_size)); } } {} -do_test multiplex-6.2.1 { file size [multiplex_name test.db 0] } [list $g_chunk_size] -do_test multiplex-6.2.2 { file size [multiplex_name test.db 1] } [list $g_chunk_size] +do_test multiplex-6.2.1 { file size [multiplex_name test.x 0] } [list $g_chunk_size] +do_test multiplex-6.2.2 { file size [multiplex_name test.x 1] } [list $g_chunk_size] do_test multiplex-6.3.0 { execsql { VACUUM } @@ -601,7 +560,7 @@ do_test multiplex-6.3.0 { do_test multiplex-6.99 { db close - multiplex_delete test.db + multiplex_delete test.x sqlite3_multiplex_shutdown } {SQLITE_OK} From 877eeb3b56f8f2829fab39d61fb3afb1ba07029c Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 20 Jul 2011 17:13:30 +0000 Subject: [PATCH 45/52] All the SQLITE_OPEN_URI flag to propagate down into the VFS. FossilOrigin-Name: 29866f9598502a007816410fade34f1d0952dea0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c0583658f0..692bd49cf1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sthe\smultiplexor\sopens\san\sauxiliary\sfile,\sit\snow\spersists\sthe\sname\sof\sthat\nfile\suntil\sit\sis\sclosed,\sas\sit\sshould.\s\sRemove\sthe\slimit\son\sthe\snumber\sof\nauxiliary\sfiles\sused\sby\sthe\smultiplexor. -D 2011-07-20T16:35:31.624 +C All\sthe\sSQLITE_OPEN_URI\sflag\sto\spropagate\sdown\sinto\sthe\sVFS. +D 2011-07-20T17:13:30.035 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F src/mutex_os2.c 882d735098c07c8c6a5472b8dd66e19675fe117f F src/mutex_unix.c b4f4e923bb8de93ec3f251fadb50855f23df9579 F src/mutex_w32.c 5e54f3ba275bcb5d00248b8c23107df2e2f73e33 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 -F src/os.c 22ac61d06e72a0dac900400147333b07b13d8e1d +F src/os.c fcc717427a80b2ed225373f07b642dc1aad7490b F src/os.h 9dbed8c2b9c1f2f2ebabc09e49829d4777c26bf9 F src/os_common.h a8f95b81eca8a1ab8593d23e94f8a35f35d4078f F src/os_os2.c 4a75888ba3dfc820ad5e8177025972d74d7f2440 @@ -952,7 +952,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P ed5f0aad6b21066bacd01521e82c22e96991f400 -R e63c3d9c3ff7ac85b18419c4f45127d4 +P 1ffa542bf913200a18ef77447aec4fc3ca1ed618 +R a8712e3b34abd0caa57ff420866bce41 U drh -Z 1541751b97b37082064c8e76500e9b76 +Z aa653174b90521d7bcecad6884ea5241 diff --git a/manifest.uuid b/manifest.uuid index 04e86b35f3..2d93c3ac97 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1ffa542bf913200a18ef77447aec4fc3ca1ed618 \ No newline at end of file +29866f9598502a007816410fade34f1d0952dea0 \ No newline at end of file diff --git a/src/os.c b/src/os.c index ba0438adef..41ec69ce45 100644 --- a/src/os.c +++ b/src/os.c @@ -136,7 +136,7 @@ int sqlite3OsOpen( ** down into the VFS layer. Some SQLITE_OPEN_ flags (for example, ** SQLITE_OPEN_FULLMUTEX or SQLITE_OPEN_SHAREDCACHE) are blocked before ** reaching the VFS. */ - rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x87f3f, pFlagsOut); + rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x87f7f, pFlagsOut); assert( rc==SQLITE_OK || pFile->pMethods==0 ); return rc; } From 0a0ca697d86e0a4fda38334933e263f904b62ae6 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 20 Jul 2011 17:59:38 +0000 Subject: [PATCH 46/52] For an existing multiplexed database, try to set the chunk size automatically based on the sizes of the preexisting pieces. FossilOrigin-Name: 427a9a5120a68bfa12fec97cfd02eb9b28b3fa6b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/test_multiplex.c | 37 ++++++++++++++++++++++++++++--------- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 692bd49cf1..b768ecaf18 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C All\sthe\sSQLITE_OPEN_URI\sflag\sto\spropagate\sdown\sinto\sthe\sVFS. -D 2011-07-20T17:13:30.035 +C For\san\sexisting\smultiplexed\sdatabase,\stry\sto\sset\sthe\schunk\ssize\sautomatically\nbased\son\sthe\ssizes\sof\sthe\spreexisting\spieces. +D 2011-07-20T17:59:38.568 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -213,7 +213,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c 03313c693cca72959dcaaf79f8d76f21c01e19ff F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 7ca7be34e0e09ef0ed6619544552ed95732e41f6 -F src/test_multiplex.c 282f315fd035c8d8342a4cc59748c60216b61fef +F src/test_multiplex.c 7f5b2ec63ff514f12e95ffbe134fca87eaac7e96 F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec @@ -952,7 +952,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 1ffa542bf913200a18ef77447aec4fc3ca1ed618 -R a8712e3b34abd0caa57ff420866bce41 +P 29866f9598502a007816410fade34f1d0952dea0 +R f7447480ebd05c67794755dbb19b5608 U drh -Z aa653174b90521d7bcecad6884ea5241 +Z ce5790e6bda5ced4f7371ad338df0dcd diff --git a/manifest.uuid b/manifest.uuid index 2d93c3ac97..372fbaa188 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -29866f9598502a007816410fade34f1d0952dea0 \ No newline at end of file +427a9a5120a68bfa12fec97cfd02eb9b28b3fa6b \ No newline at end of file diff --git a/src/test_multiplex.c b/src/test_multiplex.c index b6fe9e8b63..c5725e07a2 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -453,7 +453,7 @@ static int multiplexOpen( int rc = SQLITE_OK; /* Result code */ multiplexConn *pMultiplexOpen; /* The new multiplex file descriptor */ multiplexGroup *pGroup; /* Corresponding multiplexGroup object */ - sqlite3_file *pSubOpen; /* Real file descriptor */ + sqlite3_file *pSubOpen = 0; /* Real file descriptor */ sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ int nName; int sz; @@ -513,16 +513,35 @@ static int multiplexOpen( memcpy(pGroup->zName, zName, nName+1); pGroup->nName = nName; pGroup->flags = flags; - pSubOpen = multiplexSubOpen(pGroup, 0, &rc, pOutFlags); + rc = multiplexSubFilename(pGroup, 1); + if( rc==SQLITE_OK ){ + pSubOpen = multiplexSubOpen(pGroup, 0, &rc, pOutFlags); + } if( pSubOpen ){ - /* if this file is already larger than chunk size, disable - ** the multiplex feature. - */ + int exists, rc2, rc3; sqlite3_int64 sz; - int rc2 = pSubOpen->pMethods->xFileSize(pSubOpen, &sz); - if( (rc2==SQLITE_OK) && (sz>pGroup->nChunkSize) ){ - pGroup->bEnabled = 0; + + rc2 = pSubOpen->pMethods->xFileSize(pSubOpen, &sz); + if( rc2==SQLITE_OK ){ + /* If the first overflow file exists and if the size of the main file + ** is different from the chunk size, that means the chunk size is set + ** set incorrectly. So fix it. + ** + ** Or, if the first overflow file does not exist and the main file is + ** larger than the chunk size, that means the chunk size is too small. + ** But we have no way of determining the intended chunk size, so + ** just disable the multiplexor all togethre. + */ + rc3 = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[1].z, + SQLITE_ACCESS_EXISTS, &exists); + if( rc3==SQLITE_OK && exists && sz==(sz&0xffff0000) && sz>0 + && sz!=pGroup->nChunkSize ){ + pGroup->nChunkSize = sz; + }else if( rc3==SQLITE_OK && !exists && sz>pGroup->nChunkSize ){ + pGroup->bEnabled = 0; + } } + if( pSubOpen->pMethods->iVersion==1 ){ pMultiplexOpen->base.pMethods = &gMultiplex.sIoMethodsV1; }else{ @@ -780,10 +799,10 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){ rc = pSubOpen->pMethods->xFileSize(pSubOpen, pSize); } }else{ + sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; *pSize = 0; for(i=0; 1; i++){ sqlite3_file *pSubOpen = 0; - sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; int exists = 0; rc = multiplexSubFilename(pGroup, i); if( rc ) break; From e6deb204589077c46427b1f25f0a248980419b29 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 21 Jul 2011 20:59:58 +0000 Subject: [PATCH 47/52] All multiplexor chunk sizes up to 4GiB. Disable the multiplexor if the chunk size is set to 0. FossilOrigin-Name: 83191ad6f31536b0c1929938e1fbeb4cf6121ab0 --- manifest | 14 +++++------ manifest.uuid | 2 +- src/test_multiplex.c | 60 +++++++++++++++++++++++++------------------- test/multiplex.test | 2 -- 4 files changed, 42 insertions(+), 36 deletions(-) diff --git a/manifest b/manifest index b768ecaf18..5bca43402e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\san\sexisting\smultiplexed\sdatabase,\stry\sto\sset\sthe\schunk\ssize\sautomatically\nbased\son\sthe\ssizes\sof\sthe\spreexisting\spieces. -D 2011-07-20T17:59:38.568 +C All\smultiplexor\schunk\ssizes\sup\sto\s4GiB.\s\sDisable\sthe\smultiplexor\sif\sthe\nchunk\ssize\sis\sset\sto\s0. +D 2011-07-21T20:59:58.248 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -213,7 +213,7 @@ F src/test_intarray.h 489edb9068bb926583445cb02589344961054207 F src/test_journal.c 03313c693cca72959dcaaf79f8d76f21c01e19ff F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e F src/test_malloc.c 7ca7be34e0e09ef0ed6619544552ed95732e41f6 -F src/test_multiplex.c 7f5b2ec63ff514f12e95ffbe134fca87eaac7e96 +F src/test_multiplex.c 991a60733dbde8c529043d466c5c44d180762561 F src/test_multiplex.h e99c571bc4968b7a9363b661481f3934bfead61d F src/test_mutex.c a6bd7b9cf6e19d989e31392b06ac8d189f0d573e F src/test_onefile.c 40cf9e212a377a6511469384a64b01e6e34b2eec @@ -595,7 +595,7 @@ F test/misc5.test 9f9338f8211c7f5d1cbe16331fa65d019501aa50 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test 29032efcd3d826fbd409e2a7af873e7939f4a4e3 F test/misuse.test 30b3a458e5a70c31e74c291937b6c82204c59f33 -F test/multiplex.test 2256e90e8780de6eeafc07d4084427c3a97d3bcb +F test/multiplex.test b45367b1dac7dfa4c5b8ff0f3844260804a0034d F test/mutex1.test 78b2b9bb320e51d156c4efdb71b99b051e7a4b41 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test dc212a22b36109fd1ae37154292444ef249c5ec2 @@ -952,7 +952,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 29866f9598502a007816410fade34f1d0952dea0 -R f7447480ebd05c67794755dbb19b5608 +P 427a9a5120a68bfa12fec97cfd02eb9b28b3fa6b +R 046ebc91a200dc382cece52b5754a0c1 U drh -Z ce5790e6bda5ced4f7371ad338df0dcd +Z 032caca1acf96a44c6601a79dc41d32f diff --git a/manifest.uuid b/manifest.uuid index 372fbaa188..3b6b943327 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -427a9a5120a68bfa12fec97cfd02eb9b28b3fa6b \ No newline at end of file +83191ad6f31536b0c1929938e1fbeb4cf6121ab0 \ No newline at end of file diff --git a/src/test_multiplex.c b/src/test_multiplex.c index c5725e07a2..732df82cd7 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -134,7 +134,7 @@ struct multiplexGroup { char *zName; /* Base filename of this group */ int nName; /* Length of base filename */ int flags; /* Flags used for original opening */ - int nChunkSize; /* Chunk size used for this group */ + unsigned int szChunk; /* Chunk size used for this group */ int bEnabled; /* TRUE to use Multiplex VFS for this file */ multiplexGroup *pNext, *pPrev; /* Doubly linked list of all group objects */ }; @@ -494,18 +494,27 @@ static int multiplexOpen( } if( rc==SQLITE_OK ){ - const char *zChunkSize; /* assign pointers to extra space allocated */ char *p = (char *)&pGroup[1]; pMultiplexOpen->pGroup = pGroup; memset(pGroup, 0, sz); pGroup->bEnabled = -1; - pGroup->nChunkSize = SQLITE_MULTIPLEX_CHUNK_SIZE; + pGroup->szChunk = SQLITE_MULTIPLEX_CHUNK_SIZE; if( flags & SQLITE_OPEN_URI ){ + const char *zChunkSize; zChunkSize = sqlite3_uri_parameter(zName, "chunksize"); if( zChunkSize ){ - int n = atoi(zChunkSize); - if( n>0 ) pGroup->nChunkSize = (n+0xffff)&~0xffff; + unsigned int n = 0; + int i; + for(i=0; zChunkSize[i]>='0' && zChunkSize[i]<='9'; i++){ + n = n*10 + zChunkSize[i] - '0'; + } + if( n>0 ){ + pGroup->szChunk = (n+0xffff)&~0xffff; + }else{ + /* A zero or negative chunksize disabled the multiplexor */ + pGroup->bEnabled = 0; + } } } pGroup->zName = p; @@ -535,9 +544,9 @@ static int multiplexOpen( rc3 = pOrigVfs->xAccess(pOrigVfs, pGroup->aReal[1].z, SQLITE_ACCESS_EXISTS, &exists); if( rc3==SQLITE_OK && exists && sz==(sz&0xffff0000) && sz>0 - && sz!=pGroup->nChunkSize ){ - pGroup->nChunkSize = sz; - }else if( rc3==SQLITE_OK && !exists && sz>pGroup->nChunkSize ){ + && sz!=pGroup->szChunk ){ + pGroup->szChunk = sz; + }else if( rc3==SQLITE_OK && !exists && sz>pGroup->szChunk ){ pGroup->bEnabled = 0; } } @@ -656,15 +665,14 @@ static int multiplexRead( } }else{ while( iAmt > 0 ){ - int i = (int)(iOfst / pGroup->nChunkSize); + int i = (int)(iOfst / pGroup->szChunk); sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL); if( pSubOpen ){ - int extra = ((int)(iOfst % pGroup->nChunkSize) + iAmt) - - pGroup->nChunkSize; + int extra = ((int)(iOfst % pGroup->szChunk) + iAmt) - pGroup->szChunk; if( extra<0 ) extra = 0; iAmt -= extra; rc = pSubOpen->pMethods->xRead(pSubOpen, pBuf, iAmt, - iOfst % pGroup->nChunkSize); + iOfst % pGroup->szChunk); if( rc!=SQLITE_OK ) break; pBuf = (char *)pBuf + iAmt; iOfst += iAmt; @@ -702,15 +710,15 @@ static int multiplexWrite( } }else{ while( iAmt > 0 ){ - int i = (int)(iOfst / pGroup->nChunkSize); + int i = (int)(iOfst / pGroup->szChunk); sqlite3_file *pSubOpen = multiplexSubOpen(pGroup, i, &rc, NULL); if( pSubOpen ){ - int extra = ((int)(iOfst % pGroup->nChunkSize) + iAmt) - - pGroup->nChunkSize; + int extra = ((int)(iOfst % pGroup->szChunk) + iAmt) - + pGroup->szChunk; if( extra<0 ) extra = 0; iAmt -= extra; rc = pSubOpen->pMethods->xWrite(pSubOpen, pBuf, iAmt, - iOfst % pGroup->nChunkSize); + iOfst % pGroup->szChunk); if( rc!=SQLITE_OK ) break; pBuf = (char *)pBuf + iAmt; iOfst += iAmt; @@ -747,12 +755,12 @@ static int multiplexTruncate(sqlite3_file *pConn, sqlite3_int64 size){ sqlite3_file *pSubOpen; sqlite3_vfs *pOrigVfs = gMultiplex.pOrigVfs; /* Real VFS */ /* delete the chunks above the truncate limit */ - for(i=(int)(size / pGroup->nChunkSize)+1; inReal; i++){ + for(i=(int)(size / pGroup->szChunk)+1; inReal; i++){ multiplexSubClose(pGroup, i, pOrigVfs); } - pSubOpen = multiplexSubOpen(pGroup, (int)(size/pGroup->nChunkSize), &rc2,0); + pSubOpen = multiplexSubOpen(pGroup, (int)(size/pGroup->szChunk), &rc2,0); if( pSubOpen ){ - rc2 = pSubOpen->pMethods->xTruncate(pSubOpen, size % pGroup->nChunkSize); + rc2 = pSubOpen->pMethods->xTruncate(pSubOpen, size % pGroup->szChunk); if( rc2!=SQLITE_OK ) rc = rc2; }else{ rc = SQLITE_IOERR_TRUNCATE; @@ -821,7 +829,7 @@ static int multiplexFileSize(sqlite3_file *pConn, sqlite3_int64 *pSize){ if( rc2!=SQLITE_OK ){ rc = rc2; }else{ - if( sz>pGroup->nChunkSize ){ + if( sz>pGroup->szChunk ){ rc = SQLITE_IOERR_FSTAT; } *pSize += sz; @@ -891,14 +899,14 @@ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){ break; case MULTIPLEX_CTRL_SET_CHUNK_SIZE: if( pArg ) { - int nChunkSize = *(int *)pArg; - if( nChunkSize<1 ){ + unsigned int szChunk = *(unsigned*)pArg; + if( szChunk<1 ){ rc = SQLITE_MISUSE; }else{ /* Round up to nearest multiple of MAX_PAGE_SIZE. */ - nChunkSize = (nChunkSize + (MAX_PAGE_SIZE-1)); - nChunkSize &= ~(MAX_PAGE_SIZE-1); - pGroup->nChunkSize = nChunkSize; + szChunk = (szChunk + (MAX_PAGE_SIZE-1)); + szChunk &= ~(MAX_PAGE_SIZE-1); + pGroup->szChunk = szChunk; rc = SQLITE_OK; } } @@ -1194,7 +1202,7 @@ static int test_multiplex_dump( Tcl_NewIntObj(nChunks)); Tcl_ListObjAppendElement(interp, pGroupTerm, - Tcl_NewIntObj(pGroup->nChunkSize)); + Tcl_NewIntObj(pGroup->szChunk)); Tcl_ListObjAppendElement(interp, pGroupTerm, Tcl_NewIntObj(pGroup->nReal)); diff --git a/test/multiplex.test b/test/multiplex.test index 93d4aafc16..28ce1c77e7 100644 --- a/test/multiplex.test +++ b/test/multiplex.test @@ -80,7 +80,6 @@ do_test multiplex-1.9.1 { sqlite3_multiplex_initialize "" 1 } {SQLITE_OK} do_test multiplex-1.9.2 { sqlite3 db test.db } {} do_test multiplex-1.9.3 { multiplex_set db main 32768 16 } {SQLITE_OK} do_test multiplex-1.9.4 { multiplex_set db main 32768 -1 } {SQLITE_OK} -do_test multiplex-1.9.5 { multiplex_set db main -1 16 } {SQLITE_MISUSE} do_test multiplex-1.9.6 { multiplex_set db main 31 16 } {SQLITE_OK} do_test multiplex-1.9.7 { multiplex_set db main 32768 100 } {SQLITE_OK} do_test multiplex-1.9.8 { multiplex_set db main 1073741824 1 } {SQLITE_OK} @@ -91,7 +90,6 @@ do_test multiplex-1.10.1 { sqlite3_multiplex_initialize "" 1 } do_test multiplex-1.10.2 { sqlite3 db test.db } {} do_test multiplex-1.10.3 { lindex [ catchsql { SELECT multiplex_control(2, 32768); } ] 0 } {0} do_test multiplex-1.10.4 { lindex [ catchsql { SELECT multiplex_control(3, -1); } ] 0 } {0} -do_test multiplex-1.10.5 { lindex [ catchsql { SELECT multiplex_control(2, -1); } ] 0 } {1} do_test multiplex-1.10.6 { lindex [ catchsql { SELECT multiplex_control(2, 31); } ] 0 } {0} do_test multiplex-1.10.7 { lindex [ catchsql { SELECT multiplex_control(3, 100); } ] 0 } {0} do_test multiplex-1.10.8 { lindex [ catchsql { SELECT multiplex_control(2, 1073741824); } ] 0 } {0} From b51bf43966fdc82055195c03f7739564c5bd12df Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 21 Jul 2011 21:29:35 +0000 Subject: [PATCH 48/52] Compile with the SQLITE_ENABLE_8_3_NAME macro set to 2 to force 8+3 filenames to be on all the time. FossilOrigin-Name: ae83dac70173fecf203bf7e9f7bcb189d3e48f08 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/util.c | 10 ++++++++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 5bca43402e..7053b7dba5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C All\smultiplexor\schunk\ssizes\sup\sto\s4GiB.\s\sDisable\sthe\smultiplexor\sif\sthe\nchunk\ssize\sis\sset\sto\s0. -D 2011-07-21T20:59:58.248 +C Compile\swith\sthe\sSQLITE_ENABLE_8_3_NAME\smacro\sset\sto\s2\sto\sforce\s8+3\sfilenames\nto\sbe\son\sall\sthe\stime. +D 2011-07-21T21:29:35.633 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -236,7 +236,7 @@ F src/tokenize.c c819d9f72168a035d545a5bdafe9b085b20df705 F src/trigger.c 1cfb80e2290ef66ea89cb4e821caae65a02c0d56 F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0 -F src/util.c 0f33bbbdfcc4a2d8cf20c3b2a16ffc3b57c58a70 +F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e F src/vdbe.c a9ced64f380bbd8b04da3a1c3a9602d3942704b5 F src/vdbe.h 5cf09e7ee8a3f7d93bc51f196a96550786afe7a1 @@ -952,7 +952,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 427a9a5120a68bfa12fec97cfd02eb9b28b3fa6b -R 046ebc91a200dc382cece52b5754a0c1 +P 83191ad6f31536b0c1929938e1fbeb4cf6121ab0 +R f36be610ec3236a1941508a82c6dca72 U drh -Z 032caca1acf96a44c6601a79dc41d32f +Z be7120c77d6b184261113de7aebad3e1 diff --git a/manifest.uuid b/manifest.uuid index 3b6b943327..35b1e172d6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83191ad6f31536b0c1929938e1fbeb4cf6121ab0 \ No newline at end of file +ae83dac70173fecf203bf7e9f7bcb189d3e48f08 \ No newline at end of file diff --git a/src/util.c b/src/util.c index de73577203..67e43b4ba8 100644 --- a/src/util.c +++ b/src/util.c @@ -1149,12 +1149,15 @@ int sqlite3AbsInt32(int x){ #ifdef SQLITE_ENABLE_8_3_NAMES /* -** If SQLITE_ENABLE_8_3_NAME is set at compile-time and if the database +** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database ** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and ** if filename in z[] has a suffix (a.k.a. "extension") that is longer than ** three characters, then shorten the suffix on z[] to be the last three ** characters of the original suffix. ** +** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always +** do the suffix shortening regardless of URI parameter. +** ** Examples: ** ** test.db-journal => test.nal @@ -1162,9 +1165,12 @@ int sqlite3AbsInt32(int x){ ** test.db-shm => test.shm */ void sqlite3FileSuffix3(const char *zBaseFilename, char *z){ +#if SQLITE_ENABLE_8_3_NAMES<2 const char *zOk; zOk = sqlite3_uri_parameter(zBaseFilename, "8_3_names"); - if( zOk && sqlite3GetBoolean(zOk) ){ + if( zOk && sqlite3GetBoolean(zOk) ) +#endif + { int i, sz; sz = sqlite3Strlen30(z); for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} From 16b92eef35f6cbac591171ddad58d43f3350ebec Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 22 Jul 2011 10:33:04 +0000 Subject: [PATCH 49/52] Use $(TCLSH_CMD) instead of tclsh in the autoconf makefile. FossilOrigin-Name: fd36d8067b5a29d9acbf8c96b9d837406d0c0cce --- Makefile.in | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile.in b/Makefile.in index 4a81e15664..2b3311ac99 100644 --- a/Makefile.in +++ b/Makefile.in @@ -785,7 +785,7 @@ parse.c: $(TOP)/src/parse.y lemon$(BEXE) $(TOP)/addopcodes.awk $(NAWK) -f $(TOP)/addopcodes.awk parse.h.temp >parse.h sqlite3.h: $(TOP)/src/sqlite.h.in $(TOP)/manifest.uuid $(TOP)/VERSION - tclsh $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h + $(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h keywordhash.h: $(TOP)/tool/mkkeywordhash.c $(BCC) -o mkkeywordhash$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c diff --git a/manifest b/manifest index 7053b7dba5..3ca5f67419 100644 --- a/manifest +++ b/manifest @@ -1,7 +1,7 @@ -C Compile\swith\sthe\sSQLITE_ENABLE_8_3_NAME\smacro\sset\sto\s2\sto\sforce\s8+3\sfilenames\nto\sbe\son\sall\sthe\stime. -D 2011-07-21T21:29:35.633 +C Use\s$(TCLSH_CMD)\sinstead\sof\stclsh\sin\sthe\sautoconf\smakefile. +D 2011-07-22T10:33:04.219 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e +F Makefile.in b85ed0fdfb95fb6822bc1c8c6bd5873fc53a84c2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc 9c76731c20561a33ce1257c70be3f029cedb62a6 F Makefile.vxworks c85ec1d8597fe2f7bc225af12ac1666e21379151 @@ -952,7 +952,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P 83191ad6f31536b0c1929938e1fbeb4cf6121ab0 -R f36be610ec3236a1941508a82c6dca72 +P ae83dac70173fecf203bf7e9f7bcb189d3e48f08 +R b476c21b2eaeefe9f084a953d275e8e3 U drh -Z be7120c77d6b184261113de7aebad3e1 +Z ff014274f8b77ee9173b86c8fcceebd1 diff --git a/manifest.uuid b/manifest.uuid index 35b1e172d6..f7999cc05f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ae83dac70173fecf203bf7e9f7bcb189d3e48f08 \ No newline at end of file +fd36d8067b5a29d9acbf8c96b9d837406d0c0cce \ No newline at end of file From fe3499654962bf51410db26812d412bda7560dcb Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 22 Jul 2011 10:53:05 +0000 Subject: [PATCH 50/52] Adjust the maximum number of open database connections in wal3.test down for OpenBSD. FossilOrigin-Name: b38e52f67574997de9f4b4318f2eb68ff0133f35 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/wal3.test | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 3ca5f67419..8d739ad206 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\s$(TCLSH_CMD)\sinstead\sof\stclsh\sin\sthe\sautoconf\smakefile. -D 2011-07-22T10:33:04.219 +C Adjust\sthe\smaximum\snumber\sof\sopen\sdatabase\sconnections\sin\swal3.test\sdown\nfor\sOpenBSD. +D 2011-07-22T10:53:05.152 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b85ed0fdfb95fb6822bc1c8c6bd5873fc53a84c2 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -879,7 +879,7 @@ F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d F test/wal.test 5617ad308bfdb8a8885220d8a261a6096a8d7e57 F test/wal2.test aa0fb2314b3235be4503c06873e41ebfc0757782 -F test/wal3.test 5c396cc22497244d627306f4c1d360167353f8dd +F test/wal3.test d512a5c8b4aa345722d11e8f1671db7eb15a0e39 F test/wal4.test 3404b048fa5e10605facaf70384e6d2943412e30 F test/wal5.test f06a0427e06db00347e32eb9fa99d6a5c0f2d088 F test/wal6.test 2e3bc767d9c2ce35c47106148d43fcbd072a93b3 @@ -952,7 +952,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P ae83dac70173fecf203bf7e9f7bcb189d3e48f08 -R b476c21b2eaeefe9f084a953d275e8e3 +P fd36d8067b5a29d9acbf8c96b9d837406d0c0cce +R d12689e1e57ff777372910ecacb2cd46 U drh -Z ff014274f8b77ee9173b86c8fcceebd1 +Z af1cbe1963416f046ac52ddc2f1578ee diff --git a/manifest.uuid b/manifest.uuid index f7999cc05f..eb651d0c9b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fd36d8067b5a29d9acbf8c96b9d837406d0c0cce \ No newline at end of file +b38e52f67574997de9f4b4318f2eb68ff0133f35 \ No newline at end of file diff --git a/test/wal3.test b/test/wal3.test index bd296154e0..f7b55eb9bc 100644 --- a/test/wal3.test +++ b/test/wal3.test @@ -707,7 +707,7 @@ T delete # and continues. # set nConn 50 -if { [string match *BSD $tcl_platform(os)] } { set nConn 35 } +if { [string match *BSD $tcl_platform(os)] } { set nConn 25 } do_test wal3-9.0 { file delete -force test.db test.db-journal test.db wal sqlite3 db test.db @@ -784,4 +784,3 @@ do_multiclient_test tn { } finish_test - From 07516dd5dfca8367bcdf5fed3fa293ff0afc3fa6 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 22 Jul 2011 11:16:39 +0000 Subject: [PATCH 51/52] Add makefile targets sqlite3-all.c. FossilOrigin-Name: 71f7be586f5a18493cd516f89c4ecc460bb24462 --- Makefile.in | 3 +++ main.mk | 3 +++ manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/Makefile.in b/Makefile.in index 2b3311ac99..a9fc93af7e 100644 --- a/Makefile.in +++ b/Makefile.in @@ -517,6 +517,9 @@ sqlite3$(TEXE): $(TOP)/src/shell.c libsqlite3.la sqlite3.h sqlite3.c: .target_source $(TOP)/tool/mksqlite3c.tcl $(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl +sqlite3-all.c: sqlite3.c $(TOP)/tool/split-sqlite3c.tcl + $(TCLSH_CMD) $(TOP)/tool/split-sqlite3c.tcl + # Rule to build the amalgamation # sqlite3.lo: sqlite3.c diff --git a/main.mk b/main.mk index 16bb7115c9..1ab502af6e 100644 --- a/main.mk +++ b/main.mk @@ -383,6 +383,9 @@ sqlite3.c: target_source $(TOP)/tool/mksqlite3c.tcl echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c +sqlite3-all.c: sqlite3.c $(TOP)/tool/split-sqlite3c.tcl + tclsh $(TOP)/tool/split-sqlite3c.tcl + fts2amal.c: target_source $(TOP)/ext/fts2/mkfts2amal.tcl tclsh $(TOP)/ext/fts2/mkfts2amal.tcl diff --git a/manifest b/manifest index 8d739ad206..791d74f8eb 100644 --- a/manifest +++ b/manifest @@ -1,7 +1,7 @@ -C Adjust\sthe\smaximum\snumber\sof\sopen\sdatabase\sconnections\sin\swal3.test\sdown\nfor\sOpenBSD. -D 2011-07-22T10:53:05.152 +C Add\smakefile\stargets\ssqlite3-all.c. +D 2011-07-22T11:16:39.376 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in b85ed0fdfb95fb6822bc1c8c6bd5873fc53a84c2 +F Makefile.in 1e6988b3c11dee9bd5edc0c804bd4468d74a9cdc F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc 9c76731c20561a33ce1257c70be3f029cedb62a6 F Makefile.vxworks c85ec1d8597fe2f7bc225af12ac1666e21379151 @@ -104,7 +104,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk d81d86f0f70444f3abc241eccf5ace4a79ff9b69 +F main.mk 50adb7368bd0faa888eb4634b874d542d44f3de3 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -952,7 +952,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P fd36d8067b5a29d9acbf8c96b9d837406d0c0cce -R d12689e1e57ff777372910ecacb2cd46 +P b38e52f67574997de9f4b4318f2eb68ff0133f35 +R ada945a14522405a76d61d09ae93a2d9 U drh -Z af1cbe1963416f046ac52ddc2f1578ee +Z 925c4df3e98ac05a9968b9d94b638861 diff --git a/manifest.uuid b/manifest.uuid index eb651d0c9b..b207b87304 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b38e52f67574997de9f4b4318f2eb68ff0133f35 \ No newline at end of file +71f7be586f5a18493cd516f89c4ecc460bb24462 \ No newline at end of file From 75e7bc1db7027404e65841b2da934324a3dc5dbd Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 22 Jul 2011 11:23:49 +0000 Subject: [PATCH 52/52] Also add the sqlite3-all.c target to the MSVC makefile. FossilOrigin-Name: 8ce2b74a82264550b0e19da3e0e1a145db940a1c --- Makefile.msc | 3 +++ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 718464d6ec..799970d5f9 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -486,6 +486,9 @@ sqlite3.exe: $(TOP)\src\shell.c libsqlite3.lib sqlite3.h sqlite3.c: .target_source $(TOP)\tool\mksqlite3c.tcl $(TCLSH_CMD) $(TOP)\tool\mksqlite3c.tcl +sqlite3-all.c: sqlite3.c $(TOP)/tool/split-sqlite3c.tcl + $(TCLSH_CMD) $(TOP)/tool/split-sqlite3c.tcl + # Rule to build the amalgamation # sqlite3.lo: sqlite3.c diff --git a/manifest b/manifest index 791d74f8eb..42fc3e12f1 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Add\smakefile\stargets\ssqlite3-all.c. -D 2011-07-22T11:16:39.376 +C Also\sadd\sthe\ssqlite3-all.c\starget\sto\sthe\sMSVC\smakefile. +D 2011-07-22T11:23:49.199 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 1e6988b3c11dee9bd5edc0c804bd4468d74a9cdc F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 9c76731c20561a33ce1257c70be3f029cedb62a6 +F Makefile.msc 8e04f517922b9e31a5bf975f07bb66df79ae7076 F Makefile.vxworks c85ec1d8597fe2f7bc225af12ac1666e21379151 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION f724de7326e87b7f3b0a55f16ef4b4d993680d54 @@ -952,7 +952,7 @@ F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 2ebae31e1eb352696f3c2f7706a34c084b28c262 -P b38e52f67574997de9f4b4318f2eb68ff0133f35 -R ada945a14522405a76d61d09ae93a2d9 +P 71f7be586f5a18493cd516f89c4ecc460bb24462 +R 645e7d58a7503109e1bb3e986974849f U drh -Z 925c4df3e98ac05a9968b9d94b638861 +Z aaeed83d2a39a45645a69e9a638df017 diff --git a/manifest.uuid b/manifest.uuid index b207b87304..e12fc96311 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -71f7be586f5a18493cd516f89c4ecc460bb24462 \ No newline at end of file +8ce2b74a82264550b0e19da3e0e1a145db940a1c \ No newline at end of file