From 09643ab745aee64145ddacf3579f9c5eaf6a4452 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 24 May 2011 18:49:45 +0000 Subject: [PATCH 01/20] If the fts4 option prefix=1 is specified, have the fts4 module maintain an index of prefixes as well as terms. FossilOrigin-Name: b5bdc639898ee22eebedeb560810e94e74de8aa4 --- ext/fts3/fts3.c | 105 +++++++++++++------- ext/fts3/fts3Int.h | 30 +++++- ext/fts3/fts3_aux.c | 2 +- ext/fts3/fts3_term.c | 15 ++- ext/fts3/fts3_write.c | 222 ++++++++++++++++++++++++++++++++---------- manifest | 26 ++--- manifest.uuid | 2 +- test/fts3prefix.test | 175 +++++++++++++++++++++++++++++++++ 8 files changed, 469 insertions(+), 108 deletions(-) create mode 100644 test/fts3prefix.test diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index fecf092148..a127773449 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -865,6 +865,7 @@ static int fts3InitVtab( int nName; /* Bytes required to hold table name */ int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ int bNoDocsize = 0; /* True to omit %_docsize table */ + int bPrefix = 0; /* True to include a prefix-search index */ const char **aCol; /* Array of column names */ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ @@ -927,6 +928,8 @@ static int fts3InitVtab( }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){ zUncompress = zVal; zVal = 0; + }else if( nKey==6 && 0==sqlite3_strnicmp(z, "prefix", 6) ){ + bPrefix = 1; }else{ *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); rc = SQLITE_ERROR; @@ -979,7 +982,9 @@ static int fts3InitVtab( p->bHasStat = isFts4; TESTONLY( p->inTransaction = -1 ); TESTONLY( p->mxSavepoint = -1 ); + p->bPrefix = bPrefix; fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1); + fts3HashInit(&p->pendingPrefixes, FTS3_HASH_STRING, 1); /* Fill in the zName and zDb fields of the vtab structure. */ zCsr = (char *)&p->azColumn[nCol]; @@ -2139,6 +2144,28 @@ static int fts3DeferredTermSelect( return SQLITE_OK; } +static int fts3SegReaderCursorAppend( + Fts3SegReaderCursor *pCsr, + Fts3SegReader *pNew +){ + if( (pCsr->nSegment%16)==0 ){ + Fts3SegReader **apNew; + int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); + apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); + if( !apNew ){ + sqlite3Fts3SegReaderFree(pNew); + return SQLITE_NOMEM; + } + pCsr->apSegment = apNew; + } + pCsr->apSegment[pCsr->nSegment++] = pNew; + return SQLITE_OK; +} + +/* +** Set up a cursor object for iterating through the full-text index or +** a single level therein. +*/ int sqlite3Fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ int iLevel, /* Level of segments to scan */ @@ -2152,42 +2179,52 @@ int sqlite3Fts3SegReaderCursor( int rc2; int iAge = 0; sqlite3_stmt *pStmt = 0; - Fts3SegReader *pPending = 0; - assert( iLevel==FTS3_SEGCURSOR_ALL + assert( iLevel==FTS3_SEGCURSOR_ALL_TERM || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel==FTS3_SEGCURSOR_PENDING_PREFIX + || iLevel==FTS3_SEGCURSOR_ALL_PREFIX || iLevel>=0 ); - assert( FTS3_SEGCURSOR_PENDING<0 ); - assert( FTS3_SEGCURSOR_ALL<0 ); - assert( iLevel==FTS3_SEGCURSOR_ALL || (zTerm==0 && isPrefix==1) ); + assert( 0>FTS3_SEGCURSOR_ALL_TERM + && 0>FTS3_SEGCURSOR_PENDING + && 0>FTS3_SEGCURSOR_PENDING_PREFIX + && 0>FTS3_SEGCURSOR_ALL_PREFIX + ); + assert( iLevel==FTS3_SEGCURSOR_ALL_TERM + || iLevel==FTS3_SEGCURSOR_ALL_PREFIX + || (zTerm==0 && isPrefix==1) + ); assert( isPrefix==0 || isScan==0 ); - memset(pCsr, 0, sizeof(Fts3SegReaderCursor)); - /* If iLevel is less than 0, include a seg-reader for the pending-terms. */ + /* "isScan" is only set to true by the ft4aux module, not an ordinary + ** full-text table. The pendingTerms and pendingPrefixes tables must be + ** empty in this case. */ assert( isScan==0 || fts3HashCount(&p->pendingTerms)==0 ); + assert( isScan==0 || fts3HashCount(&p->pendingPrefixes)==0 ); + + /* If iLevel is less than 0, include a seg-reader for the pending-terms. */ if( iLevel<0 && isScan==0 ){ - rc = sqlite3Fts3SegReaderPending(p, zTerm, nTerm, isPrefix, &pPending); + int bPrefix = ( + iLevel==FTS3_SEGCURSOR_PENDING_PREFIX + || iLevel==FTS3_SEGCURSOR_ALL_PREFIX + ); + Fts3SegReader *pPending = 0; + + rc = sqlite3Fts3SegReaderPending(p,zTerm,nTerm,isPrefix,bPrefix,&pPending); if( rc==SQLITE_OK && pPending ){ - int nByte = (sizeof(Fts3SegReader *) * 16); - pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); - if( pCsr->apSegment==0 ){ - rc = SQLITE_NOMEM; - }else{ - pCsr->apSegment[0] = pPending; - pCsr->nSegment = 1; - pPending = 0; - } + rc = fts3SegReaderCursorAppend(pCsr, pPending); } } - if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + if( iLevel!=FTS3_SEGCURSOR_PENDING && iLevel!=FTS3_SEGCURSOR_PENDING_PREFIX ){ if( rc==SQLITE_OK ){ rc = sqlite3Fts3AllSegdirs(p, iLevel, &pStmt); } while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + Fts3SegReader *pSeg = 0; /* Read the values returned by the SELECT into local variables. */ sqlite3_int64 iStartBlock = sqlite3_column_int64(pStmt, 1); @@ -2196,18 +2233,6 @@ int sqlite3Fts3SegReaderCursor( int nRoot = sqlite3_column_bytes(pStmt, 4); char const *zRoot = sqlite3_column_blob(pStmt, 4); - /* If nSegment is a multiple of 16 the array needs to be extended. */ - if( (pCsr->nSegment%16)==0 ){ - Fts3SegReader **apNew; - int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); - apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); - if( !apNew ){ - rc = SQLITE_NOMEM; - goto finished; - } - pCsr->apSegment = apNew; - } - /* If zTerm is not NULL, and this segment is not stored entirely on its ** root node, the range of leaves scanned can be reduced. Do this. */ if( iStartBlock && zTerm ){ @@ -2218,10 +2243,10 @@ int sqlite3Fts3SegReaderCursor( } rc = sqlite3Fts3SegReaderNew(iAge, iStartBlock, iLeavesEndBlock, - iEndBlock, zRoot, nRoot, &pCsr->apSegment[pCsr->nSegment] + iEndBlock, zRoot, nRoot, &pSeg ); if( rc!=SQLITE_OK ) goto finished; - pCsr->nSegment++; + rc = fts3SegReaderCursorAppend(pCsr, pSeg); iAge++; } } @@ -2229,7 +2254,6 @@ int sqlite3Fts3SegReaderCursor( finished: rc2 = sqlite3_reset(pStmt); if( rc==SQLITE_DONE ) rc = rc2; - sqlite3Fts3SegReaderFree(pPending); return rc; } @@ -2247,11 +2271,18 @@ static int fts3TermSegReaderCursor( pSegcsr = sqlite3_malloc(sizeof(Fts3SegReaderCursor)); if( pSegcsr ){ - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; int i; int nCost = 0; - rc = sqlite3Fts3SegReaderCursor( - p, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr); + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + + if( isPrefix && p->bPrefix && nTerm<=FTS3_MAX_PREFIX ){ + rc = sqlite3Fts3SegReaderCursor( + p, FTS3_SEGCURSOR_ALL_PREFIX, zTerm, nTerm, 0, 0, pSegcsr); + + }else{ + rc = sqlite3Fts3SegReaderCursor( + p, FTS3_SEGCURSOR_ALL_TERM, zTerm, nTerm, isPrefix, 0, pSegcsr); + } for(i=0; rc==SQLITE_OK && inSegment; i++){ rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost); @@ -3309,6 +3340,7 @@ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ assert( p->inTransaction!=0 ); TESTONLY( p->inTransaction = 0 ); TESTONLY( p->mxSavepoint = -1; ); + sqlite3Fts3PendingPrefixesClear((Fts3Table *)pVtab); return SQLITE_OK; } @@ -3683,6 +3715,7 @@ static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ assert( p->mxSavepoint >= iSavepoint ); TESTONLY( p->mxSavepoint = iSavepoint ); sqlite3Fts3PendingTermsClear(p); + sqlite3Fts3PendingPrefixesClear((Fts3Table *)pVtab); return SQLITE_OK; } diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index bb21165ffc..785a6aa655 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -23,6 +23,8 @@ #include "fts3_tokenizer.h" #include "fts3_hash.h" +#define FTS3_MAX_PREFIX 8 + /* ** This constant controls how often segments are merged. Once there are ** FTS3_MERGE_COUNT segments of level N, they are merged into a single @@ -53,6 +55,20 @@ */ #define FTS3_VARINT_MAX 10 +/* +** FTS4 virtual tables may maintain two separate indexes. One that indexes +** all document terms (the same index FTS3 tables maintain) and another used +** for prefixes. B+-trees that are part of the prefix index have values for +** the %_segdir.level column that are equal to or greater than the following +** value. +** +** It is considered impossible for the regular index to use levels this large. +** In theory it could, but that would require that at least 2^1024 separate +** write operations to be made within the lifetime of the database. +*/ +#define FTS3_SEGDIR_PREFIXLEVEL 1024 +#define FTS3_SEGDIR_PREFIXLEVEL_STR "1024" + /* ** The testcase() macro is only used by the amalgamation. If undefined, ** make it a no-op. @@ -148,7 +164,7 @@ struct Fts3Table { /* Precompiled statements used by the implementation. Each of these ** statements is run and reset within a single virtual table API call. */ - sqlite3_stmt *aStmt[24]; + sqlite3_stmt *aStmt[27]; char *zReadExprlist; char *zWriteExprlist; @@ -156,6 +172,7 @@ struct Fts3Table { int nNodeSize; /* Soft limit for node size */ u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ + u8 bPrefix; /* True if there is a prefix index */ int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ @@ -171,6 +188,7 @@ struct Fts3Table { int nPendingData; sqlite_int64 iPrevDocid; Fts3Hash pendingTerms; + Fts3Hash pendingPrefixes; #if defined(SQLITE_DEBUG) /* State variables used for validating that the transaction control @@ -317,7 +335,8 @@ void sqlite3Fts3PendingTermsClear(Fts3Table *); int sqlite3Fts3Optimize(Fts3Table *); int sqlite3Fts3SegReaderNew(int, sqlite3_int64, sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); -int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**); +int sqlite3Fts3SegReaderPending( + Fts3Table*,const char*,int,int,int,Fts3SegReader**); void sqlite3Fts3SegReaderFree(Fts3SegReader *); int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); int sqlite3Fts3AllSegdirs(Fts3Table*, int, sqlite3_stmt **); @@ -334,8 +353,11 @@ void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *); void sqlite3Fts3SegmentsClose(Fts3Table *); -#define FTS3_SEGCURSOR_PENDING -1 -#define FTS3_SEGCURSOR_ALL -2 +/* Special values interpreted by sqlite3SegReaderCursor() */ +#define FTS3_SEGCURSOR_PENDING -1 +#define FTS3_SEGCURSOR_PENDING_PREFIX -2 +#define FTS3_SEGCURSOR_ALL_PREFIX -3 +#define FTS3_SEGCURSOR_ALL_TERM -4 int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*); int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *); diff --git a/ext/fts3/fts3_aux.c b/ext/fts3/fts3_aux.c index 6108689ae1..cb91cda347 100644 --- a/ext/fts3/fts3_aux.c +++ b/ext/fts3/fts3_aux.c @@ -375,7 +375,7 @@ static int fts3auxFilterMethod( if( pCsr->zStop==0 ) return SQLITE_NOMEM; } - rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL, + rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL_TERM, pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr ); if( rc==SQLITE_OK ){ diff --git a/ext/fts3/fts3_term.c b/ext/fts3/fts3_term.c index e207ff870d..ca46269689 100644 --- a/ext/fts3/fts3_term.c +++ b/ext/fts3/fts3_term.c @@ -27,6 +27,7 @@ typedef struct Fts3termCursor Fts3termCursor; struct Fts3termTable { sqlite3_vtab base; /* Base class used by SQLite core */ + int bPrefix; /* True for an fts4prefix table */ Fts3Table *pFts3Tab; }; @@ -56,7 +57,7 @@ struct Fts3termCursor { */ static int fts3termConnectMethod( sqlite3 *db, /* Database connection */ - void *pUnused, /* Unused */ + void *pCtx, /* Non-zero for an fts4prefix table */ int argc, /* Number of elements in argv array */ const char * const *argv, /* xCreate/xConnect argument array */ sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */ @@ -70,8 +71,6 @@ static int fts3termConnectMethod( int rc; /* value returned by declare_vtab() */ Fts3termTable *p; /* Virtual table object to return */ - UNUSED_PARAMETER(pUnused); - /* The user should specify a single argument - the name of an fts3 table. */ if( argc!=4 ){ *pzErr = sqlite3_mprintf( @@ -97,6 +96,7 @@ static int fts3termConnectMethod( p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; p->pFts3Tab->db = db; + p->bPrefix = (int)pCtx; memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); @@ -244,7 +244,8 @@ static int fts3termFilterMethod( sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ Fts3termCursor *pCsr = (Fts3termCursor *)pCursor; - Fts3Table *pFts3 = ((Fts3termTable *)pCursor->pVtab)->pFts3Tab; + Fts3termTable *p = (Fts3termTable *)pCursor->pVtab; + Fts3Table *pFts3 = p->pFts3Tab; int rc; UNUSED_PARAMETER(nVal); @@ -262,7 +263,8 @@ static int fts3termFilterMethod( pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; pCsr->filter.flags |= FTS3_SEGMENT_SCAN; - rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL, + rc = sqlite3Fts3SegReaderCursor(pFts3, + p->bPrefix ? FTS3_SEGCURSOR_ALL_PREFIX : FTS3_SEGCURSOR_ALL_TERM, pCsr->filter.zTerm, pCsr->filter.nTerm, 0, 1, &pCsr->csr ); if( rc==SQLITE_OK ){ @@ -353,6 +355,9 @@ int sqlite3Fts3InitTerm(sqlite3 *db){ int rc; /* Return code */ rc = sqlite3_create_module(db, "fts4term", &fts3term_module, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "fts4prefix", &fts3term_module, (void*)1); + } return rc; } diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index d00632a75d..5ff7e0b1aa 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -145,6 +145,14 @@ struct SegmentWriter { ** fts3NodeAddTerm() ** fts3NodeWrite() ** fts3NodeFree() +** +** When a b+tree is written to the database (either as a result of a merge +** or the pending-terms table being flushed), leaves are written into the +** database file as soon as they are completely populated. The interior of +** the tree is assembled in memory and written out only once all leaves have +** been populated and stored. This is Ok, as the b+-tree fanout is usually +** very large, meaning that the interior of the tree consumes relatively +** little memory. */ struct SegmentNode { SegmentNode *pParent; /* Parent node (or NULL for root node) */ @@ -177,7 +185,7 @@ struct SegmentNode { #define SQL_SELECT_LEVEL 12 #define SQL_SELECT_ALL_LEVEL 13 #define SQL_SELECT_LEVEL_COUNT 14 -#define SQL_SELECT_SEGDIR_COUNT_MAX 15 +#define SQL_SELECT_SEGDIR_MAX_LEVEL 15 #define SQL_DELETE_SEGDIR_BY_LEVEL 16 #define SQL_DELETE_SEGMENTS_RANGE 17 #define SQL_CONTENT_INSERT 18 @@ -186,6 +194,10 @@ struct SegmentNode { #define SQL_SELECT_DOCSIZE 21 #define SQL_SELECT_DOCTOTAL 22 #define SQL_REPLACE_DOCTOTAL 23 +#define SQL_SELECT_ALL_PREFIX_LEVEL 24 + +#define SQL_DELETE_ALL_TERMS_SEGDIR 25 +#define SQL_DELETE_ALL_PREFIX_SEGDIR 26 /* ** This function is used to obtain an SQLite prepared statement handle @@ -222,10 +234,12 @@ static int fts3SqlStmt( /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", /* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " - "FROM %Q.'%q_segdir' ORDER BY level DESC, idx ASC", + "FROM %Q.'%q_segdir' WHERE level < " FTS3_SEGDIR_PREFIXLEVEL_STR + " ORDER BY level DESC, idx ASC", /* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", -/* 15 */ "SELECT count(*), max(level) FROM %Q.'%q_segdir'", +/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level < (?+1)*" + FTS3_SEGDIR_PREFIXLEVEL_STR, /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", @@ -235,6 +249,11 @@ static int fts3SqlStmt( /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", /* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", /* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", +/* 24 */ "SELECT idx, start_block, leaves_end_block, end_block, root " + "FROM %Q.'%q_segdir' WHERE level >= " FTS3_SEGDIR_PREFIXLEVEL_STR + " ORDER BY level DESC, idx ASC", +/* 25 */ "DELETE FROM %Q.'%q_segdir' WHERE level<" FTS3_SEGDIR_PREFIXLEVEL_STR, +/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level>=" FTS3_SEGDIR_PREFIXLEVEL_STR, }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -393,9 +412,12 @@ int sqlite3Fts3ReadLock(Fts3Table *p){ int sqlite3Fts3AllSegdirs(Fts3Table *p, int iLevel, sqlite3_stmt **ppStmt){ int rc; sqlite3_stmt *pStmt = 0; - if( iLevel<0 ){ + if( iLevel==FTS3_SEGCURSOR_ALL_PREFIX ){ + rc = fts3SqlStmt(p, SQL_SELECT_ALL_PREFIX_LEVEL, &pStmt, 0); + }else if( iLevel==FTS3_SEGCURSOR_ALL_TERM ){ rc = fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, &pStmt, 0); }else{ + assert( iLevel>=0 ); rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); if( rc==SQLITE_OK ) sqlite3_bind_int(pStmt, 1, iLevel); } @@ -512,6 +534,40 @@ static int fts3PendingListAppend( return 0; } +static int fts3PendingTermsAddOne( + Fts3Table *p, + int iCol, + int iPos, + Fts3Hash *pHash, + const char *zToken, + int nToken +){ + PendingList *pList; + int rc = SQLITE_OK; + + pList = (PendingList *)fts3HashFind(pHash, zToken, nToken); + if( pList ){ + p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); + } + if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ + if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ + /* Malloc failed while inserting the new entry. This can only + ** happen if there was no previous entry for this token. + */ + assert( 0==fts3HashFind(pHash, zToken, nToken) ); + sqlite3_free(pList); + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); + } + return rc; +} + + + + /* ** Tokenize the nul-terminated string zText and add all tokens to the ** pending-terms hash-table. The docid used is that currently stored in @@ -560,8 +616,6 @@ static int fts3PendingTermsAdd( while( SQLITE_OK==rc && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) ){ - PendingList *pList; - if( iPos>=nWord ) nWord = iPos+1; /* Positions cannot be negative; we use -1 as a terminator internally. @@ -572,23 +626,13 @@ static int fts3PendingTermsAdd( break; } - pList = (PendingList *)fts3HashFind(&p->pendingTerms, zToken, nToken); - if( pList ){ - p->nPendingData -= (pList->nData + nToken + sizeof(Fts3HashElem)); - } - if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ - if( pList==fts3HashInsert(&p->pendingTerms, zToken, nToken, pList) ){ - /* Malloc failed while inserting the new entry. This can only - ** happen if there was no previous entry for this token. - */ - assert( 0==fts3HashFind(&p->pendingTerms, zToken, nToken) ); - sqlite3_free(pList); - rc = SQLITE_NOMEM; + rc = fts3PendingTermsAddOne(p,iCol,iPos,&p->pendingTerms,zToken,nToken); + if( p->bPrefix ){ + int n = (nToken > FTS3_MAX_PREFIX ? FTS3_MAX_PREFIX : nToken); + for(; n>0 && rc==SQLITE_OK; n--){ + rc = fts3PendingTermsAddOne(p,iCol,iPos,&p->pendingPrefixes,zToken,n); } } - if( rc==SQLITE_OK ){ - p->nPendingData += (pList->nData + nToken + sizeof(Fts3HashElem)); - } } pModule->xClose(pCsr); @@ -628,6 +672,17 @@ void sqlite3Fts3PendingTermsClear(Fts3Table *p){ p->nPendingData = 0; } +/* +** Discard the contents of the pending-prefixes hash table. +*/ +void sqlite3Fts3PendingPrefixesClear(Fts3Table *p){ + Fts3HashElem *pElem; + for(pElem=fts3HashFirst(&p->pendingPrefixes); pElem; pElem=fts3HashNext(pElem)){ + sqlite3_free(fts3HashData(pElem)); + } + fts3HashClear(&p->pendingPrefixes); +} + /* ** This function is called by the xUpdate() method as part of an INSERT ** operation. It adds entries for each term in the new record to the @@ -725,6 +780,7 @@ static int fts3DeleteAll(Fts3Table *p){ /* Discard the contents of the pending-terms hash table. */ sqlite3Fts3PendingTermsClear(p); + sqlite3Fts3PendingPrefixesClear(p); /* Delete everything from the %_content, %_segments and %_segdir tables. */ fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); @@ -1235,12 +1291,28 @@ static int fts3CompareElemByTerm(const void *lhs, const void *rhs){ /* ** This function is used to allocate an Fts3SegReader that iterates through ** a subset of the terms stored in the Fts3Table.pendingTerms array. +** +** If the isPrefixIter parameter is zero, then the returned SegReader iterates +** through each term in the pending-terms table. Or, if isPrefixIter is +** non-zero, it iterates through each term and its prefixes. For example, if +** the pending terms hash table contains the terms "sqlite", "mysql" and +** "firebird", then the iterator visits the following 'terms' (in the order +** shown): +** +** f fi fir fire fireb firebi firebir firebird +** m my mys mysq mysql +** s sq sql sqli sqlit sqlite +** +** Whereas if isPrefixIter is zero, the terms visited are: +** +** firebird mysql sqlite */ int sqlite3Fts3SegReaderPending( Fts3Table *p, /* Virtual table handle */ const char *zTerm, /* Term to search for */ int nTerm, /* Size of buffer zTerm */ - int isPrefix, /* True for a term-prefix query */ + int isMultiTerm, /* True to visit multiple terms */ + int isPrefixIter, /* 0->pendingTerms, 1->pendingPrefixes */ Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ ){ Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ @@ -1248,11 +1320,14 @@ int sqlite3Fts3SegReaderPending( int nElem = 0; /* Size of array at aElem */ int rc = SQLITE_OK; /* Return Code */ - if( isPrefix ){ + if( isMultiTerm ){ int nAlloc = 0; /* Size of allocated array at aElem */ Fts3HashElem *pE = 0; /* Iterator variable */ + Fts3Hash *pHash; - for(pE=fts3HashFirst(&p->pendingTerms); pE; pE=fts3HashNext(pE)){ + pHash = (isPrefixIter ? &p->pendingPrefixes : &p->pendingTerms); + + for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ char *zKey = (char *)fts3HashKey(pE); int nKey = fts3HashKeysize(pE); if( nTerm==0 || (nKey>=nTerm && 0==memcmp(zKey, zTerm, nTerm)) ){ @@ -1269,6 +1344,7 @@ int sqlite3Fts3SegReaderPending( } aElem = aElem2; } + aElem[nElem++] = pE; } } @@ -1282,6 +1358,8 @@ int sqlite3Fts3SegReaderPending( } }else{ + /* The query is a simple term lookup that matches at most one term in + ** the index. All that is required is a straight hash-lookup. */ Fts3HashElem *pE = fts3HashFindElem(&p->pendingTerms, zTerm, nTerm); if( pE ){ aElem = &pE; @@ -1302,7 +1380,7 @@ int sqlite3Fts3SegReaderPending( } } - if( isPrefix ){ + if( isMultiTerm ){ sqlite3_free(aElem); } *ppReader = pReader; @@ -1914,21 +1992,30 @@ static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ } /* -** Set *pnSegment to the total number of segments in the database. Set -** *pnMax to the largest segment level in the database (segment levels -** are stored in the 'level' column of the %_segdir table). +** Set *pnMax to the largest segment level in the database for either the +** terms index (if parameter bPrefixIndex is 0) or the prefixes index (if +** parameter bPrefixIndex is 1). +** +** Segment levels are stored in the 'level' column of the %_segdir table. ** ** Return SQLITE_OK if successful, or an SQLite error code if not. */ -static int fts3SegmentCountMax(Fts3Table *p, int *pnSegment, int *pnMax){ +static int fts3SegmentMaxLevel(Fts3Table *p, int bPrefixIndex, int *pnMax){ sqlite3_stmt *pStmt; int rc; + assert( bPrefixIndex==0 || bPrefixIndex==1 ); - rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_COUNT_MAX, &pStmt, 0); + /* Set pStmt to the compiled version of: + ** + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level < (?+1) * 1024 + ** + ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). + */ + rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int(pStmt, 1, bPrefixIndex); if( SQLITE_ROW==sqlite3_step(pStmt) ){ - *pnSegment = sqlite3_column_int(pStmt, 0); - *pnMax = sqlite3_column_int(pStmt, 1); + *pnMax = sqlite3_column_int(pStmt, 0); } return sqlite3_reset(pStmt); } @@ -1971,12 +2058,21 @@ static int fts3DeleteSegdir( return rc; } - if( iLevel==FTS3_SEGCURSOR_ALL ){ - fts3SqlExec(&rc, p, SQL_DELETE_ALL_SEGDIR, 0); + assert( iLevel>=0 + || iLevel==FTS3_SEGCURSOR_ALL_TERM + || iLevel==FTS3_SEGCURSOR_ALL_PREFIX + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel==FTS3_SEGCURSOR_PENDING_PREFIX + ); + if( iLevel==FTS3_SEGCURSOR_ALL_TERM ){ + fts3SqlExec(&rc, p, SQL_DELETE_ALL_TERMS_SEGDIR, 0); + }else if( iLevel==FTS3_SEGCURSOR_ALL_PREFIX ){ + fts3SqlExec(&rc, p, SQL_DELETE_ALL_PREFIX_SEGDIR, 0); + }else if( iLevel==FTS3_SEGCURSOR_PENDING_PREFIX ){ + sqlite3Fts3PendingPrefixesClear(p); }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ sqlite3Fts3PendingTermsClear(p); - }else{ - assert( iLevel>=0 ); + }else if( iLevel>=0 ){ rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int(pDelete, 1, iLevel); @@ -2234,27 +2330,34 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ Fts3SegReaderCursor csr; /* Cursor to iterate through level(s) */ + int bIgnoreEmpty = 0; /* True to ignore empty segments */ rc = sqlite3Fts3SegReaderCursor(p, iLevel, 0, 0, 1, 0, &csr); if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; - if( iLevel==FTS3_SEGCURSOR_ALL ){ + if( iLevel==FTS3_SEGCURSOR_ALL_TERM || iLevel==FTS3_SEGCURSOR_ALL_PREFIX ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the the numerically ** greatest segment level currently present in the database. The index ** of the new segment is always 0. */ - int nDummy; /* TODO: Remove this */ if( csr.nSegment==1 ){ rc = SQLITE_DONE; goto finished; } - rc = fts3SegmentCountMax(p, &nDummy, &iNewLevel); + rc = fts3SegmentMaxLevel(p, iLevel==FTS3_SEGCURSOR_ALL_PREFIX, &iNewLevel); + bIgnoreEmpty = 1; }else{ - /* This call is to merge all segments at level iLevel. Find the next + /* This call is to merge all segments at level iLevel. find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ - iNewLevel = iLevel+1; + if( iLevel==FTS3_SEGCURSOR_PENDING ){ + iNewLevel = 0; + }else if( iLevel==FTS3_SEGCURSOR_PENDING_PREFIX ){ + iNewLevel = FTS3_SEGDIR_PREFIXLEVEL; + }else{ + iNewLevel = iLevel+1; + } rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx); } if( rc!=SQLITE_OK ) goto finished; @@ -2263,7 +2366,7 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ memset(&filter, 0, sizeof(Fts3SegFilter)); filter.flags = FTS3_SEGMENT_REQUIRE_POS; - filter.flags |= (iLevel==FTS3_SEGCURSOR_ALL ? FTS3_SEGMENT_IGNORE_EMPTY : 0); + filter.flags |= (bIgnoreEmpty ? FTS3_SEGMENT_IGNORE_EMPTY : 0); rc = sqlite3Fts3SegReaderStart(p, &csr, &filter); while( SQLITE_OK==rc ){ @@ -2290,7 +2393,17 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ ** Flush the contents of pendingTerms to a level 0 segment. */ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ - return fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING); + int rc = SQLITE_OK; + if( p->bPrefix ){ + rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING_PREFIX); + } + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ + rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING); + } + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + } + return rc; } /* @@ -2457,11 +2570,9 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ if( !zVal ){ return SQLITE_NOMEM; }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL); - if( rc==SQLITE_DONE ){ - rc = SQLITE_OK; - }else{ - sqlite3Fts3PendingTermsClear(p); + rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_PREFIX); + if( rc==SQLITE_OK ){ + rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_TERM); } #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ @@ -2476,6 +2587,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ } sqlite3Fts3SegmentsClose(p); + sqlite3Fts3PendingTermsClear(p); + sqlite3Fts3PendingPrefixesClear(p); return rc; } @@ -2792,13 +2905,22 @@ int sqlite3Fts3UpdateMethod( */ int sqlite3Fts3Optimize(Fts3Table *p){ int rc; + int bReturnDone = 0; rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); if( rc==SQLITE_OK ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL); + rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_PREFIX); + if( rc==SQLITE_OK ){ + rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_TERM); + } + if( rc==SQLITE_DONE ){ + bReturnDone = 1; + rc = SQLITE_OK; + } if( rc==SQLITE_OK ){ rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); if( rc==SQLITE_OK ){ sqlite3Fts3PendingTermsClear(p); + sqlite3Fts3PendingPrefixesClear(p); } }else{ sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); @@ -2806,7 +2928,7 @@ int sqlite3Fts3Optimize(Fts3Table *p){ } } sqlite3Fts3SegmentsClose(p); - return rc; + return ((rc==SQLITE_OK && bReturnDone) ? SQLITE_DONE : rc); } #endif diff --git a/manifest b/manifest index 51c28b73df..bf93a84328 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sinvoke\sthe\sxRollbackTo\sor\sxRelease\smethods\sof\sa\svirtual\stable\swithout\nhaving\sfirst\sinvoked\san\sappropriate\sxSavepoint\smethod.\s\sAdd\sassert()\sstatements\nto\sFTS3/4\sto\sverify\sthat\sthis\sis\shappening\sin\sall\scases. -D 2011-05-24T15:36:01.532 +C If\sthe\sfts4\soption\sprefix=1\sis\sspecified,\shave\sthe\sfts4\smodule\smaintain\san\sindex\sof\sprefixes\sas\swell\sas\sterms. +D 2011-05-24T18:49:45.786 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,21 +61,21 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c e7600e285b58027657ebb46c6132b7d5a100accb +F ext/fts3/fts3.c 4a48bfef342badba0a71bdeb5354edaa3ad83382 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h c8c0011c5e5b3a7703376ea6cd7deb91cfb96a06 -F ext/fts3/fts3_aux.c 9e931f55eed8498dafe7bc1160f10cbb1a652fdf +F ext/fts3/fts3Int.h 02699211c0b6cf5aa713cc68c527c9a6e9159fbe +F ext/fts3/fts3_aux.c d68d8e4d39e0342302d2c834618755af7c8058ea F ext/fts3/fts3_expr.c 5f49e0deaf723724b08100bb3ff40aab02ad0c93 F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2 F ext/fts3/fts3_snippet.c 92b40397b28422c35c4127492d7ac6da34d1966a -F ext/fts3/fts3_term.c f115f5a5f4298303d3b22fc6c524b8d565c7b950 +F ext/fts3/fts3_term.c cd226a311940b8ef414d5c1f7c74971a47cacedb F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c b50181e5ecf484c2f56e98d651424e4b69f96c89 +F ext/fts3/fts3_write.c 0fd6a55c774731852f889007fc6edb1b99819ee5 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -473,6 +473,7 @@ F test/fts3fault2.test dc96203af6ba31ce20163fc35460e1556e8edf4d F test/fts3malloc.test 9c8cc3f885bb4dfc66d0460c52f68f45e4710d1b F test/fts3matchinfo.test cc0b009edbbf575283d5fdb53271179e0d8019ba F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 +F test/fts3prefix.test 5b4e08c63d5d4a79e54754dc6b2209b03c885200 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 F test/fts3rnd.test 2b1a579be557ab8ac54a51b39caa4aa8043cc4ad F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 @@ -938,7 +939,10 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P a9d095660ca0e99b226e0fe669c11a0be6c49710 -R 19875cc50e6351fd8eaf432c5942eda7 -U drh -Z f15ff60c160c7edadf458f2c05939bd4 +P 651ef24249d8c22c4f13e4c0bb98a60099cfd23a +R c499b030bb5d74c4afca730dec7a05bc +T *branch * fts3-prefix-search +T *sym-fts3-prefix-search * +T -sym-trunk * +U dan +Z 0c60a0f2885eb6df2e24e22407faffbe diff --git a/manifest.uuid b/manifest.uuid index 47cc55cc06..747bc2e8b4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -651ef24249d8c22c4f13e4c0bb98a60099cfd23a \ No newline at end of file +b5bdc639898ee22eebedeb560810e94e74de8aa4 \ No newline at end of file diff --git a/test/fts3prefix.test b/test/fts3prefix.test new file mode 100644 index 0000000000..95a5c89629 --- /dev/null +++ b/test/fts3prefix.test @@ -0,0 +1,175 @@ +# 2011 May 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS3 module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix fts3prefix + +# This proc tests that the prefixes index appears to represent the same content +# as the terms index. +# +proc fts3_terms_and_prefixes {db tbl} { + $db eval "CREATE VIRTUAL TABLE fts3check1 USING fts4term($tbl);" + $db eval "CREATE VIRTUAL TABLE fts3check2 USING fts4prefix($tbl);" + + $db eval { + CREATE TEMP TABLE terms AS SELECT * FROM fts3check1; + CREATE TEMP TABLE prefixes AS SELECT * FROM fts3check2; + CREATE INDEX temp.idx ON prefixes(term); + DROP TABLE fts3check1; + DROP TABLE fts3check2; + } + + $db eval { SELECT term, docid, col, pos FROM temp.terms } a { + set nMax [expr [string length $a(term)] - 1] + if {$nMax>8} {set nMax 8} + for {set n 0} {$n < $nMax} {incr n} { + set t [string range $a(term) 0 $n] + set r [$db one { + SELECT count(*) FROM temp.prefixes WHERE + term = $t AND docid = $a(docid) AND col = $a(col) AND pos = $a(pos) + }] + if {$r != 1} { + error "$t, $a(docid), $a(col), $a(pos)" + } + } + } + + execsql { DROP TABLE temp.prefixes } + execsql { DROP TABLE temp.terms } + + set terms_layout [$db eval " + SELECT level, idx FROM ${tbl}_segdir WHERE level < 1024 ORDER by 1, 2 + "] + set prefixes_layout [$db eval " + SELECT level-1024, idx FROM ${tbl}_segdir WHERE level >= 1024 ORDER by 1, 2 + "] + + if {$terms_layout != $prefixes_layout} { + puts "TERMS LAYOUT: $terms_layout" + puts "PREFIX LAYOUT: $prefixes_layout" + error "Terms and prefixes are comprised of different b-trees" + } + + return "" +} +proc fts3_tap_test {tn db tbl} { + uplevel [list do_test $tn [list fts3_terms_and_prefixes $db $tbl] ""] +} + +#------------------------------------------------------------------------- +# Test cases 1.* are a sanity check. They test that the prefixes index is +# being constructed correctly for the simplest possible case. +# +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE t1 USING fts4(prefix=1); + CREATE VIRTUAL TABLE prefixes USING fts4prefix(t1); + CREATE VIRTUAL TABLE terms USING fts4term(t1); +} +do_execsql_test 1.2 { + INSERT INTO t1 VALUES('sqlite mysql firebird'); +} +do_execsql_test 1.3 { + SELECT term FROM prefixes; +} {f fi fir fire fireb firebi firebir firebird m my mys mysq mysql s sq sql sqli sqlit sqlite} +do_execsql_test 1.4 { + SELECT term FROM terms; +} {firebird mysql sqlite} + +fts3_tap_test 1.5 db t1 + +#------------------------------------------------------------------------- +# A slightly more complicated dataset. This test also verifies that DELETE +# operations do not corrupt the prefixes index. +# +do_execsql_test 2.1 { + INSERT INTO t1 VALUES('FTS3 and FTS4 are an SQLite virtual table modules'); + INSERT INTO t1 VALUES('that allows users to perform full-text searches on'); + INSERT INTO t1 VALUES('a set of documents. The most common (and'); + INSERT INTO t1 VALUES('effective) way to describe full-text searches is'); + INSERT INTO t1 VALUES('"what Google, Yahoo and Altavista do with'); + INSERT INTO t1 VALUES('documents placed on the World Wide Web". Users'); + INSERT INTO t1 VALUES('input a term, or series of terms, perhaps'); + INSERT INTO t1 VALUES('connected by a binary operator or grouped together'); + INSERT INTO t1 VALUES('into a phrase, and the full-text query system'); + INSERT INTO t1 VALUES('finds the set of documents that best matches those'); + INSERT INTO t1 VALUES('terms considering the operators and groupings the'); + INSERT INTO t1 VALUES('user has specified. This article describes the'); + INSERT INTO t1 VALUES('deployment and usage of FTS3 and FTS4.'); + INSERT INTO t1 VALUES('FTS1 and FTS2 are obsolete full-text search'); + INSERT INTO t1 VALUES('modules for SQLite. There are known issues with'); + INSERT INTO t1 VALUES('these older modules and their use should be'); + INSERT INTO t1 VALUES('avoided. Portions of the original FTS3 code were'); + INSERT INTO t1 VALUES('contributed to the SQLite project by Scott Hess of'); + INSERT INTO t1 VALUES('Google. It is now developed and maintained as part'); + INSERT INTO t1 VALUES('of SQLite. '); +} +fts3_tap_test 2.2 db t1 +do_execsql_test 2.3 { DELETE FROM t1 WHERE docid%2; } +fts3_tap_test 2.4 db t1 + +do_execsql_test 2.5 { INSERT INTO t1(t1) VALUES('optimize') } +fts3_tap_test 2.6 db t1 + +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE t2 USING fts4(prefix=1); + INSERT INTO t2 VALUES('On 12 September the wind direction turned and'); + INSERT INTO t2 VALUES('William''s fleet sailed. A storm blew up and the'); + INSERT INTO t2 VALUES('fleet was forced to take shelter at'); + INSERT INTO t2 VALUES('Saint-Valery-sur-Somme and again wait for the wind'); + INSERT INTO t2 VALUES('to change. On 27 September the Norman fleet'); + INSERT INTO t2 VALUES('finally set sail, landing in England at Pevensey'); + INSERT INTO t2 VALUES('Bay (Sussex) on 28 September. William then moved'); + INSERT INTO t2 VALUES('to Hastings, a few miles to the east, where he'); + INSERT INTO t2 VALUES('built a prefabricated wooden castle for a base of'); + INSERT INTO t2 VALUES('operations. From there, he ravaged the hinterland'); + INSERT INTO t2 VALUES('and waited for Harold''s return from the north.'); + INSERT INTO t2 VALUES('On 12 September the wind direction turned and'); + INSERT INTO t2 VALUES('William''s fleet sailed. A storm blew up and the'); + INSERT INTO t2 VALUES('fleet was forced to take shelter at'); + INSERT INTO t2 VALUES('Saint-Valery-sur-Somme and again wait for the wind'); + INSERT INTO t2 VALUES('to change. On 27 September the Norman fleet'); + INSERT INTO t2 VALUES('finally set sail, landing in England at Pevensey'); + INSERT INTO t2 VALUES('Bay (Sussex) on 28 September. William then moved'); + INSERT INTO t2 VALUES('to Hastings, a few miles to the east, where he'); + INSERT INTO t2 VALUES('built a prefabricated wooden castle for a base of'); + INSERT INTO t2 VALUES('operations. From there, he ravaged the hinterland'); + INSERT INTO t2 VALUES('and waited for Harold''s return from the north.'); +} + +fts3_tap_test 3.2 db t2 +do_execsql_test 3.3 { SELECT optimize(t2) FROM t2 LIMIT 1 } {{Index optimized}} +fts3_tap_test 3.4 db t2 + + +#------------------------------------------------------------------------- +# Simple tests for reading the prefix-index. +# +do_execsql_test 4.1 { + CREATE VIRTUAL TABLE t3 USING fts4(prefix=1); + INSERT INTO t3 VALUES('one two three'); + INSERT INTO t3 VALUES('four five six'); + INSERT INTO t3 VALUES('seven eight nine'); +} +do_execsql_test 4.2 { + SELECT * FROM t3 WHERE t3 MATCH 'f*' +} {{four five six}} +do_execsql_test 4.3 { + SELECT * FROM t3 WHERE t3 MATCH 'four*' +} {{four five six}} +do_execsql_test 4.4 { + SELECT * FROM t3 WHERE t3 MATCH 's*' +} {{four five six} {seven eight nine}} + +finish_test From 9d1f874ab30b038d7e0bda416dfeee6ecf1be790 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 25 May 2011 18:34:53 +0000 Subject: [PATCH 02/20] Change fts4 so that the prefix= parameter is passes a comma-separated list of integers. For each integer N, a separate index of all prefixes of length N bytes is created. FossilOrigin-Name: be59bf49402d2e2f4b95fb6668849f3745cb7bf2 --- ext/fts3/fts3.c | 174 +++++++++++++++++-------- ext/fts3/fts3Int.h | 59 +++++---- ext/fts3/fts3_aux.c | 3 +- ext/fts3/fts3_term.c | 17 +-- ext/fts3/fts3_write.c | 286 ++++++++++++++++++++++-------------------- manifest | 25 ++-- manifest.uuid | 2 +- test/fts3prefix.test | 110 +++++++++------- 8 files changed, 400 insertions(+), 276 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index a127773449..6a5c2ae4ef 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -832,6 +832,58 @@ static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ return zRet; } +static int fts3GobbleInt(const char **pp, int *pnOut){ + const char *p = *pp; + int nInt = 0; + for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ + nInt = nInt * 10 + (p[0] - '0'); + } + if( p==*pp ) return SQLITE_ERROR; + *pnOut = nInt; + *pp = p; + return SQLITE_OK; +} + + +static int fts3PrefixParameter( + const char *zParam, /* ABC in prefix=ABC parameter to parse */ + int *pnIndex, /* OUT: size of *apIndex[] array */ + struct Fts3Index **apIndex, /* OUT: Array of indexes for this table */ + struct Fts3Index **apFree /* OUT: Free this with sqlite3_free() */ +){ + struct Fts3Index *aIndex; + int nIndex = 1; + + if( zParam && zParam[0] ){ + const char *p; + nIndex++; + for(p=zParam; *p; p++){ + if( *p==',' ) nIndex++; + } + } + + aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); + *apIndex = *apFree = aIndex; + *pnIndex = nIndex; + if( !aIndex ){ + return SQLITE_NOMEM; + } + + memset(aIndex, 0, sizeof(struct Fts3Index) * nIndex); + if( zParam ){ + const char *p = zParam; + int i; + for(i=1; ibHasStat = isFts4; TESTONLY( p->inTransaction = -1 ); TESTONLY( p->mxSavepoint = -1 ); - p->bPrefix = bPrefix; - fts3HashInit(&p->pendingTerms, FTS3_HASH_STRING, 1); - fts3HashInit(&p->pendingPrefixes, FTS3_HASH_STRING, 1); + + p->aIndex = (struct Fts3Index *)&p->azColumn[nCol]; + memcpy(p->aIndex, aIndex, sizeof(struct Fts3Index) * nIndex); + p->nIndex = nIndex; + for(i=0; iaIndex[i].hPending, FTS3_HASH_STRING, 1); + } /* Fill in the zName and zDb fields of the vtab structure. */ - zCsr = (char *)&p->azColumn[nCol]; + zCsr = (char *)&p->aIndex[nIndex]; p->zName = zCsr; memcpy(zCsr, argv[2], nName); zCsr += nName; @@ -1034,6 +1104,8 @@ static int fts3InitVtab( fts3DeclareVtab(&rc, p); fts3_init_out: + sqlite3_free(zPrefix); + sqlite3_free(aFree); sqlite3_free(zCompress); sqlite3_free(zUncompress); sqlite3_free((void *)aCol); @@ -2144,6 +2216,9 @@ static int fts3DeferredTermSelect( return SQLITE_OK; } +/* +** Append SegReader object pNew to the end of the pCsr->apSegment[] array. +*/ static int fts3SegReaderCursorAppend( Fts3SegReaderCursor *pCsr, Fts3SegReader *pNew @@ -2163,11 +2238,12 @@ static int fts3SegReaderCursorAppend( } /* -** Set up a cursor object for iterating through the full-text index or -** a single level therein. +** Set up a cursor object for iterating through a full-text index or a +** single level therein. */ int sqlite3Fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ int iLevel, /* Level of segments to scan */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ @@ -2180,49 +2256,41 @@ int sqlite3Fts3SegReaderCursor( int iAge = 0; sqlite3_stmt *pStmt = 0; - assert( iLevel==FTS3_SEGCURSOR_ALL_TERM + assert( iIndex>=0 && iIndexnIndex ); + assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel==FTS3_SEGCURSOR_PENDING - || iLevel==FTS3_SEGCURSOR_PENDING_PREFIX - || iLevel==FTS3_SEGCURSOR_ALL_PREFIX || iLevel>=0 ); - assert( 0>FTS3_SEGCURSOR_ALL_TERM - && 0>FTS3_SEGCURSOR_PENDING - && 0>FTS3_SEGCURSOR_PENDING_PREFIX - && 0>FTS3_SEGCURSOR_ALL_PREFIX - ); - assert( iLevel==FTS3_SEGCURSOR_ALL_TERM - || iLevel==FTS3_SEGCURSOR_ALL_PREFIX - || (zTerm==0 && isPrefix==1) - ); + assert( iLevelaIndex==0 ); + memset(pCsr, 0, sizeof(Fts3SegReaderCursor)); - /* "isScan" is only set to true by the ft4aux module, not an ordinary - ** full-text table. The pendingTerms and pendingPrefixes tables must be - ** empty in this case. */ - assert( isScan==0 || fts3HashCount(&p->pendingTerms)==0 ); - assert( isScan==0 || fts3HashCount(&p->pendingPrefixes)==0 ); - - /* If iLevel is less than 0, include a seg-reader for the pending-terms. */ - if( iLevel<0 && isScan==0 ){ - int bPrefix = ( - iLevel==FTS3_SEGCURSOR_PENDING_PREFIX - || iLevel==FTS3_SEGCURSOR_ALL_PREFIX - ); - Fts3SegReader *pPending = 0; - - rc = sqlite3Fts3SegReaderPending(p,zTerm,nTerm,isPrefix,bPrefix,&pPending); - if( rc==SQLITE_OK && pPending ){ - rc = fts3SegReaderCursorAppend(pCsr, pPending); + /* If iLevel is less than 0 and this is not a scan, include a seg-reader + ** for the pending-terms. If this is a scan, then this call must be being + ** made by an fts4aux module, not an FTS table. In this case calling + ** Fts3SegReaderPending might segfault, as the data structures used by + ** fts4aux are not completely populated. So it's easiest to filter these + ** calls out here. */ + if( iLevel<0 && p->aIndex ){ + Fts3SegReader *pSeg = 0; + rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix, &pSeg); + if( rc==SQLITE_OK && pSeg ){ + rc = fts3SegReaderCursorAppend(pCsr, pSeg); } } - if( iLevel!=FTS3_SEGCURSOR_PENDING && iLevel!=FTS3_SEGCURSOR_PENDING_PREFIX ){ + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ if( rc==SQLITE_OK ){ - rc = sqlite3Fts3AllSegdirs(p, iLevel, &pStmt); + rc = sqlite3Fts3AllSegdirs(p, iIndex, iLevel, &pStmt); } + while( rc==SQLITE_OK && SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ Fts3SegReader *pSeg = 0; @@ -2273,17 +2341,25 @@ static int fts3TermSegReaderCursor( if( pSegcsr ){ int i; int nCost = 0; + int bFound = 0; /* True once an index has been found */ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; - if( isPrefix && p->bPrefix && nTerm<=FTS3_MAX_PREFIX ){ - rc = sqlite3Fts3SegReaderCursor( - p, FTS3_SEGCURSOR_ALL_PREFIX, zTerm, nTerm, 0, 0, pSegcsr); - - }else{ - rc = sqlite3Fts3SegReaderCursor( - p, FTS3_SEGCURSOR_ALL_TERM, zTerm, nTerm, isPrefix, 0, pSegcsr); + if( isPrefix ){ + for(i=1; inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor( + p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr); + break; + } + } + } + + if( bFound==0 ){ + rc = sqlite3Fts3SegReaderCursor( + p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr + ); } - for(i=0; rc==SQLITE_OK && inSegment; i++){ rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost); } @@ -3340,7 +3416,6 @@ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ assert( p->inTransaction!=0 ); TESTONLY( p->inTransaction = 0 ); TESTONLY( p->mxSavepoint = -1; ); - sqlite3Fts3PendingPrefixesClear((Fts3Table *)pVtab); return SQLITE_OK; } @@ -3715,7 +3790,6 @@ static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ assert( p->mxSavepoint >= iSavepoint ); TESTONLY( p->mxSavepoint = iSavepoint ); sqlite3Fts3PendingTermsClear(p); - sqlite3Fts3PendingPrefixesClear((Fts3Table *)pVtab); return SQLITE_OK; } diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 785a6aa655..00e1c37d2b 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -23,8 +23,6 @@ #include "fts3_tokenizer.h" #include "fts3_hash.h" -#define FTS3_MAX_PREFIX 8 - /* ** This constant controls how often segments are merged. Once there are ** FTS3_MERGE_COUNT segments of level N, they are merged into a single @@ -56,18 +54,22 @@ #define FTS3_VARINT_MAX 10 /* -** FTS4 virtual tables may maintain two separate indexes. One that indexes -** all document terms (the same index FTS3 tables maintain) and another used -** for prefixes. B+-trees that are part of the prefix index have values for -** the %_segdir.level column that are equal to or greater than the following -** value. +** FTS4 virtual tables may maintain multiple indexes - one index of all terms +** in the document set and zero or more prefix indexes. All indexes are stored +** as one or more b+-trees in the %_segments and %_segdir tables. ** -** It is considered impossible for the regular index to use levels this large. -** In theory it could, but that would require that at least 2^1024 separate -** write operations to be made within the lifetime of the database. +** It is possible to determine which index a b+-tree belongs to based on the +** value stored in the "%_segdir.level" column. Given this value L, the index +** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with +** level values between 0 and 1023 (inclusive) belong to index 0, all levels +** between 1024 and 2047 to index 1, and so on. +** +** It is considered impossible for an index to use more than 1024 levels. In +** theory though this may happen, but only after at least +** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. */ -#define FTS3_SEGDIR_PREFIXLEVEL 1024 -#define FTS3_SEGDIR_PREFIXLEVEL_STR "1024" +#define FTS3_SEGDIR_MAXLEVEL 1024 +#define FTS3_SEGDIR_MAXLEVEL_STR "1024" /* ** The testcase() macro is only used by the amalgamation. If undefined, @@ -172,23 +174,32 @@ struct Fts3Table { int nNodeSize; /* Soft limit for node size */ u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ - u8 bPrefix; /* True if there is a prefix index */ int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ - /* The following hash table is used to buffer pending index updates during + /* TODO: Fix the first paragraph of this comment. + ** + ** The following hash table is used to buffer pending index updates during ** transactions. Variable nPendingData estimates the memory size of the ** pending data, including hash table overhead, but not malloc overhead. ** When nPendingData exceeds nMaxPendingData, the buffer is flushed ** automatically. Variable iPrevDocid is the docid of the most recently ** inserted record. + ** + ** A single FTS4 table may have multiple full-text indexes. For each index + ** there is an entry in the aIndex[] array. Index 0 is an index of all the + ** terms that appear in the document set. Each subsequent index in aIndex[] + ** is an index of prefixes of a specific length. */ - int nMaxPendingData; - int nPendingData; - sqlite_int64 iPrevDocid; - Fts3Hash pendingTerms; - Fts3Hash pendingPrefixes; + int nIndex; /* Size of aIndex[] */ + struct Fts3Index { + int nPrefix; /* Prefix length (0 for main terms index) */ + Fts3Hash hPending; /* Pending terms table for this index */ + } *aIndex; + int nMaxPendingData; /* Max pending data before flush to disk */ + int nPendingData; /* Current bytes of pending data */ + sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */ #if defined(SQLITE_DEBUG) /* State variables used for validating that the transaction control @@ -336,10 +347,10 @@ int sqlite3Fts3Optimize(Fts3Table *); int sqlite3Fts3SegReaderNew(int, sqlite3_int64, sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**); int sqlite3Fts3SegReaderPending( - Fts3Table*,const char*,int,int,int,Fts3SegReader**); + Fts3Table*,int,const char*,int,int,Fts3SegReader**); void sqlite3Fts3SegReaderFree(Fts3SegReader *); int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); -int sqlite3Fts3AllSegdirs(Fts3Table*, int, sqlite3_stmt **); +int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **); int sqlite3Fts3ReadLock(Fts3Table *); int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*); @@ -355,15 +366,13 @@ void sqlite3Fts3SegmentsClose(Fts3Table *); /* Special values interpreted by sqlite3SegReaderCursor() */ #define FTS3_SEGCURSOR_PENDING -1 -#define FTS3_SEGCURSOR_PENDING_PREFIX -2 -#define FTS3_SEGCURSOR_ALL_PREFIX -3 -#define FTS3_SEGCURSOR_ALL_TERM -4 +#define FTS3_SEGCURSOR_ALL -2 int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*); int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *); void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *); int sqlite3Fts3SegReaderCursor( - Fts3Table *, int, const char *, int, int, int, Fts3SegReaderCursor *); + Fts3Table *, int, int, const char *, int, int, int, Fts3SegReaderCursor *); /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 diff --git a/ext/fts3/fts3_aux.c b/ext/fts3/fts3_aux.c index cb91cda347..5c7c7b0b0f 100644 --- a/ext/fts3/fts3_aux.c +++ b/ext/fts3/fts3_aux.c @@ -96,6 +96,7 @@ static int fts3auxConnectMethod( p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; p->pFts3Tab->db = db; + p->pFts3Tab->nIndex = 1; memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); @@ -375,7 +376,7 @@ static int fts3auxFilterMethod( if( pCsr->zStop==0 ) return SQLITE_NOMEM; } - rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL_TERM, + rc = sqlite3Fts3SegReaderCursor(pFts3, 0, FTS3_SEGCURSOR_ALL, pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr ); if( rc==SQLITE_OK ){ diff --git a/ext/fts3/fts3_term.c b/ext/fts3/fts3_term.c index ca46269689..d7aa66855a 100644 --- a/ext/fts3/fts3_term.c +++ b/ext/fts3/fts3_term.c @@ -27,7 +27,7 @@ typedef struct Fts3termCursor Fts3termCursor; struct Fts3termTable { sqlite3_vtab base; /* Base class used by SQLite core */ - int bPrefix; /* True for an fts4prefix table */ + int iIndex; /* Index for Fts3Table.aIndex[] */ Fts3Table *pFts3Tab; }; @@ -70,6 +70,12 @@ static int fts3termConnectMethod( int nByte; /* Bytes of space to allocate here */ int rc; /* value returned by declare_vtab() */ Fts3termTable *p; /* Virtual table object to return */ + int iIndex = 0; + + if( argc==5 ){ + iIndex = atoi(argv[4]); + argc--; + } /* The user should specify a single argument - the name of an fts3 table. */ if( argc!=4 ){ @@ -96,7 +102,8 @@ static int fts3termConnectMethod( p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1]; p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1]; p->pFts3Tab->db = db; - p->bPrefix = (int)pCtx; + p->pFts3Tab->nIndex = iIndex+1; + p->iIndex = iIndex; memcpy((char *)p->pFts3Tab->zDb, zDb, nDb); memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3); @@ -263,8 +270,7 @@ static int fts3termFilterMethod( pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; pCsr->filter.flags |= FTS3_SEGMENT_SCAN; - rc = sqlite3Fts3SegReaderCursor(pFts3, - p->bPrefix ? FTS3_SEGCURSOR_ALL_PREFIX : FTS3_SEGCURSOR_ALL_TERM, + rc = sqlite3Fts3SegReaderCursor(pFts3, p->iIndex, FTS3_SEGCURSOR_ALL, pCsr->filter.zTerm, pCsr->filter.nTerm, 0, 1, &pCsr->csr ); if( rc==SQLITE_OK ){ @@ -355,9 +361,6 @@ int sqlite3Fts3InitTerm(sqlite3 *db){ int rc; /* Return code */ rc = sqlite3_create_module(db, "fts4term", &fts3term_module, 0); - if( rc==SQLITE_OK ){ - rc = sqlite3_create_module(db, "fts4prefix", &fts3term_module, (void*)1); - } return rc; } diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 5ff7e0b1aa..9a136c3a65 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -183,10 +183,10 @@ struct SegmentNode { #define SQL_NEXT_SEGMENTS_ID 10 #define SQL_INSERT_SEGDIR 11 #define SQL_SELECT_LEVEL 12 -#define SQL_SELECT_ALL_LEVEL 13 +#define SQL_SELECT_LEVEL_RANGE 13 #define SQL_SELECT_LEVEL_COUNT 14 #define SQL_SELECT_SEGDIR_MAX_LEVEL 15 -#define SQL_DELETE_SEGDIR_BY_LEVEL 16 +#define SQL_DELETE_SEGDIR_LEVEL 16 #define SQL_DELETE_SEGMENTS_RANGE 17 #define SQL_CONTENT_INSERT 18 #define SQL_DELETE_DOCSIZE 19 @@ -194,10 +194,11 @@ struct SegmentNode { #define SQL_SELECT_DOCSIZE 21 #define SQL_SELECT_DOCTOTAL 22 #define SQL_REPLACE_DOCTOTAL 23 -#define SQL_SELECT_ALL_PREFIX_LEVEL 24 +#define SQL_SELECT_ALL_PREFIX_LEVEL 24 #define SQL_DELETE_ALL_TERMS_SEGDIR 25 -#define SQL_DELETE_ALL_PREFIX_SEGDIR 26 + +#define SQL_DELETE_SEGDIR_RANGE 26 /* ** This function is used to obtain an SQLite prepared statement handle @@ -234,12 +235,11 @@ static int fts3SqlStmt( /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", /* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " - "FROM %Q.'%q_segdir' WHERE level < " FTS3_SEGDIR_PREFIXLEVEL_STR - " ORDER BY level DESC, idx ASC", + "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?" + "ORDER BY level DESC, idx ASC", /* 14 */ "SELECT count(*) FROM %Q.'%q_segdir' WHERE level = ?", -/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level < (?+1)*" - FTS3_SEGDIR_PREFIXLEVEL_STR, +/* 15 */ "SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", /* 16 */ "DELETE FROM %Q.'%q_segdir' WHERE level = ?", /* 17 */ "DELETE FROM %Q.'%q_segments' WHERE blockid BETWEEN ? AND ?", @@ -249,11 +249,11 @@ static int fts3SqlStmt( /* 21 */ "SELECT size FROM %Q.'%q_docsize' WHERE docid=?", /* 22 */ "SELECT value FROM %Q.'%q_stat' WHERE id=0", /* 23 */ "REPLACE INTO %Q.'%q_stat' VALUES(0,?)", -/* 24 */ "SELECT idx, start_block, leaves_end_block, end_block, root " - "FROM %Q.'%q_segdir' WHERE level >= " FTS3_SEGDIR_PREFIXLEVEL_STR - " ORDER BY level DESC, idx ASC", -/* 25 */ "DELETE FROM %Q.'%q_segdir' WHERE level<" FTS3_SEGDIR_PREFIXLEVEL_STR, -/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level>=" FTS3_SEGDIR_PREFIXLEVEL_STR, +/* 24 */ "", +/* 25 */ "", + +/* 26 */ "DELETE FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ?", + }; int rc = SQLITE_OK; sqlite3_stmt *pStmt; @@ -409,17 +409,32 @@ int sqlite3Fts3ReadLock(Fts3Table *p){ ** 3: end_block ** 4: root */ -int sqlite3Fts3AllSegdirs(Fts3Table *p, int iLevel, sqlite3_stmt **ppStmt){ +int sqlite3Fts3AllSegdirs( + Fts3Table *p, /* FTS3 table */ + int iIndex, /* Index for p->aIndex[] */ + int iLevel, /* Level to select */ + sqlite3_stmt **ppStmt /* OUT: Compiled statement */ +){ int rc; sqlite3_stmt *pStmt = 0; - if( iLevel==FTS3_SEGCURSOR_ALL_PREFIX ){ - rc = fts3SqlStmt(p, SQL_SELECT_ALL_PREFIX_LEVEL, &pStmt, 0); - }else if( iLevel==FTS3_SEGCURSOR_ALL_TERM ){ - rc = fts3SqlStmt(p, SQL_SELECT_ALL_LEVEL, &pStmt, 0); + + assert( iLevel==FTS3_SEGCURSOR_ALL || iLevel>=0 ); + assert( iLevel=0 && iIndexnIndex ); + + if( iLevel==FTS3_SEGCURSOR_ALL ){ + /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ + rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); + sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL-1); + } }else{ - assert( iLevel>=0 ); + /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); - if( rc==SQLITE_OK ) sqlite3_bind_int(pStmt, 1, iLevel); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pStmt, 1, iLevel+iIndex*FTS3_SEGDIR_MAXLEVEL); + } } *ppStmt = pStmt; return rc; @@ -565,9 +580,6 @@ static int fts3PendingTermsAddOne( return rc; } - - - /* ** Tokenize the nul-terminated string zText and add all tokens to the ** pending-terms hash-table. The docid used is that currently stored in @@ -616,6 +628,7 @@ static int fts3PendingTermsAdd( while( SQLITE_OK==rc && SQLITE_OK==(rc = xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos)) ){ + int i; if( iPos>=nWord ) nWord = iPos+1; /* Positions cannot be negative; we use -1 as a terminator internally. @@ -626,12 +639,19 @@ static int fts3PendingTermsAdd( break; } - rc = fts3PendingTermsAddOne(p,iCol,iPos,&p->pendingTerms,zToken,nToken); - if( p->bPrefix ){ - int n = (nToken > FTS3_MAX_PREFIX ? FTS3_MAX_PREFIX : nToken); - for(; n>0 && rc==SQLITE_OK; n--){ - rc = fts3PendingTermsAddOne(p,iCol,iPos,&p->pendingPrefixes,zToken,n); - } + /* Add the term to the terms index */ + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken + ); + + /* Add the term to each of the prefix indexes that it is not too + ** short for. */ + for(i=1; rc==SQLITE_OK && inIndex; i++){ + struct Fts3Index *pIndex = &p->aIndex[i]; + if( nTokennPrefix ) continue; + rc = fts3PendingTermsAddOne( + p, iCol, iPos, &pIndex->hPending, zToken, pIndex->nPrefix + ); } } @@ -661,28 +681,21 @@ static int fts3PendingTermsDocid(Fts3Table *p, sqlite_int64 iDocid){ } /* -** Discard the contents of the pending-terms hash table. +** Discard the contents of the pending-terms hash tables. */ void sqlite3Fts3PendingTermsClear(Fts3Table *p){ - Fts3HashElem *pElem; - for(pElem=fts3HashFirst(&p->pendingTerms); pElem; pElem=fts3HashNext(pElem)){ - sqlite3_free(fts3HashData(pElem)); + int i; + for(i=0; inIndex; i++){ + Fts3HashElem *pElem; + Fts3Hash *pHash = &p->aIndex[i].hPending; + for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ + sqlite3_free(fts3HashData(pElem)); + } + fts3HashClear(pHash); } - fts3HashClear(&p->pendingTerms); p->nPendingData = 0; } -/* -** Discard the contents of the pending-prefixes hash table. -*/ -void sqlite3Fts3PendingPrefixesClear(Fts3Table *p){ - Fts3HashElem *pElem; - for(pElem=fts3HashFirst(&p->pendingPrefixes); pElem; pElem=fts3HashNext(pElem)){ - sqlite3_free(fts3HashData(pElem)); - } - fts3HashClear(&p->pendingPrefixes); -} - /* ** This function is called by the xUpdate() method as part of an INSERT ** operation. It adds entries for each term in the new record to the @@ -780,7 +793,6 @@ static int fts3DeleteAll(Fts3Table *p){ /* Discard the contents of the pending-terms hash table. */ sqlite3Fts3PendingTermsClear(p); - sqlite3Fts3PendingPrefixesClear(p); /* Delete everything from the %_content, %_segments and %_segdir tables. */ fts3SqlExec(&rc, p, SQL_DELETE_ALL_CONTENT, 0); @@ -836,7 +848,7 @@ static void fts3DeleteTerms( ** Forward declaration to account for the circular dependency between ** functions fts3SegmentMerge() and fts3AllocateSegdirIdx(). */ -static int fts3SegmentMerge(Fts3Table *, int); +static int fts3SegmentMerge(Fts3Table *, int, int); /* ** This function allocates a new level iLevel index in the segdir table. @@ -853,7 +865,12 @@ static int fts3SegmentMerge(Fts3Table *, int); ** If successful, *piIdx is set to the allocated index slot and SQLITE_OK ** returned. Otherwise, an SQLite error code is returned. */ -static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ +static int fts3AllocateSegdirIdx( + Fts3Table *p, + int iIndex, /* Index for p->aIndex */ + int iLevel, + int *piIdx +){ int rc; /* Return Code */ sqlite3_stmt *pNextIdx; /* Query for next idx at level iLevel */ int iNext = 0; /* Result of query pNextIdx */ @@ -861,7 +878,7 @@ static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ /* Set variable iNext to the next available segdir index at level iLevel. */ rc = fts3SqlStmt(p, SQL_NEXT_SEGMENT_INDEX, &pNextIdx, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pNextIdx, 1, iLevel); + sqlite3_bind_int(pNextIdx, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); if( SQLITE_ROW==sqlite3_step(pNextIdx) ){ iNext = sqlite3_column_int(pNextIdx, 0); } @@ -875,7 +892,7 @@ static int fts3AllocateSegdirIdx(Fts3Table *p, int iLevel, int *piIdx){ ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. */ if( iNext>=FTS3_MERGE_COUNT ){ - rc = fts3SegmentMerge(p, iLevel); + rc = fts3SegmentMerge(p, iIndex, iLevel); *piIdx = 0; }else{ *piIdx = iNext; @@ -1309,23 +1326,22 @@ static int fts3CompareElemByTerm(const void *lhs, const void *rhs){ */ int sqlite3Fts3SegReaderPending( Fts3Table *p, /* Virtual table handle */ + int iIndex, /* Index for p->aIndex */ const char *zTerm, /* Term to search for */ int nTerm, /* Size of buffer zTerm */ - int isMultiTerm, /* True to visit multiple terms */ - int isPrefixIter, /* 0->pendingTerms, 1->pendingPrefixes */ + int bPrefix, /* True for a prefix iterator */ Fts3SegReader **ppReader /* OUT: SegReader for pending-terms */ ){ Fts3SegReader *pReader = 0; /* Fts3SegReader object to return */ Fts3HashElem **aElem = 0; /* Array of term hash entries to scan */ int nElem = 0; /* Size of array at aElem */ int rc = SQLITE_OK; /* Return Code */ + Fts3Hash *pHash; - if( isMultiTerm ){ + pHash = &p->aIndex[iIndex].hPending; + if( bPrefix ){ int nAlloc = 0; /* Size of allocated array at aElem */ Fts3HashElem *pE = 0; /* Iterator variable */ - Fts3Hash *pHash; - - pHash = (isPrefixIter ? &p->pendingPrefixes : &p->pendingTerms); for(pE=fts3HashFirst(pHash); pE; pE=fts3HashNext(pE)){ char *zKey = (char *)fts3HashKey(pE); @@ -1360,7 +1376,7 @@ int sqlite3Fts3SegReaderPending( }else{ /* The query is a simple term lookup that matches at most one term in ** the index. All that is required is a straight hash-lookup. */ - Fts3HashElem *pE = fts3HashFindElem(&p->pendingTerms, zTerm, nTerm); + Fts3HashElem *pE = fts3HashFindElem(pHash, zTerm, nTerm); if( pE ){ aElem = &pE; nElem = 1; @@ -1380,7 +1396,7 @@ int sqlite3Fts3SegReaderPending( } } - if( isMultiTerm ){ + if( bPrefix ){ sqlite3_free(aElem); } *ppReader = pReader; @@ -1992,28 +2008,28 @@ static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ } /* -** Set *pnMax to the largest segment level in the database for either the -** terms index (if parameter bPrefixIndex is 0) or the prefixes index (if -** parameter bPrefixIndex is 1). +** Set *pnMax to the largest segment level in the database for the index +** iIndex. ** ** Segment levels are stored in the 'level' column of the %_segdir table. ** ** Return SQLITE_OK if successful, or an SQLite error code if not. */ -static int fts3SegmentMaxLevel(Fts3Table *p, int bPrefixIndex, int *pnMax){ +static int fts3SegmentMaxLevel(Fts3Table *p, int iIndex, int *pnMax){ sqlite3_stmt *pStmt; int rc; - assert( bPrefixIndex==0 || bPrefixIndex==1 ); + assert( iIndex>=0 && iIndexnIndex ); /* Set pStmt to the compiled version of: ** - ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level < (?+1) * 1024 + ** SELECT max(level) FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? ** ** (1024 is actually the value of macro FTS3_SEGDIR_PREFIXLEVEL_STR). */ rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_int(pStmt, 1, bPrefixIndex); + sqlite3_bind_int(pStmt, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); + sqlite3_bind_int(pStmt, 2, (iIndex+1)*FTS3_SEGDIR_MAXLEVEL - 1); if( SQLITE_ROW==sqlite3_step(pStmt) ){ *pnMax = sqlite3_column_int(pStmt, 0); } @@ -2036,6 +2052,7 @@ static int fts3SegmentMaxLevel(Fts3Table *p, int bPrefixIndex, int *pnMax){ */ static int fts3DeleteSegdir( Fts3Table *p, /* Virtual table handle */ + int iIndex, /* Index for p->aIndex */ int iLevel, /* Level of %_segdir entries to delete */ Fts3SegReader **apSegment, /* Array of SegReader objects */ int nReader /* Size of array apSegment */ @@ -2058,27 +2075,23 @@ static int fts3DeleteSegdir( return rc; } - assert( iLevel>=0 - || iLevel==FTS3_SEGCURSOR_ALL_TERM - || iLevel==FTS3_SEGCURSOR_ALL_PREFIX - || iLevel==FTS3_SEGCURSOR_PENDING - || iLevel==FTS3_SEGCURSOR_PENDING_PREFIX - ); - if( iLevel==FTS3_SEGCURSOR_ALL_TERM ){ - fts3SqlExec(&rc, p, SQL_DELETE_ALL_TERMS_SEGDIR, 0); - }else if( iLevel==FTS3_SEGCURSOR_ALL_PREFIX ){ - fts3SqlExec(&rc, p, SQL_DELETE_ALL_PREFIX_SEGDIR, 0); - }else if( iLevel==FTS3_SEGCURSOR_PENDING_PREFIX ){ - sqlite3Fts3PendingPrefixesClear(p); - }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ - sqlite3Fts3PendingTermsClear(p); - }else if( iLevel>=0 ){ - rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_BY_LEVEL, &pDelete, 0); + assert( iLevel>=0 || iLevel==FTS3_SEGCURSOR_ALL ); + if( iLevel==FTS3_SEGCURSOR_ALL ){ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); if( rc==SQLITE_OK ){ - sqlite3_bind_int(pDelete, 1, iLevel); - sqlite3_step(pDelete); - rc = sqlite3_reset(pDelete); + sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL); + sqlite3_bind_int(pDelete, 2, (iIndex+1) * FTS3_SEGDIR_MAXLEVEL - 1); } + }else{ + rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pDelete, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int(pDelete, 1, iIndex*FTS3_SEGDIR_MAXLEVEL + iLevel); + } + } + + if( rc==SQLITE_OK ){ + sqlite3_step(pDelete); + rc = sqlite3_reset(pDelete); } return rc; @@ -2323,46 +2336,52 @@ void sqlite3Fts3SegReaderFinish( ** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, ** an SQLite error code is returned. */ -static int fts3SegmentMerge(Fts3Table *p, int iLevel){ +static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){ int rc; /* Return code */ int iIdx = 0; /* Index of new segment */ - int iNewLevel = 0; /* Level to create new segment at */ + int iNewLevel = 0; /* Level/index to create new segment at */ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ Fts3SegReaderCursor csr; /* Cursor to iterate through level(s) */ int bIgnoreEmpty = 0; /* True to ignore empty segments */ - rc = sqlite3Fts3SegReaderCursor(p, iLevel, 0, 0, 1, 0, &csr); + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevel=0 && iIndexnIndex ); + + rc = sqlite3Fts3SegReaderCursor(p, iIndex, iLevel, 0, 0, 1, 0, &csr); if( rc!=SQLITE_OK || csr.nSegment==0 ) goto finished; - if( iLevel==FTS3_SEGCURSOR_ALL_TERM || iLevel==FTS3_SEGCURSOR_ALL_PREFIX ){ + if( iLevel==FTS3_SEGCURSOR_ALL ){ /* This call is to merge all segments in the database to a single ** segment. The level of the new segment is equal to the the numerically - ** greatest segment level currently present in the database. The index - ** of the new segment is always 0. */ + ** greatest segment level currently present in the database for this + ** index. The idx of the new segment is always 0. */ if( csr.nSegment==1 ){ rc = SQLITE_DONE; goto finished; } - rc = fts3SegmentMaxLevel(p, iLevel==FTS3_SEGCURSOR_ALL_PREFIX, &iNewLevel); + rc = fts3SegmentMaxLevel(p, iIndex, &iNewLevel); bIgnoreEmpty = 1; + + }else if( iLevel==FTS3_SEGCURSOR_PENDING ){ + iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL; + rc = fts3AllocateSegdirIdx(p, iIndex, 0, &iIdx); }else{ /* This call is to merge all segments at level iLevel. find the next ** available segment index at level iLevel+1. The call to ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ - if( iLevel==FTS3_SEGCURSOR_PENDING ){ - iNewLevel = 0; - }else if( iLevel==FTS3_SEGCURSOR_PENDING_PREFIX ){ - iNewLevel = FTS3_SEGDIR_PREFIXLEVEL; - }else{ - iNewLevel = iLevel+1; - } - rc = fts3AllocateSegdirIdx(p, iNewLevel, &iIdx); + rc = fts3AllocateSegdirIdx(p, iIndex, iLevel+1, &iIdx); + iNewLevel = iIndex * FTS3_SEGDIR_MAXLEVEL + iLevel+1; } if( rc!=SQLITE_OK ) goto finished; assert( csr.nSegment>0 ); - assert( iNewLevel>=0 ); + assert( iNewLevel>=(iIndex*FTS3_SEGDIR_MAXLEVEL) ); + assert( iNewLevel<((iIndex+1)*FTS3_SEGDIR_MAXLEVEL) ); memset(&filter, 0, sizeof(Fts3SegFilter)); filter.flags = FTS3_SEGMENT_REQUIRE_POS; @@ -2378,8 +2397,10 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ if( rc!=SQLITE_OK ) goto finished; assert( pWriter ); - rc = fts3DeleteSegdir(p, iLevel, csr.apSegment, csr.nSegment); - if( rc!=SQLITE_OK ) goto finished; + if( iLevel!=FTS3_SEGCURSOR_PENDING ){ + rc = fts3DeleteSegdir(p, iIndex, iLevel, csr.apSegment, csr.nSegment); + if( rc!=SQLITE_OK ) goto finished; + } rc = fts3SegWriterFlush(p, pWriter, iNewLevel, iIdx); finished: @@ -2390,19 +2411,16 @@ static int fts3SegmentMerge(Fts3Table *p, int iLevel){ /* -** Flush the contents of pendingTerms to a level 0 segment. +** Flush the contents of pendingTerms to level 0 segments. */ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ int rc = SQLITE_OK; - if( p->bPrefix ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING_PREFIX); - } - if( rc==SQLITE_OK || rc==SQLITE_DONE ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_PENDING); - } - if( rc==SQLITE_DONE ){ - rc = SQLITE_OK; + int i; + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_PENDING); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; } + sqlite3Fts3PendingTermsClear(p); return rc; } @@ -2554,6 +2572,23 @@ static void fts3UpdateDocTotals( sqlite3_free(a); } +static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ + int i; + int bSeenDone = 0; + int rc = SQLITE_OK; + for(i=0; rc==SQLITE_OK && inIndex; i++){ + rc = fts3SegmentMerge(p, i, FTS3_SEGCURSOR_ALL); + if( rc==SQLITE_DONE ){ + bSeenDone = 1; + rc = SQLITE_OK; + } + } + sqlite3Fts3SegmentsClose(p); + sqlite3Fts3PendingTermsClear(p); + + return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; +} + /* ** Handle a 'special' INSERT of the form: ** @@ -2570,10 +2605,7 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ if( !zVal ){ return SQLITE_NOMEM; }else if( nVal==8 && 0==sqlite3_strnicmp(zVal, "optimize", 8) ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_PREFIX); - if( rc==SQLITE_OK ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_TERM); - } + rc = fts3DoOptimize(p, 0); #ifdef SQLITE_TEST }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ p->nNodeSize = atoi(&zVal[9]); @@ -2586,9 +2618,6 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ rc = SQLITE_ERROR; } - sqlite3Fts3SegmentsClose(p); - sqlite3Fts3PendingTermsClear(p); - sqlite3Fts3PendingPrefixesClear(p); return rc; } @@ -2905,30 +2934,19 @@ int sqlite3Fts3UpdateMethod( */ int sqlite3Fts3Optimize(Fts3Table *p){ int rc; - int bReturnDone = 0; rc = sqlite3_exec(p->db, "SAVEPOINT fts3", 0, 0, 0); if( rc==SQLITE_OK ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_PREFIX); - if( rc==SQLITE_OK ){ - rc = fts3SegmentMerge(p, FTS3_SEGCURSOR_ALL_TERM); - } - if( rc==SQLITE_DONE ){ - bReturnDone = 1; - rc = SQLITE_OK; - } - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); - if( rc==SQLITE_OK ){ - sqlite3Fts3PendingTermsClear(p); - sqlite3Fts3PendingPrefixesClear(p); - } + rc = fts3DoOptimize(p, 1); + if( rc==SQLITE_OK || rc==SQLITE_DONE ){ + int rc2 = sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); + if( rc2!=SQLITE_OK ) rc = rc2; }else{ sqlite3_exec(p->db, "ROLLBACK TO fts3", 0, 0, 0); sqlite3_exec(p->db, "RELEASE fts3", 0, 0, 0); } } sqlite3Fts3SegmentsClose(p); - return ((rc==SQLITE_OK && bReturnDone) ? SQLITE_DONE : rc); + return rc; } #endif diff --git a/manifest b/manifest index bf93a84328..aff0cdf712 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sthe\sfts4\soption\sprefix=1\sis\sspecified,\shave\sthe\sfts4\smodule\smaintain\san\sindex\sof\sprefixes\sas\swell\sas\sterms. -D 2011-05-24T18:49:45.786 +C Change\sfts4\sso\sthat\sthe\sprefix=\sparameter\sis\spasses\sa\scomma-separated\slist\sof\sintegers.\sFor\seach\sinteger\sN,\sa\sseparate\sindex\sof\sall\sprefixes\sof\slength\sN\sbytes\sis\screated. +D 2011-05-25T18:34:53.807 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,21 +61,21 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 4a48bfef342badba0a71bdeb5354edaa3ad83382 +F ext/fts3/fts3.c 9ebda4cd406e5aa234977caa97c2045d36048753 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 02699211c0b6cf5aa713cc68c527c9a6e9159fbe -F ext/fts3/fts3_aux.c d68d8e4d39e0342302d2c834618755af7c8058ea +F ext/fts3/fts3Int.h deeeac21d17da06683a79b40ae119b93cf86f90a +F ext/fts3/fts3_aux.c 2817a1ec9ffbad673cb1a1527ad807811bc7645b F ext/fts3/fts3_expr.c 5f49e0deaf723724b08100bb3ff40aab02ad0c93 F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2 F ext/fts3/fts3_snippet.c 92b40397b28422c35c4127492d7ac6da34d1966a -F ext/fts3/fts3_term.c cd226a311940b8ef414d5c1f7c74971a47cacedb +F ext/fts3/fts3_term.c 0ade1812c0e97f394b58299810dfd5d2fb7ba501 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c 0fd6a55c774731852f889007fc6edb1b99819ee5 +F ext/fts3/fts3_write.c 416b367f56f3100314fa76615535178718c645e1 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -473,7 +473,7 @@ F test/fts3fault2.test dc96203af6ba31ce20163fc35460e1556e8edf4d F test/fts3malloc.test 9c8cc3f885bb4dfc66d0460c52f68f45e4710d1b F test/fts3matchinfo.test cc0b009edbbf575283d5fdb53271179e0d8019ba F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 -F test/fts3prefix.test 5b4e08c63d5d4a79e54754dc6b2209b03c885200 +F test/fts3prefix.test c51b04f38ce232cd0c9edd3cc60dc981db5e2726 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 F test/fts3rnd.test 2b1a579be557ab8ac54a51b39caa4aa8043cc4ad F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 @@ -939,10 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 651ef24249d8c22c4f13e4c0bb98a60099cfd23a -R c499b030bb5d74c4afca730dec7a05bc -T *branch * fts3-prefix-search -T *sym-fts3-prefix-search * -T -sym-trunk * +P b5bdc639898ee22eebedeb560810e94e74de8aa4 +R e9f427c5bc714eae400763155f710174 U dan -Z 0c60a0f2885eb6df2e24e22407faffbe +Z 6f0f673a9e9eaf0f72d887565c9f2227 diff --git a/manifest.uuid b/manifest.uuid index 747bc2e8b4..3ae4c6e5e2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b5bdc639898ee22eebedeb560810e94e74de8aa4 \ No newline at end of file +be59bf49402d2e2f4b95fb6668849f3745cb7bf2 \ No newline at end of file diff --git a/test/fts3prefix.test b/test/fts3prefix.test index 95a5c89629..2e0c7a8ede 100644 --- a/test/fts3prefix.test +++ b/test/fts3prefix.test @@ -16,56 +16,75 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix fts3prefix +ifcapable !fts3 { + finish_test + return +} + # This proc tests that the prefixes index appears to represent the same content # as the terms index. # -proc fts3_terms_and_prefixes {db tbl} { - $db eval "CREATE VIRTUAL TABLE fts3check1 USING fts4term($tbl);" - $db eval "CREATE VIRTUAL TABLE fts3check2 USING fts4prefix($tbl);" +proc fts3_terms_and_prefixes {db tbl prefixlengths} { - $db eval { - CREATE TEMP TABLE terms AS SELECT * FROM fts3check1; - CREATE TEMP TABLE prefixes AS SELECT * FROM fts3check2; - CREATE INDEX temp.idx ON prefixes(term); - DROP TABLE fts3check1; - DROP TABLE fts3check2; - } + set iIndex 0 + foreach len $prefixlengths { + incr iIndex + $db eval { + DROP TABLE IF EXISTS fts3check1; + DROP TABLE IF EXISTS fts3check2; + } + $db eval "CREATE VIRTUAL TABLE fts3check1 USING fts4term($tbl, 0);" + $db eval "CREATE VIRTUAL TABLE fts3check2 USING fts4term($tbl, $iIndex);" - $db eval { SELECT term, docid, col, pos FROM temp.terms } a { - set nMax [expr [string length $a(term)] - 1] - if {$nMax>8} {set nMax 8} - for {set n 0} {$n < $nMax} {incr n} { - set t [string range $a(term) 0 $n] + $db eval { + DROP TABLE IF EXISTS temp.terms; + DROP TABLE IF EXISTS temp.prefixes; + CREATE TEMP TABLE terms AS SELECT * FROM fts3check1; + CREATE TEMP TABLE prefixes AS SELECT * FROM fts3check2; + CREATE INDEX temp.idx ON prefixes(term); + DROP TABLE fts3check1; + DROP TABLE fts3check2; + } + + set nExpect 0 + $db eval { SELECT term, docid, col, pos FROM temp.terms } a { + if {[string length $a(term)]<$len} continue + incr nExpect + set prefix [string range $a(term) 0 [expr $len-1]] set r [$db one { SELECT count(*) FROM temp.prefixes WHERE - term = $t AND docid = $a(docid) AND col = $a(col) AND pos = $a(pos) + term = $prefix AND docid = $a(docid) AND col = $a(col) AND pos = $a(pos) }] if {$r != 1} { error "$t, $a(docid), $a(col), $a(pos)" } } - } - execsql { DROP TABLE temp.prefixes } - execsql { DROP TABLE temp.terms } + set nCount [$db one {SELECT count(*) FROM temp.prefixes}] + if {$nCount != $nExpect} { + error "prefixes.count(*) is $nCount expected $nExpect" + } + + execsql { DROP TABLE temp.prefixes } + execsql { DROP TABLE temp.terms } - set terms_layout [$db eval " - SELECT level, idx FROM ${tbl}_segdir WHERE level < 1024 ORDER by 1, 2 - "] - set prefixes_layout [$db eval " - SELECT level-1024, idx FROM ${tbl}_segdir WHERE level >= 1024 ORDER by 1, 2 - "] + set list [list] + $db eval " + SELECT sum( 1 << (16*(level%1024)) ) AS total, (level/1024) AS tree + FROM ${tbl}_segdir GROUP BY tree + " { + lappend list [list $total $tree] + } - if {$terms_layout != $prefixes_layout} { - puts "TERMS LAYOUT: $terms_layout" - puts "PREFIX LAYOUT: $prefixes_layout" - error "Terms and prefixes are comprised of different b-trees" + if { [lsort -integer -index 0 $list] != [lsort -integer -index 1 $list] } { + error "inconsistent tree structures: $list" + } } return "" } -proc fts3_tap_test {tn db tbl} { - uplevel [list do_test $tn [list fts3_terms_and_prefixes $db $tbl] ""] +proc fts3_tap_test {tn db tbl lens} { + uplevel [list do_test $tn [list fts3_terms_and_prefixes $db $tbl $lens] ""] } #------------------------------------------------------------------------- @@ -73,21 +92,24 @@ proc fts3_tap_test {tn db tbl} { # being constructed correctly for the simplest possible case. # do_execsql_test 1.1 { - CREATE VIRTUAL TABLE t1 USING fts4(prefix=1); - CREATE VIRTUAL TABLE prefixes USING fts4prefix(t1); + CREATE VIRTUAL TABLE t1 USING fts4(prefix='1,3,6'); + + CREATE VIRTUAL TABLE p1 USING fts4term(t1, 1); + CREATE VIRTUAL TABLE p2 USING fts4term(t1, 2); + CREATE VIRTUAL TABLE p3 USING fts4term(t1, 3); CREATE VIRTUAL TABLE terms USING fts4term(t1); } do_execsql_test 1.2 { INSERT INTO t1 VALUES('sqlite mysql firebird'); } -do_execsql_test 1.3 { - SELECT term FROM prefixes; -} {f fi fir fire fireb firebi firebir firebird m my mys mysq mysql s sq sql sqli sqlit sqlite} +do_execsql_test 1.3.1 { SELECT term FROM p1 } {f m s} +do_execsql_test 1.3.2 { SELECT term FROM p2 } {fir mys sql} +do_execsql_test 1.3.3 { SELECT term FROM p3 } {firebi sqlite} do_execsql_test 1.4 { SELECT term FROM terms; } {firebird mysql sqlite} -fts3_tap_test 1.5 db t1 +fts3_tap_test 1.5 db t1 {1 3 6} #------------------------------------------------------------------------- # A slightly more complicated dataset. This test also verifies that DELETE @@ -115,15 +137,15 @@ do_execsql_test 2.1 { INSERT INTO t1 VALUES('Google. It is now developed and maintained as part'); INSERT INTO t1 VALUES('of SQLite. '); } -fts3_tap_test 2.2 db t1 +fts3_tap_test 2.2 db t1 {1 3 6} do_execsql_test 2.3 { DELETE FROM t1 WHERE docid%2; } -fts3_tap_test 2.4 db t1 +fts3_tap_test 2.4 db t1 {1 3 6} do_execsql_test 2.5 { INSERT INTO t1(t1) VALUES('optimize') } -fts3_tap_test 2.6 db t1 +fts3_tap_test 2.6 db t1 {1 3 6} do_execsql_test 3.1 { - CREATE VIRTUAL TABLE t2 USING fts4(prefix=1); + CREATE VIRTUAL TABLE t2 USING fts4(prefix='1,2,3'); INSERT INTO t2 VALUES('On 12 September the wind direction turned and'); INSERT INTO t2 VALUES('William''s fleet sailed. A storm blew up and the'); INSERT INTO t2 VALUES('fleet was forced to take shelter at'); @@ -148,16 +170,16 @@ do_execsql_test 3.1 { INSERT INTO t2 VALUES('and waited for Harold''s return from the north.'); } -fts3_tap_test 3.2 db t2 +fts3_tap_test 3.2 db t2 {1 2 3} do_execsql_test 3.3 { SELECT optimize(t2) FROM t2 LIMIT 1 } {{Index optimized}} -fts3_tap_test 3.4 db t2 +fts3_tap_test 3.4 db t2 {1 2 3} #------------------------------------------------------------------------- # Simple tests for reading the prefix-index. # do_execsql_test 4.1 { - CREATE VIRTUAL TABLE t3 USING fts4(prefix=1); + CREATE VIRTUAL TABLE t3 USING fts4(prefix="1,4"); INSERT INTO t3 VALUES('one two three'); INSERT INTO t3 VALUES('four five six'); INSERT INTO t3 VALUES('seven eight nine'); From 98b08e71f28a0ba5a365e927c7423d79c2b733f8 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 25 May 2011 19:17:32 +0000 Subject: [PATCH 03/20] If a prefix index of size N is not present, use a prefix index of size N+1 along with the terms index for queries for prefixes of length N. FossilOrigin-Name: cc83991caae7c7d647432d5711b6cd80228c3002 --- ext/fts3/fts3.c | 86 +++++++++++++++++++++++++++++-------------- ext/fts3/fts3_write.c | 2 +- manifest | 16 ++++---- manifest.uuid | 2 +- test/fts3prefix.test | 6 +++ 5 files changed, 75 insertions(+), 37 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 26239be95d..541d6c539d 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -2237,11 +2237,7 @@ static int fts3SegReaderCursorAppend( return SQLITE_OK; } -/* -** Set up a cursor object for iterating through a full-text index or a -** single level therein. -*/ -int sqlite3Fts3SegReaderCursor( +static int fts3SegReaderCursor( Fts3Table *p, /* FTS3 table handle */ int iIndex, /* Index to search (from 0 to p->nIndex-1) */ int iLevel, /* Level of segments to scan */ @@ -2253,25 +2249,8 @@ int sqlite3Fts3SegReaderCursor( ){ int rc = SQLITE_OK; int rc2; - int iAge = 0; sqlite3_stmt *pStmt = 0; - assert( iIndex>=0 && iIndexnIndex ); - assert( iLevel==FTS3_SEGCURSOR_ALL - || iLevel==FTS3_SEGCURSOR_PENDING - || iLevel>=0 - ); - assert( iLevelaIndex==0 ); - - memset(pCsr, 0, sizeof(Fts3SegReaderCursor)); - /* If iLevel is less than 0 and this is not a scan, include a seg-reader ** for the pending-terms. If this is a scan, then this call must be being ** made by an fts4aux module, not an FTS table. In this case calling @@ -2310,12 +2289,11 @@ int sqlite3Fts3SegReaderCursor( if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock; } - rc = sqlite3Fts3SegReaderNew(iAge, iStartBlock, iLeavesEndBlock, - iEndBlock, zRoot, nRoot, &pSeg + rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, + iStartBlock, iLeavesEndBlock, iEndBlock, zRoot, nRoot, &pSeg ); if( rc!=SQLITE_OK ) goto finished; rc = fts3SegReaderCursorAppend(pCsr, pSeg); - iAge++; } } @@ -2326,6 +2304,49 @@ int sqlite3Fts3SegReaderCursor( return rc; } +/* +** Set up a cursor object for iterating through a full-text index or a +** single level therein. +*/ +int sqlite3Fts3SegReaderCursor( + Fts3Table *p, /* FTS3 table handle */ + int iIndex, /* Index to search (from 0 to p->nIndex-1) */ + int iLevel, /* Level of segments to scan */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + int isScan, /* True to scan from zTerm to EOF */ + Fts3SegReaderCursor *pCsr /* Cursor object to populate */ +){ + assert( iIndex>=0 && iIndexnIndex ); + assert( iLevel==FTS3_SEGCURSOR_ALL + || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel>=0 + ); + assert( iLevelaIndex==0 ); + + memset(pCsr, 0, sizeof(Fts3SegReaderCursor)); + + return fts3SegReaderCursor( + p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr + ); +} + +static int fts3SegReaderCursorAddZero( + Fts3Table *p, + const char *zTerm, + int nTerm, + Fts3SegReaderCursor *pCsr +){ + return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr); +} + static int fts3TermSegReaderCursor( Fts3Cursor *pCsr, /* Virtual table cursor handle */ @@ -2345,12 +2366,23 @@ static int fts3TermSegReaderCursor( Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; if( isPrefix ){ - for(i=1; inIndex; i++){ + for(i=1; bFound==0 && inIndex; i++){ if( p->aIndex[i].nPrefix==nTerm ){ bFound = 1; rc = sqlite3Fts3SegReaderCursor( p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr); - break; + } + } + + for(i=1; bFound==0 && inIndex; i++){ + if( p->aIndex[i].nPrefix==nTerm+1 ){ + bFound = 1; + rc = sqlite3Fts3SegReaderCursor( + p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr + ); + if( rc==SQLITE_OK ){ + rc = fts3SegReaderCursorAddZero(p, zTerm, nTerm, pSegcsr); + } } } } diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 9a136c3a65..3d4e022298 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -422,7 +422,7 @@ int sqlite3Fts3AllSegdirs( assert( iLevel=0 && iIndexnIndex ); - if( iLevel==FTS3_SEGCURSOR_ALL ){ + if( iLevel<0 ){ /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); if( rc==SQLITE_OK ){ diff --git a/manifest b/manifest index d0a1e83625..7cd9a1f2c5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sexperimental\sfts3-prefix-search\sbranch. -D 2011-05-25T18:47:26.259 +C If\sa\sprefix\sindex\sof\ssize\sN\sis\snot\spresent,\suse\sa\sprefix\sindex\sof\ssize\sN+1\salong\swith\sthe\sterms\sindex\sfor\squeries\sfor\sprefixes\sof\slength\sN. +D 2011-05-25T19:17:32.713 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 925360f6c043fb87719e56520470265d69c5f870 +F ext/fts3/fts3.c b44e0abc7aaef8d5489533b3f0556b28097378f9 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h deeeac21d17da06683a79b40ae119b93cf86f90a F ext/fts3/fts3_aux.c 2817a1ec9ffbad673cb1a1527ad807811bc7645b @@ -75,7 +75,7 @@ F ext/fts3/fts3_term.c 0ade1812c0e97f394b58299810dfd5d2fb7ba501 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c 416b367f56f3100314fa76615535178718c645e1 +F ext/fts3/fts3_write.c beaa93bcbe1664fcada75b70893f9b221acf2777 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -473,7 +473,7 @@ F test/fts3fault2.test dc96203af6ba31ce20163fc35460e1556e8edf4d F test/fts3malloc.test 9c8cc3f885bb4dfc66d0460c52f68f45e4710d1b F test/fts3matchinfo.test cc0b009edbbf575283d5fdb53271179e0d8019ba F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 -F test/fts3prefix.test c51b04f38ce232cd0c9edd3cc60dc981db5e2726 +F test/fts3prefix.test 36246609111ec1683f7ea5ed27666ce2cefb5676 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 F test/fts3rnd.test 2b1a579be557ab8ac54a51b39caa4aa8043cc4ad F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P be59bf49402d2e2f4b95fb6668849f3745cb7bf2 d807304a695fc85402b86e1cd32a6e3bbb2823c8 -R 2e09e5d0236bf18c24c777dcd7acf739 +P f0f0a03db214b68a37069f64c27ae8520220c900 +R 95d28d5ab6425c93022acec043aee69b U dan -Z fd8764569a495dba32fc732cd25f7598 +Z 2f1dd28b688335ffa6210dbb33afadf5 diff --git a/manifest.uuid b/manifest.uuid index 00ceff796f..3079457335 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f0f0a03db214b68a37069f64c27ae8520220c900 \ No newline at end of file +cc83991caae7c7d647432d5711b6cd80228c3002 \ No newline at end of file diff --git a/test/fts3prefix.test b/test/fts3prefix.test index 2e0c7a8ede..f5e31f3208 100644 --- a/test/fts3prefix.test +++ b/test/fts3prefix.test @@ -193,5 +193,11 @@ do_execsql_test 4.3 { do_execsql_test 4.4 { SELECT * FROM t3 WHERE t3 MATCH 's*' } {{four five six} {seven eight nine}} +do_execsql_test 4.5 { + SELECT * FROM t3 WHERE t3 MATCH 'sev*' +} {{seven eight nine}} +do_execsql_test 4.6 { + SELECT * FROM t3 WHERE t3 MATCH 'one*' +} {{one two three}} finish_test From 382874fc5c4e79c5e7ea7b26854afd2e5195db2a Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 28 May 2011 15:57:40 +0000 Subject: [PATCH 04/20] Minor changes made while planning a larger change. FossilOrigin-Name: 84097a4c759b1d65890af885f137d3cb16eef584 --- ext/fts3/fts3.c | 71 +++++++++++++---------- ext/fts3/fts3Int.h | 36 ++++++------ ext/fts3/fts3_expr.c | 124 ++++++++++++++++++++++++---------------- ext/fts3/fts3_snippet.c | 18 +++--- ext/fts3/fts3_write.c | 16 +++--- manifest | 22 +++---- manifest.uuid | 2 +- test/hook.test | 28 +++++++++ 8 files changed, 191 insertions(+), 126 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 541d6c539d..536c47fc78 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -2767,11 +2767,13 @@ static int fts3NearMerge( ** each doclist that are not within nNear tokens of a corresponding entry ** in the other doclist. */ -int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){ +int sqlite3Fts3ExprNearTrim(Fts3Expr *pELeft, Fts3Expr *pERight, int nNear){ int rc; /* Return code */ + Fts3Phrase *pLeft = pELeft->pPhrase; + Fts3Phrase *pRight = pERight->pPhrase; - assert( pLeft->eType==FTSQUERY_PHRASE ); - assert( pRight->eType==FTSQUERY_PHRASE ); + assert( pELeft->eType==FTSQUERY_PHRASE && pLeft ); + assert( pERight->eType==FTSQUERY_PHRASE && pRight ); assert( pLeft->isLoaded && pRight->isLoaded ); if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){ @@ -2785,8 +2787,8 @@ int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){ int nOut; /* Size of buffer aOut in bytes */ rc = fts3NearMerge(MERGE_POS_NEAR, nNear, - pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist, - pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist, + pLeft->nToken, pLeft->aDoclist, pLeft->nDoclist, + pRight->nToken, pRight->aDoclist, pRight->nDoclist, &aOut, &nOut ); if( rc!=SQLITE_OK ) return rc; @@ -2795,8 +2797,8 @@ int sqlite3Fts3ExprNearTrim(Fts3Expr *pLeft, Fts3Expr *pRight, int nNear){ pRight->nDoclist = nOut; rc = fts3NearMerge(MERGE_POS_NEAR, nNear, - pRight->pPhrase->nToken, pRight->aDoclist, pRight->nDoclist, - pLeft->pPhrase->nToken, pLeft->aDoclist, pLeft->nDoclist, + pRight->nToken, pRight->aDoclist, pRight->nDoclist, + pLeft->nToken, pLeft->aDoclist, pLeft->nDoclist, &aOut, &nOut ); sqlite3_free(pLeft->aDoclist); @@ -3459,12 +3461,19 @@ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ */ int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){ int rc; - assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); + Fts3Phrase *pPhrase = pExpr->pPhrase; + assert( pExpr->eType==FTSQUERY_PHRASE && pPhrase ); assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); - rc = fts3EvalExpr(pCsr, pExpr, &pExpr->aDoclist, &pExpr->nDoclist, 1); + rc = fts3EvalExpr(pCsr, pExpr, &pPhrase->aDoclist, &pPhrase->nDoclist, 1); return rc; } +/* +** TODO: This is something to do with matchinfo(). Similar to +** sqlite3ExprLoadDoclists() but slightly different. +** +** UPDATE: Only used when there are deferred tokens. +*/ int sqlite3Fts3ExprLoadFtDoclist( Fts3Cursor *pCsr, Fts3Expr *pExpr, @@ -3510,44 +3519,46 @@ char *sqlite3Fts3FindPositions( sqlite3_int64 iDocid, /* Docid associated with requested pos-list */ int iCol /* Column of requested pos-list */ ){ - assert( pExpr->isLoaded ); - if( pExpr->aDoclist ){ - char *pEnd = &pExpr->aDoclist[pExpr->nDoclist]; + Fts3Phrase *pPhrase = pExpr->pPhrase; + assert( pPhrase->isLoaded ); + + if( pPhrase->aDoclist ){ + char *pEnd = &pPhrase->aDoclist[pPhrase->nDoclist]; char *pCsr; - if( pExpr->pCurrent==0 ){ + if( pPhrase->pCurrent==0 ){ if( pCursor->desc==0 ){ - pExpr->pCurrent = pExpr->aDoclist; - pExpr->iCurrent = 0; - fts3GetDeltaVarint(&pExpr->pCurrent, &pExpr->iCurrent); + pPhrase->pCurrent = pPhrase->aDoclist; + pPhrase->iCurrent = 0; + fts3GetDeltaVarint(&pPhrase->pCurrent, &pPhrase->iCurrent); }else{ - pCsr = pExpr->aDoclist; + pCsr = pPhrase->aDoclist; while( pCsriCurrent); + fts3GetDeltaVarint(&pCsr, &pPhrase->iCurrent); fts3PoslistCopy(0, &pCsr); } - fts3ReversePoslist(pExpr->aDoclist, &pCsr); - pExpr->pCurrent = pCsr; + fts3ReversePoslist(pPhrase->aDoclist, &pCsr); + pPhrase->pCurrent = pCsr; } } - pCsr = pExpr->pCurrent; + pCsr = pPhrase->pCurrent; assert( pCsr ); while( (pCursor->desc==0 && pCsrdesc && pCsr>pExpr->aDoclist) + || (pCursor->desc && pCsr>pPhrase->aDoclist) ){ - if( pCursor->desc==0 && pExpr->iCurrentdesc==0 && pPhrase->iCurrentiCurrent); + fts3GetDeltaVarint(&pCsr, &pPhrase->iCurrent); } - pExpr->pCurrent = pCsr; - }else if( pCursor->desc && pExpr->iCurrent>iDocid ){ - fts3GetReverseDeltaVarint(&pCsr, pExpr->aDoclist, &pExpr->iCurrent); - fts3ReversePoslist(pExpr->aDoclist, &pCsr); - pExpr->pCurrent = pCsr; + pPhrase->pCurrent = pCsr; + }else if( pCursor->desc && pPhrase->iCurrent>iDocid ){ + fts3GetReverseDeltaVarint(&pCsr, pPhrase->aDoclist, &pPhrase->iCurrent); + fts3ReversePoslist(pPhrase->aDoclist, &pCsr); + pPhrase->pCurrent = pCsr; }else{ - if( pExpr->iCurrent==iDocid ){ + if( pPhrase->iCurrent==iDocid ){ int iThis = 0; if( iCol<0 ){ /* If iCol is negative, return a pointer to the start of the diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 00e1c37d2b..42be10773a 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -268,17 +268,16 @@ struct Fts3Cursor { ** sequence. A single token is the base case and the most common case. ** For a sequence of tokens contained in double-quotes (i.e. "one two three") ** nToken will be the number of tokens in the string. -** -** The nDocMatch and nMatch variables contain data that may be used by the -** matchinfo() function. They are populated when the full-text index is -** queried for hits on the phrase. If one or more tokens in the phrase -** are deferred, the nDocMatch and nMatch variables are populated based -** on the assumption that the */ struct Fts3PhraseToken { char *z; /* Text of the token */ int n; /* Number of bytes in buffer z */ int isPrefix; /* True if token ends with a "*" character */ + + /* Variables above this point are populated when the expression is + ** parsed (by code in fts3_expr.c). Below this point the variables are + ** used when evaluating the expression. */ + int bFulltext; /* True if full-text index was used */ Fts3SegReaderCursor *pSegcsr; /* Segment-reader for this token */ Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ @@ -288,18 +287,24 @@ struct Fts3Phrase { /* Variables populated by fts3_expr.c when parsing a MATCH expression */ int nToken; /* Number of tokens in the phrase */ int iColumn; /* Index of column this phrase must match */ - int isNot; /* Phrase prefixed by unary not (-) operator */ + + int isLoaded; /* True if aDoclist/nDoclist are initialized. */ + char *aDoclist; /* Buffer containing doclist */ + int nDoclist; /* Size of aDoclist in bytes */ + sqlite3_int64 iCurrent; + char *pCurrent; + Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ }; /* ** A tree of these objects forms the RHS of a MATCH operator. ** -** If Fts3Expr.eType is either FTSQUERY_NEAR or FTSQUERY_PHRASE and isLoaded -** is true, then aDoclist points to a malloced buffer, size nDoclist bytes, -** containing the results of the NEAR or phrase query in FTS3 doclist -** format. As usual, the initial "Length" field found in doclists stored -** on disk is omitted from this buffer. +** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist +** points to a malloced buffer, size nDoclist bytes, containing the results +** of this phrase query in FTS3 doclist format. As usual, the initial +** "Length" field found in doclists stored on disk is omitted from this +** buffer. ** ** Variable pCurrent always points to the start of a docid field within ** aDoclist. Since the doclist is usually scanned in docid order, this can @@ -312,13 +317,6 @@ struct Fts3Expr { Fts3Expr *pLeft; /* Left operand */ Fts3Expr *pRight; /* Right operand */ Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ - - int isLoaded; /* True if aDoclist/nDoclist are initialized. */ - char *aDoclist; /* Buffer containing doclist */ - int nDoclist; /* Size of aDoclist in bytes */ - - sqlite3_int64 iCurrent; - char *pCurrent; }; /* diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 43f6d84a84..095841d142 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -81,12 +81,21 @@ int sqlite3_fts3_enable_parentheses = 0; #include #include +/* +** isNot: +** This variable is used by function getNextNode(). When getNextNode() is +** called, it sets ParseContext.isNot to true if the 'next node' is a +** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the +** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to +** zero. +*/ typedef struct ParseContext ParseContext; struct ParseContext { sqlite3_tokenizer *pTokenizer; /* Tokenizer module */ const char **azCol; /* Array of column names for fts3 table */ int nCol; /* Number of entries in azCol[] */ int iDefaultCol; /* Default column to query */ + int isNot; /* True if getNextNode() sees a unary - */ sqlite3_context *pCtx; /* Write error message here */ int nNest; /* Number of nested brackets */ }; @@ -172,7 +181,7 @@ static int getNextToken( iEnd++; } if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){ - pRet->pPhrase->isNot = 1; + pParse->isNot = 1; } } nConsumed = iEnd; @@ -224,36 +233,55 @@ static int getNextString( char *zTemp = 0; int nTemp = 0; + const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase); + int nToken = 0; + + /* The final Fts3Expr data structure, including the Fts3Phrase, + ** Fts3PhraseToken structures token buffers are all stored as a single + ** allocation so that the expression can be freed with a single call to + ** sqlite3_free(). Setting this up requires a two pass approach. + ** + ** The first pass, in the block below, uses a tokenizer cursor to iterate + ** through the tokens in the expression. This pass uses fts3ReallocOrFree() + ** to assemble data in two dynamic buffers: + ** + ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase + ** structure, followed by the array of Fts3PhraseToken + ** structures. This pass only populates the Fts3PhraseToken array. + ** + ** Buffer zTemp: Contains copies of all tokens. + ** + ** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below, + ** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase + ** structures. + */ rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor); if( rc==SQLITE_OK ){ int ii; pCursor->pTokenizer = pTokenizer; for(ii=0; rc==SQLITE_OK; ii++){ - const char *zToken; - int nToken, iBegin, iEnd, iPos; - rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos); + const char *zByte; + int nByte, iBegin, iEnd, iPos; + rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos); if( rc==SQLITE_OK ){ - int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase); - p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken)); - zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken); - if( !p || !zTemp ){ - goto no_mem; - } - if( ii==0 ){ - memset(p, 0, nByte); - p->pPhrase = (Fts3Phrase *)&p[1]; - } - p->pPhrase = (Fts3Phrase *)&p[1]; - memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken)); - p->pPhrase->nToken = ii+1; - p->pPhrase->aToken[ii].n = nToken; - memcpy(&zTemp[nTemp], zToken, nToken); - nTemp += nToken; - if( iEndpPhrase->aToken[ii].isPrefix = 1; - }else{ - p->pPhrase->aToken[ii].isPrefix = 0; - } + Fts3PhraseToken *pToken; + + p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken)); + if( !p ) goto no_mem; + + zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte); + if( !zTemp ) goto no_mem; + + assert( nToken==ii ); + pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii]; + memset(pToken, 0, sizeof(Fts3PhraseToken)); + + memcpy(&zTemp[nTemp], zByte, nByte); + nTemp += nByte; + + pToken->n = nByte; + pToken->isPrefix = (iEndpPhrase->nToken-1):0) * sizeof(Fts3PhraseToken); - p = fts3ReallocOrFree(p, nByte + nTemp); - if( !p ){ - goto no_mem; - } - if( zTemp ){ - zNew = &(((char *)p)[nByte]); - memcpy(zNew, zTemp, nTemp); - }else{ - memset(p, 0, nByte+nTemp); - } - p->pPhrase = (Fts3Phrase *)&p[1]; - for(jj=0; jjpPhrase->nToken; jj++){ - p->pPhrase->aToken[jj].z = &zNew[nNew]; - nNew += p->pPhrase->aToken[jj].n; - } - sqlite3_free(zTemp); + char *zBuf = 0; + + p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp); + if( !p ) goto no_mem; + memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p); p->eType = FTSQUERY_PHRASE; + p->pPhrase = (Fts3Phrase *)&p[1]; p->pPhrase->iColumn = pParse->iDefaultCol; + p->pPhrase->nToken = nToken; + + zBuf = (char *)&p->pPhrase->aToken[nToken]; + memcpy(zBuf, zTemp, nTemp); + sqlite3_free(zTemp); + + for(jj=0; jjpPhrase->nToken; jj++){ + p->pPhrase->aToken[jj].z = zBuf; + zBuf += p->pPhrase->aToken[jj].n; + } rc = SQLITE_OK; } @@ -341,6 +365,8 @@ static int getNextNode( const char *zInput = z; int nInput = n; + pParse->isNot = 0; + /* Skip over any whitespace before checking for a keyword, an open or ** close bracket, or a quoted string. */ @@ -559,7 +585,7 @@ static int fts3ExprParse( int isPhrase; if( !sqlite3_fts3_enable_parentheses - && p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot + && p->eType==FTSQUERY_PHRASE && pParse->isNot ){ /* Create an implicit NOT operator. */ Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); @@ -577,7 +603,6 @@ static int fts3ExprParse( p = pPrev; }else{ int eType = p->eType; - assert( eType!=FTSQUERY_PHRASE || !p->pPhrase->isNot ); isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft); /* The isRequirePhrase variable is set to true if a phrase or @@ -740,9 +765,10 @@ int sqlite3Fts3ExprParse( */ void sqlite3Fts3ExprFree(Fts3Expr *p){ if( p ){ + assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); sqlite3Fts3ExprFree(p->pLeft); sqlite3Fts3ExprFree(p->pRight); - sqlite3_free(p->aDoclist); + if( p->pPhrase ) sqlite3_free(p->pPhrase->aDoclist); sqlite3_free(p); } } @@ -800,7 +826,7 @@ static char *exprToString(Fts3Expr *pExpr, char *zBuf){ Fts3Phrase *pPhrase = pExpr->pPhrase; int i; zBuf = sqlite3_mprintf( - "%zPHRASE %d %d", zBuf, pPhrase->iColumn, pPhrase->isNot); + "%zPHRASE %d 0", zBuf, pPhrase->iColumn); for(i=0; zBuf && inToken; i++){ zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, pPhrase->aToken[i].n, pPhrase->aToken[i].z, diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 8bf9ed2d5a..b31a1cf359 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -228,16 +228,17 @@ static int fts3ExprNearTrim(Fts3Expr *pExpr){ */ static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ int rc = SQLITE_OK; + Fts3Phrase *pPhrase = pExpr->pPhrase; LoadDoclistCtx *p = (LoadDoclistCtx *)ctx; UNUSED_PARAMETER(iPhrase); p->nPhrase++; - p->nToken += pExpr->pPhrase->nToken; + p->nToken += pPhrase->nToken; - if( pExpr->isLoaded==0 ){ + if( pPhrase->isLoaded==0 ){ rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr); - pExpr->isLoaded = 1; + pPhrase->isLoaded = 1; if( rc==SQLITE_OK ){ rc = fts3ExprNearTrim(pExpr); } @@ -826,16 +827,15 @@ static int fts3ExprGlobalHitsCb( ){ MatchInfo *p = (MatchInfo *)pCtx; Fts3Cursor *pCsr = p->pCursor; + Fts3Phrase *pPhrase = pExpr->pPhrase; char *pIter; char *pEnd; char *pFree = 0; u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol]; - assert( pExpr->isLoaded ); - assert( pExpr->eType==FTSQUERY_PHRASE ); + assert( pPhrase->isLoaded ); if( pCsr->pDeferred ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; int ii; for(ii=0; iinToken; ii++){ if( pPhrase->aToken[ii].bFulltext ) break; @@ -855,8 +855,8 @@ static int fts3ExprGlobalHitsCb( return SQLITE_OK; } }else{ - pIter = pExpr->aDoclist; - pEnd = &pExpr->aDoclist[pExpr->nDoclist]; + pIter = pPhrase->aDoclist; + pEnd = &pPhrase->aDoclist[pPhrase->nDoclist]; } /* Fill in the global hit count matrix row for this phrase. */ @@ -885,7 +885,7 @@ static int fts3ExprLocalHitsCb( for(i=0; inCol; i++) p->aMatchinfo[iStart+i*3] = 0; - if( pExpr->aDoclist ){ + if( pExpr->pPhrase->aDoclist ){ char *pCsr; pCsr = sqlite3Fts3FindPositions(p->pCursor, pExpr, p->pCursor->iPrevId, -1); diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 3d4e022298..f664dec8cf 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -2642,15 +2642,17 @@ char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){ */ static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ if( pExpr ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; fts3DeferredDoclistClear(pExpr->pLeft); fts3DeferredDoclistClear(pExpr->pRight); - if( pExpr->isLoaded ){ - sqlite3_free(pExpr->aDoclist); - pExpr->isLoaded = 0; - pExpr->aDoclist = 0; - pExpr->nDoclist = 0; - pExpr->pCurrent = 0; - pExpr->iCurrent = 0; + if( pPhrase ){ + assert( pExpr->eType==FTSQUERY_PHRASE ); + sqlite3_free(pPhrase->aDoclist); + pPhrase->isLoaded = 0; + pPhrase->aDoclist = 0; + pPhrase->nDoclist = 0; + pPhrase->pCurrent = 0; + pPhrase->iCurrent = 0; } } } diff --git a/manifest b/manifest index 7cd9a1f2c5..950ee87f2e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sa\sprefix\sindex\sof\ssize\sN\sis\snot\spresent,\suse\sa\sprefix\sindex\sof\ssize\sN+1\salong\swith\sthe\sterms\sindex\sfor\squeries\sfor\sprefixes\sof\slength\sN. -D 2011-05-25T19:17:32.713 +C Minor\schanges\smade\swhile\splanning\sa\slarger\schange. +D 2011-05-28T15:57:40.694 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,21 +61,21 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c b44e0abc7aaef8d5489533b3f0556b28097378f9 +F ext/fts3/fts3.c eb59cdd89e9309ab9b2dca196a7c52f9e8927319 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h deeeac21d17da06683a79b40ae119b93cf86f90a +F ext/fts3/fts3Int.h 832f4d421f03a9d364186e779ee51994df500c62 F ext/fts3/fts3_aux.c 2817a1ec9ffbad673cb1a1527ad807811bc7645b -F ext/fts3/fts3_expr.c 5f49e0deaf723724b08100bb3ff40aab02ad0c93 +F ext/fts3/fts3_expr.c 25e30cf24198333f2ed545af905b168e88f56903 F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2 -F ext/fts3/fts3_snippet.c 92b40397b28422c35c4127492d7ac6da34d1966a +F ext/fts3/fts3_snippet.c 6ee626017ddcf7d72ca4f587724e3506434fc0d7 F ext/fts3/fts3_term.c 0ade1812c0e97f394b58299810dfd5d2fb7ba501 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c beaa93bcbe1664fcada75b70893f9b221acf2777 +F ext/fts3/fts3_write.c ed58c53fbcbc2ea79258e734159a5951ffeb1bd4 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -489,7 +489,7 @@ F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5 F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b F test/fuzz_malloc.test dd7001ac86d09c154a7dff064f4739c60e2b312c F test/fuzzer1.test 3105b5a89a6cb0d475f0877debec942fe4143462 -F test/hook.test f04c3412463f8ec117c1c704c74ca0f627ce733a +F test/hook.test f2277c309e4ee8067d95d6b9b315568e9d5329b2 F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 19b642bb134308980a92249750ea4ce3f6c75c2d F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P f0f0a03db214b68a37069f64c27ae8520220c900 -R 95d28d5ab6425c93022acec043aee69b +P cc83991caae7c7d647432d5711b6cd80228c3002 +R eef52aec1faead4921ebaf39c1605d2b U dan -Z 2f1dd28b688335ffa6210dbb33afadf5 +Z 01d8f7aea2ce8ebb35c05ccc519b614f diff --git a/manifest.uuid b/manifest.uuid index 3079457335..6768a6838c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cc83991caae7c7d647432d5711b6cd80228c3002 \ No newline at end of file +84097a4c759b1d65890af885f137d3cb16eef584 \ No newline at end of file diff --git a/test/hook.test b/test/hook.test index 6496d41e13..c4dfb2e4f9 100644 --- a/test/hook.test +++ b/test/hook.test @@ -274,6 +274,34 @@ ifcapable compound&&attach { set ::update_hook } [list] } + +do_test hook-4.4 { + execsql { + CREATE TABLE t4(a UNIQUE, b); + INSERT INTO t4 VALUES(1, 'a'); + INSERT INTO t4 VALUES(2, 'b'); + } + set ::update_hook [list] + execsql { + REPLACE INTO t4 VALUES(1, 'c'); + } + set ::update_hook +} [list INSERT main t4 3 ] +do_execsql_test hook-4.4.1 { + SELECT * FROM t4 ORDER BY a; +} {1 c 2 b} +do_test hook-4.4.2 { + set ::update_hook [list] + execsql { + PRAGMA recursive_triggers = on; + REPLACE INTO t4 VALUES(1, 'd'); + } + set ::update_hook +} [list INSERT main t4 4 ] +do_execsql_test hook-4.4.3 { + SELECT * FROM t4 ORDER BY a; +} {1 d 2 b} + db update_hook {} # #---------------------------------------------------------------------------- From e414854800a7d87c2a207774b5d65007e3592eb9 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 2 Jun 2011 19:57:24 +0000 Subject: [PATCH 05/20] Changes to improve performance and support LIMIT clauses on fts3 tables. This branch is unstable for now. FossilOrigin-Name: 28149a7882a1e9dfe4a75ec5b91d176ebe6284e9 --- ext/fts3/fts3.c | 1120 +++++++++++++++++++++++++++++++++------ ext/fts3/fts3Int.h | 86 ++- ext/fts3/fts3_aux.c | 2 +- ext/fts3/fts3_expr.c | 5 +- ext/fts3/fts3_snippet.c | 110 ++-- ext/fts3/fts3_term.c | 2 +- ext/fts3/fts3_write.c | 170 +++++- manifest | 28 +- manifest.uuid | 2 +- test/fts3defer.test | 29 +- test/permutations.test | 1 + 11 files changed, 1274 insertions(+), 281 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 536c47fc78..a7f176a700 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -313,6 +313,9 @@ SQLITE_EXTENSION_INIT1 #endif +static char *fts3EvalPhrasePoslist(Fts3Phrase *, int *); +static sqlite3_int64 fts3EvalPhraseDocid(Fts3Phrase *); + /* ** Write a 64-bit variable-length integer to memory starting at p[0]. ** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. @@ -1207,14 +1210,16 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ */ if( pInfo->nOrderBy==1 ){ struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; - if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){ + if( pOrder->desc==0 + && (pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1) + ){ if( pOrder->desc ){ pInfo->idxStr = "DESC"; }else{ pInfo->idxStr = "ASC"; } + pInfo->orderByConsumed = 1; } - pInfo->orderByConsumed = 1; } return SQLITE_OK; @@ -1256,6 +1261,8 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ return SQLITE_OK; } +static int fts3RowidMethod(sqlite3_vtab_cursor *, sqlite3_int64*); + /* ** Position the pCsr->pStmt statement so that it is on the row ** of the %_content table that contains the last match. Return @@ -1263,8 +1270,8 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ */ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ if( pCsr->isRequireSeek ){ - pCsr->isRequireSeek = 0; sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); + pCsr->isRequireSeek = 0; if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ return SQLITE_OK; }else{ @@ -2220,7 +2227,7 @@ static int fts3DeferredTermSelect( ** Append SegReader object pNew to the end of the pCsr->apSegment[] array. */ static int fts3SegReaderCursorAppend( - Fts3SegReaderCursor *pCsr, + Fts3MultiSegReader *pCsr, Fts3SegReader *pNew ){ if( (pCsr->nSegment%16)==0 ){ @@ -2245,7 +2252,7 @@ static int fts3SegReaderCursor( int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ int isScan, /* True to scan from zTerm to EOF */ - Fts3SegReaderCursor *pCsr /* Cursor object to populate */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ ){ int rc = SQLITE_OK; int rc2; @@ -2316,7 +2323,7 @@ int sqlite3Fts3SegReaderCursor( int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ int isScan, /* True to scan from zTerm to EOF */ - Fts3SegReaderCursor *pCsr /* Cursor object to populate */ + Fts3MultiSegReader *pCsr /* Cursor object to populate */ ){ assert( iIndex>=0 && iIndexnIndex ); assert( iLevel==FTS3_SEGCURSOR_ALL @@ -2331,7 +2338,7 @@ int sqlite3Fts3SegReaderCursor( ** full-text tables. */ assert( isScan==0 || p->aIndex==0 ); - memset(pCsr, 0, sizeof(Fts3SegReaderCursor)); + memset(pCsr, 0, sizeof(Fts3MultiSegReader)); return fts3SegReaderCursor( p, iIndex, iLevel, zTerm, nTerm, isPrefix, isScan, pCsr @@ -2342,23 +2349,23 @@ static int fts3SegReaderCursorAddZero( Fts3Table *p, const char *zTerm, int nTerm, - Fts3SegReaderCursor *pCsr + Fts3MultiSegReader *pCsr ){ return fts3SegReaderCursor(p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0,pCsr); } -static int fts3TermSegReaderCursor( +int sqlite3Fts3TermSegReaderCursor( Fts3Cursor *pCsr, /* Virtual table cursor handle */ const char *zTerm, /* Term to query for */ int nTerm, /* Size of zTerm in bytes */ int isPrefix, /* True for a prefix search */ - Fts3SegReaderCursor **ppSegcsr /* OUT: Allocated seg-reader cursor */ + Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ ){ - Fts3SegReaderCursor *pSegcsr; /* Object to allocate and return */ + Fts3MultiSegReader *pSegcsr; /* Object to allocate and return */ int rc = SQLITE_NOMEM; /* Return code */ - pSegcsr = sqlite3_malloc(sizeof(Fts3SegReaderCursor)); + pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader)); if( pSegcsr ){ int i; int nCost = 0; @@ -2371,6 +2378,7 @@ static int fts3TermSegReaderCursor( bFound = 1; rc = sqlite3Fts3SegReaderCursor( p, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr); + pSegcsr->bLookup = 1; } } @@ -2391,6 +2399,7 @@ static int fts3TermSegReaderCursor( rc = sqlite3Fts3SegReaderCursor( p, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr ); + pSegcsr->bLookup = !isPrefix; } for(i=0; rc==SQLITE_OK && inSegment; i++){ rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost); @@ -2402,7 +2411,7 @@ static int fts3TermSegReaderCursor( return rc; } -static void fts3SegReaderCursorFree(Fts3SegReaderCursor *pSegcsr){ +static void fts3SegReaderCursorFree(Fts3MultiSegReader *pSegcsr){ sqlite3Fts3SegReaderFinish(pSegcsr); sqlite3_free(pSegcsr); } @@ -2427,7 +2436,7 @@ static int fts3TermSelect( char **ppOut /* OUT: Malloced result buffer */ ){ int rc; /* Return code */ - Fts3SegReaderCursor *pSegcsr; /* Seg-reader cursor for this term */ + Fts3MultiSegReader *pSegcsr; /* Seg-reader cursor for this term */ TermSelect tsc; /* Context object for fts3TermSelectCb() */ Fts3SegFilter filter; /* Segment term filter configuration */ @@ -2554,7 +2563,7 @@ static void fts3DoclistStripPositions( } } -/* +/* ** Return a DocList corresponding to the phrase *pPhrase. ** ** If this function returns SQLITE_OK, but *pnOut is set to a negative value, @@ -2582,26 +2591,6 @@ static int fts3PhraseSelect( int iPrevTok = 0; int nDoc = 0; - /* If this is an xFilter() evaluation, create a segment-reader for each - ** phrase token. Or, if this is an xNext() or snippet/offsets/matchinfo - ** evaluation, only create segment-readers if there are no Fts3DeferredToken - ** objects attached to the phrase-tokens. - */ - for(ii=0; iinToken; ii++){ - Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; - if( pTok->pSegcsr==0 ){ - if( (pCsr->eEvalmode==FTS3_EVAL_FILTER) - || (pCsr->eEvalmode==FTS3_EVAL_NEXT && pCsr->pDeferred==0) - || (pCsr->eEvalmode==FTS3_EVAL_MATCHINFO && pTok->bFulltext) - ){ - rc = fts3TermSegReaderCursor( - pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr - ); - if( rc!=SQLITE_OK ) return rc; - } - } - } - for(ii=0; iinToken; ii++){ Fts3PhraseToken *pTok; /* Token to find doclist for */ int iTok = 0; /* The token being queried this iteration */ @@ -2626,7 +2615,7 @@ static int fts3PhraseSelect( /* Find the remaining token with the lowest cost. */ for(jj=0; jjnToken; jj++){ - Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[jj].pSegcsr; + Fts3MultiSegReader *pSegcsr = pPhrase->aToken[jj].pSegcsr; if( pSegcsr && pSegcsr->nCostnCost; @@ -2828,13 +2817,13 @@ static int fts3ExprAllocateSegReaders( } if( pExpr->eType==FTSQUERY_PHRASE ){ + int ii; /* Used to iterate through phrase tokens */ Fts3Phrase *pPhrase = pExpr->pPhrase; - int ii; for(ii=0; rc==SQLITE_OK && iinToken; ii++){ Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; if( pTok->pSegcsr==0 ){ - rc = fts3TermSegReaderCursor( + rc = sqlite3Fts3TermSegReaderCursor( pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr ); } @@ -2880,7 +2869,7 @@ static int fts3ExprCost(Fts3Expr *pExpr){ int ii; nCost = 0; for(ii=0; iinToken; ii++){ - Fts3SegReaderCursor *pSegcsr = pPhrase->aToken[ii].pSegcsr; + Fts3MultiSegReader *pSegcsr = pPhrase->aToken[ii].pSegcsr; if( pSegcsr ) nCost += pSegcsr->nCost; } }else{ @@ -3175,34 +3164,38 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ int rc = SQLITE_OK; /* Return code */ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - pCsr->eEvalmode = FTS3_EVAL_NEXT; - do { - if( pCsr->aDoclist==0 ){ - if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ - pCsr->isEof = 1; - rc = sqlite3_reset(pCsr->pStmt); - break; - } - pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); - }else{ - if( pCsr->desc==0 ){ - if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ + if( pCsr->bIncremental ){ + rc = sqlite3Fts3EvalNext(pCsr, pCsr->pExpr); + }else{ + pCsr->eEvalmode = FTS3_EVAL_NEXT; + do { + if( pCsr->aDoclist==0 ){ + if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ pCsr->isEof = 1; + rc = sqlite3_reset(pCsr->pStmt); break; } - fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); + pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); }else{ - fts3GetReverseDeltaVarint(&pCsr->pNextId,pCsr->aDoclist,&pCsr->iPrevId); - if( pCsr->pNextId<=pCsr->aDoclist ){ - pCsr->isEof = 1; - break; + if( pCsr->desc==0 ){ + if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ + pCsr->isEof = 1; + break; + } + fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); + }else{ + fts3GetReverseDeltaVarint(&pCsr->pNextId,pCsr->aDoclist,&pCsr->iPrevId); + if( pCsr->pNextId<=pCsr->aDoclist ){ + pCsr->isEof = 1; + break; + } } + sqlite3_reset(pCsr->pStmt); + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; } - sqlite3_reset(pCsr->pStmt); - pCsr->isRequireSeek = 1; - pCsr->isMatchinfoNeeded = 1; - } - }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 ); + }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 ); + } return rc; } @@ -3230,11 +3223,7 @@ static int fts3FilterMethod( int nVal, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ - const char *azSql[] = { - "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?", /* non-full-scan */ - "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s", /* full-scan */ - }; - int rc; /* Return code */ + int rc; char *zSql; /* SQL statement used to access %_content */ Fts3Table *p = (Fts3Table *)pCursor->pVtab; Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; @@ -3266,8 +3255,8 @@ static int fts3FilterMethod( ); if( rc!=SQLITE_OK ){ if( rc==SQLITE_ERROR ){ - p->base.zErrMsg = sqlite3_mprintf("malformed MATCH expression: [%s]", - zQuery); + static const char *zErr = "malformed MATCH expression: [%s]"; + p->base.zErrMsg = sqlite3_mprintf(zErr, zQuery); } return rc; } @@ -3275,7 +3264,9 @@ static int fts3FilterMethod( rc = sqlite3Fts3ReadLock(p); if( rc!=SQLITE_OK ) return rc; - rc = fts3EvalExpr(pCsr, pCsr->pExpr, &pCsr->aDoclist, &pCsr->nDoclist, 0); + pCsr->bIncremental = 1; + rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1); + sqlite3Fts3SegmentsClose(p); if( rc!=SQLITE_OK ) return rc; pCsr->pNextId = pCsr->aDoclist; @@ -3287,23 +3278,26 @@ static int fts3FilterMethod( ** full-text query or docid lookup, the statement retrieves a single ** row by docid. */ - zSql = (char *)azSql[idxNum==FTS3_FULLSCAN_SEARCH]; - zSql = sqlite3_mprintf( - zSql, p->zReadExprlist, p->zDb, p->zName, (idxStr ? idxStr : "ASC") - ); - if( !zSql ){ - rc = SQLITE_NOMEM; + if( idxNum==FTS3_FULLSCAN_SEARCH ){ + const char *zSort = (idxStr ? idxStr : "ASC"); + const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s"; + zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort); }else{ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); + const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x WHERE docid = ?"; + zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName); } - if( rc==SQLITE_OK && idxNum==FTS3_DOCID_SEARCH ){ + if( !zSql ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + if( rc!=SQLITE_OK ) return rc; + + if( idxNum==FTS3_DOCID_SEARCH ){ rc = sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); + if( rc!=SQLITE_OK ) return rc; } - pCsr->eSearch = (i16)idxNum; assert( pCsr->desc==0 ); - if( rc!=SQLITE_OK ) return rc; + pCsr->eSearch = (i16)idxNum; if( rc==SQLITE_OK && pCsr->nDoclist>0 && idxStr && idxStr[0]=='D' ){ sqlite3_int64 iDocid = 0; char *csr = pCsr->aDoclist; @@ -3337,7 +3331,9 @@ static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ */ static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; - if( pCsr->aDoclist ){ + if( pCsr->bIncremental ){ + *pRowid = sqlite3Fts3EvalDocid(pCsr, pCsr->pExpr); + }else if( pCsr->aDoclist ){ *pRowid = pCsr->iPrevId; }else{ /* This branch runs if the query is implemented using a full-table scan @@ -3460,11 +3456,13 @@ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ ** functions. */ int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){ - int rc; - Fts3Phrase *pPhrase = pExpr->pPhrase; - assert( pExpr->eType==FTSQUERY_PHRASE && pPhrase ); - assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); - rc = fts3EvalExpr(pCsr, pExpr, &pPhrase->aDoclist, &pPhrase->nDoclist, 1); + int rc = SQLITE_OK; + if( pCsr->bIncremental==0 ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + assert( pExpr->eType==FTSQUERY_PHRASE && pPhrase ); + assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); + rc = fts3EvalExpr(pCsr, pExpr, &pPhrase->aDoclist, &pPhrase->nDoclist, 1); + } return rc; } @@ -3480,9 +3478,10 @@ int sqlite3Fts3ExprLoadFtDoclist( char **paDoclist, int *pnDoclist ){ - int rc; - assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); + int rc = SQLITE_OK; assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); + assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); + assert( pCsr->bIncremental==0 ); pCsr->eEvalmode = FTS3_EVAL_MATCHINFO; rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1); pCsr->eEvalmode = FTS3_EVAL_NEXT; @@ -3507,82 +3506,6 @@ static void fts3ReversePoslist(char *pStart, char **ppPoslist){ *ppPoslist = p; } - -/* -** After ExprLoadDoclist() (see above) has been called, this function is -** used to iterate/search through the position lists that make up the doclist -** stored in pExpr->aDoclist. -*/ -char *sqlite3Fts3FindPositions( - Fts3Cursor *pCursor, /* Associate FTS3 cursor */ - Fts3Expr *pExpr, /* Access this expressions doclist */ - sqlite3_int64 iDocid, /* Docid associated with requested pos-list */ - int iCol /* Column of requested pos-list */ -){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - assert( pPhrase->isLoaded ); - - if( pPhrase->aDoclist ){ - char *pEnd = &pPhrase->aDoclist[pPhrase->nDoclist]; - char *pCsr; - - if( pPhrase->pCurrent==0 ){ - if( pCursor->desc==0 ){ - pPhrase->pCurrent = pPhrase->aDoclist; - pPhrase->iCurrent = 0; - fts3GetDeltaVarint(&pPhrase->pCurrent, &pPhrase->iCurrent); - }else{ - pCsr = pPhrase->aDoclist; - while( pCsriCurrent); - fts3PoslistCopy(0, &pCsr); - } - fts3ReversePoslist(pPhrase->aDoclist, &pCsr); - pPhrase->pCurrent = pCsr; - } - } - pCsr = pPhrase->pCurrent; - assert( pCsr ); - - while( (pCursor->desc==0 && pCsrdesc && pCsr>pPhrase->aDoclist) - ){ - if( pCursor->desc==0 && pPhrase->iCurrentiCurrent); - } - pPhrase->pCurrent = pCsr; - }else if( pCursor->desc && pPhrase->iCurrent>iDocid ){ - fts3GetReverseDeltaVarint(&pCsr, pPhrase->aDoclist, &pPhrase->iCurrent); - fts3ReversePoslist(pPhrase->aDoclist, &pCsr); - pPhrase->pCurrent = pCsr; - }else{ - if( pPhrase->iCurrent==iDocid ){ - int iThis = 0; - if( iCol<0 ){ - /* If iCol is negative, return a pointer to the start of the - ** position-list (instead of a pointer to the start of a list - ** of offsets associated with a specific column). - */ - return pCsr; - } - while( iThiseType==FTSQUERY_PHRASE ){ + int i; + int nToken = pExpr->pPhrase->nToken; + *pnToken += nToken; + for(i=0; ipPhrase->aToken[i]; + int rc = sqlite3Fts3TermSegReaderCursor(pCsr, + pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr + ); + if( rc!=SQLITE_OK ){ + *pRc = rc; + return; + } + } + }else{ + fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pRc); + fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pRc); + } + } +} + +static int fts3EvalPhraseLoad( + Fts3Cursor *pCsr, + Fts3Phrase *p +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int iToken; + int rc = SQLITE_OK; + + char *aDoclist = 0; + int nDoclist = 0; + int iPrev = -1; + + for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &p->aToken[iToken]; + assert( pToken->pSegcsr || pToken->pDeferred ); + + if( pToken->pDeferred==0 ){ + int nThis = 0; + char *pThis = 0; + rc = fts3TermSelect(pTab, pToken, p->iColumn, 1, &nThis, &pThis); + if( rc==SQLITE_OK ){ + if( pThis==0 ){ + sqlite3_free(aDoclist); + aDoclist = 0; + nDoclist = 0; + break; + }else if( aDoclist==0 ){ + aDoclist = pThis; + nDoclist = nThis; + }else{ + assert( iPrev>=0 ); + fts3DoclistMerge(MERGE_POS_PHRASE, iToken-iPrev, + 0, pThis, &nThis, aDoclist, nDoclist, pThis, nThis, 0 + ); + sqlite3_free(aDoclist); + aDoclist = pThis; + nDoclist = nThis; + } + iPrev = iToken; + } + } + } + + if( rc==SQLITE_OK ){ + p->doclist.aAll = aDoclist; + p->doclist.nAll = nDoclist; + }else{ + sqlite3_free(aDoclist); + } + return rc; +} + +static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int iToken; + int rc = SQLITE_OK; + + int nMaxUndeferred = -1; + char *aPoslist = 0; + int nPoslist = 0; + int iPrev = -1; + + for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ + Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; + Fts3DeferredToken *pDeferred = pToken->pDeferred; + + if( pDeferred ){ + char *pList; + int nList; + rc = sqlite3Fts3DeferredTokenList(pDeferred, &pList, &nList); + if( rc!=SQLITE_OK ) return rc; + + if( pList==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + + }else if( aPoslist==0 ){ + aPoslist = pList; + nPoslist = nList; + + }else{ + assert( iPrev>=0 ); + + char *aOut = pList; + char *p1 = aPoslist; + char *p2 = aOut; + + fts3PoslistPhraseMerge(&aOut, iToken-iPrev, 0, 1, &p1, &p2); + sqlite3_free(aPoslist); + aPoslist = pList; + nPoslist = aOut - aPoslist; + if( nPoslist==0 ){ + sqlite3_free(aPoslist); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + return SQLITE_OK; + } + } + iPrev = iToken; + }else{ + nMaxUndeferred = iToken; + } + } + + if( iPrev>=0 ){ + if( nMaxUndeferred<0 ){ + pPhrase->doclist.pList = aPoslist; + pPhrase->doclist.nList = nPoslist; + pPhrase->doclist.iDocid = pCsr->iPrevId; + }else{ + int nDistance; + char *p1; + char *p2; + char *aOut; + + if( nMaxUndeferred>iPrev ){ + p1 = aPoslist; + p2 = pPhrase->doclist.pList; + nDistance = nMaxUndeferred - iPrev; + }else{ + p1 = pPhrase->doclist.pList; + p2 = aPoslist; + nDistance = iPrev - nMaxUndeferred; + } + + aOut = (char *)sqlite3_malloc(nPoslist+8); + if( !aOut ){ + sqlite3_free(aPoslist); + return SQLITE_NOMEM; + } + + pPhrase->doclist.pList = aOut; + if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ + pPhrase->doclist.nList = (aOut - pPhrase->doclist.pList); + sqlite3_free(aPoslist); + }else{ + sqlite3_free(aOut); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + } + } + } + + return SQLITE_OK; +} + + +/* +** The following three functions: +** +** fts3EvalPhraseStart() +** fts3EvalPhraseNext() +** fts3EvalPhraseReset() +** +** May be used with a phrase object after fts3EvalAllocateReaders() has been +** called to iterate through the set of docids that match the phrase. +** +** After a successful call to fts3EvalPhraseNext(), the following two +** functions may be called to access the current docid and position-list. +** +** fts3EvalPhraseDocid() +** fts3EvalPhrasePoslist() +*/ +static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ + int rc; + Fts3Doclist *pList = &p->doclist; + Fts3PhraseToken *pFirst = &p->aToken[0]; + + assert( pList->aAll==0 ); + + if( p->nToken==1 && bOptOk==1 + && pFirst->pSegcsr && pFirst->pSegcsr->bLookup + ){ + /* Use the incremental approach. */ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); + rc = sqlite3Fts3MsrIncrStart( + pTab, pFirst->pSegcsr, iCol, pFirst->z, pFirst->n); + p->bIncr = 1; + + }else{ + + /* Load the full doclist for the phrase into memory. */ + rc = fts3EvalPhraseLoad(pCsr, p); + p->bIncr = 0; + } + + assert( rc!=SQLITE_OK || p->nToken<1 || p->aToken[0].pSegcsr==0 || p->bIncr ); + return rc; +} + +/* +** Attempt to move the phrase iterator to point to the next matching docid. +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. +** +** If there is no "next" entry and no error occurs, then *pbEof is set to +** 1 before returning. Otherwise, if no error occurs and the iterator is +** successfully advanced, *pbEof is set to 0. +*/ +static int fts3EvalPhraseNext(Fts3Cursor *pCsr, Fts3Phrase *p, u8 *pbEof){ + int rc = SQLITE_OK; + + if( p->bIncr ){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + assert( p->nToken==1 ); + rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, + &p->doclist.iDocid, &p->doclist.pList, &p->doclist.nList + ); + if( rc==SQLITE_OK && !p->doclist.pList ){ + *pbEof = 1; + } + }else{ + char *pIter; + Fts3Doclist *pDL = &p->doclist; + + if( pDL->pNextDocid ){ + pIter = pDL->pNextDocid; + }else{ + pIter = pDL->aAll; + } + + if( pIter>=&pDL->aAll[pDL->nAll] ){ + /* We have already reached the end of this doclist. EOF. */ + *pbEof = 1; + }else{ + fts3GetDeltaVarint(&pIter, &pDL->iDocid); + pDL->pList = pIter; + fts3PoslistCopy(0, &pIter); + pDL->nList = (pIter - pDL->pList); + pDL->pNextDocid = pIter; + *pbEof = 0; + } + } + + return rc; +} + +static int fts3EvalPhraseReset(Fts3Cursor *pCsr, Fts3Phrase *p){ + return SQLITE_OK; +} + +static sqlite3_int64 fts3EvalPhraseDocid(Fts3Phrase *p){ + return p->doclist.iDocid; +} + +static char *fts3EvalPhrasePoslist(Fts3Phrase *p, int *pnList){ + if( pnList ){ + *pnList = p->doclist.nList; + } + return p->doclist.pList; +} + +static void fts3EvalStartReaders( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int bOptOk, + int *pRc +){ + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + int i; + int nToken = pExpr->pPhrase->nToken; + for(i=0; ipPhrase->aToken[i].pDeferred==0 ) break; + } + pExpr->bDeferred = (i==nToken); + *pRc = fts3EvalPhraseStart(pCsr, bOptOk, pExpr->pPhrase); + }else{ + if( pExpr->eType==FTSQUERY_NEAR ){ + bOptOk = 0; + } + fts3EvalStartReaders(pCsr, pExpr->pLeft, bOptOk, pRc); + fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc); + pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); + } + } +} + +static void fts3EvalNearMerge( + Fts3Expr *p1, + Fts3Expr *p2, + int nNear, + int *pRc +){ + if( *pRc==SQLITE_OK ){ + int rc; /* Return code */ + Fts3Phrase *pLeft = p1->pPhrase; + Fts3Phrase *pRight = p2->pPhrase; + + assert( p2->eType==FTSQUERY_PHRASE && pLeft ); + assert( p2->eType==FTSQUERY_PHRASE && pRight ); + + if( pLeft->doclist.aAll==0 ){ + sqlite3_free(pRight->doclist.aAll); + pRight->doclist.aAll = 0; + pRight->doclist.nAll = 0; + }else if( pRight->doclist.aAll ){ + char *aOut; /* Buffer in which to assemble new doclist */ + int nOut; /* Size of buffer aOut in bytes */ + + *pRc = fts3NearMerge(MERGE_POS_NEAR, nNear, + pLeft->nToken, pLeft->doclist.aAll, pLeft->doclist.nAll, + pRight->nToken, pRight->doclist.aAll, pRight->doclist.nAll, + &aOut, &nOut + ); + sqlite3_free(pRight->doclist.aAll); + pRight->doclist.aAll = aOut; + pRight->doclist.nAll = nOut; + } + } +} + +static void fts3EvalNearTrim(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ + + if( pExpr && SQLITE_OK==*pRc ){ + if( pExpr->eType==FTSQUERY_NEAR ){ + Fts3Expr *pLeft = pExpr->pLeft; + int nPhrase = 2; + Fts3Expr **aPhrase; + + assert( pLeft ); + assert( pExpr->pRight ); + assert( pExpr->pRight->eType==FTSQUERY_PHRASE ); + + while( pLeft->eType!=FTSQUERY_PHRASE ){ + assert( pLeft->eType==FTSQUERY_NEAR ); + assert( pLeft->pRight->eType==FTSQUERY_PHRASE ); + pLeft = pLeft->pLeft; + nPhrase++; + } + + aPhrase = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nPhrase); + if( !aPhrase ){ + *pRc = SQLITE_NOMEM; + }else{ + int i = 1; + aPhrase[0] = pLeft; + do { + pLeft = pLeft->pParent; + aPhrase[i++] = pLeft->pRight; + }while( pLeft!=pExpr ); + + for(i=0; i<(nPhrase-1); i++){ + int nNear = aPhrase[i+1]->pParent->nNear; + fts3EvalNearMerge(aPhrase[i], aPhrase[i+1], nNear, pRc); + } + for(i=nPhrase-2; i>=0; i--){ + int nNear = aPhrase[i+1]->pParent->nNear; + fts3EvalNearMerge(aPhrase[i+1], aPhrase[i], nNear, pRc); + } + + sqlite3_free(aPhrase); + } + + }else{ + fts3EvalNearTrim(pCsr, pExpr->pLeft, pRc); + fts3EvalNearTrim(pCsr, pExpr->pRight, pRc); + } + } +} + +typedef struct Fts3TokenAndCost Fts3TokenAndCost; +struct Fts3TokenAndCost { + Fts3PhraseToken *pToken; + int nOvfl; + int iCol; +}; + +static void fts3EvalTokenCosts( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + Fts3TokenAndCost **ppTC, + int *pRc +){ + if( *pRc==SQLITE_OK && pExpr ){ + if( pExpr->eType==FTSQUERY_PHRASE ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + int i; + for(i=0; *pRc==SQLITE_OK && inToken; i++){ + Fts3TokenAndCost *pTC = (*ppTC)++; + pTC->pToken = &pPhrase->aToken[i]; + pTC->iCol = pPhrase->iColumn; + *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); + } + }else if( pExpr->eType==FTSQUERY_AND ){ + fts3EvalTokenCosts(pCsr, pExpr->pLeft, ppTC, pRc); + fts3EvalTokenCosts(pCsr, pExpr->pRight, ppTC, pRc); + } + } +} + +static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ + if( pCsr->nRowAvg==0 ){ + /* The average document size, which is required to calculate the cost + ** of each doclist, has not yet been determined. Read the required + ** data from the %_stat table to calculate it. + ** + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 + ** varints, where nCol is the number of columns in the FTS3 table. + ** The first varint is the number of documents currently stored in + ** the table. The following nCol varints contain the total amount of + ** data stored in all rows of each column of the table, from left + ** to right. + */ + int rc; + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + sqlite3_stmt *pStmt; + sqlite3_int64 nDoc = 0; + sqlite3_int64 nByte = 0; + const char *pEnd; + const char *a; + + rc = sqlite3Fts3SelectDoctotal(p, &pStmt); + if( rc!=SQLITE_OK ) return rc; + a = sqlite3_column_blob(pStmt, 0); + assert( a ); + + pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; + a += sqlite3Fts3GetVarint(a, &nDoc); + while( anRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); + assert( pCsr->nRowAvg>0 ); + rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ) return rc; + } + + *pnPage = pCsr->nRowAvg; + return SQLITE_OK; +} + +int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int nToken = 0; + + /* Allocate a MultiSegReader for each token in the expression. */ + fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &rc); + + /* Call fts3EvalPhraseStart() on all phrases in the expression. TODO: + ** This call will eventually also be responsible for determining which + ** tokens are 'deferred' until the document text is loaded into memory. + ** + ** Each token in each phrase is dealt with using one of the following + ** three strategies: + ** + ** 1. Entire doclist loaded into memory as part of the + ** fts3EvalStartReaders() call. + ** + ** 2. Doclist loaded into memory incrementally, as part of each + ** sqlite3Fts3EvalNext() call. + ** + ** 3. Token doclist is never loaded. Instead, documents are loaded into + ** memory and scanned for the token as part of the sqlite3Fts3EvalNext() + ** call. This is known as a "deferred" token. + */ + + /* If bOptOk is true, check if there are any tokens that can be + ** deferred (strategy 3). */ + if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){ + Fts3TokenAndCost *aTC; + aTC = (Fts3TokenAndCost *)sqlite3_malloc(sizeof(Fts3TokenAndCost) * nToken); + if( !aTC ){ + rc = SQLITE_NOMEM; + }else{ + int ii; + int nDocEst = 0; + int nDocSize; + Fts3TokenAndCost *pTC = aTC; + + rc = fts3EvalAverageDocsize(pCsr, &nDocSize); + fts3EvalTokenCosts(pCsr, pExpr, &pTC, &rc); + nToken = pTC-aTC; + + for(ii=0; rc==SQLITE_OK && iinOvfl) ){ + pTC = &aTC[jj]; + } + } + assert( pTC ); + + /* At this point pTC points to the cheapest remaining token. */ + if( ii==0 ){ + if( pTC->nOvfl ){ + nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10; + }else{ + /* TODO: Fix this so that the doclist need not be read twice. */ + Fts3PhraseToken *pToken = pTC->pToken; + int nList = 0; + char *pList = 0; + rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList); + if( rc==SQLITE_OK ){ + nDocEst = fts3DoclistCountDocids(1, pList, nList); + } + sqlite3_free(pList); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3TermSegReaderCursor(pCsr, + pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr + ); + } + } + }else{ + if( pTC->nOvfl>=(nDocEst*nDocSize) ){ + Fts3PhraseToken *pToken = pTC->pToken; + rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); + fts3SegReaderCursorFree(pToken->pSegcsr); + pToken->pSegcsr = 0; + } + nDocEst = 1 + (nDocEst/4); + } + pTC->pToken = 0; + } + sqlite3_free(aTC); + } + } + + fts3EvalStartReaders(pCsr, pExpr, bOptOk, &rc); + + /* Fix the results of NEAR expressions. */ + fts3EvalNearTrim(pCsr, pExpr, &rc); + + return rc; +} + +static void fts3EvalNext( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pRc +){ + if( *pRc==SQLITE_OK ){ + + pExpr->bStart = 1; + switch( pExpr->eType ){ + + case FTSQUERY_NEAR: + case FTSQUERY_AND: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + + assert( !pLeft->bDeferred || !pRight->bDeferred ); + if( pLeft->bDeferred ){ + fts3EvalNext(pCsr, pRight, pRc); + pExpr->iDocid = pRight->iDocid; + pExpr->bEof = pRight->bEof; + }else if( pRight->bDeferred ){ + fts3EvalNext(pCsr, pLeft, pRc); + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + }else{ + fts3EvalNext(pCsr, pLeft, pRc); + fts3EvalNext(pCsr, pRight, pRc); + + while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ + int iDiff = pLeft->iDocid - pRight->iDocid; + if( iDiff==0 ) break; + if( iDiff<0 ){ + fts3EvalNext(pCsr, pLeft, pRc); + }else{ + fts3EvalNext(pCsr, pRight, pRc); + } + } + + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = (pLeft->bEof || pRight->bEof); + } + break; + } + + case FTSQUERY_OR: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + + assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); + assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); + + if( pLeft->iDocid==pRight->iDocid ){ + fts3EvalNext(pCsr, pLeft, pRc); + fts3EvalNext(pCsr, pRight, pRc); + }else if( + pRight->bEof || (pLeft->bEof==0 && pLeft->iDocidiDocid) + ){ + fts3EvalNext(pCsr, pLeft, pRc); + }else{ + fts3EvalNext(pCsr, pRight, pRc); + } + + pExpr->bEof = (pLeft->bEof && pRight->bEof); + if( pRight->bEof || (pLeft->bEof==0 && pLeft->iDocidiDocid) ){ + pExpr->iDocid = pLeft->iDocid; + }else{ + pExpr->iDocid = pRight->iDocid; + } + + break; + } + + case FTSQUERY_NOT: { + Fts3Expr *pLeft = pExpr->pLeft; + Fts3Expr *pRight = pExpr->pRight; + + if( pRight->bStart==0 ){ + fts3EvalNext(pCsr, pRight, pRc); + assert( *pRc!=SQLITE_OK || pRight->bStart ); + } + do { + fts3EvalNext(pCsr, pLeft, pRc); + if( pLeft->bEof ) break; + while( !*pRc && !pRight->bEof && pRight->iDocidiDocid ){ + fts3EvalNext(pCsr, pRight, pRc); + } + }while( !pRight->bEof && pRight->iDocid==pLeft->iDocid && !*pRc ); + pExpr->iDocid = pLeft->iDocid; + pExpr->bEof = pLeft->bEof; + break; + } + + default: + assert( pExpr->eType==FTSQUERY_PHRASE ); + *pRc = fts3EvalPhraseNext(pCsr, pExpr->pPhrase, &pExpr->bEof); + pExpr->iDocid = fts3EvalPhraseDocid(pExpr->pPhrase); + break; + } + } +} + +static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ + int bHit = 0; + if( *pRc==SQLITE_OK ){ + switch( pExpr->eType ){ + case FTSQUERY_NEAR: + case FTSQUERY_AND: + bHit = ( + fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) + && fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + ); + break; + + case FTSQUERY_OR: + bHit = ( + fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) + || fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + ); + break; + + case FTSQUERY_NOT: + bHit = ( + fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) + && !fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + ); + break; + + default: + assert( pExpr->eType==FTSQUERY_PHRASE ); + *pRc = fts3EvalDeferredPhrase(pCsr, pExpr->pPhrase); + bHit = (pExpr->pPhrase->doclist.pList!=0); + pExpr->iDocid = pCsr->iPrevId; + break; + } + } + return bHit; +} + +/* +** Return 1 if both of the following are true: +** +** 1. *pRc is SQLITE_OK when this function returns, and +** +** 2. After scanning the current FTS table row for the deferred tokens, +** it is determined that the row does not match the query. +*/ +static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){ + int rc = *pRc; + int bMiss = 0; + if( rc==SQLITE_OK && pCsr->pDeferred ){ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK ){ + sqlite3Fts3FreeDeferredDoclists(pCsr); + rc = sqlite3Fts3CacheDeferredDoclists(pCsr); + } + bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc)); + sqlite3Fts3FreeDeferredDoclists(pCsr); + *pRc = rc; + } + return (rc==SQLITE_OK && bMiss); +} + +/* +** Advance to the next document that matches the expression passed as an +** argument. +*/ +int sqlite3Fts3EvalNext(Fts3Cursor *pCsr, Fts3Expr *pExpr){ + int rc = SQLITE_OK; /* Return Code */ + assert( pCsr->isEof==0 ); + assert( pCsr->bIncremental ); + if( pExpr==0 ){ + pCsr->isEof = 1; + }else{ + do { + sqlite3_reset(pCsr->pStmt); + fts3EvalNext(pCsr, pExpr, &rc); + pCsr->isEof = pExpr->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pExpr->iDocid; + }while( pCsr->isEof==0 && fts3EvalLoadDeferred(pCsr, &rc) ); + } + return rc; +} + +int sqlite3Fts3EvalFinish(Fts3Cursor *pCsr, Fts3Expr *pExpr){ + return SQLITE_OK; +} + +sqlite3_int64 sqlite3Fts3EvalDocid(Fts3Cursor *pCsr, Fts3Expr *pExpr){ + return pExpr->iDocid; +} + +/* +** Return a pointer to the entire doclist, including positions, associated +** with the phrase passed as the second argument. +*/ +int sqlite3Fts3EvalPhraseDoclist( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + Fts3Expr *pExpr, /* Phrase to return doclist for */ + const char **ppList, /* OUT: Buffer containing doclist */ + int *pnList /* OUT: Size of returned buffer, in bytes */ +){ + int rc = SQLITE_OK; + Fts3Phrase *pPhrase = pExpr->pPhrase; + + if( pPhrase->bIncr ){ + /* This phrase was being loaded from disk incrementally. But the + ** matchinfo() function requires that the entire doclist be loaded into + ** memory. This block loads the doclist into memory and modifies the + ** Fts3Phrase structure so that it does not use the incremental strategy. + */ + TESTONLY( int bEof = pExpr->bEof; ) + TESTONLY( int bStart = pExpr->bStart; ) + sqlite3_int64 iDocid = pExpr->iDocid; + + sqlite3Fts3EvalPhraseCleanup(pPhrase); + pExpr->iDocid = 0; + + rc = sqlite3Fts3EvalStart(pCsr, pExpr, 0); + assert( pExpr->bEof==bEof ); + assert( pExpr->bStart==bStart ); + assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); + if( pExpr->bStart && !pExpr->bEof ){ + pExpr->bStart = 0; + while( rc==SQLITE_OK && pExpr->bEof==0 && pExpr->iDocid!=iDocid ){ + fts3EvalNext(pCsr, pExpr, &rc); + } + } + } + + *pnList = pPhrase->doclist.nAll; + *ppList = pPhrase->doclist.aAll; + return rc; +} + +char *sqlite3Fts3EvalPhrasePoslist( + Fts3Cursor *pCsr, /* FTS3 cursor object */ + Fts3Expr *pExpr, /* Phrase to return doclist for */ + sqlite3_int64 iDocid, /* Docid to return position list for */ + int iCol /* Column to return position list for */ +){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + char *pIter = pPhrase->doclist.pList; + int iThis; + + assert( iCol>=0 && iColnColumn ); + if( !pIter + || pExpr->bEof + || pExpr->iDocid!=iDocid + || (pPhrase->iColumnnColumn && pPhrase->iColumn!=iCol) + ){ + return 0; + } + + assert( pPhrase->doclist.nList>0 ); + if( *pIter==0x01 ){ + pIter++; + pIter += sqlite3Fts3GetVarint32(pIter, &iThis); + }else{ + iThis = 0; + } + while( iThisdoclist.aAll); + memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); + for(i=0; inToken; i++){ + fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); + pPhrase->aToken[i].pSegcsr = 0; + } +} + #endif diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 42be10773a..0d69029dfb 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -47,6 +47,11 @@ */ #define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0]))) + +#ifndef MIN +# define MIN(x,y) ((x)<(y)?(x):(y)) +#endif + /* ** Maximum length of a varint encoded integer. The varint format is different ** from that used by SQLite, so the maximum length is 10, not 9. @@ -142,10 +147,11 @@ typedef struct Fts3Expr Fts3Expr; typedef struct Fts3Phrase Fts3Phrase; typedef struct Fts3PhraseToken Fts3PhraseToken; +typedef struct Fts3Doclist Fts3Doclist; typedef struct Fts3SegFilter Fts3SegFilter; typedef struct Fts3DeferredToken Fts3DeferredToken; typedef struct Fts3SegReader Fts3SegReader; -typedef struct Fts3SegReaderCursor Fts3SegReaderCursor; +typedef struct Fts3MultiSegReader Fts3MultiSegReader; /* ** A connection to a fulltext index is an instance of the following @@ -224,6 +230,7 @@ struct Fts3Cursor { u8 isRequireSeek; /* True if must seek pStmt to %_content row */ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ Fts3Expr *pExpr; /* Parsed MATCH query string */ + int bIncremental; /* True to use incremental querying */ int nPhrase; /* Number of matchable phrases in query */ Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ @@ -263,6 +270,17 @@ struct Fts3Cursor { #define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */ #define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */ + +struct Fts3Doclist { + char *aAll; /* Array containing doclist (or NULL) */ + int nAll; /* Size of a[] in bytes */ + + sqlite3_int64 iDocid; /* Current docid (if p!=0) */ + char *pNextDocid; /* Pointer to next docid */ + char *pList; /* Pointer to position list following iDocid */ + int nList; /* Length of position list */ +} doclist; + /* ** A "phrase" is a sequence of one or more tokens that must match in ** sequence. A single token is the base case and the most common case. @@ -277,23 +295,29 @@ struct Fts3PhraseToken { /* Variables above this point are populated when the expression is ** parsed (by code in fts3_expr.c). Below this point the variables are ** used when evaluating the expression. */ - int bFulltext; /* True if full-text index was used */ - Fts3SegReaderCursor *pSegcsr; /* Segment-reader for this token */ Fts3DeferredToken *pDeferred; /* Deferred token object for this token */ + Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */ }; struct Fts3Phrase { - /* Variables populated by fts3_expr.c when parsing a MATCH expression */ - int nToken; /* Number of tokens in the phrase */ - int iColumn; /* Index of column this phrase must match */ + /* Cache of doclist for this phrase. */ + Fts3Doclist doclist; + int bIncr; /* True if doclist is loaded incrementally */ +#if 1 int isLoaded; /* True if aDoclist/nDoclist are initialized. */ char *aDoclist; /* Buffer containing doclist */ int nDoclist; /* Size of aDoclist in bytes */ sqlite3_int64 iCurrent; char *pCurrent; +#endif + /* Variables below this point are populated by fts3_expr.c when parsing + ** a MATCH expression. Everything above is part of the evaluation phase. + */ + int nToken; /* Number of tokens in the phrase */ + int iColumn; /* Index of column this phrase must match */ Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */ }; @@ -317,6 +341,12 @@ struct Fts3Expr { Fts3Expr *pLeft; /* Left operand */ Fts3Expr *pRight; /* Right operand */ Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */ + + /* The following are used by the fts3_eval.c module. */ + sqlite3_int64 iDocid; /* Current docid */ + u8 bEof; /* True this expression is at EOF already */ + u8 bStart; /* True if iDocid is valid */ + u8 bDeferred; /* True if this expression is entirely deferred */ }; /* @@ -366,11 +396,12 @@ void sqlite3Fts3SegmentsClose(Fts3Table *); #define FTS3_SEGCURSOR_PENDING -1 #define FTS3_SEGCURSOR_ALL -2 -int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*); -int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *); -void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *); +int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*); +int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); +void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); + int sqlite3Fts3SegReaderCursor( - Fts3Table *, int, int, const char *, int, int, int, Fts3SegReaderCursor *); + Fts3Table *, int, int, const char *, int, int, int, Fts3MultiSegReader *); /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ #define FTS3_SEGMENT_REQUIRE_POS 0x00000001 @@ -387,7 +418,7 @@ struct Fts3SegFilter { int flags; }; -struct Fts3SegReaderCursor { +struct Fts3MultiSegReader { /* Used internally by sqlite3Fts3SegReaderXXX() calls */ Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */ int nSegment; /* Size of apSegment array */ @@ -396,8 +427,11 @@ struct Fts3SegReaderCursor { char *aBuffer; /* Buffer to merge doclists in */ int nBuffer; /* Allocated size of aBuffer[] in bytes */ - /* Cost of running this iterator. Used by fts3.c only. */ - int nCost; + int iColFilter; /* If >=0, filter for this column */ + + /* Used by fts3.c only. */ + int nCost; /* Cost of running iterator */ + int bLookup; /* True if a lookup of a single entry. */ /* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */ char *zTerm; /* Pointer to term buffer */ @@ -413,7 +447,6 @@ int sqlite3Fts3GetVarint32(const char *, int *); int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); -char *sqlite3Fts3FindPositions(Fts3Cursor *, Fts3Expr *, sqlite3_int64, int); int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *); int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *); int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int); @@ -446,4 +479,29 @@ int sqlite3Fts3InitTerm(sqlite3 *db); /* fts3_aux.c */ int sqlite3Fts3InitAux(sqlite3 *db); +int sqlite3Fts3TermSegReaderCursor( + Fts3Cursor *pCsr, /* Virtual table cursor handle */ + const char *zTerm, /* Term to query for */ + int nTerm, /* Size of zTerm in bytes */ + int isPrefix, /* True for a prefix search */ + Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ +); + +int sqlite3Fts3EvalPhraseCache(Fts3Cursor *, Fts3Phrase *); +sqlite3_int64 sqlite3Fts3EvalDocid(Fts3Cursor *, Fts3Expr *); +int sqlite3Fts3EvalPhraseDoclist(Fts3Cursor*, Fts3Expr*, const char**,int*); +void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); + +int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int); +int sqlite3Fts3EvalNext(Fts3Cursor *pCsr, Fts3Expr *pExpr); +int sqlite3Fts3MsrIncrStart( + Fts3Table*, Fts3MultiSegReader*, int, const char*, int); +int sqlite3Fts3MsrIncrNext( + Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); +char *sqlite3Fts3EvalPhrasePoslist( + Fts3Cursor *, Fts3Expr *, sqlite3_int64, int iCol); +int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); + +int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); + #endif /* _FTSINT_H */ diff --git a/ext/fts3/fts3_aux.c b/ext/fts3/fts3_aux.c index 5c7c7b0b0f..ded0202731 100644 --- a/ext/fts3/fts3_aux.c +++ b/ext/fts3/fts3_aux.c @@ -28,7 +28,7 @@ struct Fts3auxTable { struct Fts3auxCursor { sqlite3_vtab_cursor base; /* Base class used by SQLite core */ - Fts3SegReaderCursor csr; /* Must be right after "base" */ + Fts3MultiSegReader csr; /* Must be right after "base" */ Fts3SegFilter filter; char *zStop; int nStop; /* Byte-length of string zStop */ diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 095841d142..44636ed3fc 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -768,7 +768,10 @@ void sqlite3Fts3ExprFree(Fts3Expr *p){ assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); sqlite3Fts3ExprFree(p->pLeft); sqlite3Fts3ExprFree(p->pRight); - if( p->pPhrase ) sqlite3_free(p->pPhrase->aDoclist); + if( p->pPhrase ){ + sqlite3Fts3EvalPhraseCleanup(p->pPhrase); + sqlite3_free(p->pPhrase->aDoclist); + } sqlite3_free(p); } } diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index b31a1cf359..187ecbb57f 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -416,7 +416,7 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ pPhrase->nToken = pExpr->pPhrase->nToken; - pCsr = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->pCsr->iPrevId, p->iCol); + pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->pCsr->iPrevId,p->iCol); if( pCsr ){ int iFirst = 0; pPhrase->pList = pCsr; @@ -826,46 +826,31 @@ static int fts3ExprGlobalHitsCb( void *pCtx /* Pointer to MatchInfo structure */ ){ MatchInfo *p = (MatchInfo *)pCtx; - Fts3Cursor *pCsr = p->pCursor; - Fts3Phrase *pPhrase = pExpr->pPhrase; - char *pIter; - char *pEnd; - char *pFree = 0; u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol]; - assert( pPhrase->isLoaded ); - - if( pCsr->pDeferred ){ - int ii; - for(ii=0; iinToken; ii++){ - if( pPhrase->aToken[ii].bFulltext ) break; - } - if( iinToken ){ - int nFree = 0; - int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree); - if( rc!=SQLITE_OK ) return rc; - pIter = pFree; - pEnd = &pFree[nFree]; - }else{ - int iCol; /* Column index */ - for(iCol=0; iColnCol; iCol++){ - aOut[iCol*3 + 1] = (u32)p->nDoc; - aOut[iCol*3 + 2] = (u32)p->nDoc; - } - return SQLITE_OK; + if( pExpr->bDeferred ){ + int iCol; /* Column index */ + for(iCol=0; iColnCol; iCol++){ + aOut[iCol*3 + 1] = (u32)p->nDoc; + aOut[iCol*3 + 2] = (u32)p->nDoc; } }else{ - pIter = pPhrase->aDoclist; - pEnd = &pPhrase->aDoclist[pPhrase->nDoclist]; + char *pIter; + char *pEnd; + int n; + int rc = sqlite3Fts3EvalPhraseDoclist( + p->pCursor, pExpr, (const char **)&pIter, &n + ); + if( rc!=SQLITE_OK ) return rc; + pEnd = &pIter[n]; + + /* Fill in the global hit count matrix row for this phrase. */ + while( pIternCol * 3; int i; + sqlite3_int64 iDocid = p->pCursor->iPrevId; - for(i=0; inCol; i++) p->aMatchinfo[iStart+i*3] = 0; - - if( pExpr->pPhrase->aDoclist ){ + for(i=0; inCol; i++){ char *pCsr; - - pCsr = sqlite3Fts3FindPositions(p->pCursor, pExpr, p->pCursor->iPrevId, -1); + pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, iDocid, i); if( pCsr ){ - fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0); + p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr); + }else{ + p->aMatchinfo[iStart+i*3] = 0; } } @@ -976,9 +961,8 @@ static int fts3MatchinfoSelectDoctotal( typedef struct LcsIterator LcsIterator; struct LcsIterator { Fts3Expr *pExpr; /* Pointer to phrase expression */ - char *pRead; /* Cursor used to iterate through aDoclist */ int iPosOffset; /* Tokens count up to end of this phrase */ - int iCol; /* Current column number */ + char *pRead; /* Cursor used to iterate through aDoclist */ int iPos; /* Current position */ }; @@ -1009,17 +993,10 @@ static int fts3LcsIteratorAdvance(LcsIterator *pIter){ int rc = 0; pRead += sqlite3Fts3GetVarint(pRead, &iRead); - if( iRead==0 ){ - pIter->iCol = LCS_ITERATOR_FINISHED; + if( iRead==0 || iRead==1 ){ + pRead = 0; rc = 1; }else{ - if( iRead==1 ){ - pRead += sqlite3Fts3GetVarint(pRead, &iRead); - pIter->iCol = (int)iRead; - pIter->iPos = pIter->iPosOffset; - pRead += sqlite3Fts3GetVarint(pRead, &iRead); - rc = 1; - } pIter->iPos += (int)(iRead-2); } @@ -1043,6 +1020,7 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ int i; int iCol; int nToken = 0; + sqlite3_int64 iDocid = pCsr->iPrevId; /* Allocate and populate the array of LcsIterator objects. The array ** contains one element for each matchable phrase in the query. @@ -1051,42 +1029,34 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ if( !aIter ) return SQLITE_NOMEM; memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase); (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); + for(i=0; inPhrase; i++){ LcsIterator *pIter = &aIter[i]; nToken -= pIter->pExpr->pPhrase->nToken; pIter->iPosOffset = nToken; - pIter->pRead = sqlite3Fts3FindPositions(pCsr,pIter->pExpr,pCsr->iPrevId,-1); - if( pIter->pRead ){ - pIter->iPos = pIter->iPosOffset; - fts3LcsIteratorAdvance(&aIter[i]); - }else{ - pIter->iCol = LCS_ITERATOR_FINISHED; - } } for(iCol=0; iColnCol; iCol++){ int nLcs = 0; /* LCS value for this column */ int nLive = 0; /* Number of iterators in aIter not at EOF */ - /* Loop through the iterators in aIter[]. Set nLive to the number of - ** iterators that point to a position-list corresponding to column iCol. - */ for(i=0; inPhrase; i++){ - assert( aIter[i].iCol>=iCol ); - if( aIter[i].iCol==iCol ) nLive++; + LcsIterator *pIt = &aIter[i]; + pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iDocid, iCol); + if( pIt->pRead ){ + pIt->iPos = pIt->iPosOffset; + fts3LcsIteratorAdvance(&aIter[i]); + nLive++; + } } - /* The following loop runs until all iterators in aIter[] have finished - ** iterating through positions in column iCol. Exactly one of the - ** iterators is advanced each time the body of the loop is run. - */ while( nLive>0 ){ LcsIterator *pAdv = 0; /* The iterator to advance by one position */ int nThisLcs = 0; /* LCS for the current iterator positions */ for(i=0; inPhrase; i++){ LcsIterator *pIter = &aIter[i]; - if( iCol!=pIter->iCol ){ + if( pIter->pRead==0 ){ /* This iterator is already at EOF for this column. */ nThisLcs = 0; }else{ @@ -1426,7 +1396,7 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ int iPos = 0; /* First position in position-list */ UNUSED_PARAMETER(iPhrase); - pList = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->iDocid, p->iCol); + pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iDocid, p->iCol); nTerm = pExpr->pPhrase->nToken; if( pList ){ fts3GetDeltaPosition(&pList, &iPos); diff --git a/ext/fts3/fts3_term.c b/ext/fts3/fts3_term.c index d7aa66855a..eff65b7bd1 100644 --- a/ext/fts3/fts3_term.c +++ b/ext/fts3/fts3_term.c @@ -33,7 +33,7 @@ struct Fts3termTable { struct Fts3termCursor { sqlite3_vtab_cursor base; /* Base class used by SQLite core */ - Fts3SegReaderCursor csr; /* Must be right after "base" */ + Fts3MultiSegReader csr; /* Must be right after "base" */ Fts3SegFilter filter; int isEof; /* True if cursor is at EOF */ diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index f664dec8cf..deeff3c418 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -1155,6 +1155,8 @@ int sqlite3Fts3SegReaderCost( int nCost = 0; /* Cost in bytes to return */ int pgsz = p->nPgsz; /* Database page size */ + assert( pgsz>0 ); + /* If this seg-reader is reading the pending-terms table, or if all data ** for the segment is stored on the root page of the b-tree, then the cost ** is zero. In this case all required data is already in main memory. @@ -1223,6 +1225,40 @@ int sqlite3Fts3SegReaderCost( return rc; } +int sqlite3Fts3MsrOvfl( + Fts3Cursor *pCsr, + Fts3MultiSegReader *pMsr, + int *pnOvfl +){ + Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; + int nOvfl = 0; + int ii; + int rc = SQLITE_OK; + int pgsz = p->nPgsz; + + assert( p->bHasStat ); + assert( pgsz>0 ); + + for(ii=0; rc==SQLITE_OK && iinSegment; ii++){ + Fts3SegReader *pReader = pMsr->apSegment[ii]; + if( !fts3SegReaderIsPending(pReader) + && !fts3SegReaderIsRootOnly(pReader) + ){ + int jj; + for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){ + int nBlob; + rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob); + if( rc!=SQLITE_OK ) break; + if( (nBlob+35)>pgsz ){ + nOvfl += (nBlob + 34)/pgsz; + } + } + } + } + *pnOvfl = nOvfl; + return rc; +} + /* ** Free all allocations associated with the iterator passed as the ** second argument. @@ -2140,9 +2176,107 @@ static void fts3ColumnFilter( *pnList = nList; } +int sqlite3Fts3MsrIncrStart( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pCsr, /* Cursor object */ + int iCol, /* Column to match on. */ + const char *zTerm, /* Term to iterate through a doclist for */ + int nTerm /* Number of bytes in zTerm */ +){ + int i; + int nSegment = pCsr->nSegment; + + assert( pCsr->pFilter==0 ); + assert( zTerm && nTerm>0 ); + + /* Advance each segment iterator until it points to the term zTerm/nTerm. */ + for(i=0; iapSegment[i]; + do { + int rc = fts3SegReaderNext(p, pSeg); + if( rc!=SQLITE_OK ) return rc; + }while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); + } + fts3SegReaderSort(pCsr->apSegment, nSegment, nSegment, fts3SegReaderCmp); + + /* Determine how many of the segments actually point to zTerm/nTerm. */ + for(i=0; iapSegment[i]; + if( !pSeg->aNode || fts3SegReaderTermCmp(pSeg, zTerm, nTerm) ){ + break; + } + } + pCsr->nAdvance = i; + + /* Advance each of the segments to point to the first docid. */ + for(i=0; inAdvance; i++){ + fts3SegReaderFirstDocid(pCsr->apSegment[i]); + } + + assert( iCol<0 || iColnColumn ); + pCsr->iColFilter = iCol; + + return SQLITE_OK; +} + +int sqlite3Fts3MsrIncrNext( + Fts3Table *p, /* Virtual table handle */ + Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ + sqlite3_int64 *piDocid, /* OUT: Docid value */ + char **paPoslist, /* OUT: Pointer to position list */ + int *pnPoslist /* OUT: Size of position list in bytes */ +){ + int rc = SQLITE_OK; + int nMerge = pMsr->nAdvance; + Fts3SegReader **apSegment = pMsr->apSegment; + + if( nMerge==0 ){ + *paPoslist = 0; + return SQLITE_OK; + } + + while( 1 ){ + Fts3SegReader *pSeg; + fts3SegReaderSort(pMsr->apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp); + pSeg = pMsr->apSegment[0]; + + if( pSeg->pOffsetList==0 ){ + *paPoslist = 0; + break; + }else{ + char *pList; + int nList; + int j; + sqlite3_int64 iDocid = apSegment[0]->iDocid; + + fts3SegReaderNextDocid(apSegment[0], &pList, &nList); + j = 1; + while( jpOffsetList + && apSegment[j]->iDocid==iDocid + ){ + fts3SegReaderNextDocid(apSegment[j], 0, 0); + } + + if( pMsr->iColFilter>=0 ){ + fts3ColumnFilter(pMsr->iColFilter, &pList, &nList); + } + + if( nList>0 ){ + *piDocid = iDocid; + *paPoslist = pList; + *pnPoslist = nList; + break; + } + } + } + + return rc; +} + int sqlite3Fts3SegReaderStart( Fts3Table *p, /* Virtual table handle */ - Fts3SegReaderCursor *pCsr, /* Cursor object */ + Fts3MultiSegReader *pCsr, /* Cursor object */ Fts3SegFilter *pFilter /* Restrictions on range of iteration */ ){ int i; @@ -2173,7 +2307,7 @@ int sqlite3Fts3SegReaderStart( int sqlite3Fts3SegReaderStep( Fts3Table *p, /* Virtual table handle */ - Fts3SegReaderCursor *pCsr /* Cursor object */ + Fts3MultiSegReader *pCsr /* Cursor object */ ){ int rc = SQLITE_OK; @@ -2308,8 +2442,9 @@ int sqlite3Fts3SegReaderStep( return rc; } + void sqlite3Fts3SegReaderFinish( - Fts3SegReaderCursor *pCsr /* Cursor object */ + Fts3MultiSegReader *pCsr /* Cursor object */ ){ if( pCsr ){ int i; @@ -2342,7 +2477,7 @@ static int fts3SegmentMerge(Fts3Table *p, int iIndex, int iLevel){ int iNewLevel = 0; /* Level/index to create new segment at */ SegmentWriter *pWriter = 0; /* Used to write the new, merged, segment */ Fts3SegFilter filter; /* Segment term filter condition */ - Fts3SegReaderCursor csr; /* Cursor to iterate through level(s) */ + Fts3MultiSegReader csr; /* Cursor to iterate through level(s) */ int bIgnoreEmpty = 0; /* True to ignore empty segments */ assert( iLevel==FTS3_SEGCURSOR_ALL @@ -2746,6 +2881,33 @@ int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ return rc; } +int sqlite3Fts3DeferredTokenList( + Fts3DeferredToken *p, + char **ppData, + int *pnData +){ + char *pRet; + int nSkip; + sqlite3_int64 dummy; + + *ppData = 0; + *pnData = 0; + + if( p->pList==0 ){ + return SQLITE_OK; + } + + pRet = (char *)sqlite3_malloc(p->pList->nData); + if( !pRet ) return SQLITE_NOMEM; + + nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); + *pnData = p->pList->nData - nSkip; + *ppData = pRet; + + memcpy(pRet, &p->pList->aData[nSkip], *pnData); + return SQLITE_OK; +} + /* ** Add an entry for token pToken to the pCsr->pDeferred list. */ diff --git a/manifest b/manifest index 950ee87f2e..499805878d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\schanges\smade\swhile\splanning\sa\slarger\schange. -D 2011-05-28T15:57:40.694 +C Changes\sto\simprove\sperformance\sand\ssupport\sLIMIT\sclauses\son\sfts3\stables.\sThis\sbranch\sis\sunstable\sfor\snow. +D 2011-06-02T19:57:24.733 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,21 +61,21 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c eb59cdd89e9309ab9b2dca196a7c52f9e8927319 +F ext/fts3/fts3.c f92b6e0241a77a715d30dbeffd7c901053dbfda4 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 832f4d421f03a9d364186e779ee51994df500c62 -F ext/fts3/fts3_aux.c 2817a1ec9ffbad673cb1a1527ad807811bc7645b -F ext/fts3/fts3_expr.c 25e30cf24198333f2ed545af905b168e88f56903 +F ext/fts3/fts3Int.h ab1489076e7d54714d20bbbc7aaef8e694a7db50 +F ext/fts3/fts3_aux.c 28fc512608e147015c36080025456f57f571f76f +F ext/fts3/fts3_expr.c 5c789c744f98a007512f49d9cda4d2bb4cd56517 F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2 -F ext/fts3/fts3_snippet.c 6ee626017ddcf7d72ca4f587724e3506434fc0d7 -F ext/fts3/fts3_term.c 0ade1812c0e97f394b58299810dfd5d2fb7ba501 +F ext/fts3/fts3_snippet.c 10e0b0995ec82a2d93fbdf3159641cdf30f3c274 +F ext/fts3/fts3_term.c 6c7f33ab732a2a0f281898685650e3a492e1e2f1 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c ed58c53fbcbc2ea79258e734159a5951ffeb1bd4 +F ext/fts3/fts3_write.c b145547430af9451f81cfed92fb7065da2efd870 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -463,7 +463,7 @@ F test/fts3corrupt.test 7b0f91780ca36118d73324ec803187208ad33b32 F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52 -F test/fts3defer.test d6cb0db9b5997ecf863d96ff419f83f8f2c87f4f +F test/fts3defer.test 7c8a38d5f617d7b52ae1c43ed73c536e7e895a35 F test/fts3defer2.test 288bef6de15557319b8c12d476ebdc83688ef96c F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c @@ -612,7 +612,7 @@ F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806 F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16 F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16 -F test/permutations.test 5b2a4cb756ffb2407cb4743163668d1d769febb6 +F test/permutations.test d27eac16dae111ff7cec331dab4bca08625ba65a F test/pragma.test fdfc09067ea104a0c247a1a79d8093b56656f850 F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47 F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P cc83991caae7c7d647432d5711b6cd80228c3002 -R eef52aec1faead4921ebaf39c1605d2b +P 84097a4c759b1d65890af885f137d3cb16eef584 +R e38e0cc84edbfdb5eae395b3be2f7a86 U dan -Z 01d8f7aea2ce8ebb35c05ccc519b614f +Z 53d59cf3dcd8991c66b0afd5fb898b1a diff --git a/manifest.uuid b/manifest.uuid index 6768a6838c..9e0b1f8e4f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84097a4c759b1d65890af885f137d3cb16eef584 \ No newline at end of file +28149a7882a1e9dfe4a75ec5b91d176ebe6284e9 \ No newline at end of file diff --git a/test/fts3defer.test b/test/fts3defer.test index 1c9056fd9d..4bc0b0a7c3 100644 --- a/test/fts3defer.test +++ b/test/fts3defer.test @@ -20,6 +20,8 @@ ifcapable !fts3 { set sqlite_fts3_enable_parentheses 1 +set fts3_simple_deferred_tokens_only 1 + set ::testprefix fts3defer #-------------------------------------------------------------------------- @@ -257,7 +259,6 @@ foreach {tn setup} { do_select_test 1.2 { SELECT rowid FROM t1 WHERE t1 MATCH 'jk eh' } {100} -if {$tn==3} breakpoint do_select_test 1.3 { SELECT rowid FROM t1 WHERE t1 MATCH 'jk ubwrfqnbjf' } {7 70 98} @@ -282,13 +283,16 @@ if {$tn==3} breakpoint do_select_test 1.10 { SELECT rowid FROM t1 WHERE t1 MATCH 'z* vgsld' } {10 13 17 31 35 51 58 88 89 90 93 100} - do_select_test 1.11 { - SELECT rowid FROM t1 - WHERE t1 MATCH '( - zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR - zk OR zkhdvkw OR zm OR zsmhnf - ) vgsld' - } {10 13 17 31 35 51 58 88 89 90 93 100} + + if { $fts3_simple_deferred_tokens_only==0 } { + do_select_test 1.11 { + SELECT rowid FROM t1 + WHERE t1 MATCH '( + zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR + zk OR zkhdvkw OR zm OR zsmhnf + ) vgsld' + } {10 13 17 31 35 51 58 88 89 90 93 100} + } do_select_test 2.1 { SELECT rowid FROM t1 WHERE t1 MATCH '"zm agmckuiu"' @@ -364,6 +368,7 @@ if {$tn==3} breakpoint foreach DO_MALLOC_TEST $dmt_modes { # Phrase search. + # do_select_test 5.$DO_MALLOC_TEST.1 { SELECT rowid FROM t1 WHERE t1 MATCH '"jk mjpavjuhw"' } {8 15 36 64 67 72} @@ -416,9 +421,11 @@ if {$tn==3} breakpoint do_select_test 6.2.2 { SELECT rowid FROM t1 WHERE t1 MATCH '"zm azavwm"' } {15 26 92 96} - do_select_test 6.2.3 { - SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"' - } {8 15 26 92 96} + if {$fts3_simple_deferred_tokens_only==0} { + do_select_test 6.2.3 { + SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"' + } {8 15 26 92 96} + } } set testprefix fts3defer diff --git a/test/permutations.test b/test/permutations.test index 9c48d9aa2a..4640ed1139 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -179,6 +179,7 @@ test_suite "fts3" -prefix "" -description { fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test fts3near.test fts3query.test fts3shared.test fts3snippet.test + fts3sort.test fts3fault.test fts3malloc.test fts3matchinfo.test From 126ba6c0acdc107ae5907b1f97f4cde9baf57c33 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 3 Jun 2011 18:00:19 +0000 Subject: [PATCH 06/20] FTS changes: Remove unreachable code. Fix bugs. When processing a large doclist incrementally, read from disk incrementally too. FossilOrigin-Name: a4c7e2820824e82580730c36f85aede2efa66754 --- ext/fts3/fts3.c | 991 ++++++++-------------------------------- ext/fts3/fts3Int.h | 25 +- ext/fts3/fts3_expr.c | 5 +- ext/fts3/fts3_snippet.c | 63 +-- ext/fts3/fts3_write.c | 174 +++++-- manifest | 20 +- manifest.uuid | 2 +- 7 files changed, 345 insertions(+), 935 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index a7f176a700..90b081b8c5 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -313,9 +313,6 @@ SQLITE_EXTENSION_INIT1 #endif -static char *fts3EvalPhrasePoslist(Fts3Phrase *, int *); -static sqlite3_int64 fts3EvalPhraseDocid(Fts3Phrase *); - /* ** Write a 64-bit variable-length integer to memory starting at p[0]. ** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. @@ -1210,9 +1207,7 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ */ if( pInfo->nOrderBy==1 ){ struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; - if( pOrder->desc==0 - && (pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1) - ){ + if( pOrder->iColumn<0 || pOrder->iColumn==p->nColumn+1 ){ if( pOrder->desc ){ pInfo->idxStr = "DESC"; }else{ @@ -1261,8 +1256,6 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ return SQLITE_OK; } -static int fts3RowidMethod(sqlite3_vtab_cursor *, sqlite3_int64*); - /* ** Position the pCsr->pStmt statement so that it is on the row ** of the %_content table that contains the last match. Return @@ -1452,7 +1445,7 @@ static int fts3SelectLeaf( int nBlob; /* Size of zBlob in bytes */ if( piLeaf && piLeaf2 && (*piLeaf!=*piLeaf2) ){ - rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob); + rc = sqlite3Fts3ReadBlock(p, *piLeaf, &zBlob, &nBlob, 0); if( rc==SQLITE_OK ){ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, 0); } @@ -1462,7 +1455,7 @@ static int fts3SelectLeaf( } if( rc==SQLITE_OK ){ - rc = sqlite3Fts3ReadBlock(p, piLeaf ? *piLeaf : *piLeaf2, &zBlob, &nBlob); + rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0); } if( rc==SQLITE_OK ){ rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); @@ -2194,35 +2187,6 @@ static int fts3TermSelectCb( return SQLITE_OK; } -static int fts3DeferredTermSelect( - Fts3DeferredToken *pToken, /* Phrase token */ - int isTermPos, /* True to include positions */ - int *pnOut, /* OUT: Size of list */ - char **ppOut /* OUT: Body of list */ -){ - char *aSource; - int nSource; - - aSource = sqlite3Fts3DeferredDoclist(pToken, &nSource); - if( !aSource ){ - *pnOut = 0; - *ppOut = 0; - }else if( isTermPos ){ - *ppOut = sqlite3_malloc(nSource); - if( !*ppOut ) return SQLITE_NOMEM; - memcpy(*ppOut, aSource, nSource); - *pnOut = nSource; - }else{ - sqlite3_int64 docid; - *pnOut = sqlite3Fts3GetVarint(aSource, &docid); - *ppOut = sqlite3_malloc(*pnOut); - if( !*ppOut ) return SQLITE_NOMEM; - sqlite3Fts3PutVarint(*ppOut, docid); - } - - return SQLITE_OK; -} - /* ** Append SegReader object pNew to the end of the pCsr->apSegment[] array. */ @@ -2512,198 +2476,6 @@ static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){ return nDoc; } -/* -** Call sqlite3Fts3DeferToken() for each token in the expression pExpr. -*/ -static int fts3DeferExpression(Fts3Cursor *pCsr, Fts3Expr *pExpr){ - int rc = SQLITE_OK; - if( pExpr ){ - rc = fts3DeferExpression(pCsr, pExpr->pLeft); - if( rc==SQLITE_OK ){ - rc = fts3DeferExpression(pCsr, pExpr->pRight); - } - if( pExpr->eType==FTSQUERY_PHRASE ){ - int iCol = pExpr->pPhrase->iColumn; - int i; - for(i=0; rc==SQLITE_OK && ipPhrase->nToken; i++){ - Fts3PhraseToken *pToken = &pExpr->pPhrase->aToken[i]; - if( pToken->pDeferred==0 ){ - rc = sqlite3Fts3DeferToken(pCsr, pToken, iCol); - } - } - } - } - return rc; -} - -/* -** This function removes the position information from a doclist. When -** called, buffer aList (size *pnList bytes) contains a doclist that includes -** position information. This function removes the position information so -** that aList contains only docids, and adjusts *pnList to reflect the new -** (possibly reduced) size of the doclist. -*/ -static void fts3DoclistStripPositions( - char *aList, /* IN/OUT: Buffer containing doclist */ - int *pnList /* IN/OUT: Size of doclist in bytes */ -){ - if( aList ){ - char *aEnd = &aList[*pnList]; /* Pointer to one byte after EOF */ - char *p = aList; /* Input cursor */ - char *pOut = aList; /* Output cursor */ - - while( piColumn; - int isTermPos = (pPhrase->nToken>1 || isReqPos); - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; - int isFirst = 1; - - int iPrevTok = 0; - int nDoc = 0; - - for(ii=0; iinToken; ii++){ - Fts3PhraseToken *pTok; /* Token to find doclist for */ - int iTok = 0; /* The token being queried this iteration */ - char *pList = 0; /* Pointer to token doclist */ - int nList = 0; /* Size of buffer at pList */ - - /* Select a token to process. If this is an xFilter() call, then tokens - ** are processed in order from least to most costly. Otherwise, tokens - ** are processed in the order in which they occur in the phrase. - */ - if( pCsr->eEvalmode==FTS3_EVAL_MATCHINFO ){ - assert( isReqPos ); - iTok = ii; - pTok = &pPhrase->aToken[iTok]; - if( pTok->bFulltext==0 ) continue; - }else if( pCsr->eEvalmode==FTS3_EVAL_NEXT || isReqPos ){ - iTok = ii; - pTok = &pPhrase->aToken[iTok]; - }else{ - int nMinCost = 0x7FFFFFFF; - int jj; - - /* Find the remaining token with the lowest cost. */ - for(jj=0; jjnToken; jj++){ - Fts3MultiSegReader *pSegcsr = pPhrase->aToken[jj].pSegcsr; - if( pSegcsr && pSegcsr->nCostnCost; - } - } - pTok = &pPhrase->aToken[iTok]; - - /* This branch is taken if it is determined that loading the doclist - ** for the next token would require more IO than loading all documents - ** currently identified by doclist pOut/nOut. No further doclists will - ** be loaded from the full-text index for this phrase. - */ - if( nMinCost>nDoc && ii>0 ){ - rc = fts3DeferExpression(pCsr, pCsr->pExpr); - break; - } - } - - if( pCsr->eEvalmode==FTS3_EVAL_NEXT && pTok->pDeferred ){ - rc = fts3DeferredTermSelect(pTok->pDeferred, isTermPos, &nList, &pList); - }else{ - if( pTok->pSegcsr ){ - rc = fts3TermSelect(p, pTok, iCol, isTermPos, &nList, &pList); - } - pTok->bFulltext = 1; - } - assert( rc!=SQLITE_OK || pCsr->eEvalmode || pTok->pSegcsr==0 ); - if( rc!=SQLITE_OK ) break; - - if( isFirst ){ - pOut = pList; - nOut = nList; - if( pCsr->eEvalmode==FTS3_EVAL_FILTER && pPhrase->nToken>1 ){ - nDoc = fts3DoclistCountDocids(1, pOut, nOut); - } - isFirst = 0; - iPrevTok = iTok; - }else{ - /* Merge the new term list and the current output. */ - char *aLeft, *aRight; - int nLeft, nRight; - int nDist; - int mt; - - /* If this is the final token of the phrase, and positions were not - ** requested by the caller, use MERGE_PHRASE instead of POS_PHRASE. - ** This drops the position information from the output list. - */ - mt = MERGE_POS_PHRASE; - if( ii==pPhrase->nToken-1 && !isReqPos ) mt = MERGE_PHRASE; - - assert( iPrevTok!=iTok ); - if( iPrevToknToken ){ - assert( pCsr->eEvalmode==FTS3_EVAL_FILTER && isReqPos==0 ); - fts3DoclistStripPositions(pOut, &nOut); - } - *paOut = pOut; - *pnOut = nOut; - }else{ - sqlite3_free(pOut); - } - return rc; -} - /* ** This function merges two doclists according to the requirements of a ** NEAR operator. @@ -2746,408 +2518,6 @@ static int fts3NearMerge( return rc; } -/* -** This function is used as part of the processing for the snippet() and -** offsets() functions. -** -** Both pLeft and pRight are expression nodes of type FTSQUERY_PHRASE. Both -** have their respective doclists (including position information) loaded -** in Fts3Expr.aDoclist/nDoclist. This function removes all entries from -** each doclist that are not within nNear tokens of a corresponding entry -** in the other doclist. -*/ -int sqlite3Fts3ExprNearTrim(Fts3Expr *pELeft, Fts3Expr *pERight, int nNear){ - int rc; /* Return code */ - Fts3Phrase *pLeft = pELeft->pPhrase; - Fts3Phrase *pRight = pERight->pPhrase; - - assert( pELeft->eType==FTSQUERY_PHRASE && pLeft ); - assert( pERight->eType==FTSQUERY_PHRASE && pRight ); - assert( pLeft->isLoaded && pRight->isLoaded ); - - if( pLeft->aDoclist==0 || pRight->aDoclist==0 ){ - sqlite3_free(pLeft->aDoclist); - sqlite3_free(pRight->aDoclist); - pRight->aDoclist = 0; - pLeft->aDoclist = 0; - rc = SQLITE_OK; - }else{ - char *aOut; /* Buffer in which to assemble new doclist */ - int nOut; /* Size of buffer aOut in bytes */ - - rc = fts3NearMerge(MERGE_POS_NEAR, nNear, - pLeft->nToken, pLeft->aDoclist, pLeft->nDoclist, - pRight->nToken, pRight->aDoclist, pRight->nDoclist, - &aOut, &nOut - ); - if( rc!=SQLITE_OK ) return rc; - sqlite3_free(pRight->aDoclist); - pRight->aDoclist = aOut; - pRight->nDoclist = nOut; - - rc = fts3NearMerge(MERGE_POS_NEAR, nNear, - pRight->nToken, pRight->aDoclist, pRight->nDoclist, - pLeft->nToken, pLeft->aDoclist, pLeft->nDoclist, - &aOut, &nOut - ); - sqlite3_free(pLeft->aDoclist); - pLeft->aDoclist = aOut; - pLeft->nDoclist = nOut; - } - return rc; -} - - -/* -** Allocate an Fts3SegReaderArray for each token in the expression pExpr. -** The allocated objects are stored in the Fts3PhraseToken.pArray member -** variables of each token structure. -*/ -static int fts3ExprAllocateSegReaders( - Fts3Cursor *pCsr, /* FTS3 table */ - Fts3Expr *pExpr, /* Expression to create seg-readers for */ - int *pnExpr /* OUT: Number of AND'd expressions */ -){ - int rc = SQLITE_OK; /* Return code */ - - assert( pCsr->eEvalmode==FTS3_EVAL_FILTER ); - if( pnExpr && pExpr->eType!=FTSQUERY_AND ){ - (*pnExpr)++; - pnExpr = 0; - } - - if( pExpr->eType==FTSQUERY_PHRASE ){ - int ii; /* Used to iterate through phrase tokens */ - Fts3Phrase *pPhrase = pExpr->pPhrase; - - for(ii=0; rc==SQLITE_OK && iinToken; ii++){ - Fts3PhraseToken *pTok = &pPhrase->aToken[ii]; - if( pTok->pSegcsr==0 ){ - rc = sqlite3Fts3TermSegReaderCursor( - pCsr, pTok->z, pTok->n, pTok->isPrefix, &pTok->pSegcsr - ); - } - } - }else{ - rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pLeft, pnExpr); - if( rc==SQLITE_OK ){ - rc = fts3ExprAllocateSegReaders(pCsr, pExpr->pRight, pnExpr); - } - } - return rc; -} - -/* -** Free the Fts3SegReaderArray objects associated with each token in the -** expression pExpr. In other words, this function frees the resources -** allocated by fts3ExprAllocateSegReaders(). -*/ -static void fts3ExprFreeSegReaders(Fts3Expr *pExpr){ - if( pExpr ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - if( pPhrase ){ - int kk; - for(kk=0; kknToken; kk++){ - fts3SegReaderCursorFree(pPhrase->aToken[kk].pSegcsr); - pPhrase->aToken[kk].pSegcsr = 0; - } - } - fts3ExprFreeSegReaders(pExpr->pLeft); - fts3ExprFreeSegReaders(pExpr->pRight); - } -} - -/* -** Return the sum of the costs of all tokens in the expression pExpr. This -** function must be called after Fts3SegReaderArrays have been allocated -** for all tokens using fts3ExprAllocateSegReaders(). -*/ -static int fts3ExprCost(Fts3Expr *pExpr){ - int nCost; /* Return value */ - if( pExpr->eType==FTSQUERY_PHRASE ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - int ii; - nCost = 0; - for(ii=0; iinToken; ii++){ - Fts3MultiSegReader *pSegcsr = pPhrase->aToken[ii].pSegcsr; - if( pSegcsr ) nCost += pSegcsr->nCost; - } - }else{ - nCost = fts3ExprCost(pExpr->pLeft) + fts3ExprCost(pExpr->pRight); - } - return nCost; -} - -/* -** The following is a helper function (and type) for fts3EvalExpr(). It -** must be called after Fts3SegReaders have been allocated for every token -** in the expression. See the context it is called from in fts3EvalExpr() -** for further explanation. -*/ -typedef struct ExprAndCost ExprAndCost; -struct ExprAndCost { - Fts3Expr *pExpr; - int nCost; -}; -static void fts3ExprAssignCosts( - Fts3Expr *pExpr, /* Expression to create seg-readers for */ - ExprAndCost **ppExprCost /* OUT: Write to *ppExprCost */ -){ - if( pExpr->eType==FTSQUERY_AND ){ - fts3ExprAssignCosts(pExpr->pLeft, ppExprCost); - fts3ExprAssignCosts(pExpr->pRight, ppExprCost); - }else{ - (*ppExprCost)->pExpr = pExpr; - (*ppExprCost)->nCost = fts3ExprCost(pExpr); - (*ppExprCost)++; - } -} - -/* -** Evaluate the full-text expression pExpr against FTS3 table pTab. Store -** the resulting doclist in *paOut and *pnOut. This routine mallocs for -** the space needed to store the output. The caller is responsible for -** freeing the space when it has finished. -** -** This function is called in two distinct contexts: -** -** * From within the virtual table xFilter() method. In this case, the -** output doclist contains entries for all rows in the table, based on -** data read from the full-text index. -** -** In this case, if the query expression contains one or more tokens that -** are very common, then the returned doclist may contain a superset of -** the documents that actually match the expression. -** -** * From within the virtual table xNext() method. This call is only made -** if the call from within xFilter() found that there were very common -** tokens in the query expression and did return a superset of the -** matching documents. In this case the returned doclist contains only -** entries that correspond to the current row of the table. Instead of -** reading the data for each token from the full-text index, the data is -** already available in-memory in the Fts3PhraseToken.pDeferred structures. -** See fts3EvalDeferred() for how it gets there. -** -** In the first case above, Fts3Cursor.doDeferred==0. In the second (if it is -** required) Fts3Cursor.doDeferred==1. -** -** If the SQLite invokes the snippet(), offsets() or matchinfo() function -** as part of a SELECT on an FTS3 table, this function is called on each -** individual phrase expression in the query. If there were very common tokens -** found in the xFilter() call, then this function is called once for phrase -** for each row visited, and the returned doclist contains entries for the -** current row only. Otherwise, if there were no very common tokens, then this -** function is called once only for each phrase in the query and the returned -** doclist contains entries for all rows of the table. -** -** Fts3Cursor.doDeferred==1 when this function is called on phrases as a -** result of a snippet(), offsets() or matchinfo() invocation. -*/ -static int fts3EvalExpr( - Fts3Cursor *p, /* Virtual table cursor handle */ - Fts3Expr *pExpr, /* Parsed fts3 expression */ - char **paOut, /* OUT: Pointer to malloc'd result buffer */ - int *pnOut, /* OUT: Size of buffer at *paOut */ - int isReqPos /* Require positions in output buffer */ -){ - int rc = SQLITE_OK; /* Return code */ - - /* Zero the output parameters. */ - *paOut = 0; - *pnOut = 0; - - if( pExpr ){ - assert( pExpr->eType==FTSQUERY_NEAR || pExpr->eType==FTSQUERY_OR - || pExpr->eType==FTSQUERY_AND || pExpr->eType==FTSQUERY_NOT - || pExpr->eType==FTSQUERY_PHRASE - ); - assert( pExpr->eType==FTSQUERY_PHRASE || isReqPos==0 ); - - if( pExpr->eType==FTSQUERY_PHRASE ){ - rc = fts3PhraseSelect(p, pExpr->pPhrase, - isReqPos || (pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR), - paOut, pnOut - ); - fts3ExprFreeSegReaders(pExpr); - }else if( p->eEvalmode==FTS3_EVAL_FILTER && pExpr->eType==FTSQUERY_AND ){ - ExprAndCost *aExpr = 0; /* Array of AND'd expressions and costs */ - int nExpr = 0; /* Size of aExpr[] */ - char *aRet = 0; /* Doclist to return to caller */ - int nRet = 0; /* Length of aRet[] in bytes */ - int nDoc = 0x7FFFFFFF; - - assert( !isReqPos ); - - rc = fts3ExprAllocateSegReaders(p, pExpr, &nExpr); - if( rc==SQLITE_OK ){ - assert( nExpr>1 ); - aExpr = sqlite3_malloc(sizeof(ExprAndCost) * nExpr); - if( !aExpr ) rc = SQLITE_NOMEM; - } - if( rc==SQLITE_OK ){ - int ii; /* Used to iterate through expressions */ - - fts3ExprAssignCosts(pExpr, &aExpr); - aExpr -= nExpr; - for(ii=0; iipExpr && (pBest==0 || pCand->nCostnCost) ){ - pBest = pCand; - } - } - - if( pBest->nCost>nDoc ){ - rc = fts3DeferExpression(p, p->pExpr); - break; - }else{ - rc = fts3EvalExpr(p, pBest->pExpr, &aNew, &nNew, 0); - if( rc!=SQLITE_OK ) break; - pBest->pExpr = 0; - if( ii==0 ){ - aRet = aNew; - nRet = nNew; - nDoc = fts3DoclistCountDocids(0, aRet, nRet); - }else{ - fts3DoclistMerge( - MERGE_AND, 0, 0, aRet, &nRet, aRet, nRet, aNew, nNew, &nDoc - ); - sqlite3_free(aNew); - } - } - } - } - - if( rc==SQLITE_OK ){ - *paOut = aRet; - *pnOut = nRet; - }else{ - assert( *paOut==0 ); - sqlite3_free(aRet); - } - sqlite3_free(aExpr); - fts3ExprFreeSegReaders(pExpr); - - }else{ - char *aLeft; - char *aRight; - int nLeft; - int nRight; - - assert( pExpr->eType==FTSQUERY_NEAR - || pExpr->eType==FTSQUERY_OR - || pExpr->eType==FTSQUERY_NOT - || (pExpr->eType==FTSQUERY_AND && p->eEvalmode==FTS3_EVAL_NEXT) - ); - - if( 0==(rc = fts3EvalExpr(p, pExpr->pRight, &aRight, &nRight, isReqPos)) - && 0==(rc = fts3EvalExpr(p, pExpr->pLeft, &aLeft, &nLeft, isReqPos)) - ){ - switch( pExpr->eType ){ - case FTSQUERY_NEAR: { - Fts3Expr *pLeft; - Fts3Expr *pRight; - int mergetype = MERGE_NEAR; - if( pExpr->pParent && pExpr->pParent->eType==FTSQUERY_NEAR ){ - mergetype = MERGE_POS_NEAR; - } - pLeft = pExpr->pLeft; - while( pLeft->eType==FTSQUERY_NEAR ){ - pLeft=pLeft->pRight; - } - pRight = pExpr->pRight; - assert( pRight->eType==FTSQUERY_PHRASE ); - assert( pLeft->eType==FTSQUERY_PHRASE ); - - rc = fts3NearMerge(mergetype, pExpr->nNear, - pLeft->pPhrase->nToken, aLeft, nLeft, - pRight->pPhrase->nToken, aRight, nRight, - paOut, pnOut - ); - sqlite3_free(aLeft); - break; - } - - case FTSQUERY_OR: { - /* Allocate a buffer for the output. The maximum size is the - ** sum of the sizes of the two input buffers. The +1 term is - ** so that a buffer of zero bytes is never allocated - this can - ** cause fts3DoclistMerge() to incorrectly return SQLITE_NOMEM. - */ - char *aBuffer = sqlite3_malloc(nRight+nLeft+1); - rc = fts3DoclistMerge(MERGE_OR, 0, 0, aBuffer, pnOut, - aLeft, nLeft, aRight, nRight, 0 - ); - *paOut = aBuffer; - sqlite3_free(aLeft); - break; - } - - default: { - assert( FTSQUERY_NOT==MERGE_NOT && FTSQUERY_AND==MERGE_AND ); - fts3DoclistMerge(pExpr->eType, 0, 0, aLeft, pnOut, - aLeft, nLeft, aRight, nRight, 0 - ); - *paOut = aLeft; - break; - } - } - } - sqlite3_free(aRight); - } - } - - assert( rc==SQLITE_OK || *paOut==0 ); - return rc; -} - -/* -** This function is called from within xNext() for each row visited by -** an FTS3 query. If evaluating the FTS3 query expression within xFilter() -** was able to determine the exact set of matching rows, this function sets -** *pbRes to true and returns SQLITE_IO immediately. -** -** Otherwise, if evaluating the query expression within xFilter() returned a -** superset of the matching documents instead of an exact set (this happens -** when the query includes very common tokens and it is deemed too expensive to -** load their doclists from disk), this function tests if the current row -** really does match the FTS3 query. -** -** If an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK -** is returned and *pbRes is set to true if the current row matches the -** FTS3 query (and should be included in the results returned to SQLite), or -** false otherwise. -*/ -static int fts3EvalDeferred( - Fts3Cursor *pCsr, /* FTS3 cursor pointing at row to test */ - int *pbRes /* OUT: Set to true if row is a match */ -){ - int rc = SQLITE_OK; - if( pCsr->pDeferred==0 ){ - *pbRes = 1; - }else{ - rc = fts3CursorSeek(0, pCsr); - if( rc==SQLITE_OK ){ - sqlite3Fts3FreeDeferredDoclists(pCsr); - rc = sqlite3Fts3CacheDeferredDoclists(pCsr); - } - if( rc==SQLITE_OK ){ - char *a = 0; - int n = 0; - rc = fts3EvalExpr(pCsr, pCsr->pExpr, &a, &n, 0); - assert( n>=0 ); - *pbRes = (n>0); - sqlite3_free(a); - } - } - return rc; -} - /* ** Advance the cursor to the next row in the %_content table that ** matches the search criteria. For a MATCH search, this will be @@ -3160,43 +2530,19 @@ static int fts3EvalDeferred( ** subsequently to determine whether or not an EOF was hit. */ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ - int res; - int rc = SQLITE_OK; /* Return code */ + int rc; Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; - - if( pCsr->bIncremental ){ - rc = sqlite3Fts3EvalNext(pCsr, pCsr->pExpr); + if( pCsr->eSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){ + if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ + pCsr->isEof = 1; + rc = sqlite3_reset(pCsr->pStmt); + }else{ + pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); + rc = SQLITE_OK; + } }else{ - pCsr->eEvalmode = FTS3_EVAL_NEXT; - do { - if( pCsr->aDoclist==0 ){ - if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ - pCsr->isEof = 1; - rc = sqlite3_reset(pCsr->pStmt); - break; - } - pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); - }else{ - if( pCsr->desc==0 ){ - if( pCsr->pNextId>=&pCsr->aDoclist[pCsr->nDoclist] ){ - pCsr->isEof = 1; - break; - } - fts3GetDeltaVarint(&pCsr->pNextId, &pCsr->iPrevId); - }else{ - fts3GetReverseDeltaVarint(&pCsr->pNextId,pCsr->aDoclist,&pCsr->iPrevId); - if( pCsr->pNextId<=pCsr->aDoclist ){ - pCsr->isEof = 1; - break; - } - } - sqlite3_reset(pCsr->pStmt); - pCsr->isRequireSeek = 1; - pCsr->isMatchinfoNeeded = 1; - } - }while( SQLITE_OK==(rc = fts3EvalDeferred(pCsr, &res)) && res==0 ); + rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor); } - return rc; } @@ -3242,6 +2588,9 @@ static int fts3FilterMethod( sqlite3Fts3ExprFree(pCsr->pExpr); memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); + pCsr->bDesc = (idxStr && idxStr[0]=='D'); + pCsr->eSearch = (i16)idxNum; + if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){ int iCol = idxNum-FTS3_FULLTEXT_SEARCH; const char *zQuery = (const char *)sqlite3_value_text(apVal[0]); @@ -3264,7 +2613,6 @@ static int fts3FilterMethod( rc = sqlite3Fts3ReadLock(p); if( rc!=SQLITE_OK ) return rc; - pCsr->bIncremental = 1; rc = sqlite3Fts3EvalStart(pCsr, pCsr->pExpr, 1); sqlite3Fts3SegmentsClose(p); @@ -3296,22 +2644,6 @@ static int fts3FilterMethod( if( rc!=SQLITE_OK ) return rc; } - assert( pCsr->desc==0 ); - pCsr->eSearch = (i16)idxNum; - if( rc==SQLITE_OK && pCsr->nDoclist>0 && idxStr && idxStr[0]=='D' ){ - sqlite3_int64 iDocid = 0; - char *csr = pCsr->aDoclist; - while( csr<&pCsr->aDoclist[pCsr->nDoclist] ){ - fts3GetDeltaVarint(&csr, &iDocid); - } - pCsr->pNextId = csr; - pCsr->iPrevId = iDocid; - pCsr->desc = 1; - pCsr->isRequireSeek = 1; - pCsr->isMatchinfoNeeded = 1; - pCsr->eEvalmode = FTS3_EVAL_NEXT; - return SQLITE_OK; - } return fts3NextMethod(pCursor); } @@ -3331,18 +2663,7 @@ static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ */ static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; - if( pCsr->bIncremental ){ - *pRowid = sqlite3Fts3EvalDocid(pCsr, pCsr->pExpr); - }else if( pCsr->aDoclist ){ - *pRowid = pCsr->iPrevId; - }else{ - /* This branch runs if the query is implemented using a full-table scan - ** (not using the full-text index). In this case grab the rowid from the - ** SELECT statement. - */ - assert( pCsr->isRequireSeek==0 ); - *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); - } + *pRowid = pCsr->iPrevId; return SQLITE_OK; } @@ -3449,46 +2770,6 @@ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ return SQLITE_OK; } -/* -** Load the doclist associated with expression pExpr to pExpr->aDoclist. -** The loaded doclist contains positions as well as the document ids. -** This is used by the matchinfo(), snippet() and offsets() auxillary -** functions. -*/ -int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *pCsr, Fts3Expr *pExpr){ - int rc = SQLITE_OK; - if( pCsr->bIncremental==0 ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - assert( pExpr->eType==FTSQUERY_PHRASE && pPhrase ); - assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); - rc = fts3EvalExpr(pCsr, pExpr, &pPhrase->aDoclist, &pPhrase->nDoclist, 1); - } - return rc; -} - -/* -** TODO: This is something to do with matchinfo(). Similar to -** sqlite3ExprLoadDoclists() but slightly different. -** -** UPDATE: Only used when there are deferred tokens. -*/ -int sqlite3Fts3ExprLoadFtDoclist( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, - char **paDoclist, - int *pnDoclist -){ - int rc = SQLITE_OK; - assert( pExpr->eType==FTSQUERY_PHRASE && pExpr->pPhrase ); - assert( pCsr->eEvalmode==FTS3_EVAL_NEXT ); - assert( pCsr->bIncremental==0 ); - pCsr->eEvalmode = FTS3_EVAL_MATCHINFO; - rc = fts3EvalExpr(pCsr, pExpr, paDoclist, pnDoclist, 1); - pCsr->eEvalmode = FTS3_EVAL_NEXT; - return rc; -} - - /* ** When called, *ppPoslist must point to the byte immediately following the ** end of a position-list. i.e. ( (*ppPoslist)[-1]==POS_END ). This function @@ -3909,12 +3190,6 @@ int sqlite3_extension_init( } #endif -/************************************************************************* -************************************************************************** -************************************************************************** -************************************************************************** -*************************************************************************/ - /* ** Allocate an Fts3MultiSegReader for each token in the expression headed @@ -4021,6 +3296,8 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ int nPoslist = 0; int iPrev = -1; + assert( pPhrase->doclist.bFreeList==0 ); + for(iToken=0; rc==SQLITE_OK && iTokennToken; iToken++){ Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; Fts3DeferredToken *pDeferred = pToken->pDeferred; @@ -4070,6 +3347,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ pPhrase->doclist.pList = aPoslist; pPhrase->doclist.nList = nPoslist; pPhrase->doclist.iDocid = pCsr->iPrevId; + pPhrase->doclist.bFreeList = 1; }else{ int nDistance; char *p1; @@ -4094,13 +3372,14 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ pPhrase->doclist.pList = aOut; if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ + pPhrase->doclist.bFreeList = 1; pPhrase->doclist.nList = (aOut - pPhrase->doclist.pList); - sqlite3_free(aPoslist); }else{ sqlite3_free(aOut); pPhrase->doclist.pList = 0; pPhrase->doclist.nList = 0; } + sqlite3_free(aPoslist); } } @@ -4113,16 +3392,20 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ ** ** fts3EvalPhraseStart() ** fts3EvalPhraseNext() -** fts3EvalPhraseReset() ** ** May be used with a phrase object after fts3EvalAllocateReaders() has been ** called to iterate through the set of docids that match the phrase. ** ** After a successful call to fts3EvalPhraseNext(), the following two ** functions may be called to access the current docid and position-list. -** -** fts3EvalPhraseDocid() -** fts3EvalPhrasePoslist() +*/ + + +/* +** This function is called for each Fts3Phrase in a full-text query +** expression to initialize the mechanism for returning rows. Once this +** function has been called successfully on an Fts3Phrase, it may be +** used with fts3EvalPhraseNext() to iterate through the matching docids. */ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ int rc; @@ -4131,7 +3414,7 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ assert( pList->aAll==0 ); - if( p->nToken==1 && bOptOk==1 + if( pCsr->bDesc==0 && bOptOk==1 && p->nToken==1 && pFirst->pSegcsr && pFirst->pSegcsr->bLookup ){ /* Use the incremental approach. */ @@ -4142,7 +3425,6 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ p->bIncr = 1; }else{ - /* Load the full doclist for the phrase into memory. */ rc = fts3EvalPhraseLoad(pCsr, p); p->bIncr = 0; @@ -4161,22 +3443,60 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ ** 1 before returning. Otherwise, if no error occurs and the iterator is ** successfully advanced, *pbEof is set to 0. */ -static int fts3EvalPhraseNext(Fts3Cursor *pCsr, Fts3Phrase *p, u8 *pbEof){ +static int fts3EvalPhraseNext( + Fts3Cursor *pCsr, + Fts3Phrase *p, + u8 *pbEof +){ int rc = SQLITE_OK; + Fts3Doclist *pDL = &p->doclist; if( p->bIncr ){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; assert( p->nToken==1 ); rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, - &p->doclist.iDocid, &p->doclist.pList, &p->doclist.nList + &pDL->iDocid, &pDL->pList, &pDL->nList ); - if( rc==SQLITE_OK && !p->doclist.pList ){ + if( rc==SQLITE_OK && !pDL->pList ){ *pbEof = 1; } - }else{ - char *pIter; - Fts3Doclist *pDL = &p->doclist; + }else if( pCsr->bDesc && pDL->aAll ){ + if( pDL->pNextDocid==0 ){ + sqlite3_int64 iDocid = 0; + char *pNext; + char *pDocid = pDL->aAll; + char *pEnd = &pDocid[pDL->nAll]; + + while( pDocidpNextDocid = pDocid; + pDL->pList = pDocid; + fts3PoslistCopy(0, &pDocid); + } + pDL->nList = (pEnd - pDL->pList); + pDL->iDocid = iDocid; + }else{ + + assert( *pbEof==0 ); + assert( pDL->pNextDocid>pDL->aAll ); + + fts3GetReverseDeltaVarint( + &pDL->pNextDocid, pDL->aAll, &pDL->iDocid + ); + if( pDL->pNextDocid==pDL->aAll ){ + *pbEof = 1; + }else{ + char *pSave = pDL->pNextDocid; + fts3ReversePoslist(pDL->aAll, &pDL->pNextDocid); + pDL->pList = pDL->pNextDocid; + pDL->nList = pSave - pDL->pNextDocid; + } + } + + }else{ + + char *pIter; if( pDL->pNextDocid ){ pIter = pDL->pNextDocid; }else{ @@ -4199,21 +3519,6 @@ static int fts3EvalPhraseNext(Fts3Cursor *pCsr, Fts3Phrase *p, u8 *pbEof){ return rc; } -static int fts3EvalPhraseReset(Fts3Cursor *pCsr, Fts3Phrase *p){ - return SQLITE_OK; -} - -static sqlite3_int64 fts3EvalPhraseDocid(Fts3Phrase *p){ - return p->doclist.iDocid; -} - -static char *fts3EvalPhrasePoslist(Fts3Phrase *p, int *pnList){ - if( pnList ){ - *pnList = p->doclist.nList; - } - return p->doclist.pList; -} - static void fts3EvalStartReaders( Fts3Cursor *pCsr, Fts3Expr *pExpr, @@ -4495,16 +3800,33 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ return rc; } +static void fts3EvalFreeDeferredDoclist(Fts3Phrase *pPhrase){ + if( pPhrase->doclist.bFreeList ){ + sqlite3_free(pPhrase->doclist.pList); + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + pPhrase->doclist.bFreeList = 0; + } +} + +/* +** This macro is used by the fts3EvalNext() function. The two arguments are +** 64-bit docid values. If the current query is "ORDER BY docid ASC", then +** the macro returns (i1 - i2). Or if it is "ORDER BY docid DESC", then +** it returns (i2 - i1). This allows the same code to be used for merging +** doclists in ascending or descending order. +*/ +#define DOCID_CMP(i1, i2) ((pCsr->bDesc?-1:1) * (i1-i2)) + static void fts3EvalNext( Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc ){ if( *pRc==SQLITE_OK ){ - pExpr->bStart = 1; - switch( pExpr->eType ){ + switch( pExpr->eType ){ case FTSQUERY_NEAR: case FTSQUERY_AND: { Fts3Expr *pLeft = pExpr->pLeft; @@ -4524,7 +3846,7 @@ static void fts3EvalNext( fts3EvalNext(pCsr, pRight, pRc); while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ - int iDiff = pLeft->iDocid - pRight->iDocid; + int iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); if( iDiff==0 ) break; if( iDiff<0 ){ fts3EvalNext(pCsr, pLeft, pRc); @@ -4542,23 +3864,23 @@ static void fts3EvalNext( case FTSQUERY_OR: { Fts3Expr *pLeft = pExpr->pLeft; Fts3Expr *pRight = pExpr->pRight; + int iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); - if( pLeft->iDocid==pRight->iDocid ){ + if( iCmp==0 ){ fts3EvalNext(pCsr, pLeft, pRc); fts3EvalNext(pCsr, pRight, pRc); - }else if( - pRight->bEof || (pLeft->bEof==0 && pLeft->iDocidiDocid) - ){ + }else if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ fts3EvalNext(pCsr, pLeft, pRc); }else{ fts3EvalNext(pCsr, pRight, pRc); } pExpr->bEof = (pLeft->bEof && pRight->bEof); - if( pRight->bEof || (pLeft->bEof==0 && pLeft->iDocidiDocid) ){ + iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ pExpr->iDocid = pLeft->iDocid; }else{ pExpr->iDocid = pRight->iDocid; @@ -4578,7 +3900,10 @@ static void fts3EvalNext( do { fts3EvalNext(pCsr, pLeft, pRc); if( pLeft->bEof ) break; - while( !*pRc && !pRight->bEof && pRight->iDocidiDocid ){ + while( !*pRc + && !pRight->bEof + && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 + ){ fts3EvalNext(pCsr, pRight, pRc); } }while( !pRight->bEof && pRight->iDocid==pLeft->iDocid && !*pRc ); @@ -4587,11 +3912,13 @@ static void fts3EvalNext( break; } - default: - assert( pExpr->eType==FTSQUERY_PHRASE ); - *pRc = fts3EvalPhraseNext(pCsr, pExpr->pPhrase, &pExpr->bEof); - pExpr->iDocid = fts3EvalPhraseDocid(pExpr->pPhrase); + default: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + fts3EvalFreeDeferredDoclist(pPhrase); + *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); + pExpr->iDocid = pPhrase->doclist.iDocid; break; + } } } } @@ -4622,12 +3949,14 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ ); break; - default: - assert( pExpr->eType==FTSQUERY_PHRASE ); - *pRc = fts3EvalDeferredPhrase(pCsr, pExpr->pPhrase); - bHit = (pExpr->pPhrase->doclist.pList!=0); + default: { + Fts3Phrase *pPhrase = pExpr->pPhrase; + fts3EvalFreeDeferredDoclist(pPhrase); + *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); + bHit = (pPhrase->doclist.pList!=0); pExpr->iDocid = pCsr->iPrevId; break; + } } } return bHit; @@ -4640,6 +3969,9 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ ** ** 2. After scanning the current FTS table row for the deferred tokens, ** it is determined that the row does not match the query. +** +** Or, if no error occurs and it seems the current row does match the FTS +** query, return 0. */ static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){ int rc = *pRc; @@ -4658,18 +3990,21 @@ static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){ } /* -** Advance to the next document that matches the expression passed as an -** argument. +** Advance to the next document that matches the FTS expression in +** Fts3Cursor.pExpr. */ -int sqlite3Fts3EvalNext(Fts3Cursor *pCsr, Fts3Expr *pExpr){ +int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){ int rc = SQLITE_OK; /* Return Code */ + Fts3Expr *pExpr = pCsr->pExpr; assert( pCsr->isEof==0 ); - assert( pCsr->bIncremental ); if( pExpr==0 ){ pCsr->isEof = 1; }else{ do { - sqlite3_reset(pCsr->pStmt); + if( pCsr->isRequireSeek==0 ){ + sqlite3_reset(pCsr->pStmt); + } + assert( sqlite3_data_count(pCsr->pStmt)==0 ); fts3EvalNext(pCsr, pExpr, &rc); pCsr->isEof = pExpr->bEof; pCsr->isRequireSeek = 1; @@ -4680,17 +4015,24 @@ int sqlite3Fts3EvalNext(Fts3Cursor *pCsr, Fts3Expr *pExpr){ return rc; } -int sqlite3Fts3EvalFinish(Fts3Cursor *pCsr, Fts3Expr *pExpr){ - return SQLITE_OK; -} - -sqlite3_int64 sqlite3Fts3EvalDocid(Fts3Cursor *pCsr, Fts3Expr *pExpr){ - return pExpr->iDocid; -} - /* ** Return a pointer to the entire doclist, including positions, associated -** with the phrase passed as the second argument. +** with the phrase passed as the second argument. It is illegal to call +** this function if the phrase consists entirely of deferred tokens. +** +** TODO: This function is only used by the code for the matchinfo('x') +** auxiliary function - to obtain the following two values: +** +** 1. The total number of times the phrase appears in each column in all +** rows in the FTS table. +** +** 2. For each column, the total number of rows in the FTS table for which +** the phrase appears at least once in the column. +** +** It would be better if there was an sqlite3Fts3EvalXXX() function +** specifically to retrieve these values. If that were done, the concept +** of which tokens are deferred or incremental would be entirely encapsulated +** within the sqlite3Fts3EvalXXX()/fts3EvalXXX() functions in this file. */ int sqlite3Fts3EvalPhraseDoclist( Fts3Cursor *pCsr, /* FTS3 cursor object */ @@ -4701,6 +4043,12 @@ int sqlite3Fts3EvalPhraseDoclist( int rc = SQLITE_OK; Fts3Phrase *pPhrase = pExpr->pPhrase; + /* It is illegal to call this function if the phrase is entirely deferred + ** (it may contain some deferred tokens, but must also contain at least + ** one token for which the doclist may be read from the full-text index). + */ + assert( !pExpr->bDeferred ); + if( pPhrase->bIncr ){ /* This phrase was being loaded from disk incrementally. But the ** matchinfo() function requires that the entire doclist be loaded into @@ -4731,10 +4079,28 @@ int sqlite3Fts3EvalPhraseDoclist( return rc; } +/* +** The expression pExpr passed as the second argument to this function +** must be of type FTSQUERY_PHRASE. +** +** The returned value is either NULL or a pointer to a buffer containing +** a position-list indicating the occurrences of the phrase in column iCol +** of the current row. +** +** More specifically, the returned buffer contains 1 varint for each +** occurence of the phrase in the column, stored using the normal (delta+2) +** compression and is terminated by either an 0x01 or 0x00 byte. For example, +** if the requested column contains "a b X c d X X" and the position-list +** for 'X' is requested, the buffer returned may contain: +** +** 0x04 0x05 0x03 0x01 or 0x04 0x05 0x03 0x00 +** +** This function works regardless of whether or not the phrase is deferred, +** incremental, or neither. +*/ char *sqlite3Fts3EvalPhrasePoslist( Fts3Cursor *pCsr, /* FTS3 cursor object */ Fts3Expr *pExpr, /* Phrase to return doclist for */ - sqlite3_int64 iDocid, /* Docid to return position list for */ int iCol /* Column to return position list for */ ){ Fts3Phrase *pPhrase = pExpr->pPhrase; @@ -4745,7 +4111,7 @@ char *sqlite3Fts3EvalPhrasePoslist( assert( iCol>=0 && iColnColumn ); if( !pIter || pExpr->bEof - || pExpr->iDocid!=iDocid + || pExpr->iDocid!=pCsr->iPrevId || (pPhrase->iColumnnColumn && pPhrase->iColumn!=iCol) ){ return 0; @@ -4768,13 +4134,20 @@ char *sqlite3Fts3EvalPhrasePoslist( return ((iCol==iThis)?pIter:0); } +/* +** Free all components of the Fts3Phrase structure that were allocated by +** the eval module. +*/ void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ - int i; - sqlite3_free(pPhrase->doclist.aAll); - memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); - for(i=0; inToken; i++){ - fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); - pPhrase->aToken[i].pSegcsr = 0; + if( pPhrase ){ + int i; + sqlite3_free(pPhrase->doclist.aAll); + fts3EvalFreeDeferredDoclist(pPhrase); + memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); + for(i=0; inToken; i++){ + fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); + pPhrase->aToken[i].pSegcsr = 0; + } } } diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 0d69029dfb..c6834d37b6 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -230,14 +230,13 @@ struct Fts3Cursor { u8 isRequireSeek; /* True if must seek pStmt to %_content row */ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ Fts3Expr *pExpr; /* Parsed MATCH query string */ - int bIncremental; /* True to use incremental querying */ int nPhrase; /* Number of matchable phrases in query */ Fts3DeferredToken *pDeferred; /* Deferred search tokens, if any */ sqlite3_int64 iPrevId; /* Previous id read from aDoclist */ char *pNextId; /* Pointer into the body of aDoclist */ char *aDoclist; /* List of docids for full-text queries */ int nDoclist; /* Size of buffer at aDoclist */ - int desc; /* True to sort in descending order */ + int bDesc; /* True to sort in descending order */ int eEvalmode; /* An FTS3_EVAL_XX constant */ int nRowAvg; /* Average size of database rows, in pages */ @@ -274,9 +273,10 @@ struct Fts3Cursor { struct Fts3Doclist { char *aAll; /* Array containing doclist (or NULL) */ int nAll; /* Size of a[] in bytes */ - - sqlite3_int64 iDocid; /* Current docid (if p!=0) */ char *pNextDocid; /* Pointer to next docid */ + + sqlite3_int64 iDocid; /* Current docid (if pList!=0) */ + int bFreeList; /* True if pList should be sqlite3_free()d */ char *pList; /* Pointer to position list following iDocid */ int nList; /* Length of position list */ } doclist; @@ -305,14 +305,6 @@ struct Fts3Phrase { Fts3Doclist doclist; int bIncr; /* True if doclist is loaded incrementally */ -#if 1 - int isLoaded; /* True if aDoclist/nDoclist are initialized. */ - char *aDoclist; /* Buffer containing doclist */ - int nDoclist; /* Size of aDoclist in bytes */ - sqlite3_int64 iCurrent; - char *pCurrent; -#endif - /* Variables below this point are populated by fts3_expr.c when parsing ** a MATCH expression. Everything above is part of the evaluation phase. */ @@ -380,7 +372,7 @@ void sqlite3Fts3SegReaderFree(Fts3SegReader *); int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **); int sqlite3Fts3ReadLock(Fts3Table *); -int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*); +int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **); int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **); @@ -448,7 +440,6 @@ int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *); -int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *); int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int); /* fts3_tokenizer.c */ @@ -493,13 +484,13 @@ int sqlite3Fts3EvalPhraseDoclist(Fts3Cursor*, Fts3Expr*, const char**,int*); void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int); -int sqlite3Fts3EvalNext(Fts3Cursor *pCsr, Fts3Expr *pExpr); +int sqlite3Fts3EvalNext(Fts3Cursor *pCsr); + int sqlite3Fts3MsrIncrStart( Fts3Table*, Fts3MultiSegReader*, int, const char*, int); int sqlite3Fts3MsrIncrNext( Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); -char *sqlite3Fts3EvalPhrasePoslist( - Fts3Cursor *, Fts3Expr *, sqlite3_int64, int iCol); +char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol); int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index 44636ed3fc..be40a9cfc2 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -768,10 +768,7 @@ void sqlite3Fts3ExprFree(Fts3Expr *p){ assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 ); sqlite3Fts3ExprFree(p->pLeft); sqlite3Fts3ExprFree(p->pRight); - if( p->pPhrase ){ - sqlite3Fts3EvalPhraseCleanup(p->pPhrase); - sqlite3_free(p->pPhrase->aDoclist); - } + sqlite3Fts3EvalPhraseCleanup(p->pPhrase); sqlite3_free(p); } } diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 187ecbb57f..45e3c32f07 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -176,51 +176,6 @@ static int fts3ExprIterate( return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); } -/* -** The argument to this function is always a phrase node. Its doclist -** (Fts3Expr.aDoclist[]) and the doclists associated with all phrase nodes -** to the left of this one in the query tree have already been loaded. -** -** If this phrase node is part of a series of phrase nodes joined by -** NEAR operators (and is not the left-most of said series), then elements are -** removed from the phrases doclist consistent with the NEAR restriction. If -** required, elements may be removed from the doclists of phrases to the -** left of this one that are part of the same series of NEAR operator -** connected phrases. -** -** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK. -*/ -static int fts3ExprNearTrim(Fts3Expr *pExpr){ - int rc = SQLITE_OK; - Fts3Expr *pParent = pExpr->pParent; - - assert( pExpr->eType==FTSQUERY_PHRASE ); - while( rc==SQLITE_OK - && pParent - && pParent->eType==FTSQUERY_NEAR - && pParent->pRight==pExpr - ){ - /* This expression (pExpr) is the right-hand-side of a NEAR operator. - ** Find the expression to the left of the same operator. - */ - int nNear = pParent->nNear; - Fts3Expr *pLeft = pParent->pLeft; - - if( pLeft->eType!=FTSQUERY_PHRASE ){ - assert( pLeft->eType==FTSQUERY_NEAR ); - assert( pLeft->pRight->eType==FTSQUERY_PHRASE ); - pLeft = pLeft->pRight; - } - - rc = sqlite3Fts3ExprNearTrim(pLeft, pExpr, nNear); - - pExpr = pLeft; - pParent = pExpr->pParent; - } - - return rc; -} - /* ** This is an fts3ExprIterate() callback used while loading the doclists ** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also @@ -236,14 +191,6 @@ static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ p->nPhrase++; p->nToken += pPhrase->nToken; - if( pPhrase->isLoaded==0 ){ - rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr); - pPhrase->isLoaded = 1; - if( rc==SQLITE_OK ){ - rc = fts3ExprNearTrim(pExpr); - } - } - return rc; } @@ -416,7 +363,7 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ pPhrase->nToken = pExpr->pPhrase->nToken; - pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->pCsr->iPrevId,p->iCol); + pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol); if( pCsr ){ int iFirst = 0; pPhrase->pList = pCsr; @@ -867,11 +814,10 @@ static int fts3ExprLocalHitsCb( MatchInfo *p = (MatchInfo *)pCtx; int iStart = iPhrase * p->nCol * 3; int i; - sqlite3_int64 iDocid = p->pCursor->iPrevId; for(i=0; inCol; i++){ char *pCsr; - pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, iDocid, i); + pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i); if( pCsr ){ p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr); }else{ @@ -1020,7 +966,6 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ int i; int iCol; int nToken = 0; - sqlite3_int64 iDocid = pCsr->iPrevId; /* Allocate and populate the array of LcsIterator objects. The array ** contains one element for each matchable phrase in the query. @@ -1042,7 +987,7 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ for(i=0; inPhrase; i++){ LcsIterator *pIt = &aIter[i]; - pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iDocid, iCol); + pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol); if( pIt->pRead ){ pIt->iPos = pIt->iPosOffset; fts3LcsIteratorAdvance(&aIter[i]); @@ -1396,7 +1341,7 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ int iPos = 0; /* First position in position-list */ UNUSED_PARAMETER(iPhrase); - pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iDocid, p->iCol); + pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol); nTerm = pExpr->pPhrase->nToken; if( pList ){ fts3GetDeltaPosition(&pList, &iPos); diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index deeff3c418..0470a72960 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -36,6 +36,27 @@ */ #define FTS3_NODE_PADDING (FTS3_VARINT_MAX*2) +/* +** Under certain circumstances, b-tree nodes (doclists) can be loaded into +** memory incrementally instead of all at once. This can be a big performance +** win (reduced IO and CPU) if SQLite stops calling the virtual table xNext() +** method before retrieving all query results (as may happen, for example, +** if a query has a LIMIT clause). +** +** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD +** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes. +** The code is written so that the hard lower-limit for each of these values +** is 1. Clearly such small values would be inefficient, but can be useful +** for testing purposes. +** +** TODO: Add a test interface to modify these "constants" from a script for +** this purpose. +*/ +#define FTS3_NODE_CHUNKSIZE (4*1024) +#define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) +/* #define FTS3_NODE_CHUNKSIZE 1 */ +/* #define FTS3_NODE_CHUNK_THRESHOLD 1 */ + typedef struct PendingList PendingList; typedef struct SegmentNode SegmentNode; typedef struct SegmentWriter SegmentWriter; @@ -93,6 +114,9 @@ struct Fts3SegReader { char *aNode; /* Pointer to node data (or NULL) */ int nNode; /* Size of buffer at aNode (or 0) */ + int nPopulate; /* If >0, bytes of buffer aNode[] loaded */ + sqlite3_blob *pBlob; /* If not NULL, blob handle to read node */ + Fts3HashElem **ppNextElem; /* Variables set by fts3SegReaderNext(). These may be read directly @@ -933,7 +957,8 @@ int sqlite3Fts3ReadBlock( Fts3Table *p, /* FTS3 table handle */ sqlite3_int64 iBlockid, /* Access the row with blockid=$iBlockid */ char **paBlob, /* OUT: Blob data in malloc'd buffer */ - int *pnBlob /* OUT: Size of blob data */ + int *pnBlob, /* OUT: Size of blob data */ + int *pnLoad /* OUT: Bytes actually loaded */ ){ int rc; /* Return code */ @@ -954,11 +979,16 @@ int sqlite3Fts3ReadBlock( if( rc==SQLITE_OK ){ int nByte = sqlite3_blob_bytes(p->pSegments); + *pnBlob = nByte; if( paBlob ){ char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); if( !aByte ){ rc = SQLITE_NOMEM; }else{ + if( pnLoad && nByte>(FTS3_NODE_CHUNK_THRESHOLD) ){ + nByte = FTS3_NODE_CHUNKSIZE; + *pnLoad = nByte; + } rc = sqlite3_blob_read(p->pSegments, aByte, nByte, 0); memset(&aByte[nByte], 0, FTS3_NODE_PADDING); if( rc!=SQLITE_OK ){ @@ -968,7 +998,6 @@ int sqlite3Fts3ReadBlock( } *paBlob = aByte; } - *pnBlob = nByte; } return rc; @@ -982,13 +1011,55 @@ void sqlite3Fts3SegmentsClose(Fts3Table *p){ sqlite3_blob_close(p->pSegments); p->pSegments = 0; } + +static int fts3SegReaderIncrRead(Fts3SegReader *pReader){ + int nRead; /* Number of bytes to read */ + int rc; /* Return code */ + + nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE); + rc = sqlite3_blob_read( + pReader->pBlob, + &pReader->aNode[pReader->nPopulate], + nRead, + pReader->nPopulate + ); + + if( rc==SQLITE_OK ){ + pReader->nPopulate += nRead; + memset(&pReader->aNode[pReader->nPopulate], 0, FTS3_NODE_PADDING); + if( pReader->nPopulate==pReader->nNode ){ + sqlite3_blob_close(pReader->pBlob); + pReader->pBlob = 0; + pReader->nPopulate = 0; + } + } + return rc; +} + +static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){ + int rc = SQLITE_OK; + assert( !pReader->pBlob + || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode]) + ); + while( pReader->pBlob && rc==SQLITE_OK + && (pFrom - pReader->aNode + nByte)>pReader->nPopulate + ){ + rc = fts3SegReaderIncrRead(pReader); + } + return rc; +} /* ** Move the iterator passed as the first argument to the next term in the ** segment. If successful, SQLITE_OK is returned. If there is no next term, ** SQLITE_DONE. Otherwise, an SQLite error code. */ -static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ +static int fts3SegReaderNext( + Fts3Table *p, + Fts3SegReader *pReader, + int bIncr +){ + int rc; /* Return code of various sub-routines */ char *pNext; /* Cursor variable */ int nPrefix; /* Number of bytes in term prefix */ int nSuffix; /* Number of bytes in term suffix */ @@ -1000,7 +1071,6 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ } if( !pNext || pNext>=&pReader->aNode[pReader->nNode] ){ - int rc; /* Return code from Fts3ReadBlock() */ if( fts3SegReaderIsPending(pReader) ){ Fts3HashElem *pElem = *(pReader->ppNextElem); @@ -1020,6 +1090,8 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ if( !fts3SegReaderIsRootOnly(pReader) ){ sqlite3_free(pReader->aNode); + sqlite3_blob_close(pReader->pBlob); + pReader->pBlob = 0; } pReader->aNode = 0; @@ -1031,11 +1103,20 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ } rc = sqlite3Fts3ReadBlock( - p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode + p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, + (bIncr ? &pReader->nPopulate : 0) ); if( rc!=SQLITE_OK ) return rc; + assert( pReader->pBlob==0 ); + if( bIncr && pReader->nPopulatenNode ){ + pReader->pBlob = p->pSegments; + p->pSegments = 0; + } pNext = pReader->aNode; } + + rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2); + if( rc!=SQLITE_OK ) return rc; /* Because of the FTS3_NODE_PADDING bytes of padding, the following is ** safe (no risk of overread) even if the node data is corrupted. @@ -1057,6 +1138,10 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ pReader->zTerm = zNew; pReader->nTermAlloc = nNew; } + + rc = fts3SegReaderRequire(pReader, pNext, nSuffix+FTS3_VARINT_MAX); + if( rc!=SQLITE_OK ) return rc; + memcpy(&pReader->zTerm[nPrefix], pNext, nSuffix); pReader->nTerm = nPrefix+nSuffix; pNext += nSuffix; @@ -1069,7 +1154,7 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ ** of these statements is untrue, then the data structure is corrupt. */ if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] - || pReader->aDoclist[pReader->nDoclist-1] + || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) ){ return SQLITE_CORRUPT_VTAB; } @@ -1080,12 +1165,16 @@ static int fts3SegReaderNext(Fts3Table *p, Fts3SegReader *pReader){ ** Set the SegReader to point to the first docid in the doclist associated ** with the current term. */ -static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){ - int n; +static int fts3SegReaderFirstDocid(Fts3SegReader *pReader){ + int rc; assert( pReader->aDoclist ); assert( !pReader->pOffsetList ); - n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); - pReader->pOffsetList = &pReader->aDoclist[n]; + rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); + pReader->pOffsetList = &pReader->aDoclist[n]; + } + return rc; } /* @@ -1098,11 +1187,12 @@ static void fts3SegReaderFirstDocid(Fts3SegReader *pReader){ ** *pnOffsetList is set to the length of the set of column-offset ** lists, not including the nul-terminator byte. For example: */ -static void fts3SegReaderNextDocid( +static int fts3SegReaderNextDocid( Fts3SegReader *pReader, char **ppOffsetList, int *pnOffsetList ){ + int rc = SQLITE_OK; char *p = pReader->pOffsetList; char c = 0; @@ -1110,7 +1200,16 @@ static void fts3SegReaderNextDocid( ** following two lines advance it to point one byte past the end of ** the same offset list. */ - while( *p | c ) c = *p++ & 0x80; + while( 1 ){ + int nRead; + int rc; + + while( *p | c ) c = *p++ & 0x80; + assert( *p==0 ); + if( pReader->pBlob==0 || (p - pReader->aNode)!=pReader->nPopulate ) break; + rc = fts3SegReaderIncrRead(pReader); + if( rc!=SQLITE_OK ) return rc; + } p++; /* If required, populate the output variables with a pointer to and the @@ -1129,10 +1228,15 @@ static void fts3SegReaderNextDocid( if( p>=&pReader->aDoclist[pReader->nDoclist] ){ pReader->pOffsetList = 0; }else{ - sqlite3_int64 iDelta; - pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); - pReader->iDocid += iDelta; + rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + sqlite3_int64 iDelta; + pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); + pReader->iDocid += iDelta; + } } + + return SQLITE_OK; } /* @@ -1212,7 +1316,7 @@ int sqlite3Fts3SegReaderCost( ** confirms this). */ for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){ - rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob); + rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob, 0); if( rc!=SQLITE_OK ) break; if( (nBlob+35)>pgsz ){ int nOvfl = (nBlob + 34)/pgsz; @@ -1247,7 +1351,7 @@ int sqlite3Fts3MsrOvfl( int jj; for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){ int nBlob; - rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob); + rc = sqlite3Fts3ReadBlock(p, jj, 0, &nBlob, 0); if( rc!=SQLITE_OK ) break; if( (nBlob+35)>pgsz ){ nOvfl += (nBlob + 34)/pgsz; @@ -1268,6 +1372,7 @@ void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ sqlite3_free(pReader->zTerm); if( !fts3SegReaderIsRootOnly(pReader) ){ sqlite3_free(pReader->aNode); + sqlite3_blob_close(pReader->pBlob); } } sqlite3_free(pReader); @@ -2193,7 +2298,7 @@ int sqlite3Fts3MsrIncrStart( for(i=0; iapSegment[i]; do { - int rc = fts3SegReaderNext(p, pSeg); + int rc = fts3SegReaderNext(p, pSeg, 1); if( rc!=SQLITE_OK ) return rc; }while( fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); } @@ -2210,8 +2315,10 @@ int sqlite3Fts3MsrIncrStart( /* Advance each of the segments to point to the first docid. */ for(i=0; inAdvance; i++){ - fts3SegReaderFirstDocid(pCsr->apSegment[i]); + int rc = fts3SegReaderFirstDocid(pCsr->apSegment[i]); + if( rc!=SQLITE_OK ) return rc; } + fts3SegReaderSort(pCsr->apSegment, i, i, fts3SegReaderDoclistCmp); assert( iCol<0 || iColnColumn ); pCsr->iColFilter = iCol; @@ -2226,7 +2333,6 @@ int sqlite3Fts3MsrIncrNext( char **paPoslist, /* OUT: Pointer to position list */ int *pnPoslist /* OUT: Size of position list in bytes */ ){ - int rc = SQLITE_OK; int nMerge = pMsr->nAdvance; Fts3SegReader **apSegment = pMsr->apSegment; @@ -2236,27 +2342,33 @@ int sqlite3Fts3MsrIncrNext( } while( 1 ){ + int nSort; Fts3SegReader *pSeg; - fts3SegReaderSort(pMsr->apSegment, nMerge, nMerge, fts3SegReaderDoclistCmp); pSeg = pMsr->apSegment[0]; if( pSeg->pOffsetList==0 ){ *paPoslist = 0; break; }else{ + int rc; char *pList; int nList; int j; sqlite3_int64 iDocid = apSegment[0]->iDocid; - fts3SegReaderNextDocid(apSegment[0], &pList, &nList); + rc = fts3SegReaderNextDocid(apSegment[0], &pList, &nList); j = 1; - while( jpOffsetList && apSegment[j]->iDocid==iDocid ){ fts3SegReaderNextDocid(apSegment[j], 0, 0); + j++; } + if( rc!=SQLITE_OK ) return rc; + + fts3SegReaderSort(pMsr->apSegment, nMerge, j, fts3SegReaderDoclistCmp); if( pMsr->iColFilter>=0 ){ fts3ColumnFilter(pMsr->iColFilter, &pList, &nList); @@ -2269,9 +2381,10 @@ int sqlite3Fts3MsrIncrNext( break; } } + } - return rc; + return SQLITE_OK; } int sqlite3Fts3SegReaderStart( @@ -2295,7 +2408,7 @@ int sqlite3Fts3SegReaderStart( const char *zTerm = pFilter->zTerm; Fts3SegReader *pSeg = pCsr->apSegment[i]; do { - int rc = fts3SegReaderNext(p, pSeg); + int rc = fts3SegReaderNext(p, pSeg, 0); if( rc!=SQLITE_OK ) return rc; }while( zTerm && fts3SegReaderTermCmp(pSeg, zTerm, nTerm)<0 ); } @@ -2331,7 +2444,7 @@ int sqlite3Fts3SegReaderStep( ** forward. Then sort the list in order of current term again. */ for(i=0; inAdvance; i++){ - rc = fts3SegReaderNext(p, apSegment[i]); + rc = fts3SegReaderNext(p, apSegment[i], 0); if( rc!=SQLITE_OK ) return rc; } fts3SegReaderSort(apSegment, nSegment, pCsr->nAdvance, fts3SegReaderCmp); @@ -2780,15 +2893,6 @@ static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ Fts3Phrase *pPhrase = pExpr->pPhrase; fts3DeferredDoclistClear(pExpr->pLeft); fts3DeferredDoclistClear(pExpr->pRight); - if( pPhrase ){ - assert( pExpr->eType==FTSQUERY_PHRASE ); - sqlite3_free(pPhrase->aDoclist); - pPhrase->isLoaded = 0; - pPhrase->aDoclist = 0; - pPhrase->nDoclist = 0; - pPhrase->pCurrent = 0; - pPhrase->iCurrent = 0; - } } } diff --git a/manifest b/manifest index 499805878d..058d53aadb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\simprove\sperformance\sand\ssupport\sLIMIT\sclauses\son\sfts3\stables.\sThis\sbranch\sis\sunstable\sfor\snow. -D 2011-06-02T19:57:24.733 +C FTS\schanges:\sRemove\sunreachable\scode.\sFix\sbugs.\sWhen\sprocessing\sa\slarge\sdoclist\sincrementally,\sread\sfrom\sdisk\sincrementally\stoo. +D 2011-06-03T18:00:19.691 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,21 +61,21 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c f92b6e0241a77a715d30dbeffd7c901053dbfda4 +F ext/fts3/fts3.c 3183cf6aa7bfb00f227be1950b326feb8294da7d F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h ab1489076e7d54714d20bbbc7aaef8e694a7db50 +F ext/fts3/fts3Int.h ba6f831fcde80ed5f0ccb112bb593b117cf11538 F ext/fts3/fts3_aux.c 28fc512608e147015c36080025456f57f571f76f -F ext/fts3/fts3_expr.c 5c789c744f98a007512f49d9cda4d2bb4cd56517 +F ext/fts3/fts3_expr.c 0ae554230ada457e61e8184b24faac96aad78f6b F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2 -F ext/fts3/fts3_snippet.c 10e0b0995ec82a2d93fbdf3159641cdf30f3c274 +F ext/fts3/fts3_snippet.c 0485969cce410760b50d587a77186f9c7f7e96be F ext/fts3/fts3_term.c 6c7f33ab732a2a0f281898685650e3a492e1e2f1 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c b145547430af9451f81cfed92fb7065da2efd870 +F ext/fts3/fts3_write.c c2f041d74f38e40c7487440f78d20f2aaa1f5d3c F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 84097a4c759b1d65890af885f137d3cb16eef584 -R e38e0cc84edbfdb5eae395b3be2f7a86 +P 28149a7882a1e9dfe4a75ec5b91d176ebe6284e9 +R b735e6f33e9ec766163a902e084c30a6 U dan -Z 53d59cf3dcd8991c66b0afd5fb898b1a +Z f3b0ba5ae8b73d0176f1d82c224a912a diff --git a/manifest.uuid b/manifest.uuid index 9e0b1f8e4f..36c53ac073 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -28149a7882a1e9dfe4a75ec5b91d176ebe6284e9 \ No newline at end of file +a4c7e2820824e82580730c36f85aede2efa66754 \ No newline at end of file From b46ee9172942d0ef12cbb14461b5fb1fc1367c8b Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 4 Jun 2011 20:04:35 +0000 Subject: [PATCH 07/20] Allow the "order=DESC" and "order=ASC" parameters in FTS4 "CREATE VIRTUAL TABLE" statements. Tables created with "order=DESC" store all doclists in descending order, which allows optimizations normally applied to "ORDER BY docid ASC" queries to be used with "ORDER BY docid DESC" queries instead. FossilOrigin-Name: f6a0193f5a32603eb48bddc6297042dbd2ffe96e --- ext/fts3/fts3.c | 558 ++++++++++++++++++++++++++++++------------ ext/fts3/fts3Int.h | 5 +- ext/fts3/fts3_write.c | 245 ++++++++++++------- manifest | 18 +- manifest.uuid | 2 +- test/fts3sort.test | 157 ++++++++---- 6 files changed, 682 insertions(+), 303 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 90b081b8c5..695d3e2d5e 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -423,12 +423,12 @@ static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ ** When this function is called, *pp points to the first byte following a ** varint that is part of a doclist (or position-list, or any other list ** of varints). This function moves *pp to point to the start of that varint, -** and decrements the value stored in *pVal by the varint value. +** and sets *pVal by the varint value. ** ** Argument pStart points to the first byte of the doclist that the ** varint is part of. */ -static void fts3GetReverseDeltaVarint( +static void fts3GetReverseVarint( char **pp, char *pStart, sqlite3_int64 *pVal @@ -444,7 +444,7 @@ static void fts3GetReverseDeltaVarint( *pp = p; sqlite3Fts3GetVarint(p, &iVal); - *pVal -= iVal; + *pVal = iVal; } /* @@ -916,17 +916,37 @@ static int fts3InitVtab( int nDb; /* Bytes required to hold database name */ int nName; /* Bytes required to hold table name */ int isFts4 = (argv[0][3]=='4'); /* True for FTS4, false for FTS3 */ - int bNoDocsize = 0; /* True to omit %_docsize table */ const char **aCol; /* Array of column names */ sqlite3_tokenizer *pTokenizer = 0; /* Tokenizer for this table */ - char *zPrefix = 0; /* Prefix parameter value (or NULL) */ int nIndex; /* Size of aIndex[] array */ struct Fts3Index *aIndex; /* Array of indexes for this table */ struct Fts3Index *aFree = 0; /* Free this before returning */ - char *zCompress = 0; - char *zUncompress = 0; + int bNoDocsize = 0; /* True to omit %_docsize table */ + int bDescIdx = 0; /* True to store descending indexes */ + + char *zMatchinfo = 0; /* Prefix parameter value (or NULL) */ + char *zPrefix = 0; /* Prefix parameter value (or NULL) */ + char *zCompress = 0; /* compress=? parameter (or NULL) */ + char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ + char *zOrder = 0; /* order=? parameter (or NULL) */ + struct Fts4Option { + const char *zOpt; + int nOpt; + char **pzVar; + } aFts4Opt[] = { + { "matchinfo", 9, 0 }, + { "prefix", 6, 0 }, + { "compress", 8, 0 }, + { "uncompress", 10, 0 }, + { "order", 5, 0 } + }; + aFts4Opt[0].pzVar = &zMatchinfo; + aFts4Opt[1].pzVar = &zPrefix; + aFts4Opt[2].pzVar = &zCompress; + aFts4Opt[3].pzVar = &zUncompress; + aFts4Opt[4].pzVar = &zOrder; assert( strlen(argv[0])==4 ); assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) @@ -967,32 +987,27 @@ static int fts3InitVtab( /* Check if it is an FTS4 special argument. */ else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ + int iOpt; if( !zVal ){ rc = SQLITE_NOMEM; - goto fts3_init_out; - } - if( nKey==9 && 0==sqlite3_strnicmp(z, "matchinfo", 9) ){ - if( strlen(zVal)==4 && 0==sqlite3_strnicmp(zVal, "fts3", 4) ){ - bNoDocsize = 1; - }else{ - *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); - rc = SQLITE_ERROR; - } - }else if( nKey==8 && 0==sqlite3_strnicmp(z, "compress", 8) ){ - zCompress = zVal; - zVal = 0; - }else if( nKey==10 && 0==sqlite3_strnicmp(z, "uncompress", 10) ){ - zUncompress = zVal; - zVal = 0; - }else if( nKey==6 && 0==sqlite3_strnicmp(z, "prefix", 6) ){ - sqlite3_free(zPrefix); - zPrefix = zVal; - zVal = 0; }else{ - *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); - rc = SQLITE_ERROR; + for(iOpt=0; iOptnMaxPendingData = FTS3_MAX_PENDING_DATA; p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; + p->bDescIdx = bDescIdx; TESTONLY( p->inTransaction = -1 ); TESTONLY( p->mxSavepoint = -1 ); @@ -1108,6 +1144,8 @@ fts3_init_out: sqlite3_free(aFree); sqlite3_free(zCompress); sqlite3_free(zUncompress); + sqlite3_free(zOrder); + sqlite3_free(zMatchinfo); sqlite3_free((void *)aCol); if( rc!=SQLITE_OK ){ if( p ){ @@ -1880,10 +1918,11 @@ static int fts3PoslistNearMerge( #define MERGE_NOT 2 /* D + D -> D */ #define MERGE_AND 3 /* D + D -> D */ #define MERGE_OR 4 /* D + D -> D */ -#define MERGE_POS_OR 5 /* P + P -> P */ #define MERGE_PHRASE 6 /* P + P -> D */ -#define MERGE_POS_PHRASE 7 /* P + P -> P */ #define MERGE_NEAR 8 /* P + P -> D */ + +#define MERGE_POS_OR 5 /* P + P -> P */ +#define MERGE_POS_PHRASE 7 /* P + P -> P */ #define MERGE_POS_NEAR 9 /* P + P -> P */ /* @@ -1924,6 +1963,7 @@ static int fts3DoclistMerge( || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE || mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR ); + assert( mergetype==MERGE_POS_PHRASE || mergetype==MERGE_POS_NEAR ); if( !aBuffer ){ *pnBuffer = 0; @@ -2065,6 +2105,227 @@ struct TermSelect { int anOutput[16]; /* Size of output in bytes */ }; + +static void fts3GetDeltaVarint3( + char **pp, + char *pEnd, + int bDescIdx, + sqlite3_int64 *pVal +){ + if( *pp>=pEnd ){ + *pp = 0; + }else{ + sqlite3_int64 iVal; + *pp += sqlite3Fts3GetVarint(*pp, &iVal); + if( bDescIdx ){ + *pVal -= iVal; + }else{ + *pVal += iVal; + } + } +} + +static void fts3PutDeltaVarint3( + char **pp, /* IN/OUT: Output pointer */ + int bDescIdx, /* True for descending docids */ + sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ + int *pbFirst, /* IN/OUT: True after first int written */ + sqlite3_int64 iVal /* Write this value to the list */ +){ + sqlite3_int64 iWrite; + if( bDescIdx==0 || *pbFirst==0 ){ + iWrite = iVal - *piPrev; + }else{ + iWrite = *piPrev - iVal; + } + assert( *pbFirst || *piPrev==0 ); + assert( *pbFirst==0 || iWrite>0 ); + *pp += sqlite3Fts3PutVarint(*pp, iWrite); + *piPrev = iVal; + *pbFirst = 1; +} + +#define COMPARE_DOCID(i1, i2) ((bDescIdx?-1:1) * (i1-i2)) + +static int fts3DoclistOrMerge( + int bDescIdx, /* True if arguments are desc */ + u8 *a1, int n1, /* First doclist */ + u8 *a2, int n2, /* Second doclist */ + u8 **paOut, int *pnOut /* OUT: Malloc'd doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &a1[n1]; + char *pEnd2 = &a2[n2]; + char *p1 = a1; + char *p2 = a2; + char *p; + int nOut; + char *aOut; + int bFirstOut = 0; + + *paOut = 0; + *pnOut = 0; + aOut = sqlite3_malloc(n1+n2); + if( !aOut ) return SQLITE_NOMEM; + + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + while( p1 || p2 ){ + sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + + if( p2 && p1 && iDiff==0 ){ + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PoslistMerge(&p, &p1, &p2); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + }else if( !p2 || (p1 && iDiff<0) ){ + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + fts3PoslistCopy(&p, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + }else{ + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i2); + fts3PoslistCopy(&p, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + } + } + + *paOut = aOut; + *pnOut = (p-aOut); + return SQLITE_OK; +} + +static void fts3DoclistPhraseMerge( + int bDescIdx, /* True if arguments are desc */ + int nDist, /* Distance from left to right (1=adjacent) */ + u8 *aLeft, int nLeft, /* Left doclist */ + u8 *aRight, int *pnRight /* IN/OUT: Right/output doclist */ +){ + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + char *pEnd1 = &aLeft[nLeft]; + char *pEnd2 = &aRight[*pnRight]; + char *p1 = aLeft; + char *p2 = aRight; + char *p; + int bFirstOut = 0; + char *aOut = aRight; + + assert( nDist>0 ); + + p = aOut; + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + + while( p1 && p2 ){ + sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + if( iDiff==0 ){ + char *pSave = p; + sqlite3_int64 iPrevSave = iPrev; + int bFirstOutSave = bFirstOut; + + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + if( 0==fts3PoslistPhraseMerge(&p, nDist, 0, 1, &p1, &p2) ){ + p = pSave; + iPrev = iPrevSave; + bFirstOut = bFirstOutSave; + } + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + }else if( iDiff<0 ){ + fts3PoslistCopy(0, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + }else{ + fts3PoslistCopy(0, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + } + } + + *pnRight = p - aOut; +} + +/* +** This function merges two doclists according to the requirements of a +** NEAR operator. +*/ +static int fts3DoclistNearMerge( + int bDescIdx, + int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */ + int nNear, /* Parameter to NEAR operator */ + int nTokenLeft, /* Number of tokens in LHS phrase arg */ + char *aLeft, /* Doclist for LHS (incl. positions) */ + int nLeft, /* Size of LHS doclist in bytes */ + int nTokenRight, /* As nTokenLeft */ + char *aRight, /* As aLeft */ + int nRight, /* As nRight */ + char **paOut, /* OUT: Results of merge (malloced) */ + int *pnOut /* OUT: Sized of output buffer */ +){ + char *aOut; /* Buffer to write output doclist to */ + char *aTmp; /* Temp buffer used by PoslistNearMerge() */ + + sqlite3_int64 i1 = 0; + sqlite3_int64 i2 = 0; + sqlite3_int64 iPrev = 0; + int bFirstOut = 0; + + char *pEnd1 = &aLeft[nLeft]; + char *pEnd2 = &aRight[nRight]; + char *p1 = aLeft; + char *p2 = aRight; + char *p; + + int nParam1 = nNear+nTokenRight; + int nParam2 = nNear+nTokenLeft; + + p = aOut = sqlite3_malloc(nLeft+nRight+1); + aTmp = sqlite3_malloc(2*(nLeft+nRight+1)); + if( !aOut || !aTmp ){ + sqlite3_free(aOut); + sqlite3_free(aTmp); + *paOut = 0; + *pnOut = 0; + return SQLITE_NOMEM; + } + + fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); + + while( p1 && p2 ){ + sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); + if( iDiff==0 ){ + char *pSave = p; + sqlite3_int64 iPrevSave = iPrev; + int bFirstOutSave = bFirstOut; + fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); + if( !fts3PoslistNearMerge(&p, aTmp, nParam1, nParam2, &p1, &p2) ){ + p = pSave; + iPrev = iPrevSave; + bFirstOut = bFirstOutSave; + } + + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + }else if( iDiff<0 ){ + fts3PoslistCopy(0, &p1); + fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); + }else{ + fts3PoslistCopy(0, &p2); + fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); + } + } + + sqlite3_free(aTmp); + *paOut = aOut; + *pnOut = p - aOut; + return SQLITE_OK; +} + + + /* ** Merge all doclists in the TermSelect.aaOutput[] array into a single ** doclist stored in TermSelect.aaOutput[0]. If successful, delete all @@ -2074,7 +2335,7 @@ struct TermSelect { ** the responsibility of the caller to free any doclists left in the ** TermSelect.aaOutput[] array. */ -static int fts3TermSelectMerge(TermSelect *pTS){ +static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){ int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR); char *aOut = 0; int nOut = 0; @@ -2090,15 +2351,17 @@ static int fts3TermSelectMerge(TermSelect *pTS){ nOut = pTS->anOutput[i]; pTS->aaOutput[i] = 0; }else{ - int nNew = nOut + pTS->anOutput[i]; - char *aNew = sqlite3_malloc(nNew); - if( !aNew ){ - sqlite3_free(aOut); - return SQLITE_NOMEM; - } - fts3DoclistMerge(mergetype, 0, 0, - aNew, &nNew, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, 0 + int nNew; + u8 *aNew; + + int rc = fts3DoclistOrMerge(p->bDescIdx, + pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew ); + if( rc!=SQLITE_OK ){ + sqlite3_free(aOut); + return rc; + } + sqlite3_free(pTS->aaOutput[i]); sqlite3_free(aOut); pTS->aaOutput[i] = 0; @@ -2134,9 +2397,7 @@ static int fts3TermSelectCb( if( pTS->aaOutput[0]==0 ){ /* If this is the first term selected, copy the doclist to the output - ** buffer using memcpy(). TODO: Add a way to transfer control of the - ** aDoclist buffer from the caller so as to avoid the memcpy(). - */ + ** buffer using memcpy(). */ pTS->aaOutput[0] = sqlite3_malloc(nDoclist); pTS->anOutput[0] = nDoclist; if( pTS->aaOutput[0] ){ @@ -2151,36 +2412,33 @@ static int fts3TermSelectCb( int iOut; for(iOut=0; iOutaaOutput); iOut++){ - char *aNew; - int nNew; if( pTS->aaOutput[iOut]==0 ){ assert( iOut>0 ); pTS->aaOutput[iOut] = aMerge; pTS->anOutput[iOut] = nMerge; break; - } + }else{ + u8 *aNew; + int nNew; - nNew = nMerge + pTS->anOutput[iOut]; - aNew = sqlite3_malloc(nNew); - if( !aNew ){ - if( aMerge!=aDoclist ){ - sqlite3_free(aMerge); + int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, + pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew + ); + if( rc!=SQLITE_OK ){ + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + return rc; } - return SQLITE_NOMEM; - } - fts3DoclistMerge(mergetype, 0, 0, aNew, &nNew, - pTS->aaOutput[iOut], pTS->anOutput[iOut], aMerge, nMerge, 0 - ); - if( iOut>0 ) sqlite3_free(aMerge); - sqlite3_free(pTS->aaOutput[iOut]); - pTS->aaOutput[iOut] = 0; - - aMerge = aNew; - nMerge = nNew; - if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ - pTS->aaOutput[iOut] = aMerge; - pTS->anOutput[iOut] = nMerge; + if( aMerge!=aDoclist ) sqlite3_free(aMerge); + sqlite3_free(pTS->aaOutput[iOut]); + pTS->aaOutput[iOut] = 0; + + aMerge = aNew; + nMerge = nNew; + if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ + pTS->aaOutput[iOut] = aMerge; + pTS->anOutput[iOut] = nMerge; + } } } } @@ -2426,7 +2684,7 @@ static int fts3TermSelect( } if( rc==SQLITE_OK ){ - rc = fts3TermSelectMerge(&tsc); + rc = fts3TermSelectMerge(p, &tsc); } if( rc==SQLITE_OK ){ *ppOut = tsc.aaOutput[0]; @@ -2476,48 +2734,6 @@ static int fts3DoclistCountDocids(int isPoslist, char *aList, int nList){ return nDoc; } -/* -** This function merges two doclists according to the requirements of a -** NEAR operator. -** -** Both input doclists must include position information. The output doclist -** includes position information if the first argument to this function -** is MERGE_POS_NEAR, or does not if it is MERGE_NEAR. -*/ -static int fts3NearMerge( - int mergetype, /* MERGE_POS_NEAR or MERGE_NEAR */ - int nNear, /* Parameter to NEAR operator */ - int nTokenLeft, /* Number of tokens in LHS phrase arg */ - char *aLeft, /* Doclist for LHS (incl. positions) */ - int nLeft, /* Size of LHS doclist in bytes */ - int nTokenRight, /* As nTokenLeft */ - char *aRight, /* As aLeft */ - int nRight, /* As nRight */ - char **paOut, /* OUT: Results of merge (malloced) */ - int *pnOut /* OUT: Sized of output buffer */ -){ - char *aOut; /* Buffer to write output doclist to */ - int rc; /* Return code */ - - assert( mergetype==MERGE_POS_NEAR || MERGE_NEAR ); - - aOut = sqlite3_malloc(nLeft+nRight+1); - if( aOut==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = fts3DoclistMerge(mergetype, nNear+nTokenRight, nNear+nTokenLeft, - aOut, pnOut, aLeft, nLeft, aRight, nRight, 0 - ); - if( rc!=SQLITE_OK ){ - sqlite3_free(aOut); - aOut = 0; - } - } - - *paOut = aOut; - return rc; -} - /* ** Advance the cursor to the next row in the %_content table that ** matches the search criteria. For a MATCH search, this will be @@ -2588,7 +2804,11 @@ static int fts3FilterMethod( sqlite3Fts3ExprFree(pCsr->pExpr); memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); - pCsr->bDesc = (idxStr && idxStr[0]=='D'); + if( idxStr ){ + pCsr->bDesc = (idxStr[0]=='D'); + }else{ + pCsr->bDesc = p->bDescIdx; + } pCsr->eSearch = (i16)idxNum; if( idxNum!=FTS3_DOCID_SEARCH && idxNum!=FTS3_FULLSCAN_SEARCH ){ @@ -2627,7 +2847,7 @@ static int fts3FilterMethod( ** row by docid. */ if( idxNum==FTS3_FULLSCAN_SEARCH ){ - const char *zSort = (idxStr ? idxStr : "ASC"); + const char *zSort = (pCsr->bDesc ? "DESC" : "ASC"); const char *zTmpl = "SELECT %s FROM %Q.'%q_content' AS x ORDER BY docid %s"; zSql = sqlite3_mprintf(zTmpl, p->zReadExprlist, p->zDb, p->zName, zSort); }else{ @@ -3265,8 +3485,8 @@ static int fts3EvalPhraseLoad( nDoclist = nThis; }else{ assert( iPrev>=0 ); - fts3DoclistMerge(MERGE_POS_PHRASE, iToken-iPrev, - 0, pThis, &nThis, aDoclist, nDoclist, pThis, nThis, 0 + fts3DoclistPhraseMerge(pTab->bDescIdx, + iToken-iPrev, aDoclist, nDoclist, pThis, &nThis ); sqlite3_free(aDoclist); aDoclist = pThis; @@ -3411,14 +3631,14 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ int rc; Fts3Doclist *pList = &p->doclist; Fts3PhraseToken *pFirst = &p->aToken[0]; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; assert( pList->aAll==0 ); - if( pCsr->bDesc==0 && bOptOk==1 && p->nToken==1 + if( pCsr->bDesc==pTab->bDescIdx && bOptOk==1 && p->nToken==1 && pFirst->pSegcsr && pFirst->pSegcsr->bLookup ){ /* Use the incremental approach. */ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int iCol = (p->iColumn >= pTab->nColumn ? -1 : p->iColumn); rc = sqlite3Fts3MsrIncrStart( pTab, pFirst->pSegcsr, iCol, pFirst->z, pFirst->n); @@ -3434,6 +3654,58 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ return rc; } +void sqlite3Fts3DoclistPrev( + int bDescIdx, /* True if the doclist is desc */ + char *aDoclist, /* Pointer to entire doclist */ + int nDoclist, /* Length of aDoclist in bytes */ + char **ppIter, /* IN/OUT: Iterator pointer */ + sqlite3_int64 *piDocid, /* IN/OUT: Docid pointer */ + int *pnList, /* IN/OUT: List length pointer */ + u8 *pbEof /* OUT: End-of-file flag */ +){ + char *p = *ppIter; + int iMul = (bDescIdx ? -1 : 1); + + assert( *pbEof==0 ); + assert( p || *piDocid==0 ); + assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); + + if( p==0 ){ + sqlite3_int64 iDocid = 0; + char *pNext = 0; + char *pDocid = aDoclist; + char *pEnd = &aDoclist[nDoclist]; + + pDocid += sqlite3Fts3GetVarint(pDocid, &iDocid); + pNext = pDocid; + fts3PoslistCopy(0, &pDocid); + while( pDociddoclist; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; if( p->bIncr ){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; assert( p->nToken==1 ); rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, &pDL->iDocid, &pDL->pList, &pDL->nList @@ -3460,42 +3732,12 @@ static int fts3EvalPhraseNext( if( rc==SQLITE_OK && !pDL->pList ){ *pbEof = 1; } - }else if( pCsr->bDesc && pDL->aAll ){ - - if( pDL->pNextDocid==0 ){ - sqlite3_int64 iDocid = 0; - char *pNext; - char *pDocid = pDL->aAll; - char *pEnd = &pDocid[pDL->nAll]; - - while( pDocidpNextDocid = pDocid; - pDL->pList = pDocid; - fts3PoslistCopy(0, &pDocid); - } - pDL->nList = (pEnd - pDL->pList); - pDL->iDocid = iDocid; - }else{ - - assert( *pbEof==0 ); - assert( pDL->pNextDocid>pDL->aAll ); - - fts3GetReverseDeltaVarint( - &pDL->pNextDocid, pDL->aAll, &pDL->iDocid - ); - if( pDL->pNextDocid==pDL->aAll ){ - *pbEof = 1; - }else{ - char *pSave = pDL->pNextDocid; - fts3ReversePoslist(pDL->aAll, &pDL->pNextDocid); - pDL->pList = pDL->pNextDocid; - pDL->nList = pSave - pDL->pNextDocid; - } - } - + }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->aAll ){ + sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, + &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof + ); + pDL->pList = pDL->pNextDocid; }else{ - char *pIter; if( pDL->pNextDocid ){ pIter = pDL->pNextDocid; @@ -3507,7 +3749,13 @@ static int fts3EvalPhraseNext( /* We have already reached the end of this doclist. EOF. */ *pbEof = 1; }else{ - fts3GetDeltaVarint(&pIter, &pDL->iDocid); + sqlite3_int64 iDelta; + pIter += sqlite3Fts3GetVarint(pIter, &iDelta); + if( pTab->bDescIdx==0 || pDL->pNextDocid==0 ){ + pDL->iDocid += iDelta; + }else{ + pDL->iDocid -= iDelta; + } pDL->pList = pIter; fts3PoslistCopy(0, &pIter); pDL->nList = (pIter - pDL->pList); @@ -3546,6 +3794,7 @@ static void fts3EvalStartReaders( } static void fts3EvalNearMerge( + int bDescIdx, Fts3Expr *p1, Fts3Expr *p2, int nNear, @@ -3567,7 +3816,7 @@ static void fts3EvalNearMerge( char *aOut; /* Buffer in which to assemble new doclist */ int nOut; /* Size of buffer aOut in bytes */ - *pRc = fts3NearMerge(MERGE_POS_NEAR, nNear, + *pRc = fts3DoclistNearMerge(bDescIdx, MERGE_POS_NEAR, nNear, pLeft->nToken, pLeft->doclist.aAll, pLeft->doclist.nAll, pRight->nToken, pRight->doclist.aAll, pRight->doclist.nAll, &aOut, &nOut @@ -3602,6 +3851,7 @@ static void fts3EvalNearTrim(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ if( !aPhrase ){ *pRc = SQLITE_NOMEM; }else{ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; int i = 1; aPhrase[0] = pLeft; do { @@ -3611,11 +3861,11 @@ static void fts3EvalNearTrim(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ for(i=0; i<(nPhrase-1); i++){ int nNear = aPhrase[i+1]->pParent->nNear; - fts3EvalNearMerge(aPhrase[i], aPhrase[i+1], nNear, pRc); + fts3EvalNearMerge(p->bDescIdx, aPhrase[i], aPhrase[i+1], nNear, pRc); } for(i=nPhrase-2; i>=0; i--){ int nNear = aPhrase[i+1]->pParent->nNear; - fts3EvalNearMerge(aPhrase[i+1], aPhrase[i], nNear, pRc); + fts3EvalNearMerge(p->bDescIdx, aPhrase[i+1], aPhrase[i], nNear, pRc); } sqlite3_free(aPhrase); diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index c6834d37b6..c6329c30d0 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -180,6 +180,7 @@ struct Fts3Table { int nNodeSize; /* Soft limit for node size */ u8 bHasStat; /* True if %_stat table exists */ u8 bHasDocsize; /* True if %_docsize table exists */ + u8 bDescIdx; /* True if doclists are in reverse order */ int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ @@ -236,7 +237,7 @@ struct Fts3Cursor { char *pNextId; /* Pointer into the body of aDoclist */ char *aDoclist; /* List of docids for full-text queries */ int nDoclist; /* Size of buffer at aDoclist */ - int bDesc; /* True to sort in descending order */ + u8 bDesc; /* True to sort in descending order */ int eEvalmode; /* An FTS3_EVAL_XX constant */ int nRowAvg; /* Average size of database rows, in pages */ @@ -438,6 +439,7 @@ int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); int sqlite3Fts3GetVarint32(const char *, int *); int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); +void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *); int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int); @@ -495,4 +497,5 @@ int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *); + #endif /* _FTSINT_H */ diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 0470a72960..fd964739b2 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -62,9 +62,8 @@ typedef struct SegmentNode SegmentNode; typedef struct SegmentWriter SegmentWriter; /* -** Data structure used while accumulating terms in the pending-terms hash -** table. The hash table entry maps from term (a string) to a malloc'd -** instance of this structure. +** An instance of the following data structure is used to build doclists +** incrementally. See function fts3PendingListAppend() for details. */ struct PendingList { int nData; @@ -130,8 +129,11 @@ struct Fts3SegReader { char *aDoclist; /* Pointer to doclist of current entry */ int nDoclist; /* Size of doclist in current entry */ - /* The following variables are used to iterate through the current doclist */ + /* The following variables are used by fts3SegReaderNextDocid() to iterate + ** through the current doclist (aDoclist/nDoclist). + */ char *pOffsetList; + int nOffsetList; /* For descending pending seg-readers only */ sqlite3_int64 iDocid; }; @@ -573,11 +575,21 @@ static int fts3PendingListAppend( return 0; } +/* +** Free a PendingList object allocated by fts3PendingListAppend(). +*/ +static void fts3PendingListDelete(PendingList *pList){ + sqlite3_free(pList); +} + +/* +** Add an entry to one of the pending-terms hash tables. +*/ static int fts3PendingTermsAddOne( Fts3Table *p, int iCol, int iPos, - Fts3Hash *pHash, + Fts3Hash *pHash, /* Pending terms hash table to add entry to */ const char *zToken, int nToken ){ @@ -713,7 +725,8 @@ void sqlite3Fts3PendingTermsClear(Fts3Table *p){ Fts3HashElem *pElem; Fts3Hash *pHash = &p->aIndex[i].hPending; for(pElem=fts3HashFirst(pHash); pElem; pElem=fts3HashNext(pElem)){ - sqlite3_free(fts3HashData(pElem)); + PendingList *pList = (PendingList *)fts3HashData(pElem); + fts3PendingListDelete(pList); } fts3HashClear(pHash); } @@ -1115,12 +1128,13 @@ static int fts3SegReaderNext( pNext = pReader->aNode; } + assert( !fts3SegReaderIsPending(pReader) ); + rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2); if( rc!=SQLITE_OK ) return rc; /* Because of the FTS3_NODE_PADDING bytes of padding, the following is - ** safe (no risk of overread) even if the node data is corrupted. - */ + ** safe (no risk of overread) even if the node data is corrupted. */ pNext += sqlite3Fts3GetVarint32(pNext, &nPrefix); pNext += sqlite3Fts3GetVarint32(pNext, &nSuffix); if( nPrefix<0 || nSuffix<=0 @@ -1165,14 +1179,24 @@ static int fts3SegReaderNext( ** Set the SegReader to point to the first docid in the doclist associated ** with the current term. */ -static int fts3SegReaderFirstDocid(Fts3SegReader *pReader){ - int rc; +static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){ + int rc = SQLITE_OK; assert( pReader->aDoclist ); assert( !pReader->pOffsetList ); - rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); - if( rc==SQLITE_OK ){ - int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); - pReader->pOffsetList = &pReader->aDoclist[n]; + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + u8 bEof = 0; + pReader->iDocid = 0; + pReader->nOffsetList = 0; + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, + &pReader->iDocid, &pReader->nOffsetList, &bEof + ); + }else{ + rc = fts3SegReaderRequire(pReader, pReader->aDoclist, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + int n = sqlite3Fts3GetVarint(pReader->aDoclist, &pReader->iDocid); + pReader->pOffsetList = &pReader->aDoclist[n]; + } } return rc; } @@ -1188,51 +1212,83 @@ static int fts3SegReaderFirstDocid(Fts3SegReader *pReader){ ** lists, not including the nul-terminator byte. For example: */ static int fts3SegReaderNextDocid( - Fts3SegReader *pReader, - char **ppOffsetList, - int *pnOffsetList + Fts3Table *pTab, + Fts3SegReader *pReader, /* Reader to advance to next docid */ + char **ppOffsetList, /* OUT: Pointer to current position-list */ + int *pnOffsetList /* OUT: Length of *ppOffsetList in bytes */ ){ int rc = SQLITE_OK; char *p = pReader->pOffsetList; char c = 0; - /* Pointer p currently points at the first byte of an offset list. The - ** following two lines advance it to point one byte past the end of - ** the same offset list. - */ - while( 1 ){ - int nRead; - int rc; + assert( p ); - while( *p | c ) c = *p++ & 0x80; - assert( *p==0 ); - if( pReader->pBlob==0 || (p - pReader->aNode)!=pReader->nPopulate ) break; - rc = fts3SegReaderIncrRead(pReader); - if( rc!=SQLITE_OK ) return rc; - } - p++; - - /* If required, populate the output variables with a pointer to and the - ** size of the previous offset-list. - */ - if( ppOffsetList ){ - *ppOffsetList = pReader->pOffsetList; - *pnOffsetList = (int)(p - pReader->pOffsetList - 1); - } - - /* If there are no more entries in the doclist, set pOffsetList to - ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and - ** Fts3SegReader.pOffsetList to point to the next offset list before - ** returning. - */ - if( p>=&pReader->aDoclist[pReader->nDoclist] ){ - pReader->pOffsetList = 0; + if( pTab->bDescIdx && fts3SegReaderIsPending(pReader) ){ + /* A pending-terms seg-reader for an FTS4 table that uses order=desc. + ** Pending-terms doclists are always built up in ascending order, so + ** we have to iterate through them backwards here. */ + u8 bEof = 0; + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = pReader->nOffsetList - 1; + } + sqlite3Fts3DoclistPrev(0, + pReader->aDoclist, pReader->nDoclist, &p, &pReader->iDocid, + &pReader->nOffsetList, &bEof + ); + if( bEof ){ + pReader->pOffsetList = 0; + }else{ + pReader->pOffsetList = p; + } }else{ - rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); - if( rc==SQLITE_OK ){ - sqlite3_int64 iDelta; - pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); - pReader->iDocid += iDelta; + + /* Pointer p currently points at the first byte of an offset list. The + ** following block advances it to point one byte past the end of + ** the same offset list. */ + while( 1 ){ + + /* The following line of code (and the "p++" below the while() loop) is + ** normally all that is required to move pointer p to the desired + ** position. The exception is if this node is being loaded from disk + ** incrementally and pointer "p" now points to the first byte passed + ** the populated part of pReader->aNode[]. + */ + while( *p | c ) c = *p++ & 0x80; + assert( *p==0 ); + + if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break; + rc = fts3SegReaderIncrRead(pReader); + if( rc!=SQLITE_OK ) return rc; + } + p++; + + /* If required, populate the output variables with a pointer to and the + ** size of the previous offset-list. + */ + if( ppOffsetList ){ + *ppOffsetList = pReader->pOffsetList; + *pnOffsetList = (int)(p - pReader->pOffsetList - 1); + } + + /* If there are no more entries in the doclist, set pOffsetList to + ** NULL. Otherwise, set Fts3SegReader.iDocid to the next docid and + ** Fts3SegReader.pOffsetList to point to the next offset list before + ** returning. + */ + if( p>=&pReader->aDoclist[pReader->nDoclist] ){ + pReader->pOffsetList = 0; + }else{ + rc = fts3SegReaderRequire(pReader, p, FTS3_VARINT_MAX); + if( rc==SQLITE_OK ){ + sqlite3_int64 iDelta; + pReader->pOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); + if( pTab->bDescIdx ){ + pReader->iDocid -= iDelta; + }else{ + pReader->iDocid += iDelta; + } + } } } @@ -1601,6 +1657,18 @@ static int fts3SegReaderDoclistCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ assert( pLhs->aNode && pRhs->aNode ); return rc; } +static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ + int rc = (pLhs->pOffsetList==0)-(pRhs->pOffsetList==0); + if( rc==0 ){ + if( pLhs->iDocid==pRhs->iDocid ){ + rc = pRhs->iIdx - pLhs->iIdx; + }else{ + rc = (pLhs->iDocid < pRhs->iDocid) ? 1 : -1; + } + } + assert( pLhs->aNode && pRhs->aNode ); + return rc; +} /* ** Compare the term that the Fts3SegReader object passed as the first argument @@ -2290,6 +2358,9 @@ int sqlite3Fts3MsrIncrStart( ){ int i; int nSegment = pCsr->nSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); assert( pCsr->pFilter==0 ); assert( zTerm && nTerm>0 ); @@ -2315,10 +2386,10 @@ int sqlite3Fts3MsrIncrStart( /* Advance each of the segments to point to the first docid. */ for(i=0; inAdvance; i++){ - int rc = fts3SegReaderFirstDocid(pCsr->apSegment[i]); + int rc = fts3SegReaderFirstDocid(p, pCsr->apSegment[i]); if( rc!=SQLITE_OK ) return rc; } - fts3SegReaderSort(pCsr->apSegment, i, i, fts3SegReaderDoclistCmp); + fts3SegReaderSort(pCsr->apSegment, i, i, xCmp); assert( iCol<0 || iColnColumn ); pCsr->iColFilter = iCol; @@ -2335,6 +2406,9 @@ int sqlite3Fts3MsrIncrNext( ){ int nMerge = pMsr->nAdvance; Fts3SegReader **apSegment = pMsr->apSegment; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); if( nMerge==0 ){ *paPoslist = 0; @@ -2356,19 +2430,18 @@ int sqlite3Fts3MsrIncrNext( int j; sqlite3_int64 iDocid = apSegment[0]->iDocid; - rc = fts3SegReaderNextDocid(apSegment[0], &pList, &nList); + rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); j = 1; while( rc==SQLITE_OK && jpOffsetList && apSegment[j]->iDocid==iDocid ){ - fts3SegReaderNextDocid(apSegment[j], 0, 0); + rc = fts3SegReaderNextDocid(p, apSegment[j], 0, 0); j++; } if( rc!=SQLITE_OK ) return rc; - - fts3SegReaderSort(pMsr->apSegment, nMerge, j, fts3SegReaderDoclistCmp); + fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); if( pMsr->iColFilter>=0 ){ fts3ColumnFilter(pMsr->iColFilter, &pList, &nList); @@ -2433,6 +2506,9 @@ int sqlite3Fts3SegReaderStep( Fts3SegReader **apSegment = pCsr->apSegment; int nSegment = pCsr->nSegment; Fts3SegFilter *pFilter = pCsr->pFilter; + int (*xCmp)(Fts3SegReader *, Fts3SegReader *) = ( + p->bDescIdx ? fts3SegReaderDoclistCmpRev : fts3SegReaderDoclistCmp + ); if( pCsr->nSegment==0 ) return SQLITE_OK; @@ -2483,7 +2559,10 @@ int sqlite3Fts3SegReaderStep( } assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); - if( nMerge==1 && !isIgnoreEmpty ){ + if( nMerge==1 + && !isIgnoreEmpty + && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) + ){ pCsr->aDoclist = apSegment[0]->aDoclist; pCsr->nDoclist = apSegment[0]->nDoclist; rc = SQLITE_ROW; @@ -2496,22 +2575,22 @@ int sqlite3Fts3SegReaderStep( ** and a single term returned with the merged doclist. */ for(i=0; ipOffsetList ){ int j; /* Number of segments that share a docid */ char *pList; int nList; int nByte; sqlite3_int64 iDocid = apSegment[0]->iDocid; - fts3SegReaderNextDocid(apSegment[0], &pList, &nList); + fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); j = 1; while( jpOffsetList && apSegment[j]->iDocid==iDocid ){ - fts3SegReaderNextDocid(apSegment[j], 0, 0); + fts3SegReaderNextDocid(p, apSegment[j], 0, 0); j++; } @@ -2520,7 +2599,19 @@ int sqlite3Fts3SegReaderStep( } if( !isIgnoreEmpty || nList>0 ){ - nByte = sqlite3Fts3VarintLen(iDocid-iPrev) + (isRequirePos?nList+1:0); + + /* Calculate the 'docid' delta value to write into the merged + ** doclist. */ + sqlite3_int64 iDelta; + if( p->bDescIdx && nDoclist>0 ){ + iDelta = iPrev - iDocid; + }else{ + iDelta = iDocid - iPrev; + } + assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) ); + assert( nDoclist>0 || iDelta==iDocid ); + + nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); if( nDoclist+nByte>pCsr->nBuffer ){ char *aNew; pCsr->nBuffer = (nDoclist+nByte)*2; @@ -2530,9 +2621,7 @@ int sqlite3Fts3SegReaderStep( } pCsr->aBuffer = aNew; } - nDoclist += sqlite3Fts3PutVarint( - &pCsr->aBuffer[nDoclist], iDocid-iPrev - ); + nDoclist += sqlite3Fts3PutVarint(&pCsr->aBuffer[nDoclist], iDelta); iPrev = iDocid; if( isRequirePos ){ memcpy(&pCsr->aBuffer[nDoclist], pList, nList); @@ -2541,7 +2630,7 @@ int sqlite3Fts3SegReaderStep( } } - fts3SegReaderSort(apSegment, nMerge, j, fts3SegReaderDoclistCmp); + fts3SegReaderSort(apSegment, nMerge, j, xCmp); } if( nDoclist>0 ){ pCsr->aDoclist = pCsr->aBuffer; @@ -2883,19 +2972,6 @@ char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){ return 0; } -/* -** Helper fucntion for FreeDeferredDoclists(). This function removes all -** references to deferred doclists from within the tree of Fts3Expr -** structures headed by -*/ -static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ - if( pExpr ){ - Fts3Phrase *pPhrase = pExpr->pPhrase; - fts3DeferredDoclistClear(pExpr->pLeft); - fts3DeferredDoclistClear(pExpr->pRight); - } -} - /* ** Delete all cached deferred doclists. Deferred doclists are cached ** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. @@ -2903,12 +2979,9 @@ static void fts3DeferredDoclistClear(Fts3Expr *pExpr){ void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ Fts3DeferredToken *pDef; for(pDef=pCsr->pDeferred; pDef; pDef=pDef->pNext){ - sqlite3_free(pDef->pList); + fts3PendingListDelete(pDef->pList); pDef->pList = 0; } - if( pCsr->pDeferred ){ - fts3DeferredDoclistClear(pCsr->pExpr); - } } /* @@ -2920,7 +2993,7 @@ void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ Fts3DeferredToken *pNext; for(pDef=pCsr->pDeferred; pDef; pDef=pNext){ pNext = pDef->pNext; - sqlite3_free(pDef->pList); + fts3PendingListDelete(pDef->pList); sqlite3_free(pDef); } pCsr->pDeferred = 0; diff --git a/manifest b/manifest index 058d53aadb..7bf78b4a1d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C FTS\schanges:\sRemove\sunreachable\scode.\sFix\sbugs.\sWhen\sprocessing\sa\slarge\sdoclist\sincrementally,\sread\sfrom\sdisk\sincrementally\stoo. -D 2011-06-03T18:00:19.691 +C Allow\sthe\s"order=DESC"\sand\s"order=ASC"\sparameters\sin\sFTS4\s"CREATE\sVIRTUAL\sTABLE"\sstatements.\sTables\screated\swith\s"order=DESC"\sstore\sall\sdoclists\sin\sdescending\sorder,\swhich\sallows\soptimizations\snormally\sapplied\sto\s"ORDER\sBY\sdocid\sASC"\squeries\sto\sbe\sused\swith\s"ORDER\sBY\sdocid\sDESC"\squeries\sinstead. +D 2011-06-04T20:04:35.492 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,9 +61,9 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 3183cf6aa7bfb00f227be1950b326feb8294da7d +F ext/fts3/fts3.c 432c902c697850cbc5ffc2be8a945ac7ac3328d7 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h ba6f831fcde80ed5f0ccb112bb593b117cf11538 +F ext/fts3/fts3Int.h d76b021d5b7061eff7aa4055b5938eebef2bdb6a F ext/fts3/fts3_aux.c 28fc512608e147015c36080025456f57f571f76f F ext/fts3/fts3_expr.c 0ae554230ada457e61e8184b24faac96aad78f6b F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c @@ -75,7 +75,7 @@ F ext/fts3/fts3_term.c 6c7f33ab732a2a0f281898685650e3a492e1e2f1 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c c2f041d74f38e40c7487440f78d20f2aaa1f5d3c +F ext/fts3/fts3_write.c d7eded6f5ee3032b41126047cc04b6720f59e6da F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -478,7 +478,7 @@ F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 F test/fts3rnd.test 2b1a579be557ab8ac54a51b39caa4aa8043cc4ad F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test a12f22a3ba4dd59751a57c79b031d07ab5f51ddd -F test/fts3sort.test e6f24e9cffc46484bcc9fe63d3c2ce41afcaa6c9 +F test/fts3sort.test c599f19e9452b5735c41889ea5a6da996f0ebc7c F test/fts4aa.test eadf85621c0a113d4c7ad3ccbf8441130e007b8f F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 28149a7882a1e9dfe4a75ec5b91d176ebe6284e9 -R b735e6f33e9ec766163a902e084c30a6 +P a4c7e2820824e82580730c36f85aede2efa66754 +R 2bf28954217e137f11160d52b346ca5f U dan -Z f3b0ba5ae8b73d0176f1d82c224a912a +Z b16ce48fbf2f9ba7236a3b596adb6f59 diff --git a/manifest.uuid b/manifest.uuid index 36c53ac073..2d5316d141 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4c7e2820824e82580730c36f85aede2efa66754 \ No newline at end of file +f6a0193f5a32603eb48bddc6297042dbd2ffe96e \ No newline at end of file diff --git a/test/fts3sort.test b/test/fts3sort.test index 001bef1135..d2edbc2ade 100644 --- a/test/fts3sort.test +++ b/test/fts3sort.test @@ -21,9 +21,8 @@ ifcapable !fts3 { return } -set testprefix fts3sort -proc build_database {nRow} { +proc build_database {nRow param} { db close forcedelete test.db sqlite3 db test.db @@ -31,7 +30,7 @@ proc build_database {nRow} { set vocab [list aa ab ac ba bb bc ca cb cc da] expr srand(0) - execsql { CREATE VIRTUAL TABLE t1 USING fts4 } + execsql "CREATE VIRTUAL TABLE t1 USING fts4($param)" for {set i 0} {$i < $nRow} {incr i} { set v [expr int(rand()*1000000)] set doc [list] @@ -42,13 +41,24 @@ proc build_database {nRow} { } } -set nRow 1000 -do_test 1.0 { - build_database $nRow - execsql { SELECT count(*) FROM t1 } -} $nRow +set testprefix fts3sort -foreach {tn query} { +unset -nocomplain CONTROL +foreach {t param} { + 1 "" + 2 "order=asc" + 3 "order=desc" +} { + + set testprefix fts3sort-1.$t + + set nRow 1000 + do_test 1.0 { + build_database $nRow $param + execsql { SELECT count(*) FROM t1 } + } $nRow + + foreach {tn query} { 1 "SELECT docid, * FROM t1" 2 "SELECT docid, * FROM t1 WHERE t1 MATCH 'aa'" 3 "SELECT docid, * FROM t1 WHERE t1 MATCH 'a*'" @@ -59,50 +69,93 @@ foreach {tn query} { 8 "SELECT docid, * FROM t1 WHERE t1 MATCH 'nosuchtoken'" 9 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa OR da'" 10 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa OR nosuchtoken'" + 11 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa NEAR bb'" + 12 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH '\"aa bb\"'" + 13 "SELECT docid, content FROM t1 WHERE t1 MATCH 'aa NEAR/2 bb NEAR/3 cc'" + 14 "SELECT docid, content FROM t1 WHERE t1 MATCH '\"aa bb cc\"'" + } { + + unset -nocomplain A B C D + set A_list [list] + set B_list [list] + set C_list [list] + set D_list [list] + + unset -nocomplain X + db eval "$query ORDER BY rowid ASC" X { + set A($X(docid)) [array get X] + lappend A_list $X(docid) + } + unset -nocomplain X + db eval "$query ORDER BY rowid DESC" X { + set B($X(docid)) [array get X] + lappend B_list $X(docid) + } + unset -nocomplain X + db eval "$query ORDER BY docid ASC" X { + set C($X(docid)) [array get X] + lappend C_list $X(docid) + } + unset -nocomplain X + db eval "$query ORDER BY docid DESC" X { + set D($X(docid)) [array get X] + lappend D_list $X(docid) + } + + do_test $tn.1 { set A_list } [lsort -integer -increasing $A_list] + do_test $tn.2 { set B_list } [lsort -integer -decreasing $B_list] + do_test $tn.3 { set C_list } [lsort -integer -increasing $C_list] + do_test $tn.4 { set D_list } [lsort -integer -decreasing $D_list] + + unset -nocomplain DATA + unset -nocomplain X + db eval "$query" X { + set DATA($X(docid)) [array get X] + } + + do_test $tn.5 { lsort [array get A] } [lsort [array get DATA]] + do_test $tn.6 { lsort [array get B] } [lsort [array get DATA]] + do_test $tn.7 { lsort [array get C] } [lsort [array get DATA]] + do_test $tn.8 { lsort [array get D] } [lsort [array get DATA]] + + if {[info exists CONTROL($tn)]} { + do_test $tn.9 { set CONTROL($tn) } [lsort [array get DATA]] + } else { + set CONTROL($tn) [lsort [array get DATA]] + } + } +} +unset -nocomplain CONTROL + +set testprefix fts3sort + +#------------------------------------------------------------------------- +# Tests for parsing the "order=asc" and "order=desc" directives. +# +foreach {tn param res} { + 1 "order=asc" {0 {}} + 2 "order=desc" {0 {}} + 3 "order=dec" {1 {unrecognized order: dec}} + 4 "order=xxx, order=asc" {0 {}} } { - - unset -nocomplain A B C D - set A_list [list] - set B_list [list] - set C_list [list] - set D_list [list] - - unset -nocomplain X - db eval "$query ORDER BY rowid ASC" X { - set A($X(docid)) [array get X] - lappend A_list $X(docid) - } - unset -nocomplain X - db eval "$query ORDER BY rowid DESC" X { - set B($X(docid)) [array get X] - lappend B_list $X(docid) - } - unset -nocomplain X - db eval "$query ORDER BY docid ASC" X { - set C($X(docid)) [array get X] - lappend C_list $X(docid) - } - unset -nocomplain X - db eval "$query ORDER BY docid DESC" X { - set D($X(docid)) [array get X] - lappend D_list $X(docid) - } - - do_test 1.$tn.1 { set A_list } [lsort -integer -increasing $A_list] - do_test 1.$tn.2 { set B_list } [lsort -integer -decreasing $B_list] - do_test 1.$tn.3 { set C_list } [lsort -integer -increasing $C_list] - do_test 1.$tn.4 { set D_list } [lsort -integer -decreasing $D_list] - - unset -nocomplain DATA - unset -nocomplain X - db eval "$query" X { - set DATA($X(docid)) [array get X] - } - - do_test 1.$tn.5 { lsort [array get A] } [lsort [array get DATA]] - do_test 1.$tn.6 { lsort [array get B] } [lsort [array get DATA]] - do_test 1.$tn.7 { lsort [array get C] } [lsort [array get DATA]] - do_test 1.$tn.8 { lsort [array get D] } [lsort [array get DATA]] + execsql { DROP TABLE IF EXISTS t1 } + do_catchsql_test 2.1.$tn " + CREATE VIRTUAL TABLE t1 USING fts4(a, b, $param) + " $res } +do_execsql_test 2.2 { + BEGIN; + CREATE VIRTUAL TABLE t2 USING fts4(order=desc); + INSERT INTO t2 VALUES('aa bb'); + INSERT INTO t2 VALUES('bb cc'); + INSERT INTO t2 VALUES('cc aa'); + SELECT docid FROM t2 WHERE t2 MATCH 'aa'; + END; +} {3 1} +do_execsql_test 2.3 { + SELECT docid FROM t2 WHERE t2 MATCH 'aa'; +} {3 1} + finish_test + From 76e04431ef98a94d50a64540e518df5d34c6fe0d Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 4 Jun 2011 20:13:24 +0000 Subject: [PATCH 08/20] Remove some unreachable code. FossilOrigin-Name: 650e1a79eda5a2134a1fbd305ab1f205a57c0892 --- ext/fts3/fts3.c | 201 +----------------------------------------------- manifest | 12 +-- manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 207 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 695d3e2d5e..5654dab214 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -447,20 +447,6 @@ static void fts3GetReverseVarint( *pVal = iVal; } -/* -** As long as *pp has not reached its end (pEnd), then do the same -** as fts3GetDeltaVarint(): read a single varint and add it to *pVal. -** But if we have reached the end of the varint, just set *pp=0 and -** leave *pVal unchanged. -*/ -static void fts3GetDeltaVarint2(char **pp, char *pEnd, sqlite3_int64 *pVal){ - if( *pp>=pEnd ){ - *pp = 0; - }else{ - fts3GetDeltaVarint(pp, pVal); - } -} - /* ** The xDisconnect() virtual table method. */ @@ -1912,188 +1898,6 @@ static int fts3PoslistNearMerge( } } -/* -** Values that may be used as the first parameter to fts3DoclistMerge(). -*/ -#define MERGE_NOT 2 /* D + D -> D */ -#define MERGE_AND 3 /* D + D -> D */ -#define MERGE_OR 4 /* D + D -> D */ -#define MERGE_PHRASE 6 /* P + P -> D */ -#define MERGE_NEAR 8 /* P + P -> D */ - -#define MERGE_POS_OR 5 /* P + P -> P */ -#define MERGE_POS_PHRASE 7 /* P + P -> P */ -#define MERGE_POS_NEAR 9 /* P + P -> P */ - -/* -** Merge the two doclists passed in buffer a1 (size n1 bytes) and a2 -** (size n2 bytes). The output is written to pre-allocated buffer aBuffer, -** which is guaranteed to be large enough to hold the results. The number -** of bytes written to aBuffer is stored in *pnBuffer before returning. -** -** If successful, SQLITE_OK is returned. Otherwise, if a malloc error -** occurs while allocating a temporary buffer as part of the merge operation, -** SQLITE_NOMEM is returned. -*/ -static int fts3DoclistMerge( - int mergetype, /* One of the MERGE_XXX constants */ - int nParam1, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ - int nParam2, /* Used by MERGE_NEAR and MERGE_POS_NEAR */ - char *aBuffer, /* Pre-allocated output buffer */ - int *pnBuffer, /* OUT: Bytes written to aBuffer */ - char *a1, /* Buffer containing first doclist */ - int n1, /* Size of buffer a1 */ - char *a2, /* Buffer containing second doclist */ - int n2, /* Size of buffer a2 */ - int *pnDoc /* OUT: Number of docids in output */ -){ - sqlite3_int64 i1 = 0; - sqlite3_int64 i2 = 0; - sqlite3_int64 iPrev = 0; - - char *p = aBuffer; - char *p1 = a1; - char *p2 = a2; - char *pEnd1 = &a1[n1]; - char *pEnd2 = &a2[n2]; - int nDoc = 0; - - assert( mergetype==MERGE_OR || mergetype==MERGE_POS_OR - || mergetype==MERGE_AND || mergetype==MERGE_NOT - || mergetype==MERGE_PHRASE || mergetype==MERGE_POS_PHRASE - || mergetype==MERGE_NEAR || mergetype==MERGE_POS_NEAR - ); - assert( mergetype==MERGE_POS_PHRASE || mergetype==MERGE_POS_NEAR ); - - if( !aBuffer ){ - *pnBuffer = 0; - return SQLITE_NOMEM; - } - - /* Read the first docid from each doclist */ - fts3GetDeltaVarint2(&p1, pEnd1, &i1); - fts3GetDeltaVarint2(&p2, pEnd2, &i2); - - switch( mergetype ){ - case MERGE_OR: - case MERGE_POS_OR: - while( p1 || p2 ){ - if( p2 && p1 && i1==i2 ){ - fts3PutDeltaVarint(&p, &iPrev, i1); - if( mergetype==MERGE_POS_OR ) fts3PoslistMerge(&p, &p1, &p2); - fts3GetDeltaVarint2(&p1, pEnd1, &i1); - fts3GetDeltaVarint2(&p2, pEnd2, &i2); - }else if( !p2 || (p1 && i1isReqPos ? MERGE_POS_OR : MERGE_OR); char *aOut = 0; int nOut = 0; int i; @@ -2406,7 +2208,6 @@ static int fts3TermSelectCb( return SQLITE_NOMEM; } }else{ - int mergetype = (pTS->isReqPos ? MERGE_POS_OR : MERGE_OR); char *aMerge = aDoclist; int nMerge = nDoclist; int iOut; @@ -3816,7 +3617,7 @@ static void fts3EvalNearMerge( char *aOut; /* Buffer in which to assemble new doclist */ int nOut; /* Size of buffer aOut in bytes */ - *pRc = fts3DoclistNearMerge(bDescIdx, MERGE_POS_NEAR, nNear, + *pRc = fts3DoclistNearMerge(bDescIdx, nNear, pLeft->nToken, pLeft->doclist.aAll, pLeft->doclist.nAll, pRight->nToken, pRight->doclist.aAll, pRight->doclist.nAll, &aOut, &nOut diff --git a/manifest b/manifest index 7bf78b4a1d..34f7d97ba5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sthe\s"order=DESC"\sand\s"order=ASC"\sparameters\sin\sFTS4\s"CREATE\sVIRTUAL\sTABLE"\sstatements.\sTables\screated\swith\s"order=DESC"\sstore\sall\sdoclists\sin\sdescending\sorder,\swhich\sallows\soptimizations\snormally\sapplied\sto\s"ORDER\sBY\sdocid\sASC"\squeries\sto\sbe\sused\swith\s"ORDER\sBY\sdocid\sDESC"\squeries\sinstead. -D 2011-06-04T20:04:35.492 +C Remove\ssome\sunreachable\scode. +D 2011-06-04T20:13:24.690 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 432c902c697850cbc5ffc2be8a945ac7ac3328d7 +F ext/fts3/fts3.c 31f73cae1d21daba547c471a0a78d31e7516a29e F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h d76b021d5b7061eff7aa4055b5938eebef2bdb6a F ext/fts3/fts3_aux.c 28fc512608e147015c36080025456f57f571f76f @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P a4c7e2820824e82580730c36f85aede2efa66754 -R 2bf28954217e137f11160d52b346ca5f +P f6a0193f5a32603eb48bddc6297042dbd2ffe96e +R a65937620eeff7552523045d1ea22bb7 U dan -Z b16ce48fbf2f9ba7236a3b596adb6f59 +Z 582e1f65a0e245127649a23c8e5f1248 diff --git a/manifest.uuid b/manifest.uuid index 2d5316d141..a27947ef64 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f6a0193f5a32603eb48bddc6297042dbd2ffe96e \ No newline at end of file +650e1a79eda5a2134a1fbd305ab1f205a57c0892 \ No newline at end of file From 9aab071780a76e670d20a0ce58dac6cef44bbdcb Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 6 Jun 2011 06:55:38 +0000 Subject: [PATCH 09/20] Clean up the code for processing FTS4 options a bit. FossilOrigin-Name: 0425138a2365d23b07d88fda2b1f458f112f389d --- ext/fts3/fts3.c | 113 ++++++++++++++++++++++++--------------------- manifest | 14 +++--- manifest.uuid | 2 +- test/fts3sort.test | 3 +- 4 files changed, 70 insertions(+), 62 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 5654dab214..317b2b5625 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -909,30 +909,12 @@ static int fts3InitVtab( struct Fts3Index *aIndex; /* Array of indexes for this table */ struct Fts3Index *aFree = 0; /* Free this before returning */ + /* The results of parsing supported FTS4 key=value options: */ int bNoDocsize = 0; /* True to omit %_docsize table */ int bDescIdx = 0; /* True to store descending indexes */ - - char *zMatchinfo = 0; /* Prefix parameter value (or NULL) */ char *zPrefix = 0; /* Prefix parameter value (or NULL) */ char *zCompress = 0; /* compress=? parameter (or NULL) */ char *zUncompress = 0; /* uncompress=? parameter (or NULL) */ - char *zOrder = 0; /* order=? parameter (or NULL) */ - struct Fts4Option { - const char *zOpt; - int nOpt; - char **pzVar; - } aFts4Opt[] = { - { "matchinfo", 9, 0 }, - { "prefix", 6, 0 }, - { "compress", 8, 0 }, - { "uncompress", 10, 0 }, - { "order", 5, 0 } - }; - aFts4Opt[0].pzVar = &zMatchinfo; - aFts4Opt[1].pzVar = &zPrefix; - aFts4Opt[2].pzVar = &zCompress; - aFts4Opt[3].pzVar = &zUncompress; - aFts4Opt[4].pzVar = &zOrder; assert( strlen(argv[0])==4 ); assert( (sqlite3_strnicmp(argv[0], "fts4", 4)==0 && isFts4) @@ -973,26 +955,71 @@ static int fts3InitVtab( /* Check if it is an FTS4 special argument. */ else if( isFts4 && fts3IsSpecialColumn(z, &nKey, &zVal) ){ + struct Fts4Option { + const char *zOpt; + int nOpt; + char **pzVar; + } aFts4Opt[] = { + { "matchinfo", 9, 0 }, /* 0 -> MATCHINFO */ + { "prefix", 6, 0 }, /* 1 -> PREFIX */ + { "compress", 8, 0 }, /* 2 -> COMPRESS */ + { "uncompress", 10, 0 }, /* 3 -> UNCOMPRESS */ + { "order", 5, 0 } /* 4 -> ORDER */ + }; + int iOpt; if( !zVal ){ rc = SQLITE_NOMEM; }else{ for(iOpt=0; iOptnOpt && !sqlite3_strnicmp(z, pOp->zOpt, pOp->nOpt) ){ break; } } - if( zVal ){ + if( iOpt==SizeofArray(aFts4Opt) ){ *pzErr = sqlite3_mprintf("unrecognized parameter: %s", z); rc = SQLITE_ERROR; - sqlite3_free(zVal); + }else{ + switch( iOpt ){ + case 0: /* MATCHINFO */ + if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ + *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zVal); + rc = SQLITE_ERROR; + } + bNoDocsize = 1; + break; + + case 1: /* PREFIX */ + sqlite3_free(zPrefix); + zPrefix = zVal; + zVal = 0; + break; + + case 2: /* COMPRESS */ + sqlite3_free(zCompress); + zCompress = zVal; + zVal = 0; + break; + + case 3: /* UNCOMPRESS */ + sqlite3_free(zUncompress); + zUncompress = zVal; + zVal = 0; + break; + + case 4: /* ORDER */ + if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) + && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 3)) + ){ + *pzErr = sqlite3_mprintf("unrecognized order: %s", zVal); + rc = SQLITE_ERROR; + } + bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); + break; + } } + sqlite3_free(zVal); } } @@ -1011,26 +1038,6 @@ static int fts3InitVtab( nCol = 1; } - if( zMatchinfo ){ - if( strlen(zMatchinfo)!=4 || sqlite3_strnicmp(zMatchinfo, "fts3", 4) ){ - *pzErr = sqlite3_mprintf("unrecognized matchinfo: %s", zMatchinfo); - rc = SQLITE_ERROR; - goto fts3_init_out; - } - bNoDocsize = 1; - } - - if( zOrder ){ - if( (strlen(zOrder)!=3 || sqlite3_strnicmp(zOrder, "asc", 3)) - && (strlen(zOrder)!=4 || sqlite3_strnicmp(zOrder, "desc", 3)) - ){ - *pzErr = sqlite3_mprintf("unrecognized order: %s", zOrder); - rc = SQLITE_ERROR; - goto fts3_init_out; - } - bDescIdx = (zOrder[0]=='d' || zOrder[0]=='D'); - } - if( pTokenizer==0 ){ rc = sqlite3Fts3InitTokenizer(pHash, "simple", &pTokenizer, pzErr); if( rc!=SQLITE_OK ) goto fts3_init_out; @@ -1042,8 +1049,7 @@ static int fts3InitVtab( assert( zPrefix ); *pzErr = sqlite3_mprintf("error parsing prefix parameter: %s", zPrefix); } - if( rc ) goto fts3_init_out; - + if( rc!=SQLITE_OK ) goto fts3_init_out; /* Allocate and populate the Fts3Table structure. */ nByte = sizeof(Fts3Table) + /* Fts3Table */ @@ -1130,8 +1136,6 @@ fts3_init_out: sqlite3_free(aFree); sqlite3_free(zCompress); sqlite3_free(zUncompress); - sqlite3_free(zOrder); - sqlite3_free(zMatchinfo); sqlite3_free((void *)aCol); if( rc!=SQLITE_OK ){ if( p ){ @@ -4187,7 +4191,10 @@ char *sqlite3Fts3EvalPhrasePoslist( /* ** Free all components of the Fts3Phrase structure that were allocated by -** the eval module. +** the eval module. Specifically, this means to free: +** +** * the contents of pPhrase->doclist, and +** * any Fts3MultiSegReader objects held by phrase tokens. */ void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ if( pPhrase ){ diff --git a/manifest b/manifest index 34f7d97ba5..3294d4e1c4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\sunreachable\scode. -D 2011-06-04T20:13:24.690 +C Clean\sup\sthe\scode\sfor\sprocessing\sFTS4\soptions\sa\sbit. +D 2011-06-06T06:55:38.365 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 31f73cae1d21daba547c471a0a78d31e7516a29e +F ext/fts3/fts3.c ea03065c8fbffc7258777fd9e1bf1c40404ce14a F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h d76b021d5b7061eff7aa4055b5938eebef2bdb6a F ext/fts3/fts3_aux.c 28fc512608e147015c36080025456f57f571f76f @@ -478,7 +478,7 @@ F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 F test/fts3rnd.test 2b1a579be557ab8ac54a51b39caa4aa8043cc4ad F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test a12f22a3ba4dd59751a57c79b031d07ab5f51ddd -F test/fts3sort.test c599f19e9452b5735c41889ea5a6da996f0ebc7c +F test/fts3sort.test 63d52c1812904b751f9e1ff487472e44833f5402 F test/fts4aa.test eadf85621c0a113d4c7ad3ccbf8441130e007b8f F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P f6a0193f5a32603eb48bddc6297042dbd2ffe96e -R a65937620eeff7552523045d1ea22bb7 +P 650e1a79eda5a2134a1fbd305ab1f205a57c0892 +R d2808d37d9a0cf28002d639dd1fa255c U dan -Z 582e1f65a0e245127649a23c8e5f1248 +Z 406d2ac578f3de4e57d0cc24bb1a90c8 diff --git a/manifest.uuid b/manifest.uuid index a27947ef64..f78050d023 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -650e1a79eda5a2134a1fbd305ab1f205a57c0892 \ No newline at end of file +0425138a2365d23b07d88fda2b1f458f112f389d \ No newline at end of file diff --git a/test/fts3sort.test b/test/fts3sort.test index d2edbc2ade..ce5b5df76b 100644 --- a/test/fts3sort.test +++ b/test/fts3sort.test @@ -136,7 +136,8 @@ foreach {tn param res} { 1 "order=asc" {0 {}} 2 "order=desc" {0 {}} 3 "order=dec" {1 {unrecognized order: dec}} - 4 "order=xxx, order=asc" {0 {}} + 4 "order=xxx, order=asc" {1 {unrecognized order: xxx}} + 5 "order=desc, order=asc" {0 {}} } { execsql { DROP TABLE IF EXISTS t1 } do_catchsql_test 2.1.$tn " From 5289b01524c329af93ae44dfb70c8303ed553203 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 6 Jun 2011 14:51:50 +0000 Subject: [PATCH 10/20] Modify fts3rnd.test to run tests for both "ORDER BY docid ASC" and "ORDER BY docid DESC" with both order=ASC and order=DESC FTS tables. Fixes for some bugs found. FossilOrigin-Name: 89f2f482e077241ac29a58eadf44a72a9c01f98c --- ext/fts3/fts3.c | 46 +++++++++++++++++++++++----------- ext/fts3/fts3_write.c | 14 +++++++---- manifest | 16 ++++++------ manifest.uuid | 2 +- test/fts3rnd.test | 58 ++++++++++++++++++++++++++++++------------- 5 files changed, 91 insertions(+), 45 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 317b2b5625..18d7e79b78 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1144,6 +1144,7 @@ fts3_init_out: pTokenizer->pModule->xDestroy(pTokenizer); } }else{ + assert( p->pSegments==0 ); *ppVTab = &p->base; } return rc; @@ -1245,6 +1246,7 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ } } + assert( p->pSegments==0 ); return SQLITE_OK; } @@ -1280,6 +1282,7 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ sqlite3Fts3FreeDeferredTokens(pCsr); sqlite3_free(pCsr->aDoclist); sqlite3_free(pCsr->aMatchinfo); + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); sqlite3_free(pCsr); return SQLITE_OK; } @@ -2564,6 +2567,7 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ }else{ rc = sqlite3Fts3EvalNext((Fts3Cursor *)pCursor); } + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); return rc; } @@ -2701,7 +2705,7 @@ static int fts3ColumnMethod( sqlite3_context *pContext, /* Context for sqlite3_result_xxx() calls */ int iCol /* Index of column to read value from */ ){ - int rc; /* Return Code */ + int rc = SQLITE_OK; /* Return Code */ Fts3Cursor *pCsr = (Fts3Cursor *) pCursor; Fts3Table *p = (Fts3Table *)pCursor->pVtab; @@ -2712,21 +2716,20 @@ static int fts3ColumnMethod( /* This call is a request for the "docid" column. Since "docid" is an ** alias for "rowid", use the xRowid() method to obtain the value. */ - sqlite3_int64 iRowid; - rc = fts3RowidMethod(pCursor, &iRowid); - sqlite3_result_int64(pContext, iRowid); + sqlite3_result_int64(pContext, pCsr->iPrevId); }else if( iCol==p->nColumn ){ /* The extra column whose name is the same as the table. ** Return a blob which is a pointer to the cursor. */ sqlite3_result_blob(pContext, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); - rc = SQLITE_OK; }else{ rc = fts3CursorSeek(0, pCsr); if( rc==SQLITE_OK ){ sqlite3_result_value(pContext, sqlite3_column_value(pCsr->pStmt, iCol+1)); } } + + assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); return rc; } @@ -2760,6 +2763,7 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ static int fts3BeginMethod(sqlite3_vtab *pVtab){ UNUSED_PARAMETER(pVtab); TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); + assert( p->pSegments==0 ); assert( p->nPendingData==0 ); assert( p->inTransaction!=1 ); TESTONLY( p->inTransaction = 1 ); @@ -2777,6 +2781,7 @@ static int fts3CommitMethod(sqlite3_vtab *pVtab){ TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); assert( p->nPendingData==0 ); assert( p->inTransaction!=0 ); + assert( p->pSegments==0 ); TESTONLY( p->inTransaction = 0 ); TESTONLY( p->mxSavepoint = -1; ); return SQLITE_OK; @@ -3047,7 +3052,7 @@ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ assert( p->inTransaction ); assert( p->mxSavepoint < iSavepoint ); TESTONLY( p->mxSavepoint = iSavepoint ); - return sqlite3Fts3PendingTermsFlush(p); + return fts3SyncMethod(pVtab); } static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); @@ -3459,6 +3464,10 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ return rc; } +/* +** This function is used to iterate backwards (from the end to start) +** through doclists. +*/ void sqlite3Fts3DoclistPrev( int bDescIdx, /* True if the doclist is desc */ char *aDoclist, /* Pointer to entire doclist */ @@ -3471,6 +3480,7 @@ void sqlite3Fts3DoclistPrev( char *p = *ppIter; int iMul = (bDescIdx ? -1 : 1); + assert( nDoclist>0 ); assert( *pbEof==0 ); assert( p || *piDocid==0 ); assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); @@ -3537,7 +3547,7 @@ static int fts3EvalPhraseNext( if( rc==SQLITE_OK && !pDL->pList ){ *pbEof = 1; } - }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->aAll ){ + }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof ); @@ -3879,6 +3889,7 @@ static void fts3EvalNext( int *pRc ){ if( *pRc==SQLITE_OK ){ + assert( pExpr->bEof==0 ); pExpr->bStart = 1; switch( pExpr->eType ){ @@ -3901,7 +3912,7 @@ static void fts3EvalNext( fts3EvalNext(pCsr, pRight, pRc); while( !pLeft->bEof && !pRight->bEof && *pRc==SQLITE_OK ){ - int iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + sqlite3_int64 iDiff = DOCID_CMP(pLeft->iDocid, pRight->iDocid); if( iDiff==0 ) break; if( iDiff<0 ){ fts3EvalNext(pCsr, pLeft, pRc); @@ -3919,20 +3930,20 @@ static void fts3EvalNext( case FTSQUERY_OR: { Fts3Expr *pLeft = pExpr->pLeft; Fts3Expr *pRight = pExpr->pRight; - int iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); + sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); - if( iCmp==0 ){ + if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ fts3EvalNext(pCsr, pLeft, pRc); + }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ fts3EvalNext(pCsr, pRight, pRc); - }else if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ - fts3EvalNext(pCsr, pLeft, pRc); }else{ + fts3EvalNext(pCsr, pLeft, pRc); fts3EvalNext(pCsr, pRight, pRc); } - + pExpr->bEof = (pLeft->bEof && pRight->bEof); iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ @@ -3972,6 +3983,12 @@ static void fts3EvalNext( fts3EvalFreeDeferredDoclist(pPhrase); *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); pExpr->iDocid = pPhrase->doclist.iDocid; +#if 0 + printf("token \"%.*s\" docid=%lld\n", + pPhrase->aToken[0].n, pPhrase->aToken[0].z, pExpr->iDocid + ); +#endif + break; } } @@ -4123,8 +4140,9 @@ int sqlite3Fts3EvalPhraseDoclist( assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); if( pExpr->bStart && !pExpr->bEof ){ pExpr->bStart = 0; - while( rc==SQLITE_OK && pExpr->bEof==0 && pExpr->iDocid!=iDocid ){ + while( rc==SQLITE_OK && (pExpr->bStart==0 || pExpr->iDocid!=iDocid) ){ fts3EvalNext(pCsr, pExpr, &rc); + assert( !pExpr->bEof ); } } } diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index fd964739b2..a72556384c 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -3159,7 +3159,7 @@ int sqlite3Fts3UpdateMethod( int rc = SQLITE_OK; /* Return Code */ int isRemove = 0; /* True for an UPDATE or DELETE */ sqlite3_int64 iRemove = 0; /* Rowid removed by UPDATE or DELETE */ - u32 *aSzIns; /* Sizes of inserted documents */ + u32 *aSzIns = 0; /* Sizes of inserted documents */ u32 *aSzDel; /* Sizes of deleted documents */ int nChng = 0; /* Net change in number of documents */ int bInsertDone = 0; @@ -3174,12 +3174,16 @@ int sqlite3Fts3UpdateMethod( && sqlite3_value_type(apVal[0])==SQLITE_NULL && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL ){ - return fts3SpecialInsert(p, apVal[p->nColumn+2]); + rc = fts3SpecialInsert(p, apVal[p->nColumn+2]); + goto update_out; } /* Allocate space to hold the change in document sizes */ aSzIns = sqlite3_malloc( sizeof(aSzIns[0])*(p->nColumn+1)*2 ); - if( aSzIns==0 ) return SQLITE_NOMEM; + if( aSzIns==0 ){ + rc = SQLITE_NOMEM; + goto update_out; + } aSzDel = &aSzIns[p->nColumn+1]; memset(aSzIns, 0, sizeof(aSzIns[0])*(p->nColumn+1)*2); @@ -3229,8 +3233,7 @@ int sqlite3Fts3UpdateMethod( } } if( rc!=SQLITE_OK ){ - sqlite3_free(aSzIns); - return rc; + goto update_out; } /* If this is a DELETE or UPDATE operation, remove the old record. */ @@ -3263,6 +3266,7 @@ int sqlite3Fts3UpdateMethod( fts3UpdateDocTotals(&rc, p, aSzIns, aSzDel, nChng); } + update_out: sqlite3_free(aSzIns); sqlite3Fts3SegmentsClose(p); return rc; diff --git a/manifest b/manifest index 3294d4e1c4..e6e2af3449 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clean\sup\sthe\scode\sfor\sprocessing\sFTS4\soptions\sa\sbit. -D 2011-06-06T06:55:38.365 +C Modify\sfts3rnd.test\sto\srun\stests\sfor\sboth\s"ORDER\sBY\sdocid\sASC"\sand\s"ORDER\sBY\sdocid\sDESC"\swith\sboth\sorder=ASC\sand\sorder=DESC\sFTS\stables.\sFixes\sfor\ssome\sbugs\sfound. +D 2011-06-06T14:51:50.541 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c ea03065c8fbffc7258777fd9e1bf1c40404ce14a +F ext/fts3/fts3.c aafc143b59710e85a6215e8843aa2802f38397bc F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h d76b021d5b7061eff7aa4055b5938eebef2bdb6a F ext/fts3/fts3_aux.c 28fc512608e147015c36080025456f57f571f76f @@ -75,7 +75,7 @@ F ext/fts3/fts3_term.c 6c7f33ab732a2a0f281898685650e3a492e1e2f1 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c d7eded6f5ee3032b41126047cc04b6720f59e6da +F ext/fts3/fts3_write.c ed525afd524d713abe7da174d56ad935dfc26008 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -475,7 +475,7 @@ F test/fts3matchinfo.test cc0b009edbbf575283d5fdb53271179e0d8019ba F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 F test/fts3prefix.test 36246609111ec1683f7ea5ed27666ce2cefb5676 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 -F test/fts3rnd.test 2b1a579be557ab8ac54a51b39caa4aa8043cc4ad +F test/fts3rnd.test d88ec3dbe52e81e410cd1a39554d15941f86333c F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test a12f22a3ba4dd59751a57c79b031d07ab5f51ddd F test/fts3sort.test 63d52c1812904b751f9e1ff487472e44833f5402 @@ -939,7 +939,7 @@ F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f -P 650e1a79eda5a2134a1fbd305ab1f205a57c0892 -R d2808d37d9a0cf28002d639dd1fa255c +P 0425138a2365d23b07d88fda2b1f458f112f389d +R bea111e886e30c118babca4cc7623bc5 U dan -Z 406d2ac578f3de4e57d0cc24bb1a90c8 +Z 6f031e903f656ed90101feb3a80e60ca diff --git a/manifest.uuid b/manifest.uuid index f78050d023..118c979d31 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0425138a2365d23b07d88fda2b1f458f112f389d \ No newline at end of file +89f2f482e077241ac29a58eadf44a72a9c01f98c \ No newline at end of file diff --git a/test/fts3rnd.test b/test/fts3rnd.test index 0909cee614..0ad41d8ce4 100644 --- a/test/fts3rnd.test +++ b/test/fts3rnd.test @@ -162,7 +162,7 @@ proc simple_phrase {zPrefix} { # This [proc] is used to test the FTS3 matchinfo() function. # -proc simple_token_matchinfo {zToken} { +proc simple_token_matchinfo {zToken bDesc} { set nDoc(0) 0 set nDoc(1) 0 @@ -171,6 +171,8 @@ proc simple_token_matchinfo {zToken} { set nHit(1) 0 set nHit(2) 0 + set dir -inc + if {$bDesc} { set dir -dec } foreach key [array names ::t1] { set value $::t1($key) @@ -184,7 +186,7 @@ proc simple_token_matchinfo {zToken} { } set ret [list] - foreach docid [lsort -integer [array names a]] { + foreach docid [lsort -integer $dir [array names a]] { if { [lindex [lsort -integer $a($docid)] end] } { set matchinfo [list 1 3] foreach i {0 1 2} hit $a($docid) { @@ -262,17 +264,30 @@ proc mit {blob} { return $r } db func mit mit - set sqlite_fts3_enable_parentheses 1 -foreach nodesize {50 500 1000 2000} { +proc do_orderbydocid_test {tn sql res} { + uplevel [list do_select_test $tn.asc "$sql ORDER BY docid ASC" $res] + uplevel [list do_select_test $tn.desc "$sql ORDER BY docid DESC" \ + [lsort -int -dec $res] + ] +} + +foreach {nodesize order} { + 50 DESC + 50 ASC + 500 ASC + 1000 DESC + 2000 ASC +} { catch { array unset ::t1 } + set testname "$nodesize/$order" # Create the FTS3 table. Populate it (and the Tcl array) with 100 rows. # db transaction { catchsql { DROP TABLE t1 } - execsql "CREATE VIRTUAL TABLE t1 USING fts3(a, b, c)" + execsql "CREATE VIRTUAL TABLE t1 USING fts4(a, b, c, order=$order)" execsql "INSERT INTO t1(t1) VALUES('nodesize=$nodesize')" for {set i 0} {$i < 100} {incr i} { insert_row $i } } @@ -286,6 +301,8 @@ foreach nodesize {50 500 1000 2000} { set DO_MALLOC_TEST 1 set nRep 2 } + + set ::testprefix fts3rnd-1.$testname.$iTest # Delete one row, update one row and insert one row. # @@ -307,7 +324,7 @@ foreach nodesize {50 500 1000 2000} { if {0==($iTest%2)} { execsql COMMIT } if {0==($iTest%2)} { - do_test fts3rnd-1.$nodesize.$iTest.0 { fts3_integrity_check t1 } ok + #do_test 0 { fts3_integrity_check t1 } ok } # Pick 10 terms from the vocabulary. Check that the results of querying @@ -317,9 +334,14 @@ foreach nodesize {50 500 1000 2000} { # for {set i 0} {$i < 10} {incr i} { set term [random_term] - do_select_test fts3rnd-1.$nodesize.$iTest.1.$i { + do_select_test 1.$i { SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term - } [simple_token_matchinfo $term] + ORDER BY docid ASC + } [simple_token_matchinfo $term 0] + do_select_test 1.$i { + SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term + ORDER BY docid DESC + } [simple_token_matchinfo $term 1] } # This time, use the first two characters of each term as a term prefix @@ -329,7 +351,7 @@ foreach nodesize {50 500 1000 2000} { for {set i 0} {$i < $nRep} {incr i} { set prefix [string range [random_term] 0 end-1] set match "${prefix}*" - do_select_test fts3rnd-1.$nodesize.$iTest.2.$i { + do_orderbydocid_test 2.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $match] } @@ -339,7 +361,7 @@ foreach nodesize {50 500 1000 2000} { for {set i 0} {$i < $nRep} {incr i} { set term [list [random_term] [random_term]] set match "\"$term\"" - do_select_test fts3rnd-1.$nodesize.$iTest.3.$i { + do_orderbydocid_test 3.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $term] } @@ -349,7 +371,7 @@ foreach nodesize {50 500 1000 2000} { for {set i 0} {$i < $nRep} {incr i} { set term [list [random_term] [random_term] [random_term]] set match "\"$term\"" - do_select_test fts3rnd-1.$nodesize.$iTest.4.$i { + do_orderbydocid_test 4.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $term] } @@ -362,17 +384,19 @@ foreach nodesize {50 500 1000 2000} { append query "[string range [random_term] 0 end-1]*" set match "\"$query\"" - do_select_test fts3rnd-1.$nodesize.$iTest.5.$i { + do_orderbydocid_test 5.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_phrase $query] } - # A NEAR query with terms as the arguments. + # A NEAR query with terms as the arguments: + # + # ... MATCH '$term1 NEAR $term2' ... # for {set i 0} {$i < $nRep} {incr i} { set terms [list [random_term] [random_term]] set match [join $terms " NEAR "] - do_select_test fts3rnd-1.$nodesize.$iTest.6.$i { + do_orderbydocid_test 6.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_near $terms 10] } @@ -383,7 +407,7 @@ foreach nodesize {50 500 1000 2000} { set terms [list [random_term] [random_term] [random_term]] set nNear 11 set match [join $terms " NEAR/$nNear "] - do_select_test fts3rnd-1.$nodesize.$iTest.7.$i { + do_orderbydocid_test 7.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [simple_near $terms $nNear] } @@ -399,7 +423,7 @@ foreach nodesize {50 500 1000 2000} { set term1 [random_term] set term2 [random_term] set match "$term1 $op $term2" - do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i { + do_orderbydocid_test $tn.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [$proc [simple_phrase $term1] [simple_phrase $term2]] } @@ -418,7 +442,7 @@ foreach nodesize {50 500 1000 2000} { set term3 [random_term] set term4 [random_term] set match "$term1 NEAR $term2 $op $term3 NEAR $term4" - do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i { + do_orderbydocid_test $tn.$i { SELECT docid FROM t1 WHERE t1 MATCH $match } [$proc \ [simple_near [list $term1 $term2] 10] \ From 3eabcf5f4686047b82cf31c11c90f462cd7c346f Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 7 Jun 2011 18:35:45 +0000 Subject: [PATCH 11/20] Have NEAR queries use incremental merging. Fix issues surrounding the deferred token optimization. FossilOrigin-Name: 9d10a6846b12a9cc8fd4fdc3affd931a27218b5a --- ext/fts3/fts3.c | 363 ++++++++++++++++++++++++++++++++-------- manifest | 18 +- manifest.uuid | 2 +- test/fts3defer2.test | 17 +- test/fts3matchinfo.test | 4 + 5 files changed, 320 insertions(+), 84 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 0ab0ecaef7..d3a49f9df6 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1069,7 +1069,6 @@ static int fts3InitVtab( p->nPendingData = 0; p->azColumn = (char **)&p[1]; p->pTokenizer = pTokenizer; - p->nNodeSize = 1000; p->nMaxPendingData = FTS3_MAX_PENDING_DATA; p->bHasDocsize = (isFts4 && bNoDocsize==0); p->bHasStat = isFts4; @@ -1127,6 +1126,7 @@ static int fts3InitVtab( ** function sqlite3Fts3SegReaderCost() for details). */ fts3DatabasePageSize(&rc, p); + p->nNodeSize = p->nPgsz-35; /* Declare the table schema to SQLite. */ fts3DeclareVtab(&rc, p); @@ -1862,7 +1862,19 @@ static int fts3PoslistPhraseMerge( } /* -** Merge two position-lists as required by the NEAR operator. +** Merge two position-lists as required by the NEAR operator. The argument +** position lists correspond to the left and right phrases of an expression +** like: +** +** "phrase 1" NEAR "phrase number 2" +** +** Position list *pp1 corresponds to the left-hand side of the NEAR +** expression and *pp2 to the right. As usual, the indexes in the position +** lists are the offsets of the last token in each phrase (tokens "1" and "2" +** in the example above). +** +** The output position list - written to *pp - is a copy of *pp2 with those +** entries that are not sufficiently NEAR entries in *pp1 removed. */ static int fts3PoslistNearMerge( char **pp, /* Output buffer */ @@ -1875,34 +1887,27 @@ static int fts3PoslistNearMerge( char *p1 = *pp1; char *p2 = *pp2; - if( !pp ){ - if( fts3PoslistPhraseMerge(0, nRight, 0, 0, pp1, pp2) ) return 1; - *pp1 = p1; - *pp2 = p2; - return fts3PoslistPhraseMerge(0, nLeft, 0, 0, pp2, pp1); + char *pTmp1 = aTmp; + char *pTmp2; + char *aTmp2; + int res = 1; + + fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); + aTmp2 = pTmp2 = pTmp1; + *pp1 = p1; + *pp2 = p2; + fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); + if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ + fts3PoslistMerge(pp, &aTmp, &aTmp2); + }else if( pTmp1!=aTmp ){ + fts3PoslistCopy(pp, &aTmp); + }else if( pTmp2!=aTmp2 ){ + fts3PoslistCopy(pp, &aTmp2); }else{ - char *pTmp1 = aTmp; - char *pTmp2; - char *aTmp2; - int res = 1; - - fts3PoslistPhraseMerge(&pTmp1, nRight, 0, 0, pp1, pp2); - aTmp2 = pTmp2 = pTmp1; - *pp1 = p1; - *pp2 = p2; - fts3PoslistPhraseMerge(&pTmp2, nLeft, 1, 0, pp2, pp1); - if( pTmp1!=aTmp && pTmp2!=aTmp2 ){ - fts3PoslistMerge(pp, &aTmp, &aTmp2); - }else if( pTmp1!=aTmp ){ - fts3PoslistCopy(pp, &aTmp); - }else if( pTmp2!=aTmp2 ){ - fts3PoslistCopy(pp, &aTmp2); - }else{ - res = 0; - } - - return res; + res = 0; } + + return res; } /* @@ -3244,6 +3249,7 @@ static void fts3EvalAllocateReaders( Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pnToken, /* OUT: Total number of tokens in phrase. */ + int *pnOr, /* OUT: Total number of OR nodes in expr. */ int *pRc ){ if( pExpr && SQLITE_OK==*pRc ){ @@ -3262,8 +3268,9 @@ static void fts3EvalAllocateReaders( } } }else{ - fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pRc); - fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pRc); + *pnOr += (pExpr->eType==FTSQUERY_OR); + fts3EvalAllocateReaders(pCsr, pExpr->pLeft, pnToken, pnOr, pRc); + fts3EvalAllocateReaders(pCsr, pExpr->pRight, pnToken, pnOr, pRc); } } } @@ -3545,6 +3552,7 @@ static int fts3EvalPhraseNext( if( p->bIncr ){ assert( p->nToken==1 ); + assert( pDL->pNextDocid==0 ); rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, &pDL->iDocid, &pDL->pList, &pDL->nList ); @@ -3602,9 +3610,6 @@ static void fts3EvalStartReaders( pExpr->bDeferred = (i==nToken); *pRc = fts3EvalPhraseStart(pCsr, bOptOk, pExpr->pPhrase); }else{ - if( pExpr->eType==FTSQUERY_NEAR ){ - bOptOk = 0; - } fts3EvalStartReaders(pCsr, pExpr->pLeft, bOptOk, pRc); fts3EvalStartReaders(pCsr, pExpr->pRight, bOptOk, pRc); pExpr->bDeferred = (pExpr->pLeft->bDeferred && pExpr->pRight->bDeferred); @@ -3700,14 +3705,17 @@ static void fts3EvalNearTrim(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ typedef struct Fts3TokenAndCost Fts3TokenAndCost; struct Fts3TokenAndCost { Fts3PhraseToken *pToken; + Fts3Expr *pRoot; int nOvfl; int iCol; }; static void fts3EvalTokenCosts( Fts3Cursor *pCsr, + Fts3Expr *pRoot, Fts3Expr *pExpr, Fts3TokenAndCost **ppTC, + Fts3Expr ***ppOr, int *pRc ){ if( *pRc==SQLITE_OK && pExpr ){ @@ -3716,13 +3724,19 @@ static void fts3EvalTokenCosts( int i; for(i=0; *pRc==SQLITE_OK && inToken; i++){ Fts3TokenAndCost *pTC = (*ppTC)++; + pTC->pRoot = pRoot; pTC->pToken = &pPhrase->aToken[i]; pTC->iCol = pPhrase->iColumn; *pRc = sqlite3Fts3MsrOvfl(pCsr, pTC->pToken->pSegcsr, &pTC->nOvfl); } - }else if( pExpr->eType==FTSQUERY_AND ){ - fts3EvalTokenCosts(pCsr, pExpr->pLeft, ppTC, pRc); - fts3EvalTokenCosts(pCsr, pExpr->pRight, ppTC, pRc); + }else if( pExpr->eType!=FTSQUERY_NOT ){ + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr; + **ppOr = pExpr; + (*ppOr)++; + } + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); + fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); } } } @@ -3773,13 +3787,87 @@ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ return SQLITE_OK; } +static int fts3EvalSelectDeferred( + Fts3Cursor *pCsr, + Fts3Expr *pRoot, + Fts3TokenAndCost *aTC, + int nTC +){ + int nDocSize = 0; + int nDocEst = 0; + int rc = SQLITE_OK; + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int ii; + + int nOvfl = 0; + int nTerm = 0; + + for(ii=0; iinOvfl) + ){ + pTC = &aTC[jj]; + } + } + assert( pTC ); + + /* At this point pTC points to the cheapest remaining token. */ + if( ii==0 ){ + if( pTC->nOvfl ){ + nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10; + }else{ + /* TODO: Fix this so that the doclist need not be read twice. */ + Fts3PhraseToken *pToken = pTC->pToken; + int nList = 0; + char *pList = 0; + rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList); + if( rc==SQLITE_OK ){ + nDocEst = fts3DoclistCountDocids(1, pList, nList); + } + sqlite3_free(pList); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3TermSegReaderCursor(pCsr, + pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr + ); + } + } + }else{ + if( pTC->nOvfl>=(nDocEst*nDocSize) ){ + Fts3PhraseToken *pToken = pTC->pToken; + rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); + fts3SegReaderCursorFree(pToken->pSegcsr); + pToken->pSegcsr = 0; + } + nDocEst = 1 + (nDocEst/4); + } + pTC->pToken = 0; + } + + return rc; +} + int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int rc = SQLITE_OK; int nToken = 0; + int nOr = 0; /* Allocate a MultiSegReader for each token in the expression. */ - fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &rc); + fts3EvalAllocateReaders(pCsr, pExpr, &nToken, &nOr, &rc); /* Call fts3EvalPhraseStart() on all phrases in the expression. TODO: ** This call will eventually also be responsible for determining which @@ -3799,23 +3887,35 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ ** call. This is known as a "deferred" token. */ - /* If bOptOk is true, check if there are any tokens that can be - ** deferred (strategy 3). */ + /* If bOptOk is true, check if there are any tokens that should be deferred. + */ if( rc==SQLITE_OK && bOptOk && nToken>1 && pTab->bHasStat ){ Fts3TokenAndCost *aTC; - aTC = (Fts3TokenAndCost *)sqlite3_malloc(sizeof(Fts3TokenAndCost) * nToken); + Fts3Expr **apOr; + aTC = (Fts3TokenAndCost *)sqlite3_malloc( + sizeof(Fts3TokenAndCost) * nToken + + sizeof(Fts3Expr *) * nOr + ); + apOr = (Fts3Expr **)&aTC[nToken]; + if( !aTC ){ rc = SQLITE_NOMEM; }else{ int ii; - int nDocEst = 0; int nDocSize; Fts3TokenAndCost *pTC = aTC; + Fts3Expr **ppOr = apOr; - rc = fts3EvalAverageDocsize(pCsr, &nDocSize); - fts3EvalTokenCosts(pCsr, pExpr, &pTC, &rc); + fts3EvalTokenCosts(pCsr, 0, pExpr, &pTC, &ppOr, &rc); nToken = pTC-aTC; + nOr = ppOr-apOr; + rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); + for(ii=0; rc==SQLITE_OK && iinOvfl ){ @@ -3857,15 +3958,13 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ } pTC->pToken = 0; } +#endif + sqlite3_free(aTC); } } fts3EvalStartReaders(pCsr, pExpr, bOptOk, &rc); - - /* Fix the results of NEAR expressions. */ - fts3EvalNearTrim(pCsr, pExpr, &rc); - return rc; } @@ -3878,6 +3977,98 @@ static void fts3EvalFreeDeferredDoclist(Fts3Phrase *pPhrase){ } } +static int fts3EvalNearTrim2( + int nNear, + char *aTmp, /* Temporary space to use */ + char **paPoslist, /* IN/OUT: Position list */ + int *pnToken, /* IN/OUT: Tokens in phrase of *paPoslist */ + Fts3Phrase *pPhrase /* The phrase object to trim the doclist of */ +){ + int nParam1 = nNear + pPhrase->nToken; + int nParam2 = nNear + *pnToken; + char *p2; + char *pOut; + int res; + + p2 = pOut = pPhrase->doclist.pList; + res = fts3PoslistNearMerge( + &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 + ); + pPhrase->doclist.nList = pOut - pPhrase->doclist.pList; + *paPoslist = pPhrase->doclist.pList; + *pnToken = pPhrase->nToken; + + return res; +} + +static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ + int res = 1; + + /* The following block runs if pExpr is the root of a NEAR query. + ** For example, the query: + ** + ** "w" NEAR "x" NEAR "y" NEAR "z" + ** + ** which is represented in tree form as: + ** + ** | + ** +--NEAR--+ <-- root of NEAR query + ** | | + ** +--NEAR--+ "z" + ** | | + ** +--NEAR--+ "y" + ** | | + ** "w" "x" + ** + ** The right-hand child of a NEAR node is always a phrase. The + ** left-hand child may be either a phrase or a NEAR node. There are + ** no exceptions to this. + */ + if( *pRc==SQLITE_OK + && pExpr->eType==FTSQUERY_NEAR + && pExpr->bEof==0 + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ + Fts3Expr *p; + int nTmp = 0; /* Bytes of temp space */ + char *aTmp; /* Temp space for PoslistNearMerge() */ + + /* Allocate temporary working space. */ + for(p=pExpr; p->pLeft; p=p->pLeft){ + nTmp += p->pRight->pPhrase->doclist.nList; + } + nTmp += p->pPhrase->doclist.nList; + aTmp = sqlite3_malloc(nTmp*2); + if( !aTmp ){ + *pRc = SQLITE_NOMEM; + res = 0; + }else{ + char *aPoslist = p->pPhrase->doclist.pList; + int nToken = p->pPhrase->nToken; + + for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ + Fts3Phrase *pPhrase = p->pRight->pPhrase; + int nNear = p->nNear; + res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + + aPoslist = pExpr->pRight->pPhrase->doclist.pList; + nToken = pExpr->pRight->pPhrase->nToken; + for(p=pExpr->pLeft; p && res; p=p->pLeft){ + int nNear = p->pParent->nNear; + Fts3Phrase *pPhrase = ( + p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase + ); + res = fts3EvalNearTrim2(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } + } + + sqlite3_free(aTmp); + } + + return res; +} + /* ** This macro is used by the fts3EvalNext() function. The two arguments are ** 64-bit docid values. If the current query is "ORDER BY docid ASC", then @@ -3901,7 +4092,6 @@ static void fts3EvalNext( case FTSQUERY_AND: { Fts3Expr *pLeft = pExpr->pLeft; Fts3Expr *pRight = pExpr->pRight; - assert( !pLeft->bDeferred || !pRight->bDeferred ); if( pLeft->bDeferred ){ fts3EvalNext(pCsr, pRight, pRc); @@ -3924,7 +4114,7 @@ static void fts3EvalNext( fts3EvalNext(pCsr, pRight, pRc); } } - + pExpr->iDocid = pLeft->iDocid; pExpr->bEof = (pLeft->bEof || pRight->bEof); } @@ -3987,12 +4177,6 @@ static void fts3EvalNext( fts3EvalFreeDeferredDoclist(pPhrase); *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); pExpr->iDocid = pPhrase->doclist.iDocid; -#if 0 - printf("token \"%.*s\" docid=%lld\n", - pPhrase->aToken[0].n, pPhrase->aToken[0].z, pExpr->iDocid - ); -#endif - break; } } @@ -4000,7 +4184,7 @@ static void fts3EvalNext( } static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ - int bHit = 0; + int bHit = 1; if( *pRc==SQLITE_OK ){ switch( pExpr->eType ){ case FTSQUERY_NEAR: @@ -4008,15 +4192,40 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ bHit = ( fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) && fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) + && fts3EvalNearTest(pExpr, pRc) ); + + /* If this is a NEAR node and the NEAR expression does not match + ** any rows, zero the doclist for all phrases involved in the NEAR. + ** This is because the snippet(), offsets() and matchinfo() functions + ** are not supposed to recognize any instances of phrases that are + ** part of unmatched NEAR queries. For example if this expression: + ** + ** ... MATCH 'a OR (b NEAR c)' + ** + ** is matched against a row containing: + ** + ** 'a b d e' + ** + ** then any snippet() should ony highlight the "a" term, not the "b" + ** (as "b" is part of a non-matching NEAR clause). + */ + if( pExpr->eType==FTSQUERY_NEAR && bHit==0 ){ + Fts3Expr *p; + for(p=pExpr; p->pPhrase==0; p=p->pLeft){ + p->pRight->pPhrase->doclist.pList = 0; + } + p->pPhrase->doclist.pList = 0; + } + break; - case FTSQUERY_OR: - bHit = ( - fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc) - || fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc) - ); + case FTSQUERY_OR: { + int bHit1 = fts3EvalDeferredTest(pCsr, pExpr->pLeft, pRc); + int bHit2 = fts3EvalDeferredTest(pCsr, pExpr->pRight, pRc); + bHit = bHit1 || bHit2; break; + } case FTSQUERY_NOT: bHit = ( @@ -4026,11 +4235,15 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ break; default: { - Fts3Phrase *pPhrase = pExpr->pPhrase; - fts3EvalFreeDeferredDoclist(pPhrase); - *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); - bHit = (pPhrase->doclist.pList!=0); - pExpr->iDocid = pCsr->iPrevId; + if( pCsr->pDeferred ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + fts3EvalFreeDeferredDoclist(pPhrase); + *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); + bHit = (pPhrase->doclist.pList!=0); + pExpr->iDocid = pCsr->iPrevId; + }else{ + bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId); + } break; } } @@ -4052,11 +4265,12 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ static int fts3EvalLoadDeferred(Fts3Cursor *pCsr, int *pRc){ int rc = *pRc; int bMiss = 0; - if( rc==SQLITE_OK && pCsr->pDeferred ){ - rc = fts3CursorSeek(0, pCsr); - if( rc==SQLITE_OK ){ - sqlite3Fts3FreeDeferredDoclists(pCsr); - rc = sqlite3Fts3CacheDeferredDoclists(pCsr); + if( rc==SQLITE_OK ){ + if( pCsr->pDeferred ){ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3CacheDeferredDoclists(pCsr); + } } bMiss = (0==fts3EvalDeferredTest(pCsr, pCsr->pExpr, &rc)); sqlite3Fts3FreeDeferredDoclists(pCsr); @@ -4151,6 +4365,13 @@ int sqlite3Fts3EvalPhraseDoclist( } } + if( rc==SQLITE_OK + && pExpr->pParent + && pExpr->pParent->eType==FTSQUERY_NEAR + ){ + + } + *pnList = pPhrase->doclist.nAll; *ppList = pPhrase->doclist.aAll; return rc; diff --git a/manifest b/manifest index 8000db4360..a97b87491f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\strunk\schanges\sinto\sthe\sfts3-prefix-search\sbranch. -D 2011-06-06T18:14:25.721 +C Have\sNEAR\squeries\suse\sincremental\smerging.\sFix\sissues\ssurrounding\sthe\sdeferred\stoken\soptimization. +D 2011-06-07T18:35:45.780 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c d0970d9fa0a63ac470007ac5e14645f5b874b70f +F ext/fts3/fts3.c 9d2d2cab4d64f0769046d88b6740c6e1f229d1e3 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h d76b021d5b7061eff7aa4055b5938eebef2bdb6a F ext/fts3/fts3_aux.c baed9dab7fb4604ae8cafdb2d7700abe93beffbe @@ -464,14 +464,14 @@ F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52 F test/fts3defer.test 7c8a38d5f617d7b52ae1c43ed73c536e7e895a35 -F test/fts3defer2.test 288bef6de15557319b8c12d476ebdc83688ef96c +F test/fts3defer2.test ad5bd3ae616fb719eb06b0a73eb5c8fb1c411989 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3fault.test f83e556465bb69dc8bc676339eca408dce4ca246 F test/fts3fault2.test dc96203af6ba31ce20163fc35460e1556e8edf4d F test/fts3malloc.test 9c8cc3f885bb4dfc66d0460c52f68f45e4710d1b -F test/fts3matchinfo.test cc0b009edbbf575283d5fdb53271179e0d8019ba +F test/fts3matchinfo.test f424597b6843659ecbc2009e8823380233ebf375 F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 F test/fts3prefix.test 36246609111ec1683f7ea5ed27666ce2cefb5676 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 @@ -943,7 +943,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 89f2f482e077241ac29a58eadf44a72a9c01f98c 98ccfa930e8e5f10808a518e3e22e85e8a8a65c2 -R b612d83743a894702aacbc42c433c9e9 -U drh -Z c6414e08a953b76b4b33ed29bc763b72 +P 567dd84359218245d4e6887547e2a48881f2c8e0 +R 020efe4a51ef4472e0e5c3f4175d0de6 +U dan +Z 740d1ddba83232619fca71041707ab60 diff --git a/manifest.uuid b/manifest.uuid index 577710ac6f..4da09088b8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -567dd84359218245d4e6887547e2a48881f2c8e0 \ No newline at end of file +9d10a6846b12a9cc8fd4fdc3affd931a27218b5a \ No newline at end of file diff --git a/test/fts3defer2.test b/test/fts3defer2.test index 8441803300..b7bf923550 100644 --- a/test/fts3defer2.test +++ b/test/fts3defer2.test @@ -48,17 +48,22 @@ do_execsql_test 1.1.4 { UPDATE t1_segments SET block = zeroblob(length(block)) WHERE length(block)>10000; } {2} +do_execsql_test 1.2.0 { + SELECT content FROM t1 WHERE t1 MATCH 'f (e a)'; +} {{a b c d e f a x y}} + do_execsql_test 1.2.1 { SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)'; } {{a b c d e f a x y}} +breakpoint do_execsql_test 1.2.2 { SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal')) FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)'; } [list \ {a b c d [e] [f] [a] x y} \ {0 1 8 1 0 0 10 1 0 2 12 1} \ - [list 3 1 1 1 1 1 8 8 1 8 8 8 5001 9] + [list 3 1 1 1 1 1 1 1 1 8 8 8 5001 9] ] do_execsql_test 1.2.3 { @@ -67,7 +72,7 @@ do_execsql_test 1.2.3 { } [list \ {[a] b c d [e] [f] [a] x y} \ {0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1} \ - [list 3 1 1 1 1 1 8 8 2 8 8 8 5001 9] + [list 3 1 1 1 1 1 1 1 2 8 8 8 5001 9] ] do_execsql_test 1.3.1 { DROP TABLE t1 } @@ -99,8 +104,14 @@ foreach {tn sql} { [list 2 1 1 54 54 1 3 3 54 372 7] \ ] - set sqlite_fts3_enable_parentheses 1 do_execsql_test 2.2.$tn.2 { + SELECT mit(matchinfo(t2, 'x')) FROM t2 WHERE t2 MATCH 'g z'; + } [list \ + [list 1 2 2 1 54 54] \ + ] + + set sqlite_fts3_enable_parentheses 1 + do_execsql_test 2.2.$tn.3 { SELECT mit(matchinfo(t2, 'x')) FROM t2 WHERE t2 MATCH 'g OR (g z)'; } [list \ [list 1 2 2 1 2 2 1 54 54] \ diff --git a/test/fts3matchinfo.test b/test/fts3matchinfo.test index 8f194e72cb..e26bcf1052 100644 --- a/test/fts3matchinfo.test +++ b/test/fts3matchinfo.test @@ -68,7 +68,11 @@ do_execsql_test 3.1 { do_execsql_test 3.2 { CREATE VIRTUAL TABLE xx USING FTS4; +} +do_execsql_test 3.3 { SELECT * FROM xx WHERE xx MATCH 'abc'; +} +do_execsql_test 3.4 { SELECT * FROM xx WHERE xx MATCH 'a b c'; } From abf2545ed9c058bd62502b858adbdddcdabef446 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 8 Jun 2011 18:39:07 +0000 Subject: [PATCH 12/20] Fix various issues to do with deferred tokens, NEAR expressions and matchinfo(). FossilOrigin-Name: 3972a787df5ec253b99b148385655e7b68d851fa --- ext/fts3/fts3.c | 525 +++++++++++++++++----------------------- ext/fts3/fts3Int.h | 25 +- ext/fts3/fts3_expr.c | 1 + ext/fts3/fts3_snippet.c | 49 +--- ext/fts3/fts3_write.c | 105 -------- manifest | 22 +- manifest.uuid | 2 +- test/fts3matchinfo.test | 10 +- 8 files changed, 265 insertions(+), 474 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index d3a49f9df6..5536a684ae 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1122,9 +1122,7 @@ static int fts3InitVtab( } /* Figure out the page-size for the database. This is required in order to - ** estimate the cost of loading large doclists from the database (see - ** function sqlite3Fts3SegReaderCost() for details). - */ + ** estimate the cost of loading large doclists from the database. */ fts3DatabasePageSize(&rc, p); p->nNodeSize = p->nPgsz-35; @@ -1965,9 +1963,9 @@ static void fts3PutDeltaVarint3( static int fts3DoclistOrMerge( int bDescIdx, /* True if arguments are desc */ - u8 *a1, int n1, /* First doclist */ - u8 *a2, int n2, /* Second doclist */ - u8 **paOut, int *pnOut /* OUT: Malloc'd doclist */ + char *a1, int n1, /* First doclist */ + char *a2, int n2, /* Second doclist */ + char **paOut, int *pnOut /* OUT: Malloc'd doclist */ ){ sqlite3_int64 i1 = 0; sqlite3_int64 i2 = 0; @@ -1977,7 +1975,6 @@ static int fts3DoclistOrMerge( char *p1 = a1; char *p2 = a2; char *p; - int nOut; char *aOut; int bFirstOut = 0; @@ -2016,8 +2013,8 @@ static int fts3DoclistOrMerge( static void fts3DoclistPhraseMerge( int bDescIdx, /* True if arguments are desc */ int nDist, /* Distance from left to right (1=adjacent) */ - u8 *aLeft, int nLeft, /* Left doclist */ - u8 *aRight, int *pnRight /* IN/OUT: Right/output doclist */ + char *aLeft, int nLeft, /* Left doclist */ + char *aRight, int *pnRight /* IN/OUT: Right/output doclist */ ){ sqlite3_int64 i1 = 0; sqlite3_int64 i2 = 0; @@ -2063,83 +2060,6 @@ static void fts3DoclistPhraseMerge( *pnRight = p - aOut; } -/* -** This function merges two doclists according to the requirements of a -** NEAR operator. -*/ -static int fts3DoclistNearMerge( - int bDescIdx, - int nNear, /* Parameter to NEAR operator */ - int nTokenLeft, /* Number of tokens in LHS phrase arg */ - char *aLeft, /* Doclist for LHS (incl. positions) */ - int nLeft, /* Size of LHS doclist in bytes */ - int nTokenRight, /* As nTokenLeft */ - char *aRight, /* As aLeft */ - int nRight, /* As nRight */ - char **paOut, /* OUT: Results of merge (malloced) */ - int *pnOut /* OUT: Sized of output buffer */ -){ - char *aOut; /* Buffer to write output doclist to */ - char *aTmp; /* Temp buffer used by PoslistNearMerge() */ - - sqlite3_int64 i1 = 0; - sqlite3_int64 i2 = 0; - sqlite3_int64 iPrev = 0; - int bFirstOut = 0; - - char *pEnd1 = &aLeft[nLeft]; - char *pEnd2 = &aRight[nRight]; - char *p1 = aLeft; - char *p2 = aRight; - char *p; - - int nParam1 = nNear+nTokenRight; - int nParam2 = nNear+nTokenLeft; - - p = aOut = sqlite3_malloc(nLeft+nRight+1); - aTmp = sqlite3_malloc(2*(nLeft+nRight+1)); - if( !aOut || !aTmp ){ - sqlite3_free(aOut); - sqlite3_free(aTmp); - *paOut = 0; - *pnOut = 0; - return SQLITE_NOMEM; - } - - fts3GetDeltaVarint3(&p1, pEnd1, 0, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, 0, &i2); - - while( p1 && p2 ){ - sqlite3_int64 iDiff = COMPARE_DOCID(i1, i2); - if( iDiff==0 ){ - char *pSave = p; - sqlite3_int64 iPrevSave = iPrev; - int bFirstOutSave = bFirstOut; - fts3PutDeltaVarint3(&p, bDescIdx, &iPrev, &bFirstOut, i1); - if( !fts3PoslistNearMerge(&p, aTmp, nParam1, nParam2, &p1, &p2) ){ - p = pSave; - iPrev = iPrevSave; - bFirstOut = bFirstOutSave; - } - - fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); - fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); - }else if( iDiff<0 ){ - fts3PoslistCopy(0, &p1); - fts3GetDeltaVarint3(&p1, pEnd1, bDescIdx, &i1); - }else{ - fts3PoslistCopy(0, &p2); - fts3GetDeltaVarint3(&p2, pEnd2, bDescIdx, &i2); - } - } - - sqlite3_free(aTmp); - *paOut = aOut; - *pnOut = p - aOut; - return SQLITE_OK; -} - - /* ** Merge all doclists in the TermSelect.aaOutput[] array into a single @@ -2166,7 +2086,7 @@ static int fts3TermSelectMerge(Fts3Table *p, TermSelect *pTS){ pTS->aaOutput[i] = 0; }else{ int nNew; - u8 *aNew; + char *aNew; int rc = fts3DoclistOrMerge(p->bDescIdx, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew @@ -2231,7 +2151,7 @@ static int fts3TermSelectCb( pTS->anOutput[iOut] = nMerge; break; }else{ - u8 *aNew; + char *aNew; int nNew; int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, @@ -2403,7 +2323,6 @@ int sqlite3Fts3TermSegReaderCursor( pSegcsr = sqlite3_malloc(sizeof(Fts3MultiSegReader)); if( pSegcsr ){ int i; - int nCost = 0; int bFound = 0; /* True once an index has been found */ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; @@ -2436,10 +2355,6 @@ int sqlite3Fts3TermSegReaderCursor( ); pSegcsr->bLookup = !isPrefix; } - for(i=0; rc==SQLITE_OK && inSegment; i++){ - rc = sqlite3Fts3SegReaderCost(pCsr, pSegcsr->apSegment[i], &nCost); - } - pSegcsr->nCost = nCost; } *ppSegcsr = pSegcsr; @@ -3053,11 +2968,10 @@ static int fts3RenameMethod( } static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ - Fts3Table *p = (Fts3Table*)pVtab; UNUSED_PARAMETER(iSavepoint); - assert( p->inTransaction ); - assert( p->mxSavepoint < iSavepoint ); - TESTONLY( p->mxSavepoint = iSavepoint ); + assert( ((Fts3Table *)pVtab)->inTransaction ); + assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); + TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); return fts3SyncMethod(pVtab); } static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ @@ -3328,7 +3242,6 @@ static int fts3EvalPhraseLoad( } static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ - Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; int iToken; int rc = SQLITE_OK; @@ -3450,12 +3363,10 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ */ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ int rc; - Fts3Doclist *pList = &p->doclist; Fts3PhraseToken *pFirst = &p->aToken[0]; Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; - assert( pList->aAll==0 ); - + assert( p->doclist.aAll==0 ); if( pCsr->bDesc==pTab->bDescIdx && bOptOk==1 && p->nToken==1 && pFirst->pSegcsr && pFirst->pSegcsr->bLookup ){ @@ -3565,14 +3476,15 @@ static int fts3EvalPhraseNext( ); pDL->pList = pDL->pNextDocid; }else{ - char *pIter; + char *pIter; /* Used to iterate through aAll */ + char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */ if( pDL->pNextDocid ){ pIter = pDL->pNextDocid; }else{ pIter = pDL->aAll; } - if( pIter>=&pDL->aAll[pDL->nAll] ){ + if( pIter>=pEnd ){ /* We have already reached the end of this doclist. EOF. */ *pbEof = 1; }else{ @@ -3586,7 +3498,17 @@ static int fts3EvalPhraseNext( pDL->pList = pIter; fts3PoslistCopy(0, &pIter); pDL->nList = (pIter - pDL->pList); + + /* pIter now points just past the 0x00 that terminates the position- + ** list for document pDL->iDocid. However, if this position-list was + ** edited in place by fts3EvalNearTrim2(), then pIter may not actually + ** point to the start of the next docid value. The following line deals + ** with this case by advancing pIter past the zero-padding added by + ** fts3EvalNearTrim2(). */ + while( pIterpNextDocid = pIter; + assert( *pIter || pIter>=&pDL->aAll[pDL->nAll] ); *pbEof = 0; } } @@ -3617,90 +3539,6 @@ static void fts3EvalStartReaders( } } -static void fts3EvalNearMerge( - int bDescIdx, - Fts3Expr *p1, - Fts3Expr *p2, - int nNear, - int *pRc -){ - if( *pRc==SQLITE_OK ){ - int rc; /* Return code */ - Fts3Phrase *pLeft = p1->pPhrase; - Fts3Phrase *pRight = p2->pPhrase; - - assert( p2->eType==FTSQUERY_PHRASE && pLeft ); - assert( p2->eType==FTSQUERY_PHRASE && pRight ); - - if( pLeft->doclist.aAll==0 ){ - sqlite3_free(pRight->doclist.aAll); - pRight->doclist.aAll = 0; - pRight->doclist.nAll = 0; - }else if( pRight->doclist.aAll ){ - char *aOut; /* Buffer in which to assemble new doclist */ - int nOut; /* Size of buffer aOut in bytes */ - - *pRc = fts3DoclistNearMerge(bDescIdx, nNear, - pLeft->nToken, pLeft->doclist.aAll, pLeft->doclist.nAll, - pRight->nToken, pRight->doclist.aAll, pRight->doclist.nAll, - &aOut, &nOut - ); - sqlite3_free(pRight->doclist.aAll); - pRight->doclist.aAll = aOut; - pRight->doclist.nAll = nOut; - } - } -} - -static void fts3EvalNearTrim(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ - - if( pExpr && SQLITE_OK==*pRc ){ - if( pExpr->eType==FTSQUERY_NEAR ){ - Fts3Expr *pLeft = pExpr->pLeft; - int nPhrase = 2; - Fts3Expr **aPhrase; - - assert( pLeft ); - assert( pExpr->pRight ); - assert( pExpr->pRight->eType==FTSQUERY_PHRASE ); - - while( pLeft->eType!=FTSQUERY_PHRASE ){ - assert( pLeft->eType==FTSQUERY_NEAR ); - assert( pLeft->pRight->eType==FTSQUERY_PHRASE ); - pLeft = pLeft->pLeft; - nPhrase++; - } - - aPhrase = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nPhrase); - if( !aPhrase ){ - *pRc = SQLITE_NOMEM; - }else{ - Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; - int i = 1; - aPhrase[0] = pLeft; - do { - pLeft = pLeft->pParent; - aPhrase[i++] = pLeft->pRight; - }while( pLeft!=pExpr ); - - for(i=0; i<(nPhrase-1); i++){ - int nNear = aPhrase[i+1]->pParent->nNear; - fts3EvalNearMerge(p->bDescIdx, aPhrase[i], aPhrase[i+1], nNear, pRc); - } - for(i=nPhrase-2; i>=0; i--){ - int nNear = aPhrase[i+1]->pParent->nNear; - fts3EvalNearMerge(p->bDescIdx, aPhrase[i+1], aPhrase[i], nNear, pRc); - } - - sqlite3_free(aPhrase); - } - - }else{ - fts3EvalNearTrim(pCsr, pExpr->pLeft, pRc); - fts3EvalNearTrim(pCsr, pExpr->pRight, pRc); - } - } -} typedef struct Fts3TokenAndCost Fts3TokenAndCost; struct Fts3TokenAndCost { @@ -3777,6 +3615,7 @@ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ return SQLITE_CORRUPT_VTAB; } + pCsr->nDoc = nDoc; pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); assert( pCsr->nRowAvg>0 ); rc = sqlite3_reset(pStmt); @@ -3902,7 +3741,6 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ rc = SQLITE_NOMEM; }else{ int ii; - int nDocSize; Fts3TokenAndCost *pTC = aTC; Fts3Expr **ppOr = apOr; @@ -3910,55 +3748,12 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ nToken = pTC-aTC; nOr = ppOr-apOr; - rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); - for(ii=0; rc==SQLITE_OK && iinOvfl) ){ - pTC = &aTC[jj]; - } + if( rc==SQLITE_OK ){ + rc = fts3EvalSelectDeferred(pCsr, 0, aTC, nToken); + for(ii=0; rc==SQLITE_OK && iinOvfl ){ - nDocEst = (pTC->nOvfl * pTab->nPgsz + pTab->nPgsz) / 10; - }else{ - /* TODO: Fix this so that the doclist need not be read twice. */ - Fts3PhraseToken *pToken = pTC->pToken; - int nList = 0; - char *pList = 0; - rc = fts3TermSelect(pTab, pToken, pTC->iCol, 1, &nList, &pList); - if( rc==SQLITE_OK ){ - nDocEst = fts3DoclistCountDocids(1, pList, nList); - } - sqlite3_free(pList); - if( rc==SQLITE_OK ){ - rc = sqlite3Fts3TermSegReaderCursor(pCsr, - pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr - ); - } - } - }else{ - if( pTC->nOvfl>=(nDocEst*nDocSize) ){ - Fts3PhraseToken *pToken = pTC->pToken; - rc = sqlite3Fts3DeferToken(pCsr, pToken, pTC->iCol); - fts3SegReaderCursorFree(pToken->pSegcsr); - pToken->pSegcsr = 0; - } - nDocEst = 1 + (nDocEst/4); - } - pTC->pToken = 0; } -#endif sqlite3_free(aTC); } @@ -3986,6 +3781,7 @@ static int fts3EvalNearTrim2( ){ int nParam1 = nNear + pPhrase->nToken; int nParam2 = nNear + *pnToken; + int nNew; char *p2; char *pOut; int res; @@ -3994,9 +3790,15 @@ static int fts3EvalNearTrim2( res = fts3PoslistNearMerge( &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 ); - pPhrase->doclist.nList = pOut - pPhrase->doclist.pList; - *paPoslist = pPhrase->doclist.pList; - *pnToken = pPhrase->nToken; + if( res ){ + nNew = (pOut - pPhrase->doclist.pList) - 1; + assert( pPhrase->doclist.pList[nNew]=='\0' ); + assert( nNew<=pPhrase->doclist.nList && nNew>0 ); + memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); + pPhrase->doclist.nList = nNew; + *paPoslist = pPhrase->doclist.pList; + *pnToken = pPhrase->nToken; + } return res; } @@ -4305,75 +4107,202 @@ int sqlite3Fts3EvalNext(Fts3Cursor *pCsr){ return rc; } -/* -** Return a pointer to the entire doclist, including positions, associated -** with the phrase passed as the second argument. It is illegal to call -** this function if the phrase consists entirely of deferred tokens. -** -** TODO: This function is only used by the code for the matchinfo('x') -** auxiliary function - to obtain the following two values: -** -** 1. The total number of times the phrase appears in each column in all -** rows in the FTS table. -** -** 2. For each column, the total number of rows in the FTS table for which -** the phrase appears at least once in the column. -** -** It would be better if there was an sqlite3Fts3EvalXXX() function -** specifically to retrieve these values. If that were done, the concept -** of which tokens are deferred or incremental would be entirely encapsulated -** within the sqlite3Fts3EvalXXX()/fts3EvalXXX() functions in this file. -*/ -int sqlite3Fts3EvalPhraseDoclist( - Fts3Cursor *pCsr, /* FTS3 cursor object */ - Fts3Expr *pExpr, /* Phrase to return doclist for */ - const char **ppList, /* OUT: Buffer containing doclist */ - int *pnList /* OUT: Size of returned buffer, in bytes */ +static void fts3EvalRestart( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pRc ){ - int rc = SQLITE_OK; - Fts3Phrase *pPhrase = pExpr->pPhrase; + if( pExpr && *pRc==SQLITE_OK ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; - /* It is illegal to call this function if the phrase is entirely deferred - ** (it may contain some deferred tokens, but must also contain at least - ** one token for which the doclist may be read from the full-text index). - */ - assert( !pExpr->bDeferred ); + if( pPhrase ){ + fts3EvalFreeDeferredDoclist(pPhrase); + if( pPhrase->bIncr ){ + sqlite3Fts3EvalPhraseCleanup(pPhrase); + memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); + *pRc = sqlite3Fts3EvalStart(pCsr, pExpr, 0); + }else{ + pPhrase->doclist.pNextDocid = 0; + pPhrase->doclist.iDocid = 0; + } + } - if( pPhrase->bIncr ){ - /* This phrase was being loaded from disk incrementally. But the - ** matchinfo() function requires that the entire doclist be loaded into - ** memory. This block loads the doclist into memory and modifies the - ** Fts3Phrase structure so that it does not use the incremental strategy. - */ - TESTONLY( int bEof = pExpr->bEof; ) - TESTONLY( int bStart = pExpr->bStart; ) - sqlite3_int64 iDocid = pExpr->iDocid; - - sqlite3Fts3EvalPhraseCleanup(pPhrase); pExpr->iDocid = 0; + pExpr->bEof = 0; + pExpr->bStart = 0; - rc = sqlite3Fts3EvalStart(pCsr, pExpr, 0); - assert( pExpr->bEof==bEof ); - assert( pExpr->bStart==bStart ); - assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); - if( pExpr->bStart && !pExpr->bEof ){ - pExpr->bStart = 0; - while( rc==SQLITE_OK && (pExpr->bStart==0 || pExpr->iDocid!=iDocid) ){ - fts3EvalNext(pCsr, pExpr, &rc); - assert( !pExpr->bEof ); + fts3EvalRestart(pCsr, pExpr->pLeft, pRc); + fts3EvalRestart(pCsr, pExpr->pRight, pRc); + } +} + +static void fts3EvalUpdateCounts( + Fts3Cursor *pCsr, + Fts3Expr *pExpr, + int *pRc +){ + if( pExpr && *pRc==SQLITE_OK ){ + Fts3Phrase *pPhrase = pExpr->pPhrase; + if( pPhrase && pPhrase->doclist.pList ){ + int iCol = 0; + char *p = pPhrase->doclist.pList; + + assert( *p ); + while( 1 ){ + u8 c = 0; + int iCnt = 0; + while( 0xFE & (*p | c) ){ + if( (c&0x80)==0 ) iCnt++; + c = *p++ & 0x80; + } + + /* aMI[iCol*3 + 1] = Number of occurrences + ** aMI[iCol*3 + 2] = Number of rows containing at least one instance + */ + pExpr->aMI[iCol*3 + 1] += iCnt; + pExpr->aMI[iCol*3 + 2] += (iCnt>0); + if( *p==0x00 ) break; + p++; + p += sqlite3Fts3GetVarint32(p, &iCol); + } + } + + fts3EvalUpdateCounts(pCsr, pExpr->pLeft, pRc); + fts3EvalUpdateCounts(pCsr, pExpr->pRight, pRc); + } +} + +static int fts3EvalNearStats( + Fts3Cursor *pCsr, + Fts3Expr *pExpr +){ + int rc = SQLITE_OK; /* Return code */ + + assert( pExpr->eType==FTSQUERY_PHRASE ); + if( pExpr->aMI==0 ){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + Fts3Expr *pRoot; /* Root of NEAR expression */ + Fts3Expr *p; /* Iterator used for several purposes */ + + sqlite3_int64 iPrevId = pCsr->iPrevId; + sqlite3_int64 iDocid; + u8 bEof; + + /* Find the root of the NEAR expression */ + pRoot = pExpr; + while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){ + pRoot = pRoot->pParent; + } + iDocid = pRoot->iDocid; + bEof = pRoot->bEof; + + /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ + for(p=pRoot; p; p=p->pLeft){ + Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight); + assert( pE->aMI==0 ); + pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32)); + if( !pE->aMI ) return SQLITE_NOMEM; + memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); + } + + fts3EvalRestart(pCsr, pRoot, &rc); + + while( pCsr->isEof==0 && rc==SQLITE_OK ){ + + do { + /* Ensure the %_content statement is reset. */ + if( pCsr->isRequireSeek==0 ) sqlite3_reset(pCsr->pStmt); + assert( sqlite3_data_count(pCsr->pStmt)==0 ); + + /* Advance to the next document */ + fts3EvalNext(pCsr, pRoot, &rc); + pCsr->isEof = pRoot->bEof; + pCsr->isRequireSeek = 1; + pCsr->isMatchinfoNeeded = 1; + pCsr->iPrevId = pRoot->iDocid; + }while( pCsr->isEof==0 + && pRoot->eType==FTSQUERY_NEAR + && fts3EvalLoadDeferred(pCsr, &rc) + ); + + if( pCsr->isEof==0 ){ + fts3EvalUpdateCounts(pCsr, pRoot, &rc); + } + } + + pCsr->isEof = 0; + pCsr->iPrevId = iPrevId; + + if( bEof ){ + pRoot->bEof = bEof; + }else{ + fts3EvalRestart(pCsr, pRoot, &rc); + while( pRoot->iDocidbEof==0 ); + } + fts3EvalLoadDeferred(pCsr, &rc); + } + } + return rc; +} + +/* +** This function is used by the matchinfo() module to query a phrase +** expression node for the following information: +** +** 1. The total number of occurrences of the phrase in each column of +** the FTS table (considering all rows), and +** +** 2. For each column, the number of rows in the table for which the +** column contains at least one instance of the phrase. +** +** If no error occurs, SQLITE_OK is returned and the values for each column +** written into the array aiOut as follows: +** +** aiOut[iCol*3 + 1] = Number of occurrences +** aiOut[iCol*3 + 2] = Number of rows containing at least one instance +** +** Caveats: +** +** * If a phrase consists entirely of deferred tokens, then all output +** values are set to the number of documents in the table. In other +** words we assume that very common tokens occur exactly once in each +** column of each row of the table. +** +** * If a phrase contains some deferred tokens (and some non-deferred +** tokens), count the potential occurrence identified by considering +** the non-deferred tokens instead of actual phrase occurrences. +** +** * If the phrase is part of a NEAR expression, then only phrase instances +** that meet the NEAR constraint are included in the counts. +*/ +int sqlite3Fts3EvalPhraseStats( + Fts3Cursor *pCsr, /* FTS cursor handle */ + Fts3Expr *pExpr, /* Phrase expression */ + u32 *aiOut /* Array to write results into (see above) */ +){ + Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; + int rc = SQLITE_OK; + int iCol; + + if( pExpr->bDeferred ){ + assert( pCsr->nDoc>0 ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = pCsr->nDoc; + aiOut[iCol*3 + 2] = pCsr->nDoc; + } + }else{ + rc = fts3EvalNearStats(pCsr, pExpr); + if( rc==SQLITE_OK ){ + assert( pExpr->aMI ); + for(iCol=0; iColnColumn; iCol++){ + aiOut[iCol*3 + 1] = pExpr->aMI[iCol*3 + 1]; + aiOut[iCol*3 + 2] = pExpr->aMI[iCol*3 + 2]; } } } - if( rc==SQLITE_OK - && pExpr->pParent - && pExpr->pParent->eType==FTSQUERY_NEAR - ){ - - } - - *pnList = pPhrase->doclist.nAll; - *ppList = pPhrase->doclist.aAll; return rc; } diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index c6329c30d0..80604d6fa3 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -240,6 +240,7 @@ struct Fts3Cursor { u8 bDesc; /* True to sort in descending order */ int eEvalmode; /* An FTS3_EVAL_XX constant */ int nRowAvg; /* Average size of database rows, in pages */ + int nDoc; /* Documents in table */ int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */ u32 *aMatchinfo; /* Information about most recent match */ @@ -323,9 +324,17 @@ struct Fts3Phrase { ** "Length" field found in doclists stored on disk is omitted from this ** buffer. ** -** Variable pCurrent always points to the start of a docid field within -** aDoclist. Since the doclist is usually scanned in docid order, this can -** be used to accelerate seeking to the required docid within the doclist. +** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global +** matchinfo data. If it is not NULL, it points to an array of size nCol*3, +** where nCol is the number of columns in the queried FTS table. The array +** is populated as follows: +** +** aMI[iCol*3 + 0] = Undefined +** aMI[iCol*3 + 1] = Number of occurrences +** aMI[iCol*3 + 2] = Number of rows containing at least one instance +** +** The aMI array is allocated using sqlite3_malloc(). It should be freed +** when the expression node is. */ struct Fts3Expr { int eType; /* One of the FTSQUERY_XXX values defined below */ @@ -340,6 +349,8 @@ struct Fts3Expr { u8 bEof; /* True this expression is at EOF already */ u8 bStart; /* True if iDocid is valid */ u8 bDeferred; /* True if this expression is entirely deferred */ + + u32 *aMI; }; /* @@ -370,7 +381,6 @@ int sqlite3Fts3SegReaderNew(int, sqlite3_int64, int sqlite3Fts3SegReaderPending( Fts3Table*,int,const char*,int,int,Fts3SegReader**); void sqlite3Fts3SegReaderFree(Fts3SegReader *); -int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *); int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **); int sqlite3Fts3ReadLock(Fts3Table *); int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*); @@ -382,7 +392,6 @@ void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *); int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int); int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *); void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *); -char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *); void sqlite3Fts3SegmentsClose(Fts3Table *); /* Special values interpreted by sqlite3SegReaderCursor() */ @@ -441,8 +450,7 @@ int sqlite3Fts3VarintLen(sqlite3_uint64); void sqlite3Fts3Dequote(char *); void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*); -int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *); -int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int); +int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); /* fts3_tokenizer.c */ const char *sqlite3Fts3NextToken(const char *, int *); @@ -480,9 +488,6 @@ int sqlite3Fts3TermSegReaderCursor( Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */ ); -int sqlite3Fts3EvalPhraseCache(Fts3Cursor *, Fts3Phrase *); -sqlite3_int64 sqlite3Fts3EvalDocid(Fts3Cursor *, Fts3Expr *); -int sqlite3Fts3EvalPhraseDoclist(Fts3Cursor*, Fts3Expr*, const char**,int*); void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *); int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int); diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index be40a9cfc2..0383d1a276 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -769,6 +769,7 @@ void sqlite3Fts3ExprFree(Fts3Expr *p){ sqlite3Fts3ExprFree(p->pLeft); sqlite3Fts3ExprFree(p->pRight); sqlite3Fts3EvalPhraseCleanup(p->pPhrase); + sqlite3_free(p->aMI); sqlite3_free(p); } } diff --git a/ext/fts3/fts3_snippet.c b/ext/fts3/fts3_snippet.c index 45e3c32f07..5ae3a16fc4 100644 --- a/ext/fts3/fts3_snippet.c +++ b/ext/fts3/fts3_snippet.c @@ -720,26 +720,6 @@ static int fts3ColumnlistCount(char **ppCollist){ return nEntry; } -static void fts3LoadColumnlistCounts(char **pp, u32 *aOut, int isGlobal){ - char *pCsr = *pp; - while( *pCsr ){ - int nHit; - sqlite3_int64 iCol = 0; - if( *pCsr==0x01 ){ - pCsr++; - pCsr += sqlite3Fts3GetVarint(pCsr, &iCol); - } - nHit = fts3ColumnlistCount(&pCsr); - assert( nHit>0 ); - if( isGlobal ){ - aOut[iCol*3+1]++; - } - aOut[iCol*3] += nHit; - } - pCsr++; - *pp = pCsr; -} - /* ** fts3ExprIterate() callback used to collect the "global" matchinfo stats ** for a single query. @@ -773,32 +753,9 @@ static int fts3ExprGlobalHitsCb( void *pCtx /* Pointer to MatchInfo structure */ ){ MatchInfo *p = (MatchInfo *)pCtx; - u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol]; - - if( pExpr->bDeferred ){ - int iCol; /* Column index */ - for(iCol=0; iColnCol; iCol++){ - aOut[iCol*3 + 1] = (u32)p->nDoc; - aOut[iCol*3 + 2] = (u32)p->nDoc; - } - }else{ - char *pIter; - char *pEnd; - int n; - int rc = sqlite3Fts3EvalPhraseDoclist( - p->pCursor, pExpr, (const char **)&pIter, &n - ); - if( rc!=SQLITE_OK ) return rc; - pEnd = &pIter[n]; - - /* Fill in the global hit count matrix row for this phrase. */ - while( pIterpCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol] + ); } /* diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index a72556384c..30c0a1a15b 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -94,7 +94,6 @@ struct Fts3DeferredToken { ** ** sqlite3Fts3SegReaderNew() ** sqlite3Fts3SegReaderFree() -** sqlite3Fts3SegReaderCost() ** sqlite3Fts3SegReaderIterate() ** ** Methods used to manipulate Fts3SegReader structures: @@ -1295,95 +1294,6 @@ static int fts3SegReaderNextDocid( return SQLITE_OK; } -/* -** This function is called to estimate the amount of data that will be -** loaded from the disk If SegReaderIterate() is called on this seg-reader, -** in units of average document size. -** -** This can be used as follows: If the caller has a small doclist that -** contains references to N documents, and is considering merging it with -** a large doclist (size X "average documents"), it may opt not to load -** the large doclist if X>N. -*/ -int sqlite3Fts3SegReaderCost( - Fts3Cursor *pCsr, /* FTS3 cursor handle */ - Fts3SegReader *pReader, /* Segment-reader handle */ - int *pnCost /* IN/OUT: Number of bytes read */ -){ - Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; - int rc = SQLITE_OK; /* Return code */ - int nCost = 0; /* Cost in bytes to return */ - int pgsz = p->nPgsz; /* Database page size */ - - assert( pgsz>0 ); - - /* If this seg-reader is reading the pending-terms table, or if all data - ** for the segment is stored on the root page of the b-tree, then the cost - ** is zero. In this case all required data is already in main memory. - */ - if( p->bHasStat - && !fts3SegReaderIsPending(pReader) - && !fts3SegReaderIsRootOnly(pReader) - ){ - int nBlob = 0; - sqlite3_int64 iBlock; - - if( pCsr->nRowAvg==0 ){ - /* The average document size, which is required to calculate the cost - ** of each doclist, has not yet been determined. Read the required - ** data from the %_stat table to calculate it. - ** - ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 - ** varints, where nCol is the number of columns in the FTS3 table. - ** The first varint is the number of documents currently stored in - ** the table. The following nCol varints contain the total amount of - ** data stored in all rows of each column of the table, from left - ** to right. - */ - sqlite3_stmt *pStmt; - sqlite3_int64 nDoc = 0; - sqlite3_int64 nByte = 0; - const char *pEnd; - const char *a; - - rc = sqlite3Fts3SelectDoctotal(p, &pStmt); - if( rc!=SQLITE_OK ) return rc; - a = sqlite3_column_blob(pStmt, 0); - assert( a ); - - pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; - a += sqlite3Fts3GetVarint(a, &nDoc); - while( anRowAvg = (int)(((nByte / nDoc) + pgsz) / pgsz); - assert( pCsr->nRowAvg>0 ); - rc = sqlite3_reset(pStmt); - if( rc!=SQLITE_OK ) return rc; - } - - /* Assume that a blob flows over onto overflow pages if it is larger - ** than (pgsz-35) bytes in size (the file-format documentation - ** confirms this). - */ - for(iBlock=pReader->iStartBlock; iBlock<=pReader->iLeafEndBlock; iBlock++){ - rc = sqlite3Fts3ReadBlock(p, iBlock, 0, &nBlob, 0); - if( rc!=SQLITE_OK ) break; - if( (nBlob+35)>pgsz ){ - int nOvfl = (nBlob + 34)/pgsz; - nCost += ((nOvfl + pCsr->nRowAvg - 1)/pCsr->nRowAvg); - } - } - } - - *pnCost += nCost; - return rc; -} int sqlite3Fts3MsrOvfl( Fts3Cursor *pCsr, @@ -2416,7 +2326,6 @@ int sqlite3Fts3MsrIncrNext( } while( 1 ){ - int nSort; Fts3SegReader *pSeg; pSeg = pMsr->apSegment[0]; @@ -2958,20 +2867,6 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ return rc; } -/* -** Return the deferred doclist associated with deferred token pDeferred. -** This function assumes that sqlite3Fts3CacheDeferredDoclists() has already -** been called to allocate and populate the doclist. -*/ -char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *pDeferred, int *pnByte){ - if( pDeferred->pList ){ - *pnByte = pDeferred->pList->nData; - return pDeferred->pList->aData; - } - *pnByte = 0; - return 0; -} - /* ** Delete all cached deferred doclists. Deferred doclists are cached ** (allocated) by the sqlite3Fts3CacheDeferredDoclists() function. diff --git a/manifest b/manifest index a97b87491f..c0c0a4d2ec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\sNEAR\squeries\suse\sincremental\smerging.\sFix\sissues\ssurrounding\sthe\sdeferred\stoken\soptimization. -D 2011-06-07T18:35:45.780 +C Fix\svarious\sissues\sto\sdo\swith\sdeferred\stokens,\sNEAR\sexpressions\sand\smatchinfo(). +D 2011-06-08T18:39:07.487 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,21 +61,21 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 9d2d2cab4d64f0769046d88b6740c6e1f229d1e3 +F ext/fts3/fts3.c b44083cafb9840be0927f8b9fb2ab4f373167f77 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h d76b021d5b7061eff7aa4055b5938eebef2bdb6a +F ext/fts3/fts3Int.h a999cfbf605efec293a88519f74192f5204c84d6 F ext/fts3/fts3_aux.c baed9dab7fb4604ae8cafdb2d7700abe93beffbe -F ext/fts3/fts3_expr.c 0ae554230ada457e61e8184b24faac96aad78f6b +F ext/fts3/fts3_expr.c b95f0d76bcf4507c73a838f3178c4ed8c42dc2bb F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2 -F ext/fts3/fts3_snippet.c 0485969cce410760b50d587a77186f9c7f7e96be +F ext/fts3/fts3_snippet.c 82e2c1e420c871c02f6e85ea438570118d7105c8 F ext/fts3/fts3_term.c 6c7f33ab732a2a0f281898685650e3a492e1e2f1 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d -F ext/fts3/fts3_write.c ed525afd524d713abe7da174d56ad935dfc26008 +F ext/fts3/fts3_write.c bc24cec303d86aeb4b40fcbdf9f252f93ef78fc7 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9 @@ -471,7 +471,7 @@ F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a F test/fts3fault.test f83e556465bb69dc8bc676339eca408dce4ca246 F test/fts3fault2.test dc96203af6ba31ce20163fc35460e1556e8edf4d F test/fts3malloc.test 9c8cc3f885bb4dfc66d0460c52f68f45e4710d1b -F test/fts3matchinfo.test f424597b6843659ecbc2009e8823380233ebf375 +F test/fts3matchinfo.test 08a82d18cc08abb28aec41d412b4c2ef25ba6a5f F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 F test/fts3prefix.test 36246609111ec1683f7ea5ed27666ce2cefb5676 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 @@ -943,7 +943,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 567dd84359218245d4e6887547e2a48881f2c8e0 -R 020efe4a51ef4472e0e5c3f4175d0de6 +P 9d10a6846b12a9cc8fd4fdc3affd931a27218b5a +R 37e4da2cb9907d0ccf1d8076445165fd U dan -Z 740d1ddba83232619fca71041707ab60 +Z 147c4bbcabf01e6d99dff7a301984a70 diff --git a/manifest.uuid b/manifest.uuid index 4da09088b8..f3e1134a0a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9d10a6846b12a9cc8fd4fdc3affd931a27218b5a \ No newline at end of file +3972a787df5ec253b99b148385655e7b68d851fa \ No newline at end of file diff --git a/test/fts3matchinfo.test b/test/fts3matchinfo.test index e26bcf1052..40366b6aef 100644 --- a/test/fts3matchinfo.test +++ b/test/fts3matchinfo.test @@ -244,9 +244,13 @@ do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1} } do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')"; -do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} { - x {{5 8 2 5 5 5} {3 8 2 3 5 5}} - s {2 1} +# It used to be that the second 'a' token would be deferred. That doesn't +# work any longer. +if 0 { + do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} { + x {{5 8 2 5 5 5} {3 8 2 3 5 5}} + s {2 1} + } } do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'} { s {2} } From 786b06896713494d7913976a590abe1c1420c30b Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 9 Jun 2011 10:48:02 +0000 Subject: [PATCH 13/20] Fix problems to do with using both OR and NEAR operators in a single expression. FossilOrigin-Name: 4e8dd19eef04777d800977faf1859a405e396f30 --- ext/fts3/fts3.c | 71 ++++++++++++++++++++++++++++++----------------- manifest | 14 +++++----- manifest.uuid | 2 +- test/fts3rnd.test | 14 ++++++---- 4 files changed, 62 insertions(+), 39 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 5536a684ae..18a87e73ce 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3763,13 +3763,13 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ return rc; } -static void fts3EvalFreeDeferredDoclist(Fts3Phrase *pPhrase){ +static void fts3EvalZeroPoslist(Fts3Phrase *pPhrase){ if( pPhrase->doclist.bFreeList ){ sqlite3_free(pPhrase->doclist.pList); - pPhrase->doclist.pList = 0; - pPhrase->doclist.nList = 0; - pPhrase->doclist.bFreeList = 0; } + pPhrase->doclist.pList = 0; + pPhrase->doclist.nList = 0; + pPhrase->doclist.bFreeList = 0; } static int fts3EvalNearTrim2( @@ -3786,6 +3786,8 @@ static int fts3EvalNearTrim2( char *pOut; int res; + assert( pPhrase->doclist.pList ); + p2 = pOut = pPhrase->doclist.pList; res = fts3PoslistNearMerge( &pOut, aTmp, nParam1, nParam2, paPoslist, &p2 @@ -3959,16 +3961,16 @@ static void fts3EvalNext( fts3EvalNext(pCsr, pRight, pRc); assert( *pRc!=SQLITE_OK || pRight->bStart ); } - do { - fts3EvalNext(pCsr, pLeft, pRc); - if( pLeft->bEof ) break; + + fts3EvalNext(pCsr, pLeft, pRc); + if( pLeft->bEof==0 ){ while( !*pRc && !pRight->bEof && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 ){ fts3EvalNext(pCsr, pRight, pRc); } - }while( !pRight->bEof && pRight->iDocid==pLeft->iDocid && !*pRc ); + } pExpr->iDocid = pLeft->iDocid; pExpr->bEof = pLeft->bEof; break; @@ -3976,7 +3978,7 @@ static void fts3EvalNext( default: { Fts3Phrase *pPhrase = pExpr->pPhrase; - fts3EvalFreeDeferredDoclist(pPhrase); + fts3EvalZeroPoslist(pPhrase); *pRc = fts3EvalPhraseNext(pCsr, pPhrase, &pExpr->bEof); pExpr->iDocid = pPhrase->doclist.iDocid; break; @@ -3997,11 +3999,11 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ && fts3EvalNearTest(pExpr, pRc) ); - /* If this is a NEAR node and the NEAR expression does not match - ** any rows, zero the doclist for all phrases involved in the NEAR. - ** This is because the snippet(), offsets() and matchinfo() functions - ** are not supposed to recognize any instances of phrases that are - ** part of unmatched NEAR queries. For example if this expression: + /* If the NEAR expression does not match any rows, zero the doclist for + ** all phrases involved in the NEAR. This is because the snippet(), + ** offsets() and matchinfo() functions are not supposed to recognize + ** any instances of phrases that are part of unmatched NEAR queries. + ** For example if this expression: ** ** ... MATCH 'a OR (b NEAR c)' ** @@ -4012,12 +4014,19 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ ** then any snippet() should ony highlight the "a" term, not the "b" ** (as "b" is part of a non-matching NEAR clause). */ - if( pExpr->eType==FTSQUERY_NEAR && bHit==0 ){ + if( bHit==0 + && pExpr->eType==FTSQUERY_NEAR + && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) + ){ Fts3Expr *p; for(p=pExpr; p->pPhrase==0; p=p->pLeft){ - p->pRight->pPhrase->doclist.pList = 0; + if( p->pRight->iDocid==pCsr->iPrevId ){ + fts3EvalZeroPoslist(p->pRight->pPhrase); + } + } + if( p->iDocid==pCsr->iPrevId ){ + fts3EvalZeroPoslist(p->pPhrase); } - p->pPhrase->doclist.pList = 0; } break; @@ -4037,9 +4046,14 @@ static int fts3EvalDeferredTest(Fts3Cursor *pCsr, Fts3Expr *pExpr, int *pRc){ break; default: { - if( pCsr->pDeferred ){ + if( pCsr->pDeferred + && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) + ){ Fts3Phrase *pPhrase = pExpr->pPhrase; - fts3EvalFreeDeferredDoclist(pPhrase); + assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); + if( pExpr->bDeferred ){ + fts3EvalZeroPoslist(pPhrase); + } *pRc = fts3EvalDeferredPhrase(pCsr, pPhrase); bHit = (pPhrase->doclist.pList!=0); pExpr->iDocid = pCsr->iPrevId; @@ -4116,7 +4130,7 @@ static void fts3EvalRestart( Fts3Phrase *pPhrase = pExpr->pPhrase; if( pPhrase ){ - fts3EvalFreeDeferredDoclist(pPhrase); + fts3EvalZeroPoslist(pPhrase); if( pPhrase->bIncr ){ sqlite3Fts3EvalPhraseCleanup(pPhrase); memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); @@ -4172,7 +4186,7 @@ static void fts3EvalUpdateCounts( } } -static int fts3EvalNearStats( +static int fts3EvalGatherStats( Fts3Cursor *pCsr, Fts3Expr *pExpr ){ @@ -4195,6 +4209,7 @@ static int fts3EvalNearStats( } iDocid = pRoot->iDocid; bEof = pRoot->bEof; + assert( pRoot->bStart ); /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ for(p=pRoot; p; p=p->pLeft){ @@ -4236,11 +4251,17 @@ static int fts3EvalNearStats( if( bEof ){ pRoot->bEof = bEof; }else{ + /* Caution: pRoot may iterate through docids in ascending or descending + ** order. For this reason, even though it seems more defensive, the + ** do loop can not be written: + ** + ** do {...} while( pRoot->iDocidiDocidbEof==0 ); - } + }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); fts3EvalLoadDeferred(pCsr, &rc); } } @@ -4293,7 +4314,7 @@ int sqlite3Fts3EvalPhraseStats( aiOut[iCol*3 + 2] = pCsr->nDoc; } }else{ - rc = fts3EvalNearStats(pCsr, pExpr); + rc = fts3EvalGatherStats(pCsr, pExpr); if( rc==SQLITE_OK ){ assert( pExpr->aMI ); for(iCol=0; iColnColumn; iCol++){ @@ -4372,7 +4393,7 @@ void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *pPhrase){ if( pPhrase ){ int i; sqlite3_free(pPhrase->doclist.aAll); - fts3EvalFreeDeferredDoclist(pPhrase); + fts3EvalZeroPoslist(pPhrase); memset(&pPhrase->doclist, 0, sizeof(Fts3Doclist)); for(i=0; inToken; i++){ fts3SegReaderCursorFree(pPhrase->aToken[i].pSegcsr); diff --git a/manifest b/manifest index c0c0a4d2ec..4398869eb3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\svarious\sissues\sto\sdo\swith\sdeferred\stokens,\sNEAR\sexpressions\sand\smatchinfo(). -D 2011-06-08T18:39:07.487 +C Fix\sproblems\sto\sdo\swith\susing\sboth\sOR\sand\sNEAR\soperators\sin\sa\ssingle\sexpression. +D 2011-06-09T10:48:02.352 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c b44083cafb9840be0927f8b9fb2ab4f373167f77 +F ext/fts3/fts3.c 5df3b5797522d3d17949ee12d5918d6d213b5114 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h a999cfbf605efec293a88519f74192f5204c84d6 F ext/fts3/fts3_aux.c baed9dab7fb4604ae8cafdb2d7700abe93beffbe @@ -475,7 +475,7 @@ F test/fts3matchinfo.test 08a82d18cc08abb28aec41d412b4c2ef25ba6a5f F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844 F test/fts3prefix.test 36246609111ec1683f7ea5ed27666ce2cefb5676 F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2 -F test/fts3rnd.test d88ec3dbe52e81e410cd1a39554d15941f86333c +F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0 F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2 F test/fts3snippet.test a12f22a3ba4dd59751a57c79b031d07ab5f51ddd F test/fts3sort.test 63d52c1812904b751f9e1ff487472e44833f5402 @@ -943,7 +943,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 9d10a6846b12a9cc8fd4fdc3affd931a27218b5a -R 37e4da2cb9907d0ccf1d8076445165fd +P 3972a787df5ec253b99b148385655e7b68d851fa +R 9f88ec7fe13b9820fcb74aa4a46dd50c U dan -Z 147c4bbcabf01e6d99dff7a301984a70 +Z 78f71a9a33e1daee77ff848d72b67bf4 diff --git a/manifest.uuid b/manifest.uuid index f3e1134a0a..f91e3008ee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3972a787df5ec253b99b148385655e7b68d851fa \ No newline at end of file +4e8dd19eef04777d800977faf1859a405e396f30 \ No newline at end of file diff --git a/test/fts3rnd.test b/test/fts3rnd.test index 0ad41d8ce4..97af54925f 100644 --- a/test/fts3rnd.test +++ b/test/fts3rnd.test @@ -273,6 +273,8 @@ proc do_orderbydocid_test {tn sql res} { ] } +set NUM_TRIALS 100 + foreach {nodesize order} { 50 DESC 50 ASC @@ -292,7 +294,7 @@ foreach {nodesize order} { for {set i 0} {$i < 100} {incr i} { insert_row $i } } - for {set iTest 1} {$iTest <= 100} {incr iTest} { + for {set iTest 1} {$iTest <= $NUM_TRIALS} {incr iTest} { catchsql COMMIT set DO_MALLOC_TEST 0 @@ -334,11 +336,11 @@ foreach {nodesize order} { # for {set i 0} {$i < 10} {incr i} { set term [random_term] - do_select_test 1.$i { + do_select_test 1.$i.asc { SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term ORDER BY docid ASC } [simple_token_matchinfo $term 0] - do_select_test 1.$i { + do_select_test 1.$i.desc { SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term ORDER BY docid DESC } [simple_token_matchinfo $term 1] @@ -432,9 +434,9 @@ foreach {nodesize order} { # Set operations on NEAR queries. # foreach {tn op proc} { - 8 OR setop_or - 9 NOT setop_not - 10 AND setop_and + 11 OR setop_or + 12 NOT setop_not + 13 AND setop_and } { for {set i 0} {$i < $nRep} {incr i} { set term1 [random_term] From 99ebad90e3582641eb388daf9e440cb020770c8d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 13 Jun 2011 09:11:01 +0000 Subject: [PATCH 14/20] Fix a bug exposed by combining matchinfo(), NEAR and "ORDER BY rowid DESC". FossilOrigin-Name: 5f6b87f420f21749aa7c72e020c50aca74890086 --- Makefile.in | 3 +- ext/fts3/fts3.c | 7 +- ext/fts3/fts3_test.c | 249 +++++++++++++++++++++++++++++++++++++++++++ main.mk | 1 + manifest | 22 ++-- manifest.uuid | 2 +- src/tclsqlite.c | 8 ++ test/fts3auto.test | 201 ++++++++++++++++++++++++++++++++++ test/tester.tcl | 4 +- 9 files changed, 481 insertions(+), 16 deletions(-) create mode 100644 ext/fts3/fts3_test.c create mode 100644 test/fts3auto.test diff --git a/Makefile.in b/Makefile.in index b0871ad940..4a81e15664 100644 --- a/Makefile.in +++ b/Makefile.in @@ -379,7 +379,8 @@ TESTSRC = \ $(TOP)/src/test_vfs.c \ $(TOP)/src/test_wholenumber.c \ $(TOP)/src/test_wsd.c \ - $(TOP)/ext/fts3/fts3_term.c + $(TOP)/ext/fts3/fts3_term.c \ + $(TOP)/ext/fts3/fts3_test.c # Source code to the library files needed by the test fixture # diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 18a87e73ce..4b37c4e645 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -2727,8 +2727,10 @@ static int fts3RollbackMethod(sqlite3_vtab *pVtab){ ** same position list. */ static void fts3ReversePoslist(char *pStart, char **ppPoslist){ - char *p = &(*ppPoslist)[-3]; - char c = p[1]; + char *p = &(*ppPoslist)[-2]; + char c; + + while( p>pStart && (c=*p--)==0 ); while( p>pStart && (*p & 0x80) | c ){ c = *p--; } @@ -3422,6 +3424,7 @@ void sqlite3Fts3DoclistPrev( iDocid += (iMul * iDelta); pNext = pDocid; fts3PoslistCopy(0, &pDocid); + while( pDocid +#include +#include + +#define NM_MAX_TOKEN 12 + +typedef struct NearPhrase NearPhrase; +typedef struct NearDocument NearDocument; +typedef struct NearToken NearToken; + +struct NearDocument { + int nToken; /* Length of token in bytes */ + NearToken *aToken; /* Token array */ +}; + +struct NearToken { + int n; /* Length of token in bytes */ + const char *z; /* Pointer to token string */ +}; + +struct NearPhrase { + int nNear; /* Preceding NEAR value */ + int nToken; /* Number of tokens in this phrase */ + NearToken aToken[NM_MAX_TOKEN]; /* Array of tokens in this phrase */ +}; + +static int nm_phrase_match( + NearPhrase *p, + NearToken *aToken +){ + int ii; + + for(ii=0; iinToken; ii++){ + NearToken *pToken = &p->aToken[ii]; + if( pToken->n>0 && pToken->z[pToken->n-1]=='*' ){ + if( aToken[ii].n<(pToken->n-1) ) return 0; + if( memcmp(aToken[ii].z, pToken->z, pToken->n-1) ) return 0; + }else{ + if( aToken[ii].n!=pToken->n ) return 0; + if( memcmp(aToken[ii].z, pToken->z, pToken->n) ) return 0; + } + } + + return 1; +} + +static int nm_near_chain( + int iDir, /* Direction to iterate through aPhrase[] */ + NearDocument *pDoc, /* Document to match against */ + int iPos, /* Position at which iPhrase was found */ + int nPhrase, /* Size of phrase array */ + NearPhrase *aPhrase, /* Phrase array */ + int iPhrase /* Index of phrase found */ +){ + int iStart; + int iStop; + int ii; + int nNear; + int iPhrase2; + NearPhrase *p; + NearPhrase *pPrev; + + assert( iDir==1 || iDir==-1 ); + + if( iDir==1 ){ + if( (iPhrase+1)==nPhrase ) return 1; + nNear = aPhrase[iPhrase+1].nNear; + }else{ + if( iPhrase==0 ) return 1; + nNear = aPhrase[iPhrase].nNear; + } + pPrev = &aPhrase[iPhrase]; + iPhrase2 = iPhrase+iDir; + p = &aPhrase[iPhrase2]; + + iStart = iPos - nNear - p->nToken; + iStop = iPos + nNear + pPrev->nToken; + + if( iStart<0 ) iStart = 0; + if( iStop > pDoc->nToken - p->nToken ) iStop = pDoc->nToken - p->nToken; + + for(ii=iStart; ii<=iStop; ii++){ + if( nm_phrase_match(p, &pDoc->aToken[ii]) ){ + if( nm_near_chain(iDir, pDoc, ii, nPhrase, aPhrase, iPhrase2) ) return 1; + } + } + + return 0; +} + +static int nm_match_count( + NearDocument *pDoc, /* Document to match against */ + int nPhrase, /* Size of phrase array */ + NearPhrase *aPhrase, /* Phrase array */ + int iPhrase /* Index of phrase to count matches for */ +){ + int nOcc = 0; + int ii; + NearPhrase *p = &aPhrase[iPhrase]; + + for(ii=0; ii<(pDoc->nToken + 1 - p->nToken); ii++){ + if( nm_phrase_match(p, &pDoc->aToken[ii]) ){ + /* Test forward NEAR chain (i>iPhrase) */ + if( 0==nm_near_chain(1, pDoc, ii, nPhrase, aPhrase, iPhrase) ) continue; + + /* Test reverse NEAR chain (iNM_MAX_TOKEN ){ + Tcl_AppendResult(interp, "Too many tokens in phrase", 0); + rc = TCL_ERROR; + goto near_match_out; + } + for(jj=0; jjz = Tcl_GetStringFromObj(apToken[jj], &pT->n); + } + aPhrase[ii].nToken = nToken; + } + for(ii=1; ii0)); + + near_match_out: + ckfree((char *)aPhrase); + ckfree((char *)doc.aToken); + return rc; +} + +int Sqlitetestfts3_Init(Tcl_Interp *interp){ + Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0); + return TCL_OK; +} + diff --git a/main.mk b/main.mk index f84e0dea8b..ec743be439 100644 --- a/main.mk +++ b/main.mk @@ -220,6 +220,7 @@ SRC += \ # Source code to the test files. # TESTSRC = \ + $(TOP)/ext/fts3/fts3_test.c \ $(TOP)/src/test1.c \ $(TOP)/src/test2.c \ $(TOP)/src/test3.c \ diff --git a/manifest b/manifest index 4398869eb3..c96551fae9 100644 --- a/manifest +++ b/manifest @@ -1,7 +1,7 @@ -C Fix\sproblems\sto\sdo\swith\susing\sboth\sOR\sand\sNEAR\soperators\sin\sa\ssingle\sexpression. -D 2011-06-09T10:48:02.352 +C Fix\sa\sbug\sexposed\sby\scombining\smatchinfo(),\sNEAR\sand\s"ORDER\sBY\srowid\sDESC". +D 2011-06-13T09:11:01.953 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d +F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.vxworks c85ec1d8597fe2f7bc225af12ac1666e21379151 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 5df3b5797522d3d17949ee12d5918d6d213b5114 +F ext/fts3/fts3.c e71dafb1f324358d12fd02ea12644d8c6cea577a F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h a999cfbf605efec293a88519f74192f5204c84d6 F ext/fts3/fts3_aux.c baed9dab7fb4604ae8cafdb2d7700abe93beffbe @@ -72,6 +72,7 @@ F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295 F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2 F ext/fts3/fts3_snippet.c 82e2c1e420c871c02f6e85ea438570118d7105c8 F ext/fts3/fts3_term.c 6c7f33ab732a2a0f281898685650e3a492e1e2f1 +F ext/fts3/fts3_test.c 9376cc865447e63c671f0f9ffd1a2c9a29678230 F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3 F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d @@ -102,7 +103,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk 6111163d4e9720e4212ef288e967b4aa2c2ce379 +F main.mk 6de0d92dcae3d399a6bafaeb23b57f6ef0f41955 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -185,7 +186,7 @@ F src/sqliteInt.h 6e58c558c57c8f44011736d5fa5295eb3130f9de F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e -F src/tclsqlite.c 501c9a200fd998a268be475be5858febc90b725b +F src/tclsqlite.c c83f5b4a15ed92cb35aa04320aa4a30b95071b2f F src/test1.c efca486a25fb894988e7a82e84579a4e57388a02 F src/test2.c 80d323d11e909cf0eb1b6fbb4ac22276483bcf31 F src/test3.c 124ff9735fb6bb7d41de180d6bac90e7b1509432 @@ -454,6 +455,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3 +F test/fts3auto.test 696a2d32dd64a03aa47818c26ea64f8f27e7eb07 F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 @@ -692,7 +694,7 @@ F test/tclsqlite.test 8c154101e704170c2be10f137a5499ac2c6da8d3 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test f42121a0d29a62f00f93274464164177ab1cc24a F test/temptrigger.test b0273db072ce5f37cf19140ceb1f0d524bbe9f05 -F test/tester.tcl a791ee74cb6b8f8613079ccc018bf2c8b952a26c +F test/tester.tcl 76222602e59047c6ef119473c7a2ea7c6ee73d09 F test/thread001.test a3e6a7254d1cb057836cb3145b60c10bf5b7e60f F test/thread002.test afd20095e6e845b405df4f2c920cb93301ca69db F test/thread003.test b824d4f52b870ae39fc5bae4d8070eca73085dca @@ -943,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 3972a787df5ec253b99b148385655e7b68d851fa -R 9f88ec7fe13b9820fcb74aa4a46dd50c +P 4e8dd19eef04777d800977faf1859a405e396f30 +R 6b4754b974de210b10ad796c5876a1bc U dan -Z 78f71a9a33e1daee77ff848d72b67bf4 +Z f1c93614cce7c70636f14d06ee6a3496 diff --git a/manifest.uuid b/manifest.uuid index f91e3008ee..5c8deb0eac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4e8dd19eef04777d800977faf1859a405e396f30 \ No newline at end of file +5f6b87f420f21749aa7c72e020c50aca74890086 \ No newline at end of file diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 575651d7e5..49ff8bcd8d 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -3585,6 +3585,10 @@ static void init_all(Tcl_Interp *interp){ extern int Sqlitetestfuzzer_Init(Tcl_Interp*); extern int Sqlitetestwholenumber_Init(Tcl_Interp*); +#ifdef SQLITE_ENABLE_FTS3 + extern int Sqlitetestfts3_Init(Tcl_Interp *interp); +#endif + #ifdef SQLITE_ENABLE_ZIPVFS extern int Zipvfs_Init(Tcl_Interp*); Zipvfs_Init(interp); @@ -3625,6 +3629,10 @@ static void init_all(Tcl_Interp *interp){ Sqlitetestfuzzer_Init(interp); Sqlitetestwholenumber_Init(interp); +#ifdef SQLITE_ENABLE_FTS3 + Sqlitetestfts3_Init(interp); +#endif + Tcl_CreateObjCommand(interp,"load_testfixture_extensions",init_all_cmd,0,0); #ifdef SQLITE_SSE diff --git a/test/fts3auto.test b/test/fts3auto.test new file mode 100644 index 0000000000..fe87d9f2c7 --- /dev/null +++ b/test/fts3auto.test @@ -0,0 +1,201 @@ +# 2011 June 10 +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +# If this build does not include FTS3, skip the tests in this file. +# +ifcapable !fts3 { finish_test ; return } +source $testdir/fts3_common.tcl +source $testdir/malloc_common.tcl + +set testprefix fts3rnd2 + +proc test_fts3_near_match {tn doc expr res} { + fts3_near_match $doc $expr -phrasecountvar p + uplevel do_test [list $tn] [list [list set {} $p]] [list $res] +} + +# Simple test cases for C routine [fts3_near_match]. +# +test_fts3_near_match 1.1.1 {a b c a b} a {2} +test_fts3_near_match 1.1.2 {a b c a b} {a 5 b 6 c} {2 2 1} +test_fts3_near_match 1.1.3 {a b c a b} {"a b"} {2} +test_fts3_near_match 1.1.4 {a b c a b} {"b c"} {1} +test_fts3_near_match 1.1.5 {a b c a b} {"c c"} {0} + +test_fts3_near_match 1.2.1 "a b c d e f g" {b 2 f} {0 0} +test_fts3_near_match 1.2.2 "a b c d e f g" {b 3 f} {1 1} +test_fts3_near_match 1.2.3 "a b c d e f g" {f 2 b} {0 0} +test_fts3_near_match 1.2.4 "a b c d e f g" {f 3 b} {1 1} +test_fts3_near_match 1.2.5 "a b c d e f g" {"a b" 2 "f g"} {0 0} +test_fts3_near_match 1.2.6 "a b c d e f g" {"a b" 3 "f g"} {1 1} + +set A "a b c d e f g h i j k l m n o p q r s t u v w x y z" +test_fts3_near_match 1.3.1 $A {"c d" 5 "i j" 1 "e f"} {0 0 0} +test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"} {1 1 1} + +proc mit {blob} { + set scan(littleEndian) i* + set scan(bigEndian) I* + binary scan $blob $scan($::tcl_platform(byteOrder)) r + return $r +} +db func mit mit + +proc fix_near_expr {expr} { + set out [list] + lappend out [lindex $expr 0] + foreach {a b} [lrange $expr 1 end] { + if {[string match -nocase near $a]} { set a 10 } + if {[string match -nocase near/* $a]} { set a [string range $a 5 end] } + lappend out $a + lappend out $b + } + return $out +} + +proc do_near_test {tn tbl expr} { + + set expr [fix_near_expr $expr] + + # Create the MATCH expression from $expr + # + set match [lindex $expr 0] + if {[llength $match]>1} { + set match "\"$match\"" + } + foreach {nNear phrase} [lrange $expr 1 end] { + if {[llength $phrase]>1} { + append match " NEAR/$nNear \"$phrase\"" + } else { + append match " NEAR/$nNear $phrase" + } + } + + # Calculate the expected results using [fts3_near_match]. The following + # loop populates the "hits" and "counts" arrays as follows: + # + # 1. For each document in the table that matches the NEAR expression, + # hits($docid) is set to 1. The set of docids that match the expression + # can therefore be found using [array names hits]. + # + # 2. For each column of each document in the table, counts($docid,$iCol) + # is set to the -phrasecountvar output. + # + set res [list] + catch { array unset hits } + db eval "SELECT docid, * FROM $tbl" d { + set iCol 0 + foreach col [lrange $d(*) 1 end] { + set docid $d(docid) + set hit [fts3_near_match $d($col) $expr -p counts($docid,$iCol)] + if {$hit} { set hits($docid) 1 } + incr iCol + } + } + set nPhrase [expr ([llength $expr]+1)/2] + set nCol $iCol + + # This block populates the nHit and nDoc arrays. For each phrase/column + # in the query/table, array elements are set as follows: + # + # nHit($iPhrase,$iCol) - Total number of hits for phrase $iPhrase in + # column $iCol. + # + # nDoc($iPhrase,$iCol) - Number of documents with at least one hit for + # phrase $iPhrase in column $iCol. + # + for {set iPhrase 0} {$iPhrase < $nPhrase} {incr iPhrase} { + for {set iCol 0} {$iCol < $nCol} {incr iCol} { + set nHit($iPhrase,$iCol) 0 + set nDoc($iPhrase,$iCol) 0 + } + } + foreach key [array names counts] { + set iCol [lindex [split $key ,] 1] + set iPhrase 0 + foreach c $counts($key) { + if {$c>0} { incr nHit($iPhrase,$iCol) 1 } + incr nDoc($iPhrase,$iCol) $c + incr iPhrase + } + } + + # Set up the aMatchinfo array. For each document, set aMatchinfo($docid) to + # contain the output of matchinfo('x') for the document. + # + foreach docid [array names hits] { + set mi [list] + for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { + for {set iCol 0} {$iCol<$nCol} {incr iCol} { + lappend mi [lindex $counts($docid,$iCol) $iPhrase] + lappend mi $nDoc($iPhrase,$iCol) + lappend mi $nHit($iPhrase,$iCol) + } + } + set aMatchinfo($docid) $mi + } + + set matchinfo_asc [list] + foreach docid [lsort -integer -incr [array names aMatchinfo]] { + lappend matchinfo_asc $docid $aMatchinfo($docid) + } + set matchinfo_desc [list] + foreach docid [lsort -integer -decr [array names aMatchinfo]] { + lappend matchinfo_desc $docid $aMatchinfo($docid) + } + + set title "(\"$match\" -> [llength [array names hits]] rows)" + + do_execsql_test $tn$title.1 " + SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid ASC + " [lsort -integer -incr [array names hits]] + + do_execsql_test $tn$title.2 " + SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid DESC + " [lsort -integer -decr [array names hits]] + + do_execsql_test $tn$title.3 " + SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl + WHERE $tbl MATCH '$match' ORDER BY docid DESC + " $matchinfo_desc + + do_execsql_test $tn$title.4 " + SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl + WHERE $tbl MATCH '$match' ORDER BY docid ASC + " $matchinfo_asc +} + +do_test 2.1 { + execsql { CREATE VIRTUAL TABLE t1 USING fts3(a, b) } + for {set i 0} {$i<32} {incr i} { + set doc [list] + if {$i&0x01} {lappend doc one} + if {$i&0x02} {lappend doc two} + if {$i&0x04} {lappend doc three} + if {$i&0x08} {lappend doc four} + if {$i&0x10} {lappend doc five} + execsql { INSERT INTO t1 VALUES($doc, null) } + } +} {} +foreach {tn expr} { + 1 {one} + 2 {one NEAR/1 five} + 3 {t*} + 4 {t* NEAR/0 five} + 5 {o* NEAR/1 f*} + 6 {one NEAR five NEAR two NEAR four NEAR three} +} { + do_near_test 2.2.$tn t1 $expr +} + +finish_test + diff --git a/test/tester.tcl b/test/tester.tcl index 0b66c3d788..a53723b9a1 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -374,11 +374,11 @@ proc fix_testname {varname} { proc do_execsql_test {testname sql {result {}}} { fix_testname testname - uplevel do_test $testname [list "execsql {$sql}"] [list [list {*}$result]] + uplevel do_test [list $testname] [list "execsql {$sql}"] [list [list {*}$result]] } proc do_catchsql_test {testname sql result} { fix_testname testname - uplevel do_test $testname [list "catchsql {$sql}"] [list $result] + uplevel do_test [list $testname] [list "catchsql {$sql}"] [list $result] } proc do_eqp_test {name sql res} { uplevel do_execsql_test $name [list "EXPLAIN QUERY PLAN $sql"] [list $res] From 3fd651928f7fb142f07328e367ea33f2d92899c7 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 13 Jun 2011 13:48:36 +0000 Subject: [PATCH 15/20] Changes to fts3auto.test to test OR, AND and NOT operations. FossilOrigin-Name: e4ab6cdb101bbeb804820425cf569ee7dc2397fb --- manifest | 12 +-- manifest.uuid | 2 +- test/fts3auto.test | 261 ++++++++++++++++++++++++++++++++++----------- 3 files changed, 205 insertions(+), 70 deletions(-) diff --git a/manifest b/manifest index c96551fae9..8bdca7e965 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sexposed\sby\scombining\smatchinfo(),\sNEAR\sand\s"ORDER\sBY\srowid\sDESC". -D 2011-06-13T09:11:01.953 +C Changes\sto\sfts3auto.test\sto\stest\sOR,\sAND\sand\sNOT\soperations. +D 2011-06-13T13:48:36.083 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -455,7 +455,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3 -F test/fts3auto.test 696a2d32dd64a03aa47818c26ea64f8f27e7eb07 +F test/fts3auto.test 4aca0c0631fd40e79a45d8490828152b46ad4559 F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 4e8dd19eef04777d800977faf1859a405e396f30 -R 6b4754b974de210b10ad796c5876a1bc +P 5f6b87f420f21749aa7c72e020c50aca74890086 +R ca51faeb6c7aef68bca6b7079ff7ebda U dan -Z f1c93614cce7c70636f14d06ee6a3496 +Z 9a71ae3656c163debcbb9f929d0e7d22 diff --git a/manifest.uuid b/manifest.uuid index 5c8deb0eac..45225befd7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5f6b87f420f21749aa7c72e020c50aca74890086 \ No newline at end of file +e4ab6cdb101bbeb804820425cf569ee7dc2397fb \ No newline at end of file diff --git a/test/fts3auto.test b/test/fts3auto.test index fe87d9f2c7..97b4304f0c 100644 --- a/test/fts3auto.test +++ b/test/fts3auto.test @@ -16,32 +16,13 @@ ifcapable !fts3 { finish_test ; return } source $testdir/fts3_common.tcl source $testdir/malloc_common.tcl -set testprefix fts3rnd2 +set testprefix fts3auto +set sfep $sqlite_fts3_enable_parentheses +set sqlite_fts3_enable_parentheses 1 -proc test_fts3_near_match {tn doc expr res} { - fts3_near_match $doc $expr -phrasecountvar p - uplevel do_test [list $tn] [list [list set {} $p]] [list $res] -} - -# Simple test cases for C routine [fts3_near_match]. +#-------------------------------------------------------------------------- +# Start of Tcl procs used by tests. # -test_fts3_near_match 1.1.1 {a b c a b} a {2} -test_fts3_near_match 1.1.2 {a b c a b} {a 5 b 6 c} {2 2 1} -test_fts3_near_match 1.1.3 {a b c a b} {"a b"} {2} -test_fts3_near_match 1.1.4 {a b c a b} {"b c"} {1} -test_fts3_near_match 1.1.5 {a b c a b} {"c c"} {0} - -test_fts3_near_match 1.2.1 "a b c d e f g" {b 2 f} {0 0} -test_fts3_near_match 1.2.2 "a b c d e f g" {b 3 f} {1 1} -test_fts3_near_match 1.2.3 "a b c d e f g" {f 2 b} {0 0} -test_fts3_near_match 1.2.4 "a b c d e f g" {f 3 b} {1 1} -test_fts3_near_match 1.2.5 "a b c d e f g" {"a b" 2 "f g"} {0 0} -test_fts3_near_match 1.2.6 "a b c d e f g" {"a b" 3 "f g"} {1 1} - -set A "a b c d e f g h i j k l m n o p q r s t u v w x y z" -test_fts3_near_match 1.3.1 $A {"c d" 5 "i j" 1 "e f"} {0 0 0} -test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"} {1 1 1} - proc mit {blob} { set scan(littleEndian) i* set scan(bigEndian) I* @@ -62,24 +43,13 @@ proc fix_near_expr {expr} { return $out } -proc do_near_test {tn tbl expr} { +proc get_single_near_results {tbl expr arrayvar nullvar} { + upvar $arrayvar aMatchinfo + upvar $nullvar nullentry + catch {array unset aMatchinfo} set expr [fix_near_expr $expr] - # Create the MATCH expression from $expr - # - set match [lindex $expr 0] - if {[llength $match]>1} { - set match "\"$match\"" - } - foreach {nNear phrase} [lrange $expr 1 end] { - if {[llength $phrase]>1} { - append match " NEAR/$nNear \"$phrase\"" - } else { - append match " NEAR/$nNear $phrase" - } - } - # Calculate the expected results using [fts3_near_match]. The following # loop populates the "hits" and "counts" arrays as follows: # @@ -144,6 +114,116 @@ proc do_near_test {tn tbl expr} { set aMatchinfo($docid) $mi } + # Set up the nullentry output. + # + set nullentry [list] + for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { + for {set iCol 0} {$iCol<$nCol} {incr iCol} { + lappend nullentry 0 $nDoc($iPhrase,$iCol) $nHit($iPhrase,$iCol) + } + } +} + + +proc matching_brackets {expr} { + if {[string range $expr 0 0]!="(" || [string range $expr end end] !=")"} { + return 0 + } + + set iBracket 1 + set nExpr [string length $expr] + for {set i 1} {$iBracket && $i < $nExpr} {incr i} { + set c [string range $expr $i $i] + if {$c == "("} {incr iBracket} + if {$c == ")"} {incr iBracket -1} + } + + return [expr ($iBracket==0 && $i==$nExpr)] +} + +proc get_near_results {tbl expr arrayvar {nullvar ""}} { + upvar $arrayvar aMatchinfo + if {$nullvar != ""} { upvar $nullvar nullentry } + + set expr [string trim $expr] + while { [matching_brackets $expr] } { + set expr [string trim [string range $expr 1 end-1]] + } + + set prec(NOT) 1 + set prec(AND) 2 + set prec(OR) 3 + + set currentprec 0 + set iBracket 0 + set expr_length [llength $expr] + for {set i 0} {$i < $expr_length} {incr i} { + set op [lindex $expr $i] + if {$iBracket==0 && [info exists prec($op)] && $prec($op)>=$currentprec } { + set opidx $i + set currentprec $prec($op) + } else { + for {set j 0} {$j < [string length $op]} {incr j} { + set c [string range $op $j $j] + if {$c == "("} { incr iBracket +1 } + if {$c == ")"} { incr iBracket -1 } + } + } + } + if {$iBracket!=0} { error "mismatched brackets in: $expr" } + + if {[info exists opidx]==0} { + get_single_near_results $tbl $expr aMatchinfo nullentry + } else { + set eLeft [lrange $expr 0 [expr $opidx-1]] + set eRight [lrange $expr [expr $opidx+1] end] + + get_near_results $tbl $eLeft aLeft nullleft + get_near_results $tbl $eRight aRight nullright + + switch -- [lindex $expr $opidx] { + "NOT" { + foreach hit [array names aLeft] { + if {0==[info exists aRight($hit)]} { + set aMatchinfo($hit) $aLeft($hit) + } + } + set nullentry $nullleft + } + + "AND" { + foreach hit [array names aLeft] { + if {[info exists aRight($hit)]} { + set aMatchinfo($hit) [concat $aLeft($hit) $aRight($hit)] + } + } + set nullentry [concat $nullleft $nullright] + } + + "OR" { + foreach hit [array names aLeft] { + if {[info exists aRight($hit)]} { + set aMatchinfo($hit) [concat $aLeft($hit) $aRight($hit)] + unset aRight($hit) + } else { + set aMatchinfo($hit) [concat $aLeft($hit) $nullright] + } + } + foreach hit [array names aRight] { + set aMatchinfo($hit) [concat $nullleft $aRight($hit)] + } + + set nullentry [concat $nullleft $nullright] + } + } + } +} + +proc do_near_test {tn tbl expr} { + + get_near_results $tbl $expr aMatchinfo + set match $expr + set matchinfo_asc [list] foreach docid [lsort -integer -incr [array names aMatchinfo]] { lappend matchinfo_asc $docid $aMatchinfo($docid) @@ -153,15 +233,15 @@ proc do_near_test {tn tbl expr} { lappend matchinfo_desc $docid $aMatchinfo($docid) } - set title "(\"$match\" -> [llength [array names hits]] rows)" + set title "(\"$match\" -> [llength [array names aMatchinfo]] rows)" do_execsql_test $tn$title.1 " SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid ASC - " [lsort -integer -incr [array names hits]] + " [lsort -integer -incr [array names aMatchinfo]] do_execsql_test $tn$title.2 " SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid DESC - " [lsort -integer -decr [array names hits]] + " [lsort -integer -decr [array names aMatchinfo]] do_execsql_test $tn$title.3 " SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl @@ -174,28 +254,83 @@ proc do_near_test {tn tbl expr} { " $matchinfo_asc } -do_test 2.1 { - execsql { CREATE VIRTUAL TABLE t1 USING fts3(a, b) } - for {set i 0} {$i<32} {incr i} { - set doc [list] - if {$i&0x01} {lappend doc one} - if {$i&0x02} {lappend doc two} - if {$i&0x04} {lappend doc three} - if {$i&0x08} {lappend doc four} - if {$i&0x10} {lappend doc five} - execsql { INSERT INTO t1 VALUES($doc, null) } - } -} {} -foreach {tn expr} { - 1 {one} - 2 {one NEAR/1 five} - 3 {t*} - 4 {t* NEAR/0 five} - 5 {o* NEAR/1 f*} - 6 {one NEAR five NEAR two NEAR four NEAR three} -} { - do_near_test 2.2.$tn t1 $expr + +# End of test procs. Actual tests are below this line. +#-------------------------------------------------------------------------- + +#-------------------------------------------------------------------------- +# The following test cases - fts3auto-1.* - focus on testing the Tcl +# command [fts3_near_match], which is used by other tests in this file. +# +proc test_fts3_near_match {tn doc expr res} { + fts3_near_match $doc $expr -phrasecountvar p + uplevel do_test [list $tn] [list [list set {} $p]] [list $res] } +test_fts3_near_match 1.1.1 {a b c a b} a {2} +test_fts3_near_match 1.1.2 {a b c a b} {a 5 b 6 c} {2 2 1} +test_fts3_near_match 1.1.3 {a b c a b} {"a b"} {2} +test_fts3_near_match 1.1.4 {a b c a b} {"b c"} {1} +test_fts3_near_match 1.1.5 {a b c a b} {"c c"} {0} + +test_fts3_near_match 1.2.1 "a b c d e f g" {b 2 f} {0 0} +test_fts3_near_match 1.2.2 "a b c d e f g" {b 3 f} {1 1} +test_fts3_near_match 1.2.3 "a b c d e f g" {f 2 b} {0 0} +test_fts3_near_match 1.2.4 "a b c d e f g" {f 3 b} {1 1} +test_fts3_near_match 1.2.5 "a b c d e f g" {"a b" 2 "f g"} {0 0} +test_fts3_near_match 1.2.6 "a b c d e f g" {"a b" 3 "f g"} {1 1} + +set A "a b c d e f g h i j k l m n o p q r s t u v w x y z" +test_fts3_near_match 1.3.1 $A {"c d" 5 "i j" 1 "e f"} {0 0 0} +test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"} {1 1 1} + +#-------------------------------------------------------------------------- +# Test cases fts3auto-2.* run some simple tests using the +# [do_near_test] proc. +# +foreach {tn create} { + 1 "CREATE VIRTUAL TABLE t1 USING fts4(a, b)" + 2 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=DESC)" + 3 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=ASC)" + 4 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, prefix=1)" + 5 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=DESC, prefix=1)" + 6 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=ASC, prefix=1)" +} { + do_test 2.$tn.1 { + catchsql { DROP TABLE t1 } + execsql $create + for {set i 0} {$i<32} {incr i} { + set doc [list] + if {$i&0x01} {lappend doc one} + if {$i&0x02} {lappend doc two} + if {$i&0x04} {lappend doc three} + if {$i&0x08} {lappend doc four} + if {$i&0x10} {lappend doc five} + execsql { INSERT INTO t1 VALUES($doc, null) } + } + } {} + foreach {tn2 expr} { + 1 {one} + 2 {one NEAR/1 five} + 3 {t*} + 4 {t* NEAR/0 five} + 5 {o* NEAR/1 f*} + 6 {one NEAR five NEAR two NEAR four NEAR three} + 7 {one NEAR xyz} + 8 {one OR two} + 9 {one AND two} + 10 {one NOT two} + 11 {one AND two OR three} + 12 {three OR one AND two} + 13 {(three OR one) AND two} + 14 {(three OR one) AND two NOT (five NOT four)} + 15 {"one two"} + 16 {"one two" NOT "three four"} + } { + do_near_test 2.$tn.2.$tn2 t1 $expr + } +} + +set sqlite_fts3_enable_parentheses $sfep finish_test From 2f779343854941321b64995cd608e6fe7346ad2d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 13 Jun 2011 17:00:12 +0000 Subject: [PATCH 16/20] Add tests for deferred tokens to fts3auto.test. Fix a problem with OR queries and deferred tokens. FossilOrigin-Name: b9fb69e55bb05a8819688ee63987f2a05c33d59b --- ext/fts3/fts3.c | 13 +++-- manifest | 16 +++--- manifest.uuid | 2 +- test/fts3auto.test | 125 ++++++++++++++++++++++++++++++++++++++----- test/fts3defer2.test | 5 +- 5 files changed, 131 insertions(+), 30 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 4b37c4e645..17a7b2ccaa 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3572,11 +3572,16 @@ static void fts3EvalTokenCosts( } }else if( pExpr->eType!=FTSQUERY_NOT ){ if( pExpr->eType==FTSQUERY_OR ){ - pRoot = pExpr; - **ppOr = pExpr; + pRoot = pExpr->pLeft; + **ppOr = pRoot; (*ppOr)++; } fts3EvalTokenCosts(pCsr, pRoot, pExpr->pLeft, ppTC, ppOr, pRc); + if( pExpr->eType==FTSQUERY_OR ){ + pRoot = pExpr->pRight; + **ppOr = pRoot; + (*ppOr)++; + } fts3EvalTokenCosts(pCsr, pRoot, pExpr->pRight, ppTC, ppOr, pRc); } } @@ -3736,7 +3741,7 @@ int sqlite3Fts3EvalStart(Fts3Cursor *pCsr, Fts3Expr *pExpr, int bOptOk){ Fts3Expr **apOr; aTC = (Fts3TokenAndCost *)sqlite3_malloc( sizeof(Fts3TokenAndCost) * nToken - + sizeof(Fts3Expr *) * nOr + + sizeof(Fts3Expr *) * nOr * 2 ); apOr = (Fts3Expr **)&aTC[nToken]; @@ -4310,7 +4315,7 @@ int sqlite3Fts3EvalPhraseStats( int rc = SQLITE_OK; int iCol; - if( pExpr->bDeferred ){ + if( pExpr->bDeferred && pExpr->pParent->eType!=FTSQUERY_NEAR ){ assert( pCsr->nDoc>0 ); for(iCol=0; iColnColumn; iCol++){ aiOut[iCol*3 + 1] = pCsr->nDoc; diff --git a/manifest b/manifest index 8bdca7e965..15429e6135 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\sfts3auto.test\sto\stest\sOR,\sAND\sand\sNOT\soperations. -D 2011-06-13T13:48:36.083 +C Add\stests\sfor\sdeferred\stokens\sto\sfts3auto.test.\sFix\sa\sproblem\swith\sOR\squeries\sand\sdeferred\stokens. +D 2011-06-13T17:00:12.630 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c e71dafb1f324358d12fd02ea12644d8c6cea577a +F ext/fts3/fts3.c 2a48945d199d5d0fe66a06c8cf73ad2f0e7a4b5f F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h a999cfbf605efec293a88519f74192f5204c84d6 F ext/fts3/fts3_aux.c baed9dab7fb4604ae8cafdb2d7700abe93beffbe @@ -455,7 +455,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3 -F test/fts3auto.test 4aca0c0631fd40e79a45d8490828152b46ad4559 +F test/fts3auto.test bf01a1d793155d744d23e4092871ca116f1db64c F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 @@ -466,7 +466,7 @@ F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52 F test/fts3defer.test 7c8a38d5f617d7b52ae1c43ed73c536e7e895a35 -F test/fts3defer2.test ad5bd3ae616fb719eb06b0a73eb5c8fb1c411989 +F test/fts3defer2.test 35867d33ba6db03f6c73bd6f5fc333ae14f68c81 F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851 F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 5f6b87f420f21749aa7c72e020c50aca74890086 -R ca51faeb6c7aef68bca6b7079ff7ebda +P e4ab6cdb101bbeb804820425cf569ee7dc2397fb +R 7fcdbf104822a42776783c2cd9e89101 U dan -Z 9a71ae3656c163debcbb9f929d0e7d22 +Z 0999cf6b26be67f673d9db0918d86701 diff --git a/manifest.uuid b/manifest.uuid index 45225befd7..41624e6daa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e4ab6cdb101bbeb804820425cf569ee7dc2397fb \ No newline at end of file +b9fb69e55bb05a8819688ee63987f2a05c33d59b \ No newline at end of file diff --git a/test/fts3auto.test b/test/fts3auto.test index 97b4304f0c..68c82c8a85 100644 --- a/test/fts3auto.test +++ b/test/fts3auto.test @@ -93,12 +93,45 @@ proc get_single_near_results {tbl expr arrayvar nullvar} { set iCol [lindex [split $key ,] 1] set iPhrase 0 foreach c $counts($key) { - if {$c>0} { incr nHit($iPhrase,$iCol) 1 } - incr nDoc($iPhrase,$iCol) $c + if {$c>0} { incr nDoc($iPhrase,$iCol) 1 } + incr nHit($iPhrase,$iCol) $c incr iPhrase } } + if {[info exists ::fts3_deferred] && [llength $expr]==1} { + set phrase [lindex $expr 0] + set rewritten [list] + set partial 0 + foreach tok $phrase { + if {[lsearch $::fts3_deferred $tok]>=0} { + lappend rewritten * + } else { + lappend rewritten $tok + set partial 1 + } + } + if {$partial==0} { + set tblsize [db one "SELECT count(*) FROM $tbl"] + for {set iCol 0} {$iCol < $nCol} {incr iCol} { + set nHit(0,$iCol) $tblsize + set nDoc(0,$iCol) $tblsize + } + } elseif {$rewritten != $phrase} { + while {[lindex $rewritten end] == "*"} { + set rewritten [lrange $rewritten 0 end-1] + } + while {[lindex $rewritten 0] == "*"} { + set rewritten [lrange $rewritten 1 end] + } + get_single_near_results $tbl [list $rewritten] aRewrite nullentry + foreach docid [array names hits] { + set aMatchinfo($docid) $aRewrite($docid) + } + return + } + } + # Set up the aMatchinfo array. For each document, set aMatchinfo($docid) to # contain the output of matchinfo('x') for the document. # @@ -107,8 +140,8 @@ proc get_single_near_results {tbl expr arrayvar nullvar} { for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { for {set iCol 0} {$iCol<$nCol} {incr iCol} { lappend mi [lindex $counts($docid,$iCol) $iPhrase] - lappend mi $nDoc($iPhrase,$iCol) lappend mi $nHit($iPhrase,$iCol) + lappend mi $nDoc($iPhrase,$iCol) } } set aMatchinfo($docid) $mi @@ -119,7 +152,7 @@ proc get_single_near_results {tbl expr arrayvar nullvar} { set nullentry [list] for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} { for {set iCol 0} {$iCol<$nCol} {incr iCol} { - lappend nullentry 0 $nDoc($iPhrase,$iCol) $nHit($iPhrase,$iCol) + lappend nullentry 0 $nHit($iPhrase,$iCol) $nDoc($iPhrase,$iCol) } } } @@ -219,7 +252,7 @@ proc get_near_results {tbl expr arrayvar {nullvar ""}} { } } -proc do_near_test {tn tbl expr} { +proc do_fts3query_test {tn tbl expr} { get_near_results $tbl $expr aMatchinfo set match $expr @@ -286,19 +319,20 @@ test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"} {1 1 1} #-------------------------------------------------------------------------- # Test cases fts3auto-2.* run some simple tests using the -# [do_near_test] proc. +# [do_fts3query_test] proc. # foreach {tn create} { - 1 "CREATE VIRTUAL TABLE t1 USING fts4(a, b)" - 2 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=DESC)" - 3 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=ASC)" - 4 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, prefix=1)" - 5 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=DESC, prefix=1)" - 6 "CREATE VIRTUAL TABLE t1 USING fts4(a, b, order=ASC, prefix=1)" + 1 "fts4(a, b)" + 2 "fts4(a, b, order=DESC)" + 3 "fts4(a, b, order=ASC)" + 4 "fts4(a, b, prefix=1)" + 5 "fts4(a, b, order=DESC, prefix=1)" + 6 "fts4(a, b, order=ASC, prefix=1)" } { +break do_test 2.$tn.1 { catchsql { DROP TABLE t1 } - execsql $create + execsql "CREATE VIRTUAL TABLE t1 USING $create" for {set i 0} {$i<32} {incr i} { set doc [list] if {$i&0x01} {lappend doc one} @@ -309,6 +343,7 @@ foreach {tn create} { execsql { INSERT INTO t1 VALUES($doc, null) } } } {} + foreach {tn2 expr} { 1 {one} 2 {one NEAR/1 five} @@ -327,7 +362,69 @@ foreach {tn create} { 15 {"one two"} 16 {"one two" NOT "three four"} } { - do_near_test 2.$tn.2.$tn2 t1 $expr + do_fts3query_test 2.$tn.2.$tn2 t1 $expr + } +} + +#-------------------------------------------------------------------------- +# Some test cases involving deferred tokens. +# +proc make_token_deferrable {tbl token} { + set nRow [db one "SELECT count(*) FROM $tbl"] + set pgsz [db one "PRAGMA page_size"] + + execsql "INSERT INTO $tbl ($tbl) VALUES('maxpending=100000000')" + execsql BEGIN + for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} { + set doc [string repeat "$token " 100] + execsql "INSERT INTO $tbl VALUES(\$doc)" + } + execsql "INSERT INTO $tbl VALUES('aaaaaaa ${token}aaaaa')" + execsql COMMIT +} + +foreach {tn create} { + 1 "fts4(x)" + 2 "fts4(x, order=DESC)" +} { + catchsql { DROP TABLE t1 } + execsql "CREATE VIRTUAL TABLE t1 USING $create" + do_execsql_test 3.$tn.1 { + INSERT INTO t1 VALUES('a b c d e f g h i j k'); + INSERT INTO t1 VALUES('b c d e f g h i j k a'); + INSERT INTO t1 VALUES('c d e f g h i j k a b'); + INSERT INTO t1 VALUES('d e f g h i j k a b c'); + INSERT INTO t1 VALUES('e f g h i j k a b c d'); + INSERT INTO t1 VALUES('f g h i j k a b c d e'); + INSERT INTO t1 VALUES('a c e g i k'); + INSERT INTO t1 VALUES('a d g j'); + INSERT INTO t1 VALUES('c a b'); + } + + make_token_deferrable t1 c + + set ::fts3_deferred [list] + foreach {tn2 expr} { + 1 {a OR c} + } { + do_fts3query_test 3.$tn.2.$tn2 t1 $expr + } + + set ::fts3_deferred [list c] + execsql { + UPDATE t1_segments + SET block = zeroblob(length(block)) + WHERE length(block)>10000 AND 0 + } + foreach {tn2 expr} { + 1 {a NEAR c} + 2 {a AND c} + 3 {"a c"} + 4 {"c a"} + 5 {"a c" NEAR/1 g} + 6 {"a c" NEAR/0 g} + } { + do_fts3query_test 3.$tn.2.$tn2 t1 $expr } } diff --git a/test/fts3defer2.test b/test/fts3defer2.test index b7bf923550..92a4491efd 100644 --- a/test/fts3defer2.test +++ b/test/fts3defer2.test @@ -56,14 +56,13 @@ do_execsql_test 1.2.1 { SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)'; } {{a b c d e f a x y}} -breakpoint do_execsql_test 1.2.2 { SELECT snippet(t1, '[', ']'), offsets(t1), mit(matchinfo(t1, 'pcxnal')) FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)'; } [list \ {a b c d [e] [f] [a] x y} \ {0 1 8 1 0 0 10 1 0 2 12 1} \ - [list 3 1 1 1 1 1 1 1 1 8 8 8 5001 9] + [list 3 1 1 1 1 1 1 1 1 1 1 8 5001 9] ] do_execsql_test 1.2.3 { @@ -72,7 +71,7 @@ do_execsql_test 1.2.3 { } [list \ {[a] b c d [e] [f] [a] x y} \ {0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1} \ - [list 3 1 1 1 1 1 1 1 2 8 8 8 5001 9] + [list 3 1 1 1 1 1 1 1 2 2 1 8 5001 9] ] do_execsql_test 1.3.1 { DROP TABLE t1 } From 653265d9ed4a3eb1d088147dd0e750821e49b857 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 13 Jun 2011 18:21:11 +0000 Subject: [PATCH 17/20] Update trace2.test to account for new FTS queries. FossilOrigin-Name: 2c20129297b64f4113b8edb551385eb918279471 --- manifest | 14 +++--- manifest.uuid | 2 +- test/fts3auto.test | 110 ++++++++++++++++++++++++++------------------- test/trace2.test | 6 +-- 4 files changed, 75 insertions(+), 57 deletions(-) diff --git a/manifest b/manifest index 15429e6135..13395c5a81 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\sdeferred\stokens\sto\sfts3auto.test.\sFix\sa\sproblem\swith\sOR\squeries\sand\sdeferred\stokens. -D 2011-06-13T17:00:12.630 +C Update\strace2.test\sto\saccount\sfor\snew\sFTS\squeries. +D 2011-06-13T18:21:11.056 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -455,7 +455,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3 -F test/fts3auto.test bf01a1d793155d744d23e4092871ca116f1db64c +F test/fts3auto.test 0173f3fc3e03d66beb6dd65f7b99e7eb4ae2a27d F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 @@ -821,7 +821,7 @@ F test/tkt3997.test a335fa41ca3985660a139df7b734a26ef53284bd F test/tkt4018.test 7c2c9ba4df489c676a0a7a0e809a1fb9b2185bd1 F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7 F test/trace.test 4b36a41a3e9c7842151af6da5998f5080cdad9e5 -F test/trace2.test 0ce11265c83333d8f5beeca19e71ed93a88d386c +F test/trace2.test 962175290996d5f06dc4402ca218bbfc7df4cb20 F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test d5337e61de45e66b1fcbf9db833fa8c82e624b22 F test/trans3.test d728abaa318ca364dc370e06576aa7e5fbed7e97 @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P e4ab6cdb101bbeb804820425cf569ee7dc2397fb -R 7fcdbf104822a42776783c2cd9e89101 +P b9fb69e55bb05a8819688ee63987f2a05c33d59b +R 4f421cd82a93c9e8973177e58e4c866f U dan -Z 0999cf6b26be67f673d9db0918d86701 +Z 2a2f406b9fac55b2e6e57d2ed45fc4da diff --git a/manifest.uuid b/manifest.uuid index 41624e6daa..5186fdc744 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b9fb69e55bb05a8819688ee63987f2a05c33d59b \ No newline at end of file +2c20129297b64f4113b8edb551385eb918279471 \ No newline at end of file diff --git a/test/fts3auto.test b/test/fts3auto.test index 68c82c8a85..f8f1e77b42 100644 --- a/test/fts3auto.test +++ b/test/fts3auto.test @@ -21,8 +21,62 @@ set sfep $sqlite_fts3_enable_parentheses set sqlite_fts3_enable_parentheses 1 #-------------------------------------------------------------------------- -# Start of Tcl procs used by tests. +# Start of Tcl infrastructure used by tests. The entry point is +# [do_fts3query_test] (described below). # + +# do_fts3query_test TESTNAME ?OPTIONS? TABLE MATCHEXPR +# +# This proc runs several test cases on FTS3/4 table $TABLE using match +# expression $MATCHEXPR. All documents in $TABLE must be formatted so that +# they can be "tokenized" using the Tcl list commands (llength, lindex etc.). +# The name and column names used by $TABLE must not require any quoting or +# escaping when used in SQL statements. +# +# $MATCHINFO may be any expression accepted by the FTS4 MATCH operator, +# except that the ":token" syntax is not supported. Tcl list +# commands are used to tokenize the expression. Any parenthesis must appear +# either as separate list elements, or as the first (for opening) or last +# (for closing) character of a list element. i.e. the expression "(a OR b)c" +# will not be parsed correctly, but "( a OR b) c" will. +# +set sqlite_fts3_enable_parentheses 1 +proc do_fts3query_test {tn tbl expr} { + + get_near_results $tbl $expr aMatchinfo + set match $expr + + set matchinfo_asc [list] + foreach docid [lsort -integer -incr [array names aMatchinfo]] { + lappend matchinfo_asc $docid $aMatchinfo($docid) + } + set matchinfo_desc [list] + foreach docid [lsort -integer -decr [array names aMatchinfo]] { + lappend matchinfo_desc $docid $aMatchinfo($docid) + } + + set title "(\"$match\" -> [llength [array names aMatchinfo]] rows)" + + do_execsql_test $tn$title.1 " + SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid ASC + " [lsort -integer -incr [array names aMatchinfo]] + + do_execsql_test $tn$title.2 " + SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid DESC + " [lsort -integer -decr [array names aMatchinfo]] + + do_execsql_test $tn$title.3 " + SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl + WHERE $tbl MATCH '$match' ORDER BY docid DESC + " $matchinfo_desc + + do_execsql_test $tn$title.4 " + SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl + WHERE $tbl MATCH '$match' ORDER BY docid ASC + " $matchinfo_asc +} + + proc mit {blob} { set scan(littleEndian) i* set scan(bigEndian) I* @@ -252,41 +306,6 @@ proc get_near_results {tbl expr arrayvar {nullvar ""}} { } } -proc do_fts3query_test {tn tbl expr} { - - get_near_results $tbl $expr aMatchinfo - set match $expr - - set matchinfo_asc [list] - foreach docid [lsort -integer -incr [array names aMatchinfo]] { - lappend matchinfo_asc $docid $aMatchinfo($docid) - } - set matchinfo_desc [list] - foreach docid [lsort -integer -decr [array names aMatchinfo]] { - lappend matchinfo_desc $docid $aMatchinfo($docid) - } - - set title "(\"$match\" -> [llength [array names aMatchinfo]] rows)" - - do_execsql_test $tn$title.1 " - SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid ASC - " [lsort -integer -incr [array names aMatchinfo]] - - do_execsql_test $tn$title.2 " - SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid DESC - " [lsort -integer -decr [array names aMatchinfo]] - - do_execsql_test $tn$title.3 " - SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl - WHERE $tbl MATCH '$match' ORDER BY docid DESC - " $matchinfo_desc - - do_execsql_test $tn$title.4 " - SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl - WHERE $tbl MATCH '$match' ORDER BY docid ASC - " $matchinfo_asc -} - # End of test procs. Actual tests are below this line. #-------------------------------------------------------------------------- @@ -329,7 +348,6 @@ foreach {tn create} { 5 "fts4(a, b, order=DESC, prefix=1)" 6 "fts4(a, b, order=ASC, prefix=1)" } { -break do_test 2.$tn.1 { catchsql { DROP TABLE t1 } execsql "CREATE VIRTUAL TABLE t1 USING $create" @@ -390,15 +408,15 @@ foreach {tn create} { catchsql { DROP TABLE t1 } execsql "CREATE VIRTUAL TABLE t1 USING $create" do_execsql_test 3.$tn.1 { - INSERT INTO t1 VALUES('a b c d e f g h i j k'); - INSERT INTO t1 VALUES('b c d e f g h i j k a'); - INSERT INTO t1 VALUES('c d e f g h i j k a b'); - INSERT INTO t1 VALUES('d e f g h i j k a b c'); - INSERT INTO t1 VALUES('e f g h i j k a b c d'); - INSERT INTO t1 VALUES('f g h i j k a b c d e'); - INSERT INTO t1 VALUES('a c e g i k'); - INSERT INTO t1 VALUES('a d g j'); - INSERT INTO t1 VALUES('c a b'); + INSERT INTO t1(docid, x) VALUES(-2, 'a b c d e f g h i j k'); + INSERT INTO t1(docid, x) VALUES(-1, 'b c d e f g h i j k a'); + INSERT INTO t1(docid, x) VALUES(0, 'c d e f g h i j k a b'); + INSERT INTO t1(docid, x) VALUES(1, 'd e f g h i j k a b c'); + INSERT INTO t1(docid, x) VALUES(2, 'e f g h i j k a b c d'); + INSERT INTO t1(docid, x) VALUES(3, 'f g h i j k a b c d e'); + INSERT INTO t1(docid, x) VALUES(4, 'a c e g i k'); + INSERT INTO t1(docid, x) VALUES(5, 'a d g j'); + INSERT INTO t1(docid, x) VALUES(6, 'c a b'); } make_token_deferrable t1 c diff --git a/test/trace2.test b/test/trace2.test index a70dd89d1d..42738db3aa 100644 --- a/test/trace2.test +++ b/test/trace2.test @@ -141,10 +141,10 @@ ifcapable fts3 { INSERT INTO x1(x1) VALUES('optimize'); } { "INSERT INTO x1(x1) VALUES('optimize');" - "-- SELECT idx, start_block, leaves_end_block, end_block, root FROM 'main'.'x1_segdir' ORDER BY level DESC, idx ASC" - "-- SELECT count(*), max(level) FROM 'main'.'x1_segdir'" + "-- SELECT idx, start_block, leaves_end_block, end_block, root FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?ORDER BY level DESC, idx ASC" + "-- SELECT max(level) FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?" "-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)" - "-- DELETE FROM 'main'.'x1_segdir'" + "-- DELETE FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?" "-- INSERT INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)" } } From 806cbfc2149da1de634c9fcf685428b732a57ca8 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 14 Jun 2011 07:14:43 +0000 Subject: [PATCH 18/20] Remove unused parameters from internal fts3 function. FossilOrigin-Name: 06de3f2cbc27cdfd9f83218c9ea576f74f60d07b --- ext/fts3/fts3.c | 54 ++++++++++++++++++++++++------------------------- manifest | 12 +++++------ manifest.uuid | 2 +- 3 files changed, 34 insertions(+), 34 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 17a7b2ccaa..60b8595e84 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3342,21 +3342,6 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ return SQLITE_OK; } - -/* -** The following three functions: -** -** fts3EvalPhraseStart() -** fts3EvalPhraseNext() -** -** May be used with a phrase object after fts3EvalAllocateReaders() has been -** called to iterate through the set of docids that match the phrase. -** -** After a successful call to fts3EvalPhraseNext(), the following two -** functions may be called to access the current docid and position-list. -*/ - - /* ** This function is called for each Fts3Phrase in a full-text query ** expression to initialize the mechanism for returning rows. Once this @@ -4158,12 +4143,16 @@ static void fts3EvalRestart( } } -static void fts3EvalUpdateCounts( - Fts3Cursor *pCsr, - Fts3Expr *pExpr, - int *pRc -){ - if( pExpr && *pRc==SQLITE_OK ){ +/* +** After allocating the Fts3Expr.aMI[] array for each phrase in the +** expression rooted at pExpr, the cursor iterates through all rows matched +** by pExpr, calling this function for each row. This function increments +** the values in Fts3Expr.aMI[] according to the position-list currently +** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase +** expression nodes. +*/ +static void fts3EvalUpdateCounts(Fts3Expr *pExpr){ + if( pExpr ){ Fts3Phrase *pPhrase = pExpr->pPhrase; if( pPhrase && pPhrase->doclist.pList ){ int iCol = 0; @@ -4189,14 +4178,25 @@ static void fts3EvalUpdateCounts( } } - fts3EvalUpdateCounts(pCsr, pExpr->pLeft, pRc); - fts3EvalUpdateCounts(pCsr, pExpr->pRight, pRc); + fts3EvalUpdateCounts(pExpr->pLeft); + fts3EvalUpdateCounts(pExpr->pRight); } } +/* +** Expression pExpr must be of type FTSQUERY_PHRASE. +** +** If it is not already allocated and populated, this function allocates and +** populates the Fts3Expr.aMI[] array for expression pExpr. If pExpr is part +** of a NEAR expression, then it also allocates and populates the same array +** for all other phrases that are part of the NEAR expression. +** +** SQLITE_OK is returned if the aMI[] array is successfully allocated and +** populated. Otherwise, if an error occurs, an SQLite error code is returned. +*/ static int fts3EvalGatherStats( - Fts3Cursor *pCsr, - Fts3Expr *pExpr + Fts3Cursor *pCsr, /* Cursor object */ + Fts3Expr *pExpr /* FTSQUERY_PHRASE expression */ ){ int rc = SQLITE_OK; /* Return code */ @@ -4248,8 +4248,8 @@ static int fts3EvalGatherStats( && fts3EvalLoadDeferred(pCsr, &rc) ); - if( pCsr->isEof==0 ){ - fts3EvalUpdateCounts(pCsr, pRoot, &rc); + if( rc==SQLITE_OK && pCsr->isEof==0 ){ + fts3EvalUpdateCounts(pRoot); } } diff --git a/manifest b/manifest index 13395c5a81..037d939c2d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\strace2.test\sto\saccount\sfor\snew\sFTS\squeries. -D 2011-06-13T18:21:11.056 +C Remove\sunused\sparameters\sfrom\sinternal\sfts3\sfunction. +D 2011-06-14T07:14:43.149 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -61,7 +61,7 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 2a48945d199d5d0fe66a06c8cf73ad2f0e7a4b5f +F ext/fts3/fts3.c f4f65273121386349993d600c4c5f710d4b3e956 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h a999cfbf605efec293a88519f74192f5204c84d6 F ext/fts3/fts3_aux.c baed9dab7fb4604ae8cafdb2d7700abe93beffbe @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P b9fb69e55bb05a8819688ee63987f2a05c33d59b -R 4f421cd82a93c9e8973177e58e4c866f +P 2c20129297b64f4113b8edb551385eb918279471 +R 24738ea4e4f82770b7e9961a6f31bf0f U dan -Z 2a2f406b9fac55b2e6e57d2ed45fc4da +Z 096f5065b2f90702df92a748235d8620 diff --git a/manifest.uuid b/manifest.uuid index 5186fdc744..6cb1ded47e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2c20129297b64f4113b8edb551385eb918279471 \ No newline at end of file +06de3f2cbc27cdfd9f83218c9ea576f74f60d07b \ No newline at end of file From 2cf1a1de4bad0aa492f94d525c68b524d394b5a7 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 14 Jun 2011 09:00:27 +0000 Subject: [PATCH 19/20] Fix another bug caused by NEAR/matchinfo/order=DESC interaction. FossilOrigin-Name: 04907fbadeb743c95cc9f3529e63ef388684799f --- ext/fts3/fts3.c | 7 ++- manifest | 16 +++---- manifest.uuid | 2 +- test/fts3auto.test | 97 ++++++++++++++++++++++++++++++++---------- test/permutations.test | 2 +- 5 files changed, 87 insertions(+), 37 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index e057eb3825..feed5220c9 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3387,7 +3387,6 @@ void sqlite3Fts3DoclistPrev( u8 *pbEof /* OUT: End-of-file flag */ ){ char *p = *ppIter; - int iMul = (bDescIdx ? -1 : 1); assert( nDoclist>0 ); assert( *pbEof==0 ); @@ -3399,10 +3398,8 @@ void sqlite3Fts3DoclistPrev( char *pNext = 0; char *pDocid = aDoclist; char *pEnd = &aDoclist[nDoclist]; + int iMul = 1; - pDocid += sqlite3Fts3GetVarint(pDocid, &iDocid); - pNext = pDocid; - fts3PoslistCopy(0, &pDocid); while( pDocid=0} { + if {[lsearch $deferred $tok]>=0} { lappend rewritten * } else { lappend rewritten $tok @@ -178,7 +205,7 @@ proc get_single_near_results {tbl expr arrayvar nullvar} { while {[lindex $rewritten 0] == "*"} { set rewritten [lrange $rewritten 1 end] } - get_single_near_results $tbl [list $rewritten] aRewrite nullentry + get_single_near_results $tbl [list $rewritten] {} aRewrite nullentry foreach docid [array names hits] { set aMatchinfo($docid) $aRewrite($docid) } @@ -228,7 +255,7 @@ proc matching_brackets {expr} { return [expr ($iBracket==0 && $i==$nExpr)] } -proc get_near_results {tbl expr arrayvar {nullvar ""}} { +proc get_near_results {tbl expr deferred arrayvar {nullvar ""}} { upvar $arrayvar aMatchinfo if {$nullvar != ""} { upvar $nullvar nullentry } @@ -260,13 +287,13 @@ proc get_near_results {tbl expr arrayvar {nullvar ""}} { if {$iBracket!=0} { error "mismatched brackets in: $expr" } if {[info exists opidx]==0} { - get_single_near_results $tbl $expr aMatchinfo nullentry + get_single_near_results $tbl $expr $deferred aMatchinfo nullentry } else { set eLeft [lrange $expr 0 [expr $opidx-1]] set eRight [lrange $expr [expr $opidx+1] end] - get_near_results $tbl $eLeft aLeft nullleft - get_near_results $tbl $eRight aRight nullright + get_near_results $tbl $eLeft $deferred aLeft nullleft + get_near_results $tbl $eRight $deferred aRight nullright switch -- [lindex $expr $opidx] { "NOT" { @@ -390,8 +417,6 @@ foreach {tn create} { proc make_token_deferrable {tbl token} { set nRow [db one "SELECT count(*) FROM $tbl"] set pgsz [db one "PRAGMA page_size"] - - execsql "INSERT INTO $tbl ($tbl) VALUES('maxpending=100000000')" execsql BEGIN for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} { set doc [string repeat "$token " 100] @@ -421,28 +446,54 @@ foreach {tn create} { make_token_deferrable t1 c - set ::fts3_deferred [list] foreach {tn2 expr} { 1 {a OR c} } { do_fts3query_test 3.$tn.2.$tn2 t1 $expr } - set ::fts3_deferred [list c] execsql { UPDATE t1_segments SET block = zeroblob(length(block)) WHERE length(block)>10000 AND 0 } - foreach {tn2 expr} { - 1 {a NEAR c} - 2 {a AND c} - 3 {"a c"} - 4 {"c a"} - 5 {"a c" NEAR/1 g} - 6 {"a c" NEAR/0 g} + foreach {tn2 expr def} { + 1 {a NEAR c} {} + 2 {a AND c} c + 3 {"a c"} c + 4 {"c a"} c + 5 {"a c" NEAR/1 g} {} + 6 {"a c" NEAR/0 g} {} } { - do_fts3query_test 3.$tn.2.$tn2 t1 $expr + do_fts3query_test 3.$tn.2.$tn2 -deferred $def t1 $expr + } +} + +#-------------------------------------------------------------------------- +# +foreach {tn create} { + 1 "fts4(x, y)" + 2 "fts4(x, y, order=DESC)" +} { + catchsql { DROP TABLE t1 } + execsql "CREATE VIRTUAL TABLE t1 USING $create" + + foreach {x y} { + {one two five four five} {} + {} {one two five four five} + {one two} {five four five} + } { + execsql {INSERT INTO t1 VALUES($x, $y)} + } + + foreach {tn2 expr} { + 1 {one AND five} + 2 {one NEAR five} + 3 {one NEAR/1 five} + 4 {one NEAR/2 five} + 5 {one NEAR/3 five} + } { + do_fts3query_test 4.$tn.2.$tn2 t1 $expr } } diff --git a/test/permutations.test b/test/permutations.test index 4640ed1139..9c95296e39 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -183,7 +183,7 @@ test_suite "fts3" -prefix "" -description { fts3fault.test fts3malloc.test fts3matchinfo.test - fts3aux1.test fts3comp1.test + fts3aux1.test fts3comp1.test fts3auto.test } From db27fc078534611a934bafad541f2157c2ba4507 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 14 Jun 2011 11:32:50 +0000 Subject: [PATCH 20/20] Add a couple of extra tests. FossilOrigin-Name: aefd46dfae7e06fbaf4f2b9a86a7f2ac6927331e --- manifest | 12 ++--- manifest.uuid | 2 +- test/fts3auto.test | 121 ++++++++++++++++++++++++++++----------------- 3 files changed, 84 insertions(+), 51 deletions(-) diff --git a/manifest b/manifest index 252014ea4d..bc0807804f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sanother\sbug\scaused\sby\sNEAR/matchinfo/order=DESC\sinteraction. -D 2011-06-14T09:00:27.140 +C Add\sa\scouple\sof\sextra\stests. +D 2011-06-14T11:32:50.766 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -455,7 +455,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9 F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3 -F test/fts3auto.test 337dff2758887bfe7d337f76d25cc48db6e26f28 +F test/fts3auto.test 2f86f2a0e8ffa26d81d570897e6cc1c2262256d5 F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0 F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984 F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958 @@ -945,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d -P 135ce30f62ebd6a1b239c18dbbd9c926ea507db4 -R 6437acdf1435a01de33437d42862b03a +P 04907fbadeb743c95cc9f3529e63ef388684799f +R 84da75d7a3a06308ee488ec7371657d8 U dan -Z 0ea30b6b1ae90b1e541e7d3c3cb40fc6 +Z 0b29b5535d6bfbaa65e9d19cac80576d diff --git a/manifest.uuid b/manifest.uuid index 79eee6ffa9..3ddb9fa573 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -04907fbadeb743c95cc9f3529e63ef388684799f \ No newline at end of file +aefd46dfae7e06fbaf4f2b9a86a7f2ac6927331e \ No newline at end of file diff --git a/test/fts3auto.test b/test/fts3auto.test index 47429308a0..5c9b85128b 100644 --- a/test/fts3auto.test +++ b/test/fts3auto.test @@ -21,10 +21,14 @@ set sfep $sqlite_fts3_enable_parentheses set sqlite_fts3_enable_parentheses 1 #-------------------------------------------------------------------------- -# Start of Tcl infrastructure used by tests. The entry point is -# [do_fts3query_test] (described below). +# Start of Tcl infrastructure used by tests. The entry points are: +# +# do_fts3query_test +# fts3_make_deferrable +# fts3_zero_long_segments # +# # do_fts3query_test TESTNAME ?OPTIONS? TABLE MATCHEXPR # # This proc runs several test cases on FTS3/4 table $TABLE using match @@ -103,6 +107,38 @@ proc do_fts3query_test {tn args} { " $matchinfo_asc } +# fts3_make_deferrable TABLE TOKEN +# +proc fts3_make_deferrable {tbl token} { + + set stmt [sqlite3_prepare db "SELECT * FROM $tbl" -1 dummy] + set name [sqlite3_column_name $stmt 0] + sqlite3_finalize $stmt + + set nRow [db one "SELECT count(*) FROM $tbl"] + set pgsz [db one "PRAGMA page_size"] + execsql BEGIN + for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} { + set doc [string repeat "$token " 100] + execsql "INSERT INTO $tbl ($name) VALUES(\$doc)" + } + execsql "INSERT INTO $tbl ($name) VALUES('aaaaaaa ${token}aaaaa')" + execsql COMMIT + + return [expr $nRow*$pgsz] +} + +# fts3_zero_long_segments TABLE ?LIMIT? +# +proc fts3_zero_long_segments {tbl limit} { + execsql " + UPDATE ${tbl}_segments + SET block = zeroblob(length(block)) + WHERE length(block)>$limit + " + return [db changes] +} + proc mit {blob} { set scan(littleEndian) i* @@ -414,17 +450,6 @@ foreach {tn create} { #-------------------------------------------------------------------------- # Some test cases involving deferred tokens. # -proc make_token_deferrable {tbl token} { - set nRow [db one "SELECT count(*) FROM $tbl"] - set pgsz [db one "PRAGMA page_size"] - execsql BEGIN - for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} { - set doc [string repeat "$token " 100] - execsql "INSERT INTO $tbl VALUES(\$doc)" - } - execsql "INSERT INTO $tbl VALUES('aaaaaaa ${token}aaaaa')" - execsql COMMIT -} foreach {tn create} { 1 "fts4(x)" @@ -444,19 +469,14 @@ foreach {tn create} { INSERT INTO t1(docid, x) VALUES(6, 'c a b'); } - make_token_deferrable t1 c + set limit [fts3_make_deferrable t1 c] - foreach {tn2 expr} { - 1 {a OR c} - } { - do_fts3query_test 3.$tn.2.$tn2 t1 $expr - } + do_fts3query_test 3.$tn.2.1 t1 {a OR c} + + do_test 3.$tn.3 { + fts3_zero_long_segments t1 $limit + } {1} - execsql { - UPDATE t1_segments - SET block = zeroblob(length(block)) - WHERE length(block)>10000 AND 0 - } foreach {tn2 expr def} { 1 {a NEAR c} {} 2 {a AND c} c @@ -465,36 +485,49 @@ foreach {tn create} { 5 {"a c" NEAR/1 g} {} 6 {"a c" NEAR/0 g} {} } { - do_fts3query_test 3.$tn.2.$tn2 -deferred $def t1 $expr + do_fts3query_test 3.$tn.4.$tn2 -deferred $def t1 $expr } } #-------------------------------------------------------------------------- -# +# foreach {tn create} { 1 "fts4(x, y)" 2 "fts4(x, y, order=DESC)" + 3 "fts4(x, y, order=DESC, prefix=2)" } { - catchsql { DROP TABLE t1 } - execsql "CREATE VIRTUAL TABLE t1 USING $create" - foreach {x y} { - {one two five four five} {} - {} {one two five four five} - {one two} {five four five} - } { - execsql {INSERT INTO t1 VALUES($x, $y)} - } + execsql [subst { + DROP TABLE t1; + CREATE VIRTUAL TABLE t1 USING $create; + INSERT INTO t1 VALUES('one two five four five', ''); + INSERT INTO t1 VALUES('', 'one two five four five'); + INSERT INTO t1 VALUES('one two', 'five four five'); + }] - foreach {tn2 expr} { - 1 {one AND five} - 2 {one NEAR five} - 3 {one NEAR/1 five} - 4 {one NEAR/2 five} - 5 {one NEAR/3 five} - } { - do_fts3query_test 4.$tn.2.$tn2 t1 $expr - } + do_fts3query_test 4.$tn.1.1 t1 {one AND five} + do_fts3query_test 4.$tn.1.2 t1 {one NEAR five} + do_fts3query_test 4.$tn.1.3 t1 {one NEAR/1 five} + do_fts3query_test 4.$tn.1.4 t1 {one NEAR/2 five} + do_fts3query_test 4.$tn.1.5 t1 {one NEAR/3 five} + + do_test 4.$tn.2 { + set limit [fts3_make_deferrable t1 five] + execsql { INSERT INTO t1(t1) VALUES('optimize') } + expr {[fts3_zero_long_segments t1 $limit]>0} + } {1} + + do_fts3query_test 4.$tn.3.1 -deferred five t1 {one AND five} + do_fts3query_test 4.$tn.3.2 -deferred five t1 {one NEAR five} + do_fts3query_test 4.$tn.3.3 -deferred five t1 {one NEAR/1 five} + do_fts3query_test 4.$tn.3.4 -deferred five t1 {one NEAR/2 five} + do_fts3query_test 4.$tn.3.5 -deferred five t1 {one NEAR/3 five} + + do_fts3query_test 4.$tn.4.1 -deferred fi* t1 {on* AND fi*} + do_fts3query_test 4.$tn.4.2 -deferred fi* t1 {on* NEAR fi*} + do_fts3query_test 4.$tn.4.3 -deferred fi* t1 {on* NEAR/1 fi*} + do_fts3query_test 4.$tn.4.4 -deferred fi* t1 {on* NEAR/2 fi*} + do_fts3query_test 4.$tn.4.5 -deferred fi* t1 {on* NEAR/3 fi*} } set sqlite_fts3_enable_parentheses $sfep