paulhorn/sqlite
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge fts3-prefix-search branch with trunk.

Browse Source 
FossilOrigin-Name: b1f9c1e0ac51cedfb05ac073a603343f6df865b5
This commit is contained in:
dan
2011-06-14 11:50:09 +00:00
parent 0a32fa6d81 db27fc0785
commit 79bd81038c
24 changed files with 3956 additions and 1870 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile.in
+2 -1
View File
@@ -379,7 +379,8 @@ TESTSRC = \
$(TOP)/src/test_vfs.c \
$(TOP)/src/test_wholenumber.c \
$(TOP)/src/test_wsd.c \
$(TOP)/ext/fts3/fts3_term.c
$(TOP)/ext/fts3/fts3_term.c \
$(TOP)/ext/fts3/fts3_test.c
# Source code to the library files needed by the test fixture
#
ext/fts3/fts3.c
+1801 -1223
View File
File diff suppressed because it is too large Load Diff
ext/fts3/fts3Int.h
+134 -48
View File
@@ -47,12 +47,35 @@
*/
#define SizeofArray(X) ((int)(sizeof(X)/sizeof(X[0])))
#ifndef MIN
# define MIN(x,y) ((x)<(y)?(x):(y))
#endif
/*
** Maximum length of a varint encoded integer. The varint format is different
** from that used by SQLite, so the maximum length is 10, not 9.
*/
#define FTS3_VARINT_MAX 10
/*
** FTS4 virtual tables may maintain multiple indexes - one index of all terms
** in the document set and zero or more prefix indexes. All indexes are stored
** as one or more b+-trees in the %_segments and %_segdir tables.
**
** It is possible to determine which index a b+-tree belongs to based on the
** value stored in the "%_segdir.level" column. Given this value L, the index
** that the b+-tree belongs to is (L<<10). In other words, all b+-trees with
** level values between 0 and 1023 (inclusive) belong to index 0, all levels
** between 1024 and 2047 to index 1, and so on.
**
** It is considered impossible for an index to use more than 1024 levels. In
** theory though this may happen, but only after at least
** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables.
*/
#define FTS3_SEGDIR_MAXLEVEL 1024
#define FTS3_SEGDIR_MAXLEVEL_STR "1024"
/*
** The testcase() macro is only used by the amalgamation. If undefined,
** make it a no-op.
@@ -124,10 +147,11 @@ typedef struct Fts3Expr Fts3Expr;
typedef struct Fts3Phrase Fts3Phrase;
typedef struct Fts3PhraseToken Fts3PhraseToken;
typedef struct Fts3Doclist Fts3Doclist;
typedef struct Fts3SegFilter Fts3SegFilter;
typedef struct Fts3DeferredToken Fts3DeferredToken;
typedef struct Fts3SegReader Fts3SegReader;
typedef struct Fts3SegReaderCursor Fts3SegReaderCursor;
typedef struct Fts3MultiSegReader Fts3MultiSegReader;
/*
** A connection to a fulltext index is an instance of the following
@@ -148,7 +172,7 @@ struct Fts3Table {
/* Precompiled statements used by the implementation. Each of these
** statements is run and reset within a single virtual table API call.
*/
sqlite3_stmt *aStmt[24];
sqlite3_stmt *aStmt[27];
char *zReadExprlist;
char *zWriteExprlist;
@@ -156,21 +180,33 @@ struct Fts3Table {
int nNodeSize; /* Soft limit for node size */
u8 bHasStat; /* True if %_stat table exists */
u8 bHasDocsize; /* True if %_docsize table exists */
u8 bDescIdx; /* True if doclists are in reverse order */
int nPgsz; /* Page size for host database */
char *zSegmentsTbl; /* Name of %_segments table */
sqlite3_blob *pSegments; /* Blob handle open on %_segments table */
/* The following hash table is used to buffer pending index updates during
/* TODO: Fix the first paragraph of this comment.
**
** The following hash table is used to buffer pending index updates during
** transactions. Variable nPendingData estimates the memory size of the
** pending data, including hash table overhead, but not malloc overhead.
** When nPendingData exceeds nMaxPendingData, the buffer is flushed
** automatically. Variable iPrevDocid is the docid of the most recently
** inserted record.
**
** A single FTS4 table may have multiple full-text indexes. For each index
** there is an entry in the aIndex[] array. Index 0 is an index of all the
** terms that appear in the document set. Each subsequent index in aIndex[]
** is an index of prefixes of a specific length.
*/
int nMaxPendingData;
int nPendingData;
sqlite_int64 iPrevDocid;
Fts3Hash pendingTerms;
int nIndex; /* Size of aIndex[] */
struct Fts3Index {
int nPrefix; /* Prefix length (0 for main terms index) */
Fts3Hash hPending; /* Pending terms table for this index */
} *aIndex;
int nMaxPendingData; /* Max pending data before flush to disk */
int nPendingData; /* Current bytes of pending data */
sqlite_int64 iPrevDocid; /* Docid of most recently inserted document */
#if defined(SQLITE_DEBUG)
/* State variables used for validating that the transaction control
@@ -201,9 +237,10 @@ struct Fts3Cursor {
char *pNextId; /* Pointer into the body of aDoclist */
char *aDoclist; /* List of docids for full-text queries */
int nDoclist; /* Size of buffer at aDoclist */
int desc; /* True to sort in descending order */
u8 bDesc; /* True to sort in descending order */
int eEvalmode; /* An FTS3_EVAL_XX constant */
int nRowAvg; /* Average size of database rows, in pages */
int nDoc; /* Documents in table */
int isMatchinfoNeeded; /* True when aMatchinfo[] needs filling in */
u32 *aMatchinfo; /* Information about most recent match */
@@ -234,47 +271,70 @@ struct Fts3Cursor {
#define FTS3_DOCID_SEARCH 1 /* Lookup by rowid on %_content table */
#define FTS3_FULLTEXT_SEARCH 2 /* Full-text index search */
struct Fts3Doclist {
char *aAll; /* Array containing doclist (or NULL) */
int nAll; /* Size of a[] in bytes */
char *pNextDocid; /* Pointer to next docid */
sqlite3_int64 iDocid; /* Current docid (if pList!=0) */
int bFreeList; /* True if pList should be sqlite3_free()d */
char *pList; /* Pointer to position list following iDocid */
int nList; /* Length of position list */
} doclist;
/*
** A "phrase" is a sequence of one or more tokens that must match in
** sequence. A single token is the base case and the most common case.
** For a sequence of tokens contained in double-quotes (i.e. "one two three")
** nToken will be the number of tokens in the string.
**
** The nDocMatch and nMatch variables contain data that may be used by the
** matchinfo() function. They are populated when the full-text index is
** queried for hits on the phrase. If one or more tokens in the phrase
** are deferred, the nDocMatch and nMatch variables are populated based
** on the assumption that the
*/
struct Fts3PhraseToken {
char *z; /* Text of the token */
int n; /* Number of bytes in buffer z */
int isPrefix; /* True if token ends with a "*" character */
/* Variables above this point are populated when the expression is
** parsed (by code in fts3_expr.c). Below this point the variables are
** used when evaluating the expression. */
int bFulltext; /* True if full-text index was used */
Fts3SegReaderCursor *pSegcsr; /* Segment-reader for this token */
Fts3DeferredToken *pDeferred; /* Deferred token object for this token */
Fts3MultiSegReader *pSegcsr; /* Segment-reader for this token */
};
struct Fts3Phrase {
/* Variables populated by fts3_expr.c when parsing a MATCH expression */
/* Cache of doclist for this phrase. */
Fts3Doclist doclist;
int bIncr; /* True if doclist is loaded incrementally */
/* Variables below this point are populated by fts3_expr.c when parsing
** a MATCH expression. Everything above is part of the evaluation phase.
*/
int nToken; /* Number of tokens in the phrase */
int iColumn; /* Index of column this phrase must match */
int isNot; /* Phrase prefixed by unary not (-) operator */
Fts3PhraseToken aToken[1]; /* One entry for each token in the phrase */
};
/*
** A tree of these objects forms the RHS of a MATCH operator.
**
** If Fts3Expr.eType is either FTSQUERY_NEAR or FTSQUERY_PHRASE and isLoaded
** is true, then aDoclist points to a malloced buffer, size nDoclist bytes,
** containing the results of the NEAR or phrase query in FTS3 doclist
** format. As usual, the initial "Length" field found in doclists stored
** on disk is omitted from this buffer.
** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist
** points to a malloced buffer, size nDoclist bytes, containing the results
** of this phrase query in FTS3 doclist format. As usual, the initial
** "Length" field found in doclists stored on disk is omitted from this
** buffer.
**
** Variable pCurrent always points to the start of a docid field within
** aDoclist. Since the doclist is usually scanned in docid order, this can
** be used to accelerate seeking to the required docid within the doclist.
** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global
** matchinfo data. If it is not NULL, it points to an array of size nCol*3,
** where nCol is the number of columns in the queried FTS table. The array
** is populated as follows:
**
** aMI[iCol*3 + 0] = Undefined
** aMI[iCol*3 + 1] = Number of occurrences
** aMI[iCol*3 + 2] = Number of rows containing at least one instance
**
** The aMI array is allocated using sqlite3_malloc(). It should be freed
** when the expression node is.
*/
struct Fts3Expr {
int eType; /* One of the FTSQUERY_XXX values defined below */
@@ -284,12 +344,13 @@ struct Fts3Expr {
Fts3Expr *pRight; /* Right operand */
Fts3Phrase *pPhrase; /* Valid if eType==FTSQUERY_PHRASE */
int isLoaded; /* True if aDoclist/nDoclist are initialized. */
char *aDoclist; /* Buffer containing doclist */
int nDoclist; /* Size of aDoclist in bytes */
/* The following are used by the fts3_eval.c module. */
sqlite3_int64 iDocid; /* Current docid */
u8 bEof; /* True this expression is at EOF already */
u8 bStart; /* True if iDocid is valid */
u8 bDeferred; /* True if this expression is entirely deferred */
sqlite3_int64 iCurrent;
char *pCurrent;
u32 *aMI;
};
/*
@@ -317,12 +378,12 @@ void sqlite3Fts3PendingTermsClear(Fts3Table *);
int sqlite3Fts3Optimize(Fts3Table *);
int sqlite3Fts3SegReaderNew(int, sqlite3_int64,
sqlite3_int64, sqlite3_int64, const char *, int, Fts3SegReader**);
int sqlite3Fts3SegReaderPending(Fts3Table*,const char*,int,int,Fts3SegReader**);
int sqlite3Fts3SegReaderPending(
Fts3Table*,int,const char*,int,int,Fts3SegReader**);
void sqlite3Fts3SegReaderFree(Fts3SegReader *);
int sqlite3Fts3SegReaderCost(Fts3Cursor *, Fts3SegReader *, int *);
int sqlite3Fts3AllSegdirs(Fts3Table*, int, sqlite3_stmt **);
int sqlite3Fts3AllSegdirs(Fts3Table*, int, int, sqlite3_stmt **);
int sqlite3Fts3ReadLock(Fts3Table *);
int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*);
int sqlite3Fts3ReadBlock(Fts3Table*, sqlite3_int64, char **, int*, int*);
int sqlite3Fts3SelectDoctotal(Fts3Table *, sqlite3_stmt **);
int sqlite3Fts3SelectDocsize(Fts3Table *, sqlite3_int64, sqlite3_stmt **);
@@ -331,17 +392,18 @@ void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *);
int sqlite3Fts3DeferToken(Fts3Cursor *, Fts3PhraseToken *, int);
int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *);
void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *);
char *sqlite3Fts3DeferredDoclist(Fts3DeferredToken *, int *);
void sqlite3Fts3SegmentsClose(Fts3Table *);
#define FTS3_SEGCURSOR_PENDING -1
#define FTS3_SEGCURSOR_ALL -2
/* Special values interpreted by sqlite3SegReaderCursor() */
#define FTS3_SEGCURSOR_PENDING -1
#define FTS3_SEGCURSOR_ALL -2
int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Fts3SegFilter*);
int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *);
void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *);
int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3SegReaderCursor*, Fts3SegFilter*);
int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3SegReaderCursor *);
void sqlite3Fts3SegReaderFinish(Fts3SegReaderCursor *);
int sqlite3Fts3SegReaderCursor(
Fts3Table *, int, const char *, int, int, int, Fts3SegReaderCursor *);
Fts3Table *, int, int, const char *, int, int, int, Fts3MultiSegReader *);
/* Flags allowed as part of the 4th argument to SegmentReaderIterate() */
#define FTS3_SEGMENT_REQUIRE_POS 0x00000001
@@ -358,7 +420,7 @@ struct Fts3SegFilter {
int flags;
};
struct Fts3SegReaderCursor {
struct Fts3MultiSegReader {
/* Used internally by sqlite3Fts3SegReaderXXX() calls */
Fts3SegReader **apSegment; /* Array of Fts3SegReader objects */
int nSegment; /* Size of apSegment array */
@@ -367,8 +429,11 @@ struct Fts3SegReaderCursor {
char *aBuffer; /* Buffer to merge doclists in */
int nBuffer; /* Allocated size of aBuffer[] in bytes */
/* Cost of running this iterator. Used by fts3.c only. */
int nCost;
int iColFilter; /* If >=0, filter for this column */
/* Used by fts3.c only. */
int nCost; /* Cost of running iterator */
int bLookup; /* True if a lookup of a single entry. */
/* Output values. Valid only after Fts3SegReaderStep() returns SQLITE_ROW. */
char *zTerm; /* Pointer to term buffer */
@@ -383,11 +448,9 @@ int sqlite3Fts3GetVarint(const char *, sqlite_int64 *);
int sqlite3Fts3GetVarint32(const char *, int *);
int sqlite3Fts3VarintLen(sqlite3_uint64);
void sqlite3Fts3Dequote(char *);
void sqlite3Fts3DoclistPrev(int,char*,int,char**,sqlite3_int64*,int*,u8*);
char *sqlite3Fts3FindPositions(Fts3Cursor *, Fts3Expr *, sqlite3_int64, int);
int sqlite3Fts3ExprLoadDoclist(Fts3Cursor *, Fts3Expr *);
int sqlite3Fts3ExprLoadFtDoclist(Fts3Cursor *, Fts3Expr *, char **, int *);
int sqlite3Fts3ExprNearTrim(Fts3Expr *, Fts3Expr *, int);
int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *);
/* fts3_tokenizer.c */
const char *sqlite3Fts3NextToken(const char *, int *);
@@ -417,4 +480,27 @@ int sqlite3Fts3InitTerm(sqlite3 *db);
/* fts3_aux.c */
int sqlite3Fts3InitAux(sqlite3 *db);
int sqlite3Fts3TermSegReaderCursor(
Fts3Cursor *pCsr, /* Virtual table cursor handle */
const char *zTerm, /* Term to query for */
int nTerm, /* Size of zTerm in bytes */
int isPrefix, /* True for a prefix search */
Fts3MultiSegReader **ppSegcsr /* OUT: Allocated seg-reader cursor */
);
void sqlite3Fts3EvalPhraseCleanup(Fts3Phrase *);
int sqlite3Fts3EvalStart(Fts3Cursor *, Fts3Expr *, int);
int sqlite3Fts3EvalNext(Fts3Cursor *pCsr);
int sqlite3Fts3MsrIncrStart(
Fts3Table*, Fts3MultiSegReader*, int, const char*, int);
int sqlite3Fts3MsrIncrNext(
Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *);
char *sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol);
int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *);
int sqlite3Fts3DeferredTokenList(Fts3DeferredToken *, char **, int *);
#endif /* _FTSINT_H */
ext/fts3/fts3_aux.c
+3 -2
View File
@@ -28,7 +28,7 @@ struct Fts3auxTable {
struct Fts3auxCursor {
sqlite3_vtab_cursor base; /* Base class used by SQLite core */
Fts3SegReaderCursor csr; /* Must be right after "base" */
Fts3MultiSegReader csr; /* Must be right after "base" */
Fts3SegFilter filter;
char *zStop;
int nStop; /* Byte-length of string zStop */
@@ -96,6 +96,7 @@ static int fts3auxConnectMethod(
p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
p->pFts3Tab->db = db;
p->pFts3Tab->nIndex = 1;
memcpy((char *)p->pFts3Tab->zDb, zDb, nDb);
memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3);
@@ -376,7 +377,7 @@ static int fts3auxFilterMethod(
if( pCsr->zStop==0 ) return SQLITE_NOMEM;
}
rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL,
rc = sqlite3Fts3SegReaderCursor(pFts3, 0, FTS3_SEGCURSOR_ALL,
pCsr->filter.zTerm, pCsr->filter.nTerm, 0, isScan, &pCsr->csr
);
if( rc==SQLITE_OK ){
ext/fts3/fts3_expr.c
+76 -49
View File
@@ -81,12 +81,21 @@ int sqlite3_fts3_enable_parentheses = 0;
#include <string.h>
#include <assert.h>
/*
** isNot:
** This variable is used by function getNextNode(). When getNextNode() is
** called, it sets ParseContext.isNot to true if the 'next node' is a
** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the
** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to
** zero.
*/
typedef struct ParseContext ParseContext;
struct ParseContext {
sqlite3_tokenizer *pTokenizer; /* Tokenizer module */
const char **azCol; /* Array of column names for fts3 table */
int nCol; /* Number of entries in azCol[] */
int iDefaultCol; /* Default column to query */
int isNot; /* True if getNextNode() sees a unary - */
sqlite3_context *pCtx; /* Write error message here */
int nNest; /* Number of nested brackets */
};
@@ -172,7 +181,7 @@ static int getNextToken(
iEnd++;
}
if( !sqlite3_fts3_enable_parentheses && iStart>0 && z[iStart-1]=='-' ){
pRet->pPhrase->isNot = 1;
pParse->isNot = 1;
}
}
nConsumed = iEnd;
@@ -224,36 +233,55 @@ static int getNextString(
char *zTemp = 0;
int nTemp = 0;
const int nSpace = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
int nToken = 0;
/* The final Fts3Expr data structure, including the Fts3Phrase,
** Fts3PhraseToken structures token buffers are all stored as a single
** allocation so that the expression can be freed with a single call to
** sqlite3_free(). Setting this up requires a two pass approach.
**
** The first pass, in the block below, uses a tokenizer cursor to iterate
** through the tokens in the expression. This pass uses fts3ReallocOrFree()
** to assemble data in two dynamic buffers:
**
** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase
** structure, followed by the array of Fts3PhraseToken
** structures. This pass only populates the Fts3PhraseToken array.
**
** Buffer zTemp: Contains copies of all tokens.
**
** The second pass, in the block that begins "if( rc==SQLITE_DONE )" below,
** appends buffer zTemp to buffer p, and fills in the Fts3Expr and Fts3Phrase
** structures.
*/
rc = pModule->xOpen(pTokenizer, zInput, nInput, &pCursor);
if( rc==SQLITE_OK ){
int ii;
pCursor->pTokenizer = pTokenizer;
for(ii=0; rc==SQLITE_OK; ii++){
const char *zToken;
int nToken, iBegin, iEnd, iPos;
rc = pModule->xNext(pCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos);
const char *zByte;
int nByte, iBegin, iEnd, iPos;
rc = pModule->xNext(pCursor, &zByte, &nByte, &iBegin, &iEnd, &iPos);
if( rc==SQLITE_OK ){
int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
p = fts3ReallocOrFree(p, nByte+ii*sizeof(Fts3PhraseToken));
zTemp = fts3ReallocOrFree(zTemp, nTemp + nToken);
if( !p || !zTemp ){
goto no_mem;
}
if( ii==0 ){
memset(p, 0, nByte);
p->pPhrase = (Fts3Phrase *)&p[1];
}
p->pPhrase = (Fts3Phrase *)&p[1];
memset(&p->pPhrase->aToken[ii], 0, sizeof(Fts3PhraseToken));
p->pPhrase->nToken = ii+1;
p->pPhrase->aToken[ii].n = nToken;
memcpy(&zTemp[nTemp], zToken, nToken);
nTemp += nToken;
if( iEnd<nInput && zInput[iEnd]=='*' ){
p->pPhrase->aToken[ii].isPrefix = 1;
}else{
p->pPhrase->aToken[ii].isPrefix = 0;
}
Fts3PhraseToken *pToken;
p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken));
if( !p ) goto no_mem;
zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte);
if( !zTemp ) goto no_mem;
assert( nToken==ii );
pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii];
memset(pToken, 0, sizeof(Fts3PhraseToken));
memcpy(&zTemp[nTemp], zByte, nByte);
nTemp += nByte;
pToken->n = nByte;
pToken->isPrefix = (iEnd<nInput && zInput[iEnd]=='*');
nToken = ii+1;
}
}
@@ -263,28 +291,24 @@ static int getNextString(
if( rc==SQLITE_DONE ){
int jj;
char *zNew = NULL;
int nNew = 0;
int nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase);
nByte += (p?(p->pPhrase->nToken-1):0) * sizeof(Fts3PhraseToken);
p = fts3ReallocOrFree(p, nByte + nTemp);
if( !p ){
goto no_mem;
}
if( zTemp ){
zNew = &(((char *)p)[nByte]);
memcpy(zNew, zTemp, nTemp);
}else{
memset(p, 0, nByte+nTemp);
}
p->pPhrase = (Fts3Phrase *)&p[1];
for(jj=0; jj<p->pPhrase->nToken; jj++){
p->pPhrase->aToken[jj].z = &zNew[nNew];
nNew += p->pPhrase->aToken[jj].n;
}
sqlite3_free(zTemp);
char *zBuf = 0;
p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp);
if( !p ) goto no_mem;
memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p);
p->eType = FTSQUERY_PHRASE;
p->pPhrase = (Fts3Phrase *)&p[1];
p->pPhrase->iColumn = pParse->iDefaultCol;
p->pPhrase->nToken = nToken;
zBuf = (char *)&p->pPhrase->aToken[nToken];
memcpy(zBuf, zTemp, nTemp);
sqlite3_free(zTemp);
for(jj=0; jj<p->pPhrase->nToken; jj++){
p->pPhrase->aToken[jj].z = zBuf;
zBuf += p->pPhrase->aToken[jj].n;
}
rc = SQLITE_OK;
}
@@ -341,6 +365,8 @@ static int getNextNode(
const char *zInput = z;
int nInput = n;
pParse->isNot = 0;
/* Skip over any whitespace before checking for a keyword, an open or
** close bracket, or a quoted string.
*/
@@ -559,7 +585,7 @@ static int fts3ExprParse(
int isPhrase;
if( !sqlite3_fts3_enable_parentheses
&& p->eType==FTSQUERY_PHRASE && p->pPhrase->isNot
&& p->eType==FTSQUERY_PHRASE && pParse->isNot
){
/* Create an implicit NOT operator. */
Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr));
@@ -577,7 +603,6 @@ static int fts3ExprParse(
p = pPrev;
}else{
int eType = p->eType;
assert( eType!=FTSQUERY_PHRASE || !p->pPhrase->isNot );
isPhrase = (eType==FTSQUERY_PHRASE || p->pLeft);
/* The isRequirePhrase variable is set to true if a phrase or
@@ -740,9 +765,11 @@ int sqlite3Fts3ExprParse(
*/
void sqlite3Fts3ExprFree(Fts3Expr *p){
if( p ){
assert( p->eType==FTSQUERY_PHRASE || p->pPhrase==0 );
sqlite3Fts3ExprFree(p->pLeft);
sqlite3Fts3ExprFree(p->pRight);
sqlite3_free(p->aDoclist);
sqlite3Fts3EvalPhraseCleanup(p->pPhrase);
sqlite3_free(p->aMI);
sqlite3_free(p);
}
}
@@ -800,7 +827,7 @@ static char *exprToString(Fts3Expr *pExpr, char *zBuf){
Fts3Phrase *pPhrase = pExpr->pPhrase;
int i;
zBuf = sqlite3_mprintf(
"%zPHRASE %d %d", zBuf, pPhrase->iColumn, pPhrase->isNot);
"%zPHRASE %d 0", zBuf, pPhrase->iColumn);
for(i=0; zBuf && i<pPhrase->nToken; i++){
zBuf = sqlite3_mprintf("%z %.*s%s", zBuf,
pPhrase->aToken[i].n, pPhrase->aToken[i].z,
ext/fts3/fts3_snippet.c
+24 -152
View File
@@ -176,51 +176,6 @@ static int fts3ExprIterate(
return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
}
/*
** The argument to this function is always a phrase node. Its doclist
** (Fts3Expr.aDoclist[]) and the doclists associated with all phrase nodes
** to the left of this one in the query tree have already been loaded.
**
** If this phrase node is part of a series of phrase nodes joined by
** NEAR operators (and is not the left-most of said series), then elements are
** removed from the phrases doclist consistent with the NEAR restriction. If
** required, elements may be removed from the doclists of phrases to the
** left of this one that are part of the same series of NEAR operator
** connected phrases.
**
** If an OOM error occurs, SQLITE_NOMEM is returned. Otherwise, SQLITE_OK.
*/
static int fts3ExprNearTrim(Fts3Expr *pExpr){
int rc = SQLITE_OK;
Fts3Expr *pParent = pExpr->pParent;
assert( pExpr->eType==FTSQUERY_PHRASE );
while( rc==SQLITE_OK
&& pParent
&& pParent->eType==FTSQUERY_NEAR
&& pParent->pRight==pExpr
){
/* This expression (pExpr) is the right-hand-side of a NEAR operator.
** Find the expression to the left of the same operator.
*/
int nNear = pParent->nNear;
Fts3Expr *pLeft = pParent->pLeft;
if( pLeft->eType!=FTSQUERY_PHRASE ){
assert( pLeft->eType==FTSQUERY_NEAR );
assert( pLeft->pRight->eType==FTSQUERY_PHRASE );
pLeft = pLeft->pRight;
}
rc = sqlite3Fts3ExprNearTrim(pLeft, pExpr, nNear);
pExpr = pLeft;
pParent = pExpr->pParent;
}
return rc;
}
/*
** This is an fts3ExprIterate() callback used while loading the doclists
** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
@@ -228,20 +183,13 @@ static int fts3ExprNearTrim(Fts3Expr *pExpr){
*/
static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
int rc = SQLITE_OK;
Fts3Phrase *pPhrase = pExpr->pPhrase;
LoadDoclistCtx *p = (LoadDoclistCtx *)ctx;
UNUSED_PARAMETER(iPhrase);
p->nPhrase++;
p->nToken += pExpr->pPhrase->nToken;
if( pExpr->isLoaded==0 ){
rc = sqlite3Fts3ExprLoadDoclist(p->pCsr, pExpr);
pExpr->isLoaded = 1;
if( rc==SQLITE_OK ){
rc = fts3ExprNearTrim(pExpr);
}
}
p->nToken += pPhrase->nToken;
return rc;
}
@@ -415,7 +363,7 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
pPhrase->nToken = pExpr->pPhrase->nToken;
pCsr = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->pCsr->iPrevId, p->iCol);
pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
if( pCsr ){
int iFirst = 0;
pPhrase->pList = pCsr;
@@ -772,26 +720,6 @@ static int fts3ColumnlistCount(char **ppCollist){
return nEntry;
}
static void fts3LoadColumnlistCounts(char **pp, u32 *aOut, int isGlobal){
char *pCsr = *pp;
while( *pCsr ){
int nHit;
sqlite3_int64 iCol = 0;
if( *pCsr==0x01 ){
pCsr++;
pCsr += sqlite3Fts3GetVarint(pCsr, &iCol);
}
nHit = fts3ColumnlistCount(&pCsr);
assert( nHit>0 );
if( isGlobal ){
aOut[iCol*3+1]++;
}
aOut[iCol*3] += nHit;
}
pCsr++;
*pp = pCsr;
}
/*
** fts3ExprIterate() callback used to collect the "global" matchinfo stats
** for a single query.
@@ -825,48 +753,9 @@ static int fts3ExprGlobalHitsCb(
void *pCtx /* Pointer to MatchInfo structure */
){
MatchInfo *p = (MatchInfo *)pCtx;
Fts3Cursor *pCsr = p->pCursor;
char *pIter;
char *pEnd;
char *pFree = 0;
u32 *aOut = &p->aMatchinfo[3*iPhrase*p->nCol];
assert( pExpr->isLoaded );
assert( pExpr->eType==FTSQUERY_PHRASE );
if( pCsr->pDeferred ){
Fts3Phrase *pPhrase = pExpr->pPhrase;
int ii;
for(ii=0; ii<pPhrase->nToken; ii++){
if( pPhrase->aToken[ii].bFulltext ) break;
}
if( ii<pPhrase->nToken ){
int nFree = 0;
int rc = sqlite3Fts3ExprLoadFtDoclist(pCsr, pExpr, &pFree, &nFree);
if( rc!=SQLITE_OK ) return rc;
pIter = pFree;
pEnd = &pFree[nFree];
}else{
int iCol; /* Column index */
for(iCol=0; iCol<p->nCol; iCol++){
aOut[iCol*3 + 1] = (u32)p->nDoc;
aOut[iCol*3 + 2] = (u32)p->nDoc;
}
return SQLITE_OK;
}
}else{
pIter = pExpr->aDoclist;
pEnd = &pExpr->aDoclist[pExpr->nDoclist];
}
/* Fill in the global hit count matrix row for this phrase. */
while( pIter<pEnd ){
while( *pIter++ & 0x80 ); /* Skip past docid. */
fts3LoadColumnlistCounts(&pIter, &aOut[1], 1);
}
sqlite3_free(pFree);
return SQLITE_OK;
return sqlite3Fts3EvalPhraseStats(
p->pCursor, pExpr, &p->aMatchinfo[3*iPhrase*p->nCol]
);
}
/*
@@ -883,14 +772,13 @@ static int fts3ExprLocalHitsCb(
int iStart = iPhrase * p->nCol * 3;
int i;
for(i=0; i<p->nCol; i++) p->aMatchinfo[iStart+i*3] = 0;
if( pExpr->aDoclist ){
for(i=0; i<p->nCol; i++){
char *pCsr;
pCsr = sqlite3Fts3FindPositions(p->pCursor, pExpr, p->pCursor->iPrevId, -1);
pCsr = sqlite3Fts3EvalPhrasePoslist(p->pCursor, pExpr, i);
if( pCsr ){
fts3LoadColumnlistCounts(&pCsr, &p->aMatchinfo[iStart], 0);
p->aMatchinfo[iStart+i*3] = fts3ColumnlistCount(&pCsr);
}else{
p->aMatchinfo[iStart+i*3] = 0;
}
}
@@ -976,9 +864,8 @@ static int fts3MatchinfoSelectDoctotal(
typedef struct LcsIterator LcsIterator;
struct LcsIterator {
Fts3Expr *pExpr; /* Pointer to phrase expression */
char *pRead; /* Cursor used to iterate through aDoclist */
int iPosOffset; /* Tokens count up to end of this phrase */
int iCol; /* Current column number */
char *pRead; /* Cursor used to iterate through aDoclist */
int iPos; /* Current position */
};
@@ -1009,17 +896,10 @@ static int fts3LcsIteratorAdvance(LcsIterator *pIter){
int rc = 0;
pRead += sqlite3Fts3GetVarint(pRead, &iRead);
if( iRead==0 ){
pIter->iCol = LCS_ITERATOR_FINISHED;
if( iRead==0 || iRead==1 ){
pRead = 0;
rc = 1;
}else{
if( iRead==1 ){
pRead += sqlite3Fts3GetVarint(pRead, &iRead);
pIter->iCol = (int)iRead;
pIter->iPos = pIter->iPosOffset;
pRead += sqlite3Fts3GetVarint(pRead, &iRead);
rc = 1;
}
pIter->iPos += (int)(iRead-2);
}
@@ -1051,42 +931,34 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
if( !aIter ) return SQLITE_NOMEM;
memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase);
(void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
for(i=0; i<pInfo->nPhrase; i++){
LcsIterator *pIter = &aIter[i];
nToken -= pIter->pExpr->pPhrase->nToken;
pIter->iPosOffset = nToken;
pIter->pRead = sqlite3Fts3FindPositions(pCsr,pIter->pExpr,pCsr->iPrevId,-1);
if( pIter->pRead ){
pIter->iPos = pIter->iPosOffset;
fts3LcsIteratorAdvance(&aIter[i]);
}else{
pIter->iCol = LCS_ITERATOR_FINISHED;
}
}
for(iCol=0; iCol<pInfo->nCol; iCol++){
int nLcs = 0; /* LCS value for this column */
int nLive = 0; /* Number of iterators in aIter not at EOF */
/* Loop through the iterators in aIter[]. Set nLive to the number of
** iterators that point to a position-list corresponding to column iCol.
*/
for(i=0; i<pInfo->nPhrase; i++){
assert( aIter[i].iCol>=iCol );
if( aIter[i].iCol==iCol ) nLive++;
LcsIterator *pIt = &aIter[i];
pIt->pRead = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol);
if( pIt->pRead ){
pIt->iPos = pIt->iPosOffset;
fts3LcsIteratorAdvance(&aIter[i]);
nLive++;
}
}
/* The following loop runs until all iterators in aIter[] have finished
** iterating through positions in column iCol. Exactly one of the
** iterators is advanced each time the body of the loop is run.
*/
while( nLive>0 ){
LcsIterator *pAdv = 0; /* The iterator to advance by one position */
int nThisLcs = 0; /* LCS for the current iterator positions */
for(i=0; i<pInfo->nPhrase; i++){
LcsIterator *pIter = &aIter[i];
if( iCol!=pIter->iCol ){
if( pIter->pRead==0 ){
/* This iterator is already at EOF for this column. */
nThisLcs = 0;
}else{
@@ -1426,7 +1298,7 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){
int iPos = 0; /* First position in position-list */
UNUSED_PARAMETER(iPhrase);
pList = sqlite3Fts3FindPositions(p->pCsr, pExpr, p->iDocid, p->iCol);
pList = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol);
nTerm = pExpr->pPhrase->nToken;
if( pList ){
fts3GetDeltaPosition(&pList, &iPos);
ext/fts3/fts3_term.c
+13 -5
View File
@@ -27,12 +27,13 @@ typedef struct Fts3termCursor Fts3termCursor;
struct Fts3termTable {
sqlite3_vtab base; /* Base class used by SQLite core */
int iIndex; /* Index for Fts3Table.aIndex[] */
Fts3Table *pFts3Tab;
};
struct Fts3termCursor {
sqlite3_vtab_cursor base; /* Base class used by SQLite core */
Fts3SegReaderCursor csr; /* Must be right after "base" */
Fts3MultiSegReader csr; /* Must be right after "base" */
Fts3SegFilter filter;
int isEof; /* True if cursor is at EOF */
@@ -56,7 +57,7 @@ struct Fts3termCursor {
*/
static int fts3termConnectMethod(
sqlite3 *db, /* Database connection */
void *pUnused, /* Unused */
void *pCtx, /* Non-zero for an fts4prefix table */
int argc, /* Number of elements in argv array */
const char * const *argv, /* xCreate/xConnect argument array */
sqlite3_vtab **ppVtab, /* OUT: New sqlite3_vtab object */
@@ -69,8 +70,12 @@ static int fts3termConnectMethod(
int nByte; /* Bytes of space to allocate here */
int rc; /* value returned by declare_vtab() */
Fts3termTable *p; /* Virtual table object to return */
int iIndex = 0;
UNUSED_PARAMETER(pUnused);
if( argc==5 ){
iIndex = atoi(argv[4]);
argc--;
}
/* The user should specify a single argument - the name of an fts3 table. */
if( argc!=4 ){
@@ -97,6 +102,8 @@ static int fts3termConnectMethod(
p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
p->pFts3Tab->db = db;
p->pFts3Tab->nIndex = iIndex+1;
p->iIndex = iIndex;
memcpy((char *)p->pFts3Tab->zDb, zDb, nDb);
memcpy((char *)p->pFts3Tab->zName, zFts3, nFts3);
@@ -244,7 +251,8 @@ static int fts3termFilterMethod(
sqlite3_value **apVal /* Arguments for the indexing scheme */
){
Fts3termCursor *pCsr = (Fts3termCursor *)pCursor;
Fts3Table *pFts3 = ((Fts3termTable *)pCursor->pVtab)->pFts3Tab;
Fts3termTable *p = (Fts3termTable *)pCursor->pVtab;
Fts3Table *pFts3 = p->pFts3Tab;
int rc;
UNUSED_PARAMETER(nVal);
@@ -262,7 +270,7 @@ static int fts3termFilterMethod(
pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY;
pCsr->filter.flags |= FTS3_SEGMENT_SCAN;
rc = sqlite3Fts3SegReaderCursor(pFts3, FTS3_SEGCURSOR_ALL,
rc = sqlite3Fts3SegReaderCursor(pFts3, p->iIndex, FTS3_SEGCURSOR_ALL,
pCsr->filter.zTerm, pCsr->filter.nTerm, 0, 1, &pCsr->csr
);
if( rc==SQLITE_OK ){
ext/fts3/fts3_test.c
+249
View File
@@ -0,0 +1,249 @@
/*
** 2011 Jun 13
**
** The author disclaims copyright to this source code. In place of
** a legal notice, here is a blessing:
**
** May you do good and not evil.
** May you find forgiveness for yourself and forgive others.
** May you share freely, never taking more than you give.
**
******************************************************************************
**
** This file is not part of the production FTS code. It is only used for
** testing. It contains a Tcl command that can be used to test if a document
** matches an FTS NEAR expression.
*/
#include <tcl.h>
#include <string.h>
#include <assert.h>
#define NM_MAX_TOKEN 12
typedef struct NearPhrase NearPhrase;
typedef struct NearDocument NearDocument;
typedef struct NearToken NearToken;
struct NearDocument {
int nToken; /* Length of token in bytes */
NearToken *aToken; /* Token array */
};
struct NearToken {
int n; /* Length of token in bytes */
const char *z; /* Pointer to token string */
};
struct NearPhrase {
int nNear; /* Preceding NEAR value */
int nToken; /* Number of tokens in this phrase */
NearToken aToken[NM_MAX_TOKEN]; /* Array of tokens in this phrase */
};
static int nm_phrase_match(
NearPhrase *p,
NearToken *aToken
){
int ii;
for(ii=0; ii<p->nToken; ii++){
NearToken *pToken = &p->aToken[ii];
if( pToken->n>0 && pToken->z[pToken->n-1]=='*' ){
if( aToken[ii].n<(pToken->n-1) ) return 0;
if( memcmp(aToken[ii].z, pToken->z, pToken->n-1) ) return 0;
}else{
if( aToken[ii].n!=pToken->n ) return 0;
if( memcmp(aToken[ii].z, pToken->z, pToken->n) ) return 0;
}
}
return 1;
}
static int nm_near_chain(
int iDir, /* Direction to iterate through aPhrase[] */
NearDocument *pDoc, /* Document to match against */
int iPos, /* Position at which iPhrase was found */
int nPhrase, /* Size of phrase array */
NearPhrase *aPhrase, /* Phrase array */
int iPhrase /* Index of phrase found */
){
int iStart;
int iStop;
int ii;
int nNear;
int iPhrase2;
NearPhrase *p;
NearPhrase *pPrev;
assert( iDir==1 || iDir==-1 );
if( iDir==1 ){
if( (iPhrase+1)==nPhrase ) return 1;
nNear = aPhrase[iPhrase+1].nNear;
}else{
if( iPhrase==0 ) return 1;
nNear = aPhrase[iPhrase].nNear;
}
pPrev = &aPhrase[iPhrase];
iPhrase2 = iPhrase+iDir;
p = &aPhrase[iPhrase2];
iStart = iPos - nNear - p->nToken;
iStop = iPos + nNear + pPrev->nToken;
if( iStart<0 ) iStart = 0;
if( iStop > pDoc->nToken - p->nToken ) iStop = pDoc->nToken - p->nToken;
for(ii=iStart; ii<=iStop; ii++){
if( nm_phrase_match(p, &pDoc->aToken[ii]) ){
if( nm_near_chain(iDir, pDoc, ii, nPhrase, aPhrase, iPhrase2) ) return 1;
}
}
return 0;
}
static int nm_match_count(
NearDocument *pDoc, /* Document to match against */
int nPhrase, /* Size of phrase array */
NearPhrase *aPhrase, /* Phrase array */
int iPhrase /* Index of phrase to count matches for */
){
int nOcc = 0;
int ii;
NearPhrase *p = &aPhrase[iPhrase];
for(ii=0; ii<(pDoc->nToken + 1 - p->nToken); ii++){
if( nm_phrase_match(p, &pDoc->aToken[ii]) ){
/* Test forward NEAR chain (i>iPhrase) */
if( 0==nm_near_chain(1, pDoc, ii, nPhrase, aPhrase, iPhrase) ) continue;
/* Test reverse NEAR chain (i<iPhrase) */
if( 0==nm_near_chain(-1, pDoc, ii, nPhrase, aPhrase, iPhrase) ) continue;
/* This is a real match. Increment the counter. */
nOcc++;
}
}
return nOcc;
}
/*
** Tclcmd: fts3_near_match DOCUMENT EXPR ?OPTIONS?
*/
static int fts3_near_match_cmd(
ClientData clientData,
Tcl_Interp *interp,
int objc,
Tcl_Obj *CONST objv[]
){
int nTotal = 0;
int rc;
int ii;
int nPhrase;
NearPhrase *aPhrase = 0;
NearDocument doc = {0, 0};
Tcl_Obj **apDocToken;
Tcl_Obj *pRet;
Tcl_Obj *pPhrasecount = 0;
Tcl_Obj **apExprToken;
int nExprToken;
/* Must have 3 or more arguments. */
if( objc<3 || (objc%2)==0 ){
Tcl_WrongNumArgs(interp, 1, objv, "DOCUMENT EXPR ?OPTION VALUE?...");
rc = TCL_ERROR;
goto near_match_out;
}
for(ii=3; ii<objc; ii+=2){
enum NM_enum { NM_PHRASECOUNTS };
struct TestnmSubcmd {
char *zName;
enum NM_enum eOpt;
} aOpt[] = {
{ "-phrasecountvar", NM_PHRASECOUNTS },
{ 0, 0 }
};
int iOpt;
if( Tcl_GetIndexFromObjStruct(
interp, objv[ii], aOpt, sizeof(aOpt[0]), "option", 0, &iOpt)
){
return TCL_ERROR;
}
switch( aOpt[iOpt].eOpt ){
case NM_PHRASECOUNTS:
pPhrasecount = objv[ii+1];
break;
}
}
rc = Tcl_ListObjGetElements(interp, objv[1], &doc.nToken, &apDocToken);
if( rc!=TCL_OK ) goto near_match_out;
doc.aToken = (NearToken *)ckalloc(doc.nToken*sizeof(NearToken));
for(ii=0; ii<doc.nToken; ii++){
doc.aToken[ii].z = Tcl_GetStringFromObj(apDocToken[ii], &doc.aToken[ii].n);
}
rc = Tcl_ListObjGetElements(interp, objv[2], &nExprToken, &apExprToken);
if( rc!=TCL_OK ) goto near_match_out;
nPhrase = (nExprToken + 1) / 2;
aPhrase = (NearPhrase *)ckalloc(nPhrase * sizeof(NearPhrase));
memset(aPhrase, 0, nPhrase * sizeof(NearPhrase));
for(ii=0; ii<nPhrase; ii++){
Tcl_Obj *pPhrase = apExprToken[ii*2];
Tcl_Obj **apToken;
int nToken;
int jj;
rc = Tcl_ListObjGetElements(interp, pPhrase, &nToken, &apToken);
if( rc!=TCL_OK ) goto near_match_out;
if( nToken>NM_MAX_TOKEN ){
Tcl_AppendResult(interp, "Too many tokens in phrase", 0);
rc = TCL_ERROR;
goto near_match_out;
}
for(jj=0; jj<nToken; jj++){
NearToken *pT = &aPhrase[ii].aToken[jj];
pT->z = Tcl_GetStringFromObj(apToken[jj], &pT->n);
}
aPhrase[ii].nToken = nToken;
}
for(ii=1; ii<nPhrase; ii++){
Tcl_Obj *pNear = apExprToken[2*ii-1];
int nNear;
rc = Tcl_GetIntFromObj(interp, pNear, &nNear);
if( rc!=TCL_OK ) goto near_match_out;
aPhrase[ii].nNear = nNear;
}
pRet = Tcl_NewObj();
Tcl_IncrRefCount(pRet);
for(ii=0; ii<nPhrase; ii++){
int nOcc = nm_match_count(&doc, nPhrase, aPhrase, ii);
Tcl_ListObjAppendElement(interp, pRet, Tcl_NewIntObj(nOcc));
nTotal += nOcc;
}
if( pPhrasecount ){
Tcl_ObjSetVar2(interp, pPhrasecount, 0, pRet, 0);
}
Tcl_DecrRefCount(pRet);
Tcl_SetObjResult(interp, Tcl_NewBooleanObj(nTotal>0));
near_match_out:
ckfree((char *)aPhrase);
ckfree((char *)doc.aToken);
return rc;
}
int Sqlitetestfts3_Init(Tcl_Interp *interp){
Tcl_CreateObjCommand(interp, "fts3_near_match", fts3_near_match_cmd, 0, 0);
return TCL_OK;
}
ext/fts3/fts3_write.c
+648 -268
View File
File diff suppressed because it is too large Load Diff
main.mk
+1
View File
@@ -220,6 +220,7 @@ SRC += \
# Source code to the test files.
#
TESTSRC = \
$(TOP)/ext/fts3/fts3_test.c \
$(TOP)/src/test1.c \
$(TOP)/src/test2.c \
$(TOP)/src/test3.c \
manifest
+28 -25
View File
@@ -1,7 +1,7 @@
C Use\sonly\sunsigned\svalues\sin\sthe\simplementatin\sof\sLIKE\sand\sGLOB\sso\sthat\nvalues\swon't\soverflow\sto\snegative\swhen\sdealing\swith\smalformed\sUTF8.
D 2011-06-13T12:19:21.072
C Merge\sfts3-prefix-search\sbranch\swith\strunk.
D 2011-06-14T11:50:09.808
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in 11dcc00a8d0e5202def00e81732784fb0cc4fe1d
F Makefile.in c1d7a7f4fd8da6b1815032efca950e3d5125407e
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
F Makefile.vxworks c85ec1d8597fe2f7bc225af12ac1666e21379151
F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6
@@ -61,21 +61,22 @@ F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
F ext/fts3/fts3.c b3a10a1a320aaeb56a1dd6710bf09eb5c2370839
F ext/fts3/fts3.c ae3ae91d204d177fc37a22c7bc5a3a9df70e65a0
F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
F ext/fts3/fts3Int.h c8c0011c5e5b3a7703376ea6cd7deb91cfb96a06
F ext/fts3/fts3_aux.c 97c960b1b0d371c08eae6b8565dfac619eb9d979
F ext/fts3/fts3_expr.c 5f49e0deaf723724b08100bb3ff40aab02ad0c93
F ext/fts3/fts3Int.h a999cfbf605efec293a88519f74192f5204c84d6
F ext/fts3/fts3_aux.c baed9dab7fb4604ae8cafdb2d7700abe93beffbe
F ext/fts3/fts3_expr.c b95f0d76bcf4507c73a838f3178c4ed8c42dc2bb
F ext/fts3/fts3_hash.c 3c8f6387a4a7f5305588b203fa7c887d753e1f1c
F ext/fts3/fts3_hash.h 8331fb2206c609f9fc4c4735b9ab5ad6137c88ec
F ext/fts3/fts3_icu.c ac494aed69835008185299315403044664bda295
F ext/fts3/fts3_porter.c d61cfd81fb0fd8fbcb25adcaee0ba671aefaa5c2
F ext/fts3/fts3_snippet.c 92b40397b28422c35c4127492d7ac6da34d1966a
F ext/fts3/fts3_term.c f115f5a5f4298303d3b22fc6c524b8d565c7b950
F ext/fts3/fts3_snippet.c 82e2c1e420c871c02f6e85ea438570118d7105c8
F ext/fts3/fts3_term.c 6c7f33ab732a2a0f281898685650e3a492e1e2f1
F ext/fts3/fts3_test.c 9376cc865447e63c671f0f9ffd1a2c9a29678230
F ext/fts3/fts3_tokenizer.c 055f3dc7369585350b28db1ee0f3b214dca6724d
F ext/fts3/fts3_tokenizer.h 13ffd9fcb397fec32a05ef5cd9e0fa659bf3dbd3
F ext/fts3/fts3_tokenizer1.c 6e5cbaa588924ac578263a598e4fb9f5c9bb179d
F ext/fts3/fts3_write.c b50181e5ecf484c2f56e98d651424e4b69f96c89
F ext/fts3/fts3_write.c bc24cec303d86aeb4b40fcbdf9f252f93ef78fc7
F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
@@ -102,7 +103,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de
F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
F main.mk 6111163d4e9720e4212ef288e967b4aa2c2ce379
F main.mk 6de0d92dcae3d399a6bafaeb23b57f6ef0f41955
F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a
F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f
F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac
@@ -185,7 +186,7 @@ F src/sqliteInt.h 7b7ec2394b94fc4516930cd9dae37af0f9312215
F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf
F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
F src/tclsqlite.c 501c9a200fd998a268be475be5858febc90b725b
F src/tclsqlite.c c83f5b4a15ed92cb35aa04320aa4a30b95071b2f
F src/test1.c efca486a25fb894988e7a82e84579a4e57388a02
F src/test2.c 80d323d11e909cf0eb1b6fbb4ac22276483bcf31
F src/test3.c 124ff9735fb6bb7d41de180d6bac90e7b1509432
@@ -454,6 +455,7 @@ F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8
F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18
F test/fts3ao.test b83f99f70e9eec85f27d75801a974b3f820e01f9
F test/fts3atoken.test 402ef2f7c2fb4b3d4fa0587df6441c1447e799b3
F test/fts3auto.test 2f86f2a0e8ffa26d81d570897e6cc1c2262256d5
F test/fts3aux1.test 0b02743955d56fc0d4d66236a26177bd1b726de0
F test/fts3b.test e93bbb653e52afde110ad53bbd793f14fe7a8984
F test/fts3c.test fc723a9cf10b397fdfc2b32e73c53c8b1ec02958
@@ -463,21 +465,22 @@ F test/fts3corrupt.test 7b0f91780ca36118d73324ec803187208ad33b32
F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba
F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7
F test/fts3d.test 95fb3c862cbc4297c93fceb9a635543744e9ef52
F test/fts3defer.test d6cb0db9b5997ecf863d96ff419f83f8f2c87f4f
F test/fts3defer2.test 288bef6de15557319b8c12d476ebdc83688ef96c
F test/fts3defer.test 7c8a38d5f617d7b52ae1c43ed73c536e7e895a35
F test/fts3defer2.test 35867d33ba6db03f6c73bd6f5fc333ae14f68c81
F test/fts3e.test 1f6c6ac9cc8b772ca256e6b22aaeed50c9350851
F test/fts3expr.test 5e745b2b6348499d9ef8d59015de3182072c564c
F test/fts3expr2.test 18da930352e5693eaa163a3eacf96233b7290d1a
F test/fts3fault.test f83e556465bb69dc8bc676339eca408dce4ca246
F test/fts3fault2.test dc96203af6ba31ce20163fc35460e1556e8edf4d
F test/fts3malloc.test 9c8cc3f885bb4dfc66d0460c52f68f45e4710d1b
F test/fts3matchinfo.test cc0b009edbbf575283d5fdb53271179e0d8019ba
F test/fts3matchinfo.test 08a82d18cc08abb28aec41d412b4c2ef25ba6a5f
F test/fts3near.test 2e318ee434d32babd27c167142e2b94ddbab4844
F test/fts3prefix.test 36246609111ec1683f7ea5ed27666ce2cefb5676
F test/fts3query.test ef79d31fdb355d094baec1c1b24b60439a1fb8a2
F test/fts3rnd.test 2b1a579be557ab8ac54a51b39caa4aa8043cc4ad
F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0
F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2
F test/fts3snippet.test a12f22a3ba4dd59751a57c79b031d07ab5f51ddd
F test/fts3sort.test e6f24e9cffc46484bcc9fe63d3c2ce41afcaa6c9
F test/fts3sort.test 63d52c1812904b751f9e1ff487472e44833f5402
F test/fts4aa.test eadf85621c0a113d4c7ad3ccbf8441130e007b8f
F test/func.test 6c5ce11e3a0021ca3c0649234e2d4454c89110ca
F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f
@@ -488,7 +491,7 @@ F test/fuzz3.test aec64345184d1662bd30e6a17851ff659d596dc5
F test/fuzz_common.tcl a87dfbb88c2a6b08a38e9a070dabd129e617b45b
F test/fuzz_malloc.test dd7001ac86d09c154a7dff064f4739c60e2b312c
F test/fuzzer1.test 3105b5a89a6cb0d475f0877debec942fe4143462
F test/hook.test f04c3412463f8ec117c1c704c74ca0f627ce733a
F test/hook.test f2277c309e4ee8067d95d6b9b315568e9d5329b2
F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4
F test/in.test 19b642bb134308980a92249750ea4ce3f6c75c2d
F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75
@@ -611,7 +614,7 @@ F test/pageropt.test 8146bf448cf09e87bb1867c2217b921fb5857806
F test/pagesize.test 76aa9f23ecb0741a4ed9d2e16c5fa82671f28efb
F test/pcache.test 065aa286e722ab24f2e51792c1f093bf60656b16
F test/pcache2.test 0d85f2ab6963aee28c671d4c71bec038c00a1d16
F test/permutations.test 5b2a4cb756ffb2407cb4743163668d1d769febb6
F test/permutations.test 3a0d43a3e92f74f49be42ddc580e9b1ec361ba0e
F test/pragma.test fdfc09067ea104a0c247a1a79d8093b56656f850
F test/pragma2.test 5364893491b9231dd170e3459bfc2e2342658b47
F test/printf.test 05970cde31b1a9f54bd75af60597be75a5c54fea
@@ -691,7 +694,7 @@ F test/tclsqlite.test 8c154101e704170c2be10f137a5499ac2c6da8d3
F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c
F test/temptable.test f42121a0d29a62f00f93274464164177ab1cc24a
F test/temptrigger.test b0273db072ce5f37cf19140ceb1f0d524bbe9f05
F test/tester.tcl a791ee74cb6b8f8613079ccc018bf2c8b952a26c
F test/tester.tcl 76222602e59047c6ef119473c7a2ea7c6ee73d09
F test/thread001.test a3e6a7254d1cb057836cb3145b60c10bf5b7e60f
F test/thread002.test afd20095e6e845b405df4f2c920cb93301ca69db
F test/thread003.test b824d4f52b870ae39fc5bae4d8070eca73085dca
@@ -818,7 +821,7 @@ F test/tkt3997.test a335fa41ca3985660a139df7b734a26ef53284bd
F test/tkt4018.test 7c2c9ba4df489c676a0a7a0e809a1fb9b2185bd1
F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7
F test/trace.test 4b36a41a3e9c7842151af6da5998f5080cdad9e5
F test/trace2.test 0ce11265c83333d8f5beeca19e71ed93a88d386c
F test/trace2.test 962175290996d5f06dc4402ca218bbfc7df4cb20
F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6
F test/trans2.test d5337e61de45e66b1fcbf9db833fa8c82e624b22
F test/trans3.test d728abaa318ca364dc370e06576aa7e5fbed7e97
@@ -942,7 +945,7 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c
F tool/symbols.sh bc2a3709940d47c8ac8e0a1fdf17ec801f015a00
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
F tool/warnings.sh 347d974d143cf132f953b565fbc03026f19fcb4d
P 2b3d9996a829c62fbaf7c92d50e44636340b07c6
R 901f548a763eaba41771c9603ccfd6ef
U drh
Z 700a00fc01c336d2e1728d856de33900
P 77f01578bb565d1bc884b374b68bae10ce34a084 aefd46dfae7e06fbaf4f2b9a86a7f2ac6927331e
R 84da75d7a3a06308ee488ec7371657d8
U dan
Z 26ae4ebe97429ca9da8c05064a989555
manifest.uuid
+1 -1
View File
@@ -1 +1 @@
77f01578bb565d1bc884b374b68bae10ce34a084
b1f9c1e0ac51cedfb05ac073a603343f6df865b5
src/tclsqlite.c
+8
View File
@@ -3585,6 +3585,10 @@ static void init_all(Tcl_Interp *interp){
extern int Sqlitetestfuzzer_Init(Tcl_Interp*);
extern int Sqlitetestwholenumber_Init(Tcl_Interp*);
#ifdef SQLITE_ENABLE_FTS3
extern int Sqlitetestfts3_Init(Tcl_Interp *interp);
#endif
#ifdef SQLITE_ENABLE_ZIPVFS
extern int Zipvfs_Init(Tcl_Interp*);
Zipvfs_Init(interp);
@@ -3625,6 +3629,10 @@ static void init_all(Tcl_Interp *interp){
Sqlitetestfuzzer_Init(interp);
Sqlitetestwholenumber_Init(interp);
#ifdef SQLITE_ENABLE_FTS3
Sqlitetestfts3_Init(interp);
#endif
Tcl_CreateObjCommand(interp,"load_testfixture_extensions",init_all_cmd,0,0);
#ifdef SQLITE_SSE
test/fts3auto.test
+535
View File
@@ -0,0 +1,535 @@
# 2011 June 10
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
# If this build does not include FTS3, skip the tests in this file.
#
ifcapable !fts3 { finish_test ; return }
source $testdir/fts3_common.tcl
source $testdir/malloc_common.tcl
set testprefix fts3auto
set sfep $sqlite_fts3_enable_parentheses
set sqlite_fts3_enable_parentheses 1
#--------------------------------------------------------------------------
# Start of Tcl infrastructure used by tests. The entry points are:
#
# do_fts3query_test
# fts3_make_deferrable
# fts3_zero_long_segments
#
#
# do_fts3query_test TESTNAME ?OPTIONS? TABLE MATCHEXPR
#
# This proc runs several test cases on FTS3/4 table $TABLE using match
# expression $MATCHEXPR. All documents in $TABLE must be formatted so that
# they can be "tokenized" using the Tcl list commands (llength, lindex etc.).
# The name and column names used by $TABLE must not require any quoting or
# escaping when used in SQL statements.
#
# $MATCHINFO may be any expression accepted by the FTS4 MATCH operator,
# except that the "<column-name>:token" syntax is not supported. Tcl list
# commands are used to tokenize the expression. Any parenthesis must appear
# either as separate list elements, or as the first (for opening) or last
# (for closing) character of a list element. i.e. the expression "(a OR b)c"
# will not be parsed correctly, but "( a OR b) c" will.
#
# Available OPTIONS are:
#
# -deferred TOKENLIST
#
# If the "deferred" option is supplied, it is passed a list of tokens that
# are deferred by FTS and result in the relevant matchinfo() stats being an
# approximation.
#
set sqlite_fts3_enable_parentheses 1
proc do_fts3query_test {tn args} {
set nArg [llength $args]
if {$nArg < 2 || ($nArg % 2)} {
set cmd do_fts3query_test
error "wrong # args: should be \"$cmd ?-deferred LIST? TABLE MATCHEXPR\""
}
set tbl [lindex $args [expr $nArg-2]]
set match [lindex $args [expr $nArg-1]]
set deferred [list]
foreach {k v} [lrange $args 0 [expr $nArg-3]] {
switch -- $k {
-deferred {
set deferred $v
}
default {
error "bad option \"$k\": must be -deferred"
}
}
}
get_near_results $tbl $match $deferred aMatchinfo
set matchinfo_asc [list]
foreach docid [lsort -integer -incr [array names aMatchinfo]] {
lappend matchinfo_asc $docid $aMatchinfo($docid)
}
set matchinfo_desc [list]
foreach docid [lsort -integer -decr [array names aMatchinfo]] {
lappend matchinfo_desc $docid $aMatchinfo($docid)
}
set title "(\"$match\" -> [llength [array names aMatchinfo]] rows)"
do_execsql_test $tn$title.1 "
SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid ASC
" [lsort -integer -incr [array names aMatchinfo]]
do_execsql_test $tn$title.2 "
SELECT docid FROM $tbl WHERE $tbl MATCH '$match' ORDER BY docid DESC
" [lsort -integer -decr [array names aMatchinfo]]
do_execsql_test $tn$title.3 "
SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl
WHERE $tbl MATCH '$match' ORDER BY docid DESC
" $matchinfo_desc
do_execsql_test $tn$title.4 "
SELECT docid, mit(matchinfo($tbl, 'x')) FROM $tbl
WHERE $tbl MATCH '$match' ORDER BY docid ASC
" $matchinfo_asc
}
# fts3_make_deferrable TABLE TOKEN
#
proc fts3_make_deferrable {tbl token} {
set stmt [sqlite3_prepare db "SELECT * FROM $tbl" -1 dummy]
set name [sqlite3_column_name $stmt 0]
sqlite3_finalize $stmt
set nRow [db one "SELECT count(*) FROM $tbl"]
set pgsz [db one "PRAGMA page_size"]
execsql BEGIN
for {set i 0} {$i < ($nRow * $pgsz * 1.2)/100} {incr i} {
set doc [string repeat "$token " 100]
execsql "INSERT INTO $tbl ($name) VALUES(\$doc)"
}
execsql "INSERT INTO $tbl ($name) VALUES('aaaaaaa ${token}aaaaa')"
execsql COMMIT
return [expr $nRow*$pgsz]
}
# fts3_zero_long_segments TABLE ?LIMIT?
#
proc fts3_zero_long_segments {tbl limit} {
execsql "
UPDATE ${tbl}_segments
SET block = zeroblob(length(block))
WHERE length(block)>$limit
"
return [db changes]
}
proc mit {blob} {
set scan(littleEndian) i*
set scan(bigEndian) I*
binary scan $blob $scan($::tcl_platform(byteOrder)) r
return $r
}
db func mit mit
proc fix_near_expr {expr} {
set out [list]
lappend out [lindex $expr 0]
foreach {a b} [lrange $expr 1 end] {
if {[string match -nocase near $a]} { set a 10 }
if {[string match -nocase near/* $a]} { set a [string range $a 5 end] }
lappend out $a
lappend out $b
}
return $out
}
proc get_single_near_results {tbl expr deferred arrayvar nullvar} {
upvar $arrayvar aMatchinfo
upvar $nullvar nullentry
catch {array unset aMatchinfo}
set expr [fix_near_expr $expr]
# Calculate the expected results using [fts3_near_match]. The following
# loop populates the "hits" and "counts" arrays as follows:
#
# 1. For each document in the table that matches the NEAR expression,
# hits($docid) is set to 1. The set of docids that match the expression
# can therefore be found using [array names hits].
#
# 2. For each column of each document in the table, counts($docid,$iCol)
# is set to the -phrasecountvar output.
#
set res [list]
catch { array unset hits }
db eval "SELECT docid, * FROM $tbl" d {
set iCol 0
foreach col [lrange $d(*) 1 end] {
set docid $d(docid)
set hit [fts3_near_match $d($col) $expr -p counts($docid,$iCol)]
if {$hit} { set hits($docid) 1 }
incr iCol
}
}
set nPhrase [expr ([llength $expr]+1)/2]
set nCol $iCol
# This block populates the nHit and nDoc arrays. For each phrase/column
# in the query/table, array elements are set as follows:
#
# nHit($iPhrase,$iCol) - Total number of hits for phrase $iPhrase in
# column $iCol.
#
# nDoc($iPhrase,$iCol) - Number of documents with at least one hit for
# phrase $iPhrase in column $iCol.
#
for {set iPhrase 0} {$iPhrase < $nPhrase} {incr iPhrase} {
for {set iCol 0} {$iCol < $nCol} {incr iCol} {
set nHit($iPhrase,$iCol) 0
set nDoc($iPhrase,$iCol) 0
}
}
foreach key [array names counts] {
set iCol [lindex [split $key ,] 1]
set iPhrase 0
foreach c $counts($key) {
if {$c>0} { incr nDoc($iPhrase,$iCol) 1 }
incr nHit($iPhrase,$iCol) $c
incr iPhrase
}
}
if {[llength $deferred] && [llength $expr]==1} {
set phrase [lindex $expr 0]
set rewritten [list]
set partial 0
foreach tok $phrase {
if {[lsearch $deferred $tok]>=0} {
lappend rewritten *
} else {
lappend rewritten $tok
set partial 1
}
}
if {$partial==0} {
set tblsize [db one "SELECT count(*) FROM $tbl"]
for {set iCol 0} {$iCol < $nCol} {incr iCol} {
set nHit(0,$iCol) $tblsize
set nDoc(0,$iCol) $tblsize
}
} elseif {$rewritten != $phrase} {
while {[lindex $rewritten end] == "*"} {
set rewritten [lrange $rewritten 0 end-1]
}
while {[lindex $rewritten 0] == "*"} {
set rewritten [lrange $rewritten 1 end]
}
get_single_near_results $tbl [list $rewritten] {} aRewrite nullentry
foreach docid [array names hits] {
set aMatchinfo($docid) $aRewrite($docid)
}
return
}
}
# Set up the aMatchinfo array. For each document, set aMatchinfo($docid) to
# contain the output of matchinfo('x') for the document.
#
foreach docid [array names hits] {
set mi [list]
for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
for {set iCol 0} {$iCol<$nCol} {incr iCol} {
lappend mi [lindex $counts($docid,$iCol) $iPhrase]
lappend mi $nHit($iPhrase,$iCol)
lappend mi $nDoc($iPhrase,$iCol)
}
}
set aMatchinfo($docid) $mi
}
# Set up the nullentry output.
#
set nullentry [list]
for {set iPhrase 0} {$iPhrase<$nPhrase} {incr iPhrase} {
for {set iCol 0} {$iCol<$nCol} {incr iCol} {
lappend nullentry 0 $nHit($iPhrase,$iCol) $nDoc($iPhrase,$iCol)
}
}
}
proc matching_brackets {expr} {
if {[string range $expr 0 0]!="(" || [string range $expr end end] !=")"} {
return 0
}
set iBracket 1
set nExpr [string length $expr]
for {set i 1} {$iBracket && $i < $nExpr} {incr i} {
set c [string range $expr $i $i]
if {$c == "("} {incr iBracket}
if {$c == ")"} {incr iBracket -1}
}
return [expr ($iBracket==0 && $i==$nExpr)]
}
proc get_near_results {tbl expr deferred arrayvar {nullvar ""}} {
upvar $arrayvar aMatchinfo
if {$nullvar != ""} { upvar $nullvar nullentry }
set expr [string trim $expr]
while { [matching_brackets $expr] } {
set expr [string trim [string range $expr 1 end-1]]
}
set prec(NOT) 1
set prec(AND) 2
set prec(OR) 3
set currentprec 0
set iBracket 0
set expr_length [llength $expr]
for {set i 0} {$i < $expr_length} {incr i} {
set op [lindex $expr $i]
if {$iBracket==0 && [info exists prec($op)] && $prec($op)>=$currentprec } {
set opidx $i
set currentprec $prec($op)
} else {
for {set j 0} {$j < [string length $op]} {incr j} {
set c [string range $op $j $j]
if {$c == "("} { incr iBracket +1 }
if {$c == ")"} { incr iBracket -1 }
}
}
}
if {$iBracket!=0} { error "mismatched brackets in: $expr" }
if {[info exists opidx]==0} {
get_single_near_results $tbl $expr $deferred aMatchinfo nullentry
} else {
set eLeft [lrange $expr 0 [expr $opidx-1]]
set eRight [lrange $expr [expr $opidx+1] end]
get_near_results $tbl $eLeft $deferred aLeft nullleft
get_near_results $tbl $eRight $deferred aRight nullright
switch -- [lindex $expr $opidx] {
"NOT" {
foreach hit [array names aLeft] {
if {0==[info exists aRight($hit)]} {
set aMatchinfo($hit) $aLeft($hit)
}
}
set nullentry $nullleft
}
"AND" {
foreach hit [array names aLeft] {
if {[info exists aRight($hit)]} {
set aMatchinfo($hit) [concat $aLeft($hit) $aRight($hit)]
}
}
set nullentry [concat $nullleft $nullright]
}
"OR" {
foreach hit [array names aLeft] {
if {[info exists aRight($hit)]} {
set aMatchinfo($hit) [concat $aLeft($hit) $aRight($hit)]
unset aRight($hit)
} else {
set aMatchinfo($hit) [concat $aLeft($hit) $nullright]
}
}
foreach hit [array names aRight] {
set aMatchinfo($hit) [concat $nullleft $aRight($hit)]
}
set nullentry [concat $nullleft $nullright]
}
}
}
}
# End of test procs. Actual tests are below this line.
#--------------------------------------------------------------------------
#--------------------------------------------------------------------------
# The following test cases - fts3auto-1.* - focus on testing the Tcl
# command [fts3_near_match], which is used by other tests in this file.
#
proc test_fts3_near_match {tn doc expr res} {
fts3_near_match $doc $expr -phrasecountvar p
uplevel do_test [list $tn] [list [list set {} $p]] [list $res]
}
test_fts3_near_match 1.1.1 {a b c a b} a {2}
test_fts3_near_match 1.1.2 {a b c a b} {a 5 b 6 c} {2 2 1}
test_fts3_near_match 1.1.3 {a b c a b} {"a b"} {2}
test_fts3_near_match 1.1.4 {a b c a b} {"b c"} {1}
test_fts3_near_match 1.1.5 {a b c a b} {"c c"} {0}
test_fts3_near_match 1.2.1 "a b c d e f g" {b 2 f} {0 0}
test_fts3_near_match 1.2.2 "a b c d e f g" {b 3 f} {1 1}
test_fts3_near_match 1.2.3 "a b c d e f g" {f 2 b} {0 0}
test_fts3_near_match 1.2.4 "a b c d e f g" {f 3 b} {1 1}
test_fts3_near_match 1.2.5 "a b c d e f g" {"a b" 2 "f g"} {0 0}
test_fts3_near_match 1.2.6 "a b c d e f g" {"a b" 3 "f g"} {1 1}
set A "a b c d e f g h i j k l m n o p q r s t u v w x y z"
test_fts3_near_match 1.3.1 $A {"c d" 5 "i j" 1 "e f"} {0 0 0}
test_fts3_near_match 1.3.2 $A {"c d" 5 "i j" 2 "e f"} {1 1 1}
#--------------------------------------------------------------------------
# Test cases fts3auto-2.* run some simple tests using the
# [do_fts3query_test] proc.
#
foreach {tn create} {
1 "fts4(a, b)"
2 "fts4(a, b, order=DESC)"
3 "fts4(a, b, order=ASC)"
4 "fts4(a, b, prefix=1)"
5 "fts4(a, b, order=DESC, prefix=1)"
6 "fts4(a, b, order=ASC, prefix=1)"
} {
do_test 2.$tn.1 {
catchsql { DROP TABLE t1 }
execsql "CREATE VIRTUAL TABLE t1 USING $create"
for {set i 0} {$i<32} {incr i} {
set doc [list]
if {$i&0x01} {lappend doc one}
if {$i&0x02} {lappend doc two}
if {$i&0x04} {lappend doc three}
if {$i&0x08} {lappend doc four}
if {$i&0x10} {lappend doc five}
execsql { INSERT INTO t1 VALUES($doc, null) }
}
} {}
foreach {tn2 expr} {
1 {one}
2 {one NEAR/1 five}
3 {t*}
4 {t* NEAR/0 five}
5 {o* NEAR/1 f*}
6 {one NEAR five NEAR two NEAR four NEAR three}
7 {one NEAR xyz}
8 {one OR two}
9 {one AND two}
10 {one NOT two}
11 {one AND two OR three}
12 {three OR one AND two}
13 {(three OR one) AND two}
14 {(three OR one) AND two NOT (five NOT four)}
15 {"one two"}
16 {"one two" NOT "three four"}
} {
do_fts3query_test 2.$tn.2.$tn2 t1 $expr
}
}
#--------------------------------------------------------------------------
# Some test cases involving deferred tokens.
#
foreach {tn create} {
1 "fts4(x)"
2 "fts4(x, order=DESC)"
} {
catchsql { DROP TABLE t1 }
execsql "CREATE VIRTUAL TABLE t1 USING $create"
do_execsql_test 3.$tn.1 {
INSERT INTO t1(docid, x) VALUES(-2, 'a b c d e f g h i j k');
INSERT INTO t1(docid, x) VALUES(-1, 'b c d e f g h i j k a');
INSERT INTO t1(docid, x) VALUES(0, 'c d e f g h i j k a b');
INSERT INTO t1(docid, x) VALUES(1, 'd e f g h i j k a b c');
INSERT INTO t1(docid, x) VALUES(2, 'e f g h i j k a b c d');
INSERT INTO t1(docid, x) VALUES(3, 'f g h i j k a b c d e');
INSERT INTO t1(docid, x) VALUES(4, 'a c e g i k');
INSERT INTO t1(docid, x) VALUES(5, 'a d g j');
INSERT INTO t1(docid, x) VALUES(6, 'c a b');
}
set limit [fts3_make_deferrable t1 c]
do_fts3query_test 3.$tn.2.1 t1 {a OR c}
do_test 3.$tn.3 {
fts3_zero_long_segments t1 $limit
} {1}
foreach {tn2 expr def} {
1 {a NEAR c} {}
2 {a AND c} c
3 {"a c"} c
4 {"c a"} c
5 {"a c" NEAR/1 g} {}
6 {"a c" NEAR/0 g} {}
} {
do_fts3query_test 3.$tn.4.$tn2 -deferred $def t1 $expr
}
}
#--------------------------------------------------------------------------
#
foreach {tn create} {
1 "fts4(x, y)"
2 "fts4(x, y, order=DESC)"
3 "fts4(x, y, order=DESC, prefix=2)"
} {
execsql [subst {
DROP TABLE t1;
CREATE VIRTUAL TABLE t1 USING $create;
INSERT INTO t1 VALUES('one two five four five', '');
INSERT INTO t1 VALUES('', 'one two five four five');
INSERT INTO t1 VALUES('one two', 'five four five');
}]
do_fts3query_test 4.$tn.1.1 t1 {one AND five}
do_fts3query_test 4.$tn.1.2 t1 {one NEAR five}
do_fts3query_test 4.$tn.1.3 t1 {one NEAR/1 five}
do_fts3query_test 4.$tn.1.4 t1 {one NEAR/2 five}
do_fts3query_test 4.$tn.1.5 t1 {one NEAR/3 five}
do_test 4.$tn.2 {
set limit [fts3_make_deferrable t1 five]
execsql { INSERT INTO t1(t1) VALUES('optimize') }
expr {[fts3_zero_long_segments t1 $limit]>0}
} {1}
do_fts3query_test 4.$tn.3.1 -deferred five t1 {one AND five}
do_fts3query_test 4.$tn.3.2 -deferred five t1 {one NEAR five}
do_fts3query_test 4.$tn.3.3 -deferred five t1 {one NEAR/1 five}
do_fts3query_test 4.$tn.3.4 -deferred five t1 {one NEAR/2 five}
do_fts3query_test 4.$tn.3.5 -deferred five t1 {one NEAR/3 five}
do_fts3query_test 4.$tn.4.1 -deferred fi* t1 {on* AND fi*}
do_fts3query_test 4.$tn.4.2 -deferred fi* t1 {on* NEAR fi*}
do_fts3query_test 4.$tn.4.3 -deferred fi* t1 {on* NEAR/1 fi*}
do_fts3query_test 4.$tn.4.4 -deferred fi* t1 {on* NEAR/2 fi*}
do_fts3query_test 4.$tn.4.5 -deferred fi* t1 {on* NEAR/3 fi*}
}
set sqlite_fts3_enable_parentheses $sfep
finish_test
test/fts3defer.test
+18 -11
View File
@@ -20,6 +20,8 @@ ifcapable !fts3 {
set sqlite_fts3_enable_parentheses 1
set fts3_simple_deferred_tokens_only 1
set ::testprefix fts3defer
#--------------------------------------------------------------------------
@@ -257,7 +259,6 @@ foreach {tn setup} {
do_select_test 1.2 {
SELECT rowid FROM t1 WHERE t1 MATCH 'jk eh'
} {100}
if {$tn==3} breakpoint
do_select_test 1.3 {
SELECT rowid FROM t1 WHERE t1 MATCH 'jk ubwrfqnbjf'
} {7 70 98}
@@ -282,13 +283,16 @@ if {$tn==3} breakpoint
do_select_test 1.10 {
SELECT rowid FROM t1 WHERE t1 MATCH 'z* vgsld'
} {10 13 17 31 35 51 58 88 89 90 93 100}
do_select_test 1.11 {
SELECT rowid FROM t1
WHERE t1 MATCH '(
zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR
zk OR zkhdvkw OR zm OR zsmhnf
) vgsld'
} {10 13 17 31 35 51 58 88 89 90 93 100}
if { $fts3_simple_deferred_tokens_only==0 } {
do_select_test 1.11 {
SELECT rowid FROM t1
WHERE t1 MATCH '(
zdu OR zexh OR zf OR zhbrzadb OR zidhxhbtv OR
zk OR zkhdvkw OR zm OR zsmhnf
) vgsld'
} {10 13 17 31 35 51 58 88 89 90 93 100}
}
do_select_test 2.1 {
SELECT rowid FROM t1 WHERE t1 MATCH '"zm agmckuiu"'
@@ -364,6 +368,7 @@ if {$tn==3} breakpoint
foreach DO_MALLOC_TEST $dmt_modes {
# Phrase search.
#
do_select_test 5.$DO_MALLOC_TEST.1 {
SELECT rowid FROM t1 WHERE t1 MATCH '"jk mjpavjuhw"'
} {8 15 36 64 67 72}
@@ -416,9 +421,11 @@ if {$tn==3} breakpoint
do_select_test 6.2.2 {
SELECT rowid FROM t1 WHERE t1 MATCH '"zm azavwm"'
} {15 26 92 96}
do_select_test 6.2.3 {
SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"'
} {8 15 26 92 96}
if {$fts3_simple_deferred_tokens_only==0} {
do_select_test 6.2.3 {
SELECT rowid FROM t1 WHERE t1 MATCH '"jk xduvfhk" OR "zm azavwm"'
} {8 15 26 92 96}
}
}
set testprefix fts3defer
test/fts3defer2.test
+13 -3
View File
@@ -48,6 +48,10 @@ do_execsql_test 1.1.4 {
UPDATE t1_segments SET block = zeroblob(length(block)) WHERE length(block)>10000;
} {2}
do_execsql_test 1.2.0 {
SELECT content FROM t1 WHERE t1 MATCH 'f (e a)';
} {{a b c d e f a x y}}
do_execsql_test 1.2.1 {
SELECT content FROM t1 WHERE t1 MATCH 'f (e NEAR/2 a)';
} {{a b c d e f a x y}}
@@ -58,7 +62,7 @@ do_execsql_test 1.2.2 {
} [list \
{a b c d [e] [f] [a] x y} \
{0 1 8 1 0 0 10 1 0 2 12 1} \
[list 3 1 1 1 1 1 8 8 1 8 8 8 5001 9]
[list 3 1 1 1 1 1 1 1 1 1 1 8 5001 9]
]
do_execsql_test 1.2.3 {
@@ -67,7 +71,7 @@ do_execsql_test 1.2.3 {
} [list \
{[a] b c d [e] [f] [a] x y} \
{0 2 0 1 0 1 8 1 0 0 10 1 0 2 12 1} \
[list 3 1 1 1 1 1 8 8 2 8 8 8 5001 9]
[list 3 1 1 1 1 1 1 1 2 2 1 8 5001 9]
]
do_execsql_test 1.3.1 { DROP TABLE t1 }
@@ -99,8 +103,14 @@ foreach {tn sql} {
[list 2 1 1 54 54 1 3 3 54 372 7] \
]
set sqlite_fts3_enable_parentheses 1
do_execsql_test 2.2.$tn.2 {
SELECT mit(matchinfo(t2, 'x')) FROM t2 WHERE t2 MATCH 'g z';
} [list \
[list 1 2 2 1 54 54] \
]
set sqlite_fts3_enable_parentheses 1
do_execsql_test 2.2.$tn.3 {
SELECT mit(matchinfo(t2, 'x')) FROM t2 WHERE t2 MATCH 'g OR (g z)';
} [list \
[list 1 2 2 1 2 2 1 54 54] \
test/fts3matchinfo.test
+11 -3
View File
@@ -68,7 +68,11 @@ do_execsql_test 3.1 {
do_execsql_test 3.2 {
CREATE VIRTUAL TABLE xx USING FTS4;
}
do_execsql_test 3.3 {
SELECT * FROM xx WHERE xx MATCH 'abc';
}
do_execsql_test 3.4 {
SELECT * FROM xx WHERE xx MATCH 'a b c';
}
@@ -240,9 +244,13 @@ do_matchinfo_test 4.2.6 t5 {t5 MATCH 'a OR b'} { s {1 2 1} }
do_execsql_test 4.3.0 "INSERT INTO t5 VALUES('x y [string repeat {b } 50000]')";
do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} {
x {{5 8 2 5 5 5} {3 8 2 3 5 5}}
s {2 1}
# It used to be that the second 'a' token would be deferred. That doesn't
# work any longer.
if 0 {
do_matchinfo_test 4.3.1 t5 {t5 MATCH 'a a'} {
x {{5 8 2 5 5 5} {3 8 2 3 5 5}}
s {2 1}
}
}
do_matchinfo_test 4.3.2 t5 {t5 MATCH 'a b'} { s {2} }
test/fts3prefix.test
+203
View File
@@ -0,0 +1,203 @@
# 2011 May 04
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#*************************************************************************
# This file implements regression tests for SQLite library. The
# focus of this script is testing the FTS3 module.
#
set testdir [file dirname $argv0]
source $testdir/tester.tcl
set testprefix fts3prefix
ifcapable !fts3 {
finish_test
return
}
# This proc tests that the prefixes index appears to represent the same content
# as the terms index.
#
proc fts3_terms_and_prefixes {db tbl prefixlengths} {
set iIndex 0
foreach len $prefixlengths {
incr iIndex
$db eval {
DROP TABLE IF EXISTS fts3check1;
DROP TABLE IF EXISTS fts3check2;
}
$db eval "CREATE VIRTUAL TABLE fts3check1 USING fts4term($tbl, 0);"
$db eval "CREATE VIRTUAL TABLE fts3check2 USING fts4term($tbl, $iIndex);"
$db eval {
DROP TABLE IF EXISTS temp.terms;
DROP TABLE IF EXISTS temp.prefixes;
CREATE TEMP TABLE terms AS SELECT * FROM fts3check1;
CREATE TEMP TABLE prefixes AS SELECT * FROM fts3check2;
CREATE INDEX temp.idx ON prefixes(term);
DROP TABLE fts3check1;
DROP TABLE fts3check2;
}
set nExpect 0
$db eval { SELECT term, docid, col, pos FROM temp.terms } a {
if {[string length $a(term)]<$len} continue
incr nExpect
set prefix [string range $a(term) 0 [expr $len-1]]
set r [$db one {
SELECT count(*) FROM temp.prefixes WHERE
term = $prefix AND docid = $a(docid) AND col = $a(col) AND pos = $a(pos)
}]
if {$r != 1} {
error "$t, $a(docid), $a(col), $a(pos)"
}
}
set nCount [$db one {SELECT count(*) FROM temp.prefixes}]
if {$nCount != $nExpect} {
error "prefixes.count(*) is $nCount expected $nExpect"
}
execsql { DROP TABLE temp.prefixes }
execsql { DROP TABLE temp.terms }
set list [list]
$db eval "
SELECT sum( 1 << (16*(level%1024)) ) AS total, (level/1024) AS tree
FROM ${tbl}_segdir GROUP BY tree
" {
lappend list [list $total $tree]
}
if { [lsort -integer -index 0 $list] != [lsort -integer -index 1 $list] } {
error "inconsistent tree structures: $list"
}
}
return ""
}
proc fts3_tap_test {tn db tbl lens} {
uplevel [list do_test $tn [list fts3_terms_and_prefixes $db $tbl $lens] ""]
}
#-------------------------------------------------------------------------
# Test cases 1.* are a sanity check. They test that the prefixes index is
# being constructed correctly for the simplest possible case.
#
do_execsql_test 1.1 {
CREATE VIRTUAL TABLE t1 USING fts4(prefix='1,3,6');
CREATE VIRTUAL TABLE p1 USING fts4term(t1, 1);
CREATE VIRTUAL TABLE p2 USING fts4term(t1, 2);
CREATE VIRTUAL TABLE p3 USING fts4term(t1, 3);
CREATE VIRTUAL TABLE terms USING fts4term(t1);
}
do_execsql_test 1.2 {
INSERT INTO t1 VALUES('sqlite mysql firebird');
}
do_execsql_test 1.3.1 { SELECT term FROM p1 } {f m s}
do_execsql_test 1.3.2 { SELECT term FROM p2 } {fir mys sql}
do_execsql_test 1.3.3 { SELECT term FROM p3 } {firebi sqlite}
do_execsql_test 1.4 {
SELECT term FROM terms;
} {firebird mysql sqlite}
fts3_tap_test 1.5 db t1 {1 3 6}
#-------------------------------------------------------------------------
# A slightly more complicated dataset. This test also verifies that DELETE
# operations do not corrupt the prefixes index.
#
do_execsql_test 2.1 {
INSERT INTO t1 VALUES('FTS3 and FTS4 are an SQLite virtual table modules');
INSERT INTO t1 VALUES('that allows users to perform full-text searches on');
INSERT INTO t1 VALUES('a set of documents. The most common (and');
INSERT INTO t1 VALUES('effective) way to describe full-text searches is');
INSERT INTO t1 VALUES('"what Google, Yahoo and Altavista do with');
INSERT INTO t1 VALUES('documents placed on the World Wide Web". Users');
INSERT INTO t1 VALUES('input a term, or series of terms, perhaps');
INSERT INTO t1 VALUES('connected by a binary operator or grouped together');
INSERT INTO t1 VALUES('into a phrase, and the full-text query system');
INSERT INTO t1 VALUES('finds the set of documents that best matches those');
INSERT INTO t1 VALUES('terms considering the operators and groupings the');
INSERT INTO t1 VALUES('user has specified. This article describes the');
INSERT INTO t1 VALUES('deployment and usage of FTS3 and FTS4.');
INSERT INTO t1 VALUES('FTS1 and FTS2 are obsolete full-text search');
INSERT INTO t1 VALUES('modules for SQLite. There are known issues with');
INSERT INTO t1 VALUES('these older modules and their use should be');
INSERT INTO t1 VALUES('avoided. Portions of the original FTS3 code were');
INSERT INTO t1 VALUES('contributed to the SQLite project by Scott Hess of');
INSERT INTO t1 VALUES('Google. It is now developed and maintained as part');
INSERT INTO t1 VALUES('of SQLite. ');
}
fts3_tap_test 2.2 db t1 {1 3 6}
do_execsql_test 2.3 { DELETE FROM t1 WHERE docid%2; }
fts3_tap_test 2.4 db t1 {1 3 6}
do_execsql_test 2.5 { INSERT INTO t1(t1) VALUES('optimize') }
fts3_tap_test 2.6 db t1 {1 3 6}
do_execsql_test 3.1 {
CREATE VIRTUAL TABLE t2 USING fts4(prefix='1,2,3');
INSERT INTO t2 VALUES('On 12 September the wind direction turned and');
INSERT INTO t2 VALUES('William''s fleet sailed. A storm blew up and the');
INSERT INTO t2 VALUES('fleet was forced to take shelter at');
INSERT INTO t2 VALUES('Saint-Valery-sur-Somme and again wait for the wind');
INSERT INTO t2 VALUES('to change. On 27 September the Norman fleet');
INSERT INTO t2 VALUES('finally set sail, landing in England at Pevensey');
INSERT INTO t2 VALUES('Bay (Sussex) on 28 September. William then moved');
INSERT INTO t2 VALUES('to Hastings, a few miles to the east, where he');
INSERT INTO t2 VALUES('built a prefabricated wooden castle for a base of');
INSERT INTO t2 VALUES('operations. From there, he ravaged the hinterland');
INSERT INTO t2 VALUES('and waited for Harold''s return from the north.');
INSERT INTO t2 VALUES('On 12 September the wind direction turned and');
INSERT INTO t2 VALUES('William''s fleet sailed. A storm blew up and the');
INSERT INTO t2 VALUES('fleet was forced to take shelter at');
INSERT INTO t2 VALUES('Saint-Valery-sur-Somme and again wait for the wind');
INSERT INTO t2 VALUES('to change. On 27 September the Norman fleet');
INSERT INTO t2 VALUES('finally set sail, landing in England at Pevensey');
INSERT INTO t2 VALUES('Bay (Sussex) on 28 September. William then moved');
INSERT INTO t2 VALUES('to Hastings, a few miles to the east, where he');
INSERT INTO t2 VALUES('built a prefabricated wooden castle for a base of');
INSERT INTO t2 VALUES('operations. From there, he ravaged the hinterland');
INSERT INTO t2 VALUES('and waited for Harold''s return from the north.');
}
fts3_tap_test 3.2 db t2 {1 2 3}
do_execsql_test 3.3 { SELECT optimize(t2) FROM t2 LIMIT 1 } {{Index optimized}}
fts3_tap_test 3.4 db t2 {1 2 3}
#-------------------------------------------------------------------------
# Simple tests for reading the prefix-index.
#
do_execsql_test 4.1 {
CREATE VIRTUAL TABLE t3 USING fts4(prefix="1,4");
INSERT INTO t3 VALUES('one two three');
INSERT INTO t3 VALUES('four five six');
INSERT INTO t3 VALUES('seven eight nine');
}
do_execsql_test 4.2 {
SELECT * FROM t3 WHERE t3 MATCH 'f*'
} {{four five six}}
do_execsql_test 4.3 {
SELECT * FROM t3 WHERE t3 MATCH 'four*'
} {{four five six}}
do_execsql_test 4.4 {
SELECT * FROM t3 WHERE t3 MATCH 's*'
} {{four five six} {seven eight nine}}
do_execsql_test 4.5 {
SELECT * FROM t3 WHERE t3 MATCH 'sev*'
} {{seven eight nine}}
do_execsql_test 4.6 {
SELECT * FROM t3 WHERE t3 MATCH 'one*'
} {{one two three}}
finish_test
test/fts3rnd.test
+47 -21
View File
@@ -162,7 +162,7 @@ proc simple_phrase {zPrefix} {
# This [proc] is used to test the FTS3 matchinfo() function.
#
proc simple_token_matchinfo {zToken} {
proc simple_token_matchinfo {zToken bDesc} {
set nDoc(0) 0
set nDoc(1) 0
@@ -171,6 +171,8 @@ proc simple_token_matchinfo {zToken} {
set nHit(1) 0
set nHit(2) 0
set dir -inc
if {$bDesc} { set dir -dec }
foreach key [array names ::t1] {
set value $::t1($key)
@@ -184,7 +186,7 @@ proc simple_token_matchinfo {zToken} {
}
set ret [list]
foreach docid [lsort -integer [array names a]] {
foreach docid [lsort -integer $dir [array names a]] {
if { [lindex [lsort -integer $a($docid)] end] } {
set matchinfo [list 1 3]
foreach i {0 1 2} hit $a($docid) {
@@ -262,22 +264,37 @@ proc mit {blob} {
return $r
}
db func mit mit
set sqlite_fts3_enable_parentheses 1
foreach nodesize {50 500 1000 2000} {
proc do_orderbydocid_test {tn sql res} {
uplevel [list do_select_test $tn.asc "$sql ORDER BY docid ASC" $res]
uplevel [list do_select_test $tn.desc "$sql ORDER BY docid DESC" \
[lsort -int -dec $res]
]
}
set NUM_TRIALS 100
foreach {nodesize order} {
50 DESC
50 ASC
500 ASC
1000 DESC
2000 ASC
} {
catch { array unset ::t1 }
set testname "$nodesize/$order"
# Create the FTS3 table. Populate it (and the Tcl array) with 100 rows.
#
db transaction {
catchsql { DROP TABLE t1 }
execsql "CREATE VIRTUAL TABLE t1 USING fts3(a, b, c)"
execsql "CREATE VIRTUAL TABLE t1 USING fts4(a, b, c, order=$order)"
execsql "INSERT INTO t1(t1) VALUES('nodesize=$nodesize')"
for {set i 0} {$i < 100} {incr i} { insert_row $i }
}
for {set iTest 1} {$iTest <= 100} {incr iTest} {
for {set iTest 1} {$iTest <= $NUM_TRIALS} {incr iTest} {
catchsql COMMIT
set DO_MALLOC_TEST 0
@@ -286,6 +303,8 @@ foreach nodesize {50 500 1000 2000} {
set DO_MALLOC_TEST 1
set nRep 2
}
set ::testprefix fts3rnd-1.$testname.$iTest
# Delete one row, update one row and insert one row.
#
@@ -307,7 +326,7 @@ foreach nodesize {50 500 1000 2000} {
if {0==($iTest%2)} { execsql COMMIT }
if {0==($iTest%2)} {
do_test fts3rnd-1.$nodesize.$iTest.0 { fts3_integrity_check t1 } ok
#do_test 0 { fts3_integrity_check t1 } ok
}
# Pick 10 terms from the vocabulary. Check that the results of querying
@@ -317,9 +336,14 @@ foreach nodesize {50 500 1000 2000} {
#
for {set i 0} {$i < 10} {incr i} {
set term [random_term]
do_select_test fts3rnd-1.$nodesize.$iTest.1.$i {
do_select_test 1.$i.asc {
SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term
} [simple_token_matchinfo $term]
ORDER BY docid ASC
} [simple_token_matchinfo $term 0]
do_select_test 1.$i.desc {
SELECT docid, mit(matchinfo(t1)) FROM t1 WHERE t1 MATCH $term
ORDER BY docid DESC
} [simple_token_matchinfo $term 1]
}
# This time, use the first two characters of each term as a term prefix
@@ -329,7 +353,7 @@ foreach nodesize {50 500 1000 2000} {
for {set i 0} {$i < $nRep} {incr i} {
set prefix [string range [random_term] 0 end-1]
set match "${prefix}*"
do_select_test fts3rnd-1.$nodesize.$iTest.2.$i {
do_orderbydocid_test 2.$i {
SELECT docid FROM t1 WHERE t1 MATCH $match
} [simple_phrase $match]
}
@@ -339,7 +363,7 @@ foreach nodesize {50 500 1000 2000} {
for {set i 0} {$i < $nRep} {incr i} {
set term [list [random_term] [random_term]]
set match "\"$term\""
do_select_test fts3rnd-1.$nodesize.$iTest.3.$i {
do_orderbydocid_test 3.$i {
SELECT docid FROM t1 WHERE t1 MATCH $match
} [simple_phrase $term]
}
@@ -349,7 +373,7 @@ foreach nodesize {50 500 1000 2000} {
for {set i 0} {$i < $nRep} {incr i} {
set term [list [random_term] [random_term] [random_term]]
set match "\"$term\""
do_select_test fts3rnd-1.$nodesize.$iTest.4.$i {
do_orderbydocid_test 4.$i {
SELECT docid FROM t1 WHERE t1 MATCH $match
} [simple_phrase $term]
}
@@ -362,17 +386,19 @@ foreach nodesize {50 500 1000 2000} {
append query "[string range [random_term] 0 end-1]*"
set match "\"$query\""
do_select_test fts3rnd-1.$nodesize.$iTest.5.$i {
do_orderbydocid_test 5.$i {
SELECT docid FROM t1 WHERE t1 MATCH $match
} [simple_phrase $query]
}
# A NEAR query with terms as the arguments.
# A NEAR query with terms as the arguments:
#
# ... MATCH '$term1 NEAR $term2' ...
#
for {set i 0} {$i < $nRep} {incr i} {
set terms [list [random_term] [random_term]]
set match [join $terms " NEAR "]
do_select_test fts3rnd-1.$nodesize.$iTest.6.$i {
do_orderbydocid_test 6.$i {
SELECT docid FROM t1 WHERE t1 MATCH $match
} [simple_near $terms 10]
}
@@ -383,7 +409,7 @@ foreach nodesize {50 500 1000 2000} {
set terms [list [random_term] [random_term] [random_term]]
set nNear 11
set match [join $terms " NEAR/$nNear "]
do_select_test fts3rnd-1.$nodesize.$iTest.7.$i {
do_orderbydocid_test 7.$i {
SELECT docid FROM t1 WHERE t1 MATCH $match
} [simple_near $terms $nNear]
}
@@ -399,7 +425,7 @@ foreach nodesize {50 500 1000 2000} {
set term1 [random_term]
set term2 [random_term]
set match "$term1 $op $term2"
do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i {
do_orderbydocid_test $tn.$i {
SELECT docid FROM t1 WHERE t1 MATCH $match
} [$proc [simple_phrase $term1] [simple_phrase $term2]]
}
@@ -408,9 +434,9 @@ foreach nodesize {50 500 1000 2000} {
# Set operations on NEAR queries.
#
foreach {tn op proc} {
8 OR setop_or
9 NOT setop_not
10 AND setop_and
11 OR setop_or
12 NOT setop_not
13 AND setop_and
} {
for {set i 0} {$i < $nRep} {incr i} {
set term1 [random_term]
@@ -418,7 +444,7 @@ foreach nodesize {50 500 1000 2000} {
set term3 [random_term]
set term4 [random_term]
set match "$term1 NEAR $term2 $op $term3 NEAR $term4"
do_select_test fts3rnd-1.$nodesize.$iTest.$tn.$i {
do_orderbydocid_test $tn.$i {
SELECT docid FROM t1 WHERE t1 MATCH $match
} [$proc \
[simple_near [list $term1 $term2] 10] \
test/fts3sort.test
+106 -52
View File
@@ -21,9 +21,8 @@ ifcapable !fts3 {
return
}
set testprefix fts3sort
proc build_database {nRow} {
proc build_database {nRow param} {
db close
forcedelete test.db
sqlite3 db test.db
@@ -31,7 +30,7 @@ proc build_database {nRow} {
set vocab [list aa ab ac ba bb bc ca cb cc da]
expr srand(0)
execsql { CREATE VIRTUAL TABLE t1 USING fts4 }
execsql "CREATE VIRTUAL TABLE t1 USING fts4($param)"
for {set i 0} {$i < $nRow} {incr i} {
set v [expr int(rand()*1000000)]
set doc [list]
@@ -42,13 +41,24 @@ proc build_database {nRow} {
}
}
set nRow 1000
do_test 1.0 {
build_database $nRow
execsql { SELECT count(*) FROM t1 }
} $nRow
set testprefix fts3sort
foreach {tn query} {
unset -nocomplain CONTROL
foreach {t param} {
1 ""
2 "order=asc"
3 "order=desc"
} {
set testprefix fts3sort-1.$t
set nRow 1000
do_test 1.0 {
build_database $nRow $param
execsql { SELECT count(*) FROM t1 }
} $nRow
foreach {tn query} {
1 "SELECT docid, * FROM t1"
2 "SELECT docid, * FROM t1 WHERE t1 MATCH 'aa'"
3 "SELECT docid, * FROM t1 WHERE t1 MATCH 'a*'"
@@ -59,50 +69,94 @@ foreach {tn query} {
8 "SELECT docid, * FROM t1 WHERE t1 MATCH 'nosuchtoken'"
9 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa OR da'"
10 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa OR nosuchtoken'"
11 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH 'aa NEAR bb'"
12 "SELECT docid, snippet(t1) FROM t1 WHERE t1 MATCH '\"aa bb\"'"
13 "SELECT docid, content FROM t1 WHERE t1 MATCH 'aa NEAR/2 bb NEAR/3 cc'"
14 "SELECT docid, content FROM t1 WHERE t1 MATCH '\"aa bb cc\"'"
} {
unset -nocomplain A B C D
set A_list [list]
set B_list [list]
set C_list [list]
set D_list [list]
unset -nocomplain X
db eval "$query ORDER BY rowid ASC" X {
set A($X(docid)) [array get X]
lappend A_list $X(docid)
}
unset -nocomplain X
db eval "$query ORDER BY rowid DESC" X {
set B($X(docid)) [array get X]
lappend B_list $X(docid)
}
unset -nocomplain X
db eval "$query ORDER BY docid ASC" X {
set C($X(docid)) [array get X]
lappend C_list $X(docid)
}
unset -nocomplain X
db eval "$query ORDER BY docid DESC" X {
set D($X(docid)) [array get X]
lappend D_list $X(docid)
}
do_test $tn.1 { set A_list } [lsort -integer -increasing $A_list]
do_test $tn.2 { set B_list } [lsort -integer -decreasing $B_list]
do_test $tn.3 { set C_list } [lsort -integer -increasing $C_list]
do_test $tn.4 { set D_list } [lsort -integer -decreasing $D_list]
unset -nocomplain DATA
unset -nocomplain X
db eval "$query" X {
set DATA($X(docid)) [array get X]
}
do_test $tn.5 { lsort [array get A] } [lsort [array get DATA]]
do_test $tn.6 { lsort [array get B] } [lsort [array get DATA]]
do_test $tn.7 { lsort [array get C] } [lsort [array get DATA]]
do_test $tn.8 { lsort [array get D] } [lsort [array get DATA]]
if {[info exists CONTROL($tn)]} {
do_test $tn.9 { set CONTROL($tn) } [lsort [array get DATA]]
} else {
set CONTROL($tn) [lsort [array get DATA]]
}
}
}
unset -nocomplain CONTROL
set testprefix fts3sort
#-------------------------------------------------------------------------
# Tests for parsing the "order=asc" and "order=desc" directives.
#
foreach {tn param res} {
1 "order=asc" {0 {}}
2 "order=desc" {0 {}}
3 "order=dec" {1 {unrecognized order: dec}}
4 "order=xxx, order=asc" {1 {unrecognized order: xxx}}
5 "order=desc, order=asc" {0 {}}
} {
unset -nocomplain A B C D
set A_list [list]
set B_list [list]
set C_list [list]
set D_list [list]
unset -nocomplain X
db eval "$query ORDER BY rowid ASC" X {
set A($X(docid)) [array get X]
lappend A_list $X(docid)
}
unset -nocomplain X
db eval "$query ORDER BY rowid DESC" X {
set B($X(docid)) [array get X]
lappend B_list $X(docid)
}
unset -nocomplain X
db eval "$query ORDER BY docid ASC" X {
set C($X(docid)) [array get X]
lappend C_list $X(docid)
}
unset -nocomplain X
db eval "$query ORDER BY docid DESC" X {
set D($X(docid)) [array get X]
lappend D_list $X(docid)
}
do_test 1.$tn.1 { set A_list } [lsort -integer -increasing $A_list]
do_test 1.$tn.2 { set B_list } [lsort -integer -decreasing $B_list]
do_test 1.$tn.3 { set C_list } [lsort -integer -increasing $C_list]
do_test 1.$tn.4 { set D_list } [lsort -integer -decreasing $D_list]
unset -nocomplain DATA
unset -nocomplain X
db eval "$query" X {
set DATA($X(docid)) [array get X]
}
do_test 1.$tn.5 { lsort [array get A] } [lsort [array get DATA]]
do_test 1.$tn.6 { lsort [array get B] } [lsort [array get DATA]]
do_test 1.$tn.7 { lsort [array get C] } [lsort [array get DATA]]
do_test 1.$tn.8 { lsort [array get D] } [lsort [array get DATA]]
execsql { DROP TABLE IF EXISTS t1 }
do_catchsql_test 2.1.$tn "
CREATE VIRTUAL TABLE t1 USING fts4(a, b, $param)
" $res
}
do_execsql_test 2.2 {
BEGIN;
CREATE VIRTUAL TABLE t2 USING fts4(order=desc);
INSERT INTO t2 VALUES('aa bb');
INSERT INTO t2 VALUES('bb cc');
INSERT INTO t2 VALUES('cc aa');
SELECT docid FROM t2 WHERE t2 MATCH 'aa';
END;
} {3 1}
do_execsql_test 2.3 {
SELECT docid FROM t2 WHERE t2 MATCH 'aa';
} {3 1}
finish_test
test/hook.test
+28
View File
@@ -274,6 +274,34 @@ ifcapable compound&&attach {
set ::update_hook
} [list]
}
do_test hook-4.4 {
execsql {
CREATE TABLE t4(a UNIQUE, b);
INSERT INTO t4 VALUES(1, 'a');
INSERT INTO t4 VALUES(2, 'b');
}
set ::update_hook [list]
execsql {
REPLACE INTO t4 VALUES(1, 'c');
}
set ::update_hook
} [list INSERT main t4 3 ]
do_execsql_test hook-4.4.1 {
SELECT * FROM t4 ORDER BY a;
} {1 c 2 b}
do_test hook-4.4.2 {
set ::update_hook [list]
execsql {
PRAGMA recursive_triggers = on;
REPLACE INTO t4 VALUES(1, 'd');
}
set ::update_hook
} [list INSERT main t4 4 ]
do_execsql_test hook-4.4.3 {
SELECT * FROM t4 ORDER BY a;
} {1 d 2 b}
db update_hook {}
#
#----------------------------------------------------------------------------
test/permutations.test
+2 -1
View File
@@ -179,10 +179,11 @@ test_suite "fts3" -prefix "" -description {
fts3atoken.test fts3b.test fts3c.test fts3cov.test fts3d.test
fts3defer.test fts3defer2.test fts3e.test fts3expr.test fts3expr2.test
fts3near.test fts3query.test fts3shared.test fts3snippet.test
fts3sort.test
fts3fault.test fts3malloc.test fts3matchinfo.test
fts3aux1.test fts3comp1.test
fts3aux1.test fts3comp1.test fts3auto.test
}
test/tester.tcl
+2 -2
View File
@@ -374,11 +374,11 @@ proc fix_testname {varname} {
proc do_execsql_test {testname sql {result {}}} {
fix_testname testname
uplevel do_test $testname [list "execsql {$sql}"] [list [list {*}$result]]
uplevel do_test [list $testname] [list "execsql {$sql}"] [list [list {*}$result]]
}
proc do_catchsql_test {testname sql result} {
fix_testname testname
uplevel do_test $testname [list "catchsql {$sql}"] [list $result]
uplevel do_test [list $testname] [list "catchsql {$sql}"] [list $result]
}
proc do_eqp_test {name sql res} {
uplevel do_execsql_test $name [list "EXPLAIN QUERY PLAN $sql"] [list $res]
test/trace2.test
+3 -3
View File
@@ -141,10 +141,10 @@ ifcapable fts3 {
INSERT INTO x1(x1) VALUES('optimize');
} {
"INSERT INTO x1(x1) VALUES('optimize');"
"-- SELECT idx, start_block, leaves_end_block, end_block, root FROM 'main'.'x1_segdir' ORDER BY level DESC, idx ASC"
"-- SELECT count(*), max(level) FROM 'main'.'x1_segdir'"
"-- SELECT idx, start_block, leaves_end_block, end_block, root FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?ORDER BY level DESC, idx ASC"
"-- SELECT max(level) FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?"
"-- SELECT coalesce((SELECT max(blockid) FROM 'main'.'x1_segments') + 1, 1)"
"-- DELETE FROM 'main'.'x1_segdir'"
"-- DELETE FROM 'main'.'x1_segdir' WHERE level BETWEEN ? AND ?"
"-- INSERT INTO 'main'.'x1_segdir' VALUES(?,?,?,?,?,?)"
}
}