\n"
continue
}
+ if {[regexp {SYNONYM SUPPORT} $line]} {
+ set line "
Synonym Support
"
+ }
if {[string trim $line] == ""} {
append res "\n"
} else {
append res "$line\n"
}
}
- append res "\n"
set res
}
@@ -208,6 +210,10 @@ proc main {data} {
fts5_tokenizer {
output [get_fts5_struct $data "typedef struct Fts5Tokenizer" "^\};"]
+ output [get_fts5_struct $data \
+ "Flags that may be passed as the third argument to xTokenize()" \
+ "#define FTS5_TOKEN_COLOCATED"
+ ]
}
fts5_extension {
diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h
index c123d6444c..6872918e94 100644
--- a/ext/fts5/fts5.h
+++ b/ext/fts5/fts5.h
@@ -217,7 +217,7 @@ struct Fts5ExtensionApi {
int (*xTokenize)(Fts5Context*,
const char *pText, int nText, /* Text to tokenize */
void *pCtx, /* Context passed to xToken() */
- int (*xToken)(void*, const char*, int, int, int) /* Callback */
+ int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
);
int (*xPhraseCount)(Fts5Context*);
@@ -278,18 +278,46 @@ struct Fts5ExtensionApi {
**
** xTokenize:
** This function is expected to tokenize the nText byte string indicated
-** by argument pText. pText may not be nul-terminated. The first argument
-** passed to this function is a pointer to an Fts5Tokenizer object returned
-** by an earlier call to xCreate().
+** by argument pText. pText may or may not be nul-terminated. The first
+** argument passed to this function is a pointer to an Fts5Tokenizer object
+** returned by an earlier call to xCreate().
+**
+** The second argument indicates the reason that FTS5 is requesting
+** tokenization of the supplied text. This is always one of the following
+** four values:
+**
+**
- FTS5_TOKENIZE_DOCUMENT - A document is being inserted into
+** or removed from the FTS table. The tokenizer is being invoked to
+** determine the set of tokens to add to (or delete from) the
+** FTS index.
+**
+**
- FTS5_TOKENIZE_QUERY - A MATCH query is being executed
+** against the FTS index. The tokenizer is being called to tokenize
+** a bareword or quoted string specified as part of the query.
+**
+**
- (FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX) - Same as
+** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is
+** followed by a "*" character, indicating that the last token
+** returned by the tokenizer will be treated as a token prefix.
+**
+**
- FTS5_TOKENIZE_AUX - The tokenizer is being invoked to
+** satisfy an fts5_api.xTokenize() request made by an auxiliary
+** function. Or an fts5_api.xColumnSize() request made by the same
+** on a columnsize=0 database.
+**
**
** For each token in the input string, the supplied callback xToken() must
** be invoked. The first argument to it should be a copy of the pointer
-** passed as the second argument to xTokenize(). The next two arguments
-** are a pointer to a buffer containing the token text, and the size of
-** the token in bytes. The 4th and 5th arguments are the byte offsets of
-** the first byte of and first byte immediately following the text from
+** passed as the second argument to xTokenize(). The third and fourth
+** arguments are a pointer to a buffer containing the token text, and the
+** size of the token in bytes. The 4th and 5th arguments are the byte offsets
+** of the first byte of and first byte immediately following the text from
** which the token is derived within the input.
**
+** The second argument passed to the xToken() callback ("tflags") should
+** normally be set to 0. The exception is if the tokenizer supports
+** synonyms. In this case see the discussion below for details.
+**
** FTS5 assumes the xToken() callback is invoked for each token in the
** order that they occur within the input text.
**
@@ -301,6 +329,112 @@ struct Fts5ExtensionApi {
** may abandon the tokenization and return any error code other than
** SQLITE_OK or SQLITE_DONE.
**
+** SYNONYM SUPPORT
+**
+** Custom tokenizers may also support synonyms. Consider a case in which a
+** user wishes to query for a phrase such as "first place". Using the
+** built-in tokenizers, the FTS5 query 'first + place' will match instances
+** of "first place" within the document set, but not alternative forms
+** such as "1st place". In some applications, it would be better to match
+** all instances of "first place" or "1st place" regardless of which form
+** the user specified in the MATCH query text.
+**
+** There are several ways to approach this in FTS5:
+**
+** - By mapping all synonyms to a single token. In this case, the
+** In the above example, this means that the tokenizer returns the
+** same token for inputs "first" and "1st". Say that token is in
+** fact "first", so that when the user inserts the document "I won
+** 1st place" entries are added to the index for tokens "i", "won",
+** "first" and "place". If the user then queries for '1st + place',
+** the tokenizer substitutes "first" for "1st" and the query works
+** as expected.
+**
+**
- By adding multiple synonyms for a single term to the FTS index.
+** In this case, when tokenizing query text, the tokenizer may
+** provide multiple synonyms for a single term within the document.
+** FTS5 then queries the index for each synonym individually. For
+** example, faced with the query:
+**
+**
+** ... MATCH 'first place'
+**
+** the tokenizer offers both "1st" and "first" as synonyms for the
+** first token in the MATCH query and FTS5 effectively runs a query
+** similar to:
+**
+**
+** ... MATCH '(first OR 1st) place'
+**
+** except that, for the purposes of auxiliary functions, the query
+** still appears to contain just two phrases - "(first OR 1st)"
+** being treated as a single phrase.
+**
+**
- By adding multiple synonyms for a single term to the FTS index.
+** Using this method, when tokenizing document text, the tokenizer
+** provides multiple synonyms for each token. So that when a
+** document such as "I won first place" is tokenized, entries are
+** added to the FTS index for "i", "won", "first", "1st" and
+** "place".
+**
+** This way, even if the tokenizer does not provide synonyms
+** when tokenizing query text (it should not - to do would be
+** inefficient), it doesn't matter if the user queries for
+** 'first + place' or '1st + place', as there are entires in the
+** FTS index corresponding to both forms of the first token.
+**
+**
+** Whether is is parsing document or query text, any call to xToken that
+** specifies a tflags argument with the FTS5_TOKEN_COLOCATED bit
+** is considered to supply a synonym for the previous token. For example,
+** when parsing the document "I won first place", a tokenizer that supports
+** synonyms would call xToken() 5 times, as follows:
+**
+**
+** xToken(pCtx, 0, "i", 1, 0, 1);
+** xToken(pCtx, 0, "won", 3, 2, 5);
+** xToken(pCtx, 0, "first", 5, 6, 11);
+** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11);
+** xToken(pCtx, 0, "place", 5, 12, 17);
+**
+**
+** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time
+** xToken() is called. Multiple synonyms may be specified for a single token
+** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence.
+** There is no limit to the number of synonyms that may be provided for a
+** single token.
+**
+** In many cases, method (1) above is the best approach. It does not add
+** extra data to the FTS index or require FTS5 to query for multiple terms,
+** so it is efficient in terms of disk space and query speed. However, it
+** does not support prefix queries very well. If, as suggested above, the
+** token "first" is subsituted for "1st" by the tokenizer, then the query:
+**
+**
+** ... MATCH '1s*'
+**
+** will not match documents that contain the token "1st" (as the tokenizer
+** will probably not map "1s" to any prefix of "first").
+**
+** For full prefix support, method (3) may be preferred. In this case,
+** because the index contains entries for both "first" and "1st", prefix
+** queries such as 'fi*' or '1s*' will match correctly. However, because
+** extra entries are added to the FTS index, this method uses more space
+** within the database.
+**
+** Method (2) offers a midpoint between (1) and (3). Using this method,
+** a query such as '1s*' will match documents that contain the literal
+** token "1st", but not "first" (assuming the tokenizer is not able to
+** provide synonyms for prefixes). However, a non-prefix query like '1st'
+** will match against "1st" and "first". This method does not require
+** extra disk space, as no extra entries are added to the FTS index.
+** On the other hand, it may require more CPU cycles to run MATCH queries,
+** as separate queries of the FTS index are required for each synonym.
+**
+** When using methods (2) or (3), it is important that the tokenizer only
+** provide synonyms when tokenizing document text (method (2)) or query
+** text (method (3)), not both. Doing so will not cause any errors, but is
+** inefficient.
*/
typedef struct Fts5Tokenizer Fts5Tokenizer;
typedef struct fts5_tokenizer fts5_tokenizer;
@@ -309,9 +443,11 @@ struct fts5_tokenizer {
void (*xDelete)(Fts5Tokenizer*);
int (*xTokenize)(Fts5Tokenizer*,
void *pCtx,
+ int flags, /* Mask of FTS5_TOKENIZE_* flags */
const char *pText, int nText,
int (*xToken)(
void *pCtx, /* Copy of 2nd argument to xTokenize() */
+ int tflags, /* Mask of FTS5_TOKEN_* flags */
const char *pToken, /* Pointer to buffer containing token */
int nToken, /* Size of token in bytes */
int iStart, /* Byte offset of token within input text */
@@ -320,6 +456,16 @@ struct fts5_tokenizer {
);
};
+/* Flags that may be passed as the third argument to xTokenize() */
+#define FTS5_TOKENIZE_QUERY 0x0001
+#define FTS5_TOKENIZE_PREFIX 0x0002
+#define FTS5_TOKENIZE_DOCUMENT 0x0004
+#define FTS5_TOKENIZE_AUX 0x0008
+
+/* Flags that may be passed by the tokenizer implementation back to FTS5
+** as the third argument to the supplied xToken callback. */
+#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */
+
/*
** END OF CUSTOM TOKENIZERS
*************************************************************************/
@@ -329,7 +475,7 @@ struct fts5_tokenizer {
*/
typedef struct fts5_api fts5_api;
struct fts5_api {
- int iVersion; /* Currently always set to 1 */
+ int iVersion; /* Currently always set to 2 */
/* Create a new tokenizer */
int (*xCreateTokenizer)(
diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h
index 5298429437..837ecb1ccd 100644
--- a/ext/fts5/fts5Int.h
+++ b/ext/fts5/fts5Int.h
@@ -166,9 +166,10 @@ int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig);
int sqlite3Fts5Tokenize(
Fts5Config *pConfig, /* FTS5 Configuration object */
+ int flags, /* FTS5_TOKENIZE_* flags */
const char *pText, int nText, /* Text to tokenize */
void *pCtx, /* Context passed to xToken() */
- int (*xToken)(void*, const char*, int, int, int) /* Callback */
+ int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
);
void sqlite3Fts5Dequote(char *z);
@@ -234,8 +235,10 @@ struct Fts5PoslistReader {
int n; /* Size of buffer at a[] in bytes */
int i; /* Current offset in a[] */
+ u8 bFlag; /* For client use (any custom purpose) */
+
/* Output variables */
- int bEof; /* Set to true at EOF */
+ u8 bEof; /* Set to true at EOF */
i64 iPos; /* (iCol<<32) + iPos */
};
int sqlite3Fts5PoslistReaderInit(
@@ -381,9 +384,9 @@ int sqlite3Fts5IndexErrcode(Fts5Index*);
void sqlite3Fts5IndexReset(Fts5Index*);
/*
-** Get or set the "averages" record.
+** Get or set the "averages" values.
*/
-int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf);
+int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize);
int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int);
/*
@@ -596,7 +599,7 @@ int sqlite3Fts5ExprPhraseCount(Fts5Expr*);
int sqlite3Fts5ExprPhraseSize(Fts5Expr*, int iPhrase);
int sqlite3Fts5ExprPoslist(Fts5Expr*, int, const u8 **);
-int sqlite3Fts5ExprPhraseExpr(Fts5Config*, Fts5Expr*, int, Fts5Expr**);
+int sqlite3Fts5ExprClonePhrase(Fts5Config*, Fts5Expr*, int, Fts5Expr**);
/*******************************************
** The fts5_expr.c API above this point is used by the other hand-written
diff --git a/ext/fts5/fts5_aux.c b/ext/fts5/fts5_aux.c
index 818dfcf297..2e33d5132f 100644
--- a/ext/fts5/fts5_aux.c
+++ b/ext/fts5/fts5_aux.c
@@ -148,6 +148,7 @@ static void fts5HighlightAppend(
*/
static int fts5HighlightCb(
void *pContext, /* Pointer to HighlightContext object */
+ int tflags, /* Mask of FTS5_TOKEN_* flags */
const char *pToken, /* Buffer containing token */
int nToken, /* Size of token in bytes */
int iStartOff, /* Start offset of token */
@@ -155,7 +156,10 @@ static int fts5HighlightCb(
){
HighlightContext *p = (HighlightContext*)pContext;
int rc = SQLITE_OK;
- int iPos = p->iPos++;
+ int iPos;
+
+ if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK;
+ iPos = p->iPos++;
if( p->iRangeEnd>0 ){
if( iPosiRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK;
diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c
index 7e991fc21d..74faf6dd30 100644
--- a/ext/fts5/fts5_config.c
+++ b/ext/fts5/fts5_config.c
@@ -645,12 +645,15 @@ int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){
*/
int sqlite3Fts5Tokenize(
Fts5Config *pConfig, /* FTS5 Configuration object */
+ int flags, /* FTS5_TOKENIZE_* flags */
const char *pText, int nText, /* Text to tokenize */
void *pCtx, /* Context passed to xToken() */
- int (*xToken)(void*, const char*, int, int, int) /* Callback */
+ int (*xToken)(void*, int, const char*, int, int, int) /* Callback */
){
if( pText==0 ) return SQLITE_OK;
- return pConfig->pTokApi->xTokenize(pConfig->pTok, pCtx, pText, nText, xToken);
+ return pConfig->pTokApi->xTokenize(
+ pConfig->pTok, pCtx, flags, pText, nText, xToken
+ );
}
/*
diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index a713bb7c5a..b68634694f 100644
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -22,6 +22,8 @@
*/
#define FTS5_EOF 0
+#define FTS5_LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32))
+
typedef struct Fts5ExprTerm Fts5ExprTerm;
/*
@@ -73,6 +75,7 @@ struct Fts5ExprTerm {
int bPrefix; /* True for a prefix term */
char *zTerm; /* nul-terminated term */
Fts5IndexIter *pIter; /* Iterator for this term */
+ Fts5ExprTerm *pSynonym; /* Pointer to first in list of synonyms */
};
/*
@@ -181,6 +184,10 @@ static int fts5ExprGetToken(
default: {
const char *z2;
+ if( sqlite3Fts5IsBareword(z[0])==0 ){
+ sqlite3Fts5ParseError(pParse, "fts5: syntax error near \"%.1s\"", z);
+ return FTS5_EOF;
+ }
tok = FTS5_STRING;
for(z2=&z[1]; sqlite3Fts5IsBareword(*z2); z2++);
pToken->n = (z2 - z);
@@ -244,79 +251,6 @@ int sqlite3Fts5ExprNew(
return sParse.rc;
}
-/*
-** Create a new FTS5 expression by cloning phrase iPhrase of the
-** expression passed as the second argument.
-*/
-int sqlite3Fts5ExprPhraseExpr(
- Fts5Config *pConfig,
- Fts5Expr *pExpr,
- int iPhrase,
- Fts5Expr **ppNew
-){
- int rc = SQLITE_OK; /* Return code */
- Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */
- Fts5ExprPhrase *pCopy; /* Copy of pOrig */
- Fts5Expr *pNew = 0; /* Expression to return via *ppNew */
-
- pOrig = pExpr->apExprPhrase[iPhrase];
- pCopy = (Fts5ExprPhrase*)sqlite3Fts5MallocZero(&rc,
- sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * pOrig->nTerm
- );
- if( pCopy ){
- int i; /* Used to iterate through phrase terms */
- Fts5ExprPhrase **apPhrase;
- Fts5ExprNode *pNode;
- Fts5ExprNearset *pNear;
-
- pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr));
- apPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc,
- sizeof(Fts5ExprPhrase*)
- );
- pNode = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprNode));
- pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc,
- sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)
- );
-
- for(i=0; inTerm; i++){
- pCopy->aTerm[i].zTerm = sqlite3Fts5Strndup(&rc, pOrig->aTerm[i].zTerm,-1);
- pCopy->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
- }
-
- if( rc==SQLITE_OK ){
- /* All the allocations succeeded. Put the expression object together. */
- pNew->pIndex = pExpr->pIndex;
- pNew->pRoot = pNode;
- pNew->nPhrase = 1;
- pNew->apExprPhrase = apPhrase;
- pNew->apExprPhrase[0] = pCopy;
-
- pNode->eType = (pOrig->nTerm==1 ? FTS5_TERM : FTS5_STRING);
- pNode->pNear = pNear;
-
- pNear->nPhrase = 1;
- pNear->apPhrase[0] = pCopy;
-
- pCopy->nTerm = pOrig->nTerm;
- pCopy->pNode = pNode;
- }else{
- /* At least one allocation failed. Free them all. */
- for(i=0; inTerm; i++){
- sqlite3_free(pCopy->aTerm[i].zTerm);
- }
- sqlite3_free(pCopy);
- sqlite3_free(pNear);
- sqlite3_free(pNode);
- sqlite3_free(apPhrase);
- sqlite3_free(pNew);
- pNew = 0;
- }
- }
-
- *ppNew = pNew;
- return rc;
-}
-
/*
** Free the expression node object passed as the only argument.
*/
@@ -350,6 +284,115 @@ static int fts5ExprColsetTest(Fts5ExprColset *pColset, int iCol){
return 0;
}
+/*
+** Argument pTerm must be a synonym iterator. Return the current rowid
+** that it points to.
+*/
+static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){
+ i64 iRet = 0;
+ int bRetValid = 0;
+ Fts5ExprTerm *p;
+
+ assert( pTerm->pSynonym );
+ assert( bDesc==0 || bDesc==1 );
+ for(p=pTerm; p; p=p->pSynonym){
+ if( 0==sqlite3Fts5IterEof(p->pIter) ){
+ i64 iRowid = sqlite3Fts5IterRowid(p->pIter);
+ if( bRetValid==0 || (bDesc!=(iRowidpSynonym );
+ for(p=pTerm; p; p=p->pSynonym){
+ Fts5IndexIter *pIter = p->pIter;
+ if( sqlite3Fts5IterEof(pIter)==0 && sqlite3Fts5IterRowid(pIter)==iRowid ){
+ const u8 *a;
+ int n;
+ i64 dummy;
+ rc = sqlite3Fts5IterPoslist(pIter, &a, &n, &dummy);
+ if( rc!=SQLITE_OK ) goto synonym_poslist_out;
+ if( nIter==nAlloc ){
+ int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2;
+ Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte);
+ if( aNew==0 ){
+ rc = SQLITE_NOMEM;
+ goto synonym_poslist_out;
+ }
+ memcpy(aNew, aIter, sizeof(Fts5PoslistReader) * nIter);
+ nAlloc = nAlloc*2;
+ if( aIter!=aStatic ) sqlite3_free(aIter);
+ aIter = aNew;
+ }
+ sqlite3Fts5PoslistReaderInit(-1, a, n, &aIter[nIter]);
+ assert( aIter[nIter].bEof==0 );
+ nIter++;
+ }
+ }
+
+ assert( *pbDel==0 );
+ if( nIter==1 ){
+ *pa = (u8*)aIter[0].a;
+ *pn = aIter[0].n;
+ }else{
+ Fts5PoslistWriter writer = {0};
+ Fts5Buffer buf = {0,0,0};
+ i64 iPrev = -1;
+ while( 1 ){
+ int i;
+ i64 iMin = FTS5_LARGEST_INT64;
+ for(i=0; inTerm);
/* Initialize a term iterator for each term in the phrase */
for(i=0; inTerm; i++){
+ Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
i64 dummy;
- int n;
- const u8 *a;
- rc = sqlite3Fts5IterPoslist(pPhrase->aTerm[i].pIter, &a, &n, &dummy);
- if( rc || sqlite3Fts5PoslistReaderInit(iCol, a, n, &aIter[i]) ){
- goto ismatch_out;
+ int n = 0;
+ int bFlag = 0;
+ const u8 *a = 0;
+ if( pTerm->pSynonym ){
+ rc = fts5ExprSynonymPoslist(pTerm, pNode->iRowid, &bFlag, (u8**)&a, &n);
+ }else{
+ rc = sqlite3Fts5IterPoslist(pTerm->pIter, &a, &n, &dummy);
}
+ if( rc!=SQLITE_OK ) goto ismatch_out;
+ sqlite3Fts5PoslistReaderInit(iCol, a, n, &aIter[i]);
+ aIter[i].bFlag = bFlag;
+ if( aIter[i].bEof ) goto ismatch_out;
}
while( 1 ){
@@ -431,6 +482,9 @@ static int fts5ExprPhraseIsMatch(
ismatch_out:
*pbMatch = (pPhrase->poslist.n>0);
+ for(i=0; inTerm; i++){
+ if( aIter[i].bFlag ) sqlite3_free((u8*)aIter[i].a);
+ }
if( aIter!=aStatic ) sqlite3_free(aIter);
return rc;
}
@@ -598,17 +652,55 @@ static int fts5ExprNearAdvanceFirst(
int bFromValid,
i64 iFrom
){
- Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter;
+ Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0];
int rc;
- assert( Fts5NodeIsString(pNode) );
- if( bFromValid ){
- rc = sqlite3Fts5IterNextFrom(pIter, iFrom);
+ if( pTerm->pSynonym ){
+ int bEof = 1;
+ Fts5ExprTerm *p;
+
+ /* Find the firstest rowid any synonym points to. */
+ i64 iRowid = fts5ExprSynonymRowid(pTerm, pExpr->bDesc, 0);
+
+ /* Advance each iterator that currently points to iRowid. Or, if iFrom
+ ** is valid - each iterator that points to a rowid before iFrom. */
+ for(p=pTerm; p; p=p->pSynonym){
+ if( sqlite3Fts5IterEof(p->pIter)==0 ){
+ i64 ii = sqlite3Fts5IterRowid(p->pIter);
+ if( ii==iRowid
+ || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc)
+ ){
+ if( bFromValid ){
+ rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom);
+ }else{
+ rc = sqlite3Fts5IterNext(p->pIter);
+ }
+ if( rc!=SQLITE_OK ) break;
+ if( sqlite3Fts5IterEof(p->pIter)==0 ){
+ bEof = 0;
+ }
+ }else{
+ bEof = 0;
+ }
+ }
+ }
+
+ /* Set the EOF flag if either all synonym iterators are at EOF or an
+ ** error has occurred. */
+ pNode->bEof = (rc || bEof);
}else{
- rc = sqlite3Fts5IterNext(pIter);
+ Fts5IndexIter *pIter = pTerm->pIter;
+
+ assert( Fts5NodeIsString(pNode) );
+ if( bFromValid ){
+ rc = sqlite3Fts5IterNextFrom(pIter, iFrom);
+ }else{
+ rc = sqlite3Fts5IterNext(pIter);
+ }
+
+ pNode->bEof = (rc || sqlite3Fts5IterEof(pIter));
}
- pNode->bEof = (rc || sqlite3Fts5IterEof(pIter));
return rc;
}
@@ -647,6 +739,35 @@ static int fts5ExprAdvanceto(
return 0;
}
+static int fts5ExprSynonymAdvanceto(
+ Fts5ExprTerm *pTerm, /* Term iterator to advance */
+ int bDesc, /* True if iterator is "rowid DESC" */
+ i64 *piLast, /* IN/OUT: Lastest rowid seen so far */
+ int *pRc /* OUT: Error code */
+){
+ int rc = SQLITE_OK;
+ i64 iLast = *piLast;
+ Fts5ExprTerm *p;
+ int bEof = 0;
+
+ for(p=pTerm; rc==SQLITE_OK && p; p=p->pSynonym){
+ if( sqlite3Fts5IterEof(p->pIter)==0 ){
+ i64 iRowid = sqlite3Fts5IterRowid(p->pIter);
+ if( (bDesc==0 && iLast>iRowid) || (bDesc && iLastpIter, iLast);
+ }
+ }
+ }
+
+ if( rc!=SQLITE_OK ){
+ *pRc = rc;
+ bEof = 1;
+ }else{
+ *piLast = fts5ExprSynonymRowid(pTerm, bDesc, &bEof);
+ }
+ return bEof;
+}
+
/*
** IN/OUT parameter (*pa) points to a position list n bytes in size. If
** the position list contains entries for column iCol, then (*pa) is set
@@ -717,9 +838,9 @@ static int fts5ExprNearTest(
** phrase is not a match, break out of the loop early. */
for(i=0; rc==SQLITE_OK && inPhrase; i++){
Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
- if( pPhrase->nTerm>1 || pNear->pColset ){
+ if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){
int bMatch = 0;
- rc = fts5ExprPhraseIsMatch(pExpr, pNear->pColset, pPhrase, &bMatch);
+ rc = fts5ExprPhraseIsMatch(pNode, pNear->pColset, pPhrase, &bMatch);
if( bMatch==0 ) break;
}else{
rc = sqlite3Fts5IterPoslistBuffer(
@@ -755,6 +876,7 @@ static int fts5ExprTokenTest(
assert( pNode->eType==FTS5_TERM );
assert( pNear->nPhrase==1 && pPhrase->nTerm==1 );
+ assert( pPhrase->aTerm[0].pSynonym==0 );
rc = sqlite3Fts5IterPoslist(pIter, &pPos, &nPos, &pNode->iRowid);
@@ -801,69 +923,99 @@ static int fts5ExprNearNextMatch(
i64 iLast; /* Lastest rowid any iterator points to */
int i, j; /* Phrase and token index, respectively */
int bMatch; /* True if all terms are at the same rowid */
+ const int bDesc = pExpr->bDesc;
- assert( pNear->nPhrase>1 || pNear->apPhrase[0]->nTerm>1 );
+ /* Check that this node should not be FTS5_TERM */
+ assert( pNear->nPhrase>1
+ || pNear->apPhrase[0]->nTerm>1
+ || pNear->apPhrase[0]->aTerm[0].pSynonym
+ );
/* Initialize iLast, the "lastest" rowid any iterator points to. If the
** iterator skips through rowids in the default ascending order, this means
** the maximum rowid. Or, if the iterator is "ORDER BY rowid DESC", then it
** means the minimum rowid. */
- iLast = sqlite3Fts5IterRowid(pLeft->aTerm[0].pIter);
+ if( pLeft->aTerm[0].pSynonym ){
+ iLast = fts5ExprSynonymRowid(&pLeft->aTerm[0], bDesc, 0);
+ }else{
+ iLast = sqlite3Fts5IterRowid(pLeft->aTerm[0].pIter);
+ }
do {
bMatch = 1;
for(i=0; inPhrase; i++){
Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
for(j=0; jnTerm; j++){
- Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
- i64 iRowid = sqlite3Fts5IterRowid(pIter);
- if( iRowid!=iLast ) bMatch = 0;
- if( fts5ExprAdvanceto(pIter, pExpr->bDesc, &iLast,&rc,&pNode->bEof) ){
- return rc;
+ Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
+ if( pTerm->pSynonym ){
+ Fts5ExprTerm *p;
+ int bEof = 1;
+ i64 iRowid = fts5ExprSynonymRowid(pTerm, bDesc, 0);
+ if( iRowid==iLast ) continue;
+ bMatch = 0;
+ if( fts5ExprSynonymAdvanceto(pTerm, bDesc, &iLast, &rc) ){
+ pNode->bEof = 1;
+ return rc;
+ }
+ }else{
+ Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
+ i64 iRowid = sqlite3Fts5IterRowid(pIter);
+ if( iRowid==iLast ) continue;
+ bMatch = 0;
+ if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
+ return rc;
+ }
}
}
}
}while( bMatch==0 );
- pNode->bNomatch = (0==fts5ExprNearTest(&rc, pExpr, pNode));
pNode->iRowid = iLast;
+ pNode->bNomatch = (0==fts5ExprNearTest(&rc, pExpr, pNode));
return rc;
}
/*
** Initialize all term iterators in the pNear object. If any term is found
-** to match no documents at all, set *pbEof to true and return immediately,
-** without initializing any further iterators.
+** to match no documents at all, return immediately without initializing any
+** further iterators.
*/
static int fts5ExprNearInitAll(
Fts5Expr *pExpr,
Fts5ExprNode *pNode
){
Fts5ExprNearset *pNear = pNode->pNear;
- Fts5ExprTerm *pTerm;
- Fts5ExprPhrase *pPhrase;
int i, j;
int rc = SQLITE_OK;
for(i=0; rc==SQLITE_OK && inPhrase; i++){
- pPhrase = pNear->apPhrase[i];
+ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i];
for(j=0; jnTerm; j++){
- pTerm = &pPhrase->aTerm[j];
- if( pTerm->pIter ){
- sqlite3Fts5IterClose(pTerm->pIter);
- pTerm->pIter = 0;
+ Fts5ExprTerm *pTerm = &pPhrase->aTerm[j];
+ Fts5ExprTerm *p;
+ int bEof = 1;
+
+ for(p=pTerm; p && rc==SQLITE_OK; p=p->pSynonym){
+ if( p->pIter ){
+ sqlite3Fts5IterClose(p->pIter);
+ p->pIter = 0;
+ }
+ rc = sqlite3Fts5IndexQuery(
+ pExpr->pIndex, p->zTerm, strlen(p->zTerm),
+ (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
+ (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
+ &p->pIter
+ );
+ assert( rc==SQLITE_OK || p->pIter==0 );
+ if( p->pIter && 0==sqlite3Fts5IterEof(p->pIter) ){
+ bEof = 0;
+ }
}
- rc = sqlite3Fts5IndexQuery(
- pExpr->pIndex, pTerm->zTerm, strlen(pTerm->zTerm),
- (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
- (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
- &pTerm->pIter
- );
- assert( rc==SQLITE_OK || pTerm->pIter==0 );
- if( pTerm->pIter==0 || sqlite3Fts5IterEof(pTerm->pIter) ){
+
+ if( bEof ){
pNode->bEof = 1;
- break;
+ return rc;
}
}
}
@@ -1029,10 +1181,17 @@ static int fts5ExprNodeNext(
};
case FTS5_TERM: {
- rc = fts5ExprNearAdvanceFirst(pExpr, pNode, bFromValid, iFrom);
- if( pNode->bEof==0 ){
+ Fts5IndexIter *pIter = pNode->pNear->apPhrase[0]->aTerm[0].pIter;
+ if( bFromValid ){
+ rc = sqlite3Fts5IterNextFrom(pIter, iFrom);
+ }else{
+ rc = sqlite3Fts5IterNext(pIter);
+ }
+ if( rc==SQLITE_OK && sqlite3Fts5IterEof(pIter)==0 ){
assert( rc==SQLITE_OK );
rc = fts5ExprTokenTest(pExpr, pNode);
+ }else{
+ pNode->bEof = 1;
}
return rc;
};
@@ -1266,10 +1425,16 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
if( pPhrase ){
int i;
for(i=0; inTerm; i++){
+ Fts5ExprTerm *pSyn;
+ Fts5ExprTerm *pNext;
Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
sqlite3_free(pTerm->zTerm);
- if( pTerm->pIter ){
- sqlite3Fts5IterClose(pTerm->pIter);
+ sqlite3Fts5IterClose(pTerm->pIter);
+
+ for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){
+ pNext = pSyn->pSynonym;
+ sqlite3Fts5IterClose(pSyn->pIter);
+ sqlite3_free(pSyn);
}
}
if( pPhrase->poslist.nSpace>0 ) fts5BufferFree(&pPhrase->poslist);
@@ -1331,6 +1496,7 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset(
typedef struct TokenCtx TokenCtx;
struct TokenCtx {
Fts5ExprPhrase *pPhrase;
+ int rc;
};
/*
@@ -1338,34 +1504,60 @@ struct TokenCtx {
*/
static int fts5ParseTokenize(
void *pContext, /* Pointer to Fts5InsertCtx object */
+ int tflags, /* Mask of FTS5_TOKEN_* flags */
const char *pToken, /* Buffer containing token */
int nToken, /* Size of token in bytes */
- int iStart, /* Start offset of token */
- int iEnd /* End offset of token */
+ int iUnused1, /* Start offset of token */
+ int iUnused2 /* End offset of token */
){
int rc = SQLITE_OK;
const int SZALLOC = 8;
TokenCtx *pCtx = (TokenCtx*)pContext;
Fts5ExprPhrase *pPhrase = pCtx->pPhrase;
- Fts5ExprTerm *pTerm;
- if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){
- Fts5ExprPhrase *pNew;
- int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0);
+ /* If an error has already occurred, this is a no-op */
+ if( pCtx->rc!=SQLITE_OK ) return pCtx->rc;
- pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase,
- sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew
- );
- if( pNew==0 ) return SQLITE_NOMEM;
- if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase));
- pCtx->pPhrase = pPhrase = pNew;
- pNew->nTerm = nNew - SZALLOC;
+ assert( pPhrase==0 || pPhrase->nTerm>0 );
+ if( pPhrase && (tflags & FTS5_TOKEN_COLOCATED) ){
+ Fts5ExprTerm *pSyn;
+ int nByte = sizeof(Fts5ExprTerm) + nToken+1;
+ pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte);
+ if( pSyn==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ memset(pSyn, 0, nByte);
+ pSyn->zTerm = (char*)&pSyn[1];
+ memcpy(pSyn->zTerm, pToken, nToken);
+ pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym;
+ pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn;
+ }
+ }else{
+ Fts5ExprTerm *pTerm;
+ if( pPhrase==0 || (pPhrase->nTerm % SZALLOC)==0 ){
+ Fts5ExprPhrase *pNew;
+ int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0);
+
+ pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase,
+ sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew
+ );
+ if( pNew==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ if( pPhrase==0 ) memset(pNew, 0, sizeof(Fts5ExprPhrase));
+ pCtx->pPhrase = pPhrase = pNew;
+ pNew->nTerm = nNew - SZALLOC;
+ }
+ }
+
+ if( rc==SQLITE_OK ){
+ pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
+ memset(pTerm, 0, sizeof(Fts5ExprTerm));
+ pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
+ }
}
- pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
- memset(pTerm, 0, sizeof(Fts5ExprTerm));
- pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
-
+ pCtx->rc = rc;
return rc;
}
@@ -1417,11 +1609,14 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
rc = fts5ParseStringFromToken(pToken, &z);
if( rc==SQLITE_OK ){
+ int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_QUERY : 0);
+ int n;
sqlite3Fts5Dequote(z);
- rc = sqlite3Fts5Tokenize(pConfig, z, strlen(z), &sCtx, fts5ParseTokenize);
+ n = strlen(z);
+ rc = sqlite3Fts5Tokenize(pConfig, flags, z, n, &sCtx, fts5ParseTokenize);
}
sqlite3_free(z);
- if( rc ){
+ if( rc || (rc = sCtx.rc) ){
pParse->rc = rc;
fts5ExprPhraseFree(sCtx.pPhrase);
sCtx.pPhrase = 0;
@@ -1450,6 +1645,83 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
return sCtx.pPhrase;
}
+/*
+** Create a new FTS5 expression by cloning phrase iPhrase of the
+** expression passed as the second argument.
+*/
+int sqlite3Fts5ExprClonePhrase(
+ Fts5Config *pConfig,
+ Fts5Expr *pExpr,
+ int iPhrase,
+ Fts5Expr **ppNew
+){
+ int rc = SQLITE_OK; /* Return code */
+ Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */
+ Fts5ExprPhrase *pCopy; /* Copy of pOrig */
+ int i; /* Used to iterate through phrase terms */
+
+ Fts5Expr *pNew = 0; /* Expression to return via *ppNew */
+ Fts5ExprPhrase **apPhrase; /* pNew->apPhrase */
+ Fts5ExprNode *pNode; /* pNew->pRoot */
+ Fts5ExprNearset *pNear; /* pNew->pRoot->pNear */
+
+ TokenCtx sCtx = {0,0}; /* Context object for fts5ParseTokenize */
+
+
+ pOrig = pExpr->apExprPhrase[iPhrase];
+
+ pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr));
+ if( rc==SQLITE_OK ){
+ pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc,
+ sizeof(Fts5ExprPhrase*));
+ }
+ if( rc==SQLITE_OK ){
+ pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc,
+ sizeof(Fts5ExprNode));
+ }
+ if( rc==SQLITE_OK ){
+ pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc,
+ sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*));
+ }
+
+ for(i=0; rc==SQLITE_OK && inTerm; i++){
+ int tflags = 0;
+ Fts5ExprTerm *p;
+ for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){
+ const char *zTerm = p->zTerm;
+ rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, strlen(zTerm), 0, 0);
+ tflags = FTS5_TOKEN_COLOCATED;
+ }
+ if( rc==SQLITE_OK ){
+ sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
+ }
+ }
+
+ if( rc==SQLITE_OK ){
+ /* All the allocations succeeded. Put the expression object together. */
+ pNew->pIndex = pExpr->pIndex;
+ pNew->nPhrase = 1;
+ pNew->apExprPhrase[0] = sCtx.pPhrase;
+ pNew->pRoot->pNear->apPhrase[0] = sCtx.pPhrase;
+ pNew->pRoot->pNear->nPhrase = 1;
+ sCtx.pPhrase->pNode = pNew->pRoot;
+
+ if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){
+ pNew->pRoot->eType = FTS5_TERM;
+ }else{
+ pNew->pRoot->eType = FTS5_STRING;
+ }
+ }else{
+ sqlite3Fts5ExprFree(pNew);
+ fts5ExprPhraseFree(sCtx.pPhrase);
+ pNew = 0;
+ }
+
+ *ppNew = pNew;
+ return rc;
+}
+
+
/*
** Token pTok has appeared in a MATCH expression where the NEAR operator
** is expected. If token pTok does not contain "NEAR", store an error
@@ -1630,7 +1902,10 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
for(iPhrase=0; iPhrasenPhrase; iPhrase++){
pNear->apPhrase[iPhrase]->pNode = pRet;
}
- if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 ){
+ if( pNear->nPhrase==1
+ && pNear->apPhrase[0]->nTerm==1
+ && pNear->apPhrase[0]->aTerm[0].pSynonym==0
+ ){
pRet->eType = FTS5_TERM;
}
}else{
@@ -1650,16 +1925,28 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
}
static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
- char *zQuoted = sqlite3_malloc(strlen(pTerm->zTerm) * 2 + 3 + 2);
+ int nByte = 0;
+ Fts5ExprTerm *p;
+ char *zQuoted;
+
+ /* Determine the maximum amount of space required. */
+ for(p=pTerm; p; p=p->pSynonym){
+ nByte += strlen(pTerm->zTerm) * 2 + 3 + 2;
+ }
+ zQuoted = sqlite3_malloc(nByte);
+
if( zQuoted ){
int i = 0;
- char *zIn = pTerm->zTerm;
- zQuoted[i++] = '"';
- while( *zIn ){
- if( *zIn=='"' ) zQuoted[i++] = '"';
- zQuoted[i++] = *zIn++;
+ for(p=pTerm; p; p=p->pSynonym){
+ char *zIn = p->zTerm;
+ zQuoted[i++] = '"';
+ while( *zIn ){
+ if( *zIn=='"' ) zQuoted[i++] = '"';
+ zQuoted[i++] = *zIn++;
+ }
+ zQuoted[i++] = '"';
+ if( p->pSynonym ) zQuoted[i++] = '|';
}
- zQuoted[i++] = '"';
if( pTerm->bPrefix ){
zQuoted[i++] = ' ';
zQuoted[i++] = '*';
diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c
index a7ab7c4c05..7bb744773c 100644
--- a/ext/fts5/fts5_index.c
+++ b/ext/fts5/fts5_index.c
@@ -293,7 +293,6 @@ typedef struct Fts5Data Fts5Data;
typedef struct Fts5DlidxIter Fts5DlidxIter;
typedef struct Fts5DlidxLvl Fts5DlidxLvl;
typedef struct Fts5DlidxWriter Fts5DlidxWriter;
-typedef struct Fts5NodeIter Fts5NodeIter;
typedef struct Fts5PageWriter Fts5PageWriter;
typedef struct Fts5SegIter Fts5SegIter;
typedef struct Fts5DoclistIter Fts5DoclistIter;
@@ -526,24 +525,6 @@ struct Fts5IndexIter {
};
-/*
-** Object for iterating through the conents of a single internal node in
-** memory.
-*/
-struct Fts5NodeIter {
- /* Internal. Set and managed by fts5NodeIterXXX() functions. Except,
- ** the EOF test for the iterator is (Fts5NodeIter.aData==0). */
- const u8 *aData;
- int nData;
- int iOff;
-
- /* Output variables */
- Fts5Buffer term;
- int nEmpty;
- int iChild;
- int bDlidx;
-};
-
/*
** An instance of the following type is used to iterate through the contents
** of a doclist-index record.
@@ -573,23 +554,6 @@ struct Fts5DlidxIter {
Fts5DlidxLvl aLvl[1];
};
-
-
-/*
-** The first argument passed to this macro is a pointer to an Fts5Buffer
-** object.
-*/
-#define fts5BufferSize(pBuf,n) { \
- if( pBuf->nSpacep, n); \
- if( pNew==0 ){ \
- sqlite3_free(pBuf->p); \
- } \
- pBuf->nSpace = n; \
- pBuf->p = pNew; \
- } \
-}
-
static void fts5PutU16(u8 *aOut, u16 iVal){
aOut[0] = (iVal>>8);
aOut[1] = (iVal&0xFF);
@@ -617,6 +581,7 @@ static void *fts5IdxMalloc(Fts5Index *p, int nByte){
**
** res = *pLeft - *pRight
*/
+#ifdef SQLITE_DEBUG
static int fts5BufferCompareBlob(
Fts5Buffer *pLeft, /* Left hand side of comparison */
const u8 *pRight, int nRight /* Right hand side of comparison */
@@ -625,7 +590,7 @@ static int fts5BufferCompareBlob(
int res = memcmp(pLeft->p, pRight, nCmp);
return (res==0 ? (pLeft->n - nRight) : res);
}
-
+#endif
/*
** Compare the contents of the two buffers using memcmp(). If one buffer
@@ -665,11 +630,14 @@ static void fts5CloseReader(Fts5Index *p){
}
}
-static Fts5Data *fts5DataReadOrBuffer(
- Fts5Index *p,
- Fts5Buffer *pBuf,
- i64 iRowid
-){
+
+/*
+** Retrieve a record from the %_data table.
+**
+** If an error occurs, NULL is returned and an error left in the
+** Fts5Index object.
+*/
+static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){
Fts5Data *pRet = 0;
if( p->rc==SQLITE_OK ){
int rc = SQLITE_OK;
@@ -689,8 +657,8 @@ static Fts5Data *fts5DataReadOrBuffer(
if( rc==SQLITE_ABORT ) rc = SQLITE_OK;
}
- /* If the blob handle is not yet open, open and seek it. Otherwise, use
- ** the blob_reopen() API to reseek the existing blob handle. */
+ /* If the blob handle is not open at this point, open it and seek
+ ** to the requested entry. */
if( p->pReader==0 && rc==SQLITE_OK ){
Fts5Config *pConfig = p->pConfig;
rc = sqlite3_blob_open(pConfig->db,
@@ -708,22 +676,13 @@ static Fts5Data *fts5DataReadOrBuffer(
if( rc==SQLITE_OK ){
u8 *aOut = 0; /* Read blob data into this buffer */
int nByte = sqlite3_blob_bytes(p->pReader);
- if( pBuf ){
- fts5BufferSize(pBuf, MAX(nByte, p->pConfig->pgsz) + 20);
- pBuf->n = nByte;
- aOut = pBuf->p;
- if( aOut==0 ){
- rc = SQLITE_NOMEM;
- }
+ int nAlloc = sizeof(Fts5Data) + nByte + FTS5_DATA_PADDING;
+ pRet = (Fts5Data*)sqlite3_malloc(nAlloc);
+ if( pRet ){
+ pRet->n = nByte;
+ aOut = pRet->p = (u8*)&pRet[1];
}else{
- int nSpace = nByte + FTS5_DATA_PADDING;
- pRet = (Fts5Data*)sqlite3_malloc(nSpace+sizeof(Fts5Data));
- if( pRet ){
- pRet->n = nByte;
- aOut = pRet->p = (u8*)&pRet[1];
- }else{
- rc = SQLITE_NOMEM;
- }
+ rc = SQLITE_NOMEM;
}
if( rc==SQLITE_OK ){
@@ -738,33 +697,10 @@ static Fts5Data *fts5DataReadOrBuffer(
p->nRead++;
}
- return pRet;
-}
-
-/*
-** Retrieve a record from the %_data table.
-**
-** If an error occurs, NULL is returned and an error left in the
-** Fts5Index object.
-*/
-static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){
- Fts5Data *pRet = fts5DataReadOrBuffer(p, 0, iRowid);
assert( (pRet==0)==(p->rc!=SQLITE_OK) );
return pRet;
}
-/*
-** Read a record from the %_data table into the buffer supplied as the
-** second argument.
-**
-** If an error occurs, an error is left in the Fts5Index object. If an
-** error has already occurred when this function is called, it is a
-** no-op.
-*/
-static void fts5DataBuffer(Fts5Index *p, Fts5Buffer *pBuf, i64 iRowid){
- (void)fts5DataReadOrBuffer(p, pBuf, iRowid);
-}
-
/*
** Release a reference to data record returned by an earlier call to
** fts5DataRead().
@@ -1033,19 +969,18 @@ static Fts5Structure *fts5StructureRead(Fts5Index *p){
Fts5Config *pConfig = p->pConfig;
Fts5Structure *pRet = 0; /* Object to return */
int iCookie; /* Configuration cookie */
+ Fts5Data *pData;
Fts5Buffer buf = {0, 0, 0};
- fts5DataBuffer(p, &buf, FTS5_STRUCTURE_ROWID);
- if( buf.p==0 ) return 0;
- assert( buf.nSpace>=(buf.n + FTS5_DATA_ZERO_PADDING) );
- memset(&buf.p[buf.n], 0, FTS5_DATA_ZERO_PADDING);
- p->rc = fts5StructureDecode(buf.p, buf.n, &iCookie, &pRet);
-
+ pData = fts5DataRead(p, FTS5_STRUCTURE_ROWID);
+ if( p->rc ) return 0;
+ memset(&pData->p[pData->n], 0, FTS5_DATA_PADDING);
+ p->rc = fts5StructureDecode(pData->p, pData->n, &iCookie, &pRet);
if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){
p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie);
}
- fts5BufferFree(&buf);
+ fts5DataRelease(pData);
if( p->rc!=SQLITE_OK ){
fts5StructureRelease(pRet);
pRet = 0;
@@ -1228,62 +1163,6 @@ static void fts5StructurePromote(
}
-/*
-** If the pIter->iOff offset currently points to an entry indicating one
-** or more term-less nodes, advance past it and set pIter->nEmpty to
-** the number of empty child nodes.
-*/
-static void fts5NodeIterGobbleNEmpty(Fts5NodeIter *pIter){
- if( pIter->iOffnData && 0==(pIter->aData[pIter->iOff] & 0xfe) ){
- pIter->bDlidx = pIter->aData[pIter->iOff] & 0x01;
- pIter->iOff++;
- pIter->iOff += fts5GetVarint32(&pIter->aData[pIter->iOff], pIter->nEmpty);
- }else{
- pIter->nEmpty = 0;
- pIter->bDlidx = 0;
- }
-}
-
-/*
-** Advance to the next entry within the node.
-*/
-static void fts5NodeIterNext(int *pRc, Fts5NodeIter *pIter){
- if( pIter->iOff>=pIter->nData ){
- pIter->aData = 0;
- pIter->iChild += pIter->nEmpty;
- }else{
- int nPre, nNew;
- pIter->iOff += fts5GetVarint32(&pIter->aData[pIter->iOff], nPre);
- pIter->iOff += fts5GetVarint32(&pIter->aData[pIter->iOff], nNew);
- pIter->term.n = nPre-2;
- fts5BufferAppendBlob(pRc, &pIter->term, nNew, pIter->aData+pIter->iOff);
- pIter->iOff += nNew;
- pIter->iChild += (1 + pIter->nEmpty);
- fts5NodeIterGobbleNEmpty(pIter);
- if( *pRc ) pIter->aData = 0;
- }
-}
-
-
-/*
-** Initialize the iterator object pIter to iterate through the internal
-** segment node in pData.
-*/
-static void fts5NodeIterInit(const u8 *aData, int nData, Fts5NodeIter *pIter){
- memset(pIter, 0, sizeof(*pIter));
- pIter->aData = aData;
- pIter->nData = nData;
- pIter->iOff = fts5GetVarint32(aData, pIter->iChild);
- fts5NodeIterGobbleNEmpty(pIter);
-}
-
-/*
-** Free any memory allocated by the iterator object.
-*/
-static void fts5NodeIterFree(Fts5NodeIter *pIter){
- fts5BufferFree(&pIter->term);
-}
-
/*
** Advance the iterator passed as the only argument. If the end of the
** doclist-index page is reached, return non-zero.
@@ -2041,119 +1920,6 @@ static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){
pIter->pDlidx = fts5DlidxIterInit(p, bRev, iSeg, pIter->iTermLeafPgno);
}
-#ifdef SQLITE_DEBUG
-static void fts5AssertNodeSeekOk(
- Fts5Buffer *pNode,
- const u8 *pTerm, int nTerm, /* Term to search for */
- int iExpectPg,
- int bExpectDlidx
-){
- int bDlidx;
- int iPg;
- int rc = SQLITE_OK;
- Fts5NodeIter node;
-
- fts5NodeIterInit(pNode->p, pNode->n, &node);
- assert( node.term.n==0 );
- iPg = node.iChild;
- bDlidx = node.bDlidx;
- for(fts5NodeIterNext(&rc, &node);
- node.aData && fts5BufferCompareBlob(&node.term, pTerm, nTerm)<=0;
- fts5NodeIterNext(&rc, &node)
- ){
- iPg = node.iChild;
- bDlidx = node.bDlidx;
- }
- fts5NodeIterFree(&node);
-
- assert( rc!=SQLITE_OK || iPg==iExpectPg );
- assert( rc!=SQLITE_OK || bDlidx==bExpectDlidx );
-}
-#else
-#define fts5AssertNodeSeekOk(v,w,x,y,z)
-#endif
-
-/*
-** Argument pNode is an internal b-tree node. This function searches
-** within the node for the largest term that is smaller than or equal
-** to (pTerm/nTerm).
-**
-** It returns the associated page number. Or, if (pTerm/nTerm) is smaller
-** than all terms within the node, the leftmost child page number.
-**
-** Before returning, (*pbDlidx) is set to true if the last term on the
-** returned child page number has a doclist-index. Or left as is otherwise.
-*/
-static int fts5NodeSeek(
- Fts5Buffer *pNode, /* Node to search */
- const u8 *pTerm, int nTerm, /* Term to search for */
- int *pbDlidx /* OUT: True if dlidx flag is set */
-){
- int iPg;
- u8 *pPtr = pNode->p;
- u8 *pEnd = &pPtr[pNode->n];
- int nMatch = 0; /* Number of bytes of pTerm already matched */
-
- assert( *pbDlidx==0 );
-
- pPtr += fts5GetVarint32(pPtr, iPg);
- while( pPtr=pEnd ) break;
- }
-
- /* Read the next "term" pointer. Set nKeep to the number of bytes to
- ** keep from the previous term, and nNew to the number of bytes of
- ** new data that will be appended to it. */
- nKeep = (int)*pPtr++;
- nNew = (int)*pPtr++;
- if( (nKeep | nNew) & 0x0080 ){
- pPtr -= 2;
- pPtr += fts5GetVarint32(pPtr, nKeep);
- pPtr += fts5GetVarint32(pPtr, nNew);
- }
- nKeep -= 2;
-
- /* Compare (pTerm/nTerm) to the current term on the node (the one described
- ** by nKeep/nNew). If the node term is larger, break out of the while()
- ** loop.
- **
- ** Otherwise, if (pTerm/nTerm) is larger or the two terms are equal,
- ** leave variable nMatch set to the size of the largest prefix common to
- ** both terms in bytes. */
- if( nKeep==nMatch ){
- int nTst = MIN(nNew, nTerm-nMatch);
- int i;
- for(i=0; i pTerm[nMatch]) ) break;
- }else if( nKeeprc==SQLITE_OK ){
+ do{
if( bMove ) fts5SegIterNext(p, pIter, 0);
if( pIter->pLeaf==0 ) break;
if( bRev==0 && pIter->iRowid>=iMatch ) break;
if( bRev!=0 && pIter->iRowid<=iMatch ) break;
bMove = 1;
- }
+ }while( p->rc==SQLITE_OK );
}
@@ -4459,13 +4225,9 @@ int sqlite3Fts5IndexRollback(Fts5Index *p){
*/
int sqlite3Fts5IndexReinit(Fts5Index *p){
Fts5Structure s;
-
- assert( p->rc==SQLITE_OK );
- p->rc = sqlite3Fts5IndexSetAverages(p, (const u8*)"", 0);
-
memset(&s, 0, sizeof(Fts5Structure));
+ fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0);
fts5StructureWrite(p, &s);
-
return fts5IndexReturn(p);
}
@@ -4787,13 +4549,28 @@ void sqlite3Fts5IterClose(Fts5IndexIter *pIter){
}
/*
-** Read the "averages" record into the buffer supplied as the second
-** argument. Return SQLITE_OK if successful, or an SQLite error code
-** if an error occurs.
+** Read and decode the "averages" record from the database.
+**
+** Parameter anSize must point to an array of size nCol, where nCol is
+** the number of user defined columns in the FTS table.
*/
-int sqlite3Fts5IndexGetAverages(Fts5Index *p, Fts5Buffer *pBuf){
- assert( p->rc==SQLITE_OK );
- fts5DataReadOrBuffer(p, pBuf, FTS5_AVERAGES_ROWID);
+int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize){
+ int nCol = p->pConfig->nCol;
+ Fts5Data *pData;
+
+ *pnRow = 0;
+ memset(anSize, 0, sizeof(i64) * nCol);
+ pData = fts5DataRead(p, FTS5_AVERAGES_ROWID);
+ if( p->rc==SQLITE_OK && pData->n ){
+ int i = 0;
+ int iCol;
+ i += fts5GetVarint(&pData->p[i], (u64*)pnRow);
+ for(iCol=0; in && iColp[i], (u64*)&anSize[iCol]);
+ }
+ }
+
+ fts5DataRelease(pData);
return fts5IndexReturn(p);
}
@@ -5487,73 +5264,53 @@ static void fts5DecodeFunction(
fts5DecodeStructure(&rc, &s, a, n);
}
}else{
-
Fts5Buffer term;
+ int iTermOff = 0;
+ int iRowidOff = 0;
+ int iOff;
+ int nKeep = 0;
+
memset(&term, 0, sizeof(Fts5Buffer));
- if( iHeight==0 ){
- int iTermOff = 0;
- int iRowidOff = 0;
- int iOff;
- int nKeep = 0;
-
- if( n>=4 ){
- iRowidOff = fts5GetU16(&a[0]);
- iTermOff = fts5GetU16(&a[2]);
- }else{
- sqlite3Fts5BufferSet(&rc, &s, 8, (const u8*)"corrupt");
- goto decode_out;
- }
-
- if( iRowidOff ){
- iOff = iRowidOff;
- }else if( iTermOff ){
- iOff = iTermOff;
- }else{
- iOff = n;
- }
- fts5DecodePoslist(&rc, &s, &a[4], iOff-4);
-
- assert( iRowidOff==0 || iOff==iRowidOff );
- if( iRowidOff ){
- iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], n-iOff);
- }
-
- assert( iTermOff==0 || iOff==iTermOff );
- while( iOff=4 ){
+ iRowidOff = fts5GetU16(&a[0]);
+ iTermOff = fts5GetU16(&a[2]);
}else{
- Fts5NodeIter ss;
- for(fts5NodeIterInit(a, n, &ss); ss.aData; fts5NodeIterNext(&rc, &ss)){
- if( ss.term.n==0 ){
- sqlite3Fts5BufferAppendPrintf(&rc, &s, " left=%d", ss.iChild);
- }else{
- sqlite3Fts5BufferAppendPrintf(&rc,&s, " \"%.*s\"",
- ss.term.n, ss.term.p
- );
- }
- if( ss.nEmpty ){
- sqlite3Fts5BufferAppendPrintf(&rc, &s, " empty=%d%s", ss.nEmpty,
- ss.bDlidx ? "*" : ""
- );
- }
- }
- fts5NodeIterFree(&ss);
+ sqlite3Fts5BufferSet(&rc, &s, 8, (const u8*)"corrupt");
+ goto decode_out;
}
+
+ if( iRowidOff ){
+ iOff = iRowidOff;
+ }else if( iTermOff ){
+ iOff = iTermOff;
+ }else{
+ iOff = n;
+ }
+ fts5DecodePoslist(&rc, &s, &a[4], iOff-4);
+
+ assert( iRowidOff==0 || iOff==iRowidOff );
+ if( iRowidOff ){
+ iOff += fts5DecodeDoclist(&rc, &s, &a[iOff], n-iOff);
+ }
+
+ assert( iTermOff==0 || iOff==iTermOff );
+ while( iOffbase.pVtab);
- return sqlite3Fts5Tokenize(pTab->pConfig, pText, nText, pUserData, xToken);
+ return sqlite3Fts5Tokenize(
+ pTab->pConfig, FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken
+ );
}
static int fts5ApiPhraseCount(Fts5Context *pCtx){
@@ -1655,13 +1657,16 @@ static int fts5ApiColumnText(
static int fts5ColumnSizeCb(
void *pContext, /* Pointer to int */
+ int tflags,
const char *pToken, /* Buffer containing token */
int nToken, /* Size of token in bytes */
int iStart, /* Start offset of token */
int iEnd /* End offset of token */
){
int *pCnt = (int*)pContext;
- *pCnt = *pCnt + 1;
+ if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){
+ (*pCnt)++;
+ }
return SQLITE_OK;
}
@@ -1691,7 +1696,9 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){
pCsr->aColumnSize[i] = 0;
rc = fts5ApiColumnText(pCtx, i, &z, &n);
if( rc==SQLITE_OK ){
- rc = sqlite3Fts5Tokenize(pConfig, z, n, p, fts5ColumnSizeCb);
+ rc = sqlite3Fts5Tokenize(
+ pConfig, FTS5_TOKENIZE_AUX, z, n, p, fts5ColumnSizeCb
+ );
}
}
}
@@ -1853,7 +1860,7 @@ static int fts5ApiQueryPhrase(
pNew->iFirstRowid = SMALLEST_INT64;
pNew->iLastRowid = LARGEST_INT64;
pNew->base.pVtab = (sqlite3_vtab*)pTab;
- rc = sqlite3Fts5ExprPhraseExpr(pConf, pCsr->pExpr, iPhrase, &pNew->pExpr);
+ rc = sqlite3Fts5ExprClonePhrase(pConf, pCsr->pExpr, iPhrase, &pNew->pExpr);
}
if( rc==SQLITE_OK ){
@@ -2344,7 +2351,7 @@ int sqlite3_fts5_init(
void *p = (void*)pGlobal;
memset(pGlobal, 0, sizeof(Fts5Global));
pGlobal->db = db;
- pGlobal->api.iVersion = 1;
+ pGlobal->api.iVersion = 2;
pGlobal->api.xCreateFunction = fts5CreateAux;
pGlobal->api.xCreateTokenizer = fts5CreateTokenizer;
pGlobal->api.xFindTokenizer = fts5FindTokenizer;
diff --git a/ext/fts5/fts5_storage.c b/ext/fts5/fts5_storage.c
index f09b7d9158..65c6d48a36 100644
--- a/ext/fts5/fts5_storage.c
+++ b/ext/fts5/fts5_storage.c
@@ -359,6 +359,7 @@ struct Fts5InsertCtx {
*/
static int fts5StorageInsertCallback(
void *pContext, /* Pointer to Fts5InsertCtx object */
+ int tflags,
const char *pToken, /* Buffer containing token */
int nToken, /* Size of token in bytes */
int iStart, /* Start offset of token */
@@ -366,8 +367,10 @@ static int fts5StorageInsertCallback(
){
Fts5InsertCtx *pCtx = (Fts5InsertCtx*)pContext;
Fts5Index *pIdx = pCtx->pStorage->pIndex;
- int iPos = pCtx->szCol++;
- return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, iPos, pToken, nToken);
+ if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
+ pCtx->szCol++;
+ }
+ return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken);
}
/*
@@ -394,6 +397,7 @@ static int fts5StorageDeleteFromIndex(Fts5Storage *p, i64 iDel){
if( pConfig->abUnindexed[iCol-1] ) continue;
ctx.szCol = 0;
rc = sqlite3Fts5Tokenize(pConfig,
+ FTS5_TOKENIZE_DOCUMENT,
(const char*)sqlite3_column_text(pSeek, iCol),
sqlite3_column_bytes(pSeek, iCol),
(void*)&ctx,
@@ -451,22 +455,7 @@ static int fts5StorageInsertDocsize(
static int fts5StorageLoadTotals(Fts5Storage *p, int bCache){
int rc = SQLITE_OK;
if( p->bTotalsValid==0 ){
- int nCol = p->pConfig->nCol;
- Fts5Buffer buf;
- memset(&buf, 0, sizeof(buf));
-
- memset(p->aTotalSize, 0, sizeof(i64) * nCol);
- p->nTotalRow = 0;
- rc = sqlite3Fts5IndexGetAverages(p->pIndex, &buf);
- if( rc==SQLITE_OK && buf.n ){
- int i = 0;
- int iCol;
- i += fts5GetVarint(&buf.p[i], (u64*)&p->nTotalRow);
- for(iCol=0; iaTotalSize[iCol]);
- }
- }
- sqlite3_free(buf.p);
+ rc = sqlite3Fts5IndexGetAverages(p->pIndex, &p->nTotalRow, p->aTotalSize);
p->bTotalsValid = bCache;
}
return rc;
@@ -565,6 +554,7 @@ int sqlite3Fts5StorageSpecialDelete(
if( pConfig->abUnindexed[iCol] ) continue;
ctx.szCol = 0;
rc = sqlite3Fts5Tokenize(pConfig,
+ FTS5_TOKENIZE_DOCUMENT,
(const char*)sqlite3_value_text(apVal[iCol]),
sqlite3_value_bytes(apVal[iCol]),
(void*)&ctx,
@@ -654,6 +644,7 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){
ctx.szCol = 0;
if( pConfig->abUnindexed[ctx.iCol]==0 ){
rc = sqlite3Fts5Tokenize(pConfig,
+ FTS5_TOKENIZE_DOCUMENT,
(const char*)sqlite3_column_text(pScan, ctx.iCol+1),
sqlite3_column_bytes(pScan, ctx.iCol+1),
(void*)&ctx,
@@ -771,6 +762,7 @@ int sqlite3Fts5StorageInsert(
ctx.szCol = 0;
if( pConfig->abUnindexed[ctx.iCol]==0 ){
rc = sqlite3Fts5Tokenize(pConfig,
+ FTS5_TOKENIZE_DOCUMENT,
(const char*)sqlite3_value_text(apVal[ctx.iCol+2]),
sqlite3_value_bytes(apVal[ctx.iCol+2]),
(void*)&ctx,
@@ -838,15 +830,18 @@ struct Fts5IntegrityCtx {
*/
static int fts5StorageIntegrityCallback(
void *pContext, /* Pointer to Fts5InsertCtx object */
+ int tflags,
const char *pToken, /* Buffer containing token */
int nToken, /* Size of token in bytes */
int iStart, /* Start offset of token */
int iEnd /* End offset of token */
){
Fts5IntegrityCtx *pCtx = (Fts5IntegrityCtx*)pContext;
- int iPos = pCtx->szCol++;
+ if( (tflags & FTS5_TOKEN_COLOCATED)==0 || pCtx->szCol==0 ){
+ pCtx->szCol++;
+ }
pCtx->cksum ^= sqlite3Fts5IndexCksum(
- pCtx->pConfig, pCtx->iRowid, pCtx->iCol, iPos, pToken, nToken
+ pCtx->pConfig, pCtx->iRowid, pCtx->iCol, pCtx->szCol-1, pToken, nToken
);
return SQLITE_OK;
}
@@ -881,19 +876,23 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
int i;
ctx.iRowid = sqlite3_column_int64(pScan, 0);
ctx.szCol = 0;
- rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
+ if( pConfig->bColumnsize ){
+ rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize);
+ }
for(i=0; rc==SQLITE_OK && inCol; i++){
if( pConfig->abUnindexed[i] ) continue;
ctx.iCol = i;
ctx.szCol = 0;
- rc = sqlite3Fts5Tokenize(
- pConfig,
+ rc = sqlite3Fts5Tokenize(pConfig,
+ FTS5_TOKENIZE_DOCUMENT,
(const char*)sqlite3_column_text(pScan, i+1),
sqlite3_column_bytes(pScan, i+1),
(void*)&ctx,
fts5StorageIntegrityCallback
);
- if( ctx.szCol!=aColSize[i] ) rc = FTS5_CORRUPT;
+ if( pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
+ rc = FTS5_CORRUPT;
+ }
aTotalSize[i] += ctx.szCol;
}
if( rc!=SQLITE_OK ) break;
@@ -918,7 +917,7 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p){
rc = fts5StorageCount(p, "content", &nRow);
if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT;
}
- if( rc==SQLITE_OK ){
+ if( rc==SQLITE_OK && pConfig->bColumnsize ){
i64 nRow;
rc = fts5StorageCount(p, "docsize", &nRow);
if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT;
@@ -1002,9 +1001,12 @@ static int fts5StorageDecodeSizeArray(
** otherwise.
*/
int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){
- int nCol = p->pConfig->nCol;
- sqlite3_stmt *pLookup = 0;
- int rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
+ int nCol = p->pConfig->nCol; /* Number of user columns in table */
+ sqlite3_stmt *pLookup = 0; /* Statement to query %_docsize */
+ int rc; /* Return Code */
+
+ assert( p->pConfig->bColumnsize );
+ rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
if( rc==SQLITE_OK ){
int bCorrupt = 1;
sqlite3_bind_int64(pLookup, 1, iRowid);
diff --git a/ext/fts5/fts5_tcl.c b/ext/fts5/fts5_tcl.c
index 82f3e0390d..21ce5f7bde 100644
--- a/ext/fts5/fts5_tcl.c
+++ b/ext/fts5/fts5_tcl.c
@@ -141,6 +141,7 @@ struct F5tAuxData {
static int xTokenizeCb(
void *pCtx,
+ int tflags,
const char *zToken, int nToken,
int iStart, int iEnd
){
@@ -584,6 +585,7 @@ struct F5tTokenizeCtx {
static int xTokenizeCb2(
void *pCtx,
+ int tflags,
const char *zToken, int nToken,
int iStart, int iEnd
){
@@ -666,7 +668,9 @@ static int f5tTokenize(
ctx.bSubst = (objc==5);
ctx.pRet = pRet;
ctx.zInput = zText;
- rc = tokenizer.xTokenize(pTok, (void*)&ctx, zText, nText, xTokenizeCb2);
+ rc = tokenizer.xTokenize(
+ pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, zText, nText, xTokenizeCb2
+ );
tokenizer.xDelete(pTok);
if( rc!=SQLITE_OK ){
Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", 0);
@@ -688,11 +692,11 @@ static int f5tTokenize(
typedef struct F5tTokenizerContext F5tTokenizerContext;
typedef struct F5tTokenizerCb F5tTokenizerCb;
typedef struct F5tTokenizerModule F5tTokenizerModule;
-typedef struct F5tTokenizerModule F5tTokenizerInstance;
+typedef struct F5tTokenizerInstance F5tTokenizerInstance;
struct F5tTokenizerContext {
void *pCtx;
- int (*xToken)(void*, const char*, int, int, int);
+ int (*xToken)(void*, int, const char*, int, int, int);
};
struct F5tTokenizerModule {
@@ -701,6 +705,12 @@ struct F5tTokenizerModule {
F5tTokenizerContext *pContext;
};
+struct F5tTokenizerInstance {
+ Tcl_Interp *interp;
+ Tcl_Obj *pScript;
+ F5tTokenizerContext *pContext;
+};
+
static int f5tTokenizerCreate(
void *pCtx,
const char **azArg,
@@ -748,26 +758,53 @@ static void f5tTokenizerDelete(Fts5Tokenizer *p){
static int f5tTokenizerTokenize(
Fts5Tokenizer *p,
void *pCtx,
+ int flags,
const char *pText, int nText,
- int (*xToken)(void*, const char*, int, int, int)
+ int (*xToken)(void*, int, const char*, int, int, int)
){
F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
void *pOldCtx;
- int (*xOldToken)(void*, const char*, int, int, int);
+ int (*xOldToken)(void*, int, const char*, int, int, int);
Tcl_Obj *pEval;
int rc;
+ const char *zFlags;
pOldCtx = pInst->pContext->pCtx;
xOldToken = pInst->pContext->xToken;
+ pInst->pContext->pCtx = pCtx;
+ pInst->pContext->xToken = xToken;
+
+ assert(
+ flags==FTS5_TOKENIZE_DOCUMENT
+ || flags==FTS5_TOKENIZE_AUX
+ || flags==FTS5_TOKENIZE_QUERY
+ || flags==(FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX)
+ );
pEval = Tcl_DuplicateObj(pInst->pScript);
Tcl_IncrRefCount(pEval);
- rc = Tcl_ListObjAppendElement(
- pInst->interp, pEval, Tcl_NewStringObj(pText, nText)
- );
- if( rc==TCL_OK ){
- rc = Tcl_EvalObjEx(pInst->interp, pEval, TCL_GLOBAL_ONLY);
+ switch( flags ){
+ case FTS5_TOKENIZE_DOCUMENT:
+ zFlags = "document";
+ break;
+ case FTS5_TOKENIZE_AUX:
+ zFlags = "aux";
+ break;
+ case FTS5_TOKENIZE_QUERY:
+ zFlags = "query";
+ break;
+ case (FTS5_TOKENIZE_PREFIX | FTS5_TOKENIZE_QUERY):
+ zFlags = "prefixquery";
+ break;
+ default:
+ assert( 0 );
+ zFlags = "invalid";
+ break;
}
+
+ Tcl_ListObjAppendElement(pInst->interp, pEval, Tcl_NewStringObj(zFlags, -1));
+ Tcl_ListObjAppendElement(pInst->interp, pEval, Tcl_NewStringObj(pText,nText));
+ rc = Tcl_EvalObjEx(pInst->interp, pEval, TCL_GLOBAL_ONLY);
Tcl_DecrRefCount(pEval);
pInst->pContext->pCtx = pOldCtx;
@@ -776,7 +813,7 @@ static int f5tTokenizerTokenize(
}
/*
-** sqlite3_fts5_token TEXT START END POS
+** sqlite3_fts5_token ?-colocated? TEXT START END
*/
static int f5tTokenizerReturn(
void * clientData,
@@ -788,14 +825,29 @@ static int f5tTokenizerReturn(
int iStart;
int iEnd;
int nToken;
+ int tflags = 0;
char *zToken;
int rc;
- assert( p );
- if( objc!=4 ){
- Tcl_WrongNumArgs(interp, 1, objv, "TEXT START END");
+ if( objc==5 ){
+ int nArg;
+ char *zArg = Tcl_GetStringFromObj(objv[1], &nArg);
+ if( nArg<=10 && nArg>=2 && memcmp("-colocated", zArg, nArg)==0 ){
+ tflags |= FTS5_TOKEN_COLOCATED;
+ }else{
+ goto usage;
+ }
+ }else if( objc!=4 ){
+ goto usage;
+ }
+
+ zToken = Tcl_GetStringFromObj(objv[objc-3], &nToken);
+ if( Tcl_GetIntFromObj(interp, objv[objc-2], &iStart)
+ || Tcl_GetIntFromObj(interp, objv[objc-1], &iEnd)
+ ){
return TCL_ERROR;
}
+
if( p->xToken==0 ){
Tcl_AppendResult(interp,
"sqlite3_fts5_token may only be used by tokenizer callback", 0
@@ -803,16 +855,13 @@ static int f5tTokenizerReturn(
return TCL_ERROR;
}
- zToken = Tcl_GetStringFromObj(objv[1], &nToken);
- if( Tcl_GetIntFromObj(interp, objv[2], &iStart)
- || Tcl_GetIntFromObj(interp, objv[3], &iEnd)
- ){
- return TCL_ERROR;
- }
-
- rc = p->xToken(p->pCtx, zToken, nToken, iStart, iEnd);
+ rc = p->xToken(p->pCtx, tflags, zToken, nToken, iStart, iEnd);
Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
return TCL_OK;
+
+ usage:
+ Tcl_WrongNumArgs(interp, 1, objv, "?-colocated? TEXT START END");
+ return TCL_ERROR;
}
static void f5tDelTokenizer(void *pCtx){
diff --git a/ext/fts5/fts5_test_mi.c b/ext/fts5/fts5_test_mi.c
index 8ebf5f5077..355f23330d 100644
--- a/ext/fts5/fts5_test_mi.c
+++ b/ext/fts5/fts5_test_mi.c
@@ -352,7 +352,7 @@ static void fts5MatchinfoFunc(
){
const char *zArg;
Fts5MatchinfoCtx *p;
- int rc;
+ int rc = SQLITE_OK;
if( nVal>0 ){
zArg = (const char*)sqlite3_value_text(apVal[0]);
@@ -363,11 +363,16 @@ static void fts5MatchinfoFunc(
p = (Fts5MatchinfoCtx*)pApi->xGetAuxdata(pFts, 0);
if( p==0 || sqlite3_stricmp(zArg, p->zArg) ){
p = fts5MatchinfoNew(pApi, pFts, pCtx, zArg);
- pApi->xSetAuxdata(pFts, p, sqlite3_free);
- if( p==0 ) return;
+ if( p==0 ){
+ rc = SQLITE_NOMEM;
+ }else{
+ rc = pApi->xSetAuxdata(pFts, p, sqlite3_free);
+ }
}
- rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoLocalCb);
+ if( rc==SQLITE_OK ){
+ rc = fts5MatchinfoIter(pApi, pFts, p, fts5MatchinfoLocalCb);
+ }
if( rc!=SQLITE_OK ){
sqlite3_result_error_code(pCtx, rc);
}else{
diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c
index 426e35551b..29a4579b44 100644
--- a/ext/fts5/fts5_tokenize.c
+++ b/ext/fts5/fts5_tokenize.c
@@ -116,8 +116,9 @@ static void asciiFold(char *aOut, const char *aIn, int nByte){
static int fts5AsciiTokenize(
Fts5Tokenizer *pTokenizer,
void *pCtx,
+ int flags,
const char *pText, int nText,
- int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd)
+ int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
){
AsciiTokenizer *p = (AsciiTokenizer*)pTokenizer;
int rc = SQLITE_OK;
@@ -158,7 +159,7 @@ static int fts5AsciiTokenize(
asciiFold(pFold, &pText[is], nByte);
/* Invoke the token callback */
- rc = xToken(pCtx, pFold, nByte, is, ie);
+ rc = xToken(pCtx, 0, pFold, nByte, is, ie);
is = ie+1;
}
@@ -385,8 +386,9 @@ static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){
static int fts5UnicodeTokenize(
Fts5Tokenizer *pTokenizer,
void *pCtx,
+ int flags,
const char *pText, int nText,
- int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd)
+ int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
){
Unicode61Tokenizer *p = (Unicode61Tokenizer*)pTokenizer;
int rc = SQLITE_OK;
@@ -475,7 +477,7 @@ static int fts5UnicodeTokenize(
}
/* Invoke the token callback */
- rc = xToken(pCtx, aFold, zOut-aFold, is, ie);
+ rc = xToken(pCtx, 0, aFold, zOut-aFold, is, ie);
}
tokenize_done:
@@ -553,7 +555,7 @@ static int fts5PorterCreate(
typedef struct PorterContext PorterContext;
struct PorterContext {
void *pCtx;
- int (*xToken)(void*, const char*, int, int, int);
+ int (*xToken)(void*, int, const char*, int, int, int);
char *aBuf;
};
@@ -1118,6 +1120,7 @@ static void fts5PorterStep1A(char *aBuf, int *pnBuf){
static int fts5PorterCb(
void *pCtx,
+ int tflags,
const char *pToken,
int nToken,
int iStart,
@@ -1175,10 +1178,10 @@ static int fts5PorterCb(
nBuf--;
}
- return p->xToken(p->pCtx, aBuf, nBuf, iStart, iEnd);
+ return p->xToken(p->pCtx, tflags, aBuf, nBuf, iStart, iEnd);
pass_through:
- return p->xToken(p->pCtx, pToken, nToken, iStart, iEnd);
+ return p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd);
}
/*
@@ -1187,8 +1190,9 @@ static int fts5PorterCb(
static int fts5PorterTokenize(
Fts5Tokenizer *pTokenizer,
void *pCtx,
+ int flags,
const char *pText, int nText,
- int (*xToken)(void*, const char*, int nToken, int iStart, int iEnd)
+ int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
){
PorterTokenizer *p = (PorterTokenizer*)pTokenizer;
PorterContext sCtx;
@@ -1196,7 +1200,7 @@ static int fts5PorterTokenize(
sCtx.pCtx = pCtx;
sCtx.aBuf = p->aBuf;
return p->tokenizer.xTokenize(
- p->pTokenizer, (void*)&sCtx, pText, nText, fts5PorterCb
+ p->pTokenizer, (void*)&sCtx, flags, pText, nText, fts5PorterCb
);
}
@@ -1225,7 +1229,7 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){
);
}
- return SQLITE_OK;
+ return rc;
}
diff --git a/ext/fts5/test/fts5_common.tcl b/ext/fts5/test/fts5_common.tcl
index 59058a8462..5eba1296ad 100644
--- a/ext/fts5/test/fts5_common.tcl
+++ b/ext/fts5/test/fts5_common.tcl
@@ -295,3 +295,36 @@ proc NOT {a b} {
return $a
}
+#-------------------------------------------------------------------------
+# This command is similar to [split], except that it also provides the
+# start and end offsets of each token. For example:
+#
+# [fts5_tokenize_split "abc d ef"] -> {abc 0 3 d 4 5 ef 6 8}
+#
+
+proc gobble_whitespace {textvar} {
+ upvar $textvar t
+ regexp {([ ]*)(.*)} $t -> space t
+ return [string length $space]
+}
+
+proc gobble_text {textvar wordvar} {
+ upvar $textvar t
+ upvar $wordvar w
+ regexp {([^ ]*)(.*)} $t -> w t
+ return [string length $w]
+}
+
+proc fts5_tokenize_split {text} {
+ set token ""
+ set ret [list]
+ set iOff [gobble_whitespace text]
+ while {[set nToken [gobble_text text word]]} {
+ lappend ret $word $iOff [expr $iOff+$nToken]
+ incr iOff $nToken
+ incr iOff [gobble_whitespace text]
+ }
+
+ set ret
+}
+
diff --git a/ext/fts5/test/fts5aa.test b/ext/fts5/test/fts5aa.test
index daa535cd9b..5f3a9d9e1d 100644
--- a/ext/fts5/test/fts5aa.test
+++ b/ext/fts5/test/fts5aa.test
@@ -343,7 +343,7 @@ do_execsql_test 13.5 {
} {1}
do_execsql_test 13.6 {
- SELECT rowid FROM t1 WHERE t1 MATCH '.';
+ SELECT rowid FROM t1 WHERE t1 MATCH '""';
} {}
#-------------------------------------------------------------------------
@@ -506,6 +506,36 @@ do_execsql_test 18.3 {
SELECT t1.rowid, t2.rowid FROM t2, t1 WHERE t2 MATCH t1.a AND t1.rowid = t2.c
} {1 1}
+#--------------------------------------------------------------------
+# fts5 table in the temp schema.
+#
+reset_db
+do_execsql_test 19.0 {
+ CREATE VIRTUAL TABLE temp.t1 USING fts5(x);
+ INSERT INTO t1 VALUES('x y z');
+ INSERT INTO t1 VALUES('w x 1');
+ SELECT rowid FROM t1 WHERE t1 MATCH 'x';
+} {1 2}
+
+#--------------------------------------------------------------------
+# Test that 6 and 7 byte varints can be read.
+#
+reset_db
+do_execsql_test 20.0 {
+ CREATE VIRTUAL TABLE temp.tmp USING fts5(x);
+}
+set ::ids [list \
+ 0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43]
+]
+do_test 20.1 {
+ foreach id $::ids {
+ execsql { INSERT INTO tmp(rowid, x) VALUES($id, 'x y z') }
+ }
+ execsql { SELECT rowid FROM tmp WHERE tmp MATCH 'y' }
+} $::ids
+
+
+
finish_test
diff --git a/ext/fts5/test/fts5columnsize.test b/ext/fts5/test/fts5columnsize.test
index ed0edd677e..dec9b58d3d 100644
--- a/ext/fts5/test/fts5columnsize.test
+++ b/ext/fts5/test/fts5columnsize.test
@@ -134,5 +134,18 @@ do_execsql_test 3.2.1 {
1 {-1 0 -1} 2 {-1 0 -1}
}
+#-------------------------------------------------------------------------
+# Test the integrity-check
+#
+do_execsql_test 4.1.1 {
+ CREATE VIRTUAL TABLE t5 USING fts5(x, columnsize=0);
+ INSERT INTO t5 VALUES('1 2 3 4');
+ INSERT INTO t5 VALUES('2 4 6 8');
+}
+
+breakpoint
+do_execsql_test 4.1.2 {
+ INSERT INTO t5(t5) VALUES('integrity-check');
+}
finish_test
diff --git a/ext/fts5/test/fts5ea.test b/ext/fts5/test/fts5ea.test
index ad05412ba9..3ccbd7d7a2 100644
--- a/ext/fts5/test/fts5ea.test
+++ b/ext/fts5/test/fts5ea.test
@@ -87,6 +87,12 @@ do_execsql_test 4.0 {
SELECT fts5_expr('a AND """"', 'x', 'tokenize="unicode61 tokenchars ''""''"');
} {{"a" AND """"}}
+#-------------------------------------------------------------------------
+# Experiment with a tokenizer that considers " to be a token character.
+#
+do_catchsql_test 5.0 {
+ SELECT fts5_expr('abc | def');
+} {1 {fts5: syntax error near "|"}}
diff --git a/ext/fts5/test/fts5eb.test b/ext/fts5/test/fts5eb.test
index 352e1b4a17..55c4b15cf3 100644
--- a/ext/fts5/test/fts5eb.test
+++ b/ext/fts5/test/fts5eb.test
@@ -30,18 +30,18 @@ proc do_syntax_test {tn expr res} {
}
foreach {tn expr res} {
- 1 {abc} {"abc"}
- 2 {abc .} {"abc"}
- 3 {.} {}
- 4 {abc OR .} {"abc"}
- 5 {abc NOT .} {"abc"}
- 6 {abc AND .} {"abc"}
- 7 {. OR abc} {"abc"}
- 8 {. NOT abc} {"abc"}
- 9 {. AND abc} {"abc"}
- 10 {abc + . + def} {"abc" + "def"}
- 11 {abc . def} {"abc" AND "def"}
- 12 {r+e OR w} {"r" + "e" OR "w"}
+ 1 {abc} {"abc"}
+ 2 {abc ""} {"abc"}
+ 3 {""} {}
+ 4 {abc OR ""} {"abc"}
+ 5 {abc NOT ""} {"abc"}
+ 6 {abc AND ""} {"abc"}
+ 7 {"" OR abc} {"abc"}
+ 8 {"" NOT abc} {"abc"}
+ 9 {"" AND abc} {"abc"}
+ 10 {abc + "" + def} {"abc" + "def"}
+ 11 {abc "" def} {"abc" AND "def"}
+ 12 {r+e OR w} {"r" + "e" OR "w"}
} {
do_execsql_test 1.$tn {SELECT fts5_expr($expr)} [list $res]
}
diff --git a/ext/fts5/test/fts5fault6.test b/ext/fts5/test/fts5fault6.test
index b9657be1cc..c79cf7ab70 100644
--- a/ext/fts5/test/fts5fault6.test
+++ b/ext/fts5/test/fts5fault6.test
@@ -22,6 +22,7 @@ ifcapable !fts5 {
return
}
+
#-------------------------------------------------------------------------
# OOM while rebuilding an FTS5 table.
#
@@ -148,5 +149,149 @@ do_faultsim_test 4.1 -faults oom-t* -prep {
faultsim_test_result {0 {}}
}
+#-------------------------------------------------------------------------
+#
+# 5.2.* OOM while running a query that includes synonyms and matchinfo().
+#
+# 5.3.* OOM while running a query that returns a row containing instances
+# of more than 4 synonyms for a single term.
+#
+proc mit {blob} {
+ set scan(littleEndian) i*
+ set scan(bigEndian) I*
+ binary scan $blob $scan($::tcl_platform(byteOrder)) r
+ return $r
+}
+proc tcl_tokenize {tflags text} {
+ foreach {w iStart iEnd} [fts5_tokenize_split $text] {
+ sqlite3_fts5_token $w $iStart $iEnd
+ if {$tflags=="query" && [string length $w]==1} {
+ for {set i 2} {$i < 7} {incr i} {
+ sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd
+ }
+ }
+ }
+}
+proc tcl_create {args} { return "tcl_tokenize" }
+reset_db
+sqlite3_fts5_create_tokenizer db tcl tcl_create
+db func mit mit
+sqlite3_fts5_register_matchinfo db
+do_test 5.0 {
+ execsql { CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl) }
+ execsql { INSERT INTO t1(t1, rank) VALUES('pgsz', 32) }
+ foreach {rowid text} {
+ 1 {aaaa cc b aaaaa cc aa}
+ 2 {aa aa bb a bbb}
+ 3 {bb aaaaa aaaaa b aaaa aaaaa}
+ 4 {aa a b aaaa aa}
+ 5 {aa b ccc aaaaa cc}
+ 6 {aa aaaaa bbbb cc aaa}
+ 7 {aaaaa aa aa ccccc bb}
+ 8 {ccc bbbbb ccccc bbb c}
+ 9 {cccccc bbbb a aaa cccc c}
+
+ 20 {ddd f ddd eeeee fff ffff eeee ddd fff eeeee dddddd eeee}
+ 21 {fffff eee dddd fffff dd ee ee eeeee eee eeeeee ee dd e}
+ 22 {fffff d eeee dddd fffff dddddd ffff ddddd eeeee ee eee dddd ddddd}
+ 23 {ddddd fff ddd eeeee ffff eeee ddd ff ff ffffff eeeeee dddd ffffff}
+ 24 {eee dd ee dddd dddd eeeeee e eee fff ffff}
+ 25 {ddddd ffffff dddddd fff ddd ddddd ddd f eeee fff dddd f}
+ 26 {f ffff fff fff eeeeee dddd d dddddd ddddd eee ff eeeee}
+ 27 {eee fff dddddd eeeee eeeee dddd ddddd ffff f eeeee eee dddddd ddddd d}
+ 28 {dd ddddd d ddd d fff d dddd ee dddd ee ddd dddddd dddddd}
+ 29 {eeee dddd ee dddd eeee dddd dd fffff f ddd eeeee ddd ee}
+ 30 {ff ffffff eeeeee eeeee eee ffffff ff ffff f fffff eeeee}
+ 31 {fffff eeeeee dddd eeee eeee eeeeee eee fffff d ddddd ffffff ffff dddddd}
+ 32 {dddddd fffff ee eeeeee eeee ee fff dddd fff eeee ffffff eeeeee ffffff}
+ 33 {ddddd eeee dd ffff dddddd fff eeee ddddd ffff eeee ddd}
+ 34 {ee dddd ddddd dddddd eeee eeeeee f dd ee dddddd ffffff}
+ 35 {ee dddd dd eeeeee ddddd eee d eeeeee dddddd eee dddd fffff}
+ 36 {eee ffffff ffffff e fffff eeeee ff dddddd dddddd fff}
+ 37 {eeeee fffff dddddd dddd ffffff fff f dd ee dd dd eeeee}
+ 38 {eeeeee ee d ff eeeeee eeeeee eee eeeee ee ffffff dddd eeee dddddd ee}
+ 39 {eeeeee ddd fffff e dddd ee eee eee ffffff ee f d dddd}
+ 40 {ffffff dddddd eee ee ffffff eee eeee ddddd ee eeeeee f}
+ 41 {ddd ddd fff fffff ee fffff f fff ddddd fffff}
+ 42 {dddd ee ff d f ffffff fff ffffff ff dd dddddd f eeee}
+ 43 {d dd fff fffff d f fff e dddd ee ee}
+ 44 {ff ffff eee ddd d dd ffff dddd d eeee d eeeeee}
+ 45 {eeee f eeeee ee e ffff f ddd e fff}
+ 46 {ffff d ffff eeee ffff eeeee f ffff ddddd eee}
+ 47 {dd dd dddddd ddddd fffff dddddd ddd ddddd eeeeee ffff eeee eee ee}
+ 48 {ffff ffff e dddd ffffff dd dd dddd f fffff}
+ 49 {ffffff d dddddd ffff eeeee f ffff ffff d dd fffff eeeee}
+
+ 50 {x e}
+ } {
+ execsql { INSERT INTO t1(rowid, a) VALUES($rowid, $text) }
+ }
+} {}
+
+set res [list {*}{
+ 1 {3 24 8 2 12 6}
+ 5 {2 24 8 2 12 6}
+ 6 {3 24 8 1 12 6}
+ 7 {3 24 8 1 12 6}
+ 9 {2 24 8 3 12 6}
+}]
+do_execsql_test 5.1.1 {
+ SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c'
+} $res
+do_execsql_test 5.1.2 {
+ SELECT count(*) FROM t1 WHERE t1 MATCH 'd e f'
+} 29
+
+faultsim_save_and_close
+do_faultsim_test 5.2 -faults oom* -prep {
+ faultsim_restore_and_reopen
+ sqlite3_fts5_create_tokenizer db tcl tcl_create
+ sqlite3_fts5_register_matchinfo db
+ db func mit mit
+} -body {
+ db eval {
+ SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH 'a AND c'
+ }
+} -test {
+ faultsim_test_result [list 0 $::res]
+}
+
+do_faultsim_test 5.3 -faults oom* -prep {
+ faultsim_restore_and_reopen
+ sqlite3_fts5_create_tokenizer db tcl tcl_create
+} -body {
+ db eval {
+ SELECT count(*) FROM t1 WHERE t1 MATCH 'd AND e AND f'
+ }
+} -test {
+ faultsim_test_result {0 29}
+}
+
+do_faultsim_test 5.4 -faults oom* -prep {
+ faultsim_restore_and_reopen
+ sqlite3_fts5_create_tokenizer db tcl tcl_create
+} -body {
+ db eval {
+ SELECT count(*) FROM t1 WHERE t1 MATCH 'x + e'
+ }
+} -test {
+ faultsim_test_result {0 1}
+}
+
+#-------------------------------------------------------------------------
+catch { db close }
+breakpoint
+do_faultsim_test 6 -faults oom* -prep {
+ sqlite_orig db test.db
+ sqlite3_db_config_lookaside db 0 0 0
+} -body {
+ load_static_extension db fts5
+} -test {
+ faultsim_test_result {0 {}} {1 {initialization of fts5 failed: }}
+ if {$testrc==0} {
+ db eval { CREATE VIRTUAL TABLE temp.t1 USING fts5(x) }
+ }
+ db close
+}
finish_test
diff --git a/ext/fts5/test/fts5fault7.test b/ext/fts5/test/fts5fault7.test
new file mode 100644
index 0000000000..381f9ea6ad
--- /dev/null
+++ b/ext/fts5/test/fts5fault7.test
@@ -0,0 +1,45 @@
+# 2015 September 3
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# This file is focused on OOM errors.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5fault2
+
+# If SQLITE_ENABLE_FTS3 is defined, omit this file.
+ifcapable !fts5 {
+ finish_test
+ return
+}
+
+#-------------------------------------------------------------------------
+# Test fault-injection on a query that uses xColumnSize() on columnsize=0
+# table.
+#
+do_execsql_test 1.0 {
+ CREATE VIRTUAL TABLE t1 USING fts5(x, columnsize=0);
+ INSERT INTO t1 VALUES('a b c d e f g');
+ INSERT INTO t1 VALUES('a b c d');
+ INSERT INTO t1 VALUES('a b c d e f g h i j');
+}
+
+
+fts5_aux_test_functions db
+do_faultsim_test 1 -faults oom* -body {
+ execsql { SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH 'b' }
+} -test {
+ faultsim_test_result {0 {7 4 10}} {1 SQLITE_NOMEM}
+}
+
+finish_test
+
diff --git a/ext/fts5/test/fts5matchinfo.test b/ext/fts5/test/fts5matchinfo.test
index 359702eff6..21f9b003e7 100644
--- a/ext/fts5/test/fts5matchinfo.test
+++ b/ext/fts5/test/fts5matchinfo.test
@@ -355,10 +355,10 @@ do_execsql_test 10.1 {
#---------------------------------------------------------------------------
# Test the 'y' matchinfo flag
#
-set sqlite_fts3_enable_parentheses 1
reset_db
+sqlite3_fts5_register_matchinfo db
do_execsql_test 11.0 {
- CREATE VIRTUAL TABLE tt USING fts3(x, y);
+ CREATE VIRTUAL TABLE tt USING fts5(x, y);
INSERT INTO tt VALUES('c d a c d d', 'e a g b d a'); -- 1
INSERT INTO tt VALUES('c c g a e b', 'c g d g e c'); -- 2
INSERT INTO tt VALUES('b e f d e g', 'b a c b c g'); -- 3
@@ -432,19 +432,18 @@ foreach {tn expr res} {
SELECT rowid, mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH $expr
} $r2
}
-set sqlite_fts3_enable_parentheses 0
#---------------------------------------------------------------------------
# Test the 'b' matchinfo flag
#
-set sqlite_fts3_enable_parentheses 1
reset_db
+sqlite3_fts5_register_matchinfo db
db func mit mit
do_test 12.0 {
set cols [list]
for {set i 0} {$i < 50} {incr i} { lappend cols "c$i" }
- execsql "CREATE VIRTUAL TABLE tt USING fts3([join $cols ,])"
+ execsql "CREATE VIRTUAL TABLE tt USING fts5([join $cols ,])"
} {}
do_execsql_test 12.1 {
@@ -452,6 +451,5 @@ do_execsql_test 12.1 {
SELECT mit(matchinfo(tt, 'b')) FROM tt WHERE tt MATCH 'abc';
} [list [list [expr 1<<4] [expr 1<<(45-32)]]]
-set sqlite_fts3_enable_parentheses 0
finish_test
diff --git a/ext/fts5/test/fts5synonym.test b/ext/fts5/test/fts5synonym.test
new file mode 100644
index 0000000000..008d2fc543
--- /dev/null
+++ b/ext/fts5/test/fts5synonym.test
@@ -0,0 +1,460 @@
+# 2014 Dec 20
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focusing on custom tokenizers that support synonyms.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5synonym
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+ finish_test
+ return
+}
+
+foreach S {
+ {zero 0}
+ {one 1 i}
+ {two 2 ii}
+ {three 3 iii}
+ {four 4 iv}
+ {five 5 v}
+ {six 6 vi}
+ {seven 7 vii}
+ {eight 8 viii}
+ {nine 9 ix}
+} {
+ foreach s $S {
+ set o [list]
+ foreach x $S {if {$x!=$s} {lappend o $x}}
+ set ::syn($s) $o
+ }
+}
+
+proc tcl_tokenize {tflags text} {
+ foreach {w iStart iEnd} [fts5_tokenize_split $text] {
+ sqlite3_fts5_token $w $iStart $iEnd
+ }
+}
+
+proc tcl_create {args} {
+ return "tcl_tokenize"
+}
+
+sqlite3_fts5_create_tokenizer db tcl tcl_create
+
+#-------------------------------------------------------------------------
+# Warm body test for the code in fts5_tcl.c.
+#
+do_execsql_test 1.0 {
+ CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl);
+ INSERT INTO ft VALUES('abc def ghi');
+ INSERT INTO ft VALUES('jkl mno pqr');
+ SELECT rowid, x FROM ft WHERE ft MATCH 'def';
+ SELECT x, rowid FROM ft WHERE ft MATCH 'pqr';
+} {1 {abc def ghi} {jkl mno pqr} 2}
+
+#-------------------------------------------------------------------------
+# Test a tokenizer that supports synonyms by adding extra entries to the
+# FTS index.
+#
+
+proc tcl_tokenize {tflags text} {
+ foreach {w iStart iEnd} [fts5_tokenize_split $text] {
+ sqlite3_fts5_token $w $iStart $iEnd
+ if {$tflags=="document" && [info exists ::syn($w)]} {
+ foreach s $::syn($w) {
+ sqlite3_fts5_token -colo $s $iStart $iEnd
+ }
+ }
+ }
+}
+reset_db
+sqlite3_fts5_create_tokenizer db tcl tcl_create
+
+do_execsql_test 2.0 {
+ CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl);
+ INSERT INTO ft VALUES('one two three');
+ INSERT INTO ft VALUES('four five six');
+ INSERT INTO ft VALUES('eight nine ten');
+} {}
+
+foreach {tn expr res} {
+ 1 "3" 1
+ 2 "eight OR 8 OR 5" {2 3}
+ 3 "10" {}
+ 4 "1*" {1}
+ 5 "1 + 2" {1}
+} {
+ do_execsql_test 2.1.$tn {
+ SELECT rowid FROM ft WHERE ft MATCH $expr
+ } $res
+}
+
+#-------------------------------------------------------------------------
+# Test some broken tokenizers:
+#
+# 3.1.*: A tokenizer that declares the very first token to be colocated.
+#
+# 3.2.*: A tokenizer that reports two identical tokens at the same position.
+# This is allowed.
+#
+reset_db
+sqlite3_fts5_create_tokenizer db tcl tcl_create
+proc tcl_tokenize {tflags text} {
+ set bColo 1
+ foreach {w iStart iEnd} [fts5_tokenize_split $text] {
+ if {$bColo} {
+ sqlite3_fts5_token -colo $w $iStart $iEnd
+ set bColo 0
+ } {
+ sqlite3_fts5_token $w $iStart $iEnd
+ }
+ }
+}
+do_execsql_test 3.1.0 {
+ CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl);
+ INSERT INTO ft VALUES('one two three');
+ CREATE VIRTUAL TABLE vv USING fts5vocab(ft, row);
+ SELECT * FROM vv;
+} {
+ one 1 1 three 1 1 two 1 1
+}
+
+do_execsql_test 3.1.1 {
+ INSERT INTO ft(ft) VALUES('integrity-check');
+} {}
+
+proc tcl_tokenize {tflags text} {
+ foreach {w iStart iEnd} [fts5_tokenize_split $text] {
+ sqlite3_fts5_token $w $iStart $iEnd
+ }
+}
+
+do_execsql_test 3.1.2 {
+ SELECT rowid FROM ft WHERE ft MATCH 'one two three'
+} {1}
+
+reset_db
+sqlite3_fts5_create_tokenizer db tcl tcl_create
+proc tcl_tokenize {tflags text} {
+ foreach {w iStart iEnd} [fts5_tokenize_split $text] {
+ sqlite3_fts5_token $w $iStart $iEnd
+ sqlite3_fts5_token -colo $w $iStart $iEnd
+ }
+}
+do_execsql_test 3.2.0 {
+ CREATE VIRTUAL TABLE ft USING fts5(x, tokenize = tcl);
+ INSERT INTO ft VALUES('one one two three');
+ CREATE VIRTUAL TABLE vv USING fts5vocab(ft, row);
+ SELECT * FROM vv;
+} {
+ one 1 4 three 1 2 two 1 2
+}
+do_execsql_test 3.2.1 {
+ SELECT rowid FROM ft WHERE ft MATCH 'one';
+} {1}
+do_execsql_test 3.2.2 {
+ SELECT rowid FROM ft WHERE ft MATCH 'one two three';
+} {1}
+do_execsql_test 3.2.3 {
+ SELECT rowid FROM ft WHERE ft MATCH 'one + one + two + three';
+} {1}
+do_execsql_test 3.2.4 {
+ SELECT rowid FROM ft WHERE ft MATCH 'one two two three';
+} {1}
+do_execsql_test 3.2.5 {
+ SELECT rowid FROM ft WHERE ft MATCH 'one + two + two + three';
+} {}
+
+#-------------------------------------------------------------------------
+# Check that expressions with synonyms can be parsed and executed.
+#
+reset_db
+sqlite3_fts5_create_tokenizer db tcl tcl_create
+proc tcl_tokenize {tflags text} {
+ foreach {w iStart iEnd} [fts5_tokenize_split $text] {
+ sqlite3_fts5_token $w $iStart $iEnd
+ if {$tflags=="query" && [info exists ::syn($w)]} {
+ foreach s $::syn($w) {
+ sqlite3_fts5_token -colo $s $iStart $iEnd
+ }
+ }
+ }
+}
+
+foreach {tn expr res} {
+ 1 {abc} {"abc"}
+ 2 {one} {"one"|"i"|"1"}
+ 3 {3} {"3"|"iii"|"three"}
+ 4 {3*} {"3"|"iii"|"three" *}
+} {
+ do_execsql_test 4.1.$tn {SELECT fts5_expr($expr, 'tokenize=tcl')} [list $res]
+}
+
+do_execsql_test 4.2.1 {
+ CREATE VIRTUAL TABLE xx USING fts5(x, tokenize=tcl);
+ INSERT INTO xx VALUES('one two');
+ INSERT INTO xx VALUES('three four');
+}
+
+do_execsql_test 4.2.2 {
+ SELECT rowid FROM xx WHERE xx MATCH '2'
+} {1}
+
+do_execsql_test 4.2.3 {
+ SELECT rowid FROM xx WHERE xx MATCH '3'
+} {2}
+
+do_test 5.0 {
+ execsql {
+ CREATE VIRTUAL TABLE t1 USING fts5(a, b, tokenize=tcl)
+ }
+ foreach {rowid a b} {
+ 1 {four v 4 i three} {1 3 five five 4 one}
+ 2 {5 1 3 4 i} {2 2 v two 4}
+ 3 {5 i 5 2 four 4 1} {iii ii five two 1}
+ 4 {ii four 4 one 5 three five} {one 5 1 iii 4 3}
+ 5 {three i v i four 4 1} {ii five five five iii}
+ 6 {4 2 ii two 2 iii} {three 1 four 4 iv 1 iv}
+ 7 {ii ii two three 2 5} {iii i ii iii iii one one}
+ 8 {2 ii i two 3 three 2} {two iv v iii 3 five}
+ 9 {i 2 iv 3 five four v} {iii 4 three i three ii 1}
+ } {
+ execsql { INSERT INTO t1(rowid, a, b) VALUES($rowid, $a, $b) }
+ }
+} {}
+
+
+foreach {tn q res} {
+ 1 {one} {
+ 1 {four v 4 [i] three} {[1] 3 five five 4 [one]}
+ 2 {5 [1] 3 4 [i]} {2 2 v two 4}
+ 3 {5 [i] 5 2 four 4 [1]} {iii ii five two [1]}
+ 4 {ii four 4 [one] 5 three five} {[one] 5 [1] iii 4 3}
+ 5 {three [i] v [i] four 4 [1]} {ii five five five iii}
+ 6 {4 2 ii two 2 iii} {three [1] four 4 iv [1] iv}
+ 7 {ii ii two three 2 5} {iii [i] ii iii iii [one] [one]}
+ 8 {2 ii [i] two 3 three 2} {two iv v iii 3 five}
+ 9 {[i] 2 iv 3 five four v} {iii 4 three [i] three ii [1]}
+ }
+ 2 {five four} {
+ 1 {[four] [v] [4] i three} {1 3 [five] [five] [4] one}
+ 2 {[5] 1 3 [4] i} {2 2 [v] two [4]}
+ 3 {[5] i [5] 2 [four] [4] 1} {iii ii [five] two 1}
+ 4 {ii [four] [4] one [5] three [five]} {one [5] 1 iii [4] 3}
+ 5 {three i [v] i [four] [4] 1} {ii [five] [five] [five] iii}
+ 8 {2 ii i two 3 three 2} {two [iv] [v] iii 3 [five]}
+ 9 {i 2 [iv] 3 [five] [four] [v]} {iii [4] three i three ii 1}
+ }
+ 3 {one OR two OR iii OR 4 OR v} {
+ 1 {[four] [v] [4] [i] [three]} {[1] [3] [five] [five] [4] [one]}
+ 2 {[5] [1] [3] [4] [i]} {[2] [2] [v] [two] [4]}
+ 3 {[5] [i] [5] [2] [four] [4] [1]} {[iii] [ii] [five] [two] [1]}
+ 4 {[ii] [four] [4] [one] [5] [three] [five]} {[one] [5] [1] [iii] [4] [3]}
+ 5 {[three] [i] [v] [i] [four] [4] [1]} {[ii] [five] [five] [five] [iii]}
+ 6 {[4] [2] [ii] [two] [2] [iii]} {[three] [1] [four] [4] [iv] [1] [iv]}
+ 7 {[ii] [ii] [two] [three] [2] [5]} {[iii] [i] [ii] [iii] [iii] [one] [one]}
+ 8 {[2] [ii] [i] [two] [3] [three] [2]} {[two] [iv] [v] [iii] [3] [five]}
+ 9 {[i] [2] [iv] [3] [five] [four] [v]} {[iii] [4] [three] [i] [three] [ii] [1]}
+ }
+
+ 4 {5 + 1} {
+ 2 {[5 1] 3 4 i} {2 2 v two 4}
+ 3 {[5 i] 5 2 four 4 1} {iii ii five two 1}
+ 4 {ii four 4 one 5 three five} {one [5 1] iii 4 3}
+ 5 {three i [v i] four 4 1} {ii five five five iii}
+ }
+
+ 5 {one + two + three} {
+ 7 {ii ii two three 2 5} {iii [i ii iii] iii one one}
+ 8 {2 ii [i two 3] three 2} {two iv v iii 3 five}
+ }
+
+ 6 {"v v"} {
+ 1 {four v 4 i three} {1 3 [five five] 4 one}
+ 5 {three i v i four 4 1} {ii [five five five] iii}
+ }
+} {
+ do_execsql_test 5.1.$tn {
+ SELECT rowid, highlight(t1, 0, '[', ']'), highlight(t1, 1, '[', ']')
+ FROM t1 WHERE t1 MATCH $q
+ } $res
+}
+
+# Test that the xQueryPhrase() API works with synonyms.
+#
+proc mit {blob} {
+ set scan(littleEndian) i*
+ set scan(bigEndian) I*
+ binary scan $blob $scan($::tcl_platform(byteOrder)) r
+ return $r
+}
+db func mit mit
+sqlite3_fts5_register_matchinfo db
+
+foreach {tn q res} {
+ 1 {one} {
+ 1 {1 11 7 2 12 6} 2 {2 11 7 0 12 6}
+ 3 {2 11 7 1 12 6} 4 {1 11 7 2 12 6}
+ 5 {3 11 7 0 12 6} 6 {0 11 7 2 12 6}
+ 7 {0 11 7 3 12 6} 8 {1 11 7 0 12 6}
+ 9 {1 11 7 2 12 6}
+ }
+} {
+ do_execsql_test 5.2.$tn {
+ SELECT rowid, mit(matchinfo(t1, 'x')) FROM t1 WHERE t1 MATCH $q
+ } $res
+}
+
+
+#-------------------------------------------------------------------------
+# Test terms with more than 4 synonyms.
+#
+reset_db
+sqlite3_fts5_create_tokenizer db tcl tcl_create
+proc tcl_tokenize {tflags text} {
+ foreach {w iStart iEnd} [fts5_tokenize_split $text] {
+ sqlite3_fts5_token $w $iStart $iEnd
+ if {$tflags=="query" && [string length $w]==1} {
+ for {set i 2} {$i<=10} {incr i} {
+ sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd
+ }
+ }
+ }
+}
+
+do_execsql_test 6.0.1 {
+ CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=tcl);
+ INSERT INTO t1 VALUES('yy xx qq');
+ INSERT INTO t1 VALUES('yy xx xx');
+}
+do_execsql_test 6.0.2 {
+ SELECT * FROM t1 WHERE t1 MATCH 'NEAR(y q)';
+} {{yy xx qq}}
+
+do_test 6.0.3 {
+ execsql {
+ CREATE VIRTUAL TABLE t2 USING fts5(a, b, tokenize=tcl)
+ }
+ foreach {rowid a b} {
+ 1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq aaaa}
+ 2 {ww oooooo bbbbb ssssss mm} {ffffff yy iiii rr s ccc qqqqq}
+ 3 {zzzz llll gggggg cccc uu} {hhhhhh aaaa ppppp rr ee jjjj}
+ 4 {r f i rrrrrr ww hhh} {aa yyy t x aaaaa ii}
+ 5 {fffff mm vvvv ooo ffffff kkkk tttt} {cccccc bb e zzz d n}
+ 6 {iii dddd hh qqqq ddd ooo} {ttt d c b aaaaaa qqqq}
+ 7 {jjjj rrrr v zzzzz u tt t} {ppppp pp dddd mm hhh uuu}
+ 8 {gggg rrrrrr kkkk vvvv gggg jjjjjj b} {dddddd jj r w cccc wwwwww ss}
+ 9 {kkkkk qqq oooo e tttttt mmm} {e ss qqqqqq hhhh llllll gg}
+ } {
+ execsql { INSERT INTO t2(rowid, a, b) VALUES($rowid, $a, $b) }
+ }
+} {}
+
+foreach {tn q res} {
+ 1 {a} {
+ 1 {yyyy vvvvv qq oo yyyyyy vvvv eee} {ffff uu r qq [aaaa]}
+ 3 {zzzz llll gggggg cccc uu} {hhhhhh [aaaa] ppppp rr ee jjjj}
+ 4 {r f i rrrrrr ww hhh} {[aa] yyy t x [aaaaa] ii}
+ 6 {iii dddd hh qqqq ddd ooo} {ttt d c b [aaaaaa] qqqq}
+ }
+
+ 2 {a AND q} {
+ 1 {yyyy vvvvv [qq] oo yyyyyy vvvv eee} {ffff uu r [qq] [aaaa]}
+ 6 {iii dddd hh [qqqq] ddd ooo} {ttt d c b [aaaaaa] [qqqq]}
+ }
+
+ 3 {o OR (q AND a)} {
+ 1 {yyyy vvvvv [qq] [oo] yyyyyy vvvv eee} {ffff uu r [qq] [aaaa]}
+ 2 {ww [oooooo] bbbbb ssssss mm} {ffffff yy iiii rr s ccc qqqqq}
+ 5 {fffff mm vvvv [ooo] ffffff kkkk tttt} {cccccc bb e zzz d n}
+ 6 {iii dddd hh [qqqq] ddd [ooo]} {ttt d c b [aaaaaa] [qqqq]}
+ 9 {kkkkk qqq [oooo] e tttttt mmm} {e ss qqqqqq hhhh llllll gg}
+ }
+
+ 4 {NEAR(q y, 20)} {
+ 1 {[yyyy] vvvvv [qq] oo [yyyyyy] vvvv eee} {ffff uu r qq aaaa}
+ 2 {ww oooooo bbbbb ssssss mm} {ffffff [yy] iiii rr s ccc [qqqqq]}
+ }
+} {
+ do_execsql_test 6.1.$tn.asc {
+ SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']')
+ FROM t2 WHERE t2 MATCH $q
+ } $res
+
+ set res2 [list]
+ foreach {rowid a b} $res {
+ set res2 [concat [list $rowid $a $b] $res2]
+ }
+
+ do_execsql_test 6.1.$tn.desc {
+ SELECT rowid, highlight(t2, 0, '[', ']'), highlight(t2, 1, '[', ']')
+ FROM t2 WHERE t2 MATCH $q ORDER BY rowid DESC
+ } $res2
+}
+
+do_execsql_test 6.2.1 {
+ INSERT INTO t2(rowid, a, b) VALUES(13,
+ 'x xx xxx xxxx xxxxx xxxxxx xxxxxxx', 'y yy yyy yyyy yyyyy yyyyyy yyyyyyy'
+ );
+ SELECT rowid, highlight(t2, 0, '<', '>'), highlight(t2, 1, '(', ')')
+ FROM t2 WHERE t2 MATCH 'x OR y'
+} {
+ 1 { vvvvv qq oo vvvv eee} {ffff uu r qq aaaa}
+ 2 {ww oooooo bbbbb ssssss mm} {ffffff (yy) iiii rr s ccc qqqqq}
+ 4 {r f i rrrrrr ww hhh} {aa (yyy) t (x) aaaaa ii}
+ 13 { }
+ {(y) (yy) (yyy) (yyyy) (yyyyy) (yyyyyy) (yyyyyyy)}
+}
+
+#-------------------------------------------------------------------------
+# Test that the xColumnSize() API is not confused by colocated tokens.
+#
+reset_db
+sqlite3_fts5_create_tokenizer db tcl tcl_create
+fts5_aux_test_functions db
+proc tcl_tokenize {tflags text} {
+ foreach {w iStart iEnd} [fts5_tokenize_split $text] {
+ sqlite3_fts5_token $w $iStart $iEnd
+ if {[string length $w]==1} {
+ for {set i 2} {$i<=10} {incr i} {
+ sqlite3_fts5_token -colo [string repeat $w $i] $iStart $iEnd
+ }
+ }
+ }
+}
+
+do_execsql_test 7.0.1 {
+ CREATE VIRTUAL TABLE t1 USING fts5(a, b, columnsize=1, tokenize=tcl);
+ INSERT INTO t1 VALUES('0 2 3', '4 5 6 7');
+ INSERT INTO t1 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0');
+ SELECT fts5_test_columnsize(t1) FROM t1 WHERE t1 MATCH '000 AND 00 AND 0';
+} {{3 4} {2 10}}
+
+do_execsql_test 7.0.2 {
+ INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+do_execsql_test 7.1.1 {
+ CREATE VIRTUAL TABLE t2 USING fts5(a, b, columnsize=0, tokenize=tcl);
+ INSERT INTO t2 VALUES('0 2 3', '4 5 6 7');
+ INSERT INTO t2 VALUES('8 9', '0 0 0 0 0 0 0 0 0 0');
+ SELECT fts5_test_columnsize(t2) FROM t2 WHERE t2 MATCH '000 AND 00 AND 0';
+} {{3 4} {2 10}}
+
+do_execsql_test 7.1.2 {
+ INSERT INTO t2(t2) VALUES('integrity-check');
+}
+
+finish_test
+
diff --git a/main.mk b/main.mk
index 7cc09d6c8d..d07e473be6 100644
--- a/main.mk
+++ b/main.mk
@@ -47,6 +47,7 @@
TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP)
TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3
TCCX += -I$(TOP)/ext/async -I$(TOP)/ext/userauth
+TCCX += -I$(TOP)/ext/fts5
# Object files for the SQLite library.
#
@@ -230,6 +231,29 @@ SRC += \
$(TOP)/ext/rbu/sqlite3rbu.h
+# FTS5 things
+#
+FTS5_HDR = \
+ $(TOP)/ext/fts5/fts5.h \
+ $(TOP)/ext/fts5/fts5Int.h \
+ fts5parse.h
+
+FTS5_SRC = \
+ $(TOP)/ext/fts5/fts5_aux.c \
+ $(TOP)/ext/fts5/fts5_buffer.c \
+ $(TOP)/ext/fts5/fts5_main.c \
+ $(TOP)/ext/fts5/fts5_config.c \
+ $(TOP)/ext/fts5/fts5_expr.c \
+ $(TOP)/ext/fts5/fts5_hash.c \
+ $(TOP)/ext/fts5/fts5_index.c \
+ fts5parse.c \
+ $(TOP)/ext/fts5/fts5_storage.c \
+ $(TOP)/ext/fts5/fts5_tokenize.c \
+ $(TOP)/ext/fts5/fts5_unicode2.c \
+ $(TOP)/ext/fts5/fts5_varint.c \
+ $(TOP)/ext/fts5/fts5_vocab.c \
+
+
# Generated source code files
#
SRC += \
@@ -636,25 +660,6 @@ fts3_write.o: $(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR)
rtree.o: $(TOP)/ext/rtree/rtree.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/rtree/rtree.c
-# FTS5 things
-#
-FTS5_SRC = \
- $(TOP)/ext/fts5/fts5.h \
- $(TOP)/ext/fts5/fts5Int.h \
- $(TOP)/ext/fts5/fts5_aux.c \
- $(TOP)/ext/fts5/fts5_buffer.c \
- $(TOP)/ext/fts5/fts5_main.c \
- $(TOP)/ext/fts5/fts5_config.c \
- $(TOP)/ext/fts5/fts5_expr.c \
- $(TOP)/ext/fts5/fts5_hash.c \
- $(TOP)/ext/fts5/fts5_index.c \
- fts5parse.c fts5parse.h \
- $(TOP)/ext/fts5/fts5_storage.c \
- $(TOP)/ext/fts5/fts5_tokenize.c \
- $(TOP)/ext/fts5/fts5_unicode2.c \
- $(TOP)/ext/fts5/fts5_varint.c \
- $(TOP)/ext/fts5/fts5_vocab.c \
-
fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon
cp $(TOP)/ext/fts5/fts5parse.y .
rm -f fts5parse.h
@@ -662,11 +667,10 @@ fts5parse.c: $(TOP)/ext/fts5/fts5parse.y lemon
fts5parse.h: fts5parse.c
-fts5.c: $(FTS5_SRC)
+fts5.c: $(FTS5_SRC) $(FTS5_HDR)
tclsh $(TOP)/ext/fts5/tool/mkfts5c.tcl
cp $(TOP)/ext/fts5/fts5.h .
-
userauth.o: $(TOP)/ext/userauth/userauth.c $(HDR) $(EXTHDR)
$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/userauth/userauth.c
diff --git a/manifest b/manifest
index bfd4ab3109..e7ca9c9ceb 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Simplification\sof\sthe\sLRU\slist\shandling\sin\spcache1.
-D 2015-09-04T04:31:56.886
+C Modify\sthe\sfts5\scustom\stokenizer\sinterface\sto\spermit\ssynonym\ssupport.\sThe\sfts5_api.iVersion\svalue\sis\snow\sset\sto\s2.\sExisting\sfts5\scustom\stokenizers\s(if\sthere\sare\ssuch\sthings)\swill\sneed\sto\sbe\supdated\sto\suse\sthe\snew\sapi\sversion.
+D 2015-09-04T10:31:51.624
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in f85066ce844a28b671aaeeff320921cd0ce36239
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -104,27 +104,27 @@ F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
F ext/fts3/unicode/mkunicode.tcl 95cf7ec186e48d4985e433ff8a1c89090a774252
F ext/fts3/unicode/parseunicode.tcl da577d1384810fb4e2b209bf3313074353193e95
-F ext/fts5/extract_api_docs.tcl 06583c935f89075ea0b32f85efa5dd7619fcbd03
-F ext/fts5/fts5.h 1950ec0544de667a24c1d8af9b2fde5db7db3bc9
-F ext/fts5/fts5Int.h 45f2ceb3c030f70e2cc4c199e9f700c2f2367f77
-F ext/fts5/fts5_aux.c 044cb176a815f4388308738437f6e130aa384fb0
+F ext/fts5/extract_api_docs.tcl a36e54ec777172ddd3f9a88daf593b00848368e0
+F ext/fts5/fts5.h f04659e0df5af83731b102189a32280f74f4a6bc
+F ext/fts5/fts5Int.h f65d41f66accad0a289d6bd66b13c07d2932f9be
+F ext/fts5/fts5_aux.c 7a307760a9c57c750d043188ec0bad59f5b5ec7e
F ext/fts5/fts5_buffer.c 80f9ba4431848cb857e3d2158f5280093dcd8015
-F ext/fts5/fts5_config.c fdfa63ae8e527ecfaa50f94063c610429cc887cf
-F ext/fts5/fts5_expr.c d075d36c84975a1cfcf070442d28e28027b61c25
+F ext/fts5/fts5_config.c 80b61fd2c6844b64a3e72a64572d50a812da9384
+F ext/fts5/fts5_expr.c 1c24e1a2ffb286bfe37e537a43b7fadabfe993d4
F ext/fts5/fts5_hash.c 4bf4b99708848357b8a2b5819e509eb6d3df9246
-F ext/fts5/fts5_index.c 076c4995bf06a6d1559a6e31f9a86b90f2105374
-F ext/fts5/fts5_main.c fc47ad734dfb55765b7542a345cee981170e7caa
-F ext/fts5/fts5_storage.c 22ec9b5d35a39e2b5b65daf4ba7cd47fbb2d0df5
-F ext/fts5/fts5_tcl.c 96a3b9e982c4a64a242eefd752fa6669cd405a67
-F ext/fts5/fts5_test_mi.c 80a9e86fb4c5b6b58f8fefac05e9b96d1a6574e1
-F ext/fts5/fts5_tokenize.c 2836f6728bd74c7efac7487f5d9c27ca3e1b509c
+F ext/fts5/fts5_index.c febb68173333ae3248eb15928a18b21112d45135
+F ext/fts5/fts5_main.c e9d0892424bb7f0a8b58613d4ff75cb650cf286e
+F ext/fts5/fts5_storage.c 120f7b143688b5b7710dacbd48cff211609b8059
+F ext/fts5/fts5_tcl.c 6da58d6e8f42a93c4486b5ba9b187a7f995dee37
+F ext/fts5/fts5_test_mi.c e96be827aa8f571031e65e481251dc1981d608bf
+F ext/fts5/fts5_tokenize.c f380f46f341af9c9a9908e1aade685ba1eaa157a
F ext/fts5/fts5_unicode2.c 78273fbd588d1d9bd0a7e4e0ccc9207348bae33c
F ext/fts5/fts5_varint.c 3f86ce09cab152e3d45490d7586b7ed2e40c13f1
F ext/fts5/fts5_vocab.c 4622e0b7d84a488a1585aaa56eb214ee67a988bc
F ext/fts5/fts5parse.y 833db1101b78c0c47686ab1b84918e38c36e9452
F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
-F ext/fts5/test/fts5_common.tcl 3338968de1880ca12b0451ae8f9b8b12d14e0ba7
-F ext/fts5/test/fts5aa.test c6e680a0d1b6c2616a382f1006d5d91eca697bd0
+F ext/fts5/test/fts5_common.tcl b6e6a40ef5d069c8e86ca4fbad491e1195485dbc
+F ext/fts5/test/fts5aa.test be961d10b0eff5e973c37b3b7a32a18cda87a77c
F ext/fts5/test/fts5ab.test 6fe3a56731d15978afbb74ae51b355fc9310f2ad
F ext/fts5/test/fts5ac.test 9737992d08c56bfd4803e933744d2d764e23795c
F ext/fts5/test/fts5ad.test b2edee8b7de0c21d2c88f8a18c195034aad6952d
@@ -141,7 +141,7 @@ F ext/fts5/test/fts5auto.test caa5bcf917db11944655a2a9bd38c67c520376ca
F ext/fts5/test/fts5aux.test 8c687c948cc98e9a94be014df7d518acc1b3b74f
F ext/fts5/test/fts5auxdata.test 141a7cbffcceb1bd2799b4b29c183ff8780d586e
F ext/fts5/test/fts5bigpl.test 04ee0d7eebbebf17c31f5a0b5c5f9494eac3a0cb
-F ext/fts5/test/fts5columnsize.test 97dc6bd66c91009d00407aa078dd5e9e8eb22f99
+F ext/fts5/test/fts5columnsize.test a8cfef21ffa1c264b9f670a7d94eeaccb5341c07
F ext/fts5/test/fts5config.test ad2ff42ddc856aed2d05bf89dc1c578c8a39ea3b
F ext/fts5/test/fts5content.test 9a952c95518a14182dc3b59e3c8fa71cda82a4e1
F ext/fts5/test/fts5corrupt.test 928c9c91d40690d301f943a7ed0ffc19e0d0e7b6
@@ -149,18 +149,19 @@ F ext/fts5/test/fts5corrupt2.test 1a830ccd6dbe1b601c7e3f5bbc1cf77bd8c8803b
F ext/fts5/test/fts5corrupt3.test 1ccf575f5126e79f9fec7979fd02a1f40a076be3
F ext/fts5/test/fts5dlidx.test 59b80bbe34169a082c575d9c26f0a7019a7b79c1
F ext/fts5/test/fts5doclist.test 8edb5b57e5f144030ed74ec00ef6fa4294fed79b
-F ext/fts5/test/fts5ea.test 451bb37310ee6df8ef72e4354fda5621b3b51448
-F ext/fts5/test/fts5eb.test 46f49497edc25ef3b2bff9fb6d75b6d201e2b39e
+F ext/fts5/test/fts5ea.test b01e3a18cdfabbff8104a96a5242a06a68a998a0
+F ext/fts5/test/fts5eb.test 3e5869af2008cbc4ad03a175a0b6f6e58134cd43
F ext/fts5/test/fts5fault1.test 7a562367cb4a735b57b410dbdb62dcc8d971faec
F ext/fts5/test/fts5fault2.test 28c36c843bb39ae855ba79827417ecc37f114341
F ext/fts5/test/fts5fault3.test d6e9577d4312e331a913c72931bf131704efc8f3
F ext/fts5/test/fts5fault4.test 762991d526ee67c2b374351a17248097ea38bee7
F ext/fts5/test/fts5fault5.test 54da9fd4c3434a1d4f6abdcb6469299d91cf5875
-F ext/fts5/test/fts5fault6.test 234dc6355f8d3f8b5be2763f30699d770247c215
+F ext/fts5/test/fts5fault6.test 97bce1a36b7a64e3203fea504ae8e5cfd5ada423
+F ext/fts5/test/fts5fault7.test f4a9b796f8b20c78ec7cf9f4e11144d15d7c3fd4
F ext/fts5/test/fts5full.test 6f6143af0c6700501d9fd597189dfab1555bb741
F ext/fts5/test/fts5hash.test 42eb066f667e9a389a63437cb7038c51974d4fc6
F ext/fts5/test/fts5integrity.test 29f41d2c7126c6122fbb5d54e556506456876145
-F ext/fts5/test/fts5matchinfo.test ee6e7b130096c708c12049fa9c1ceb628954c4f9
+F ext/fts5/test/fts5matchinfo.test 2163b0013e824bba65499da9e34ea4da41349cc2
F ext/fts5/test/fts5merge.test 8f3cdba2ec9c5e7e568246e81b700ad37f764367
F ext/fts5/test/fts5near.test b214cddb1c1f1bddf45c75af768f20145f7e71cc
F ext/fts5/test/fts5optimize.test 42741e7c085ee0a1276140a752d4407d97c2c9f5
@@ -172,6 +173,7 @@ F ext/fts5/test/fts5rank.test 11dcebba31d822f7e99685b4ea2c2ae3ec0b16f1
F ext/fts5/test/fts5rebuild.test 03935f617ace91ed23a6099c7c74d905227ff29b
F ext/fts5/test/fts5restart.test c17728fdea26e7d0f617d22ad5b4b2862b994c17
F ext/fts5/test/fts5rowid.test 6f9833b23b176dc4aa15b7fc02afeb2b220fd460
+F ext/fts5/test/fts5synonym.test cf88c0a56d5ea9591e3939ef1f6e294f7f2d0671
F ext/fts5/test/fts5tokenizer.test ea4df698b35cc427ebf2ba22829d0e28386d8c89
F ext/fts5/test/fts5unicode.test fbef8d8a3b4b88470536cc57604a82ca52e51841
F ext/fts5/test/fts5unicode2.test c1dd890ba32b7609adba78e420faa847abe43b59
@@ -258,7 +260,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60
-F main.mk 8da13ed011a7ae19450b7554910ff4afa3bd22b7
+F main.mk 61821e43596648bfacce2d6283377bee35986131
F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea
F mkopcodeh.awk 0e7f04a8eb90f92259e47d80110e4e98d7ce337a
F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83
@@ -1380,7 +1382,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
F tool/warnings.sh 48bd54594752d5be3337f12c72f28d2080cb630b
F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P f00a9e1e998c4bd249a45444dc2d71a7e4903b8b
-R 231caff1c142ba151d728fb853afd78c
-U drh
-Z ce45274979109dc4b492f06dc4bc7386
+P 05a3a2cd140587265b5427d23c93c5be1f39e199 443a5eb8e17fd4f0b83ecc5bba74848e2c8a968c
+R 3837e1c9ccb54eb6541c962b4dbcd417
+U dan
+Z 9252bca216b9264b05e49a4db9d079d2
diff --git a/manifest.uuid b/manifest.uuid
index 562662b19b..36dfbd374c 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-05a3a2cd140587265b5427d23c93c5be1f39e199
\ No newline at end of file
+0b7e4ab8abde3ae32459233df115c433dd58d2c1
\ No newline at end of file