From e25a267e812e918a78577c919ad426afe57b3b1f Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 14 Sep 2024 20:30:14 +0000 Subject: [PATCH 1/8] Fix the fts5 xInstToken() API for prefix queries that do not use prefix-indexes. This is experimental. FossilOrigin-Name: 97c2824f471e7e622c4a166947a6e8162cae891345101539829a6fcec83373fe --- ext/fts5/fts5_expr.c | 18 +++-- ext/fts5/fts5_index.c | 19 ++++-- ext/fts5/test/fts5origintext.test | 11 +++- ext/fts5/test/fts5tokendata.test | 105 ++++++++++++++++++++++++++++++ manifest | 22 ++++--- manifest.uuid | 2 +- 6 files changed, 148 insertions(+), 29 deletions(-) create mode 100644 ext/fts5/test/fts5tokendata.test diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index cd44b96bda..0124a1cc93 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -3046,7 +3046,7 @@ static int fts5ExprPopulatePoslistsCb( int rc = sqlite3Fts5PoslistWriterAppend( &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff ); - if( rc==SQLITE_OK && pExpr->pConfig->bTokendata && !pT->bPrefix ){ + if( rc==SQLITE_OK && (pExpr->pConfig->bTokendata || pT->bPrefix) ){ int iCol = p->iOff>>32; int iTokOff = p->iOff & 0x7FFFFFFF; rc = sqlite3Fts5IndexIterWriteTokendata( @@ -3239,15 +3239,13 @@ int sqlite3Fts5ExprInstToken( return SQLITE_RANGE; } pTerm = &pPhrase->aTerm[iToken]; - if( pTerm->bPrefix==0 ){ - if( pExpr->pConfig->bTokendata ){ - rc = sqlite3Fts5IterToken( - pTerm->pIter, iRowid, iCol, iOff+iToken, ppOut, pnOut - ); - }else{ - *ppOut = pTerm->pTerm; - *pnOut = pTerm->nFullTerm; - } + if( pExpr->pConfig->bTokendata || pTerm->bPrefix ){ + rc = sqlite3Fts5IterToken( + pTerm->pIter, iRowid, iCol, iOff+iToken, ppOut, pnOut + ); + }else{ + *ppOut = pTerm->pTerm; + *pnOut = pTerm->nFullTerm; } return rc; } diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 4363305a56..ded1ec59cf 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -6571,12 +6571,13 @@ int sqlite3Fts5IndexWrite( static int fts5IsTokendataPrefix( Fts5Buffer *pBuf, const u8 *pToken, - int nToken + int nToken, + int bPrefix ){ return ( pBuf->n>=nToken && 0==memcmp(pBuf->p, pToken, nToken) - && (pBuf->n==nToken || pBuf->p[nToken]==0x00) + && (bPrefix || pBuf->n==nToken || pBuf->p[nToken]==0x00) ); } @@ -6879,7 +6880,8 @@ static Fts5Iter *fts5SetupTokendataIter( Fts5Index *p, /* FTS index to query */ const u8 *pToken, /* Buffer containing query term */ int nToken, /* Size of buffer pToken in bytes */ - Fts5Colset *pColset /* Colset to filter on */ + Fts5Colset *pColset, /* Colset to filter on */ + int bPrefix /* True to match any prefix */ ){ Fts5Iter *pRet = 0; Fts5TokenDataIter *pSet = 0; @@ -6961,7 +6963,7 @@ static Fts5Iter *fts5SetupTokendataIter( pSmall = 0; for(ii=0; iinSeg; ii++){ Fts5SegIter *pII = &pNew->aSeg[ii]; - if( 0==fts5IsTokendataPrefix(&pII->term, pToken, nToken) ){ + if( 0==fts5IsTokendataPrefix(&pII->term, pToken, nToken, bPrefix) ){ fts5SegIterSetEOF(pII); } if( pII->pLeaf && (!pSmall || fts5BufferCompare(pSmall, &pII->term)>0) ){ @@ -7037,6 +7039,9 @@ int sqlite3Fts5IndexQuery( int bTokendata = pConfig->bTokendata; if( nToken>0 ) memcpy(&buf.p[1], pToken, nToken); + /* The NOTOKENDATA flag is set when it is known that tokendata data will + ** not be required. e.g. for queries performed as part of an + ** integrity-check, or by the fts5vocab module. */ if( flags & (FTS5INDEX_QUERY_NOTOKENDATA|FTS5INDEX_QUERY_SCAN) ){ bTokendata = 0; } @@ -7066,9 +7071,9 @@ int sqlite3Fts5IndexQuery( } } - if( bTokendata && iIdx==0 ){ - buf.p[0] = '0'; - pRet = fts5SetupTokendataIter(p, buf.p, nToken+1, pColset); + if( (bTokendata && iIdx==0) || iIdx>pConfig->nPrefix ){ + buf.p[0] = FTS5_MAIN_PREFIX; + pRet = fts5SetupTokendataIter(p, buf.p, nToken+1, pColset, iIdx>0); }else if( iIdx<=pConfig->nPrefix ){ /* Straight index lookup */ Fts5Structure *pStruct = fts5StructureRead(p); diff --git a/ext/fts5/test/fts5origintext.test b/ext/fts5/test/fts5origintext.test index 8e975fa17c..9741f786e8 100644 --- a/ext/fts5/test/fts5origintext.test +++ b/ext/fts5/test/fts5origintext.test @@ -166,6 +166,13 @@ do_execsql_test 3.3.3 { SELECT rowid FROM ft2('HELLO') } {1 2 3} do_execsql_test 3.3.4 { SELECT rowid FROM ft2('hello*') } {1 2 3 10} +do_execsql_test 3.3.5.1 { SELECT rowid FROM ft2('HELLO') ORDER BY rowid DESC} { + 3 2 1 +} +do_execsql_test 3.3.5.2 { SELECT rowid FROM ft2('HELLO') ORDER BY +rowid DESC} { + 3 2 1 +} + #------------------------------------------------------------------------- # reset_db @@ -285,11 +292,11 @@ do_execsql_test 6.1 { do_execsql_test 6.2 { SELECT rowid, tokens(ft) FROM ft('on*'); -} {1 {{}} 2 {{} {}}} +} {1 one.One 2 {one one.One}} do_execsql_test 6.3 { SELECT rowid, tokens(ft) FROM ft('Three*'); -} {1 {{}} 2 {{}}} +} {1 three.Three 2 three.Three} fts5_aux_test_functions db do_catchsql_test 6.4 { diff --git a/ext/fts5/test/fts5tokendata.test b/ext/fts5/test/fts5tokendata.test new file mode 100644 index 0000000000..7f75f4fa8e --- /dev/null +++ b/ext/fts5/test/fts5tokendata.test @@ -0,0 +1,105 @@ +# 2014 Jan 08 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focused on phrase queries. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5tokendata + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +foreach_detail_mode $testprefix { + + sqlite3_fts5_register_origintext db + fts5_aux_test_functions db + proc b {x} { string map [list "\0" "."] $x } + db func b b + + do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft USING fts5(a, b, tokendata=1, + tokenize="origintext unicode61", detail=%DETAIL% + ); + CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance); + } + + do_execsql_test 1.1 { + INSERT INTO ft(rowid, a, b) VALUES + (1, 'Pedagog Pedal Pedant', 'Peculier Day Today'), + (2, 'Pedant pedantic pecked', 'Peck Penalize Pen'); + + INSERT INTO ft(rowid, a, b) VALUES + (3, 'Penalty Pence Penciled', 'One Two Three'), + (4, 'Pedant Pedal Pedant', 'Peculier Day Today'); + } + + do_execsql_test 1.2 { + SELECT DISTINCT b(term) FROM vocab + } { + day.Day one.One peck.Peck pecked peculier.Peculier pedagog.Pedagog + pedal.Pedal pedant.Pedant pedantic pen.Pen penalize.Penalize + penalty.Penalty pence.Pence penciled.Penciled three.Three + today.Today two.Two + } + + do_execsql_test 1.3.1 { + SELECT rowid FROM ft('pe*') + } { + 1 2 3 4 + } + + do_execsql_test 1.3.2 { + SELECT rowid FROM ft('pe*') ORDER BY rowid DESC + } { + 4 3 2 1 + } + + if {"%DETAIL%"!="none"} { + do_execsql_test 1.3.3 { + SELECT rowid FROM ft WHERE a MATCH 'pe*' ORDER BY rowid DESC + } { + 4 3 2 1 + } + } + + do_execsql_test 1.4 { + SELECT rowid, b( fts5_test_insttoken(ft, 0, 0) ) FROM ft('pedant') + } { + 1 pedant.Pedant + 2 pedant.Pedant + 4 pedant.Pedant + } + + do_execsql_test 1.5 { + SELECT rowid, b( fts5_test_insttoken(ft, 0, 0) ) FROM ft('pe*') + } { + 1 pedagog.Pedagog + 2 pedant.Pedant + 3 penalty.Penalty + 4 pedant.Pedant + } + + do_execsql_test 1.6 { + SELECT rowid, fts5_test_poslist(ft) FROM ft('pe*') + } { + 1 {0.0.0 0.0.1 0.0.2 0.1.0} + 2 {0.0.0 0.0.1 0.0.2 0.1.0 0.1.1 0.1.2} + 3 {0.0.0 0.0.1 0.0.2} + 4 {0.0.0 0.0.1 0.0.2 0.1.0} + } +} + +finish_test + diff --git a/manifest b/manifest index 3a6304a539..3fc85c61ed 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\ssubsequent\schanges\sto\ssqlite-rsync\sthat\swere\saccidentally\smissed\sby\nthe\sprior\scheck-in. -D 2024-09-14T16:52:45.740 +C Fix\sthe\sfts5\sxInstToken()\sAPI\sfor\sprefix\squeries\sthat\sdo\snot\suse\sprefix-indexes.\sThis\sis\sexperimental. +D 2024-09-14T20:30:14.441 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -97,9 +97,9 @@ F ext/fts5/fts5Int.h 93aba03ca417f403b07b2ab6f50aa0e0c1b8b031917a9026b81520e7047 F ext/fts5/fts5_aux.c 65a0468dd177d6093aa9ae1622e6d86b0136b8d267c62c0ad6493ad1e9a3d759 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 F ext/fts5/fts5_config.c da21548ddbc1a457cb42545f527065221ede8ada6a734891b8c34317a7a9506b -F ext/fts5/fts5_expr.c 9a56f53700d1860f0ee2f373c2b9074eaf2a7aa0637d0e27a6476de26a3fee33 +F ext/fts5/fts5_expr.c 1f60d81aa4703435f98f46bbb41fb2a2efa898423fec070a2b3f7a02f177ac64 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c 571483823193f09439356741669aa8c81da838ae6f5e1bfa7517f7ee2fb3addd +F ext/fts5/fts5_index.c aadd271f3c2048418298377908dd09d496753a5c7da84161a9c86ca8c1e78e9a F ext/fts5/fts5_main.c 4503498d3453e29a3cd89dacaba029011e89cb8c481a6241611d106e7a369bd4 F ext/fts5/fts5_storage.c 3332497823c3d171cf56379f2bd8c971ce15a19aadacff961106462022c92470 F ext/fts5/fts5_tcl.c 4db9258a7882c5eac0da4433042132aaf15b87dd1e1636c7a6ca203abd2c8bfe @@ -201,7 +201,7 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618 F ext/fts5/test/fts5optimize.test 264b9101721c17d06d1d174feb743fda3ddc89fad41dee980fef821428258e47 F ext/fts5/test/fts5optimize2.test 795d4ae5f66a7239cf8d5aef4c2ea96aeb8bcd907bd9be0cfe22064fc71a44ed F ext/fts5/test/fts5optimize3.test 1653029284e10e0715246819893ba30565c4ead0d0fc470adae92c353ea857d3 -F ext/fts5/test/fts5origintext.test 2015f69bc8abd111152a8e66211fd2d45026378001e07c054159aa4f84e6691d +F ext/fts5/test/fts5origintext.test 63d5b0dc00f0104add8960da0705a70bffd4d86b6feb6ddbb38bff21141d42f0 F ext/fts5/test/fts5origintext2.test f4505ff79bf7369f2b8b10b9cef7476049d844e20b37f29cad3a8b8d5ac6f9ba F ext/fts5/test/fts5origintext3.test 45c33cf0c91a9ca0e36d298462db3edc7c8fe45fd185649a9dbfd66bb670058b F ext/fts5/test/fts5origintext4.test 0d3ef0a8038f471dbc83001c34fe5f7ae39b571bfc209670771eb28bc0fc50e8 @@ -234,6 +234,7 @@ F ext/fts5/test/fts5synonym.test becc8cea6cfc958a50b30c572c68cbfdf7455971d0fe988 F ext/fts5/test/fts5synonym2.test 58f357b997cf2fedeeb9d0de4db9f880fa96fa2fe27a743bfe7d7b96895bdd87 F ext/fts5/test/fts5tok1.test 1f7817499f5971450d8c4a652114b3d833393c8134e32422d0af27884ffe9cef F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2 +F ext/fts5/test/fts5tokendata.test 7cad79af82e8e81b7a36b450087233d2fca051bb0d584421afc375d6dd26d6f6 F ext/fts5/test/fts5tokenizer.test 7937cec672b148223fff8746d21d3e7ed0965fd7caf35ccdc888a005bb452f98 F ext/fts5/test/fts5tokenizer2.test ddb8b10fbe4b84b2a75812671f127774c1d2e3e2bf82d2e0e4f0bb1cd8a2b2d6 F ext/fts5/test/fts5tokenizer3.test eea778f7bb7024c3e904e28915f9d53286141671b138722148be22a9c758bdc3 @@ -2213,8 +2214,11 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P b7a8ce4c8c5fc6a3b4744d412d96f99d2452eb4086ad84472511da3b4d6afec6 129aca54f6b791c222b51f3eb01569e1e569269860e153b005140eb65af378b9 -R 3fe2d7eb1050bd322cb4e381c1c28c99 -U drh -Z 5104838c8e0c55c4caf222e0f2dc7147 +P 50762ba0783a04e0dcd9456a1ae17d875b0a9272f2f09854a23d9d5253761e9f +R 7d932f341b9d462f57fdca7d1de99fb1 +T *branch * fts5-tokendata-prefix +T *sym-fts5-tokendata-prefix * +T -sym-trunk * +U dan +Z 82ecce364e38343eb4f3cc0cc45881c8 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index e011175dd4..77dd09f207 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -50762ba0783a04e0dcd9456a1ae17d875b0a9272f2f09854a23d9d5253761e9f +97c2824f471e7e622c4a166947a6e8162cae891345101539829a6fcec83373fe From 2eff8f225292c7fd17299910338e01706dba083a Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 25 Sep 2024 18:55:11 +0000 Subject: [PATCH 2/8] Change the way tokendata indexes are collected for prefix queries. FossilOrigin-Name: 204ddf4e726b695dd12ab4a945ec2461655aa0bcc38b74e970f07ed2ac43c6ff --- ext/fts5/fts5Int.h | 9 +- ext/fts5/fts5_expr.c | 3 +- ext/fts5/fts5_index.c | 246 ++++++++++++++++++++++++++++++++++++------ manifest | 16 +-- manifest.uuid | 2 +- 5 files changed, 235 insertions(+), 41 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index b15521f163..a71fb13e06 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -502,7 +502,14 @@ int sqlite3Fts5StructureTest(Fts5Index*, void*); /* ** Used by xInstToken(): */ -int sqlite3Fts5IterToken(Fts5IndexIter*, i64, int, int, const char**, int*); +int sqlite3Fts5IterToken( + Fts5IndexIter *pIndexIter, + const char *pToken, int nToken, + i64 iRowid, + int iCol, + int iOff, + const char **ppOut, int *pnOut +); /* ** Insert or remove data to or from the index. Each time a document is diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c index 0124a1cc93..877c3f79c5 100644 --- a/ext/fts5/fts5_expr.c +++ b/ext/fts5/fts5_expr.c @@ -3241,7 +3241,8 @@ int sqlite3Fts5ExprInstToken( pTerm = &pPhrase->aTerm[iToken]; if( pExpr->pConfig->bTokendata || pTerm->bPrefix ){ rc = sqlite3Fts5IterToken( - pTerm->pIter, iRowid, iCol, iOff+iToken, ppOut, pnOut + pTerm->pIter, pTerm->pTerm, pTerm->nQueryTerm, + iRowid, iCol, iOff+iToken, ppOut, pnOut ); }else{ *ppOut = pTerm->pTerm; diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index ded1ec59cf..1efbe5a7b4 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -6571,13 +6571,12 @@ int sqlite3Fts5IndexWrite( static int fts5IsTokendataPrefix( Fts5Buffer *pBuf, const u8 *pToken, - int nToken, - int bPrefix + int nToken ){ return ( pBuf->n>=nToken && 0==memcmp(pBuf->p, pToken, nToken) - && (bPrefix || pBuf->n==nToken || pBuf->p[nToken]==0x00) + && (pBuf->n==nToken || pBuf->p[nToken]==0x00) ); } @@ -6602,20 +6601,25 @@ static void fts5SegIterSetEOF(Fts5SegIter *pSeg){ struct Fts5TokenDataMap { i64 iRowid; /* Row this token is located in */ i64 iPos; /* Position of token */ + int iIter; /* Iterator token was read from */ + int nByte; /* Length of token in bytes (or 0) */ }; /* ** An object used to supplement Fts5Iter for tokendata=1 iterators. */ struct Fts5TokenDataIter { - int nIter; - int nIterAlloc; - int nMap; int nMapAlloc; Fts5TokenDataMap *aMap; + /* The following are used for prefix-queries only. */ + Fts5Buffer terms; + + /* The following are used for other full-token tokendata queries only. */ + int nIter; + int nIterAlloc; Fts5PoslistReader *aPoslistReader; int *aPoslistToIter; Fts5Iter *apIter[1]; @@ -6666,6 +6670,7 @@ static void fts5TokendataIterDelete(Fts5TokenDataIter *pSet){ for(ii=0; iinIter; ii++){ fts5MultiIterFree(pSet->apIter[ii]); } + fts5BufferFree(&pSet->terms); sqlite3_free(pSet->aPoslistReader); sqlite3_free(pSet->aMap); sqlite3_free(pSet); @@ -6679,6 +6684,7 @@ static void fts5TokendataIterAppendMap( Fts5Index *p, Fts5TokenDataIter *pT, int iIter, + int nByte, i64 iRowid, i64 iPos ){ @@ -6701,6 +6707,7 @@ static void fts5TokendataIterAppendMap( pT->aMap[pT->nMap].iRowid = iRowid; pT->aMap[pT->nMap].iPos = iPos; pT->aMap[pT->nMap].iIter = iIter; + pT->aMap[pT->nMap].nByte = nByte; pT->nMap++; } } @@ -6745,7 +6752,7 @@ static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){ pIter->base.iRowid = iRowid; if( nHit==1 && eDetail==FTS5_DETAIL_FULL ){ - fts5TokendataIterAppendMap(pIter->pIndex, pT, iMin, iRowid, -1); + fts5TokendataIterAppendMap(pIter->pIndex, pT, iMin, 0, iRowid, -1); }else if( nHit>1 && eDetail!=FTS5_DETAIL_NONE ){ int nReader = 0; @@ -6880,8 +6887,7 @@ static Fts5Iter *fts5SetupTokendataIter( Fts5Index *p, /* FTS index to query */ const u8 *pToken, /* Buffer containing query term */ int nToken, /* Size of buffer pToken in bytes */ - Fts5Colset *pColset, /* Colset to filter on */ - int bPrefix /* True to match any prefix */ + Fts5Colset *pColset /* Colset to filter on */ ){ Fts5Iter *pRet = 0; Fts5TokenDataIter *pSet = 0; @@ -6963,7 +6969,7 @@ static Fts5Iter *fts5SetupTokendataIter( pSmall = 0; for(ii=0; iinSeg; ii++){ Fts5SegIter *pII = &pNew->aSeg[ii]; - if( 0==fts5IsTokendataPrefix(&pII->term, pToken, nToken, bPrefix) ){ + if( 0==fts5IsTokendataPrefix(&pII->term, pToken, nToken) ){ fts5SegIterSetEOF(pII); } if( pII->pLeaf && (!pSmall || fts5BufferCompare(pSmall, &pII->term)>0) ){ @@ -6999,6 +7005,7 @@ static Fts5Iter *fts5SetupTokendataIter( pRet = fts5MultiIterAlloc(p, 0); } if( pRet ){ + pRet->nSeg = 0; pRet->pTokenDataIter = pSet; if( pSet ){ fts5IterSetOutputsTokendata(pRet); @@ -7071,9 +7078,9 @@ int sqlite3Fts5IndexQuery( } } - if( (bTokendata && iIdx==0) || iIdx>pConfig->nPrefix ){ + if( bTokendata && iIdx==0 ){ buf.p[0] = FTS5_MAIN_PREFIX; - pRet = fts5SetupTokendataIter(p, buf.p, nToken+1, pColset, iIdx>0); + pRet = fts5SetupTokendataIter(p, buf.p, nToken+1, pColset); }else if( iIdx<=pConfig->nPrefix ){ /* Straight index lookup */ Fts5Structure *pStruct = fts5StructureRead(p); @@ -7121,7 +7128,8 @@ int sqlite3Fts5IndexQuery( int sqlite3Fts5IterNext(Fts5IndexIter *pIndexIter){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; assert( pIter->pIndex->rc==SQLITE_OK ); - if( pIter->pTokenDataIter ){ + if( pIter->nSeg==0 ){ + assert( pIter->pTokenDataIter ); fts5TokendataIterNext(pIter, 0, 0); }else{ fts5MultiIterNext(pIter->pIndex, pIter, 0, 0); @@ -7158,7 +7166,8 @@ int sqlite3Fts5IterNextScan(Fts5IndexIter *pIndexIter){ */ int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIndexIter, i64 iMatch){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; - if( pIter->pTokenDataIter ){ + if( pIter->nSeg==0 ){ + assert( pIter->pTokenDataIter ); fts5TokendataIterNext(pIter, 1, iMatch); }else{ fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch); @@ -7177,14 +7186,161 @@ const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){ return (z ? &z[1] : 0); } +static void fts5TokendataMerge( + Fts5TokenDataMap *a1, int n1, + Fts5TokenDataMap *a2, int n2, + Fts5TokenDataMap *aOut +){ + int i1 = 0; + int i2 = 0; + + assert( n1>=0 && n2>=0 ); + while( i1=n2 || (i1nMap * sizeof(Fts5TokenDataMap); + + aTmp = (Fts5TokenDataMap*)sqlite3Fts5MallocZero(&p->rc, nByte); + if( aTmp ){ + Fts5TokenDataMap *a1 = pT->aMap; + Fts5TokenDataMap *a2 = aTmp; + i64 nHalf; + + for(nHalf=1; nHalfnMap; nHalf=nHalf*2){ + int i1; + for(i1=0; i1nMap; i1+=(nHalf*2)){ + int n1 = MIN(nHalf, pT->nMap-i1); + int n2 = MIN(nHalf, pT->nMap-i1-n1); + fts5TokendataMerge(&a1[i1], n1, &a1[i1+n1], n2, &a2[i1]); + } + SWAPVAL(Fts5TokenDataMap*, a1, a2); + } + + if( a1!=pT->aMap ){ + memcpy(pT->aMap, a1, pT->nMap*sizeof(Fts5TokenDataMap)); + } + sqlite3_free(aTmp); + +#ifdef SQLITE_DEBUG + { + int ii; + for(ii=1; iinMap; ii++){ + Fts5TokenDataMap *p1 = &pT->aMap[ii-1]; + Fts5TokenDataMap *p2 = &pT->aMap[ii]; + assert( p1->iRowidiRowid + || (p1->iRowid==p2->iRowid && p1->iPos<=p2->iPos) + ); + } + } +#endif + } +} + +static int fts5SetupPrefixIterTokendata( + Fts5Iter *pIter, + const char *pToken, + int nToken +){ + Fts5Index *p = pIter->pIndex; + Fts5Buffer token = {0, 0, 0}; + Fts5TokenDataIter *pT = 0; + + fts5BufferGrow(&p->rc, &token, nToken+1); + pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(*pT)); + + if( p->rc==SQLITE_OK ){ + const int flags = FTS5INDEX_QUERY_SCAN + | FTS5INDEX_QUERY_SKIPEMPTY + | FTS5INDEX_QUERY_NOOUTPUT; + Fts5Structure *pStruct = 0; + Fts5Iter *p1 = 0; /* Iterator used to find tokendata */ + + int bNewTerm = 1; + int iTermOff = 0; + int nTermByte = 0; + + /* Fill in the token prefix to search for */ + token.p[0] = FTS5_MAIN_PREFIX; + memcpy(&token.p[1], pToken, nToken); + token.n = nToken+1; + + /* Grab a reference to the table structure. That will be released before + ** this function returns. */ + pStruct = fts5StructureRead(p); + + fts5MultiIterNew(p, pStruct, flags, 0, token.p, token.n, -1, 0, &p1); + fts5IterSetOutputCb(&p->rc, p1); + for( /* no-op */ ; + fts5MultiIterEof(p, p1)==0; + fts5MultiIterNext2(p, p1, &bNewTerm) + ){ + i64 iPos = 0; + int iPosOff = 0; + + Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; + p1->xSetOutputs(p1, pSeg); + + if( bNewTerm ){ + int nTerm = pSeg->term.n; + const u8 *pTerm = pSeg->term.p; + assert_nc( memcmp(token.p, pTerm, MIN(token.n, nTerm))<=0 ); + if( nTermterms.n; + fts5BufferAppendBlob(&p->rc, &pT->terms, nTermByte, pTerm+1); + } + + while( 0==sqlite3Fts5PoslistNext64( + p1->base.pData, p1->base.nData, &iPosOff, &iPos + ) ){ + fts5TokendataIterAppendMap( + p, pT, iTermOff, nTermByte, p1->base.iRowid, iPos + ); + } + } + + /* fts5SetupPrefixIter */ + fts5MultiIterFree(p1); + fts5StructureRelease(pStruct); + + fts5TokendataIterSortMap(p, pT); + } + + if( p->rc==SQLITE_OK ){ + pIter->pTokenDataIter = pT; + }else{ + fts5TokendataIterDelete(pT); + } + fts5BufferFree(&token); + + return fts5IndexReturn(p); +} + /* ** This is used by xInstToken() to access the token at offset iOff, column ** iCol of row iRowid. The token is returned via output variables *ppOut ** and *pnOut. The iterator passed as the first argument must be a tokendata=1 ** iterator (pIter->pTokenDataIter!=0). +** +** pToken/nToken: */ int sqlite3Fts5IterToken( Fts5IndexIter *pIndexIter, + const char *pToken, int nToken, i64 iRowid, int iCol, int iOff, @@ -7192,13 +7348,22 @@ int sqlite3Fts5IterToken( ){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; Fts5TokenDataIter *pT = pIter->pTokenDataIter; - Fts5TokenDataMap *aMap = pT->aMap; i64 iPos = (((i64)iCol)<<32) + iOff; - + Fts5TokenDataMap *aMap = 0; int i1 = 0; - int i2 = pT->nMap; + int i2 = 0; int iTest = 0; + assert( pT || (pToken && pIter->nSeg>0) ); + if( pT==0 ){ + int rc = fts5SetupPrefixIterTokendata(pIter, pToken, nToken); + if( rc!=SQLITE_OK ) return rc; + pT = pIter->pTokenDataIter; + } + + i2 = pT->nMap; + aMap = pT->aMap; + while( i2>i1 ){ iTest = (i1 + i2) / 2; @@ -7221,9 +7386,15 @@ int sqlite3Fts5IterToken( } if( i2>i1 ){ - Fts5Iter *pMap = pT->apIter[aMap[iTest].iIter]; - *ppOut = (const char*)pMap->aSeg[0].term.p+1; - *pnOut = pMap->aSeg[0].term.n-1; + if( pIter->nSeg==0 ){ + Fts5Iter *pMap = pT->apIter[aMap[iTest].iIter]; + *ppOut = (const char*)pMap->aSeg[0].term.p+1; + *pnOut = pMap->aSeg[0].term.n-1; + }else{ + Fts5TokenDataMap *p = &aMap[iTest]; + *ppOut = (const char*)&pT->terms.p[p->iIter]; + *pnOut = aMap[iTest].nByte; + } } return SQLITE_OK; @@ -7235,7 +7406,9 @@ int sqlite3Fts5IterToken( */ void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter *pIndexIter){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; - if( pIter && pIter->pTokenDataIter ){ + if( pIter && pIter->pTokenDataIter + && (pIter->nSeg==0 || pIter->pIndex->pConfig->eDetail!=FTS5_DETAIL_FULL) + ){ pIter->pTokenDataIter->nMap = 0; } } @@ -7255,17 +7428,30 @@ int sqlite3Fts5IndexIterWriteTokendata( Fts5Iter *pIter = (Fts5Iter*)pIndexIter; Fts5TokenDataIter *pT = pIter->pTokenDataIter; Fts5Index *p = pIter->pIndex; - int ii; + i64 iPos = (((i64)iCol)<<32) + iOff; assert( p->pConfig->eDetail!=FTS5_DETAIL_FULL ); - assert( pIter->pTokenDataIter ); - - for(ii=0; iinIter; ii++){ - Fts5Buffer *pTerm = &pT->apIter[ii]->aSeg[0].term; - if( nToken==pTerm->n-1 && memcmp(pToken, pTerm->p+1, nToken)==0 ) break; - } - if( iinIter ){ - fts5TokendataIterAppendMap(p, pT, ii, iRowid, (((i64)iCol)<<32) + iOff); + assert( pIter->pTokenDataIter || pIter->nSeg>0 ); + if( pIter->nSeg>0 ){ + /* This is a prefix term iterator. */ + Fts5TokenDataIter *pT = pIter->pTokenDataIter; + if( pT==0 ){ + pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(*pT)); + pIter->pTokenDataIter = pT; + } + if( pT ){ + fts5TokendataIterAppendMap(p, pT, pT->terms.n, nToken, iRowid, iPos); + fts5BufferAppendBlob(&p->rc, &pT->terms, nToken, (const u8*)pToken); + } + }else{ + int ii; + for(ii=0; iinIter; ii++){ + Fts5Buffer *pTerm = &pT->apIter[ii]->aSeg[0].term; + if( nToken==pTerm->n-1 && memcmp(pToken, pTerm->p+1, nToken)==0 ) break; + } + if( iinIter ){ + fts5TokendataIterAppendMap(p, pT, ii, 0, iRowid, iPos); + } } return fts5IndexReturn(p); } diff --git a/manifest b/manifest index 31fbff973a..91a17cd96b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sthis\sbranch. -D 2024-09-24T15:43:52.339 +C Change\sthe\sway\stokendata\sindexes\sare\scollected\sfor\sprefix\squeries. +D 2024-09-25T18:55:11.223 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -93,13 +93,13 @@ F ext/fts3/unicode/mkunicode.tcl 63db9624ccf70d4887836c320eda93ab552f21008f3be7e F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl 009cf59c77afa86d137b0cca3e3b1a5efbe2264faa2df233f9a7aa8563926d15 F ext/fts5/fts5.h efaaac0df3d3bc740383044c144b582f47921aafa21d7b10eb98f42c24c740b0 -F ext/fts5/fts5Int.h 93aba03ca417f403b07b2ab6f50aa0e0c1b8b031917a9026b81520e7047a168e +F ext/fts5/fts5Int.h 83a7af3fee07d5163bf7bf97db310544fcc143c94acb13dbced7e06ae8025a18 F ext/fts5/fts5_aux.c 65a0468dd177d6093aa9ae1622e6d86b0136b8d267c62c0ad6493ad1e9a3d759 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 F ext/fts5/fts5_config.c da21548ddbc1a457cb42545f527065221ede8ada6a734891b8c34317a7a9506b -F ext/fts5/fts5_expr.c 1f60d81aa4703435f98f46bbb41fb2a2efa898423fec070a2b3f7a02f177ac64 +F ext/fts5/fts5_expr.c 69b8d976058512c07dfe86e229521b7a871768157bd1607cedf1a5038dfd72c9 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c aadd271f3c2048418298377908dd09d496753a5c7da84161a9c86ca8c1e78e9a +F ext/fts5/fts5_index.c 8dfb22c5e42cd56d3abbe107a5561fc3b4f731fc4c821ac049482d9dedc50acc F ext/fts5/fts5_main.c 4503498d3453e29a3cd89dacaba029011e89cb8c481a6241611d106e7a369bd4 F ext/fts5/fts5_storage.c 3332497823c3d171cf56379f2bd8c971ce15a19aadacff961106462022c92470 F ext/fts5/fts5_tcl.c 4db9258a7882c5eac0da4433042132aaf15b87dd1e1636c7a6ca203abd2c8bfe @@ -2214,8 +2214,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 97c2824f471e7e622c4a166947a6e8162cae891345101539829a6fcec83373fe 42bb941584a1ac922ee6b0b6ecadce71c9259555563cf49913a6f820f3f9b887 -R cb71c4478793484afa282b1a3cf11afa +P 9945206e6e26a48a49b9747650d299eb983cc21a3a61c621cd81f0bbc85a74d7 +R d12c6f9d3e41d3b7f32c957f52650189 U dan -Z efd6491e97a3bc8cd929e608bea6d1a9 +Z 0b9676f39cb90827333f01676ed89ac5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index dc02877987..92e4aa62cf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9945206e6e26a48a49b9747650d299eb983cc21a3a61c621cd81f0bbc85a74d7 +204ddf4e726b695dd12ab4a945ec2461655aa0bcc38b74e970f07ed2ac43c6ff From 7d56669bc48ba12c23713219a2e10923758b915f Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 28 Sep 2024 20:45:11 +0000 Subject: [PATCH 3/8] Rationalize some of the new code on this branch. FossilOrigin-Name: 66f209ba40e7de49b304d7931ff38a4994038452aab08e9347286a234c6f7075 --- ext/fts5/fts5_index.c | 309 ++++++++++++++++++++++++------------------ manifest | 12 +- manifest.uuid | 2 +- 3 files changed, 181 insertions(+), 142 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 1efbe5a7b4..ede091650d 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -6198,6 +6198,101 @@ static void fts5MergePrefixLists( *p1 = out; } + +static int fts5VisitPrefixRange( + Fts5Index *p, + Fts5Colset *pColset, + u8 *pToken, + int nToken, + void (*xVisit)(Fts5Index*, void *pCtx, Fts5Iter *pIter, const u8*, int), + void *pCtx +){ + const int flags = FTS5INDEX_QUERY_SCAN + | FTS5INDEX_QUERY_SKIPEMPTY + | FTS5INDEX_QUERY_NOOUTPUT; + Fts5Iter *p1 = 0; /* Iterator used to gather data from index */ + int bNewTerm = 1; + Fts5Structure *pStruct = fts5StructureRead(p); + + fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1); + fts5IterSetOutputCb(&p->rc, p1); + for( /* no-op */ ; + fts5MultiIterEof(p, p1)==0; + fts5MultiIterNext2(p, p1, &bNewTerm) + ){ + Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; + int nNew = 0; + const u8 *pNew = 0; + + p1->xSetOutputs(p1, pSeg); + + + if( bNewTerm ){ + nNew = pSeg->term.n; + pNew = pSeg->term.p; + if( nNewrc; +} + +typedef struct PrefixSetupCtx PrefixSetupCtx; +struct PrefixSetupCtx { + void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*); + void (*xAppend)(Fts5Index*, u64, Fts5Iter*, Fts5Buffer*); + i64 iLastRowid; + int nMerge; + Fts5Buffer *aBuf; + int nBuf; + Fts5Buffer doclist; +}; + +static void prefixIterSetupCb( + Fts5Index *p, + void *pCtx, + Fts5Iter *p1, + const u8 *pNew, + int nNew +){ + PrefixSetupCtx *pSetup = (PrefixSetupCtx*)pCtx; + const int nMerge = pSetup->nMerge; + + if( p1->base.nData>0 ){ + if( p1->base.iRowid<=pSetup->iLastRowid && pSetup->doclist.n>0 ){ + int i; + for(i=0; p->rc==SQLITE_OK && pSetup->doclist.n; i++){ + int i1 = i*nMerge; + int iStore; + assert( i1+nMerge<=pSetup->nBuf ); + for(iStore=i1; iStoreaBuf[iStore].n==0 ){ + fts5BufferSwap(&pSetup->doclist, &pSetup->aBuf[iStore]); + fts5BufferZero(&pSetup->doclist); + break; + } + } + if( iStore==i1+nMerge ){ + pSetup->xMerge(p, &pSetup->doclist, nMerge, &pSetup->aBuf[i1]); + for(iStore=i1; iStoreaBuf[iStore]); + } + } + } + pSetup->iLastRowid = 0; + } + + pSetup->xAppend( + p, (u64)p1->base.iRowid-(u64)pSetup->iLastRowid, p1, &pSetup->doclist + ); + pSetup->iLastRowid = p1->base.iRowid; + } +} + static void fts5SetupPrefixIter( Fts5Index *p, /* Index to read from */ int bDesc, /* True for "ORDER BY rowid DESC" */ @@ -6208,38 +6303,30 @@ static void fts5SetupPrefixIter( Fts5Iter **ppIter /* OUT: New iterator */ ){ Fts5Structure *pStruct; - Fts5Buffer *aBuf; - int nBuf = 32; - int nMerge = 1; + PrefixSetupCtx s; + + memset(&s, 0, sizeof(s)); + s.nMerge = 1; + s.iLastRowid = 0; + s.nBuf = 32; - void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*); - void (*xAppend)(Fts5Index*, u64, Fts5Iter*, Fts5Buffer*); if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ - xMerge = fts5MergeRowidLists; - xAppend = fts5AppendRowid; + s.xMerge = fts5MergeRowidLists; + s.xAppend = fts5AppendRowid; }else{ - nMerge = FTS5_MERGE_NLIST-1; - nBuf = nMerge*8; /* Sufficient to merge (16^8)==(2^32) lists */ - xMerge = fts5MergePrefixLists; - xAppend = fts5AppendPoslist; + s.nMerge = FTS5_MERGE_NLIST-1; + s.nBuf = s.nMerge*8; /* Sufficient to merge (16^8)==(2^32) lists */ + s.xMerge = fts5MergePrefixLists; + s.xAppend = fts5AppendPoslist; } - aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); + s.aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*s.nBuf); pStruct = fts5StructureRead(p); - assert( p->rc!=SQLITE_OK || (aBuf && pStruct) ); + assert( p->rc!=SQLITE_OK || (s.aBuf && pStruct) ); if( p->rc==SQLITE_OK ){ - const int flags = FTS5INDEX_QUERY_SCAN - | FTS5INDEX_QUERY_SKIPEMPTY - | FTS5INDEX_QUERY_NOOUTPUT; int i; - i64 iLastRowid = 0; - Fts5Iter *p1 = 0; /* Iterator used to gather data from index */ Fts5Data *pData; - Fts5Buffer doclist; - int bNewTerm = 1; - - memset(&doclist, 0, sizeof(doclist)); /* If iIdx is non-zero, then it is the number of a prefix-index for ** prefixes 1 character longer than the prefix being queried for. That @@ -6247,6 +6334,7 @@ static void fts5SetupPrefixIter( ** corresponding to the prefix itself. That one is extracted from the ** main term index here. */ if( iIdx!=0 ){ + Fts5Iter *p1 = 0; /* Iterator used to gather data from index */ int dummy = 0; const int f2 = FTS5INDEX_QUERY_SKIPEMPTY|FTS5INDEX_QUERY_NOOUTPUT; pToken[0] = FTS5_MAIN_PREFIX; @@ -6259,82 +6347,41 @@ static void fts5SetupPrefixIter( Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; p1->xSetOutputs(p1, pSeg); if( p1->base.nData ){ - xAppend(p, (u64)p1->base.iRowid-(u64)iLastRowid, p1, &doclist); - iLastRowid = p1->base.iRowid; + s.xAppend(p, (u64)p1->base.iRowid-(u64)s.iLastRowid, p1, &s.doclist); + s.iLastRowid = p1->base.iRowid; } } fts5MultiIterFree(p1); } pToken[0] = FTS5_MAIN_PREFIX + iIdx; - fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1); - fts5IterSetOutputCb(&p->rc, p1); + fts5VisitPrefixRange( + p, pColset, pToken, nToken, prefixIterSetupCb, (void*)&s + ); - for( /* no-op */ ; - fts5MultiIterEof(p, p1)==0; - fts5MultiIterNext2(p, p1, &bNewTerm) - ){ - Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; - int nTerm = pSeg->term.n; - const u8 *pTerm = pSeg->term.p; - p1->xSetOutputs(p1, pSeg); - - assert_nc( memcmp(pToken, pTerm, MIN(nToken, nTerm))<=0 ); - if( bNewTerm ){ - if( nTermbase.nData==0 ) continue; - if( p1->base.iRowid<=iLastRowid && doclist.n>0 ){ - for(i=0; p->rc==SQLITE_OK && doclist.n; i++){ - int i1 = i*nMerge; - int iStore; - assert( i1+nMerge<=nBuf ); - for(iStore=i1; iStorebase.iRowid-(u64)iLastRowid, p1, &doclist); - iLastRowid = p1->base.iRowid; - } - - assert( (nBuf%nMerge)==0 ); - for(i=0; irc==SQLITE_OK ){ - xMerge(p, &doclist, nMerge, &aBuf[i]); + s.xMerge(p, &s.doclist, s.nMerge, &s.aBuf[i]); } - for(iFree=i; iFreep = (u8*)&pData[1]; - pData->nn = pData->szLeaf = doclist.n; - if( doclist.n ) memcpy(pData->p, doclist.p, doclist.n); + pData->nn = pData->szLeaf = s.doclist.n; + if( s.doclist.n ) memcpy(pData->p, s.doclist.p, s.doclist.n); fts5MultiIterNew2(p, pData, bDesc, ppIter); } - fts5BufferFree(&doclist); } + fts5BufferFree(&s.doclist); fts5StructureRelease(pStruct); - sqlite3_free(aBuf); + sqlite3_free(s.aBuf); } @@ -7021,7 +7068,6 @@ static Fts5Iter *fts5SetupTokendataIter( return pRet; } - /* ** Open a new iterator to iterate though all rowid that match the ** specified token or token prefix. @@ -7046,9 +7092,11 @@ int sqlite3Fts5IndexQuery( int bTokendata = pConfig->bTokendata; if( nToken>0 ) memcpy(&buf.p[1], pToken, nToken); - /* The NOTOKENDATA flag is set when it is known that tokendata data will - ** not be required. e.g. for queries performed as part of an - ** integrity-check, or by the fts5vocab module. */ + /* The NOTOKENDATA flag is set when each token in a tokendata=1 table + ** should be treated individually, instead of merging all those with + ** a common prefix into a single entry. This is used, for example, by + ** queries performed as part of an integrity-check, or by the fts5vocab + ** module. */ if( flags & (FTS5INDEX_QUERY_NOTOKENDATA|FTS5INDEX_QUERY_SCAN) ){ bTokendata = 0; } @@ -7092,7 +7140,7 @@ int sqlite3Fts5IndexQuery( fts5StructureRelease(pStruct); } }else{ - /* Scan multiple terms in the main index */ + /* Scan multiple terms in the main index for a prefix query. */ int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; fts5SetupPrefixIter(p, bDesc, iPrefixIdx, buf.p, nToken+1, pColset,&pRet); if( pRet==0 ){ @@ -7250,6 +7298,39 @@ static void fts5TokendataIterSortMap(Fts5Index *p, Fts5TokenDataIter *pT){ } } +typedef struct TokendataSetupCtx TokendataSetupCtx; +struct TokendataSetupCtx { + Fts5TokenDataIter *pT; + int iTermOff; + int nTermByte; +}; + +static void prefixIterSetupTokendataCb( + Fts5Index *p, + void *pCtx, + Fts5Iter *p1, + const u8 *pNew, + int nNew +){ + TokendataSetupCtx *pSetup = (TokendataSetupCtx*)pCtx; + int iPosOff = 0; + i64 iPos = 0; + + if( pNew ){ + pSetup->nTermByte = nNew-1; + pSetup->iTermOff = pSetup->pT->terms.n; + fts5BufferAppendBlob(&p->rc, &pSetup->pT->terms, nNew-1, pNew+1); + } + + while( 0==sqlite3Fts5PoslistNext64( + p1->base.pData, p1->base.nData, &iPosOff, &iPos + ) ){ + fts5TokendataIterAppendMap(p, + pSetup->pT, pSetup->iTermOff, pSetup->nTermByte, p1->base.iRowid, iPos + ); + } +} + static int fts5SetupPrefixIterTokendata( Fts5Iter *pIter, const char *pToken, @@ -7257,73 +7338,31 @@ static int fts5SetupPrefixIterTokendata( ){ Fts5Index *p = pIter->pIndex; Fts5Buffer token = {0, 0, 0}; - Fts5TokenDataIter *pT = 0; + TokendataSetupCtx ctx; + + memset(&ctx, 0, sizeof(ctx)); fts5BufferGrow(&p->rc, &token, nToken+1); - pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(*pT)); + ctx.pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(*ctx.pT)); if( p->rc==SQLITE_OK ){ - const int flags = FTS5INDEX_QUERY_SCAN - | FTS5INDEX_QUERY_SKIPEMPTY - | FTS5INDEX_QUERY_NOOUTPUT; - Fts5Structure *pStruct = 0; - Fts5Iter *p1 = 0; /* Iterator used to find tokendata */ - - int bNewTerm = 1; - int iTermOff = 0; - int nTermByte = 0; /* Fill in the token prefix to search for */ token.p[0] = FTS5_MAIN_PREFIX; memcpy(&token.p[1], pToken, nToken); token.n = nToken+1; - /* Grab a reference to the table structure. That will be released before - ** this function returns. */ - pStruct = fts5StructureRead(p); + fts5VisitPrefixRange( + p, 0, token.p, token.n, prefixIterSetupTokendataCb, (void*)&ctx + ); - fts5MultiIterNew(p, pStruct, flags, 0, token.p, token.n, -1, 0, &p1); - fts5IterSetOutputCb(&p->rc, p1); - for( /* no-op */ ; - fts5MultiIterEof(p, p1)==0; - fts5MultiIterNext2(p, p1, &bNewTerm) - ){ - i64 iPos = 0; - int iPosOff = 0; - - Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; - p1->xSetOutputs(p1, pSeg); - - if( bNewTerm ){ - int nTerm = pSeg->term.n; - const u8 *pTerm = pSeg->term.p; - assert_nc( memcmp(token.p, pTerm, MIN(token.n, nTerm))<=0 ); - if( nTermterms.n; - fts5BufferAppendBlob(&p->rc, &pT->terms, nTermByte, pTerm+1); - } - - while( 0==sqlite3Fts5PoslistNext64( - p1->base.pData, p1->base.nData, &iPosOff, &iPos - ) ){ - fts5TokendataIterAppendMap( - p, pT, iTermOff, nTermByte, p1->base.iRowid, iPos - ); - } - } - - /* fts5SetupPrefixIter */ - fts5MultiIterFree(p1); - fts5StructureRelease(pStruct); - - fts5TokendataIterSortMap(p, pT); + fts5TokendataIterSortMap(p, ctx.pT); } if( p->rc==SQLITE_OK ){ - pIter->pTokenDataIter = pT; + pIter->pTokenDataIter = ctx.pT; }else{ - fts5TokendataIterDelete(pT); + fts5TokendataIterDelete(ctx.pT); } fts5BufferFree(&token); diff --git a/manifest b/manifest index 91a17cd96b..0d2ad1c4fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sway\stokendata\sindexes\sare\scollected\sfor\sprefix\squeries. -D 2024-09-25T18:55:11.223 +C Rationalize\ssome\sof\sthe\snew\scode\son\sthis\sbranch. +D 2024-09-28T20:45:11.387 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -99,7 +99,7 @@ F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70 F ext/fts5/fts5_config.c da21548ddbc1a457cb42545f527065221ede8ada6a734891b8c34317a7a9506b F ext/fts5/fts5_expr.c 69b8d976058512c07dfe86e229521b7a871768157bd1607cedf1a5038dfd72c9 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c 8dfb22c5e42cd56d3abbe107a5561fc3b4f731fc4c821ac049482d9dedc50acc +F ext/fts5/fts5_index.c c1005920192146452a3545500761ecc8cfab84572d251e8536103a01899f67d5 F ext/fts5/fts5_main.c 4503498d3453e29a3cd89dacaba029011e89cb8c481a6241611d106e7a369bd4 F ext/fts5/fts5_storage.c 3332497823c3d171cf56379f2bd8c971ce15a19aadacff961106462022c92470 F ext/fts5/fts5_tcl.c 4db9258a7882c5eac0da4433042132aaf15b87dd1e1636c7a6ca203abd2c8bfe @@ -2214,8 +2214,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 9945206e6e26a48a49b9747650d299eb983cc21a3a61c621cd81f0bbc85a74d7 -R d12c6f9d3e41d3b7f32c957f52650189 +P 204ddf4e726b695dd12ab4a945ec2461655aa0bcc38b74e970f07ed2ac43c6ff +R da06610bae74973a44f69a92b9b60e12 U dan -Z 0b9676f39cb90827333f01676ed89ac5 +Z ca9c653ea5575a07920ac6ddffa15d1d # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 92e4aa62cf..f550314bcd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -204ddf4e726b695dd12ab4a945ec2461655aa0bcc38b74e970f07ed2ac43c6ff +66f209ba40e7de49b304d7931ff38a4994038452aab08e9347286a234c6f7075 From d2a88e961a78582050bd56e3676bf54fef01fb3f Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 1 Oct 2024 20:38:08 +0000 Subject: [PATCH 4/8] Rationalize code further. And add tests. FossilOrigin-Name: 0ca002a4ab88f3e7ae1e6e518038157eaa20759f57888c2ed7e50cb92bd96348 --- ext/fts5/fts5_index.c | 165 +++++++++++++++++++++-------- ext/fts5/test/fts5origintext3.test | 23 ++++ manifest | 14 +-- manifest.uuid | 2 +- 4 files changed, 151 insertions(+), 53 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index ede091650d..f6f51d7343 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -6199,15 +6199,45 @@ static void fts5MergePrefixLists( } -static int fts5VisitPrefixRange( - Fts5Index *p, - Fts5Colset *pColset, - u8 *pToken, - int nToken, +/* +** Iterate through a range of entries in the FTS index, invoking the xVisit +** callback for each of them. +** +** Parameter pToken points to an nToken buffer containing an FTS index term +** (i.e. a document term with the preceding 1 byte index identifier - +** FTS5_MAIN_PREFIX or similar). If bPrefix is true, then the call visits +** all entries for terms that have pToken/nToken as a prefix. If bPrefix +** is false, then only entries with pToken/nToken as the entire key are +** visited. +** +** If the current table is a tokendata=1 table, then if bPrefix is true then +** each index term is treated separately. However, if bPrefix is false, then +** all index terms corresponding to pToken/nToken are collapsed into a single +** term before the callback is invoked. +** +** The callback invoked for each entry visited is specified by paramter xVisit. +** Each time it is invoked, it is passed a pointer to the Fts5Index object, +** a copy of the 7th paramter to this function (pCtx) and a pointer to the +** iterator that indicates the current entry. If the current entry is the +** first with a new term (i.e. different from that of the previous entry, +** including the very first term), then the final two parameters are passed +** a pointer to the term and its size in bytes, respectively. If the current +** entry is not the first associated with its term, these two parameters +** are passed 0. +** +** If parameter pColset is not NULL, then it is used to filter entries before +** the callback is invoked. +*/ +static int fts5VisitEntries( + Fts5Index *p, /* Fts5 index object */ + Fts5Colset *pColset, /* Columns filter to apply, or NULL */ + u8 *pToken, /* Buffer containing token */ + int nToken, /* Size of buffer pToken in bytes */ + int bPrefix, /* True for a prefix scan */ void (*xVisit)(Fts5Index*, void *pCtx, Fts5Iter *pIter, const u8*, int), - void *pCtx + void *pCtx /* Passed as second argument to xVisit() */ ){ - const int flags = FTS5INDEX_QUERY_SCAN + const int flags = (bPrefix ? FTS5INDEX_QUERY_SCAN : 0) | FTS5INDEX_QUERY_SKIPEMPTY | FTS5INDEX_QUERY_NOOUTPUT; Fts5Iter *p1 = 0; /* Iterator used to gather data from index */ @@ -6226,7 +6256,6 @@ static int fts5VisitPrefixRange( p1->xSetOutputs(p1, pSeg); - if( bNewTerm ){ nNew = pSeg->term.n; pNew = pSeg->term.p; @@ -6241,6 +6270,9 @@ static int fts5VisitPrefixRange( return p->rc; } +/* +** Context object passed by fts5SetupPrefixIter() to fts5VisitEntries(). +*/ typedef struct PrefixSetupCtx PrefixSetupCtx; struct PrefixSetupCtx { void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*); @@ -6252,6 +6284,9 @@ struct PrefixSetupCtx { Fts5Buffer doclist; }; +/* +** fts5VisitEntries() callback used by fts5SetupPrefixIter() +*/ static void prefixIterSetupCb( Fts5Index *p, void *pCtx, @@ -6325,6 +6360,7 @@ static void fts5SetupPrefixIter( assert( p->rc!=SQLITE_OK || (s.aBuf && pStruct) ); if( p->rc==SQLITE_OK ){ + void *pCtx = (void*)&s; int i; Fts5Data *pData; @@ -6334,30 +6370,12 @@ static void fts5SetupPrefixIter( ** corresponding to the prefix itself. That one is extracted from the ** main term index here. */ if( iIdx!=0 ){ - Fts5Iter *p1 = 0; /* Iterator used to gather data from index */ - int dummy = 0; - const int f2 = FTS5INDEX_QUERY_SKIPEMPTY|FTS5INDEX_QUERY_NOOUTPUT; pToken[0] = FTS5_MAIN_PREFIX; - fts5MultiIterNew(p, pStruct, f2, pColset, pToken, nToken, -1, 0, &p1); - fts5IterSetOutputCb(&p->rc, p1); - for(; - fts5MultiIterEof(p, p1)==0; - fts5MultiIterNext2(p, p1, &dummy) - ){ - Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; - p1->xSetOutputs(p1, pSeg); - if( p1->base.nData ){ - s.xAppend(p, (u64)p1->base.iRowid-(u64)s.iLastRowid, p1, &s.doclist); - s.iLastRowid = p1->base.iRowid; - } - } - fts5MultiIterFree(p1); + fts5VisitEntries(p, pColset, pToken, nToken, 0, prefixIterSetupCb, pCtx); } pToken[0] = FTS5_MAIN_PREFIX + iIdx; - fts5VisitPrefixRange( - p, pColset, pToken, nToken, prefixIterSetupCb, (void*)&s - ); + fts5VisitEntries(p, pColset, pToken, nToken, 1, prefixIterSetupCb, pCtx); assert( (s.nBuf%s.nMerge)==0 ); for(i=0; i0, not just a container for +** Fts5TokenDataMap structures), then this variable is an index into +** the apIter[] array. The corresponding term is that which the iterator +** at apIter[iIter] currently points to. +** +** Or, if the Fts5TokenDataIter iterator is just a container object +** (nIter==0), then iIter is an index into the term.p[] buffer where +** the term is stored. +** +** nByte: +** In the case where iIter is an index into term.p[], this variable +** is the size of the term in bytes. If iIter is an index into apIter[], +** this variable is unused. */ struct Fts5TokenDataMap { i64 iRowid; /* Row this token is located in */ i64 iPos; /* Position of token */ - int iIter; /* Iterator token was read from */ int nByte; /* Length of token in bytes (or 0) */ }; /* ** An object used to supplement Fts5Iter for tokendata=1 iterators. +** +** This object serves two purposes. The first is as a container for an array +** of Fts5TokenDataMap structures, which are used to find the token required +** when the xInstToken() API is used. This is done by the nMapAlloc, nMap and +** aMap[] variables. */ struct Fts5TokenDataIter { - int nMap; - int nMapAlloc; - Fts5TokenDataMap *aMap; + int nMapAlloc; /* Allocated size of aMap[] in entries */ + int nMap; /* Number of valid entries in aMap[] */ + Fts5TokenDataMap *aMap; /* Array of (rowid+pos -> token) mappings */ /* The following are used for prefix-queries only. */ Fts5Buffer terms; @@ -7234,10 +7280,18 @@ const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){ return (z ? &z[1] : 0); } +/* +** The two input arrays - a1[] and a2[] - are in sorted order. This function +** merges the two arrays together and writes the result to output array +** aOut[]. aOut[] is guaranteed to be large enough to hold the result. +** +** Duplicate entries are copied into the output. So the size of the output +** array is always (n1+n2) entries. +*/ static void fts5TokendataMerge( - Fts5TokenDataMap *a1, int n1, - Fts5TokenDataMap *a2, int n2, - Fts5TokenDataMap *aOut + Fts5TokenDataMap *a1, int n1, /* Input array 1 */ + Fts5TokenDataMap *a2, int n2, /* Input array 2 */ + Fts5TokenDataMap *aOut /* Output array */ ){ int i1 = 0; int i2 = 0; @@ -7258,6 +7312,12 @@ static void fts5TokendataMerge( } } +/* +** Sort the contents of the pT->aMap[] array. +** +** The sorting algorithm requries a malloc(). If this fails, an error code +** is left in Fts5Index.rc before returning. +*/ static void fts5TokendataIterSortMap(Fts5Index *p, Fts5TokenDataIter *pT){ Fts5TokenDataMap *aTmp = 0; int nByte = pT->nMap * sizeof(Fts5TokenDataMap); @@ -7298,13 +7358,23 @@ static void fts5TokendataIterSortMap(Fts5Index *p, Fts5TokenDataIter *pT){ } } +/* +** fts5VisitEntries() context object used by fts5SetupPrefixIterTokendata() +** to pass data to prefixIterSetupTokendataCb(). +*/ typedef struct TokendataSetupCtx TokendataSetupCtx; struct TokendataSetupCtx { - Fts5TokenDataIter *pT; - int iTermOff; - int nTermByte; + Fts5TokenDataIter *pT; /* Object being populated with mappings */ + int iTermOff; /* Offset of current term in terms.p[] */ + int nTermByte; /* Size of current term in bytes */ }; +/* +** fts5VisitEntries() callback used by fts5SetupPrefixIterTokendata(). This +** callback adds an entry to the Fts5TokenDataIter.aMap[] array for each +** position in the current position-list. It doesn't matter that some of +** these may be out of order - they will be sorted later. +*/ static void prefixIterSetupTokendataCb( Fts5Index *p, void *pCtx, @@ -7331,10 +7401,15 @@ static void prefixIterSetupTokendataCb( } } +/* +** pIter is a prefix query. This function populates pIter->pTokenDataIter +** with an Fts5TokenDataIter object containing mappings for all rows +** matched by the query. +*/ static int fts5SetupPrefixIterTokendata( Fts5Iter *pIter, - const char *pToken, - int nToken + const char *pToken, /* Token prefix to search for */ + int nToken /* Size of pToken in bytes */ ){ Fts5Index *p = pIter->pIndex; Fts5Buffer token = {0, 0, 0}; @@ -7352,8 +7427,8 @@ static int fts5SetupPrefixIterTokendata( memcpy(&token.p[1], pToken, nToken); token.n = nToken+1; - fts5VisitPrefixRange( - p, 0, token.p, token.n, prefixIterSetupTokendataCb, (void*)&ctx + fts5VisitEntries( + p, 0, token.p, token.n, 1, prefixIterSetupTokendataCb, (void*)&ctx ); fts5TokendataIterSortMap(p, ctx.pT); diff --git a/ext/fts5/test/fts5origintext3.test b/ext/fts5/test/fts5origintext3.test index 9dda2a5748..a4bca0de9b 100644 --- a/ext/fts5/test/fts5origintext3.test +++ b/ext/fts5/test/fts5origintext3.test @@ -95,6 +95,29 @@ foreach_detail_mode $testprefix { do_execsql_test 1.8 { SELECT rowid FROM ft2('aaa AND bbb'); } {10 24} do_execsql_test 1.9 { SELECT rowid FROM ft2('bbb AND aaa'); } {10 24} + do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft3 USING fts5( + x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%, + prefix=2 + ); + } + do_execsql_test 2.1 { + INSERT INTO ft3(rowid, x) VALUES(1, 'one'); + INSERT INTO ft3(rowid, x) VALUES(2, 'ONE'); + INSERT INTO ft3(rowid, x) VALUES(3, 'ONT'); + INSERT INTO ft3(rowid, x) VALUES(4, 'on'); + INSERT INTO ft3(rowid, x) VALUES(5, 'On'); + } + + do_execsql_test 2.2 { + SELECT rowid FROM ft3('on*'); + } {1 2 3 4 5} + + do_execsql_test 2.3 { + SELECT rowid, insttoken(ft3, 0, 0) FROM ft3('on*'); + } {1 one 2 one.ONE 3 ont.ONT 4 on 5 on.On} + + } finish_test diff --git a/manifest b/manifest index 0d2ad1c4fb..7b755eff5b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rationalize\ssome\sof\sthe\snew\scode\son\sthis\sbranch. -D 2024-09-28T20:45:11.387 +C Rationalize\scode\sfurther.\sAnd\sadd\stests. +D 2024-10-01T20:38:08.239 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724 @@ -99,7 +99,7 @@ F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70 F ext/fts5/fts5_config.c da21548ddbc1a457cb42545f527065221ede8ada6a734891b8c34317a7a9506b F ext/fts5/fts5_expr.c 69b8d976058512c07dfe86e229521b7a871768157bd1607cedf1a5038dfd72c9 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c c1005920192146452a3545500761ecc8cfab84572d251e8536103a01899f67d5 +F ext/fts5/fts5_index.c 9b2b9636ccefd6140c0ad7a44c51c2ea39f377753a13f06a2e6792215b62cede F ext/fts5/fts5_main.c 4503498d3453e29a3cd89dacaba029011e89cb8c481a6241611d106e7a369bd4 F ext/fts5/fts5_storage.c 3332497823c3d171cf56379f2bd8c971ce15a19aadacff961106462022c92470 F ext/fts5/fts5_tcl.c 4db9258a7882c5eac0da4433042132aaf15b87dd1e1636c7a6ca203abd2c8bfe @@ -203,7 +203,7 @@ F ext/fts5/test/fts5optimize2.test 795d4ae5f66a7239cf8d5aef4c2ea96aeb8bcd907bd9b F ext/fts5/test/fts5optimize3.test 1653029284e10e0715246819893ba30565c4ead0d0fc470adae92c353ea857d3 F ext/fts5/test/fts5origintext.test 63d5b0dc00f0104add8960da0705a70bffd4d86b6feb6ddbb38bff21141d42f0 F ext/fts5/test/fts5origintext2.test f4505ff79bf7369f2b8b10b9cef7476049d844e20b37f29cad3a8b8d5ac6f9ba -F ext/fts5/test/fts5origintext3.test 45c33cf0c91a9ca0e36d298462db3edc7c8fe45fd185649a9dbfd66bb670058b +F ext/fts5/test/fts5origintext3.test 1f5174a9f4cf42f58f833dbfb314940793ca4723854ec2651e7530ddb35a66a6 F ext/fts5/test/fts5origintext4.test 0d3ef0a8038f471dbc83001c34fe5f7ae39b571bfc209670771eb28bc0fc50e8 F ext/fts5/test/fts5origintext5.test ee12b440ec335e5b422d1668aca0051b52ff28b6ee67073e8bbc29f509fd562b F ext/fts5/test/fts5phrase.test bb2554bb61d15f859678c96dc89a7de415cd5fc3b7b54c29b82a0d0ad138091c @@ -2214,8 +2214,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0 -P 204ddf4e726b695dd12ab4a945ec2461655aa0bcc38b74e970f07ed2ac43c6ff -R da06610bae74973a44f69a92b9b60e12 +P 66f209ba40e7de49b304d7931ff38a4994038452aab08e9347286a234c6f7075 +R 364baf490f2f60461704cce12defe7d5 U dan -Z ca9c653ea5575a07920ac6ddffa15d1d +Z 98928b751e601e5ab0ec38779c287b09 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index f550314bcd..7b4f75a7f2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -66f209ba40e7de49b304d7931ff38a4994038452aab08e9347286a234c6f7075 +0ca002a4ab88f3e7ae1e6e518038157eaa20759f57888c2ed7e50cb92bd96348 From 090b8649be3603920327a0ae45c2cf9384d8615b Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 2 Nov 2024 19:10:50 +0000 Subject: [PATCH 5/8] Allow an fts5 table or query to be configured to collect xInstToken data for any prefix terms as part of the first parse of the main index, if any. FossilOrigin-Name: 46929ae92b26f02bc70de9931b21a8a7cf9a2453d5fb07f68b712f62e28e9152 --- ext/fts5/fts5Int.h | 3 +- ext/fts5/fts5_config.c | 13 + ext/fts5/fts5_index.c | 501 +++++++++++++++-------------- ext/fts5/fts5_main.c | 28 +- ext/fts5/test/fts5origintext.test | 121 ++++--- ext/fts5/test/fts5origintext3.test | 47 ++- manifest | 24 +- manifest.uuid | 2 +- 8 files changed, 430 insertions(+), 309 deletions(-) diff --git a/ext/fts5/fts5Int.h b/ext/fts5/fts5Int.h index bb72f45c7b..832f9ad477 100644 --- a/ext/fts5/fts5Int.h +++ b/ext/fts5/fts5Int.h @@ -247,7 +247,8 @@ struct Fts5Config { char *zRank; /* Name of rank function */ char *zRankArgs; /* Arguments to rank function */ int bSecureDelete; /* 'secure-delete' */ - int nDeleteMerge; /* 'deletemerge' */ + int nDeleteMerge; /* 'deletemerge' */ + int bPrefixInsttoken; /* 'prefix-insttoken' */ /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ char **pzErrmsg; diff --git a/ext/fts5/fts5_config.c b/ext/fts5/fts5_config.c index a674b44d0b..eea82b046d 100644 --- a/ext/fts5/fts5_config.c +++ b/ext/fts5/fts5_config.c @@ -1026,6 +1026,19 @@ int sqlite3Fts5ConfigSetValue( }else{ pConfig->bSecureDelete = (bVal ? 1 : 0); } + } + + else if( 0==sqlite3_stricmp(zKey, "insttoken") ){ + int bVal = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + bVal = sqlite3_value_int(pVal); + } + if( bVal<0 ){ + *pbBadkey = 1; + }else{ + pConfig->bPrefixInsttoken = (bVal ? 1 : 0); + } + }else{ *pbBadkey = 1; } diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index 04f0a6740c..e7028e411c 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -6260,6 +6260,7 @@ static int fts5VisitEntries( const u8 *pNew = 0; p1->xSetOutputs(p1, pSeg); + if( p->rc ) break; if( bNewTerm ){ nNew = pSeg->term.n; @@ -6275,6 +6276,247 @@ static int fts5VisitEntries( return p->rc; } + +/* +** Usually, a tokendata=1 iterator (struct Fts5TokenDataIter) accumulates an +** array of these for each row it visits (so all iRowid fields are the same). +** Or, for an iterator used by an "ORDER BY rank" query, it accumulates an +** array of these for the entire query (in which case iRowid fields may take +** a variety of values). +** +** Each instance in the array indicates the iterator (and therefore term) +** associated with position iPos of rowid iRowid. This is used by the +** xInstToken() API. +** +** iRowid: +** Rowid for the current entry. +** +** iPos: +** Position of current entry within row. In the usual ((iCol<<32)+iOff) +** format (e.g. see macros FTS5_POS2COLUMN() and FTS5_POS2OFFSET()). +** +** iIter: +** If the Fts5TokenDataIter iterator that the entry is part of is +** actually an iterator (i.e. with nIter>0, not just a container for +** Fts5TokenDataMap structures), then this variable is an index into +** the apIter[] array. The corresponding term is that which the iterator +** at apIter[iIter] currently points to. +** +** Or, if the Fts5TokenDataIter iterator is just a container object +** (nIter==0), then iIter is an index into the term.p[] buffer where +** the term is stored. +** +** nByte: +** In the case where iIter is an index into term.p[], this variable +** is the size of the term in bytes. If iIter is an index into apIter[], +** this variable is unused. +*/ +struct Fts5TokenDataMap { + i64 iRowid; /* Row this token is located in */ + i64 iPos; /* Position of token */ + int iIter; /* Iterator token was read from */ + int nByte; /* Length of token in bytes (or 0) */ +}; + +/* +** An object used to supplement Fts5Iter for tokendata=1 iterators. +** +** This object serves two purposes. The first is as a container for an array +** of Fts5TokenDataMap structures, which are used to find the token required +** when the xInstToken() API is used. This is done by the nMapAlloc, nMap and +** aMap[] variables. +*/ +struct Fts5TokenDataIter { + int nMapAlloc; /* Allocated size of aMap[] in entries */ + int nMap; /* Number of valid entries in aMap[] */ + Fts5TokenDataMap *aMap; /* Array of (rowid+pos -> token) mappings */ + + /* The following are used for prefix-queries only. */ + Fts5Buffer terms; + + /* The following are used for other full-token tokendata queries only. */ + int nIter; + int nIterAlloc; + Fts5PoslistReader *aPoslistReader; + int *aPoslistToIter; + Fts5Iter *apIter[1]; +}; + +/* +** The two input arrays - a1[] and a2[] - are in sorted order. This function +** merges the two arrays together and writes the result to output array +** aOut[]. aOut[] is guaranteed to be large enough to hold the result. +** +** Duplicate entries are copied into the output. So the size of the output +** array is always (n1+n2) entries. +*/ +static void fts5TokendataMerge( + Fts5TokenDataMap *a1, int n1, /* Input array 1 */ + Fts5TokenDataMap *a2, int n2, /* Input array 2 */ + Fts5TokenDataMap *aOut /* Output array */ +){ + int i1 = 0; + int i2 = 0; + + assert( n1>=0 && n2>=0 ); + while( i1=n2 || (i1rc==SQLITE_OK ){ + if( pT->nMap==pT->nMapAlloc ){ + int nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64; + int nByte = nNew * sizeof(Fts5TokenDataMap); + Fts5TokenDataMap *aNew; + + aNew = (Fts5TokenDataMap*)sqlite3_realloc(pT->aMap, nByte); + if( aNew==0 ){ + p->rc = SQLITE_NOMEM; + return; + } + + pT->aMap = aNew; + pT->nMapAlloc = nNew; + } + + pT->aMap[pT->nMap].iRowid = iRowid; + pT->aMap[pT->nMap].iPos = iPos; + pT->aMap[pT->nMap].iIter = iIter; + pT->aMap[pT->nMap].nByte = nByte; + pT->nMap++; + } +} + +/* +** Sort the contents of the pT->aMap[] array. +** +** The sorting algorithm requries a malloc(). If this fails, an error code +** is left in Fts5Index.rc before returning. +*/ +static void fts5TokendataIterSortMap(Fts5Index *p, Fts5TokenDataIter *pT){ + Fts5TokenDataMap *aTmp = 0; + int nByte = pT->nMap * sizeof(Fts5TokenDataMap); + + aTmp = (Fts5TokenDataMap*)sqlite3Fts5MallocZero(&p->rc, nByte); + if( aTmp ){ + Fts5TokenDataMap *a1 = pT->aMap; + Fts5TokenDataMap *a2 = aTmp; + i64 nHalf; + + for(nHalf=1; nHalfnMap; nHalf=nHalf*2){ + int i1; + for(i1=0; i1nMap; i1+=(nHalf*2)){ + int n1 = MIN(nHalf, pT->nMap-i1); + int n2 = MIN(nHalf, pT->nMap-i1-n1); + fts5TokendataMerge(&a1[i1], n1, &a1[i1+n1], n2, &a2[i1]); + } + SWAPVAL(Fts5TokenDataMap*, a1, a2); + } + + if( a1!=pT->aMap ){ + memcpy(pT->aMap, a1, pT->nMap*sizeof(Fts5TokenDataMap)); + } + sqlite3_free(aTmp); + +#ifdef SQLITE_DEBUG + { + int ii; + for(ii=1; iinMap; ii++){ + Fts5TokenDataMap *p1 = &pT->aMap[ii-1]; + Fts5TokenDataMap *p2 = &pT->aMap[ii]; + assert( p1->iRowidiRowid + || (p1->iRowid==p2->iRowid && p1->iPos<=p2->iPos) + ); + } + } +#endif + } +} + +/* +** Delete an Fts5TokenDataIter structure and its contents. +*/ +static void fts5TokendataIterDelete(Fts5TokenDataIter *pSet){ + if( pSet ){ + int ii; + for(ii=0; iinIter; ii++){ + fts5MultiIterFree(pSet->apIter[ii]); + } + fts5BufferFree(&pSet->terms); + sqlite3_free(pSet->aPoslistReader); + sqlite3_free(pSet->aMap); + sqlite3_free(pSet); + } +} + + +/* +** fts5VisitEntries() context object used by fts5SetupPrefixIterTokendata() +** to pass data to prefixIterSetupTokendataCb(). +*/ +typedef struct TokendataSetupCtx TokendataSetupCtx; +struct TokendataSetupCtx { + Fts5TokenDataIter *pT; /* Object being populated with mappings */ + int iTermOff; /* Offset of current term in terms.p[] */ + int nTermByte; /* Size of current term in bytes */ +}; + +/* +** fts5VisitEntries() callback used by fts5SetupPrefixIterTokendata(). This +** callback adds an entry to the Fts5TokenDataIter.aMap[] array for each +** position in the current position-list. It doesn't matter that some of +** these may be out of order - they will be sorted later. +*/ +static void prefixIterSetupTokendataCb( + Fts5Index *p, + void *pCtx, + Fts5Iter *p1, + const u8 *pNew, + int nNew +){ + TokendataSetupCtx *pSetup = (TokendataSetupCtx*)pCtx; + int iPosOff = 0; + i64 iPos = 0; + + if( pNew ){ + pSetup->nTermByte = nNew-1; + pSetup->iTermOff = pSetup->pT->terms.n; + fts5BufferAppendBlob(&p->rc, &pSetup->pT->terms, nNew-1, pNew+1); + } + + while( 0==sqlite3Fts5PoslistNext64( + p1->base.pData, p1->base.nData, &iPosOff, &iPos + ) ){ + fts5TokendataIterAppendMap(p, + pSetup->pT, pSetup->iTermOff, pSetup->nTermByte, p1->base.iRowid, iPos + ); + } +} + + /* ** Context object passed by fts5SetupPrefixIter() to fts5VisitEntries(). */ @@ -6287,6 +6529,7 @@ struct PrefixSetupCtx { Fts5Buffer *aBuf; int nBuf; Fts5Buffer doclist; + TokendataSetupCtx *pTokendata; }; /* @@ -6331,6 +6574,10 @@ static void prefixIterSetupCb( ); pSetup->iLastRowid = p1->base.iRowid; } + + if( pSetup->pTokendata ){ + prefixIterSetupTokendataCb(p, (void*)pSetup->pTokendata, p1, pNew, nNew); + } } static void fts5SetupPrefixIter( @@ -6344,11 +6591,21 @@ static void fts5SetupPrefixIter( ){ Fts5Structure *pStruct; PrefixSetupCtx s; + TokendataSetupCtx s2; memset(&s, 0, sizeof(s)); + memset(&s2, 0, sizeof(s2)); + s.nMerge = 1; s.iLastRowid = 0; s.nBuf = 32; + if( iIdx==0 + && p->pConfig->eDetail==FTS5_DETAIL_FULL + && p->pConfig->bPrefixInsttoken + ){ + s.pTokendata = &s2; + s2.pT = (Fts5TokenDataIter*)fts5IdxMalloc(p, sizeof(*s2.pT)); + } if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ s.xMerge = fts5MergeRowidLists; @@ -6400,8 +6657,15 @@ static void fts5SetupPrefixIter( if( s.doclist.n ) memcpy(pData->p, s.doclist.p, s.doclist.n); fts5MultiIterNew2(p, pData, bDesc, ppIter); } + + if( p->rc==SQLITE_OK && s.pTokendata ){ + fts5TokendataIterSortMap(p, s2.pT); + (*ppIter)->pTokenDataIter = s2.pT; + s2.pT = 0; + } } + fts5TokendataIterDelete(s2.pT); fts5BufferFree(&s.doclist); fts5StructureRelease(pStruct); sqlite3_free(s.aBuf); @@ -6658,71 +6922,6 @@ static void fts5SegIterSetEOF(Fts5SegIter *pSeg){ pSeg->pLeaf = 0; } -/* -** Usually, a tokendata=1 iterator (struct Fts5TokenDataIter) accumulates an -** array of these for each row it visits (so all iRowid fields are the same). -** Or, for an iterator used by an "ORDER BY rank" query, it accumulates an -** array of these for the entire query (in which case iRowid fields may take -** a variety of values). -** -** Each instance in the array indicates the iterator (and therefore term) -** associated with position iPos of rowid iRowid. This is used by the -** xInstToken() API. -** -** iRowid: -** Rowid for the current entry. -** -** iPos: -** Position of current entry within row. In the usual ((iCol<<32)+iOff) -** format (e.g. see macros FTS5_POS2COLUMN() and FTS5_POS2OFFSET()). -** -** iIter: -** If the Fts5TokenDataIter iterator that the entry is part of is -** actually an iterator (i.e. with nIter>0, not just a container for -** Fts5TokenDataMap structures), then this variable is an index into -** the apIter[] array. The corresponding term is that which the iterator -** at apIter[iIter] currently points to. -** -** Or, if the Fts5TokenDataIter iterator is just a container object -** (nIter==0), then iIter is an index into the term.p[] buffer where -** the term is stored. -** -** nByte: -** In the case where iIter is an index into term.p[], this variable -** is the size of the term in bytes. If iIter is an index into apIter[], -** this variable is unused. -*/ -struct Fts5TokenDataMap { - i64 iRowid; /* Row this token is located in */ - i64 iPos; /* Position of token */ - int iIter; /* Iterator token was read from */ - int nByte; /* Length of token in bytes (or 0) */ -}; - -/* -** An object used to supplement Fts5Iter for tokendata=1 iterators. -** -** This object serves two purposes. The first is as a container for an array -** of Fts5TokenDataMap structures, which are used to find the token required -** when the xInstToken() API is used. This is done by the nMapAlloc, nMap and -** aMap[] variables. -*/ -struct Fts5TokenDataIter { - int nMapAlloc; /* Allocated size of aMap[] in entries */ - int nMap; /* Number of valid entries in aMap[] */ - Fts5TokenDataMap *aMap; /* Array of (rowid+pos -> token) mappings */ - - /* The following are used for prefix-queries only. */ - Fts5Buffer terms; - - /* The following are used for other full-token tokendata queries only. */ - int nIter; - int nIterAlloc; - Fts5PoslistReader *aPoslistReader; - int *aPoslistToIter; - Fts5Iter *apIter[1]; -}; - /* ** This function appends iterator pAppend to Fts5TokenDataIter pIn and ** returns the result. @@ -6759,57 +6958,6 @@ static Fts5TokenDataIter *fts5AppendTokendataIter( return pRet; } -/* -** Delete an Fts5TokenDataIter structure and its contents. -*/ -static void fts5TokendataIterDelete(Fts5TokenDataIter *pSet){ - if( pSet ){ - int ii; - for(ii=0; iinIter; ii++){ - fts5MultiIterFree(pSet->apIter[ii]); - } - fts5BufferFree(&pSet->terms); - sqlite3_free(pSet->aPoslistReader); - sqlite3_free(pSet->aMap); - sqlite3_free(pSet); - } -} - -/* -** Append a mapping to the token-map belonging to object pT. -*/ -static void fts5TokendataIterAppendMap( - Fts5Index *p, - Fts5TokenDataIter *pT, - int iIter, - int nByte, - i64 iRowid, - i64 iPos -){ - if( p->rc==SQLITE_OK ){ - if( pT->nMap==pT->nMapAlloc ){ - int nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64; - int nByte = nNew * sizeof(Fts5TokenDataMap); - Fts5TokenDataMap *aNew; - - aNew = (Fts5TokenDataMap*)sqlite3_realloc(pT->aMap, nByte); - if( aNew==0 ){ - p->rc = SQLITE_NOMEM; - return; - } - - pT->aMap = aNew; - pT->nMapAlloc = nNew; - } - - pT->aMap[pT->nMap].iRowid = iRowid; - pT->aMap[pT->nMap].iPos = iPos; - pT->aMap[pT->nMap].iIter = iIter; - pT->aMap[pT->nMap].nByte = nByte; - pT->nMap++; - } -} - /* ** The iterator passed as the only argument must be a tokendata=1 iterator ** (pIter->pTokenDataIter!=0). This function sets the iterator output @@ -7285,127 +7433,6 @@ const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){ return (z ? &z[1] : 0); } -/* -** The two input arrays - a1[] and a2[] - are in sorted order. This function -** merges the two arrays together and writes the result to output array -** aOut[]. aOut[] is guaranteed to be large enough to hold the result. -** -** Duplicate entries are copied into the output. So the size of the output -** array is always (n1+n2) entries. -*/ -static void fts5TokendataMerge( - Fts5TokenDataMap *a1, int n1, /* Input array 1 */ - Fts5TokenDataMap *a2, int n2, /* Input array 2 */ - Fts5TokenDataMap *aOut /* Output array */ -){ - int i1 = 0; - int i2 = 0; - - assert( n1>=0 && n2>=0 ); - while( i1=n2 || (i1aMap[] array. -** -** The sorting algorithm requries a malloc(). If this fails, an error code -** is left in Fts5Index.rc before returning. -*/ -static void fts5TokendataIterSortMap(Fts5Index *p, Fts5TokenDataIter *pT){ - Fts5TokenDataMap *aTmp = 0; - int nByte = pT->nMap * sizeof(Fts5TokenDataMap); - - aTmp = (Fts5TokenDataMap*)sqlite3Fts5MallocZero(&p->rc, nByte); - if( aTmp ){ - Fts5TokenDataMap *a1 = pT->aMap; - Fts5TokenDataMap *a2 = aTmp; - i64 nHalf; - - for(nHalf=1; nHalfnMap; nHalf=nHalf*2){ - int i1; - for(i1=0; i1nMap; i1+=(nHalf*2)){ - int n1 = MIN(nHalf, pT->nMap-i1); - int n2 = MIN(nHalf, pT->nMap-i1-n1); - fts5TokendataMerge(&a1[i1], n1, &a1[i1+n1], n2, &a2[i1]); - } - SWAPVAL(Fts5TokenDataMap*, a1, a2); - } - - if( a1!=pT->aMap ){ - memcpy(pT->aMap, a1, pT->nMap*sizeof(Fts5TokenDataMap)); - } - sqlite3_free(aTmp); - -#ifdef SQLITE_DEBUG - { - int ii; - for(ii=1; iinMap; ii++){ - Fts5TokenDataMap *p1 = &pT->aMap[ii-1]; - Fts5TokenDataMap *p2 = &pT->aMap[ii]; - assert( p1->iRowidiRowid - || (p1->iRowid==p2->iRowid && p1->iPos<=p2->iPos) - ); - } - } -#endif - } -} - -/* -** fts5VisitEntries() context object used by fts5SetupPrefixIterTokendata() -** to pass data to prefixIterSetupTokendataCb(). -*/ -typedef struct TokendataSetupCtx TokendataSetupCtx; -struct TokendataSetupCtx { - Fts5TokenDataIter *pT; /* Object being populated with mappings */ - int iTermOff; /* Offset of current term in terms.p[] */ - int nTermByte; /* Size of current term in bytes */ -}; - -/* -** fts5VisitEntries() callback used by fts5SetupPrefixIterTokendata(). This -** callback adds an entry to the Fts5TokenDataIter.aMap[] array for each -** position in the current position-list. It doesn't matter that some of -** these may be out of order - they will be sorted later. -*/ -static void prefixIterSetupTokendataCb( - Fts5Index *p, - void *pCtx, - Fts5Iter *p1, - const u8 *pNew, - int nNew -){ - TokendataSetupCtx *pSetup = (TokendataSetupCtx*)pCtx; - int iPosOff = 0; - i64 iPos = 0; - - if( pNew ){ - pSetup->nTermByte = nNew-1; - pSetup->iTermOff = pSetup->pT->terms.n; - fts5BufferAppendBlob(&p->rc, &pSetup->pT->terms, nNew-1, pNew+1); - } - - while( 0==sqlite3Fts5PoslistNext64( - p1->base.pData, p1->base.nData, &iPosOff, &iPos - ) ){ - fts5TokendataIterAppendMap(p, - pSetup->pT, pSetup->iTermOff, pSetup->nTermByte, p1->base.iRowid, iPos - ); - } -} - /* ** pIter is a prefix query. This function populates pIter->pTokenDataIter ** with an Fts5TokenDataIter object containing mappings for all rows diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c index 5713fccdd1..a65750f8e1 100644 --- a/ext/fts5/fts5_main.c +++ b/ext/fts5/fts5_main.c @@ -93,6 +93,7 @@ struct Fts5Global { #define FTS5_LOCALE_HDR_SIZE ((int)sizeof( ((Fts5Global*)0)->aLocaleHdr )) #define FTS5_LOCALE_HDR(pConfig) ((const u8*)(pConfig->pGlobal->aLocaleHdr)) +#define FTS5_INSTTOKEN_SUBTYPE 73 /* ** Each auxiliary function registered with the FTS5 module is represented @@ -1417,6 +1418,7 @@ static int fts5FilterMethod( sqlite3_value *pRowidGe = 0; /* rowid >= ? expression (or NULL) */ int iCol; /* Column on LHS of MATCH operator */ char **pzErrmsg = pConfig->pzErrmsg; + int bPrefixInsttoken = pConfig->bPrefixInsttoken; int i; int iIdxStr = 0; Fts5Expr *pExpr = 0; @@ -1452,6 +1454,9 @@ static int fts5FilterMethod( rc = fts5ExtractExprText(pConfig, apVal[i], &zText, &bFreeAndReset); if( rc!=SQLITE_OK ) goto filter_out; if( zText==0 ) zText = ""; + if( sqlite3_value_subtype(apVal[i])==FTS5_INSTTOKEN_SUBTYPE ){ + pConfig->bPrefixInsttoken = 1; + } iCol = 0; do{ @@ -1592,6 +1597,7 @@ static int fts5FilterMethod( filter_out: sqlite3Fts5ExprFree(pExpr); pConfig->pzErrmsg = pzErrmsg; + pConfig->bPrefixInsttoken = bPrefixInsttoken; return rc; } @@ -3651,6 +3657,19 @@ static void fts5LocaleFunc( } } +/* +** Implementation of fts5_insttoken() function. +*/ +static void fts5InsttokenFunc( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apArg /* Function arguments */ +){ + assert( nArg==1 ); + sqlite3_result_value(pCtx, apArg[0]); + sqlite3_result_subtype(pCtx, FTS5_INSTTOKEN_SUBTYPE); +} + /* ** Return true if zName is the extension on one of the shadow tables used ** by this module. @@ -3780,10 +3799,17 @@ static int fts5Init(sqlite3 *db){ if( rc==SQLITE_OK ){ rc = sqlite3_create_function( db, "fts5_locale", 2, - SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE|SQLITE_SUBTYPE, p, fts5LocaleFunc, 0, 0 ); } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5_insttoken", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE, + p, fts5InsttokenFunc, 0, 0 + ); + } } /* If SQLITE_FTS5_ENABLE_TEST_MI is defined, assume that the file diff --git a/ext/fts5/test/fts5origintext.test b/ext/fts5/test/fts5origintext.test index 9741f786e8..be77cbfca5 100644 --- a/ext/fts5/test/fts5origintext.test +++ b/ext/fts5/test/fts5origintext.test @@ -22,34 +22,40 @@ ifcapable !fts5 { } foreach_detail_mode $testprefix { +foreach {tn insttoken} { + 1 0 + 2 1 +} { +reset_db sqlite3_fts5_register_origintext db -do_execsql_test 1.0 { +do_execsql_test $tn.1.0 { CREATE VIRTUAL TABLE ft USING fts5( x, tokenize="origintext unicode61", detail=%DETAIL% ); + INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken); CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance); } -do_execsql_test 1.1 { +do_execsql_test $tn.1.1 { INSERT INTO ft VALUES('Hello world'); } -do_execsql_test 1.2 { +do_execsql_test $tn.1.2 { INSERT INTO ft(ft) VALUES('integrity-check'); } proc b {x} { string map [list "\0" "."] $x } db func b b -do_execsql_test 1.3 { +do_execsql_test $tn.1.3 { select b(term) from vocab; } { hello.Hello world } -do_execsql_test 1.4 { +do_execsql_test $tn.1.4 { SELECT rowid FROM ft('Hello'); } {1} @@ -88,33 +94,34 @@ proc document {} { db func document document sqlite3_fts5_register_origintext db -do_execsql_test 2.0 { +do_execsql_test $tn.2.0 { CREATE VIRTUAL TABLE ft USING fts5( x, tokenize="origintext unicode61", detail=%DETAIL% ); + INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken); INSERT INTO ft(ft, rank) VALUES('pgsz', 128); CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance); } -do_test 2.1 { +do_test $tn.2.1 { for {set ii 0} {$ii < 500} {incr ii} { execsql { INSERT INTO ft VALUES( document() ) } } } {} -do_execsql_test 2.2 { +do_execsql_test $tn.2.2 { INSERT INTO ft(ft) VALUES('integrity-check'); } -do_execsql_test 2.3 { +do_execsql_test $tn.2.3 { INSERT INTO ft(ft, rank) VALUES('merge', 16); } -do_execsql_test 2.4 { +do_execsql_test $tn.2.4 { INSERT INTO ft(ft) VALUES('integrity-check'); } -do_execsql_test 2.5 { +do_execsql_test $tn.2.5 { INSERT INTO ft(ft) VALUES('optimize'); } @@ -122,10 +129,11 @@ do_execsql_test 2.5 { reset_db sqlite3_fts5_register_origintext db -do_execsql_test 3.0 { +do_execsql_test $tn.3.0 { CREATE VIRTUAL TABLE ft USING fts5( x, tokenize="origintext unicode61", detail=%DETAIL% ); + INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken); CREATE VIRTUAL TABLE vocab USING fts5vocab(ft, instance); INSERT INTO ft(rowid, x) VALUES(1, 'hello'); @@ -137,16 +145,17 @@ do_execsql_test 3.0 { #db func b b #execsql_pp { SELECT b(term) FROM vocab } -do_execsql_test 3.1.1 { SELECT rowid FROM ft('hello') } 1 -do_execsql_test 3.1.2 { SELECT rowid FROM ft('Hello') } 2 -do_execsql_test 3.1.3 { SELECT rowid FROM ft('HELLO') } 3 +do_execsql_test $tn.3.1.1 { SELECT rowid FROM ft('hello') } 1 +do_execsql_test $tn.3.1.2 { SELECT rowid FROM ft('Hello') } 2 +do_execsql_test $tn.3.1.3 { SELECT rowid FROM ft('HELLO') } 3 -do_execsql_test 3.2 { +do_execsql_test $tn.3.2 { CREATE VIRTUAL TABLE ft2 USING fts5(x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL% ); + INSERT INTO ft2(ft2, rank) VALUES('insttoken', $insttoken); CREATE VIRTUAL TABLE vocab2 USING fts5vocab(ft2, instance); INSERT INTO ft2(rowid, x) VALUES(1, 'hello'); @@ -160,16 +169,16 @@ do_execsql_test 3.2 { #db func b b #execsql_pp { SELECT b(term) FROM vocab } -do_execsql_test 3.3.1 { SELECT rowid FROM ft2('hello') } {1 2 3} -do_execsql_test 3.3.2 { SELECT rowid FROM ft2('Hello') } {1 2 3} -do_execsql_test 3.3.3 { SELECT rowid FROM ft2('HELLO') } {1 2 3} +do_execsql_test $tn.3.3.1 { SELECT rowid FROM ft2('hello') } {1 2 3} +do_execsql_test $tn.3.3.2 { SELECT rowid FROM ft2('Hello') } {1 2 3} +do_execsql_test $tn.3.3.3 { SELECT rowid FROM ft2('HELLO') } {1 2 3} -do_execsql_test 3.3.4 { SELECT rowid FROM ft2('hello*') } {1 2 3 10} +do_execsql_test $tn.3.3.4 { SELECT rowid FROM ft2('hello*') } {1 2 3 10} -do_execsql_test 3.3.5.1 { SELECT rowid FROM ft2('HELLO') ORDER BY rowid DESC} { +do_execsql_test $tn.3.3.5.1 { SELECT rowid FROM ft2('HELLO') ORDER BY rowid DESC} { 3 2 1 } -do_execsql_test 3.3.5.2 { SELECT rowid FROM ft2('HELLO') ORDER BY +rowid DESC} { +do_execsql_test $tn.3.3.5.2 { SELECT rowid FROM ft2('HELLO') ORDER BY +rowid DESC} { 3 2 1 } @@ -183,36 +192,37 @@ proc querytoken {cmd iPhrase iToken} { } sqlite3_fts5_create_function db querytoken querytoken -do_execsql_test 4.0 { +do_execsql_test $tn.4.0 { CREATE VIRTUAL TABLE ft USING fts5( x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL% ); + INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken); INSERT INTO ft VALUES('one two three four'); } -do_execsql_test 4.1 { +do_execsql_test $tn.4.1 { SELECT rowid, querytoken(ft, 0, 0) FROM ft('TwO') } {1 two.TwO} -do_execsql_test 4.2 { +do_execsql_test $tn.4.2 { SELECT rowid, querytoken(ft, 0, 0) FROM ft('one TWO ThreE') } {1 one} -do_execsql_test 4.3 { +do_execsql_test $tn.4.3 { SELECT rowid, querytoken(ft, 1, 0) FROM ft('one TWO ThreE') } {1 two.TWO} if {"%DETAIL%"=="full"} { # Phrase queries are only supported for detail=full. # - do_execsql_test 4.4 { + do_execsql_test $tn.4.4 { SELECT rowid, querytoken(ft, 0, 2) FROM ft('"one TWO ThreE"') } {1 three.ThreE} - do_catchsql_test 4.5 { + do_catchsql_test $tn.4.5 { SELECT rowid, querytoken(ft, 0, 3) FROM ft('"one TWO ThreE"') } {1 SQLITE_RANGE} - do_catchsql_test 4.6 { + do_catchsql_test $tn.4.6 { SELECT rowid, querytoken(ft, 1, 0) FROM ft('"one TWO ThreE"') } {1 SQLITE_RANGE} - do_catchsql_test 4.7 { + do_catchsql_test $tn.4.7 { SELECT rowid, querytoken(ft, -1, 0) FROM ft('"one TWO ThreE"') } {1 SQLITE_RANGE} } @@ -228,14 +238,15 @@ proc insttoken {cmd iIdx iToken} { sqlite3_fts5_create_function db insttoken insttoken fts5_aux_test_functions db -do_execsql_test 5.0 { +do_execsql_test $tn.5.0 { CREATE VIRTUAL TABLE ft USING fts5( x, tokenize='origintext unicode61', tokendata=1, detail=%DETAIL% ); + INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken); INSERT INTO ft VALUES('one ONE One oNe oNE one'); } -do_execsql_test 5.1 { +do_execsql_test $tn.5.1 { SELECT insttoken(ft, 0, 0), insttoken(ft, 1, 0), insttoken(ft, 2, 0), @@ -247,13 +258,37 @@ do_execsql_test 5.1 { one one.ONE one.One one.oNe one.oNE one } -do_execsql_test 5.2 { +do_execsql_test $tn.5.2 { + SELECT insttoken(ft, 0, 0), + insttoken(ft, 1, 0), + insttoken(ft, 2, 0), + insttoken(ft, 3, 0), + insttoken(ft, 4, 0), + insttoken(ft, 5, 0) + FROM ft('on*'); +} { + one one.ONE one.One one.oNe one.oNE one +} + +do_execsql_test $tn.5.3 { + SELECT insttoken(ft, 0, 0), + insttoken(ft, 1, 0), + insttoken(ft, 2, 0), + insttoken(ft, 3, 0), + insttoken(ft, 4, 0), + insttoken(ft, 5, 0) + FROM ft(fts5_insttoken('on*')); +} { + one one.ONE one.One one.oNe one.oNE one +} + +do_execsql_test $tn.5.4 { SELECT insttoken(ft, 1, 0) FROM ft('one'); } { one.ONE } -do_execsql_test 5.3 { +do_execsql_test $tn.5.5 { SELECT fts5_test_poslist(ft) FROM ft('one'); } { {0.0.0 0.0.1 0.0.2 0.0.3 0.0.4 0.0.5} @@ -267,10 +302,11 @@ do_execsql_test 5.3 { # reset_db sqlite3_fts5_register_origintext db -do_execsql_test 6.0 { +do_execsql_test $tn.6.0 { CREATE VIRTUAL TABLE ft USING fts5( x, y, tokenize='origintext unicode61', detail=%DETAIL%, tokendata=0 ); + INSERT INTO ft(ft, rank) VALUES('insttoken', $insttoken); INSERT INTO ft VALUES('One Two', 'Three two'); INSERT INTO ft VALUES('three Three', 'one One'); @@ -286,34 +322,35 @@ proc tokens {cmd} { } sqlite3_fts5_create_function db tokens tokens -do_execsql_test 6.1 { +do_execsql_test $tn.6.1 { SELECT rowid, tokens(ft) FROM ft('One'); } {1 one.One 2 one.One} -do_execsql_test 6.2 { +do_execsql_test $tn.6.2 { SELECT rowid, tokens(ft) FROM ft('on*'); } {1 one.One 2 {one one.One}} -do_execsql_test 6.3 { +do_execsql_test $tn.6.3 { SELECT rowid, tokens(ft) FROM ft('Three*'); } {1 three.Three 2 three.Three} fts5_aux_test_functions db -do_catchsql_test 6.4 { +do_catchsql_test $tn.6.4 { SELECT fts5_test_insttoken(ft, -1, 0) FROM ft('one'); } {1 SQLITE_RANGE} -do_catchsql_test 6.5 { +do_catchsql_test $tn.6.5 { SELECT fts5_test_insttoken(ft, 1, 0) FROM ft('one'); } {1 SQLITE_RANGE} -do_catchsql_test 6.6 { +do_catchsql_test $tn.6.6 { CREATE VIRTUAL TABLE ft2 USING fts5(x, tokendata=2); } {1 {malformed tokendata=... directive}} -do_catchsql_test 6.7 { +do_catchsql_test $tn.6.7 { CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', tokendata=11); } {1 {malformed tokendata=... directive}} +} } finish_test diff --git a/ext/fts5/test/fts5origintext3.test b/ext/fts5/test/fts5origintext3.test index a4bca0de9b..351ab1f617 100644 --- a/ext/fts5/test/fts5origintext3.test +++ b/ext/fts5/test/fts5origintext3.test @@ -22,6 +22,11 @@ ifcapable !fts5 { } foreach_detail_mode $testprefix { + foreach {tn insttoken} { + 1 0 + 2 1 + } { + reset_db sqlite3_fts5_register_origintext db @@ -32,21 +37,25 @@ foreach_detail_mode $testprefix { } sqlite3_fts5_create_function db insttoken insttoken - do_execsql_test 1.0 { + do_execsql_test $tn.1.0 { CREATE VIRTUAL TABLE ft USING fts5( x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL% ); } + + do_execsql_test $tn.1.0.1 { + INSERT INTO ft(ft, rank) VALUES('insttoken', 1); + } - do_execsql_test 1.1 { + do_execsql_test $tn.1.1 { INSERT INTO ft VALUES('Hello world HELLO WORLD hello'); } - do_execsql_test 1.2 { + do_execsql_test $tn.1.2 { SELECT fts5_test_poslist(ft) FROM ft('hello'); } {{0.0.0 0.0.2 0.0.4}} - do_execsql_test 1.3 { + do_execsql_test $tn.1.3 { SELECT insttoken(ft, 0, 0), insttoken(ft, 1, 0), @@ -54,7 +63,15 @@ foreach_detail_mode $testprefix { FROM ft('hello'); } {hello.Hello hello.HELLO hello} - do_execsql_test 1.4 { + do_execsql_test $tn.1.3.1 { + SELECT + insttoken(ft, 0, 0), + insttoken(ft, 1, 0), + insttoken(ft, 2, 0) + FROM ft('hel*'); + } {hello.Hello hello.HELLO hello} + + do_execsql_test $tn.1.4 { SELECT insttoken(ft, 0, 0), insttoken(ft, 1, 0), @@ -62,7 +79,7 @@ foreach_detail_mode $testprefix { FROM ft('hello') ORDER BY rank; } {hello.Hello hello.HELLO hello} - do_execsql_test 1.5 { + do_execsql_test $tn.1.5 { CREATE VIRTUAL TABLE ft2 USING fts5( x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL% ); @@ -71,11 +88,11 @@ foreach_detail_mode $testprefix { INSERT INTO ft2(rowid, x) VALUES(3, 'THREE one two three THREE'); } - do_execsql_test 1.6 { + do_execsql_test $tn.1.6 { SELECT insttoken(ft2, 0, 0), rowid FROM ft2('three') ORDER BY rank; } {three.THREE 3 three 1 three 2} - do_execsql_test 1.7 { + do_execsql_test $tn.1.7 { INSERT INTO ft2(rowid, x) VALUES(10, 'aaa bbb BBB'); INSERT INTO ft2(rowid, x) VALUES(12, 'bbb bbb bbb'); INSERT INTO ft2(rowid, x) VALUES(13, 'bbb bbb bbb'); @@ -92,16 +109,16 @@ foreach_detail_mode $testprefix { INSERT INTO ft2(rowid, x) VALUES(24, 'aaa bbb BBB'); } - do_execsql_test 1.8 { SELECT rowid FROM ft2('aaa AND bbb'); } {10 24} - do_execsql_test 1.9 { SELECT rowid FROM ft2('bbb AND aaa'); } {10 24} + do_execsql_test $tn.1.8 { SELECT rowid FROM ft2('aaa AND bbb'); } {10 24} + do_execsql_test $tn.1.9 { SELECT rowid FROM ft2('bbb AND aaa'); } {10 24} - do_execsql_test 2.0 { + do_execsql_test $tn.2.0 { CREATE VIRTUAL TABLE ft3 USING fts5( x, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%, prefix=2 ); } - do_execsql_test 2.1 { + do_execsql_test $tn.2.1 { INSERT INTO ft3(rowid, x) VALUES(1, 'one'); INSERT INTO ft3(rowid, x) VALUES(2, 'ONE'); INSERT INTO ft3(rowid, x) VALUES(3, 'ONT'); @@ -109,15 +126,15 @@ foreach_detail_mode $testprefix { INSERT INTO ft3(rowid, x) VALUES(5, 'On'); } - do_execsql_test 2.2 { + do_execsql_test $tn.2.2 { SELECT rowid FROM ft3('on*'); } {1 2 3 4 5} - do_execsql_test 2.3 { + do_execsql_test $tn.2.3 { SELECT rowid, insttoken(ft3, 0, 0) FROM ft3('on*'); } {1 one 2 one.ONE 3 ont.ONT 4 on 5 on.On} - + } } finish_test diff --git a/manifest b/manifest index 2bd0c9cf8d..603ced6491 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\sinto\sthis\sbranch. -D 2024-11-01T19:41:22.452 +C Allow\san\sfts5\stable\sor\squery\sto\sbe\sconfigured\sto\scollect\sxInstToken\sdata\sfor\sany\sprefix\sterms\sas\spart\sof\sthe\sfirst\sparse\sof\sthe\smain\sindex,\sif\sany. +D 2024-11-02T19:10:50.264 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md c5b4009dca54d127d2d6033c22fd9cc34f53bedb6ef12c7cbaa468381c74ab28 @@ -49,7 +49,7 @@ F autosetup/cc.tcl 7e2fe943ae9d45cf39e9f5b05b6230df8e719415edea5af06c30eb68680bd F autosetup/default.auto 5cdf016de2140e50f1db190a02039dc42fb390af1dda4cc4853e3042a9ef0e82 F autosetup/jimsh0.c 27ea5f221359ef6c58780fc6c185aadbf8d3bee9a021331a3e5de0eba0dc6de6 F autosetup/pkg-config.tcl 4e635bf39022ff65e0d5434339dd41503ea48fc53822c9c5bde88b02d3d952ba -F autosetup/proj.tcl 68362ca12e1a32fe73ece32b59a8e8e02a3983295f2dd82c9c4700507bade02c w autosetup/hwaci-common.tcl +F autosetup/proj.tcl 68362ca12e1a32fe73ece32b59a8e8e02a3983295f2dd82c9c4700507bade02c F autosetup/system.tcl 3a39d6e0b3bfba526fd39afe07c1d0d325e5a31925013a1ba7c671e1128e31bb F autosetup/tmake.auto eaebc74ad538dfdd3c817c27eefc31930c20510c4f3a3704071f6cb0629ed71f F autosetup/tmake.tcl a275793ec1b6f8708179af0acef1f6f10d46c2920739743f7a8720c6d700c7a9 @@ -106,14 +106,14 @@ F ext/fts3/unicode/mkunicode.tcl 63db9624ccf70d4887836c320eda93ab552f21008f3be7e F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl 009cf59c77afa86d137b0cca3e3b1a5efbe2264faa2df233f9a7aa8563926d15 F ext/fts5/fts5.h 6b4b92df890965567360db5f1ead24fd13a72cb23b95e4ed2ff58d1d89f7aa42 -F ext/fts5/fts5Int.h a282f33a260ddce09dc2b0334d41d83aab0893b2b1656eb83c595a3d0eec2975 +F ext/fts5/fts5Int.h 6abff7dd770dc5969c994c281e6e77fc277ce414d56cc4a62c145cc7036b0b67 F ext/fts5/fts5_aux.c 65a0468dd177d6093aa9ae1622e6d86b0136b8d267c62c0ad6493ad1e9a3d759 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 -F ext/fts5/fts5_config.c a6633d88596758941c625b526075b85d3d9fd1089d8d9eab5db6e8a71fd347ad +F ext/fts5/fts5_config.c e7d8dd062b44a66cd77e5a0f74f23a2354cd1f3f8575afb967b2773c3384f7f8 F ext/fts5/fts5_expr.c 69b8d976058512c07dfe86e229521b7a871768157bd1607cedf1a5038dfd72c9 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c ee650a838fc0591776f7582de578f414959a76cc0118851e4c1f7d13e7365379 -F ext/fts5/fts5_main.c 50eb059e51d730e8e0c77df4e568b018079e112a755c094488b0d5b1aa06afbb +F ext/fts5/fts5_index.c 2cef40d6fdd761229dd4127e0b1ddcb61dfd6a4ac7e73653b7fddbe0075e50be +F ext/fts5/fts5_main.c b2ec6bf97fc378906c0e78c61f10ca8e64f15e03237f2521f7d81736983be378 F ext/fts5/fts5_storage.c 337b05e4c66fc822d031e264d65bde807ec2fab08665ca2cc8aaf9c5fa06792c F ext/fts5/fts5_tcl.c 5b16a249962809b2aaaab964bf58838ea72f30b8b12373cafe612f8cc71e2a40 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee @@ -214,9 +214,9 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618 F ext/fts5/test/fts5optimize.test 264b9101721c17d06d1d174feb743fda3ddc89fad41dee980fef821428258e47 F ext/fts5/test/fts5optimize2.test 795d4ae5f66a7239cf8d5aef4c2ea96aeb8bcd907bd9be0cfe22064fc71a44ed F ext/fts5/test/fts5optimize3.test 1653029284e10e0715246819893ba30565c4ead0d0fc470adae92c353ea857d3 -F ext/fts5/test/fts5origintext.test 63d5b0dc00f0104add8960da0705a70bffd4d86b6feb6ddbb38bff21141d42f0 +F ext/fts5/test/fts5origintext.test 3b73aa036ce5244bb7c5782c5441b979585bdca026accf75d16026a2a8119c09 F ext/fts5/test/fts5origintext2.test f4505ff79bf7369f2b8b10b9cef7476049d844e20b37f29cad3a8b8d5ac6f9ba -F ext/fts5/test/fts5origintext3.test 1f5174a9f4cf42f58f833dbfb314940793ca4723854ec2651e7530ddb35a66a6 +F ext/fts5/test/fts5origintext3.test 4988b6375acc3bbb0515667765f57e389caf449814af9c1095c053f7de2b4223 F ext/fts5/test/fts5origintext4.test 0d3ef0a8038f471dbc83001c34fe5f7ae39b571bfc209670771eb28bc0fc50e8 F ext/fts5/test/fts5origintext5.test ee12b440ec335e5b422d1668aca0051b52ff28b6ee67073e8bbc29f509fd562b F ext/fts5/test/fts5phrase.test bb2554bb61d15f859678c96dc89a7de415cd5fc3b7b54c29b82a0d0ad138091c @@ -2199,8 +2199,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bce9a524de6dda87daa90395cd55713d2d3ccfc090e53a947548e434db5eef5e 2dcc465a7090811ddbc210673f37b4c3a4439c501874abefd403efe1e98f5b17 -R 86617b1841d68e4c10c087c197cdd65c +P 790c56d493c66a2136e24d349d169639809d70bfab6996975a403be568a267a5 +R 71ba4975c4c76073cda6dd2f314d94d1 U dan -Z c01ad82d76d71ffe9e5032c32219c470 +Z 95da39a03d7bb4b9bc58c6dbf7b809e5 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 05f1de1961..94e1307210 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -790c56d493c66a2136e24d349d169639809d70bfab6996975a403be568a267a5 +46929ae92b26f02bc70de9931b21a8a7cf9a2453d5fb07f68b712f62e28e9152 From 8acaa6d039199a11231f087790500c72397d791a Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 6 Nov 2024 16:20:16 +0000 Subject: [PATCH 6/8] Add tests for xInstToken() and prefix queries with various fts5 configurations. FossilOrigin-Name: 9cc04331a01760189d88697233009dbe8a60eda589792ad01b56300499e9f54d --- ext/fts5/test/fts5origintext6.test | 209 +++++++++++++++++++++++++++++ manifest | 11 +- manifest.uuid | 2 +- 3 files changed, 216 insertions(+), 6 deletions(-) create mode 100644 ext/fts5/test/fts5origintext6.test diff --git a/ext/fts5/test/fts5origintext6.test b/ext/fts5/test/fts5origintext6.test new file mode 100644 index 0000000000..7b27e310b9 --- /dev/null +++ b/ext/fts5/test/fts5origintext6.test @@ -0,0 +1,209 @@ +# 2014 Jan 08 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests focused on phrase queries. +# + +source [file join [file dirname [info script]] fts5_common.tcl] +set testprefix fts5origintext6 + +# If SQLITE_ENABLE_FTS5 is not defined, omit this file. +ifcapable !fts5 { + finish_test + return +} + +proc insert_data {tbl} { + db eval " + INSERT INTO $tbl (rowid, x, y) VALUES + (1, 'ChH BDd HhG efc BjJ BGi GBG FdD','ciJ AFf ADf fBJ fhC GFI JEH fcA'), + (2, 'deg AIG Fie jII cCd Hbf igF fEE','GeA Ija gJg EDc HFi DDI dCf aDd'), + (3, 'IJC hga deC Jfa Aeg hfh CcH dfb','ajD hgC Jaf IfH CHe jIG AjD adF'), + (4, 'FiH GJH IDA AiG bBc CGG Eih bIH','hHg JaH aii IHE Ggd gcH gji CGc'), + (5, 'ceg CAd jFI GAB BGg EeC IdH acG','bBC eIG ifH eDE Adj bjb GCj ebA'), + (6, 'Eac Fbh aFF Eea jeG EIj HCc JJH','hbd giE Gfe eiI dEF abE cJf cAb'), + (7, 'dic hAc jEC AiG FEF jHc HiD HBI','aEd ebE Gfi AJG EBA faj GiG jjE'), + (8, 'Fca iEe EgE jjJ gce ijf EGc EBi','gaI dhH bFg CFc HeC CjI Jfg ccH'), + (9, 'cfd iaa HCf iHJ HjG ffh ABb ibi','CfG bia Dai eii Ejg Jeg fCg hDb'), + (10, 'Jjf hJC IID HJj bGB EbJ cgg eBj','jci jhi JAF jIg Bei Bcd cAC AJd'), + (11, 'egG Cdi bFf fEB hfH jDH jia Efd','FAd eCg fAi aiC baC eJG acF iGE'), + (12, 'Ada Gde CJI ADG gJA Cbb ccF iAB','eAE ajC FBB ccd Jgh fJg ieg hGE'), + (13, 'gBb fDG Jdd HdD fiJ Bed Cig iGg','heC FeI iaj gdg ebB giC HaD FIe'), + (14, 'FiI iDd Ffe igI bgB EJf FHG hDF','cjC AeI abf Fah cbJ ffH jEb aib'), + (15, 'jaF hBI jIH Gdh FEc Fij hgj jFh','dGA ADH feh AAI AfJ DbC gBi hGH'), + (16, 'gjH BGg iGj aFE CAH edI idf HEH','hIf DDg fjB hGi cHF BCH FjG Bgd'), + (17, 'iaI JGH hji gcj Dda eeG jDd CBi','cHg jeh caG gIc feF ihG hgJ Abj'), + (18, 'jHI iDB eFf AiH EFB CDb IAj GbC','Ghe dEI gdI jai gib dAG BIa djb'), + (19, 'abI fHG Ccf aAc FDa fiC agF bdB','afi hde IgE bGF cfg DHD diE aca'), + (20, 'IFh eDJ jfh cDg dde JGJ GAf fIJ','IBa EfH faE aeI FIF baJ FGj EIH'), + (21, 'Dee bFC bBA dEI CEj aJI ghA dCH','hBA ddA HJh dfj egI Dij dFE bGE'), + (22, 'JFE BCj FgA afc Jda FGD iHJ HDh','eAI jHe BHD Gah bbD Bgj gbh eGB'), + (23, 'edE CJE FjG aFI edA Cea FId iFe','ABG jcA ddj EEc Dcg hAI agA biA'), + (24, 'AgE cfc eef cGh aFB DcH efJ hcH','eGF HaB diG fgi bdc iGJ FGJ fFB'), + (25, 'aCa AgI GhC DDI hGJ Hgc Gcg bbG','iID Fga jHa jIj idj DFD bAC AFJ'), + (26, 'gjC JGh Fge faa eCA iGG gHE Gai','bDi hFE BbI DHD Adb Fgi hCa Hij'), + (27, 'Eji jEI jhF DFC afH cDh AGc dHA','IDe GcA ChF DIb Bif HfH agD DGh'), + (28, 'gDD AEE Dfg ICf Cbi JdE jgH eEi','eEb dBG FDE jgf cAI FaJ jaA cDd'), + (29, 'cbe Gec hgB Egi bca dHg bAJ jBf','EFB DgD GJc fDb EeE bBA GFC Hbe'), + (30, 'Adc eHB afI hDc Bhh baE hcJ BBd','JAH deg bcF Dab Bgj Gbb JHi FIB'), + (31, 'agF dIj AJJ Hfg cCG hED Igc fHC','JEf eia dHf Ggc Agj geD bEE Gei'), + (32, 'DAd cCe cbJ FjG gJe gba dJA GCf','eAf hFc bGE ABI hHA IcE abF CCE'), + (33, 'fFh jJe DhJ cDJ EBi AfD eFI IhG','fEG GCc Bjd EFF ggg CFe EHd ciB'), + (34, 'Ejb BjI eAF HaD eEJ FaG Eda AHC','Iah hgD EJG fdD cIE Daj IFf eJh'), + (35, 'aHG eCe FjA djJ dAJ jiJ IaE GGB','Acg iEF JfB FIC Eei ggj dic Iii'), + (36, 'Fdb EDF GaF JjB ehH IgC hgi DCG','cag DHI Fah hAJ bbh egG Hia hgJ'), + (37, 'HGg icC JEC AFJ Ddh dhi hfC Ich','fEg bED Bff hCJ EiA cIf bfG cGA'), + (38, 'aEJ jGI BCi FaA ebA BHj cIJ GcC','dCH ADd bGB cFE AgF geD cbG jIc'), + (39, 'JFB bBi heA BFA hgB Ahj EIE CgI','EIJ JFG FJE GeA Hdg HeH ACh GiA'), + (40, 'agB DDC CED igC Dfc DhI eiC fHi','dAB dcg iJF cej Fcc cAc AfB Fdd'), + (41, 'BdF DHj Ege hcG DEd eFa dCf gBb','FBG ChB cej iGd Hbh fCc Ibe Abh'), + (42, 'Bgc DjI cbC jGD bdb hHB IJA IJH','heg cii abb IGf eDe hJc dii fcE'), + (43, 'fhf ECa FiA aDh Jbf CiB Jhe ajD','GFE bIF aeD gDE BIE Jea DfC BEc'), + (44, 'GjE dBj DbJ ICF aDh EEH Ejb jFb','dJj aEc IBg bEG Faf fjA hjf FAF'), + (45, 'BfA efd IIJ AHG dDF eGg dIJ Gcb','Bfj jeb Ahc dAE ACH Dfb ieb dhC'), + (46, 'Ibj ege geC dJh CIi hbD EAG fGA','DEb BFe Bjg FId Fhg HeF JAc BbE'), + (47, 'dhB afC hgG bEJ aIe Cbe iEE JCD','bdg Ajc FGA jbh Jge iAj fIA jbE'), + (48, 'egH iDi bfH iiI hGC jFF Hfd AHB','bjE Beb iCc haB gIH Dea bga dfd'), + (49, 'jgf chc jGc Baj HBb jdE hgh heI','FFB aBd iEB EIG HGf Bbj EIi JbI'), + (50, 'jhe EGi ajA fbH geh EHe FdC bij','jDE bBC gbH HeE dcH iBH IFE AHi'), + (51, 'aCb JiD cgJ Bjj iAI Hbe IAF FhH','ijf bhE Jdf FED dCH bbG HcJ ebH'); + " +} + +foreach_detail_mode $testprefix { +foreach external {0 1 2} { + reset_db + + proc tokens {cmd} { + set ret [list] + for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} { + set txt [$cmd xInstToken $iTok 0] + set txt [string map [list "\0" "."] $txt] + lappend ret $txt + } + set ret + } + sqlite3_fts5_create_function db tokens tokens + sqlite3_fts5_register_origintext db + + set E(0) internal + set E(1) external + set E(2) contentless + set e $E($external) + + db eval { CREATE TABLE ex(x, y) } + switch -- $external { + 0 { + do_execsql_test 1.$e.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, y, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL% + ); + } + } + + 1 { + do_execsql_test 1.$e.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, y, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%, + content=ex + ); + } + } + + 2 { + do_execsql_test 1.$e.0 { + CREATE VIRTUAL TABLE ft USING fts5( + x, y, tokenize="origintext unicode61", tokendata=1, detail=%DETAIL%, + content= + ); + } + } + } + insert_data ex + insert_data ft + + proc prefixquery {prefix bInst bYOnly} { + set ret [list] + db eval { SELECT rowid, x, y FROM ex ORDER BY rowid } { + set row [list] + set bSeen 0 + + set T [concat $x $y] + if {$bYOnly} { set T $y } + + foreach w $T { + if {[string match -nocase $prefix $w]} { + set bSeen 1 + if {$bInst} { + set v [string tolower $w] + if {$w != $v} { append v ".$w" } + lappend row $v + } + } + } + + if {$bSeen} { + lappend ret $rowid + lappend ret $row + } + } + + set ret + } + + proc do_prefixquery_test {tn prefix} { + set bInst [expr {$::e!="contentless" || "%DETAIL%"=="full"}] + set expect [prefixquery $prefix $bInst 0] + set expect2 [prefixquery $prefix $bInst 1] + + uplevel [list do_execsql_test $tn.1 " + SELECT rowid, tokens(ft) FROM ft('$prefix') + " $expect] + uplevel [list do_execsql_test $tn.2 " + SELECT rowid, tokens(ft) FROM ft(fts5_insttoken('$prefix')) + " $expect] + db eval { INSERT INTO ft(ft, rank) VALUES('insttoken', 1) } + uplevel [list do_execsql_test $tn.3 " + SELECT rowid, tokens(ft) FROM ft('$prefix') + " $expect] + db eval { INSERT INTO ft(ft, rank) VALUES('insttoken', 0) } + + if {"%DETAIL%"!="none"} { + uplevel [list do_execsql_test $tn.4 " + SELECT rowid, tokens(ft) FROM ft('y: $prefix') + " $expect2] + uplevel [list do_execsql_test $tn.5 " + SELECT rowid, tokens(ft) FROM ft(fts5_insttoken('y: $prefix')) + " $expect2] + db eval { INSERT INTO ft(ft, rank) VALUES('insttoken', 1) } + uplevel [list do_execsql_test $tn.6 " + SELECT rowid, tokens(ft) FROM ft('y: $prefix') + " $expect2] + db eval { INSERT INTO ft(ft, rank) VALUES('insttoken', 0) } + } + } + + do_prefixquery_test 1.$e.1 a* + do_prefixquery_test 1.$e.2 b* + do_prefixquery_test 1.$e.3 c* + do_prefixquery_test 1.$e.4 d* + do_prefixquery_test 1.$e.5 e* + do_prefixquery_test 1.$e.6 f* + do_prefixquery_test 1.$e.7 g* + do_prefixquery_test 1.$e.8 h* + do_prefixquery_test 1.$e.9 i* + do_prefixquery_test 1.$e.10 j* +}} + + + +finish_test + diff --git a/manifest b/manifest index 603ced6491..d34164b757 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\san\sfts5\stable\sor\squery\sto\sbe\sconfigured\sto\scollect\sxInstToken\sdata\sfor\sany\sprefix\sterms\sas\spart\sof\sthe\sfirst\sparse\sof\sthe\smain\sindex,\sif\sany. -D 2024-11-02T19:10:50.264 +C Add\stests\sfor\sxInstToken()\sand\sprefix\squeries\swith\svarious\sfts5\sconfigurations. +D 2024-11-06T16:20:16.715 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md c5b4009dca54d127d2d6033c22fd9cc34f53bedb6ef12c7cbaa468381c74ab28 @@ -219,6 +219,7 @@ F ext/fts5/test/fts5origintext2.test f4505ff79bf7369f2b8b10b9cef7476049d844e20b3 F ext/fts5/test/fts5origintext3.test 4988b6375acc3bbb0515667765f57e389caf449814af9c1095c053f7de2b4223 F ext/fts5/test/fts5origintext4.test 0d3ef0a8038f471dbc83001c34fe5f7ae39b571bfc209670771eb28bc0fc50e8 F ext/fts5/test/fts5origintext5.test ee12b440ec335e5b422d1668aca0051b52ff28b6ee67073e8bbc29f509fd562b +F ext/fts5/test/fts5origintext6.test 09eb1347cb0dceaebbebf3d3e6bd5d24c7c1006efddc2984540450324bbdafa4 F ext/fts5/test/fts5phrase.test bb2554bb61d15f859678c96dc89a7de415cd5fc3b7b54c29b82a0d0ad138091c F ext/fts5/test/fts5plan.test f8b0d752a818059a934cdc96c0f77de058a67a0a57bb3a8181d28307ab5b1626 F ext/fts5/test/fts5porter.test 15b514fac8690b58e99c330efe5bf5615bc43f2fae4a3cca3f923dbaff55a0c0 @@ -2199,8 +2200,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 790c56d493c66a2136e24d349d169639809d70bfab6996975a403be568a267a5 -R 71ba4975c4c76073cda6dd2f314d94d1 +P 46929ae92b26f02bc70de9931b21a8a7cf9a2453d5fb07f68b712f62e28e9152 +R a22fb948708e266e2094d3645362e84e U dan -Z 95da39a03d7bb4b9bc58c6dbf7b809e5 +Z 3041da3c7f9ea1d87e0998786f60280f # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 94e1307210..20c99b7fae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -46929ae92b26f02bc70de9931b21a8a7cf9a2453d5fb07f68b712f62e28e9152 +9cc04331a01760189d88697233009dbe8a60eda589792ad01b56300499e9f54d From 032bcf72fb8768a4bd995c4bb874e5c2e25165bb Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 20 Nov 2024 20:39:18 +0000 Subject: [PATCH 7/8] Fix compiler warnings caused by variable shadowing. FossilOrigin-Name: 211b305791980b24c4192ffc57a0471473de3fca32bfc146c0eeacedef7a88aa --- ext/fts5/fts5_index.c | 5 ++--- main.mk | 1 - manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/ext/fts5/fts5_index.c b/ext/fts5/fts5_index.c index e7028e411c..8730b69509 100644 --- a/ext/fts5/fts5_index.c +++ b/ext/fts5/fts5_index.c @@ -6389,10 +6389,10 @@ static void fts5TokendataIterAppendMap( if( p->rc==SQLITE_OK ){ if( pT->nMap==pT->nMapAlloc ){ int nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64; - int nByte = nNew * sizeof(Fts5TokenDataMap); + int nAlloc = nNew * sizeof(Fts5TokenDataMap); Fts5TokenDataMap *aNew; - aNew = (Fts5TokenDataMap*)sqlite3_realloc(pT->aMap, nByte); + aNew = (Fts5TokenDataMap*)sqlite3_realloc(pT->aMap, nAlloc); if( aNew==0 ){ p->rc = SQLITE_NOMEM; return; @@ -7580,7 +7580,6 @@ int sqlite3Fts5IndexIterWriteTokendata( assert( pIter->pTokenDataIter || pIter->nSeg>0 ); if( pIter->nSeg>0 ){ /* This is a prefix term iterator. */ - Fts5TokenDataIter *pT = pIter->pTokenDataIter; if( pT==0 ){ pT = (Fts5TokenDataIter*)sqlite3Fts5MallocZero(&p->rc, sizeof(*pT)); pIter->pTokenDataIter = pT; diff --git a/main.mk b/main.mk index d4cfb3b5e7..71b730c0bf 100644 --- a/main.mk +++ b/main.mk @@ -778,7 +778,6 @@ TESTSRC2 = \ $(TOP)/ext/fts3/fts3.c \ $(TOP)/ext/fts3/fts3_aux.c \ $(TOP)/ext/fts3/fts3_expr.c \ - $(TOP)/ext/fts3/fts3_term.c \ $(TOP)/ext/fts3/fts3_tokenizer.c \ $(TOP)/ext/fts3/fts3_write.c \ $(TOP)/ext/session/sqlite3session.c \ diff --git a/manifest b/manifest index 69261eb226..75828653be 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\sinto\sthis\sbranch. -D 2024-11-06T17:31:48.766 +C Fix\scompiler\swarnings\scaused\sby\svariable\sshadowing. +D 2024-11-20T20:39:18.431 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -113,7 +113,7 @@ F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70 F ext/fts5/fts5_config.c e7d8dd062b44a66cd77e5a0f74f23a2354cd1f3f8575afb967b2773c3384f7f8 F ext/fts5/fts5_expr.c 69b8d976058512c07dfe86e229521b7a871768157bd1607cedf1a5038dfd72c9 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1 -F ext/fts5/fts5_index.c 2cef40d6fdd761229dd4127e0b1ddcb61dfd6a4ac7e73653b7fddbe0075e50be +F ext/fts5/fts5_index.c cef6791bd9f9db4305494292d6dd5d24a7379aabf370a4d6b559e16b740fa88e F ext/fts5/fts5_main.c b2ec6bf97fc378906c0e78c61f10ca8e64f15e03237f2521f7d81736983be378 F ext/fts5/fts5_storage.c 337b05e4c66fc822d031e264d65bde807ec2fab08665ca2cc8aaf9c5fa06792c F ext/fts5/fts5_tcl.c aee6ae6d0c6968564c392bf0d09aaabb4d8bea9ca69fd224dc9b44243324acbf @@ -701,7 +701,7 @@ F ext/wasm/tests/opfs/concurrency/test.js d08889a5bb6e61937d0b8cbb78c9efbefbf65a F ext/wasm/tests/opfs/concurrency/worker.js 0a8c1a3e6ebb38aabbee24f122693f1fb29d599948915c76906681bb7da1d3d2 F ext/wasm/wasmfs.make bc8bb227f35d5bd3863a7bd2233437c37472a0d81585979f058f9b9b503bef35 F magic.txt 5ade0bc977aa135e79e3faaea894d5671b26107cc91e70783aa7dc83f22f3ba0 -F main.mk c1a91dc50ffc1cb828b85d767d02d80d9eabeb0ef47e77e57bd72617749c5dbf +F main.mk 3dbee6d7e7678b79531b78048fea187146cb7e0352047ed774003ba4df1fb93a F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test 61e61469e257df0850df4293d7d4d6c2af301421 @@ -2202,8 +2202,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9cc04331a01760189d88697233009dbe8a60eda589792ad01b56300499e9f54d 5495b12569c318d5020b4b5a625a392ef8e777b81c0200624fbbc2a6b5eddef9 -R 43b03038eb634e877d13ab18dcd80180 +P edb842349320eda9550bdfcd5a327949c5512e02f4b993782587b2131a425746 +R 87e3d58f4f7440e00e0a55bcb770c2ab U dan -Z 70a74ef30cc348badfc60dd108f53b23 +Z e1bc2b76da9d8c387af12aec8bd8d459 # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index b5f3f21ebe..259f7f2dfa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -edb842349320eda9550bdfcd5a327949c5512e02f4b993782587b2131a425746 +211b305791980b24c4192ffc57a0471473de3fca32bfc146c0eeacedef7a88aa From e2a3c7232b061e03b05ac4d891737d57a77dd030 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 26 Nov 2024 20:16:34 +0000 Subject: [PATCH 8/8] Update documentation in fts5.h. FossilOrigin-Name: f1e44f703acb415e2ff7d7f87fa05fc874ef1c432095bb3ccb45c6ad2fd085a9 --- ext/fts5/fts5.h | 19 ++++++++++++++++++- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/ext/fts5/fts5.h b/ext/fts5/fts5.h index 5305a6915d..6c4d238959 100644 --- a/ext/fts5/fts5.h +++ b/ext/fts5/fts5.h @@ -306,8 +306,25 @@ struct Fts5PhraseIter { ** It is the output of the tokenizer module. For tokendata=1 tables, this ** includes any embedded 0x00 and trailing data. ** +** This API may be slow in some cases if the token identified by parameters +** iIdx and iToken matched a prefix token in the query. In most cases, the +** first call to this API for each prefix token in the query is forced +** to scan the portion of the full-text index that matches the prefix +** token to collect the extra data required by this API. If the prefix +** token matches a large number of token instances in the document set, +** this may be a performance problem. +** +** If the user knows in advance that a query may use this API for a +** prefix token, FTS5 may be configured to collect all required data as part +** of the initial querying of the full-text index, avoiding the second scan +** entirely. This also causes prefix queries that do not use this API to +** run more slowly and use more memory. FTS5 may be configured in this way +** either on a per-table basis using the [FTS5 insttoken | 'insttoken'] +** option, or on a per-query basis using the +** [fts5_insttoken | fts5_insttoken()] user function. +** ** This API can be quite slow if used with an FTS5 table created with the -** "detail=none" or "detail=column" option. +** "detail=none" or "detail=column" option. ** ** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale) ** If parameter iCol is less than zero, or greater than or equal to the diff --git a/manifest b/manifest index 75828653be..8dfeec1334 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompiler\swarnings\scaused\sby\svariable\sshadowing. -D 2024-11-20T20:39:18.431 +C Update\sdocumentation\sin\sfts5.h. +D 2024-11-26T20:16:34.989 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea F LICENSE.md e108e1e69ae8e8a59e93c455654b8ac9356a11720d3345df2a4743e9590fb20d @@ -106,7 +106,7 @@ F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl 63db9624ccf70d4887836c320eda93ab552f21008f3be7ede551eac3ead62baa F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb F ext/fts5/extract_api_docs.tcl 009cf59c77afa86d137b0cca3e3b1a5efbe2264faa2df233f9a7aa8563926d15 -F ext/fts5/fts5.h 6b4b92df890965567360db5f1ead24fd13a72cb23b95e4ed2ff58d1d89f7aa42 +F ext/fts5/fts5.h df1741b4dbf8446f663ba30ce6eca51ef1cef0dbc481806273fc8a0cc159f4b8 F ext/fts5/fts5Int.h 6abff7dd770dc5969c994c281e6e77fc277ce414d56cc4a62c145cc7036b0b67 F ext/fts5/fts5_aux.c 65a0468dd177d6093aa9ae1622e6d86b0136b8d267c62c0ad6493ad1e9a3d759 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09 @@ -2202,8 +2202,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7 F tool/warnings.sh 49a486c5069de041aedcbde4de178293e0463ae9918ecad7539eedf0ec77a139 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P edb842349320eda9550bdfcd5a327949c5512e02f4b993782587b2131a425746 -R 87e3d58f4f7440e00e0a55bcb770c2ab +P 211b305791980b24c4192ffc57a0471473de3fca32bfc146c0eeacedef7a88aa +R ce521c664e19e9b02f777e0b20a87e4f U dan -Z e1bc2b76da9d8c387af12aec8bd8d459 +Z 1b49bcb3c99dc54eba71f1872e2bcc4a # Remove this line to create a well-formed Fossil manifest. diff --git a/manifest.uuid b/manifest.uuid index 259f7f2dfa..5bd79817a5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -211b305791980b24c4192ffc57a0471473de3fca32bfc146c0eeacedef7a88aa +f1e44f703acb415e2ff7d7f87fa05fc874ef1c432095bb3ccb45c6ad2fd085a9