Fixes for xInstToken() with tokendata=0 tables. And with prefix queries.

FossilOrigin-Name: 78fbb71598b1ca756acc078253880a1d0f7983a5a26b9efc683e6488122505a1

ext/fts5/fts5_expr.c

@@ -3208,7 +3208,8 @@ int sqlite3Fts5ExprInstToken(
   int *pnOut
 ){
   Fts5ExprPhrase *pPhrase = 0;
-  Fts5IndexIter *pIter = 0;
+  Fts5ExprTerm *pTerm = 0;
+  int rc = SQLITE_OK;
 
   if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
     return SQLITE_RANGE;
@@ -3217,9 +3218,18 @@ int sqlite3Fts5ExprInstToken(
   if( iToken<0 || iToken>=pPhrase->nTerm ){
     return SQLITE_RANGE;
   }
-  pIter = pPhrase->aTerm[iToken].pIter;
-
-  return sqlite3Fts5IterToken(pIter, iRowid, iCol, iOff+iToken, ppOut, pnOut);
+  pTerm = &pPhrase->aTerm[iToken];
+  if( pTerm->bPrefix==0 ){
+    if( pExpr->pConfig->bTokendata ){
+      rc = sqlite3Fts5IterToken(
+          pTerm->pIter, iRowid, iCol, iOff+iToken, ppOut, pnOut
+      );
+    }else{
+      *ppOut = pTerm->pTerm;
+      *pnOut = pTerm->nFullTerm;
+    }
+  }
+  return rc;
 }
 
 /*

ext/fts5/fts5_main.c

@@ -2365,7 +2365,6 @@ static int fts5ApiInstToken(
       int iCol = pCsr->aInst[iIdx*3 + 1];
       int iOff = pCsr->aInst[iIdx*3 + 2];
       i64 iRowid = fts5CursorRowid(pCsr);
-
       rc = sqlite3Fts5ExprInstToken(
           pCsr->pExpr, iRowid, iPhrase, iCol, iOff, iToken, ppOut, pnOut
       );

ext/fts5/test/fts5origintext.test

@@ -252,6 +252,45 @@ do_execsql_test 5.3 {
   {0.0.0 0.0.1 0.0.2 0.0.3 0.0.4 0.0.5}
 }
 
+#-------------------------------------------------------------------------
+# Test the xInstToken() API with:
+#
+#   * a non tokendata=1 table.
+#   * prefix queries.
+#
+reset_db
+sqlite3_fts5_register_origintext db
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, y, tokenize='origintext unicode61', detail=%DETAIL%
+  );
+
+  INSERT INTO ft VALUES('One Two', 'Three two');
+  INSERT INTO ft VALUES('three Three', 'one One');
+}
+proc tokens {cmd} { 
+  set ret [list]
+  for {set iTok 0} {$iTok < [$cmd xInstCount]} {incr iTok} {
+    set txt [$cmd xInstToken $iTok 0]
+    set txt [string map [list "\0" "."] $txt]
+    lappend ret $txt
+  }
+  set ret
+}
+sqlite3_fts5_create_function db tokens tokens
+
+do_execsql_test 6.1 {
+  SELECT rowid, tokens(ft) FROM ft('One');
+} {1 one.One 2 one.One}
+
+do_execsql_test 6.2 {
+  SELECT rowid, tokens(ft) FROM ft('on*');
+} {1 {{}} 2 {{} {}}}
+
+do_execsql_test 6.3 {
+  SELECT rowid, tokens(ft) FROM ft('Three*');
+} {1 {{}} 2 {{}}}
+
 }
 
 finish_test

manifest

@@ -1,5 +1,5 @@
-C Fix\smemory\sleak\sin\snew\scode\son\sthis\sbranch.
-D 2023-12-04T17:58:56.324
+C Fixes\sfor\sxInstToken()\swith\stokendata=0\stables.\sAnd\swith\sprefix\squeries.
+D 2023-12-04T18:45:14.192
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -94,10 +94,10 @@ F ext/fts5/fts5Int.h defa43c0932265138ee910ca416e6baccf8b774e0f3d610e74be1ab2880
 F ext/fts5/fts5_aux.c ee770eec0af8646db9e18fc01a0dad7345b5f5e8cbba236704cfae2d777022ad
 F ext/fts5/fts5_buffer.c 3001fbabb585d6de52947b44b455235072b741038391f830d6b729225eeaf6a5
 F ext/fts5/fts5_config.c 8072a207034b51ae9b7694121d1b5715c794e94b275e088f70ae532378ca5cdf
-F ext/fts5/fts5_expr.c aec893108d90cc8eae6801283d38d4e705d8c22aba12717fcc56e1910b8e3b32
+F ext/fts5/fts5_expr.c 920516be4aac44eccdd9e747fe26f367442848b9a58971e4b36edb0c8284b6b8
 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
 F ext/fts5/fts5_index.c 7111fed6c01048e227cc8c681306fa2db1e5ad29429b1373e665b8e95a7868ba
-F ext/fts5/fts5_main.c 1fbadf63a7381fd68753efdccce19683f587668489b1c5a9c7bf53d6e9b64872
+F ext/fts5/fts5_main.c fb7ec495d663f40d18e420e1986316591041a70e1e4b4696ab2a7384e4c7fd7a
 F ext/fts5/fts5_storage.c 5d10b9bdcce5b90656cad13c7d12ad4148677d4b9e3fca0481fca56d6601426d
 F ext/fts5/fts5_tcl.c cf0fd0dbe64ec272491b749e0d594f563cda03336aeb60900129e6d18b0aefb8
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
@@ -190,7 +190,7 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618
 F ext/fts5/test/fts5optimize.test 36a752d24c818792032e4ff502936fc9cc5ef938721696396fdc79214b2717f1
 F ext/fts5/test/fts5optimize2.test 93e742c36b487d8874621360af5b1ce4d39b04fb9e71ce9bc34015c5fc811785
 F ext/fts5/test/fts5optimize3.test bf9c91bb927d0fb2b9a06318a217a0419183ac5913842e062c7e0b98ea5d0fca
-F ext/fts5/test/fts5origintext.test 6574e8d2121460cda72866afe3e582693d9992f150b0703aff5981625b527e62
+F ext/fts5/test/fts5origintext.test d2796fa08ee7aecfabdc0c45bb8a2fb16a00ea8757e63fbc153b718dbe430a39
 F ext/fts5/test/fts5origintext2.test 43b07dd62d087743322b0003a27c8efdbda6c8659a27fde71f32ead27b5a0969
 F ext/fts5/test/fts5origintext3.test e0d47c187e7c279d25aa27aa3de8dd0d26b050a74db90670c9b20d0ecfcfb52a
 F ext/fts5/test/fts5origintext4.test 296b1b1e6630d492b99db0769e8127087548f0e939376047716a68b77ca3c871
@@ -2150,8 +2150,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 317a50563d9e8586fda136e513727241b414e7267d50a06571c8ebd0eae710bc
-R 4cc8f92a68d4e11e53934c982deb6dc3
+P ebc160b9a05568df66f86e30804399ee29d34b44a60c57e062f98cb92826353f
+R 4a97cb4432f9a2f4a872da3bc4853a9c
 U dan
-Z c8f15e3d33aff662e610fc8982efce75
+Z d5e27dba49298422b322a3619fa6b5f7
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-ebc160b9a05568df66f86e30804399ee29d34b44a60c57e062f98cb92826353f
+78fbb71598b1ca756acc078253880a1d0f7983a5a26b9efc683e6488122505a1