Add further tests for xInstToken().

FossilOrigin-Name: 8582707f16133f003a6687f68cbea03d4eb6c2a0e2e07746b7cace0c44e84fa4

ext/fts5/test/fts5origintext5.test

@@ -82,7 +82,7 @@ db func document document
 expr srand(6)
 
 set NDOC  200
-set NLOOP 100
+set NLOOP 50
 
 sqlite3_fts5_register_origintext db
 
@@ -102,15 +102,17 @@ proc rankfunc {cmd} {
 }
 sqlite3_fts5_create_function db rankfunc rankfunc
 
-proc ctrl_tokens {doc term} {
+proc ctrl_tokens {term args} {
   set ret [list]
   set term [string tolower $term]
-  foreach a $doc {
-    if {[string tolower $a]==$term} {
-      if {$a==$term} {
-        lappend ret $a
-      } else {
-        lappend ret [string tolower $a].$a
+  foreach doc $args {
+    foreach a $doc {
+      if {[string tolower $a]==$term} {
+        if {$a==$term} {
+          lappend ret $a
+        } else {
+          lappend ret [string tolower $a].$a
+        }
       }
     }
   }
@@ -118,6 +120,20 @@ proc ctrl_tokens {doc term} {
 }
 db func ctrl_tokens ctrl_tokens
 
+proc do_all_vocab_test {tn} {
+  foreach ::v [vocab] {
+    set answer [execsql {
+      SELECT id, ctrl_tokens($::v, x) FROM ctrl WHERE x LIKE '%' || $::v || '%'
+    }]
+    do_execsql_test $tn.$::v.1 {
+      SELECT rowid, tokens(ft) FROM ft($::v)
+    } $answer
+    do_execsql_test $tn.$::v.2 {
+      SELECT rowid, tokens(ft) FROM ft($::v) ORDER BY rank
+    } $answer
+  }
+}
+
 
 do_execsql_test 1.0 {
   CREATE VIRTUAL TABLE ft USING fts5(
@@ -139,20 +155,6 @@ do_test 1.1 {
   }
 } {}
 
-proc do_all_vocab_test {tn} {
-  foreach ::v [vocab] {
-    set answer [execsql {
-      SELECT id, ctrl_tokens(x, $::v) FROM ctrl WHERE x LIKE '%' || $::v || '%'
-    }]
-    do_execsql_test $tn.$::v.1 {
-      SELECT rowid, tokens(ft) FROM ft($::v)
-    } $answer
-    do_execsql_test $tn.$::v.2 {
-      SELECT rowid, tokens(ft) FROM ft($::v) ORDER BY rank
-    } $answer
-  }
-}
-
 #execsql_pp { SELECT * FROM ctrl }
 #execsql_pp { SELECT * FROM ft }
 #fts5_aux_test_functions db
@@ -173,9 +175,59 @@ for {set ii 0} {$ii < $NLOOP} {incr ii} {
   do_all_vocab_test 1.3.$ii
 }
 
+#-------------------------------------------------------------------------
 
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft2 USING fts5(
+      x, y, tokenize="origintext unicode61", content=, contentless_delete=1,
+      tokendata=1
+  );
 
+  CREATE TABLE ctrl2(id INTEGER PRIMARY KEY, x TEXT, y TEXT);
+  INSERT INTO ft2(ft2, rank) VALUES('pgsz', 64);
+  INSERT INTO ft2(ft2, rank) VALUES('rank', 'rankfunc()');
+}
+do_test 2.1 {
+  for {set ii 0} {$ii < $NDOC} {incr ii} {
+    set doc1 [document]
+    set doc2 [document]
+    execsql {
+      INSERT INTO ft2(rowid, x, y) VALUES($ii, $doc, $doc2);
+      INSERT INTO ctrl2(id, x, y) VALUES($ii, $doc, $doc2);
+    }
+  }
+} {}
 
+proc do_all_vocab_test2 {tn} {
+  foreach ::v [vocab] {
+    set answer [execsql {
+      SELECT id, ctrl_tokens($::v, x, y) FROM ctrl2
+      WHERE x LIKE '%' || $::v || '%' OR y LIKE '%' || $::v || '%';
+    }]
+    do_execsql_test $tn.$::v.1 {
+      SELECT rowid, tokens(ft2) FROM ft2($::v)
+    } $answer
+    do_execsql_test $tn.$::v.2 {
+      SELECT rowid, tokens(ft2) FROM ft2($::v) ORDER BY rank
+    } $answer
+  }
+}
+
+do_all_vocab_test2 2.2
+
+for {set ii 0} {$ii < $NLOOP} {incr ii} {
+  set lRowid [execsql { SELECT id FROM ctrl2 WHERE random() % 2 }]
+  foreach r $lRowid {
+    execsql { DELETE FROM ft2 WHERE rowid = $r }
+    execsql { DELETE FROM ctrl2 WHERE rowid = $r }
+
+    set doc1 [document]
+    set doc2 [document]
+    execsql { INSERT INTO ft2(rowid, x, y) VALUES($r, $doc, $doc1) }
+    execsql { INSERT INTO ctrl2(id, x, y) VALUES($r, $doc, $doc2) }
+  }
+  do_all_vocab_test 2.3.$ii
+}
 
 finish_test
 

manifest

@@ -1,5 +1,5 @@
-C Fixes\sfor\sxInstToken()\swith\stokendata=0\stables.\sAnd\swith\sprefix\squeries.
-D 2023-12-04T18:45:14.192
+C Add\sfurther\stests\sfor\sxInstToken().
+D 2023-12-04T19:48:08.530
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -194,7 +194,7 @@ F ext/fts5/test/fts5origintext.test d2796fa08ee7aecfabdc0c45bb8a2fb16a00ea8757e6
 F ext/fts5/test/fts5origintext2.test 43b07dd62d087743322b0003a27c8efdbda6c8659a27fde71f32ead27b5a0969
 F ext/fts5/test/fts5origintext3.test e0d47c187e7c279d25aa27aa3de8dd0d26b050a74db90670c9b20d0ecfcfb52a
 F ext/fts5/test/fts5origintext4.test 296b1b1e6630d492b99db0769e8127087548f0e939376047716a68b77ca3c871
-F ext/fts5/test/fts5origintext5.test f9dfb005248d764fdc52859308875f680b523a897258a2cc40100f9e2356b5a9
+F ext/fts5/test/fts5origintext5.test 067bfb3008323585df640ab29e8ef7c4ca6dec62c597be07a9f896d88f98cd10
 F ext/fts5/test/fts5phrase.test 13e5d8e9083077b3d9c74315b3c92ec723cc6eb37c8155e0bfe1bba00559f07b
 F ext/fts5/test/fts5plan.test b65cfcca9ddd6fdaa118c61e17aeec8e8433bc5b6bb307abd116514f79c49c5a
 F ext/fts5/test/fts5porter.test 8d08010c28527db66bc3feebd2b8767504aaeb9b101a986342fa7833d49d0d15
@@ -2150,8 +2150,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P ebc160b9a05568df66f86e30804399ee29d34b44a60c57e062f98cb92826353f
-R 4a97cb4432f9a2f4a872da3bc4853a9c
+P 78fbb71598b1ca756acc078253880a1d0f7983a5a26b9efc683e6488122505a1
+R ced1cc2c59284a9ef3026043e080f745
 U dan
-Z d5e27dba49298422b322a3619fa6b5f7
+Z e2cce7acff968ef3c9472723192ae452
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-78fbb71598b1ca756acc078253880a1d0f7983a5a26b9efc683e6488122505a1
+8582707f16133f003a6687f68cbea03d4eb6c2a0e2e07746b7cace0c44e84fa4