Add the "tokenchars=" and "separators=" options, for customizing the set of characters considered to be token separators, to the unicode61 tokenizer.

FossilOrigin-Name: e56fb462aa1f11bb23303ae0dc62815c21e26a52

test/fts4unicode.test

@@ -31,6 +31,18 @@ proc do_unicode_token_test2 {tn input res} {
   " [list [list {*}$res]]]
 }
 
+proc do_unicode_token_test3 {tn args} {
+  set res   [lindex $args end]
+  set sql "SELECT fts3_tokenizer_test('unicode61'"
+  foreach a [lrange $args 0 end-1] {
+    append sql ", '"
+    append sql [string map {' ''} $a]
+    append sql "'"
+  }
+  append sql ")"
+  uplevel [list do_execsql_test $tn $sql [list [list {*}$res]]]
+}
+
 do_unicode_token_test 1.0 {a B c D} {0 a a 1 b B 2 c c 3 d D}
 do_unicode_token_test 1.1 {Ä Ö Ü} {0 ä Ä 1 ö Ö 2 ü Ü}
 do_unicode_token_test 1.2 {xÄx xÖx xÜx} {0 xäx xÄx 1 xöx xÖx 2 xüx xÜx}
@@ -236,6 +248,82 @@ do_test 4.3 {
   }
 } {}
 
+#-------------------------------------------------------------------------
+
+do_unicode_token_test3 5.1 {tokenchars=} {
+  sqlite3_reset sqlite3_column_int
+} {
+  0 sqlite3 sqlite3 
+  1 reset reset 
+  2 sqlite3 sqlite3 
+  3 column column 
+  4 int int
+}
+
+do_unicode_token_test3 5.2 {tokenchars=_} {
+  sqlite3_reset sqlite3_column_int
+} {
+  0 sqlite3_reset sqlite3_reset 
+  1 sqlite3_column_int sqlite3_column_int
+}
+
+do_unicode_token_test3 5.3 {separators=xyz} {
+  Laotianxhorseyrunszfast
+} {
+  0 laotian Laotian
+  1 horse horse
+  2 runs runs
+  3 fast fast
+}
+
+do_unicode_token_test3 5.4 {tokenchars=xyz} {
+  Laotianxhorseyrunszfast
+} {
+  0 laotianxhorseyrunszfast Laotianxhorseyrunszfast
+}
+
+do_unicode_token_test3 5.5 {tokenchars=_} {separators=zyx} {
+  sqlite3_resetxsqlite3_column_intyhonda_phantom
+} {
+  0 sqlite3_reset sqlite3_reset 
+  1 sqlite3_column_int sqlite3_column_int
+  2 honda_phantom honda_phantom
+}
+
+do_unicode_token_test3 5.6 "separators=\u05D1" "abc\u05D1def" {
+  0 abc abc 1 def def
+}
+
+do_unicode_token_test3 5.7                             \
+  "tokenchars=\u2444\u2445"                            \
+  "separators=\u05D0\u05D1\u05D2"                      \
+  "\u2444fre\u2445sh\u05D0water\u05D2fish.\u2445timer" \
+  [list                                                \
+    0 \u2444fre\u2445sh \u2444fre\u2445sh              \
+    1 water water                                      \
+    2 fish fish                                        \
+    3 \u2445timer \u2445timer                          \
+  ]
+
+# Check that it is not possible to add a standalone diacritic codepoint 
+# to either separators or tokenchars.
+do_unicode_token_test3 5.8 "separators=\u0301" \
+  "hello\u0301world \u0301helloworld"          \
+  "0 helloworld hello\u0301world 1 helloworld helloworld"
+
+do_unicode_token_test3 5.9 "tokenchars=\u0301" \
+  "hello\u0301world \u0301helloworld"          \
+  "0 helloworld hello\u0301world 1 helloworld helloworld"
+
+do_unicode_token_test3 5.10 "separators=\u0301" \
+  "remove_diacritics=0"                        \
+  "hello\u0301world \u0301helloworld"          \
+  "0 hello\u0301world hello\u0301world 1 helloworld helloworld"
+
+do_unicode_token_test3 5.11 "tokenchars=\u0301" \
+  "remove_diacritics=0"                         \
+  "hello\u0301world \u0301helloworld"           \
+  "0 hello\u0301world hello\u0301world 1 helloworld helloworld"
 
 
 finish_test