Add the xLanguageid method to sqlite3_fts3_tokenizer versions 1 and greater.

FossilOrigin-Name: f8e9c445dd358c40e5a7bf3756b9f291909dbea7

test/fts4langid.test

@@ -40,7 +40,6 @@ set ::testprefix fts4langid
 #   3.* - Tests with content= tables. Both where there is a real 
 #         underlying content table and where there is not.
 #
-#
 #   4.* - Test that if one is provided, the tokenizer xLanguage method
 #         is called to configure the tokenizer before tokenizing query
 #         or document text.
@@ -342,7 +341,45 @@ do_test_query1 3.3.4 {"zero one" OR "one two"} {
   or_merge_lists [rowid_list "zero one"] [rowid_list "one two"]
 }
 
+#-------------------------------------------------------------------------
+# Test cases 4.*
+#
+proc build_multilingual_db_2 {db} {
+  $db eval {
+    CREATE VIRTUAL TABLE t4 USING fts4(
+        tokenize=testtokenizer, 
+        languageid=lid
+    );
+  }
+  for {set i 0} {$i < 50} {incr i} {
+    execsql { 
+      INSERT INTO t4(docid, content, lid) VALUES($i, 'The Quick Brown Fox', $i) 
+    }
+  }
+}
 
+do_test 4.1.0 {
+  reset_db
+  set ptr [fts3_test_tokenizer]
+  execsql { SELECT fts3_tokenizer('testtokenizer', $ptr) }
+  build_multilingual_db_2 db
+} {}
+do_execsql_test 4.1.1 {
+  SELECT docid FROM t4 WHERE t4 MATCH 'quick';
+} {0}
+do_execsql_test 4.1.2 {
+  SELECT docid FROM t4 WHERE t4 MATCH 'quick' AND lid=1;
+} {}
+do_execsql_test 4.1.3 {
+  SELECT docid FROM t4 WHERE t4 MATCH 'Quick' AND lid=1;
+} {1}
+for {set i 0} {$i < 50} {incr i} {
+  do_execsql_test 4.1.4.$i {
+    SELECT count(*) FROM t4 WHERE t4 MATCH 'fox' AND lid=$i;
+  } [expr 0==($i%2)]
+}
+do_catchsql_test 4.1.5 {
+  INSERT INTO t4(content, lid) VALUES('hello world', 101)
+} {1 {SQL logic error or missing database}}
 
 finish_test
-