Minor tweaks to the hebrew transliteration tables.

FossilOrigin-Name: 7b6de5c35d1c2e141b1eb666c8dd5ef6201ab579

Makefile.in

@@ -165,8 +165,9 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
          backup.lo bitvec.lo btmutex.lo btree.lo build.lo \
          callback.lo complete.lo ctime.lo date.lo delete.lo \
          expr.lo fault.lo fkey.lo \
-         fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo fts3_porter.lo \
-         fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo fts3_write.lo \
+         fts3.lo fts3_aux.lo fts3_expr.lo fts3_hash.lo fts3_icu.lo \
+         fts3_porter.lo fts3_snippet.lo fts3_tokenizer.lo fts3_tokenizer1.lo \
+         fts3_tokenizer2.lo fts3_write.lo \
          func.lo global.lo hash.lo \
          icu.lo insert.lo journal.lo legacy.lo loadext.lo \
          main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
@@ -317,6 +318,7 @@ SRC += \
   $(TOP)/ext/fts3/fts3_tokenizer.h \
   $(TOP)/ext/fts3/fts3_tokenizer.c \
   $(TOP)/ext/fts3/fts3_tokenizer1.c \
+  $(TOP)/ext/fts3/fts3_tokenizer2.c \
   $(TOP)/ext/fts3/fts3_write.c
 SRC += \
   $(TOP)/ext/icu/sqliteicu.h \
@@ -856,6 +858,9 @@ fts3_tokenizer.lo:	$(TOP)/ext/fts3/fts3_tokenizer.c $(HDR) $(EXTHDR)
 fts3_tokenizer1.lo:	$(TOP)/ext/fts3/fts3_tokenizer1.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer1.c
 
+fts3_tokenizer2.lo:	$(TOP)/ext/fts3/fts3_tokenizer2.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer2.c
+
 fts3_write.lo:	$(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_write.c
 

ext/fts3/README.tokenizers

@@ -11,7 +11,7 @@
     );
 
   The built-in tokenizers (valid values to pass as <tokenizer name>) are
-  "simple" and "porter".
+  "simple", "porter", "transliterate01", and "icu".
 
   <tokenizer-args> should consist of zero or more white-space separated
   arguments to pass to the selected tokenizer implementation. The 

ext/fts3/fts3.c

@@ -3543,10 +3543,11 @@ static void hashDestroy(void *p){
 }
 
 /*
-** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are 
-** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c
-** respectively. The following three forward declarations are for functions
-** declared in these files used to retrieve the respective implementations.
+** The fts3 built-in tokenizers - "simple", "porter", "transliterate01,
+** and "icu"- are implemented in files fts3_tokenizer1.c, fts3_porter.c,
+** fts3_transliterate01 and fts3_icu.c respectively. The following three
+** forward declarations are for functions declared in these files used
+** to retrieve the respective implementations.
 **
 ** Calling sqlite3Fts3SimpleTokenizerModule() sets the value pointed
 ** to by the argument to point to the "simple" tokenizer implementation.
@@ -3554,6 +3555,7 @@ static void hashDestroy(void *p){
 */
 void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
 void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+void sqlite3Fts3TranslitTokenizerModule(sqlite3_tokenizer_module const**);
 #ifdef SQLITE_ENABLE_ICU
 void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const**ppModule);
 #endif
@@ -3569,6 +3571,7 @@ int sqlite3Fts3Init(sqlite3 *db){
   Fts3Hash *pHash = 0;
   const sqlite3_tokenizer_module *pSimple = 0;
   const sqlite3_tokenizer_module *pPorter = 0;
+  const sqlite3_tokenizer_module *pTranslit = 0;
 
 #ifdef SQLITE_ENABLE_ICU
   const sqlite3_tokenizer_module *pIcu = 0;
@@ -3585,6 +3588,7 @@ int sqlite3Fts3Init(sqlite3 *db){
 
   sqlite3Fts3SimpleTokenizerModule(&pSimple);
   sqlite3Fts3PorterTokenizerModule(&pPorter);
+  sqlite3Fts3TranslitTokenizerModule(&pTranslit);
 
   /* Allocate and initialise the hash-table used to store tokenizers. */
   pHash = sqlite3_malloc(sizeof(Fts3Hash));
@@ -3598,6 +3602,7 @@ int sqlite3Fts3Init(sqlite3 *db){
   if( rc==SQLITE_OK ){
     if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple)
      || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) 
+     || sqlite3Fts3HashInsert(pHash, "transliterate01", 16, (void *)pTranslit) 
 #ifdef SQLITE_ENABLE_ICU
      || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu))
 #endif

main.mk

@@ -54,7 +54,7 @@ LIBOBJ+= alter.o analyze.o attach.o auth.o \
          backup.o bitvec.o btmutex.o btree.o build.o \
          callback.o complete.o ctime.o date.o delete.o expr.o fault.o fkey.o \
          fts3.o fts3_aux.o fts3_expr.o fts3_hash.o fts3_icu.o fts3_porter.o \
-         fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o \
+         fts3_snippet.o fts3_tokenizer.o fts3_tokenizer1.o fts3_tokenizer2.o \
          fts3_write.o func.o global.o hash.o \
          icu.o insert.o journal.o legacy.o loadext.o \
          main.o malloc.o mem0.o mem1.o mem2.o mem3.o mem5.o \
@@ -198,6 +198,7 @@ SRC += \
   $(TOP)/ext/fts3/fts3_tokenizer.h \
   $(TOP)/ext/fts3/fts3_tokenizer.c \
   $(TOP)/ext/fts3/fts3_tokenizer1.c \
+  $(TOP)/ext/fts3/fts3_tokenizer2.c \
   $(TOP)/ext/fts3/fts3_write.c
 SRC += \
   $(TOP)/ext/icu/sqliteicu.h \
@@ -508,6 +509,9 @@ fts3_tokenizer.o:	$(TOP)/ext/fts3/fts3_tokenizer.c $(HDR) $(EXTHDR)
 fts3_tokenizer1.o:	$(TOP)/ext/fts3/fts3_tokenizer1.c $(HDR) $(EXTHDR)
 	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer1.c
 
+fts3_tokenizer2.o:	$(TOP)/ext/fts3/fts3_tokenizer2.c $(HDR) $(EXTHDR)
+	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_tokenizer2.c
+
 fts3_write.o:	$(TOP)/ext/fts3/fts3_write.c $(HDR) $(EXTHDR)
 	$(TCCX) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3_write.c
 

manifest

@@ -1,7 +1,7 @@
-C Fix\sa\sminor\sdeviation\sfrom\sthe\scoding\sstyle\sguidelines.
-D 2012-04-27T16:38:11.705
+C Minor\stweaks\sto\sthe\shebrew\stransliteration\stables.
+D 2012-05-04T13:22:42.928
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
-F Makefile.in 2f37e468503dbe79d35c9f6dffcf3fae1ae9ec20
+F Makefile.in 502e53594fa43e920aa9d3a3a408f926e83fccb7
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
 F Makefile.msc 7849a871b6cdb20fd51baee6bbe5965a03326be4
 F Makefile.vxworks 3b7fe7a0571fdadc61363ebc1b23732d2d6363ca
@@ -53,9 +53,9 @@ F ext/fts2/fts2_tokenizer1.c 0123d21078e053bd98fd6186c5c6dc6d67969f2e
 F ext/fts2/mkfts2amal.tcl 974d5d438cb3f7c4a652639262f82418c1e4cff0
 F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51
 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a
-F ext/fts3/README.tokenizers 998756696647400de63d5ba60e9655036cb966e9
+F ext/fts3/README.tokenizers 060abb298f5bfd505f3c6b831c9902fc446cf632
 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d
-F ext/fts3/fts3.c 111626ce72b0df93f509ebd14ce31804fed24be0
+F ext/fts3/fts3.c 5718d78a09648f92c4b102e420c2a1e81330addd
 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe
 F ext/fts3/fts3Int.h 5fd2ec4e47faf17bf4a508d6b8ec5fc0f2c80bff
 F ext/fts3/fts3_aux.c 5205182bd8f372782597888156404766edf5781e
@@ -70,10 +70,12 @@ F ext/fts3/fts3_test.c 348f7d08cae05285794e23dc4fe8b8fdf66e264a
 F ext/fts3/fts3_tokenizer.c 3da7254a9881f7e270ab28e2004e0d22b3212bce
 F ext/fts3/fts3_tokenizer.h 66dec98e365854b6cd2d54f1a96bb6d428fc5a68
 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004
+F ext/fts3/fts3_tokenizer2.c 13175d138fddd041d389068cafa0d9957ac837f2
 F ext/fts3/fts3_write.c cd4af00b3b0512b4d76177a267fcaafab44cbce4
 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9
 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100
 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197
+F ext/fts3/translit01.tcl 3d9ec48eb56ab9aaa0304c7c4fda44f60d7276cf
 F ext/icu/README.txt bf8461d8cdc6b8f514c080e4e10dc3b2bbdfefa9
 F ext/icu/icu.c eb9ae1d79046bd7871aa97ee6da51eb770134b5a
 F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37
@@ -98,7 +100,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de
 F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
 F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
-F main.mk a80771d44176a0c744d9d4e048497e7ed0b4040d
+F main.mk a67ab64b7d9a4842a23e50c1a8abc9fa3d55f8f4
 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a
 F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f
 F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac
@@ -489,6 +491,7 @@ F test/fts3rnd.test 1320d8826a845e38a96e769562bf83d7a92a15d0
 F test/fts3shared.test 8bb266521d7c5495c0ae522bb4d376ad5387d4a2
 F test/fts3snippet.test 8e956051221a34c7daeb504f023cb54d5fa5a8b2
 F test/fts3sort.test 95be0b19d7e41c44b29014f13ea8bddd495fd659
+F test/fts3translit01.test e5106f72be6d28161416aaac675ebd39cd6f7c52
 F test/fts4aa.test 6e7f90420b837b2c685f3bcbe84c868492d40a68
 F test/fts4check.test 66fa274cab2b615f2fb338b257713aba8fad88a8
 F test/fts4content.test 17b2360f7d1a9a7e5aa8022783f5c5731b6dfd4f
@@ -969,7 +972,7 @@ F tool/mkkeywordhash.c bb52064aa614e1426445e4b2b9b00eeecd23cc79
 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
 F tool/mksqlite3c-noext.tcl 105023aa86f696a74b1d6a4929d1e1c3baf9471c
-F tool/mksqlite3c.tcl 9fbac513cd9d5ac95ad55630f49bb16c5347ab75
+F tool/mksqlite3c.tcl 09c8ff0c9e615cab1bb46e3d63b836ed6d0905f9
 F tool/mksqlite3h.tcl 78013ad79a5e492e5f764f3c7a8ef834255061f8
 F tool/mksqlite3internalh.tcl 7b43894e21bcb1bb39e11547ce7e38a063357e87
 F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091
@@ -995,7 +998,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
 F tool/warnings-clang.sh a8a0a3babda96dfb1ff51adda3cbbf3dfb7266c2
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
-P a49e909c8738317c8383ce93771c0a9c4cf270bc
-R b188dfa7c7a5aca6dba55435d29b09a3
+P 930115693aa20bcbeb919143098009f30907749a
+R d80972dc05430fb3727fd721db5129d8
 U drh
-Z afa94c75b9ca22f35aabedc6458ef96f
+Z 8fb82806311f067e31e89d8965325302

manifest.uuid

@@ -1 +1 @@
-1e51bffe777587cd05bd7db5e02d6291c3eb8c1a
+7b6de5c35d1c2e141b1eb666c8dd5ef6201ab579

test/fts3translit01.test

@@ -0,0 +1,68 @@
+# 2012 May 04
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS3/4 module, and in particular
+# the transliterate01 stemmer.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# If SQLITE_ENABLE_FTS4 is defined, omit this file.
+ifcapable !fts3 {
+  finish_test
+  return
+}
+
+# Test data for the Transliterator01 stemmer.  The first word of each line
+# is the input.  The second word is the desired output.
+#
+set translit01_test_data {
+  smörgåsbord      smoergaasbord
+  Ångström         aangstroem 
+  étude            etude
+  communiqué       communique
+  fiancée          fiancee
+  crèche           creche
+  Zürich           zuerich
+  Gödel            goedel
+  \u0427\u0430\u0439\u043a\u043e\u0301\u0432\u0441\u043a\u0438\u0439
+                   chaikovskii
+  \u0391\u1f30\u03c3\u03c7\u03cd\u03bb\u03bf\u03c2
+                   aschylos
+  \u03a3\u03c9\u03ba\u03c1\u03ac\u03c4\u03b7\u03c2
+                   sokratis
+  \u05d1\u05b5\u05bc\u05d9\u05ea\u05dc\u05b6\u05d7\u05b6\u05dd
+                   beaytlehem
+  \u05d9\u05b0\u05e8\u05d5\u05bc\u05e9\u05b8\u05c1\u05dc\u05b7\u05d9\u05b4\u05dd
+                   yervashashlayim
+}                         
+
+# Create a full-text index to use for testing the stemmer.
+#
+db close
+sqlite3 db :memory:
+db eval {
+  CREATE VIRTUAL TABLE t1 USING fts4(word, tokenize transliterate01);
+  CREATE VIRTUAL TABLE t1aux USING fts4aux(t1);
+}
+
+foreach {pfrom pto} $translit01_test_data {
+  do_test fts3translit01-$pfrom {
+    execsql {
+      DELETE FROM t1;
+      INSERT INTO t1(word) VALUES($pfrom);
+      SELECT term FROM t1aux where col=0;
+    }
+  } $pto
+}
+
+finish_test

tool/mksqlite3c.tcl

@@ -314,6 +314,7 @@ foreach file {
    fts3_porter.c
    fts3_tokenizer.c
    fts3_tokenizer1.c
+   fts3_tokenizer2.c
    fts3_write.c
    fts3_snippet.c