Add a test to verify that the fts3 and fts5 "unicode61" tokenizers are byte for byte compatible.

FossilOrigin-Name: 8ec8314354edc9d6f7d1c8a6370fd984a552a52f
2016-01-14 20:05:59 +00:00
parent 3349620e9e
commit 2cfe049f40
9 changed files with 692 additions and 14 deletions
@@ -333,7 +333,8 @@ TESTSRC += \
  $(TOP)/ext/misc/wholenumber.c \
  $(TOP)/ext/misc/vfslog.c \
  $(TOP)/ext/fts5/fts5_tcl.c \
-  $(TOP)/ext/fts5/fts5_test_mi.c
+  $(TOP)/ext/fts5/fts5_test_mi.c \
+  $(TOP)/ext/fts5/fts5_test_tok.c


 #TESTSRC += $(TOP)/ext/fts2/fts2_tokenizer.c