Make hi-bit characters delimiters. This is a stopgap until the tokenizer

and fulltext.c recognize UTF-8 correctly. (CVS 3370)

FossilOrigin-Name: ca850d3d80f67672172d11392fcdf60bfbb94c02

ext/fts1/simple_tokenizer.c

@@ -61,11 +61,11 @@ static int simpleCreate(
   if( argc>1 ){
     t->zDelim = string_dup(argv[1]);
   } else {
-    /* Build a string of non-alphanumeric ASCII characters */
-    char zDelim[128];               /* nul-terminated, so nul not a member */
+    /* Build a string excluding alphanumeric ASCII characters */
+    char zDelim[256];               /* nul-terminated, so nul not a member */
     int i, j;
-    for(i=1, j=0; i<0x80; i++){
-      if( !isalnum(i) ){
+    for(i=1, j=0; i<0x100; i++){
+      if( i>=0x80 || !isalnum(i) ){
         zDelim[j++] = i;
       }
     }

manifest

@@ -1,5 +1,5 @@
-C Add\ssqlite3_malloc\sand\ssqlite3_realloc\sto\sthe\ssqlite3.def\sfile.\s\sTicket\s#1943.\s(CVS\s3369)
-D 2006-08-27T14:10:39
+C Make\shi-bit\scharacters\sdelimiters.\s\sThis\sis\sa\sstopgap\suntil\sthe\stokenizer\nand\sfulltext.c\srecognize\sUTF-8\scorrectly.\s(CVS\s3370)
+D 2006-08-28T20:08:57
 F Makefile.in 8e7f9ecebab2c6e0f3db20ff129a8f9405ab64f8
 F Makefile.linux-gcc 2d8574d1ba75f129aba2019f0b959db380a90935
 F README 9c4e2d6706bdcc3efdd773ce752a8cdab4f90028
@@ -23,7 +23,7 @@ F ext/fts1/ft_hash.c 3927bd880e65329bdc6f506555b228b28924921b
 F ext/fts1/ft_hash.h 1a35e654a235c2c662d3ca0dfc3138ad60b8b7d5
 F ext/fts1/fulltext.c 2bf058ab2835531fa5fee4242c51eda19cfe88a7
 F ext/fts1/fulltext.h 08525a47852d1d62a0be81d3fc3fe2d23b094efd
-F ext/fts1/simple_tokenizer.c 289b7f35bf692e85834a7b599fd0a136e415a9eb
+F ext/fts1/simple_tokenizer.c 22501944cd77686be592382692051514ef228ec6
 F ext/fts1/tokenizer.h 0c53421b832366d20d720d21ea3e1f6e66a36ef9
 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895
 F ltmain.sh f6b283068efa69f06eb8aa1fe4bddfdbdeb35826
@@ -388,7 +388,7 @@ F www/tclsqlite.tcl bb0d1357328a42b1993d78573e587c6dcbc964b9
 F www/vdbe.tcl 87a31ace769f20d3627a64fa1fade7fed47b90d0
 F www/version3.tcl 890248cf7b70e60c383b0e84d77d5132b3ead42b
 F www/whentouse.tcl 97e2b5cd296f7d8057e11f44427dea8a4c2db513
-P bc84cb54b0df09738fd90e48820dc3cdfa7828c2
-R 0cc69f8ab1498c5143b9dcba25099f86
-U drh
-Z c38109b8330c69ee9fac74b68de526b4
+P 4a74838eac5b939058c8aebef30113c5bdb354e5
+R 4dd7009178412e6bfe81d5289193af1c
+U shess
+Z 847ff5b71d65a558b08ddb2fff6d70a4

manifest.uuid

@@ -1 +1 @@
-4a74838eac5b939058c8aebef30113c5bdb354e5
+ca850d3d80f67672172d11392fcdf60bfbb94c02