Prototype implementation of "PRAGMA reset_database". This pragma differs from

SQLITE_DBCONFIG_RESET_DATABASE in that the pragma only works if the database is reasonably well-formed, whereas the dbconfig works regardless. FossilOrigin-Name: cf0999d4f8fd120c4796240600cd42fcc8baa056efad432d0dae7da372023787
2022-09-28 17:10:23 +00:00
443 changed files with 28622 additions and 45227 deletions
@@ -35,7 +35,7 @@ TCC += -I${TOP}/ext/fts3 -I${TOP}/ext/async -I${TOP}/ext/session
 TCC += -I${TOP}/ext/userauth

 # Define this for the autoconf-based build, so that the code knows it can
-# include the generated sqlite_cfg.h
+# include the generated config.h
 #
 TCC += -D_HAVE_SQLITE_CONFIG_H -DBUILD_sqlite

@@ -184,7 +184,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \
         main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
         memdb.lo memjournal.lo \
         mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
-         notify.lo opcodes.lo os.lo os_kv.lo os_unix.lo os_win.lo \
+         notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
         pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
         random.lo resolve.lo rowset.lo rtree.lo \
         sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
@@ -257,7 +257,6 @@ SRC = \
  $(TOP)/src/os.h \
  $(TOP)/src/os_common.h \
  $(TOP)/src/os_setup.h \
-  $(TOP)/src/os_kv.c \
  $(TOP)/src/os_unix.c \
  $(TOP)/src/os_win.c \
  $(TOP)/src/os_win.h \
@@ -315,6 +314,24 @@ SRC = \

 # Source code for extensions
 #
+SRC += \
+  $(TOP)/ext/fts1/fts1.c \
+  $(TOP)/ext/fts1/fts1.h \
+  $(TOP)/ext/fts1/fts1_hash.c \
+  $(TOP)/ext/fts1/fts1_hash.h \
+  $(TOP)/ext/fts1/fts1_porter.c \
+  $(TOP)/ext/fts1/fts1_tokenizer.h \
+  $(TOP)/ext/fts1/fts1_tokenizer1.c
+SRC += \
+  $(TOP)/ext/fts2/fts2.c \
+  $(TOP)/ext/fts2/fts2.h \
+  $(TOP)/ext/fts2/fts2_hash.c \
+  $(TOP)/ext/fts2/fts2_hash.h \
+  $(TOP)/ext/fts2/fts2_icu.c \
+  $(TOP)/ext/fts2/fts2_porter.c \
+  $(TOP)/ext/fts2/fts2_tokenizer.h \
+  $(TOP)/ext/fts2/fts2_tokenizer.c \
+  $(TOP)/ext/fts2/fts2_tokenizer1.c
 SRC += \
  $(TOP)/ext/fts3/fts3.c \
  $(TOP)/ext/fts3/fts3.h \
@@ -360,7 +377,7 @@ SRC += \
  opcodes.h \
  parse.c \
  parse.h \
-  sqlite_cfg.h \
+  config.h \
  shell.c \
  sqlite3.h

@@ -373,6 +390,7 @@ TESTSRC = \
  $(TOP)/src/test4.c \
  $(TOP)/src/test5.c \
  $(TOP)/src/test6.c \
+  $(TOP)/src/test7.c \
  $(TOP)/src/test8.c \
  $(TOP)/src/test9.c \
  $(TOP)/src/test_autoext.c \
@@ -401,6 +419,7 @@ TESTSRC = \
  $(TOP)/src/test_quota.c \
  $(TOP)/src/test_rtree.c \
  $(TOP)/src/test_schema.c \
+  $(TOP)/src/test_server.c \
  $(TOP)/src/test_superlock.c \
  $(TOP)/src/test_syscall.c \
  $(TOP)/src/test_tclsh.c \
@@ -414,10 +433,7 @@ TESTSRC = \
  $(TOP)/ext/fts3/fts3_term.c \
  $(TOP)/ext/fts3/fts3_test.c  \
  $(TOP)/ext/session/test_session.c \
-  $(TOP)/ext/recover/sqlite3recover.c \
-  $(TOP)/ext/recover/dbdata.c \
-  $(TOP)/ext/recover/test_recover.c \
-  $(TOP)/ext/rbu/test_rbu.c
+  $(TOP)/ext/rbu/test_rbu.c 

 # Statically linked extensions
 #
@@ -426,7 +442,6 @@ TESTSRC += \
  $(TOP)/ext/expert/test_expert.c \
  $(TOP)/ext/misc/amatch.c \
  $(TOP)/ext/misc/appendvfs.c \
-  $(TOP)/ext/misc/basexx.c \
  $(TOP)/ext/misc/carray.c \
  $(TOP)/ext/misc/cksumvfs.c \
  $(TOP)/ext/misc/closure.c \
@@ -477,7 +492,6 @@ TESTSRC2 = \
  $(TOP)/src/main.c \
  $(TOP)/src/mem5.c \
  $(TOP)/src/os.c \
-  $(TOP)/src/os_kv.c \
  $(TOP)/src/os_unix.c \
  $(TOP)/src/os_win.c \
  $(TOP)/src/pager.c \
@@ -541,10 +555,18 @@ HDR = \
   $(TOP)/src/vdbeInt.h \
   $(TOP)/src/vxworks.h \
   $(TOP)/src/whereInt.h \
-   sqlite_cfg.h
+   config.h

 # Header files used by extensions
 #
+EXTHDR += \
+  $(TOP)/ext/fts1/fts1.h \
+  $(TOP)/ext/fts1/fts1_hash.h \
+  $(TOP)/ext/fts1/fts1_tokenizer.h
+EXTHDR += \
+  $(TOP)/ext/fts2/fts2.h \
+  $(TOP)/ext/fts2/fts2_hash.h \
+  $(TOP)/ext/fts2/fts2_tokenizer.h
 EXTHDR += \
  $(TOP)/ext/fts3/fts3.h \
  $(TOP)/ext/fts3/fts3Int.h \
@@ -588,8 +610,7 @@ TESTOPTS = --verbose=file --output=test-out.txt

 # Extra compiler options for various shell tools
 #
-SHELL_OPT += -DSQLITE_DQS=0
-SHELL_OPT += -DSQLITE_ENABLE_FTS4
+SHELL_OPT = -DSQLITE_ENABLE_FTS4
 #SHELL_OPT += -DSQLITE_ENABLE_FTS5
 SHELL_OPT += -DSQLITE_ENABLE_RTREE
 SHELL_OPT += -DSQLITE_ENABLE_EXPLAIN_COMMENTS
@@ -600,57 +621,23 @@ SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
 FUZZERSHELL_OPT = 
-FUZZCHECK_OPT += -I$(TOP)/test
-FUZZCHECK_OPT += -I$(TOP)/ext/recover
-FUZZCHECK_OPT += \
-  -DSQLITE_OSS_FUZZ \
-  -DSQLITE_ENABLE_BYTECODE_VTAB \
-  -DSQLITE_ENABLE_DBPAGE_VTAB \
-  -DSQLITE_ENABLE_DBSTAT_VTAB \
-  -DSQLITE_ENABLE_BYTECODE_VTAB \
-  -DSQLITE_ENABLE_DESERIALIZE \
-  -DSQLITE_ENABLE_EXPLAIN_COMMENTS \
-  -DSQLITE_ENABLE_FTS3_PARENTHESIS \
-  -DSQLITE_ENABLE_FTS4 \
-  -DSQLITE_ENABLE_FTS5 \
-  -DSQLITE_ENABLE_GEOPOLY \
-  -DSQLITE_ENABLE_MATH_FUNCTIONS \
-  -DSQLITE_ENABLE_MEMSYS5 \
-  -DSQLITE_ENABLE_NORMALIZE \
-  -DSQLITE_ENABLE_OFFSET_SQL_FUNC \
-  -DSQLITE_ENABLE_PREUPDATE_HOOK \
-  -DSQLITE_ENABLE_RTREE \
-  -DSQLITE_ENABLE_SESSION \
-  -DSQLITE_ENABLE_STMTVTAB \
-  -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \
-  -DSQLITE_ENABLE_STAT4 \
-  -DSQLITE_ENABLE_STMT_SCANSTATUS \
-  -DSQLITE_MAX_MEMORY=50000000 \
-  -DSQLITE_MAX_MMAP_SIZE=0 \
-  -DSQLITE_OMIT_LOAD_EXTENSION \
-  -DSQLITE_PRINTF_PRECISION_LIMIT=1000 \
-  -DSQLITE_PRIVATE=""
-
-FUZZCHECK_SRC += $(TOP)/test/fuzzcheck.c
-FUZZCHECK_SRC += $(TOP)/test/ossfuzz.c
-FUZZCHECK_SRC += $(TOP)/test/fuzzinvariants.c
-FUZZCHECK_SRC += $(TOP)/ext/recover/dbdata.c
-FUZZCHECK_SRC += $(TOP)/ext/recover/sqlite3recover.c
-FUZZCHECK_SRC += $(TOP)/test/vt02.c
+FUZZCHECK_OPT = -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
+FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
+FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
+FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS4
+FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS3_PARENTHESIS
+FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS5
+FUZZCHECK_OPT += -DSQLITE_ENABLE_RTREE
+FUZZCHECK_OPT += -DSQLITE_ENABLE_GEOPOLY
+FUZZCHECK_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
+FUZZCHECK_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB
+FUZZCHECK_SRC = $(TOP)/test/fuzzcheck.c $(TOP)/test/ossfuzz.c $(TOP)/test/fuzzinvariants.c
 DBFUZZ_OPT =
-ST_OPT = -DSQLITE_OS_KV_OPTIONAL
-
-
-# In wasi-sdk builds, disable the CLI shell build in the "all" target.
-SQLITE3_SHELL_TARGET_  = sqlite3$(TEXE)
-SQLITE3_SHELL_TARGET_1 =
-SQLITE3_SHELL_TARGET   = $(SQLITE3_SHELL_TARGET_@HAVE_WASI_SDK@)

 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
 #
-all:	sqlite3.h libsqlite3.la $(SQLITE3_SHELL_TARGET) \
-  $(HAVE_TCL:1=libtclsqlite3.la)
+all:	sqlite3.h libsqlite3.la sqlite3$(TEXE) $(HAVE_TCL:1=libtclsqlite3.la)

 Makefile: $(TOP)/Makefile.in
 	./config.status
@@ -694,12 +681,9 @@ fuzzershell$(TEXE):	$(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(FUZZERSHELL_OPT) \
 	  $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS)

-fuzzcheck$(TEXE):	$(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP)
+fuzzcheck$(TEXE):	$(FUZZCHECK_SRC) sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)

-fuzzcheck-asan$(TEXE):	$(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP)
-	$(LTLINK) -o $@ -fsanitize=address $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)
-
 ossshell$(TEXE):	$(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \
             $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS)
@@ -780,15 +764,6 @@ sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl
 	cp tsrc/sqlite3ext.h .
 	cp $(TOP)/ext/session/sqlite3session.h .

-sqlite3r.h: sqlite3.h
-	$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) --enable-recover >sqlite3r.h
-
-sqlite3r.c: sqlite3.c sqlite3r.h
-	cp $(TOP)/ext/recover/sqlite3recover.c tsrc/
-	cp $(TOP)/ext/recover/sqlite3recover.h tsrc/
-	cp $(TOP)/ext/recover/dbdata.c tsrc/
-	$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl --enable-recover $(AMALGAMATION_LINE_MACROS)
-
 sqlite3ext.h:	.target_source
 	cp tsrc/sqlite3ext.h .

@@ -963,9 +938,6 @@ pcache1.lo:	$(TOP)/src/pcache1.c $(HDR) $(TOP)/src/pcache.h
 os.lo:	$(TOP)/src/os.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/os.c

-os_kv.lo:	$(TOP)/src/os_kv.c $(HDR)
-	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/os_kv.c
-
 os_unix.lo:	$(TOP)/src/os_unix.c $(HDR)
 	$(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/os_unix.c

@@ -1119,9 +1091,6 @@ SHELL_SRC = \
        $(TOP)/ext/misc/appendvfs.c \
 	$(TOP)/ext/misc/completion.c \
        $(TOP)/ext/misc/decimal.c \
-        $(TOP)/ext/misc/basexx.c \
-        $(TOP)/ext/misc/base64.c \
-        $(TOP)/ext/misc/base85.c \
 	$(TOP)/ext/misc/fileio.c \
        $(TOP)/ext/misc/ieee754.c \
        $(TOP)/ext/misc/regexp.c \
@@ -1133,9 +1102,6 @@ SHELL_SRC = \
 	$(TOP)/ext/expert/sqlite3expert.h \
 	$(TOP)/ext/misc/zipfile.c \
 	$(TOP)/ext/misc/memtrace.c \
-	$(TOP)/ext/recover/dbdata.c \
-	$(TOP)/ext/recover/sqlite3recover.c \
-	$(TOP)/ext/recover/sqlite3recover.h \
        $(TOP)/src/test_windirent.c

 shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
@@ -1149,6 +1115,24 @@ shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
 icu.lo:	$(TOP)/ext/icu/icu.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/icu/icu.c

+fts2.lo:	$(TOP)/ext/fts2/fts2.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2.c
+
+fts2_hash.lo:	$(TOP)/ext/fts2/fts2_hash.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_hash.c
+
+fts2_icu.lo:	$(TOP)/ext/fts2/fts2_icu.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_icu.c
+
+fts2_porter.lo:	$(TOP)/ext/fts2/fts2_porter.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_porter.c
+
+fts2_tokenizer.lo:	$(TOP)/ext/fts2/fts2_tokenizer.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer.c
+
+fts2_tokenizer1.lo:	$(TOP)/ext/fts2/fts2_tokenizer1.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts2/fts2_tokenizer1.c
+
 fts3.lo:	$(TOP)/ext/fts3/fts3.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/fts3/fts3.c

@@ -1303,15 +1287,6 @@ tcltest:	./testfixture$(TEXE)
 testrunner:	testfixture$(TEXE)
 	./testfixture$(TEXE) $(TOP)/test/testrunner.tcl

-# Runs both fuzztest and testrunner, consecutively.
-#
-devtest:	testfixture$(TEXE) fuzztest testrunner
-
-# Testing for a release
-#
-releasetest: testfixture$(TEXE)
-	./testfixture$(TEXE) $(TOP)/test/testrunner.tcl release
-
 # Minimal testing that runs in less than 3 minutes
 #
 quicktest:	./testfixture$(TEXE)
@@ -1412,7 +1387,7 @@ LogEst$(TEXE):	$(TOP)/tool/logest.c sqlite3.h
 wordcount$(TEXE):	$(TOP)/test/wordcount.c sqlite3.lo
 	$(LTLINK) -o $@ $(TOP)/test/wordcount.c sqlite3.lo $(TLIBS)

-speedtest1$(TEXE):	$(TOP)/test/speedtest1.c sqlite3.c Makefile
+speedtest1$(TEXE):	$(TOP)/test/speedtest1.c sqlite3.c
 	$(LTLINK) $(ST_OPT) -o $@ $(TOP)/test/speedtest1.c sqlite3.c $(TLIBS)

 startup$(TEXE):	$(TOP)/test/startup.c sqlite3.c
@@ -1469,6 +1444,9 @@ threadtest: threadtest3$(TEXE)
 threadtest5: sqlite3.c $(TOP)/test/threadtest5.c
 	$(LTLINK) $(TOP)/test/threadtest5.c sqlite3.c -o $@ $(TLIBS)

+releasetest:
+	$(TCLSH_CMD) $(TOP)/test/releasetest.tcl
+
 # Standard install and cleanup targets
 #
 lib_install:	libsqlite3.la
@@ -1523,7 +1501,7 @@ clean:
 	rm -f threadtest5

 distclean:	clean
-	rm -f sqlite_cfg.h config.log config.status libtool Makefile sqlite3.pc \
+	rm -f config.h config.log config.status libtool Makefile sqlite3.pc \
 		$(TESTPROGS)

 #
@@ -1544,8 +1522,143 @@ sqlite3.dll: $(REAL_LIBOBJ) sqlite3.def
 	$(TCC) -shared -o $@ sqlite3.def \
 		-Wl,"--strip-all" $(REAL_LIBOBJ)

+
 #
-# Fiddle app
+# fiddle/wasm section
 #
-fiddle: sqlite3.c shell.c
-	make -C ext/wasm fiddle emcc_opt=-Os
+# Maintenance reminder: we can/should move this into the wasm-specific
+# GNU Make makefile, but we currently need it here for access to
+# $(SHELL_OPT). The rest of the wasm-related bits are handled via GNU
+# Make in ext/wasm/...
+#
+wasm_dir = ext/wasm
+wasm_dir_abs = $(TOP)/ext/wasm
+# ^^^ some emcc opts require absolute paths
+fiddle_dir = $(wasm_dir)/fiddle
+fiddle_dir_abs = $(TOP)/$(fiddle_dir)
+fiddle_module_js = $(fiddle_dir)/fiddle-module.js
+#emcc_opt = -O0
+#emcc_opt = -O1
+#emcc_opt = -O2
+#emcc_opt = -O3
+emcc_opt = -Oz
+emcc_flags = $(emcc_opt) \
+        -sALLOW_TABLE_GROWTH \
+        -sABORTING_MALLOC \
+        -sSTRICT_JS \
+        -sENVIRONMENT=web \
+        -sMODULARIZE \
+        -sEXPORTED_RUNTIME_METHODS=@$(wasm_dir_abs)/EXPORTED_RUNTIME_METHODS.fiddle \
+        -sDYNAMIC_EXECUTION=0 \
+        --minify 0 \
+        -I. $(SHELL_OPT) \
+        -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_UTF16 -DSQLITE_OMIT_DEPRECATED
+$(fiddle_module_js): Makefile sqlite3.c shell.c \
+    $(wasm_dir)/EXPORTED_RUNTIME_METHODS.fiddle \
+    $(wasm_dir)/EXPORTED_FUNCTIONS.fiddle
+	emcc -o $@ $(emcc_flags) \
+        -sEXPORT_NAME=initFiddleModule \
+        -sEXPORTED_FUNCTIONS=@$(wasm_dir_abs)/EXPORTED_FUNCTIONS.fiddle \
+        -DSQLITE_SHELL_FIDDLE \
+        sqlite3.c shell.c
+	gzip < $@ > $@.gz
+	gzip < $(fiddle_dir)/fiddle-module.wasm > $(fiddle_dir)/fiddle-module.wasm.gz
+$(fiddle_dir)/fiddle.js.gz: $(fiddle_dir)/fiddle.js
+	gzip < $< > $@
+
+fiddle_generated = $(fiddle_module_js) $(fiddle_module_js).gz \
+                   $(fiddle_dir)/fiddle-module.wasm \
+                   $(fiddle_dir)/fiddle-module.wasm.gz \
+                   $(fiddle_dir)/fiddle.js.gz
+
+clean-fiddle:
+	rm -f $(fiddle_generated)
+clean: clean-fiddle
+fiddle: $(fiddle_module_js) $(fiddle_dir)/fiddle.js.gz
+wasm: fiddle
+########################################################################
+# Explanation of the emcc build flags follows. Full docs for these can
+# be found at:
+#
+#  https://github.com/emscripten-core/emscripten/blob/main/src/settings.js
+#
+# -sENVIRONMENT=web: elides bootstrap code related to non-web JS
+#  environments like node.js. Removing this makes the output a tiny
+#  tick larger but hypothetically makes it more portable to
+#  non-browser JS environments.
+#
+# -sMODULARIZE: changes how the generated code is structured to avoid
+#  declaring a global Module object and instead installing a function
+#  which loads and initializes the module. The function is named...
+#
+# -sEXPORT_NAME=jsFunctionName (see -sMODULARIZE)
+#
+# -sEXPORTED_RUNTIME_METHODS=@/absolute/path/to/file: a file
+#  containing a list of emscripten-supplied APIs, one per line, which
+#  must be exported into the generated JS. Must be an absolute path!
+#
+# -sEXPORTED_FUNCTIONS=@/absolute/path/to/file: a file containing a
+#  list of C functions, one per line, which must be exported via wasm
+#  so they're visible to JS. C symbols names in that file must all
+#  start with an underscore for reasons known only to the emcc
+#  developers. e.g., _sqlite3_open_v2 and _sqlite3_finalize. Must be
+#  an absolute path!
+#
+# -sSTRICT_JS ensures that the emitted JS code includes the 'use
+#  strict' option. Note that -sSTRICT is more broadly-scoped and
+#  results in build errors.
+#
+# -sALLOW_TABLE_GROWTH is required for (at a minimum) the UDF-binding
+#  feature. Without it, JS functions cannot be made to proxy C-side
+#  callbacks.
+#
+# -sABORTING_MALLOC causes the JS-bound _malloc() to abort rather than
+#  return 0 on OOM. If set to 0 then all code which uses _malloc()
+#  must, just like in C, check the result before using it, else
+#  they're likely to corrupt the JS/WASM heap by writing to its
+#  address of 0. It is, as of this writing, enabled in Emscripten by
+#  default but we enable it explicitly in case that default changes.
+#
+# -sDYNAMIC_EXECUTION=0 disables eval() and the Function constructor.
+#  If the build runs without these, it's preferable to use this flag
+#  because certain execution environments disallow those constructs.
+#  This flag is not strictly necessary, however.
+#
+# -sWASM_BIGINT is UNTESTED but "should" allow the int64-using C APIs
+#  to work with JS/wasm, insofar as the JS environment supports the
+#  BigInt type. That support requires an extremely recent browser:
+#  Safari didn't get that support until late 2020.
+#
+# --no-entry: for compiling library code with no main(). If this is
+#  not supplied and the code has a main(), it is called as part of the
+#  module init process. Note that main() is #if'd out of shell.c
+#  (renamed) when building in wasm mode.
+#
+# --pre-js/--post-js=FILE relative or absolute paths to JS files to
+#  prepend/append to the emcc-generated bootstrapping JS. It's
+#  easier/faster to develop with separate JS files (reduces rebuilding
+#  requirements) but certain configurations, namely -sMODULARIZE, may
+#  require using at least a --pre-js file. They can be used
+#  individually and need not be paired.
+#
+# -O0..-O3 and -Oz: optimization levels affect not only C-style
+#  optimization but whether or not the resulting generated JS code
+#  gets minified. -O0 compiles _much_ more quickly than -O3 or -Oz,
+#  and doesn't minimize any JS code, so is recommended for
+#  development. -O3 or -Oz are recommended for deployment, but
+#  primarily because -Oz will shrink the wasm file notably. JS-side
+#  minification makes little difference in terms of overall
+#  distributable size.
+#
+# --minify 0: disables minification of the generated JS code,
+#  regardless of optimization level. Minification of the JS has
+#  minimal overall effect in the larger scheme of things and results
+#  in JS files which can neither be edited nor viewed as text files in
+#  Fossil (which flags them as binary because of their extreme line
+#  lengths). Interestingly, whether or not the comments in the
+#  generated JS file get stripped is unaffected by this setting and
+#  depends entirely on the optimization level. Higher optimization
+#  levels reduce the size of the JS considerably even without
+#  minification.
+#
+########################################################################
@@ -1251,7 +1251,7 @@ LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \
         main.lo malloc.lo mem0.lo mem1.lo mem2.lo mem3.lo mem5.lo \
         memdb.lo memjournal.lo \
         mutex.lo mutex_noop.lo mutex_unix.lo mutex_w32.lo \
-         notify.lo opcodes.lo os.lo os_kv.lo os_unix.lo os_win.lo \
+         notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \
         pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \
         random.lo resolve.lo rowset.lo rtree.lo \
         sqlite3session.lo select.lo sqlite3rbu.lo status.lo stmt.lo \
@@ -1332,7 +1332,6 @@ SRC00 = \
  $(TOP)\src\mutex_w32.c \
  $(TOP)\src\notify.c \
  $(TOP)\src\os.c \
-  $(TOP)\src\os_kv.c \
  $(TOP)\src\os_unix.c \
  $(TOP)\src\os_win.c

@@ -1412,6 +1411,20 @@ SRC05 = \
  $(TOP)\src\wal.h \
  $(TOP)\src\whereInt.h

+# Extension source code files, part 1.
+#
+SRC06 = \
+  $(TOP)\ext\fts1\fts1.c \
+  $(TOP)\ext\fts1\fts1_hash.c \
+  $(TOP)\ext\fts1\fts1_porter.c \
+  $(TOP)\ext\fts1\fts1_tokenizer1.c \
+  $(TOP)\ext\fts2\fts2.c \
+  $(TOP)\ext\fts2\fts2_hash.c \
+  $(TOP)\ext\fts2\fts2_icu.c \
+  $(TOP)\ext\fts2\fts2_porter.c \
+  $(TOP)\ext\fts2\fts2_tokenizer.c \
+  $(TOP)\ext\fts2\fts2_tokenizer1.c
+
 # Extension source code files, part 2.
 #
 SRC07 = \
@@ -1434,6 +1447,16 @@ SRC07 = \
  $(TOP)\ext\rbu\sqlite3rbu.c \
  $(TOP)\ext\misc\stmt.c

+# Extension header files, part 1.
+#
+SRC08 = \
+  $(TOP)\ext\fts1\fts1.h \
+  $(TOP)\ext\fts1\fts1_hash.h \
+  $(TOP)\ext\fts1\fts1_tokenizer.h \
+  $(TOP)\ext\fts2\fts2.h \
+  $(TOP)\ext\fts2\fts2_hash.h \
+  $(TOP)\ext\fts2\fts2_tokenizer.h
+
 # Extension header files, part 2.
 #
 SRC09 = \
@@ -1474,7 +1497,7 @@ SRC12 =

 # All source code files.
 #
-SRC = $(SRC00) $(SRC01) $(SRC03) $(SRC04) $(SRC05) $(SRC07) $(SRC09) $(SRC10) $(SRC11) $(SRC12)
+SRC = $(SRC00) $(SRC01) $(SRC03) $(SRC04) $(SRC05) $(SRC06) $(SRC07) $(SRC08) $(SRC09) $(SRC10) $(SRC11) $(SRC12)

 # Source code to the test files.
 #
@@ -1485,6 +1508,7 @@ TESTSRC = \
  $(TOP)\src\test4.c \
  $(TOP)\src\test5.c \
  $(TOP)\src\test6.c \
+  $(TOP)\src\test7.c \
  $(TOP)\src\test8.c \
  $(TOP)\src\test9.c \
  $(TOP)\src\test_autoext.c \
@@ -1513,6 +1537,7 @@ TESTSRC = \
  $(TOP)\src\test_quota.c \
  $(TOP)\src\test_rtree.c \
  $(TOP)\src\test_schema.c \
+  $(TOP)\src\test_server.c \
  $(TOP)\src\test_superlock.c \
  $(TOP)\src\test_syscall.c \
  $(TOP)\src\test_tclsh.c \
@@ -1526,7 +1551,7 @@ TESTSRC = \
  $(TOP)\ext\fts3\fts3_term.c \
  $(TOP)\ext\fts3\fts3_test.c \
  $(TOP)\ext\rbu\test_rbu.c \
-  $(TOP)\ext\session\test_session.c
+  $(TOP)\ext\session\test_session.c 

 # Statically linked extensions.
 #
@@ -1535,7 +1560,6 @@ TESTEXT = \
  $(TOP)\ext\expert\test_expert.c \
  $(TOP)\ext\misc\amatch.c \
  $(TOP)\ext\misc\appendvfs.c \
-  $(TOP)\ext\misc\basexx.c \
  $(TOP)\ext\misc\carray.c \
  $(TOP)\ext\misc\cksumvfs.c \
  $(TOP)\ext\misc\closure.c \
@@ -1563,9 +1587,6 @@ TESTEXT = \
  $(TOP)\ext\misc\unionvtab.c \
  $(TOP)\ext\misc\wholenumber.c \
  $(TOP)\ext\rtree\test_rtreedoc.c \
-  $(TOP)\ext\recover\sqlite3recover.c \
-  $(TOP)\ext\recover\test_recover.c \
-  $(TOP)\ext\recover\dbdata.c \
  fts5.c

 # If use of zlib is enabled, add the "zipfile.c" source file.
@@ -1580,6 +1601,7 @@ TESTEXT = $(TESTEXT) $(TOP)\ext\misc\zipfile.c
 TESTSRC2 = \
  $(SRC00) \
  $(SRC01) \
+  $(SRC06) \
  $(SRC07) \
  $(SRC10) \
  $(TOP)\ext\async\sqlite3async.c
@@ -1614,6 +1636,14 @@ HDR = \

 # Header files used by extensions
 #
+EXTHDR = $(EXTHDR) \
+  $(TOP)\ext\fts1\fts1.h \
+  $(TOP)\ext\fts1\fts1_hash.h \
+  $(TOP)\ext\fts1\fts1_tokenizer.h
+EXTHDR = $(EXTHDR) \
+  $(TOP)\ext\fts2\fts2.h \
+  $(TOP)\ext\fts2\fts2_hash.h \
+  $(TOP)\ext\fts2\fts2_tokenizer.h
 EXTHDR = $(EXTHDR) \
  $(TOP)\ext\fts3\fts3.h \
  $(TOP)\ext\fts3\fts3Int.h \
@@ -1657,7 +1687,6 @@ FUZZDATA = \
 # when the shell is not being dynamically linked.
 #
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_DQS=0
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
@@ -1668,52 +1697,15 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
 #
 MPTESTER_COMPILE_OPTS = -DSQLITE_ENABLE_FTS5
 FUZZERSHELL_COMPILE_OPTS =
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -I$(TOP)\test -I$(TOP)\ext\recover
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MEMSYS5
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OSS_FUZZ
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBSTAT_VTAB
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DESERIALIZE
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS3_PARENTHESIS
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS4
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS5
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_GEOPOLY
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MATH_FUNCTIONS
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MEMSYS5
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_NORMALIZE
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_PREUPDATE_HOOK
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_RTREE
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_SESSION
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_STMTVTAB
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_STAT4
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MEMORY=50000000
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MMAP_SIZE=0
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRIVATE=""
-
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MEMORY=50000000
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION
+FUZZCHECK_OPTS = -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ -DSQLITE_MAX_MEMORY=50000000 -DSQLITE_PRINTF_PRECISION_LIMIT=1000
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS4
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS5
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_RTREE
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_GEOPOLY
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBSTAT_VTAB
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB
-FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\fuzzcheck.c
-FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\ossfuzz.c
-FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\fuzzinvariants.c
-FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\vt02.c
-FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\dbdata.c
-FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\sqlite3recover.c

+FUZZCHECK_SRC = $(TOP)\test\fuzzcheck.c $(TOP)\test\ossfuzz.c $(TOP)\test\fuzzinvariants.c
 OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c
 DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION
 KV_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_DIRECT_OVERFLOW_READ
@@ -1816,9 +1808,6 @@ dbfuzz.exe:	$(TOP)\test\dbfuzz.c $(SQLITE3C) $(SQLITE3H)
 fuzzcheck.exe:	$(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(FUZZCHECK_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

-fuzzcheck-asan.exe:	$(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H)
-	$(LTLINK) $(NO_WARN) /fsanitize=address $(FUZZCHECK_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
-
 ossshell.exe:	$(OSSSHELL_SRC) $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(FUZZCHECK_OPTS) $(OSSSHELL_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

@@ -1856,7 +1845,9 @@ mptest:	mptester.exe
 	for %i in ($(SRC03)) do copy /Y %i tsrc
 	for %i in ($(SRC04)) do copy /Y %i tsrc
 	for %i in ($(SRC05)) do copy /Y %i tsrc
+	for %i in ($(SRC06)) do copy /Y %i tsrc
 	for %i in ($(SRC07)) do copy /Y %i tsrc
+	for %i in ($(SRC08)) do copy /Y %i tsrc
 	for %i in ($(SRC09)) do copy /Y %i tsrc
 	for %i in ($(SRC10)) do copy /Y %i tsrc
 	for %i in ($(SRC11)) do copy /Y %i tsrc
@@ -2062,9 +2053,6 @@ pcache1.lo:	$(TOP)\src\pcache1.c $(HDR) $(TOP)\src\pcache.h
 os.lo:	$(TOP)\src\os.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\os.c

-os_kv.lo:	$(TOP)\src\os_kv.c $(HDR)
-	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\os_kv.c
-
 os_unix.lo:	$(TOP)\src\os_unix.c $(HDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) -c $(TOP)\src\os_unix.c

@@ -2226,8 +2214,6 @@ SHELL_SRC = \
 	$(TOP)\src\shell.c.in \
 	$(TOP)\ext\misc\appendvfs.c \
 	$(TOP)\ext\misc\completion.c \
-        $(TOP)\ext\misc\base64.c \
-        $(TOP)\ext\misc\base85.c \
 	$(TOP)\ext\misc\decimal.c \
 	$(TOP)\ext\misc\fileio.c \
 	$(TOP)\ext\misc\ieee754.c \
@@ -2238,9 +2224,6 @@ SHELL_SRC = \
 	$(TOP)\ext\expert\sqlite3expert.c \
 	$(TOP)\ext\expert\sqlite3expert.h \
 	$(TOP)\ext\misc\memtrace.c \
-	$(TOP)/ext/recover/dbdata.c \
-	$(TOP)/ext/recover/sqlite3recover.c \
-	$(TOP)/ext/recover/sqlite3recover.h \
 	$(TOP)\src\test_windirent.c

 # If use of zlib is enabled, add the "zipfile.c" source file.
@@ -2261,6 +2244,24 @@ zlib:
 icu.lo:	$(TOP)\ext\icu\icu.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\icu\icu.c

+fts2.lo:	$(TOP)\ext\fts2\fts2.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2.c
+
+fts2_hash.lo:	$(TOP)\ext\fts2\fts2_hash.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_hash.c
+
+fts2_icu.lo:	$(TOP)\ext\fts2\fts2_icu.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_icu.c
+
+fts2_porter.lo:	$(TOP)\ext\fts2\fts2_porter.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_porter.c
+
+fts2_tokenizer.lo:	$(TOP)\ext\fts2\fts2_tokenizer.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_tokenizer.c
+
+fts2_tokenizer1.lo:	$(TOP)\ext\fts2\fts2_tokenizer1.c $(HDR) $(EXTHDR)
+	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts2\fts2_tokenizer1.c
+
 fts3.lo:	$(TOP)\ext\fts3\fts3.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) $(CORE_COMPILE_OPTS) $(NO_WARN) -DSQLITE_CORE -c $(TOP)\ext\fts3\fts3.c

@@ -2480,16 +2481,6 @@ tcltest: testfixture.exe
 testrunner:	testfixture.exe
 	.\testfixture.exe $(TOP)\test\testrunner.tcl

-# Runs both fuzztest and testrunner, consecutively.
-#
-devtest:	testfixture.exe fuzztest testrunner
-
-# Testing for a release
-#
-releasetest: testfixture.exe fuzztest
-	testfixture.exe $(TOP)/test/testrunner.tcl release
-
-
 smoketest:	$(TESTPROGS)
 	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\main.test $(TESTOPTS)
@@ -7,7 +7,7 @@ and most of the documentation are managed separately.

 ## Version Control

-SQLite sources are managed using
+SQLite sources are managed using the
 [Fossil](https://www.fossil-scm.org/), a distributed version control system
 that was specifically designed and written to support SQLite development.
 The [Fossil repository](https://sqlite.org/src/timeline) contains the urtext.
@@ -1 +1 @@
-3.42.0
+3.40.0
@@ -955,7 +955,6 @@ LIBRESOBJS =
 # when the shell is not being dynamically linked.
 #
 !IF $(DYNAMIC_SHELL)==0 && $(FOR_WIN10)==0
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_DQS=0
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
@@ -19,7 +19,7 @@ dnl	to configure the system for the local environment.
 # so that we create the export library with the dll.
 #-----------------------------------------------------------------------

-AC_INIT([sqlite],[3.42.0])
+AC_INIT([sqlite],[3.40.0])

 #--------------------------------------------------------------------
 # Call TEA_INIT as the first TEA_ macro to set up initial vars.
@@ -1,4 +1,4 @@
-/* sqlite_cfg.h.in.  Generated from configure.ac by autoheader.  */
+/* config.h.in.  Generated from configure.ac by autoheader.  */

 /* Define to 1 if you have the <dlfcn.h> header file. */
 #undef HAVE_DLFCN_H
@@ -45,16 +45,16 @@
 /* Define to 1 if you have the <memory.h> header file. */
 #undef HAVE_MEMORY_H

-/* Define to 1 if you have the `pread' function. */
+/* Define to 1 if you have the pread() function. */
 #undef HAVE_PREAD

-/* Define to 1 if you have the `pread64' function. */
+/* Define to 1 if you have the pread64() function. */
 #undef HAVE_PREAD64

-/* Define to 1 if you have the `pwrite' function. */
+/* Define to 1 if you have the pwrite() function. */
 #undef HAVE_PWRITE

-/* Define to 1 if you have the `pwrite64' function. */
+/* Define to 1 if you have the pwrite64() function. */
 #undef HAVE_PWRITE64

 /* Define to 1 if you have the <stdint.h> header file. */
@@ -63,7 +63,7 @@
 /* Define to 1 if you have the <stdlib.h> header file. */
 #undef HAVE_STDLIB_H

-/* Define to 1 if you have the `strchrnul' function. */
+/* Define to 1 if you have the strchrnul() function */
 #undef HAVE_STRCHRNUL

 /* Define to 1 if you have the <strings.h> header file. */
@@ -99,12 +99,9 @@
 /* Define to 1 if you have the `usleep' function. */
 #undef HAVE_USLEEP

-/* Define to 1 if you have the `utime' function. */
+/* Define to 1 if you have the utime() library function. */
 #undef HAVE_UTIME

-/* Define to 1 if you have the <zlib.h> header file. */
-#undef HAVE_ZLIB_H
-
 /* Define to the sub-directory in which libtool stores uninstalled libraries.
   */
 #undef LT_OBJDIR
@@ -121,20 +118,12 @@
 /* Define to the one symbol short name of this package. */
 #undef PACKAGE_TARNAME

-/* Define to the home page for this package. */
-#undef PACKAGE_URL
-
 /* Define to the version of this package. */
 #undef PACKAGE_VERSION

 /* Define to 1 if you have the ANSI C header files. */
 #undef STDC_HEADERS

-/* Enable large inode numbers on Mac OS X 10.5.  */
-#ifndef _DARWIN_USE_64_BIT_INODE
-# define _DARWIN_USE_64_BIT_INODE 1
-#endif
-
 /* Number of bits in a file offset, on hosts where this is settable. */
 #undef _FILE_OFFSET_BITS

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.42.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.40.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.42.0'
-PACKAGE_STRING='sqlite 3.42.0'
+PACKAGE_VERSION='3.40.0'
+PACKAGE_STRING='sqlite 3.40.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''

@@ -800,7 +800,6 @@ TEMP_STORE
 ALLOWRELEASE
 SQLITE_THREADSAFE
 BUILD_CC
-HAVE_WASI_SDK
 RELEASE
 VERSION
 program_prefix
@@ -893,7 +892,6 @@ enable_fast_install
 with_gnu_ld
 enable_libtool_lock
 enable_largefile
-with_wasi_sdk
 enable_threadsafe
 enable_releasemode
 enable_tempstore
@@ -1470,7 +1468,7 @@ if test "$ac_init_help" = "long"; then
  # Omit some internal or obsolete options to make the list less imposing.
  # This message is too long to be a string in the A/UX 3.1 sh.
  cat <<_ACEOF
-\`configure' configures sqlite 3.42.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.40.0 to adapt to many kinds of systems.

 Usage: $0 [OPTION]... [VAR=VALUE]...

@@ -1535,7 +1533,7 @@ fi

 if test -n "$ac_init_help"; then
  case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.42.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.40.0:";;
   esac
  cat <<\_ACEOF

@@ -1581,8 +1579,6 @@ Optional Packages:
  --with-pic              try to use only PIC/non-PIC objects [default=use
                          both]
  --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
-  --with-wasi-sdk=DIR     directory containing the WASI SDK. Triggers
-                          cross-compile to WASM.
  --with-tcl=DIR          directory containing tcl configuration
                          (tclConfig.sh)
  --with-readline-lib     specify readline library
@@ -1665,7 +1661,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
  cat <<\_ACEOF
-sqlite configure 3.42.0
+sqlite configure 3.40.0
 generated by GNU Autoconf 2.69

 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2084,7 +2080,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.

-It was created by sqlite $as_me 3.42.0, which was
+It was created by sqlite $as_me 3.40.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  $ $0 $@
@@ -3942,13 +3938,13 @@ if ${lt_cv_nm_interface+:} false; then :
 else
  lt_cv_nm_interface="BSD nm"
  echo "int some_variable = 0;" > conftest.$ac_ext
-  (eval echo "\"\$as_me:3945: $ac_compile\"" >&5)
+  (eval echo "\"\$as_me:3941: $ac_compile\"" >&5)
  (eval "$ac_compile" 2>conftest.err)
  cat conftest.err >&5
-  (eval echo "\"\$as_me:3948: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
+  (eval echo "\"\$as_me:3944: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
  (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
  cat conftest.err >&5
-  (eval echo "\"\$as_me:3951: output\"" >&5)
+  (eval echo "\"\$as_me:3947: output\"" >&5)
  cat conftest.out >&5
  if $GREP 'External.*some_variable' conftest.out > /dev/null; then
    lt_cv_nm_interface="MS dumpbin"
@@ -5154,7 +5150,7 @@ ia64-*-hpux*)
  ;;
 *-*-irix6*)
  # Find out which ABI we are using.
-  echo '#line 5157 "configure"' > conftest.$ac_ext
+  echo '#line 5153 "configure"' > conftest.$ac_ext
  if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
  (eval $ac_compile) 2>&5
  ac_status=$?
@@ -6679,11 +6675,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:6682: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:6678: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>conftest.err)
   ac_status=$?
   cat conftest.err >&5
-   echo "$as_me:6686: \$? = $ac_status" >&5
+   echo "$as_me:6682: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s "$ac_outfile"; then
     # The compiler can only warn and ignore the option if not recognized
     # So say no if there are warnings other than the usual output.
@@ -7018,11 +7014,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7021: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7017: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>conftest.err)
   ac_status=$?
   cat conftest.err >&5
-   echo "$as_me:7025: \$? = $ac_status" >&5
+   echo "$as_me:7021: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s "$ac_outfile"; then
     # The compiler can only warn and ignore the option if not recognized
     # So say no if there are warnings other than the usual output.
@@ -7123,11 +7119,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7126: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7122: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>out/conftest.err)
   ac_status=$?
   cat out/conftest.err >&5
-   echo "$as_me:7130: \$? = $ac_status" >&5
+   echo "$as_me:7126: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s out/conftest2.$ac_objext
   then
     # The compiler can only warn and ignore the option if not recognized
@@ -7178,11 +7174,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7181: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7177: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>out/conftest.err)
   ac_status=$?
   cat out/conftest.err >&5
-   echo "$as_me:7185: \$? = $ac_status" >&5
+   echo "$as_me:7181: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s out/conftest2.$ac_objext
   then
     # The compiler can only warn and ignore the option if not recognized
@@ -9558,7 +9554,7 @@ else
  lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  lt_status=$lt_dlunknown
  cat > conftest.$ac_ext <<_LT_EOF
-#line 9561 "configure"
+#line 9557 "configure"
 #include "confdefs.h"

 #if HAVE_DLFCN_H
@@ -9654,7 +9650,7 @@ else
  lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  lt_status=$lt_dlunknown
  cat > conftest.$ac_ext <<_LT_EOF
-#line 9657 "configure"
+#line 9653 "configure"
 #include "confdefs.h"

 #if HAVE_DLFCN_H
@@ -10397,68 +10393,6 @@ RELEASE=`cat $srcdir/VERSION`
 $as_echo "$as_me: Release set to $RELEASE" >&6;}


-##########
-# Handle --with-wasi-sdk=DIR
-#
-# This must be early because it changes the toolchain.
-#
-
-# Check whether --with-wasi-sdk was given.
-if test "${with_wasi_sdk+set}" = set; then :
-  withval=$with_wasi_sdk; with_wasisdk=${withval}
-fi
-
-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for WASI SDK directory" >&5
-$as_echo_n "checking for WASI SDK directory... " >&6; }
-if ${ac_cv_c_wasi_sdk+:} false; then :
-  $as_echo_n "(cached) " >&6
-else
-
-  # First check to see if --with-tcl was specified.
-  if test x"${with_wasi_sdk}" != x ; then
-    if ! test -d "${with_wasi_sdk}" ; then
-      as_fn_error $? "${with_wasi_sdk} directory doesn't exist" "$LINENO" 5
-    fi
-    { $as_echo "$as_me:${as_lineno-$LINENO}: result: ${with_wasi_sdk}: using wasi-sdk clang, disabling: tcl, CLI shell, DLL" >&5
-$as_echo "${with_wasi_sdk}: using wasi-sdk clang, disabling: tcl, CLI shell, DLL" >&6; }
-    use_wasi_sdk=yes
-  else
-    use_wasi_sdk=no
-  fi
-
-fi
-
-if test "${use_wasi_sdk}" = "no" ; then
-  HAVE_WASI_SDK=""
-  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; }
-else
-  HAVE_WASI_SDK=1
-# Changing --host and --target have no effect here except to possibly
-# cause confusion. autoconf has finished processing them by this
-# point.
-#
-#  host_alias=wasm32-wasi
-#  target=wasm32-wasi
-#
-# Merely changing CC and LD to the wasi-sdk's is enough to get
-# sqlite3.o building in WASM format.
-  CC="${with_wasi_sdk}/bin/clang"
-  LD="${with_wasi_sdk}/bin/wasm-ld"
-  RANLIB="${with_wasi_sdk}/bin/llvm-ranlib"
-  cross_compiling=yes
-  enable_threadsafe=no
-  use_tcl=no
-  enable_tcl=no
-  # libtool is apparently hard-coded to use gcc for linking DLLs, so
-  # we disable the DLL build...
-  enable_shared=no
-  { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
-$as_echo "yes" >&6; }
-fi
-
-
-
 #########
 # Locate a compiler for the build machine.  This compiler should
 # generate command-line programs that run on the build machine.
@@ -11333,7 +11267,6 @@ fi

 #########
 # See whether we should use the amalgamation to build
-
 # Check whether --enable-amalgamation was given.
 if test "${enable_amalgamation+set}" = set; then :
  enableval=$enable_amalgamation;
@@ -11942,7 +11875,7 @@ fi

 #########
 # Output the config header
-ac_config_headers="$ac_config_headers sqlite_cfg.h"
+ac_config_headers="$ac_config_headers config.h"


 #########
@@ -12457,7 +12390,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.42.0, which was
+This file was extended by sqlite $as_me 3.40.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  CONFIG_FILES    = $CONFIG_FILES
@@ -12523,7 +12456,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.42.0
+sqlite config.status 3.40.0
 configured by $0, generated by GNU Autoconf 2.69,
  with options \\"\$ac_cs_config\\"

@@ -12905,7 +12838,7 @@ for ac_config_target in $ac_config_targets
 do
  case $ac_config_target in
    "libtool") CONFIG_COMMANDS="$CONFIG_COMMANDS libtool" ;;
-    "sqlite_cfg.h") CONFIG_HEADERS="$CONFIG_HEADERS sqlite_cfg.h" ;;
+    "config.h") CONFIG_HEADERS="$CONFIG_HEADERS config.h" ;;
    "Makefile") CONFIG_FILES="$CONFIG_FILES Makefile" ;;
    "sqlite3.pc") CONFIG_FILES="$CONFIG_FILES sqlite3.pc" ;;

@@ -70,11 +70,11 @@
 #        target platform.  "" for Unix and ".exe" for windows.
 #
 # This configure.in file is easy to reuse on other projects.  Just
-# change the argument to AC_INIT.  And disable any features that
+# change the argument to AC_INIT().  And disable any features that
 # you don't need (for example BLT) by erasing or commenting out
 # the corresponding code.
 #
-AC_INIT([sqlite],[m4_esyscmd(cat VERSION | tr -d '\n')])
+AC_INIT(sqlite, m4_esyscmd([cat VERSION | tr -d '\n']))

 dnl Make sure the local VERSION file matches this configure script
 sqlite_version_sanity_check=`cat $srcdir/VERSION | tr -d '\n'`
@@ -88,7 +88,7 @@ fi
 #########
 # Programs needed
 #
-LT_INIT
+AC_PROG_LIBTOOL
 AC_PROG_INSTALL

 #########
@@ -158,56 +158,6 @@ RELEASE=`cat $srcdir/VERSION`
 AC_MSG_NOTICE(Release set to $RELEASE)
 AC_SUBST(RELEASE)

-##########
-# Handle --with-wasi-sdk=DIR
-#
-# This must be early because it changes the toolchain.
-#
-AC_ARG_WITH(wasi-sdk,
-AS_HELP_STRING([--with-wasi-sdk=DIR],
-       [directory containing the WASI SDK. Triggers cross-compile to WASM.]), with_wasisdk=${withval})
-AC_MSG_CHECKING([for WASI SDK directory])
-AC_CACHE_VAL(ac_cv_c_wasi_sdk,[
-  # First check to see if --with-tcl was specified.
-  if test x"${with_wasi_sdk}" != x ; then
-    if ! test -d "${with_wasi_sdk}" ; then
-      AC_MSG_ERROR([${with_wasi_sdk} directory doesn't exist])
-    fi
-    AC_MSG_RESULT([${with_wasi_sdk}: using wasi-sdk clang, disabling: tcl, CLI shell, DLL])
-    use_wasi_sdk=yes
-  else
-    use_wasi_sdk=no
-  fi
-])
-if test "${use_wasi_sdk}" = "no" ; then
-  HAVE_WASI_SDK=""
-  AC_MSG_RESULT([no])
-else
-  HAVE_WASI_SDK=1
-# Changing --host and --target have no effect here except to possibly
-# cause confusion. autoconf has finished processing them by this
-# point.
-#
-#  host_alias=wasm32-wasi
-#  target=wasm32-wasi
-#
-# Merely changing CC and LD to the wasi-sdk's is enough to get
-# sqlite3.o building in WASM format.
-  CC="${with_wasi_sdk}/bin/clang"
-  LD="${with_wasi_sdk}/bin/wasm-ld"
-  RANLIB="${with_wasi_sdk}/bin/llvm-ranlib"
-  cross_compiling=yes
-  enable_threadsafe=no
-  use_tcl=no
-  enable_tcl=no
-  # libtool is apparently hard-coded to use gcc for linking DLLs, so
-  # we disable the DLL build...
-  enable_shared=no
-  AC_MSG_RESULT([yes])
-fi
-AC_SUBST(HAVE_WASI_SDK)
-
-
 #########
 # Locate a compiler for the build machine.  This compiler should
 # generate command-line programs that run on the build machine.
@@ -229,7 +179,7 @@ AC_SUBST(BUILD_CC)
 # Do we want to support multithreaded use of sqlite
 #
 AC_ARG_ENABLE(threadsafe, 
-AS_HELP_STRING([--disable-threadsafe],[Disable mutexing]))
+AC_HELP_STRING([--disable-threadsafe],[Disable mutexing]))
 AC_MSG_CHECKING([whether to support threadsafe operation])
 if test "$enable_threadsafe" = "no"; then
  SQLITE_THREADSAFE=0
@@ -249,7 +199,7 @@ fi
 # Do we want to support release
 #
 AC_ARG_ENABLE(releasemode, 
-AS_HELP_STRING([--enable-releasemode],[Support libtool link to release mode]),,enable_releasemode=no)
+AC_HELP_STRING([--enable-releasemode],[Support libtool link to release mode]),,enable_releasemode=no)
 AC_MSG_CHECKING([whether to support shared library linked as release mode or not])
 if test "$enable_releasemode" = "no"; then
  ALLOWRELEASE=""
@@ -264,7 +214,7 @@ AC_SUBST(ALLOWRELEASE)
 # Do we want temporary databases in memory
 #
 AC_ARG_ENABLE(tempstore, 
-AS_HELP_STRING([--enable-tempstore],[Use an in-ram database for temporary tables (never,no,yes,always)]),,enable_tempstore=no)
+AC_HELP_STRING([--enable-tempstore],[Use an in-ram database for temporary tables (never,no,yes,always)]),,enable_tempstore=no)
 AC_MSG_CHECKING([whether to use an in-ram database for temporary tables])
 case "$enable_tempstore" in
  never ) 
@@ -304,15 +254,7 @@ else
  AC_MSG_RESULT(unknown)
 fi
 if test "$CYGWIN" != "yes"; then
-  m4_warn([obsolete],
-[AC_CYGWIN is obsolete: use AC_CANONICAL_HOST and check if $host_os
-matches *cygwin*])dnl
-AC_CANONICAL_HOST
-case $host_os in
-  *cygwin* ) CYGWIN=yes;;
-	 * ) CYGWIN=no;;
-esac
-
+  AC_CYGWIN
 fi
 if test "$CYGWIN" = "yes"; then
  BUILD_EXEEXT=.exe
@@ -347,10 +289,10 @@ AC_SUBST(TARGET_EXEEXT)
 # Those macros could not be used directly since we have to make some
 # minor changes to accomodate systems that do not have TCL installed.
 #
-AC_ARG_ENABLE(tcl, AS_HELP_STRING([--disable-tcl],[do not build TCL extension]),
+AC_ARG_ENABLE(tcl, AC_HELP_STRING([--disable-tcl],[do not build TCL extension]),
      [use_tcl=$enableval],[use_tcl=yes])
 if test "${use_tcl}" = "yes" ; then
-  AC_ARG_WITH(tcl, AS_HELP_STRING([--with-tcl=DIR],[directory containing tcl configuration (tclConfig.sh)]), with_tclconfig=${withval})
+  AC_ARG_WITH(tcl, AC_HELP_STRING([--with-tcl=DIR],[directory containing tcl configuration (tclConfig.sh)]), with_tclconfig=${withval})
  AC_MSG_CHECKING([for Tcl configuration])
  AC_CACHE_VAL(ac_cv_c_tclconfig,[
    # First check to see if --with-tcl was specified.
@@ -532,11 +474,11 @@ TARGET_READLINE_INC=""
 TARGET_HAVE_READLINE=0
 TARGET_HAVE_EDITLINE=0
 AC_ARG_ENABLE([editline],
-	[AS_HELP_STRING([--enable-editline],[enable BSD editline support])],
+	[AC_HELP_STRING([--enable-editline],[enable BSD editline support])],
 	[with_editline=$enableval],
 	[with_editline=auto])
 AC_ARG_ENABLE([readline],
-	[AS_HELP_STRING([--disable-readline],[disable readline support])],
+	[AC_HELP_STRING([--disable-readline],[disable readline support])],
 	[with_readline=$enableval],
 	[with_readline=auto])

@@ -552,7 +494,7 @@ if test x"$with_readline" != xno; then
 	found="yes"

 	AC_ARG_WITH([readline-lib],
-		[AS_HELP_STRING([--with-readline-lib],[specify readline library])],
+		[AC_HELP_STRING([--with-readline-lib],[specify readline library])],
 		[with_readline_lib=$withval],
 		[with_readline_lib="auto"])
 	if test "x$with_readline_lib" = xauto; then
@@ -567,7 +509,7 @@ if test x"$with_readline" != xno; then
 	fi

 	AC_ARG_WITH([readline-inc],
-		[AS_HELP_STRING([--with-readline-inc],[specify readline include paths])],
+		[AC_HELP_STRING([--with-readline-inc],[specify readline include paths])],
 		[with_readline_inc=$withval],
 		[with_readline_inc="auto"])
 	if test "x$with_readline_inc" = xauto; then
@@ -612,7 +554,7 @@ AC_SEARCH_LIBS(fdatasync, [rt])

 #########
 # check for debug enabled
-AC_ARG_ENABLE(debug, AS_HELP_STRING([--enable-debug],[enable debugging & verbose explain]))
+AC_ARG_ENABLE(debug, AC_HELP_STRING([--enable-debug],[enable debugging & verbose explain]))
 AC_MSG_CHECKING([build type])
 if test "${enable_debug}" = "yes" ; then
  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
@@ -625,8 +567,7 @@ AC_SUBST(TARGET_DEBUG)

 #########
 # See whether we should use the amalgamation to build
-
-AC_ARG_ENABLE(amalgamation, AS_HELP_STRING([--disable-amalgamation],
+AC_ARG_ENABLE(amalgamation, AC_HELP_STRING([--disable-amalgamation],
      [Disable the amalgamation and instead build all files separately]))
 if test "${enable_amalgamation}" = "no" ; then
  USE_AMALGAMATION=0
@@ -641,7 +582,7 @@ AC_SUBST(HAVE_ZLIB)

 #########
 # See whether we should allow loadable extensions
-AC_ARG_ENABLE(load-extension, AS_HELP_STRING([--disable-load-extension],
+AC_ARG_ENABLE(load-extension, AC_HELP_STRING([--disable-load-extension],
      [Disable loading of external extensions]),,[enable_load_extension=yes])
 if test "${enable_load_extension}" = "yes" ; then
  OPT_FEATURE_FLAGS=""
@@ -654,7 +595,7 @@ fi
 # Do we want to support math functions
 #
 AC_ARG_ENABLE(math, 
-AS_HELP_STRING([--disable-math],[Disable math functions]))
+AC_HELP_STRING([--disable-math],[Disable math functions]))
 AC_MSG_CHECKING([whether to support math functions])
 if test "$enable_math" = "no"; then
  AC_MSG_RESULT([no])
@@ -668,7 +609,7 @@ fi
 # Do we want to support JSON functions
 #
 AC_ARG_ENABLE(json, 
-AS_HELP_STRING([--disable-json],[Disable JSON functions]))
+AC_HELP_STRING([--disable-json],[Disable JSON functions]))
 AC_MSG_CHECKING([whether to support JSON functions])
 if test "$enable_json" = "no"; then
  AC_MSG_RESULT([no])
@@ -680,14 +621,14 @@ fi
 ########
 # The --enable-all argument is short-hand to enable
 # multiple extensions.
-AC_ARG_ENABLE(all, AS_HELP_STRING([--enable-all],
+AC_ARG_ENABLE(all, AC_HELP_STRING([--enable-all],
      [Enable FTS4, FTS5, Geopoly, RTree, Sessions]))

 ##########
 # Do we want to support memsys3 and/or memsys5
 #
 AC_ARG_ENABLE(memsys5, 
-  AS_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]))
+  AC_HELP_STRING([--enable-memsys5],[Enable MEMSYS5]))
 AC_MSG_CHECKING([whether to support MEMSYS5])
 if test "${enable_memsys5}" = "yes"; then
  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS5"
@@ -696,7 +637,7 @@ else
  AC_MSG_RESULT([no])
 fi
 AC_ARG_ENABLE(memsys3, 
-  AS_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]))
+  AC_HELP_STRING([--enable-memsys3],[Enable MEMSYS3]))
 AC_MSG_CHECKING([whether to support MEMSYS3])
 if test "${enable_memsys3}" = "yes" -a "${enable_memsys5}" = "no"; then
  OPT_FEATURE_FLAGS="${OPT_FEATURE_FLAGS} -DSQLITE_ENABLE_MEMSYS3"
@@ -707,7 +648,7 @@ fi

 #########
 # See whether we should enable Full Text Search extensions
-AC_ARG_ENABLE(fts3, AS_HELP_STRING([--enable-fts3],
+AC_ARG_ENABLE(fts3, AC_HELP_STRING([--enable-fts3],
      [Enable the FTS3 extension]))
 AC_MSG_CHECKING([whether to support FTS3])
 if test "${enable_fts3}" = "yes" ; then
@@ -716,7 +657,7 @@ if test "${enable_fts3}" = "yes" ; then
 else
  AC_MSG_RESULT([no])
 fi
-AC_ARG_ENABLE(fts4, AS_HELP_STRING([--enable-fts4],
+AC_ARG_ENABLE(fts4, AC_HELP_STRING([--enable-fts4],
      [Enable the FTS4 extension]))
 AC_MSG_CHECKING([whether to support FTS4])
 if test "${enable_fts4}" = "yes" -o "${enable_all}" = "yes" ; then
@@ -726,7 +667,7 @@ if test "${enable_fts4}" = "yes" -o "${enable_all}" = "yes" ; then
 else
  AC_MSG_RESULT([no])
 fi
-AC_ARG_ENABLE(fts5, AS_HELP_STRING([--enable-fts5],
+AC_ARG_ENABLE(fts5, AC_HELP_STRING([--enable-fts5],
      [Enable the FTS5 extension]))
 AC_MSG_CHECKING([whether to support FTS5])
 if test "${enable_fts5}" = "yes" -o "${enable_all}" = "yes" ; then
@@ -740,7 +681,7 @@ fi
 #########
 # See whether we should enable the LIMIT clause on UPDATE and DELETE
 # statements.
-AC_ARG_ENABLE(update-limit, AS_HELP_STRING([--enable-update-limit],
+AC_ARG_ENABLE(update-limit, AC_HELP_STRING([--enable-update-limit],
      [Enable the UPDATE/DELETE LIMIT clause]))
 AC_MSG_CHECKING([whether to support LIMIT on UPDATE and DELETE statements])
 if test "${enable_update_limit}" = "yes" ; then
@@ -752,7 +693,7 @@ fi

 #########
 # See whether we should enable GEOPOLY
-AC_ARG_ENABLE(geopoly, AS_HELP_STRING([--enable-geopoly],
+AC_ARG_ENABLE(geopoly, AC_HELP_STRING([--enable-geopoly],
      [Enable the GEOPOLY extension]),
      [enable_geopoly=yes],[enable_geopoly=no])
 AC_MSG_CHECKING([whether to support GEOPOLY])
@@ -766,7 +707,7 @@ fi

 #########
 # See whether we should enable RTREE
-AC_ARG_ENABLE(rtree, AS_HELP_STRING([--enable-rtree],
+AC_ARG_ENABLE(rtree, AC_HELP_STRING([--enable-rtree],
      [Enable the RTREE extension]))
 AC_MSG_CHECKING([whether to support RTREE])
 if test "${enable_rtree}" = "yes" ; then
@@ -778,7 +719,7 @@ fi

 #########
 # See whether we should enable the SESSION extension
-AC_ARG_ENABLE(session, AS_HELP_STRING([--enable-session],
+AC_ARG_ENABLE(session, AC_HELP_STRING([--enable-session],
      [Enable the SESSION extension]))
 AC_MSG_CHECKING([whether to support SESSION])
 if test "${enable_session}" = "yes" -o "${enable_all}" = "yes" ; then
@@ -842,7 +783,7 @@ BUILD_CFLAGS=$ac_temp_BUILD_CFLAGS

 #########
 # See whether we should use GCOV
-AC_ARG_ENABLE(gcov, AS_HELP_STRING([--enable-gcov],
+AC_ARG_ENABLE(gcov, AC_HELP_STRING([--enable-gcov],
      [Enable coverage testing using gcov]))
 if test "${use_gcov}" = "yes" ; then
  USE_GCOV=1
@@ -865,14 +806,13 @@ AC_SUBST(AMALGAMATION_LINE_MACROS)

 #########
 # Output the config header
-AC_CONFIG_HEADERS(sqlite_cfg.h)
+AC_CONFIG_HEADERS(config.h)

 #########
 # Generate the output files.
 #
 AC_SUBST(BUILD_CFLAGS)
-AC_CONFIG_FILES([
+AC_OUTPUT([
 Makefile
 sqlite3.pc
 ])
-AC_OUTPUT
@@ -0,0 +1,2 @@
+This folder contains source code to the first full-text search
+extension for SQLite.
@@ -0,0 +1,404 @@
+/*
+** 2001 September 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This is the implementation of generic hash-tables used in SQLite.
+** We've modified it slightly to serve as a standalone hash table
+** implementation for the full-text indexing module.
+*/
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "ft_hash.h"
+
+void *malloc_and_zero(int n){
+  void *p = malloc(n);
+  if( p ){
+    memset(p, 0, n);
+  }
+  return p;
+}
+
+/* Turn bulk memory into a hash table object by initializing the
+** fields of the Hash structure.
+**
+** "pNew" is a pointer to the hash table that is to be initialized.
+** keyClass is one of the constants HASH_INT, HASH_POINTER,
+** HASH_BINARY, or HASH_STRING.  The value of keyClass 
+** determines what kind of key the hash table will use.  "copyKey" is
+** true if the hash table should make its own private copy of keys and
+** false if it should just use the supplied pointer.  CopyKey only makes
+** sense for HASH_STRING and HASH_BINARY and is ignored
+** for other key classes.
+*/
+void HashInit(Hash *pNew, int keyClass, int copyKey){
+  assert( pNew!=0 );
+  assert( keyClass>=HASH_STRING && keyClass<=HASH_BINARY );
+  pNew->keyClass = keyClass;
+#if 0
+  if( keyClass==HASH_POINTER || keyClass==HASH_INT ) copyKey = 0;
+#endif
+  pNew->copyKey = copyKey;
+  pNew->first = 0;
+  pNew->count = 0;
+  pNew->htsize = 0;
+  pNew->ht = 0;
+  pNew->xMalloc = malloc_and_zero;
+  pNew->xFree = free;
+}
+
+/* Remove all entries from a hash table.  Reclaim all memory.
+** Call this routine to delete a hash table or to reset a hash table
+** to the empty state.
+*/
+void HashClear(Hash *pH){
+  HashElem *elem;         /* For looping over all elements of the table */
+
+  assert( pH!=0 );
+  elem = pH->first;
+  pH->first = 0;
+  if( pH->ht ) pH->xFree(pH->ht);
+  pH->ht = 0;
+  pH->htsize = 0;
+  while( elem ){
+    HashElem *next_elem = elem->next;
+    if( pH->copyKey && elem->pKey ){
+      pH->xFree(elem->pKey);
+    }
+    pH->xFree(elem);
+    elem = next_elem;
+  }
+  pH->count = 0;
+}
+
+#if 0 /* NOT USED */
+/*
+** Hash and comparison functions when the mode is HASH_INT
+*/
+static int intHash(const void *pKey, int nKey){
+  return nKey ^ (nKey<<8) ^ (nKey>>8);
+}
+static int intCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  return n2 - n1;
+}
+#endif
+
+#if 0 /* NOT USED */
+/*
+** Hash and comparison functions when the mode is HASH_POINTER
+*/
+static int ptrHash(const void *pKey, int nKey){
+  uptr x = Addr(pKey);
+  return x ^ (x<<8) ^ (x>>8);
+}
+static int ptrCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  if( pKey1==pKey2 ) return 0;
+  if( pKey1<pKey2 ) return -1;
+  return 1;
+}
+#endif
+
+/*
+** Hash and comparison functions when the mode is HASH_STRING
+*/
+static int strHash(const void *pKey, int nKey){
+  const char *z = (const char *)pKey;
+  int h = 0;
+  if( nKey<=0 ) nKey = (int) strlen(z);
+  while( nKey > 0  ){
+    h = (h<<3) ^ h ^ *z++;
+    nKey--;
+  }
+  return h & 0x7fffffff;
+}
+static int strCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  if( n1!=n2 ) return 1;
+  return strncmp((const char*)pKey1,(const char*)pKey2,n1);
+}
+
+/*
+** Hash and comparison functions when the mode is HASH_BINARY
+*/
+static int binHash(const void *pKey, int nKey){
+  int h = 0;
+  const char *z = (const char *)pKey;
+  while( nKey-- > 0 ){
+    h = (h<<3) ^ h ^ *(z++);
+  }
+  return h & 0x7fffffff;
+}
+static int binCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  if( n1!=n2 ) return 1;
+  return memcmp(pKey1,pKey2,n1);
+}
+
+/*
+** Return a pointer to the appropriate hash function given the key class.
+**
+** The C syntax in this function definition may be unfamilar to some 
+** programmers, so we provide the following additional explanation:
+**
+** The name of the function is "hashFunction".  The function takes a
+** single parameter "keyClass".  The return value of hashFunction()
+** is a pointer to another function.  Specifically, the return value
+** of hashFunction() is a pointer to a function that takes two parameters
+** with types "const void*" and "int" and returns an "int".
+*/
+static int (*hashFunction(int keyClass))(const void*,int){
+#if 0  /* HASH_INT and HASH_POINTER are never used */
+  switch( keyClass ){
+    case HASH_INT:     return &intHash;
+    case HASH_POINTER: return &ptrHash;
+    case HASH_STRING:  return &strHash;
+    case HASH_BINARY:  return &binHash;;
+    default: break;
+  }
+  return 0;
+#else
+  if( keyClass==HASH_STRING ){
+    return &strHash;
+  }else{
+    assert( keyClass==HASH_BINARY );
+    return &binHash;
+  }
+#endif
+}
+
+/*
+** Return a pointer to the appropriate hash function given the key class.
+**
+** For help in interpreted the obscure C code in the function definition,
+** see the header comment on the previous function.
+*/
+static int (*compareFunction(int keyClass))(const void*,int,const void*,int){
+#if 0 /* HASH_INT and HASH_POINTER are never used */
+  switch( keyClass ){
+    case HASH_INT:     return &intCompare;
+    case HASH_POINTER: return &ptrCompare;
+    case HASH_STRING:  return &strCompare;
+    case HASH_BINARY:  return &binCompare;
+    default: break;
+  }
+  return 0;
+#else
+  if( keyClass==HASH_STRING ){
+    return &strCompare;
+  }else{
+    assert( keyClass==HASH_BINARY );
+    return &binCompare;
+  }
+#endif
+}
+
+/* Link an element into the hash table
+*/
+static void insertElement(
+  Hash *pH,              /* The complete hash table */
+  struct _ht *pEntry,    /* The entry into which pNew is inserted */
+  HashElem *pNew         /* The element to be inserted */
+){
+  HashElem *pHead;       /* First element already in pEntry */
+  pHead = pEntry->chain;
+  if( pHead ){
+    pNew->next = pHead;
+    pNew->prev = pHead->prev;
+    if( pHead->prev ){ pHead->prev->next = pNew; }
+    else             { pH->first = pNew; }
+    pHead->prev = pNew;
+  }else{
+    pNew->next = pH->first;
+    if( pH->first ){ pH->first->prev = pNew; }
+    pNew->prev = 0;
+    pH->first = pNew;
+  }
+  pEntry->count++;
+  pEntry->chain = pNew;
+}
+
+
+/* Resize the hash table so that it cantains "new_size" buckets.
+** "new_size" must be a power of 2.  The hash table might fail 
+** to resize if sqliteMalloc() fails.
+*/
+static void rehash(Hash *pH, int new_size){
+  struct _ht *new_ht;            /* The new hash table */
+  HashElem *elem, *next_elem;    /* For looping over existing elements */
+  int (*xHash)(const void*,int); /* The hash function */
+
+  assert( (new_size & (new_size-1))==0 );
+  new_ht = (struct _ht *)pH->xMalloc( new_size*sizeof(struct _ht) );
+  if( new_ht==0 ) return;
+  if( pH->ht ) pH->xFree(pH->ht);
+  pH->ht = new_ht;
+  pH->htsize = new_size;
+  xHash = hashFunction(pH->keyClass);
+  for(elem=pH->first, pH->first=0; elem; elem = next_elem){
+    int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1);
+    next_elem = elem->next;
+    insertElement(pH, &new_ht[h], elem);
+  }
+}
+
+/* This function (for internal use only) locates an element in an
+** hash table that matches the given key.  The hash for this key has
+** already been computed and is passed as the 4th parameter.
+*/
+static HashElem *findElementGivenHash(
+  const Hash *pH,     /* The pH to be searched */
+  const void *pKey,   /* The key we are searching for */
+  int nKey,
+  int h               /* The hash for this key. */
+){
+  HashElem *elem;                /* Used to loop thru the element list */
+  int count;                     /* Number of elements left to test */
+  int (*xCompare)(const void*,int,const void*,int);  /* comparison function */
+
+  if( pH->ht ){
+    struct _ht *pEntry = &pH->ht[h];
+    elem = pEntry->chain;
+    count = pEntry->count;
+    xCompare = compareFunction(pH->keyClass);
+    while( count-- && elem ){
+      if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ 
+        return elem;
+      }
+      elem = elem->next;
+    }
+  }
+  return 0;
+}
+
+/* Remove a single entry from the hash table given a pointer to that
+** element and a hash on the element's key.
+*/
+static void removeElementGivenHash(
+  Hash *pH,         /* The pH containing "elem" */
+  HashElem* elem,   /* The element to be removed from the pH */
+  int h             /* Hash value for the element */
+){
+  struct _ht *pEntry;
+  if( elem->prev ){
+    elem->prev->next = elem->next; 
+  }else{
+    pH->first = elem->next;
+  }
+  if( elem->next ){
+    elem->next->prev = elem->prev;
+  }
+  pEntry = &pH->ht[h];
+  if( pEntry->chain==elem ){
+    pEntry->chain = elem->next;
+  }
+  pEntry->count--;
+  if( pEntry->count<=0 ){
+    pEntry->chain = 0;
+  }
+  if( pH->copyKey && elem->pKey ){
+    pH->xFree(elem->pKey);
+  }
+  pH->xFree( elem );
+  pH->count--;
+  if( pH->count<=0 ){
+    assert( pH->first==0 );
+    assert( pH->count==0 );
+    HashClear(pH);
+  }
+}
+
+/* Attempt to locate an element of the hash table pH with a key
+** that matches pKey,nKey.  Return the data for this element if it is
+** found, or NULL if there is no match.
+*/
+void *HashFind(const Hash *pH, const void *pKey, int nKey){
+  int h;             /* A hash on key */
+  HashElem *elem;    /* The element that matches key */
+  int (*xHash)(const void*,int);  /* The hash function */
+
+  if( pH==0 || pH->ht==0 ) return 0;
+  xHash = hashFunction(pH->keyClass);
+  assert( xHash!=0 );
+  h = (*xHash)(pKey,nKey);
+  assert( (pH->htsize & (pH->htsize-1))==0 );
+  elem = findElementGivenHash(pH,pKey,nKey, h & (pH->htsize-1));
+  return elem ? elem->data : 0;
+}
+
+/* Insert an element into the hash table pH.  The key is pKey,nKey
+** and the data is "data".
+**
+** If no element exists with a matching key, then a new
+** element is created.  A copy of the key is made if the copyKey
+** flag is set.  NULL is returned.
+**
+** If another element already exists with the same key, then the
+** new data replaces the old data and the old data is returned.
+** The key is not copied in this instance.  If a malloc fails, then
+** the new data is returned and the hash table is unchanged.
+**
+** If the "data" parameter to this function is NULL, then the
+** element corresponding to "key" is removed from the hash table.
+*/
+void *HashInsert(Hash *pH, const void *pKey, int nKey, void *data){
+  int hraw;             /* Raw hash value of the key */
+  int h;                /* the hash of the key modulo hash table size */
+  HashElem *elem;       /* Used to loop thru the element list */
+  HashElem *new_elem;   /* New element added to the pH */
+  int (*xHash)(const void*,int);  /* The hash function */
+
+  assert( pH!=0 );
+  xHash = hashFunction(pH->keyClass);
+  assert( xHash!=0 );
+  hraw = (*xHash)(pKey, nKey);
+  assert( (pH->htsize & (pH->htsize-1))==0 );
+  h = hraw & (pH->htsize-1);
+  elem = findElementGivenHash(pH,pKey,nKey,h);
+  if( elem ){
+    void *old_data = elem->data;
+    if( data==0 ){
+      removeElementGivenHash(pH,elem,h);
+    }else{
+      elem->data = data;
+    }
+    return old_data;
+  }
+  if( data==0 ) return 0;
+  new_elem = (HashElem*)pH->xMalloc( sizeof(HashElem) );
+  if( new_elem==0 ) return data;
+  if( pH->copyKey && pKey!=0 ){
+    new_elem->pKey = pH->xMalloc( nKey );
+    if( new_elem->pKey==0 ){
+      pH->xFree(new_elem);
+      return data;
+    }
+    memcpy((void*)new_elem->pKey, pKey, nKey);
+  }else{
+    new_elem->pKey = (void*)pKey;
+  }
+  new_elem->nKey = nKey;
+  pH->count++;
+  if( pH->htsize==0 ){
+    rehash(pH,8);
+    if( pH->htsize==0 ){
+      pH->count = 0;
+      pH->xFree(new_elem);
+      return data;
+    }
+  }
+  if( pH->count > pH->htsize ){
+    rehash(pH,pH->htsize*2);
+  }
+  assert( pH->htsize>0 );
+  assert( (pH->htsize & (pH->htsize-1))==0 );
+  h = hraw & (pH->htsize-1);
+  insertElement(pH, &pH->ht[h], new_elem);
+  new_elem->data = data;
+  return 0;
+}
@@ -0,0 +1,111 @@
+/*
+** 2001 September 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This is the header file for the generic hash-table implementation
+** used in SQLite.  We've modified it slightly to serve as a standalone
+** hash table implementation for the full-text indexing module.
+**
+*/
+#ifndef _HASH_H_
+#define _HASH_H_
+
+/* Forward declarations of structures. */
+typedef struct Hash Hash;
+typedef struct HashElem HashElem;
+
+/* A complete hash table is an instance of the following structure.
+** The internals of this structure are intended to be opaque -- client
+** code should not attempt to access or modify the fields of this structure
+** directly.  Change this structure only by using the routines below.
+** However, many of the "procedures" and "functions" for modifying and
+** accessing this structure are really macros, so we can't really make
+** this structure opaque.
+*/
+struct Hash {
+  char keyClass;          /* HASH_INT, _POINTER, _STRING, _BINARY */
+  char copyKey;           /* True if copy of key made on insert */
+  int count;              /* Number of entries in this table */
+  HashElem *first;        /* The first element of the array */
+  void *(*xMalloc)(int);  /* malloc() function to use */
+  void (*xFree)(void *);  /* free() function to use */
+  int htsize;             /* Number of buckets in the hash table */
+  struct _ht {            /* the hash table */
+    int count;               /* Number of entries with this hash */
+    HashElem *chain;         /* Pointer to first entry with this hash */
+  } *ht;
+};
+
+/* Each element in the hash table is an instance of the following 
+** structure.  All elements are stored on a single doubly-linked list.
+**
+** Again, this structure is intended to be opaque, but it can't really
+** be opaque because it is used by macros.
+*/
+struct HashElem {
+  HashElem *next, *prev;   /* Next and previous elements in the table */
+  void *data;              /* Data associated with this element */
+  void *pKey; int nKey;    /* Key associated with this element */
+};
+
+/*
+** There are 4 different modes of operation for a hash table:
+**
+**   HASH_INT         nKey is used as the key and pKey is ignored.
+**
+**   HASH_POINTER     pKey is used as the key and nKey is ignored.
+**
+**   HASH_STRING      pKey points to a string that is nKey bytes long
+**                           (including the null-terminator, if any).  Case
+**                           is respected in comparisons.
+**
+**   HASH_BINARY      pKey points to binary data nKey bytes long. 
+**                           memcmp() is used to compare keys.
+**
+** A copy of the key is made for HASH_STRING and HASH_BINARY
+** if the copyKey parameter to HashInit is 1.  
+*/
+/* #define HASH_INT       1 // NOT USED */
+/* #define HASH_POINTER   2 // NOT USED */
+#define HASH_STRING    3
+#define HASH_BINARY    4
+
+/*
+** Access routines.  To delete, insert a NULL pointer.
+*/
+void HashInit(Hash*, int keytype, int copyKey);
+void *HashInsert(Hash*, const void *pKey, int nKey, void *pData);
+void *HashFind(const Hash*, const void *pKey, int nKey);
+void HashClear(Hash*);
+
+/*
+** Macros for looping over all elements of a hash table.  The idiom is
+** like this:
+**
+**   Hash h;
+**   HashElem *p;
+**   ...
+**   for(p=HashFirst(&h); p; p=HashNext(p)){
+**     SomeStructure *pData = HashData(p);
+**     // do something with pData
+**   }
+*/
+#define HashFirst(H)  ((H)->first)
+#define HashNext(E)   ((E)->next)
+#define HashData(E)   ((E)->data)
+#define HashKey(E)    ((E)->pKey)
+#define HashKeysize(E) ((E)->nKey)
+
+/*
+** Number of entries in a hash table
+*/
+#define HashCount(H)  ((H)->count)
+
+#endif /* _HASH_H_ */
@@ -0,0 +1,11 @@
+#include "sqlite3.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif  /* __cplusplus */
+
+int sqlite3Fts1Init(sqlite3 *db);
+
+#ifdef __cplusplus
+}  /* extern "C" */
+#endif  /* __cplusplus */
@@ -0,0 +1,369 @@
+/*
+** 2001 September 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This is the implementation of generic hash-tables used in SQLite.
+** We've modified it slightly to serve as a standalone hash table
+** implementation for the full-text indexing module.
+*/
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+
+/*
+** The code in this file is only compiled if:
+**
+**     * The FTS1 module is being built as an extension
+**       (in which case SQLITE_CORE is not defined), or
+**
+**     * The FTS1 module is being built into the core of
+**       SQLite (in which case SQLITE_ENABLE_FTS1 is defined).
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)
+
+
+#include "fts1_hash.h"
+
+static void *malloc_and_zero(int n){
+  void *p = malloc(n);
+  if( p ){
+    memset(p, 0, n);
+  }
+  return p;
+}
+
+/* Turn bulk memory into a hash table object by initializing the
+** fields of the Hash structure.
+**
+** "pNew" is a pointer to the hash table that is to be initialized.
+** keyClass is one of the constants 
+** FTS1_HASH_BINARY or FTS1_HASH_STRING.  The value of keyClass 
+** determines what kind of key the hash table will use.  "copyKey" is
+** true if the hash table should make its own private copy of keys and
+** false if it should just use the supplied pointer.
+*/
+void sqlite3Fts1HashInit(fts1Hash *pNew, int keyClass, int copyKey){
+  assert( pNew!=0 );
+  assert( keyClass>=FTS1_HASH_STRING && keyClass<=FTS1_HASH_BINARY );
+  pNew->keyClass = keyClass;
+  pNew->copyKey = copyKey;
+  pNew->first = 0;
+  pNew->count = 0;
+  pNew->htsize = 0;
+  pNew->ht = 0;
+  pNew->xMalloc = malloc_and_zero;
+  pNew->xFree = free;
+}
+
+/* Remove all entries from a hash table.  Reclaim all memory.
+** Call this routine to delete a hash table or to reset a hash table
+** to the empty state.
+*/
+void sqlite3Fts1HashClear(fts1Hash *pH){
+  fts1HashElem *elem;         /* For looping over all elements of the table */
+
+  assert( pH!=0 );
+  elem = pH->first;
+  pH->first = 0;
+  if( pH->ht ) pH->xFree(pH->ht);
+  pH->ht = 0;
+  pH->htsize = 0;
+  while( elem ){
+    fts1HashElem *next_elem = elem->next;
+    if( pH->copyKey && elem->pKey ){
+      pH->xFree(elem->pKey);
+    }
+    pH->xFree(elem);
+    elem = next_elem;
+  }
+  pH->count = 0;
+}
+
+/*
+** Hash and comparison functions when the mode is FTS1_HASH_STRING
+*/
+static int strHash(const void *pKey, int nKey){
+  const char *z = (const char *)pKey;
+  int h = 0;
+  if( nKey<=0 ) nKey = (int) strlen(z);
+  while( nKey > 0  ){
+    h = (h<<3) ^ h ^ *z++;
+    nKey--;
+  }
+  return h & 0x7fffffff;
+}
+static int strCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  if( n1!=n2 ) return 1;
+  return strncmp((const char*)pKey1,(const char*)pKey2,n1);
+}
+
+/*
+** Hash and comparison functions when the mode is FTS1_HASH_BINARY
+*/
+static int binHash(const void *pKey, int nKey){
+  int h = 0;
+  const char *z = (const char *)pKey;
+  while( nKey-- > 0 ){
+    h = (h<<3) ^ h ^ *(z++);
+  }
+  return h & 0x7fffffff;
+}
+static int binCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  if( n1!=n2 ) return 1;
+  return memcmp(pKey1,pKey2,n1);
+}
+
+/*
+** Return a pointer to the appropriate hash function given the key class.
+**
+** The C syntax in this function definition may be unfamilar to some 
+** programmers, so we provide the following additional explanation:
+**
+** The name of the function is "hashFunction".  The function takes a
+** single parameter "keyClass".  The return value of hashFunction()
+** is a pointer to another function.  Specifically, the return value
+** of hashFunction() is a pointer to a function that takes two parameters
+** with types "const void*" and "int" and returns an "int".
+*/
+static int (*hashFunction(int keyClass))(const void*,int){
+  if( keyClass==FTS1_HASH_STRING ){
+    return &strHash;
+  }else{
+    assert( keyClass==FTS1_HASH_BINARY );
+    return &binHash;
+  }
+}
+
+/*
+** Return a pointer to the appropriate hash function given the key class.
+**
+** For help in interpreted the obscure C code in the function definition,
+** see the header comment on the previous function.
+*/
+static int (*compareFunction(int keyClass))(const void*,int,const void*,int){
+  if( keyClass==FTS1_HASH_STRING ){
+    return &strCompare;
+  }else{
+    assert( keyClass==FTS1_HASH_BINARY );
+    return &binCompare;
+  }
+}
+
+/* Link an element into the hash table
+*/
+static void insertElement(
+  fts1Hash *pH,            /* The complete hash table */
+  struct _fts1ht *pEntry,  /* The entry into which pNew is inserted */
+  fts1HashElem *pNew       /* The element to be inserted */
+){
+  fts1HashElem *pHead;     /* First element already in pEntry */
+  pHead = pEntry->chain;
+  if( pHead ){
+    pNew->next = pHead;
+    pNew->prev = pHead->prev;
+    if( pHead->prev ){ pHead->prev->next = pNew; }
+    else             { pH->first = pNew; }
+    pHead->prev = pNew;
+  }else{
+    pNew->next = pH->first;
+    if( pH->first ){ pH->first->prev = pNew; }
+    pNew->prev = 0;
+    pH->first = pNew;
+  }
+  pEntry->count++;
+  pEntry->chain = pNew;
+}
+
+
+/* Resize the hash table so that it cantains "new_size" buckets.
+** "new_size" must be a power of 2.  The hash table might fail 
+** to resize if sqliteMalloc() fails.
+*/
+static void rehash(fts1Hash *pH, int new_size){
+  struct _fts1ht *new_ht;          /* The new hash table */
+  fts1HashElem *elem, *next_elem;  /* For looping over existing elements */
+  int (*xHash)(const void*,int);   /* The hash function */
+
+  assert( (new_size & (new_size-1))==0 );
+  new_ht = (struct _fts1ht *)pH->xMalloc( new_size*sizeof(struct _fts1ht) );
+  if( new_ht==0 ) return;
+  if( pH->ht ) pH->xFree(pH->ht);
+  pH->ht = new_ht;
+  pH->htsize = new_size;
+  xHash = hashFunction(pH->keyClass);
+  for(elem=pH->first, pH->first=0; elem; elem = next_elem){
+    int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1);
+    next_elem = elem->next;
+    insertElement(pH, &new_ht[h], elem);
+  }
+}
+
+/* This function (for internal use only) locates an element in an
+** hash table that matches the given key.  The hash for this key has
+** already been computed and is passed as the 4th parameter.
+*/
+static fts1HashElem *findElementGivenHash(
+  const fts1Hash *pH, /* The pH to be searched */
+  const void *pKey,   /* The key we are searching for */
+  int nKey,
+  int h               /* The hash for this key. */
+){
+  fts1HashElem *elem;            /* Used to loop thru the element list */
+  int count;                     /* Number of elements left to test */
+  int (*xCompare)(const void*,int,const void*,int);  /* comparison function */
+
+  if( pH->ht ){
+    struct _fts1ht *pEntry = &pH->ht[h];
+    elem = pEntry->chain;
+    count = pEntry->count;
+    xCompare = compareFunction(pH->keyClass);
+    while( count-- && elem ){
+      if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ 
+        return elem;
+      }
+      elem = elem->next;
+    }
+  }
+  return 0;
+}
+
+/* Remove a single entry from the hash table given a pointer to that
+** element and a hash on the element's key.
+*/
+static void removeElementGivenHash(
+  fts1Hash *pH,         /* The pH containing "elem" */
+  fts1HashElem* elem,   /* The element to be removed from the pH */
+  int h                 /* Hash value for the element */
+){
+  struct _fts1ht *pEntry;
+  if( elem->prev ){
+    elem->prev->next = elem->next; 
+  }else{
+    pH->first = elem->next;
+  }
+  if( elem->next ){
+    elem->next->prev = elem->prev;
+  }
+  pEntry = &pH->ht[h];
+  if( pEntry->chain==elem ){
+    pEntry->chain = elem->next;
+  }
+  pEntry->count--;
+  if( pEntry->count<=0 ){
+    pEntry->chain = 0;
+  }
+  if( pH->copyKey && elem->pKey ){
+    pH->xFree(elem->pKey);
+  }
+  pH->xFree( elem );
+  pH->count--;
+  if( pH->count<=0 ){
+    assert( pH->first==0 );
+    assert( pH->count==0 );
+    fts1HashClear(pH);
+  }
+}
+
+/* Attempt to locate an element of the hash table pH with a key
+** that matches pKey,nKey.  Return the data for this element if it is
+** found, or NULL if there is no match.
+*/
+void *sqlite3Fts1HashFind(const fts1Hash *pH, const void *pKey, int nKey){
+  int h;                 /* A hash on key */
+  fts1HashElem *elem;    /* The element that matches key */
+  int (*xHash)(const void*,int);  /* The hash function */
+
+  if( pH==0 || pH->ht==0 ) return 0;
+  xHash = hashFunction(pH->keyClass);
+  assert( xHash!=0 );
+  h = (*xHash)(pKey,nKey);
+  assert( (pH->htsize & (pH->htsize-1))==0 );
+  elem = findElementGivenHash(pH,pKey,nKey, h & (pH->htsize-1));
+  return elem ? elem->data : 0;
+}
+
+/* Insert an element into the hash table pH.  The key is pKey,nKey
+** and the data is "data".
+**
+** If no element exists with a matching key, then a new
+** element is created.  A copy of the key is made if the copyKey
+** flag is set.  NULL is returned.
+**
+** If another element already exists with the same key, then the
+** new data replaces the old data and the old data is returned.
+** The key is not copied in this instance.  If a malloc fails, then
+** the new data is returned and the hash table is unchanged.
+**
+** If the "data" parameter to this function is NULL, then the
+** element corresponding to "key" is removed from the hash table.
+*/
+void *sqlite3Fts1HashInsert(
+  fts1Hash *pH,        /* The hash table to insert into */
+  const void *pKey,    /* The key */
+  int nKey,            /* Number of bytes in the key */
+  void *data           /* The data */
+){
+  int hraw;                 /* Raw hash value of the key */
+  int h;                    /* the hash of the key modulo hash table size */
+  fts1HashElem *elem;       /* Used to loop thru the element list */
+  fts1HashElem *new_elem;   /* New element added to the pH */
+  int (*xHash)(const void*,int);  /* The hash function */
+
+  assert( pH!=0 );
+  xHash = hashFunction(pH->keyClass);
+  assert( xHash!=0 );
+  hraw = (*xHash)(pKey, nKey);
+  assert( (pH->htsize & (pH->htsize-1))==0 );
+  h = hraw & (pH->htsize-1);
+  elem = findElementGivenHash(pH,pKey,nKey,h);
+  if( elem ){
+    void *old_data = elem->data;
+    if( data==0 ){
+      removeElementGivenHash(pH,elem,h);
+    }else{
+      elem->data = data;
+    }
+    return old_data;
+  }
+  if( data==0 ) return 0;
+  new_elem = (fts1HashElem*)pH->xMalloc( sizeof(fts1HashElem) );
+  if( new_elem==0 ) return data;
+  if( pH->copyKey && pKey!=0 ){
+    new_elem->pKey = pH->xMalloc( nKey );
+    if( new_elem->pKey==0 ){
+      pH->xFree(new_elem);
+      return data;
+    }
+    memcpy((void*)new_elem->pKey, pKey, nKey);
+  }else{
+    new_elem->pKey = (void*)pKey;
+  }
+  new_elem->nKey = nKey;
+  pH->count++;
+  if( pH->htsize==0 ){
+    rehash(pH,8);
+    if( pH->htsize==0 ){
+      pH->count = 0;
+      pH->xFree(new_elem);
+      return data;
+    }
+  }
+  if( pH->count > pH->htsize ){
+    rehash(pH,pH->htsize*2);
+  }
+  assert( pH->htsize>0 );
+  assert( (pH->htsize & (pH->htsize-1))==0 );
+  h = hraw & (pH->htsize-1);
+  insertElement(pH, &pH->ht[h], new_elem);
+  new_elem->data = data;
+  return 0;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) */
@@ -0,0 +1,112 @@
+/*
+** 2001 September 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This is the header file for the generic hash-table implementation
+** used in SQLite.  We've modified it slightly to serve as a standalone
+** hash table implementation for the full-text indexing module.
+**
+*/
+#ifndef _FTS1_HASH_H_
+#define _FTS1_HASH_H_
+
+/* Forward declarations of structures. */
+typedef struct fts1Hash fts1Hash;
+typedef struct fts1HashElem fts1HashElem;
+
+/* A complete hash table is an instance of the following structure.
+** The internals of this structure are intended to be opaque -- client
+** code should not attempt to access or modify the fields of this structure
+** directly.  Change this structure only by using the routines below.
+** However, many of the "procedures" and "functions" for modifying and
+** accessing this structure are really macros, so we can't really make
+** this structure opaque.
+*/
+struct fts1Hash {
+  char keyClass;          /* HASH_INT, _POINTER, _STRING, _BINARY */
+  char copyKey;           /* True if copy of key made on insert */
+  int count;              /* Number of entries in this table */
+  fts1HashElem *first;    /* The first element of the array */
+  void *(*xMalloc)(int);  /* malloc() function to use */
+  void (*xFree)(void *);  /* free() function to use */
+  int htsize;             /* Number of buckets in the hash table */
+  struct _fts1ht {        /* the hash table */
+    int count;               /* Number of entries with this hash */
+    fts1HashElem *chain;     /* Pointer to first entry with this hash */
+  } *ht;
+};
+
+/* Each element in the hash table is an instance of the following 
+** structure.  All elements are stored on a single doubly-linked list.
+**
+** Again, this structure is intended to be opaque, but it can't really
+** be opaque because it is used by macros.
+*/
+struct fts1HashElem {
+  fts1HashElem *next, *prev; /* Next and previous elements in the table */
+  void *data;                /* Data associated with this element */
+  void *pKey; int nKey;      /* Key associated with this element */
+};
+
+/*
+** There are 2 different modes of operation for a hash table:
+**
+**   FTS1_HASH_STRING        pKey points to a string that is nKey bytes long
+**                           (including the null-terminator, if any).  Case
+**                           is respected in comparisons.
+**
+**   FTS1_HASH_BINARY        pKey points to binary data nKey bytes long. 
+**                           memcmp() is used to compare keys.
+**
+** A copy of the key is made if the copyKey parameter to fts1HashInit is 1.  
+*/
+#define FTS1_HASH_STRING    1
+#define FTS1_HASH_BINARY    2
+
+/*
+** Access routines.  To delete, insert a NULL pointer.
+*/
+void sqlite3Fts1HashInit(fts1Hash*, int keytype, int copyKey);
+void *sqlite3Fts1HashInsert(fts1Hash*, const void *pKey, int nKey, void *pData);
+void *sqlite3Fts1HashFind(const fts1Hash*, const void *pKey, int nKey);
+void sqlite3Fts1HashClear(fts1Hash*);
+
+/*
+** Shorthand for the functions above
+*/
+#define fts1HashInit   sqlite3Fts1HashInit
+#define fts1HashInsert sqlite3Fts1HashInsert
+#define fts1HashFind   sqlite3Fts1HashFind
+#define fts1HashClear  sqlite3Fts1HashClear
+
+/*
+** Macros for looping over all elements of a hash table.  The idiom is
+** like this:
+**
+**   fts1Hash h;
+**   fts1HashElem *p;
+**   ...
+**   for(p=fts1HashFirst(&h); p; p=fts1HashNext(p)){
+**     SomeStructure *pData = fts1HashData(p);
+**     // do something with pData
+**   }
+*/
+#define fts1HashFirst(H)  ((H)->first)
+#define fts1HashNext(E)   ((E)->next)
+#define fts1HashData(E)   ((E)->data)
+#define fts1HashKey(E)    ((E)->pKey)
+#define fts1HashKeysize(E) ((E)->nKey)
+
+/*
+** Number of entries in a hash table
+*/
+#define fts1HashCount(H)  ((H)->count)
+
+#endif /* _FTS1_HASH_H_ */
@@ -0,0 +1,643 @@
+/*
+** 2006 September 30
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** Implementation of the full-text-search tokenizer that implements
+** a Porter stemmer.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+**     * The FTS1 module is being built as an extension
+**       (in which case SQLITE_CORE is not defined), or
+**
+**     * The FTS1 module is being built into the core of
+**       SQLite (in which case SQLITE_ENABLE_FTS1 is defined).
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)
+
+
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+
+#include "fts1_tokenizer.h"
+
+/*
+** Class derived from sqlite3_tokenizer
+*/
+typedef struct porter_tokenizer {
+  sqlite3_tokenizer base;      /* Base class */
+} porter_tokenizer;
+
+/*
+** Class derived from sqlit3_tokenizer_cursor
+*/
+typedef struct porter_tokenizer_cursor {
+  sqlite3_tokenizer_cursor base;
+  const char *zInput;          /* input we are tokenizing */
+  int nInput;                  /* size of the input */
+  int iOffset;                 /* current position in zInput */
+  int iToken;                  /* index of next token to be returned */
+  char *zToken;                /* storage for current token */
+  int nAllocated;              /* space allocated to zToken buffer */
+} porter_tokenizer_cursor;
+
+
+/* Forward declaration */
+static const sqlite3_tokenizer_module porterTokenizerModule;
+
+
+/*
+** Create a new tokenizer instance.
+*/
+static int porterCreate(
+  int argc, const char * const *argv,
+  sqlite3_tokenizer **ppTokenizer
+){
+  porter_tokenizer *t;
+  t = (porter_tokenizer *) calloc(sizeof(*t), 1);
+  if( t==NULL ) return SQLITE_NOMEM;
+
+  *ppTokenizer = &t->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destroy a tokenizer
+*/
+static int porterDestroy(sqlite3_tokenizer *pTokenizer){
+  free(pTokenizer);
+  return SQLITE_OK;
+}
+
+/*
+** Prepare to begin tokenizing a particular string.  The input
+** string to be tokenized is zInput[0..nInput-1].  A cursor
+** used to incrementally tokenize this string is returned in 
+** *ppCursor.
+*/
+static int porterOpen(
+  sqlite3_tokenizer *pTokenizer,         /* The tokenizer */
+  const char *zInput, int nInput,        /* String to be tokenized */
+  sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */
+){
+  porter_tokenizer_cursor *c;
+
+  c = (porter_tokenizer_cursor *) malloc(sizeof(*c));
+  if( c==NULL ) return SQLITE_NOMEM;
+
+  c->zInput = zInput;
+  if( zInput==0 ){
+    c->nInput = 0;
+  }else if( nInput<0 ){
+    c->nInput = (int)strlen(zInput);
+  }else{
+    c->nInput = nInput;
+  }
+  c->iOffset = 0;                 /* start tokenizing at the beginning */
+  c->iToken = 0;
+  c->zToken = NULL;               /* no space allocated, yet. */
+  c->nAllocated = 0;
+
+  *ppCursor = &c->base;
+  return SQLITE_OK;
+}
+
+/*
+** Close a tokenization cursor previously opened by a call to
+** porterOpen() above.
+*/
+static int porterClose(sqlite3_tokenizer_cursor *pCursor){
+  porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
+  free(c->zToken);
+  free(c);
+  return SQLITE_OK;
+}
+/*
+** Vowel or consonant
+*/
+static const char cType[] = {
+   0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0,
+   1, 1, 1, 2, 1
+};
+
+/*
+** isConsonant() and isVowel() determine if their first character in
+** the string they point to is a consonant or a vowel, according
+** to Porter ruls.  
+**
+** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'.
+** 'Y' is a consonant unless it follows another consonant,
+** in which case it is a vowel.
+**
+** In these routine, the letters are in reverse order.  So the 'y' rule
+** is that 'y' is a consonant unless it is followed by another
+** consonent.
+*/
+static int isVowel(const char*);
+static int isConsonant(const char *z){
+  int j;
+  char x = *z;
+  if( x==0 ) return 0;
+  assert( x>='a' && x<='z' );
+  j = cType[x-'a'];
+  if( j<2 ) return j;
+  return z[1]==0 || isVowel(z + 1);
+}
+static int isVowel(const char *z){
+  int j;
+  char x = *z;
+  if( x==0 ) return 0;
+  assert( x>='a' && x<='z' );
+  j = cType[x-'a'];
+  if( j<2 ) return 1-j;
+  return isConsonant(z + 1);
+}
+
+/*
+** Let any sequence of one or more vowels be represented by V and let
+** C be sequence of one or more consonants.  Then every word can be
+** represented as:
+**
+**           [C] (VC){m} [V]
+**
+** In prose:  A word is an optional consonant followed by zero or
+** vowel-consonant pairs followed by an optional vowel.  "m" is the
+** number of vowel consonant pairs.  This routine computes the value
+** of m for the first i bytes of a word.
+**
+** Return true if the m-value for z is 1 or more.  In other words,
+** return true if z contains at least one vowel that is followed
+** by a consonant.
+**
+** In this routine z[] is in reverse order.  So we are really looking
+** for an instance of of a consonant followed by a vowel.
+*/
+static int m_gt_0(const char *z){
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isConsonant(z) ){ z++; }
+  return *z!=0;
+}
+
+/* Like mgt0 above except we are looking for a value of m which is
+** exactly 1
+*/
+static int m_eq_1(const char *z){
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isConsonant(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 1;
+  while( isConsonant(z) ){ z++; }
+  return *z==0;
+}
+
+/* Like mgt0 above except we are looking for a value of m>1 instead
+** or m>0
+*/
+static int m_gt_1(const char *z){
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isConsonant(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isConsonant(z) ){ z++; }
+  return *z!=0;
+}
+
+/*
+** Return TRUE if there is a vowel anywhere within z[0..n-1]
+*/
+static int hasVowel(const char *z){
+  while( isConsonant(z) ){ z++; }
+  return *z!=0;
+}
+
+/*
+** Return TRUE if the word ends in a double consonant.
+**
+** The text is reversed here. So we are really looking at
+** the first two characters of z[].
+*/
+static int doubleConsonant(const char *z){
+  return isConsonant(z) && z[0]==z[1] && isConsonant(z+1);
+}
+
+/*
+** Return TRUE if the word ends with three letters which
+** are consonant-vowel-consonent and where the final consonant
+** is not 'w', 'x', or 'y'.
+**
+** The word is reversed here.  So we are really checking the
+** first three letters and the first one cannot be in [wxy].
+*/
+static int star_oh(const char *z){
+  return
+    z[0]!=0 && isConsonant(z) &&
+    z[0]!='w' && z[0]!='x' && z[0]!='y' &&
+    z[1]!=0 && isVowel(z+1) &&
+    z[2]!=0 && isConsonant(z+2);
+}
+
+/*
+** If the word ends with zFrom and xCond() is true for the stem
+** of the word that preceeds the zFrom ending, then change the 
+** ending to zTo.
+**
+** The input word *pz and zFrom are both in reverse order.  zTo
+** is in normal order. 
+**
+** Return TRUE if zFrom matches.  Return FALSE if zFrom does not
+** match.  Not that TRUE is returned even if xCond() fails and
+** no substitution occurs.
+*/
+static int stem(
+  char **pz,             /* The word being stemmed (Reversed) */
+  const char *zFrom,     /* If the ending matches this... (Reversed) */
+  const char *zTo,       /* ... change the ending to this (not reversed) */
+  int (*xCond)(const char*)   /* Condition that must be true */
+){
+  char *z = *pz;
+  while( *zFrom && *zFrom==*z ){ z++; zFrom++; }
+  if( *zFrom!=0 ) return 0;
+  if( xCond && !xCond(z) ) return 1;
+  while( *zTo ){
+    *(--z) = *(zTo++);
+  }
+  *pz = z;
+  return 1;
+}
+
+/*
+** This is the fallback stemmer used when the porter stemmer is
+** inappropriate.  The input word is copied into the output with
+** US-ASCII case folding.  If the input word is too long (more
+** than 20 bytes if it contains no digits or more than 6 bytes if
+** it contains digits) then word is truncated to 20 or 6 bytes
+** by taking 10 or 3 bytes from the beginning and end.
+*/
+static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
+  int i, mx, j;
+  int hasDigit = 0;
+  for(i=0; i<nIn; i++){
+    int c = zIn[i];
+    if( c>='A' && c<='Z' ){
+      zOut[i] = c - 'A' + 'a';
+    }else{
+      if( c>='0' && c<='9' ) hasDigit = 1;
+      zOut[i] = c;
+    }
+  }
+  mx = hasDigit ? 3 : 10;
+  if( nIn>mx*2 ){
+    for(j=mx, i=nIn-mx; i<nIn; i++, j++){
+      zOut[j] = zOut[i];
+    }
+    i = j;
+  }
+  zOut[i] = 0;
+  *pnOut = i;
+}
+
+
+/*
+** Stem the input word zIn[0..nIn-1].  Store the output in zOut.
+** zOut is at least big enough to hold nIn bytes.  Write the actual
+** size of the output word (exclusive of the '\0' terminator) into *pnOut.
+**
+** Any upper-case characters in the US-ASCII character set ([A-Z])
+** are converted to lower case.  Upper-case UTF characters are
+** unchanged.
+**
+** Words that are longer than about 20 bytes are stemmed by retaining
+** a few bytes from the beginning and the end of the word.  If the
+** word contains digits, 3 bytes are taken from the beginning and
+** 3 bytes from the end.  For long words without digits, 10 bytes
+** are taken from each end.  US-ASCII case folding still applies.
+** 
+** If the input word contains not digits but does characters not 
+** in [a-zA-Z] then no stemming is attempted and this routine just 
+** copies the input into the input into the output with US-ASCII
+** case folding.
+**
+** Stemming never increases the length of the word.  So there is
+** no chance of overflowing the zOut buffer.
+*/
+static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
+  int i, j, c;
+  char zReverse[28];
+  char *z, *z2;
+  if( nIn<3 || nIn>=sizeof(zReverse)-7 ){
+    /* The word is too big or too small for the porter stemmer.
+    ** Fallback to the copy stemmer */
+    copy_stemmer(zIn, nIn, zOut, pnOut);
+    return;
+  }
+  for(i=0, j=sizeof(zReverse)-6; i<nIn; i++, j--){
+    c = zIn[i];
+    if( c>='A' && c<='Z' ){
+      zReverse[j] = c + 'a' - 'A';
+    }else if( c>='a' && c<='z' ){
+      zReverse[j] = c;
+    }else{
+      /* The use of a character not in [a-zA-Z] means that we fallback
+      ** to the copy stemmer */
+      copy_stemmer(zIn, nIn, zOut, pnOut);
+      return;
+    }
+  }
+  memset(&zReverse[sizeof(zReverse)-5], 0, 5);
+  z = &zReverse[j+1];
+
+
+  /* Step 1a */
+  if( z[0]=='s' ){
+    if(
+     !stem(&z, "sess", "ss", 0) &&
+     !stem(&z, "sei", "i", 0)  &&
+     !stem(&z, "ss", "ss", 0)
+    ){
+      z++;
+    }
+  }
+
+  /* Step 1b */  
+  z2 = z;
+  if( stem(&z, "dee", "ee", m_gt_0) ){
+    /* Do nothing.  The work was all in the test */
+  }else if( 
+     (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel))
+      && z!=z2
+  ){
+     if( stem(&z, "ta", "ate", 0) ||
+         stem(&z, "lb", "ble", 0) ||
+         stem(&z, "zi", "ize", 0) ){
+       /* Do nothing.  The work was all in the test */
+     }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){
+       z++;
+     }else if( m_eq_1(z) && star_oh(z) ){
+       *(--z) = 'e';
+     }
+  }
+
+  /* Step 1c */
+  if( z[0]=='y' && hasVowel(z+1) ){
+    z[0] = 'i';
+  }
+
+  /* Step 2 */
+  switch( z[1] ){
+   case 'a':
+     stem(&z, "lanoita", "ate", m_gt_0) ||
+     stem(&z, "lanoit", "tion", m_gt_0);
+     break;
+   case 'c':
+     stem(&z, "icne", "ence", m_gt_0) ||
+     stem(&z, "icna", "ance", m_gt_0);
+     break;
+   case 'e':
+     stem(&z, "rezi", "ize", m_gt_0);
+     break;
+   case 'g':
+     stem(&z, "igol", "log", m_gt_0);
+     break;
+   case 'l':
+     stem(&z, "ilb", "ble", m_gt_0) ||
+     stem(&z, "illa", "al", m_gt_0) ||
+     stem(&z, "iltne", "ent", m_gt_0) ||
+     stem(&z, "ile", "e", m_gt_0) ||
+     stem(&z, "ilsuo", "ous", m_gt_0);
+     break;
+   case 'o':
+     stem(&z, "noitazi", "ize", m_gt_0) ||
+     stem(&z, "noita", "ate", m_gt_0) ||
+     stem(&z, "rota", "ate", m_gt_0);
+     break;
+   case 's':
+     stem(&z, "msila", "al", m_gt_0) ||
+     stem(&z, "ssenevi", "ive", m_gt_0) ||
+     stem(&z, "ssenluf", "ful", m_gt_0) ||
+     stem(&z, "ssensuo", "ous", m_gt_0);
+     break;
+   case 't':
+     stem(&z, "itila", "al", m_gt_0) ||
+     stem(&z, "itivi", "ive", m_gt_0) ||
+     stem(&z, "itilib", "ble", m_gt_0);
+     break;
+  }
+
+  /* Step 3 */
+  switch( z[0] ){
+   case 'e':
+     stem(&z, "etaci", "ic", m_gt_0) ||
+     stem(&z, "evita", "", m_gt_0)   ||
+     stem(&z, "ezila", "al", m_gt_0);
+     break;
+   case 'i':
+     stem(&z, "itici", "ic", m_gt_0);
+     break;
+   case 'l':
+     stem(&z, "laci", "ic", m_gt_0) ||
+     stem(&z, "luf", "", m_gt_0);
+     break;
+   case 's':
+     stem(&z, "ssen", "", m_gt_0);
+     break;
+  }
+
+  /* Step 4 */
+  switch( z[1] ){
+   case 'a':
+     if( z[0]=='l' && m_gt_1(z+2) ){
+       z += 2;
+     }
+     break;
+   case 'c':
+     if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e')  && m_gt_1(z+4)  ){
+       z += 4;
+     }
+     break;
+   case 'e':
+     if( z[0]=='r' && m_gt_1(z+2) ){
+       z += 2;
+     }
+     break;
+   case 'i':
+     if( z[0]=='c' && m_gt_1(z+2) ){
+       z += 2;
+     }
+     break;
+   case 'l':
+     if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){
+       z += 4;
+     }
+     break;
+   case 'n':
+     if( z[0]=='t' ){
+       if( z[2]=='a' ){
+         if( m_gt_1(z+3) ){
+           z += 3;
+         }
+       }else if( z[2]=='e' ){
+         stem(&z, "tneme", "", m_gt_1) ||
+         stem(&z, "tnem", "", m_gt_1) ||
+         stem(&z, "tne", "", m_gt_1);
+       }
+     }
+     break;
+   case 'o':
+     if( z[0]=='u' ){
+       if( m_gt_1(z+2) ){
+         z += 2;
+       }
+     }else if( z[3]=='s' || z[3]=='t' ){
+       stem(&z, "noi", "", m_gt_1);
+     }
+     break;
+   case 's':
+     if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){
+       z += 3;
+     }
+     break;
+   case 't':
+     stem(&z, "eta", "", m_gt_1) ||
+     stem(&z, "iti", "", m_gt_1);
+     break;
+   case 'u':
+     if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){
+       z += 3;
+     }
+     break;
+   case 'v':
+   case 'z':
+     if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){
+       z += 3;
+     }
+     break;
+  }
+
+  /* Step 5a */
+  if( z[0]=='e' ){
+    if( m_gt_1(z+1) ){
+      z++;
+    }else if( m_eq_1(z+1) && !star_oh(z+1) ){
+      z++;
+    }
+  }
+
+  /* Step 5b */
+  if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){
+    z++;
+  }
+
+  /* z[] is now the stemmed word in reverse order.  Flip it back
+  ** around into forward order and return.
+  */
+  *pnOut = i = strlen(z);
+  zOut[i] = 0;
+  while( *z ){
+    zOut[--i] = *(z++);
+  }
+}
+
+/*
+** Characters that can be part of a token.  We assume any character
+** whose value is greater than 0x80 (any UTF character) can be
+** part of a token.  In other words, delimiters all must have
+** values of 0x7f or lower.
+*/
+static const char isIdChar[] = {
+/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 3x */
+    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 4x */
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,  /* 5x */
+    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 6x */
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,  /* 7x */
+};
+#define idChar(C)  (((ch=C)&0x80)!=0 || (ch>0x2f && isIdChar[ch-0x30]))
+#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !isIdChar[ch-0x30]))
+
+/*
+** Extract the next token from a tokenization cursor.  The cursor must
+** have been opened by a prior call to porterOpen().
+*/
+static int porterNext(
+  sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by porterOpen */
+  const char **pzToken,               /* OUT: *pzToken is the token text */
+  int *pnBytes,                       /* OUT: Number of bytes in token */
+  int *piStartOffset,                 /* OUT: Starting offset of token */
+  int *piEndOffset,                   /* OUT: Ending offset of token */
+  int *piPosition                     /* OUT: Position integer of token */
+){
+  porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
+  const char *z = c->zInput;
+
+  while( c->iOffset<c->nInput ){
+    int iStartOffset, ch;
+
+    /* Scan past delimiter characters */
+    while( c->iOffset<c->nInput && isDelim(z[c->iOffset]) ){
+      c->iOffset++;
+    }
+
+    /* Count non-delimiter characters. */
+    iStartOffset = c->iOffset;
+    while( c->iOffset<c->nInput && !isDelim(z[c->iOffset]) ){
+      c->iOffset++;
+    }
+
+    if( c->iOffset>iStartOffset ){
+      int n = c->iOffset-iStartOffset;
+      if( n>c->nAllocated ){
+        c->nAllocated = n+20;
+        c->zToken = realloc(c->zToken, c->nAllocated);
+        if( c->zToken==NULL ) return SQLITE_NOMEM;
+      }
+      porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes);
+      *pzToken = c->zToken;
+      *piStartOffset = iStartOffset;
+      *piEndOffset = c->iOffset;
+      *piPosition = c->iToken++;
+      return SQLITE_OK;
+    }
+  }
+  return SQLITE_DONE;
+}
+
+/*
+** The set of routines that implement the porter-stemmer tokenizer
+*/
+static const sqlite3_tokenizer_module porterTokenizerModule = {
+  0,
+  porterCreate,
+  porterDestroy,
+  porterOpen,
+  porterClose,
+  porterNext,
+};
+
+/*
+** Allocate a new porter tokenizer.  Return a pointer to the new
+** tokenizer in *ppModule
+*/
+void sqlite3Fts1PorterTokenizerModule(
+  sqlite3_tokenizer_module const**ppModule
+){
+  *ppModule = &porterTokenizerModule;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) */
@@ -0,0 +1,90 @@
+/*
+** 2006 July 10
+**
+** The author disclaims copyright to this source code.
+**
+*************************************************************************
+** Defines the interface to tokenizers used by fulltext-search.  There
+** are three basic components:
+**
+** sqlite3_tokenizer_module is a singleton defining the tokenizer
+** interface functions.  This is essentially the class structure for
+** tokenizers.
+**
+** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
+** including customization information defined at creation time.
+**
+** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
+** tokens from a particular input.
+*/
+#ifndef _FTS1_TOKENIZER_H_
+#define _FTS1_TOKENIZER_H_
+
+/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
+** If tokenizers are to be allowed to call sqlite3_*() functions, then
+** we will need a way to register the API consistently.
+*/
+#include "sqlite3.h"
+
+/*
+** Structures used by the tokenizer interface.
+*/
+typedef struct sqlite3_tokenizer sqlite3_tokenizer;
+typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
+typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
+
+struct sqlite3_tokenizer_module {
+  int iVersion;                  /* currently 0 */
+
+  /*
+  ** Create and destroy a tokenizer.  argc/argv are passed down from
+  ** the fulltext virtual table creation to allow customization.
+  */
+  int (*xCreate)(int argc, const char *const*argv,
+                 sqlite3_tokenizer **ppTokenizer);
+  int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
+
+  /*
+  ** Tokenize a particular input.  Call xOpen() to prepare to
+  ** tokenize, xNext() repeatedly until it returns SQLITE_DONE, then
+  ** xClose() to free any internal state.  The pInput passed to
+  ** xOpen() must exist until the cursor is closed.  The ppToken
+  ** result from xNext() is only valid until the next call to xNext()
+  ** or until xClose() is called.
+  */
+  /* TODO(shess) current implementation requires pInput to be
+  ** nul-terminated.  This should either be fixed, or pInput/nBytes
+  ** should be converted to zInput.
+  */
+  int (*xOpen)(sqlite3_tokenizer *pTokenizer,
+               const char *pInput, int nBytes,
+               sqlite3_tokenizer_cursor **ppCursor);
+  int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
+  int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
+               const char **ppToken, int *pnBytes,
+               int *piStartOffset, int *piEndOffset, int *piPosition);
+};
+
+struct sqlite3_tokenizer {
+  const sqlite3_tokenizer_module *pModule;  /* The module for this tokenizer */
+  /* Tokenizer implementations will typically add additional fields */
+};
+
+struct sqlite3_tokenizer_cursor {
+  sqlite3_tokenizer *pTokenizer;       /* Tokenizer for this cursor. */
+  /* Tokenizer implementations will typically add additional fields */
+};
+
+/*
+** Get the module for a tokenizer which generates tokens based on a
+** set of non-token characters.  The default is to break tokens at any
+** non-alnum character, though the set of delimiters can also be
+** specified by the first argv argument to xCreate().
+*/
+/* TODO(shess) This doesn't belong here.  Need some sort of
+** registration process.
+*/
+void sqlite3Fts1SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+void sqlite3Fts1PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+
+#endif /* _FTS1_TOKENIZER_H_ */
@@ -0,0 +1,221 @@
+/*
+** The author disclaims copyright to this source code.
+**
+*************************************************************************
+** Implementation of the "simple" full-text-search tokenizer.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+**     * The FTS1 module is being built as an extension
+**       (in which case SQLITE_CORE is not defined), or
+**
+**     * The FTS1 module is being built into the core of
+**       SQLite (in which case SQLITE_ENABLE_FTS1 is defined).
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)
+
+
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+
+#include "fts1_tokenizer.h"
+
+typedef struct simple_tokenizer {
+  sqlite3_tokenizer base;
+  char delim[128];             /* flag ASCII delimiters */
+} simple_tokenizer;
+
+typedef struct simple_tokenizer_cursor {
+  sqlite3_tokenizer_cursor base;
+  const char *pInput;          /* input we are tokenizing */
+  int nBytes;                  /* size of the input */
+  int iOffset;                 /* current position in pInput */
+  int iToken;                  /* index of next token to be returned */
+  char *pToken;                /* storage for current token */
+  int nTokenAllocated;         /* space allocated to zToken buffer */
+} simple_tokenizer_cursor;
+
+
+/* Forward declaration */
+static const sqlite3_tokenizer_module simpleTokenizerModule;
+
+static int isDelim(simple_tokenizer *t, unsigned char c){
+  return c<0x80 && t->delim[c];
+}
+
+/*
+** Create a new tokenizer instance.
+*/
+static int simpleCreate(
+  int argc, const char * const *argv,
+  sqlite3_tokenizer **ppTokenizer
+){
+  simple_tokenizer *t;
+
+  t = (simple_tokenizer *) calloc(sizeof(*t), 1);
+  if( t==NULL ) return SQLITE_NOMEM;
+
+  /* TODO(shess) Delimiters need to remain the same from run to run,
+  ** else we need to reindex.  One solution would be a meta-table to
+  ** track such information in the database, then we'd only want this
+  ** information on the initial create.
+  */
+  if( argc>1 ){
+    int i, n = strlen(argv[1]);
+    for(i=0; i<n; i++){
+      unsigned char ch = argv[1][i];
+      /* We explicitly don't support UTF-8 delimiters for now. */
+      if( ch>=0x80 ){
+        free(t);
+        return SQLITE_ERROR;
+      }
+      t->delim[ch] = 1;
+    }
+  } else {
+    /* Mark non-alphanumeric ASCII characters as delimiters */
+    int i;
+    for(i=1; i<0x80; i++){
+      t->delim[i] = !isalnum(i);
+    }
+  }
+
+  *ppTokenizer = &t->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destroy a tokenizer
+*/
+static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
+  free(pTokenizer);
+  return SQLITE_OK;
+}
+
+/*
+** Prepare to begin tokenizing a particular string.  The input
+** string to be tokenized is pInput[0..nBytes-1].  A cursor
+** used to incrementally tokenize this string is returned in 
+** *ppCursor.
+*/
+static int simpleOpen(
+  sqlite3_tokenizer *pTokenizer,         /* The tokenizer */
+  const char *pInput, int nBytes,        /* String to be tokenized */
+  sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */
+){
+  simple_tokenizer_cursor *c;
+
+  c = (simple_tokenizer_cursor *) malloc(sizeof(*c));
+  if( c==NULL ) return SQLITE_NOMEM;
+
+  c->pInput = pInput;
+  if( pInput==0 ){
+    c->nBytes = 0;
+  }else if( nBytes<0 ){
+    c->nBytes = (int)strlen(pInput);
+  }else{
+    c->nBytes = nBytes;
+  }
+  c->iOffset = 0;                 /* start tokenizing at the beginning */
+  c->iToken = 0;
+  c->pToken = NULL;               /* no space allocated, yet. */
+  c->nTokenAllocated = 0;
+
+  *ppCursor = &c->base;
+  return SQLITE_OK;
+}
+
+/*
+** Close a tokenization cursor previously opened by a call to
+** simpleOpen() above.
+*/
+static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
+  simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
+  free(c->pToken);
+  free(c);
+  return SQLITE_OK;
+}
+
+/*
+** Extract the next token from a tokenization cursor.  The cursor must
+** have been opened by a prior call to simpleOpen().
+*/
+static int simpleNext(
+  sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by simpleOpen */
+  const char **ppToken,               /* OUT: *ppToken is the token text */
+  int *pnBytes,                       /* OUT: Number of bytes in token */
+  int *piStartOffset,                 /* OUT: Starting offset of token */
+  int *piEndOffset,                   /* OUT: Ending offset of token */
+  int *piPosition                     /* OUT: Position integer of token */
+){
+  simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
+  simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
+  unsigned char *p = (unsigned char *)c->pInput;
+
+  while( c->iOffset<c->nBytes ){
+    int iStartOffset;
+
+    /* Scan past delimiter characters */
+    while( c->iOffset<c->nBytes && isDelim(t, p[c->iOffset]) ){
+      c->iOffset++;
+    }
+
+    /* Count non-delimiter characters. */
+    iStartOffset = c->iOffset;
+    while( c->iOffset<c->nBytes && !isDelim(t, p[c->iOffset]) ){
+      c->iOffset++;
+    }
+
+    if( c->iOffset>iStartOffset ){
+      int i, n = c->iOffset-iStartOffset;
+      if( n>c->nTokenAllocated ){
+        c->nTokenAllocated = n+20;
+        c->pToken = realloc(c->pToken, c->nTokenAllocated);
+        if( c->pToken==NULL ) return SQLITE_NOMEM;
+      }
+      for(i=0; i<n; i++){
+        /* TODO(shess) This needs expansion to handle UTF-8
+        ** case-insensitivity.
+        */
+        unsigned char ch = p[iStartOffset+i];
+        c->pToken[i] = ch<0x80 ? tolower(ch) : ch;
+      }
+      *ppToken = c->pToken;
+      *pnBytes = n;
+      *piStartOffset = iStartOffset;
+      *piEndOffset = c->iOffset;
+      *piPosition = c->iToken++;
+
+      return SQLITE_OK;
+    }
+  }
+  return SQLITE_DONE;
+}
+
+/*
+** The set of routines that implement the simple tokenizer
+*/
+static const sqlite3_tokenizer_module simpleTokenizerModule = {
+  0,
+  simpleCreate,
+  simpleDestroy,
+  simpleOpen,
+  simpleClose,
+  simpleNext,
+};
+
+/*
+** Allocate a new simple tokenizer.  Return a pointer to the new
+** tokenizer in *ppModule
+*/
+void sqlite3Fts1SimpleTokenizerModule(
+  sqlite3_tokenizer_module const**ppModule
+){
+  *ppModule = &simpleTokenizerModule;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) */
@@ -0,0 +1,11 @@
+#include "sqlite3.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif  /* __cplusplus */
+
+int fulltext_init(sqlite3 *db);
+
+#ifdef __cplusplus
+}  /* extern "C" */
+#endif  /* __cplusplus */
@@ -0,0 +1,174 @@
+/*
+** The author disclaims copyright to this source code.
+**
+*************************************************************************
+** Implementation of the "simple" full-text-search tokenizer.
+*/
+
+#include <assert.h>
+#if !defined(__APPLE__)
+#include <malloc.h>
+#else
+#include <stdlib.h>
+#endif
+#include <stdio.h>
+#include <string.h>
+#include <ctype.h>
+
+#include "tokenizer.h"
+
+/* Duplicate a string; the caller must free() the returned string.
+ * (We don't use strdup() since it's not part of the standard C library and
+ * may not be available everywhere.) */
+/* TODO(shess) Copied from fulltext.c, consider util.c for such
+** things. */
+static char *string_dup(const char *s){
+  char *str = malloc(strlen(s) + 1);
+  strcpy(str, s);
+  return str;
+}
+
+typedef struct simple_tokenizer {
+  sqlite3_tokenizer base;
+  const char *zDelim;          /* token delimiters */
+} simple_tokenizer;
+
+typedef struct simple_tokenizer_cursor {
+  sqlite3_tokenizer_cursor base;
+  const char *pInput;          /* input we are tokenizing */
+  int nBytes;                  /* size of the input */
+  const char *pCurrent;        /* current position in pInput */
+  int iToken;                  /* index of next token to be returned */
+  char *zToken;                /* storage for current token */
+  int nTokenBytes;             /* actual size of current token */
+  int nTokenAllocated;         /* space allocated to zToken buffer */
+} simple_tokenizer_cursor;
+
+static sqlite3_tokenizer_module simpleTokenizerModule;/* forward declaration */
+
+static int simpleCreate(
+  int argc, const char **argv,
+  sqlite3_tokenizer **ppTokenizer
+){
+  simple_tokenizer *t;
+
+  t = (simple_tokenizer *) malloc(sizeof(simple_tokenizer));
+  /* TODO(shess) Delimiters need to remain the same from run to run,
+  ** else we need to reindex.  One solution would be a meta-table to
+  ** track such information in the database, then we'd only want this
+  ** information on the initial create.
+  */
+  if( argc>1 ){
+    t->zDelim = string_dup(argv[1]);
+  } else {
+    /* Build a string excluding alphanumeric ASCII characters */
+    char zDelim[0x80];               /* nul-terminated, so nul not a member */
+    int i, j;
+    for(i=1, j=0; i<0x80; i++){
+      if( !isalnum(i) ){
+        zDelim[j++] = i;
+      }
+    }
+    zDelim[j++] = '\0';
+    assert( j<=sizeof(zDelim) );
+    t->zDelim = string_dup(zDelim);
+  }
+
+  *ppTokenizer = &t->base;
+  return SQLITE_OK;
+}
+
+static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
+  simple_tokenizer *t = (simple_tokenizer *) pTokenizer;
+
+  free((void *) t->zDelim);
+  free(t);
+
+  return SQLITE_OK;
+}
+
+static int simpleOpen(
+  sqlite3_tokenizer *pTokenizer,
+  const char *pInput, int nBytes,
+  sqlite3_tokenizer_cursor **ppCursor
+){
+  simple_tokenizer_cursor *c;
+
+  c = (simple_tokenizer_cursor *) malloc(sizeof(simple_tokenizer_cursor));
+  c->pInput = pInput;
+  c->nBytes = nBytes<0 ? (int) strlen(pInput) : nBytes;
+  c->pCurrent = c->pInput;        /* start tokenizing at the beginning */
+  c->iToken = 0;
+  c->zToken = NULL;               /* no space allocated, yet. */
+  c->nTokenBytes = 0;
+  c->nTokenAllocated = 0;
+
+  *ppCursor = &c->base;
+  return SQLITE_OK;
+}
+
+static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
+  simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
+
+  if( NULL!=c->zToken ){
+    free(c->zToken);
+  }
+  free(c);
+
+  return SQLITE_OK;
+}
+
+static int simpleNext(
+  sqlite3_tokenizer_cursor *pCursor,
+  const char **ppToken, int *pnBytes,
+  int *piStartOffset, int *piEndOffset, int *piPosition
+){
+  simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
+  simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
+  int ii;
+
+  while( c->pCurrent-c->pInput<c->nBytes ){
+    int n = (int) strcspn(c->pCurrent, t->zDelim);
+    if( n>0 ){
+      if( n+1>c->nTokenAllocated ){
+        c->zToken = realloc(c->zToken, n+1);
+      }
+      for(ii=0; ii<n; ii++){
+        /* TODO(shess) This needs expansion to handle UTF-8
+        ** case-insensitivity.
+        */
+        char ch = c->pCurrent[ii];
+        c->zToken[ii] = (unsigned char)ch<0x80 ? tolower((unsigned char)ch):ch;
+      }
+      c->zToken[n] = '\0';
+      *ppToken = c->zToken;
+      *pnBytes = n;
+      *piStartOffset = (int) (c->pCurrent-c->pInput);
+      *piEndOffset = *piStartOffset+n;
+      *piPosition = c->iToken++;
+      c->pCurrent += n + 1;
+
+      return SQLITE_OK;
+    }
+    c->pCurrent += n + 1;
+    /* TODO(shess) could strspn() to skip delimiters en masse.  Needs
+    ** to happen in two places, though, which is annoying.
+    */
+  }
+  return SQLITE_DONE;
+}
+
+static sqlite3_tokenizer_module simpleTokenizerModule = {
+  0,
+  simpleCreate,
+  simpleDestroy,
+  simpleOpen,
+  simpleClose,
+  simpleNext,
+};
+
+void get_simple_tokenizer_module(
+  sqlite3_tokenizer_module **ppModule
+){
+  *ppModule = &simpleTokenizerModule;
+}
@@ -0,0 +1,89 @@
+/*
+** 2006 July 10
+**
+** The author disclaims copyright to this source code.
+**
+*************************************************************************
+** Defines the interface to tokenizers used by fulltext-search.  There
+** are three basic components:
+**
+** sqlite3_tokenizer_module is a singleton defining the tokenizer
+** interface functions.  This is essentially the class structure for
+** tokenizers.
+**
+** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
+** including customization information defined at creation time.
+**
+** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
+** tokens from a particular input.
+*/
+#ifndef _TOKENIZER_H_
+#define _TOKENIZER_H_
+
+/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
+** If tokenizers are to be allowed to call sqlite3_*() functions, then
+** we will need a way to register the API consistently.
+*/
+#include "sqlite3.h"
+
+/*
+** Structures used by the tokenizer interface.
+*/
+typedef struct sqlite3_tokenizer sqlite3_tokenizer;
+typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
+typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
+
+struct sqlite3_tokenizer_module {
+  int iVersion;                  /* currently 0 */
+
+  /*
+  ** Create and destroy a tokenizer.  argc/argv are passed down from
+  ** the fulltext virtual table creation to allow customization.
+  */
+  int (*xCreate)(int argc, const char **argv,
+                 sqlite3_tokenizer **ppTokenizer);
+  int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
+
+  /*
+  ** Tokenize a particular input.  Call xOpen() to prepare to
+  ** tokenize, xNext() repeatedly until it returns SQLITE_DONE, then
+  ** xClose() to free any internal state.  The pInput passed to
+  ** xOpen() must exist until the cursor is closed.  The ppToken
+  ** result from xNext() is only valid until the next call to xNext()
+  ** or until xClose() is called.
+  */
+  /* TODO(shess) current implementation requires pInput to be
+  ** nul-terminated.  This should either be fixed, or pInput/nBytes
+  ** should be converted to zInput.
+  */
+  int (*xOpen)(sqlite3_tokenizer *pTokenizer,
+               const char *pInput, int nBytes,
+               sqlite3_tokenizer_cursor **ppCursor);
+  int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
+  int (*xNext)(sqlite3_tokenizer_cursor *pCursor,
+               const char **ppToken, int *pnBytes,
+               int *piStartOffset, int *piEndOffset, int *piPosition);
+};
+
+struct sqlite3_tokenizer {
+  sqlite3_tokenizer_module *pModule;  /* The module for this tokenizer */
+  /* Tokenizer implementations will typically add additional fields */
+};
+
+struct sqlite3_tokenizer_cursor {
+  sqlite3_tokenizer *pTokenizer;       /* Tokenizer for this cursor. */
+  /* Tokenizer implementations will typically add additional fields */
+};
+
+/*
+** Get the module for a tokenizer which generates tokens based on a
+** set of non-token characters.  The default is to break tokens at any
+** non-alnum character, though the set of delimiters can also be
+** specified by the first argv argument to xCreate().
+*/
+/* TODO(shess) This doesn't belong here.  Need some sort of
+** registration process.
+*/
+void get_simple_tokenizer_module(sqlite3_tokenizer_module **ppModule);
+
+#endif /* _TOKENIZER_H_ */
@@ -0,0 +1,133 @@
+
+1. FTS2 Tokenizers
+
+  When creating a new full-text table, FTS2 allows the user to select
+  the text tokenizer implementation to be used when indexing text
+  by specifying a "tokenizer" clause as part of the CREATE VIRTUAL TABLE
+  statement:
+
+    CREATE VIRTUAL TABLE <table-name> USING fts2(
+      <columns ...> [, tokenizer <tokenizer-name> [<tokenizer-args>]]
+    );
+
+  The built-in tokenizers (valid values to pass as <tokenizer name>) are
+  "simple" and "porter".
+
+  <tokenizer-args> should consist of zero or more white-space separated
+  arguments to pass to the selected tokenizer implementation. The 
+  interpretation of the arguments, if any, depends on the individual 
+  tokenizer.
+
+2. Custom Tokenizers
+
+  FTS2 allows users to provide custom tokenizer implementations. The 
+  interface used to create a new tokenizer is defined and described in 
+  the fts2_tokenizer.h source file.
+
+  Registering a new FTS2 tokenizer is similar to registering a new 
+  virtual table module with SQLite. The user passes a pointer to a
+  structure containing pointers to various callback functions that
+  make up the implementation of the new tokenizer type. For tokenizers,
+  the structure (defined in fts2_tokenizer.h) is called
+  "sqlite3_tokenizer_module".
+
+  FTS2 does not expose a C-function that users call to register new
+  tokenizer types with a database handle. Instead, the pointer must
+  be encoded as an SQL blob value and passed to FTS2 through the SQL
+  engine by evaluating a special scalar function, "fts2_tokenizer()".
+  The fts2_tokenizer() function may be called with one or two arguments,
+  as follows:
+
+    SELECT fts2_tokenizer(<tokenizer-name>);
+    SELECT fts2_tokenizer(<tokenizer-name>, <sqlite3_tokenizer_module ptr>);
+  
+  Where <tokenizer-name> is a string identifying the tokenizer and
+  <sqlite3_tokenizer_module ptr> is a pointer to an sqlite3_tokenizer_module
+  structure encoded as an SQL blob. If the second argument is present,
+  it is registered as tokenizer <tokenizer-name> and a copy of it
+  returned. If only one argument is passed, a pointer to the tokenizer
+  implementation currently registered as <tokenizer-name> is returned,
+  encoded as a blob. Or, if no such tokenizer exists, an SQL exception
+  (error) is raised.
+
+  SECURITY: If the fts2 extension is used in an environment where potentially
+    malicious users may execute arbitrary SQL (i.e. gears), they should be
+    prevented from invoking the fts2_tokenizer() function, possibly using the
+    authorisation callback.
+
+  See "Sample code" below for an example of calling the fts2_tokenizer()
+  function from C code.
+
+3. ICU Library Tokenizers
+
+  If this extension is compiled with the SQLITE_ENABLE_ICU pre-processor 
+  symbol defined, then there exists a built-in tokenizer named "icu" 
+  implemented using the ICU library. The first argument passed to the
+  xCreate() method (see fts2_tokenizer.h) of this tokenizer may be
+  an ICU locale identifier. For example "tr_TR" for Turkish as used
+  in Turkey, or "en_AU" for English as used in Australia. For example:
+
+    "CREATE VIRTUAL TABLE thai_text USING fts2(text, tokenizer icu th_TH)"
+
+  The ICU tokenizer implementation is very simple. It splits the input
+  text according to the ICU rules for finding word boundaries and discards
+  any tokens that consist entirely of white-space. This may be suitable
+  for some applications in some locales, but not all. If more complex
+  processing is required, for example to implement stemming or 
+  discard punctuation, this can be done by creating a tokenizer 
+  implementation that uses the ICU tokenizer as part of its implementation.
+
+  When using the ICU tokenizer this way, it is safe to overwrite the
+  contents of the strings returned by the xNext() method (see
+  fts2_tokenizer.h).
+
+4. Sample code.
+
+  The following two code samples illustrate the way C code should invoke
+  the fts2_tokenizer() scalar function:
+
+      int registerTokenizer(
+        sqlite3 *db, 
+        char *zName, 
+        const sqlite3_tokenizer_module *p
+      ){
+        int rc;
+        sqlite3_stmt *pStmt;
+        const char zSql[] = "SELECT fts2_tokenizer(?, ?)";
+      
+        rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+        if( rc!=SQLITE_OK ){
+          return rc;
+        }
+      
+        sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+        sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
+        sqlite3_step(pStmt);
+      
+        return sqlite3_finalize(pStmt);
+      }
+      
+      int queryTokenizer(
+        sqlite3 *db, 
+        char *zName,  
+        const sqlite3_tokenizer_module **pp
+      ){
+        int rc;
+        sqlite3_stmt *pStmt;
+        const char zSql[] = "SELECT fts2_tokenizer(?)";
+      
+        *pp = 0;
+        rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+        if( rc!=SQLITE_OK ){
+          return rc;
+        }
+      
+        sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+        if( SQLITE_ROW==sqlite3_step(pStmt) ){
+          if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
+            memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
+          }
+        }
+      
+        return sqlite3_finalize(pStmt);
+      }
@@ -0,0 +1,4 @@
+This folder contains source code to the second full-text search
+extension for SQLite.  While the API is the same, this version uses a
+substantially different storage schema from fts1, so tables will need
+to be rebuilt.
@@ -0,0 +1,26 @@
+/*
+** 2006 Oct 10
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This header file is used by programs that want to link against the
+** FTS2 library.  All it does is declare the sqlite3Fts2Init() interface.
+*/
+#include "sqlite3.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif  /* __cplusplus */
+
+int sqlite3Fts2Init(sqlite3 *db);
+
+#ifdef __cplusplus
+}  /* extern "C" */
+#endif  /* __cplusplus */
@@ -0,0 +1,376 @@
+/*
+** 2001 September 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This is the implementation of generic hash-tables used in SQLite.
+** We've modified it slightly to serve as a standalone hash table
+** implementation for the full-text indexing module.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+**     * The FTS2 module is being built as an extension
+**       (in which case SQLITE_CORE is not defined), or
+**
+**     * The FTS2 module is being built into the core of
+**       SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "sqlite3.h"
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT3
+#include "fts2_hash.h"
+
+/*
+** Malloc and Free functions
+*/
+static void *fts2HashMalloc(int n){
+  void *p = sqlite3_malloc(n);
+  if( p ){
+    memset(p, 0, n);
+  }
+  return p;
+}
+static void fts2HashFree(void *p){
+  sqlite3_free(p);
+}
+
+/* Turn bulk memory into a hash table object by initializing the
+** fields of the Hash structure.
+**
+** "pNew" is a pointer to the hash table that is to be initialized.
+** keyClass is one of the constants 
+** FTS2_HASH_BINARY or FTS2_HASH_STRING.  The value of keyClass 
+** determines what kind of key the hash table will use.  "copyKey" is
+** true if the hash table should make its own private copy of keys and
+** false if it should just use the supplied pointer.
+*/
+void sqlite3Fts2HashInit(fts2Hash *pNew, int keyClass, int copyKey){
+  assert( pNew!=0 );
+  assert( keyClass>=FTS2_HASH_STRING && keyClass<=FTS2_HASH_BINARY );
+  pNew->keyClass = keyClass;
+  pNew->copyKey = copyKey;
+  pNew->first = 0;
+  pNew->count = 0;
+  pNew->htsize = 0;
+  pNew->ht = 0;
+}
+
+/* Remove all entries from a hash table.  Reclaim all memory.
+** Call this routine to delete a hash table or to reset a hash table
+** to the empty state.
+*/
+void sqlite3Fts2HashClear(fts2Hash *pH){
+  fts2HashElem *elem;         /* For looping over all elements of the table */
+
+  assert( pH!=0 );
+  elem = pH->first;
+  pH->first = 0;
+  fts2HashFree(pH->ht);
+  pH->ht = 0;
+  pH->htsize = 0;
+  while( elem ){
+    fts2HashElem *next_elem = elem->next;
+    if( pH->copyKey && elem->pKey ){
+      fts2HashFree(elem->pKey);
+    }
+    fts2HashFree(elem);
+    elem = next_elem;
+  }
+  pH->count = 0;
+}
+
+/*
+** Hash and comparison functions when the mode is FTS2_HASH_STRING
+*/
+static int strHash(const void *pKey, int nKey){
+  const char *z = (const char *)pKey;
+  int h = 0;
+  if( nKey<=0 ) nKey = (int) strlen(z);
+  while( nKey > 0  ){
+    h = (h<<3) ^ h ^ *z++;
+    nKey--;
+  }
+  return h & 0x7fffffff;
+}
+static int strCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  if( n1!=n2 ) return 1;
+  return strncmp((const char*)pKey1,(const char*)pKey2,n1);
+}
+
+/*
+** Hash and comparison functions when the mode is FTS2_HASH_BINARY
+*/
+static int binHash(const void *pKey, int nKey){
+  int h = 0;
+  const char *z = (const char *)pKey;
+  while( nKey-- > 0 ){
+    h = (h<<3) ^ h ^ *(z++);
+  }
+  return h & 0x7fffffff;
+}
+static int binCompare(const void *pKey1, int n1, const void *pKey2, int n2){
+  if( n1!=n2 ) return 1;
+  return memcmp(pKey1,pKey2,n1);
+}
+
+/*
+** Return a pointer to the appropriate hash function given the key class.
+**
+** The C syntax in this function definition may be unfamilar to some 
+** programmers, so we provide the following additional explanation:
+**
+** The name of the function is "hashFunction".  The function takes a
+** single parameter "keyClass".  The return value of hashFunction()
+** is a pointer to another function.  Specifically, the return value
+** of hashFunction() is a pointer to a function that takes two parameters
+** with types "const void*" and "int" and returns an "int".
+*/
+static int (*hashFunction(int keyClass))(const void*,int){
+  if( keyClass==FTS2_HASH_STRING ){
+    return &strHash;
+  }else{
+    assert( keyClass==FTS2_HASH_BINARY );
+    return &binHash;
+  }
+}
+
+/*
+** Return a pointer to the appropriate hash function given the key class.
+**
+** For help in interpreted the obscure C code in the function definition,
+** see the header comment on the previous function.
+*/
+static int (*compareFunction(int keyClass))(const void*,int,const void*,int){
+  if( keyClass==FTS2_HASH_STRING ){
+    return &strCompare;
+  }else{
+    assert( keyClass==FTS2_HASH_BINARY );
+    return &binCompare;
+  }
+}
+
+/* Link an element into the hash table
+*/
+static void insertElement(
+  fts2Hash *pH,            /* The complete hash table */
+  struct _fts2ht *pEntry,  /* The entry into which pNew is inserted */
+  fts2HashElem *pNew       /* The element to be inserted */
+){
+  fts2HashElem *pHead;     /* First element already in pEntry */
+  pHead = pEntry->chain;
+  if( pHead ){
+    pNew->next = pHead;
+    pNew->prev = pHead->prev;
+    if( pHead->prev ){ pHead->prev->next = pNew; }
+    else             { pH->first = pNew; }
+    pHead->prev = pNew;
+  }else{
+    pNew->next = pH->first;
+    if( pH->first ){ pH->first->prev = pNew; }
+    pNew->prev = 0;
+    pH->first = pNew;
+  }
+  pEntry->count++;
+  pEntry->chain = pNew;
+}
+
+
+/* Resize the hash table so that it cantains "new_size" buckets.
+** "new_size" must be a power of 2.  The hash table might fail 
+** to resize if sqliteMalloc() fails.
+*/
+static void rehash(fts2Hash *pH, int new_size){
+  struct _fts2ht *new_ht;          /* The new hash table */
+  fts2HashElem *elem, *next_elem;  /* For looping over existing elements */
+  int (*xHash)(const void*,int);   /* The hash function */
+
+  assert( (new_size & (new_size-1))==0 );
+  new_ht = (struct _fts2ht *)fts2HashMalloc( new_size*sizeof(struct _fts2ht) );
+  if( new_ht==0 ) return;
+  fts2HashFree(pH->ht);
+  pH->ht = new_ht;
+  pH->htsize = new_size;
+  xHash = hashFunction(pH->keyClass);
+  for(elem=pH->first, pH->first=0; elem; elem = next_elem){
+    int h = (*xHash)(elem->pKey, elem->nKey) & (new_size-1);
+    next_elem = elem->next;
+    insertElement(pH, &new_ht[h], elem);
+  }
+}
+
+/* This function (for internal use only) locates an element in an
+** hash table that matches the given key.  The hash for this key has
+** already been computed and is passed as the 4th parameter.
+*/
+static fts2HashElem *findElementGivenHash(
+  const fts2Hash *pH, /* The pH to be searched */
+  const void *pKey,   /* The key we are searching for */
+  int nKey,
+  int h               /* The hash for this key. */
+){
+  fts2HashElem *elem;            /* Used to loop thru the element list */
+  int count;                     /* Number of elements left to test */
+  int (*xCompare)(const void*,int,const void*,int);  /* comparison function */
+
+  if( pH->ht ){
+    struct _fts2ht *pEntry = &pH->ht[h];
+    elem = pEntry->chain;
+    count = pEntry->count;
+    xCompare = compareFunction(pH->keyClass);
+    while( count-- && elem ){
+      if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ 
+        return elem;
+      }
+      elem = elem->next;
+    }
+  }
+  return 0;
+}
+
+/* Remove a single entry from the hash table given a pointer to that
+** element and a hash on the element's key.
+*/
+static void removeElementGivenHash(
+  fts2Hash *pH,         /* The pH containing "elem" */
+  fts2HashElem* elem,   /* The element to be removed from the pH */
+  int h                 /* Hash value for the element */
+){
+  struct _fts2ht *pEntry;
+  if( elem->prev ){
+    elem->prev->next = elem->next; 
+  }else{
+    pH->first = elem->next;
+  }
+  if( elem->next ){
+    elem->next->prev = elem->prev;
+  }
+  pEntry = &pH->ht[h];
+  if( pEntry->chain==elem ){
+    pEntry->chain = elem->next;
+  }
+  pEntry->count--;
+  if( pEntry->count<=0 ){
+    pEntry->chain = 0;
+  }
+  if( pH->copyKey && elem->pKey ){
+    fts2HashFree(elem->pKey);
+  }
+  fts2HashFree( elem );
+  pH->count--;
+  if( pH->count<=0 ){
+    assert( pH->first==0 );
+    assert( pH->count==0 );
+    fts2HashClear(pH);
+  }
+}
+
+/* Attempt to locate an element of the hash table pH with a key
+** that matches pKey,nKey.  Return the data for this element if it is
+** found, or NULL if there is no match.
+*/
+void *sqlite3Fts2HashFind(const fts2Hash *pH, const void *pKey, int nKey){
+  int h;                 /* A hash on key */
+  fts2HashElem *elem;    /* The element that matches key */
+  int (*xHash)(const void*,int);  /* The hash function */
+
+  if( pH==0 || pH->ht==0 ) return 0;
+  xHash = hashFunction(pH->keyClass);
+  assert( xHash!=0 );
+  h = (*xHash)(pKey,nKey);
+  assert( (pH->htsize & (pH->htsize-1))==0 );
+  elem = findElementGivenHash(pH,pKey,nKey, h & (pH->htsize-1));
+  return elem ? elem->data : 0;
+}
+
+/* Insert an element into the hash table pH.  The key is pKey,nKey
+** and the data is "data".
+**
+** If no element exists with a matching key, then a new
+** element is created.  A copy of the key is made if the copyKey
+** flag is set.  NULL is returned.
+**
+** If another element already exists with the same key, then the
+** new data replaces the old data and the old data is returned.
+** The key is not copied in this instance.  If a malloc fails, then
+** the new data is returned and the hash table is unchanged.
+**
+** If the "data" parameter to this function is NULL, then the
+** element corresponding to "key" is removed from the hash table.
+*/
+void *sqlite3Fts2HashInsert(
+  fts2Hash *pH,        /* The hash table to insert into */
+  const void *pKey,    /* The key */
+  int nKey,            /* Number of bytes in the key */
+  void *data           /* The data */
+){
+  int hraw;                 /* Raw hash value of the key */
+  int h;                    /* the hash of the key modulo hash table size */
+  fts2HashElem *elem;       /* Used to loop thru the element list */
+  fts2HashElem *new_elem;   /* New element added to the pH */
+  int (*xHash)(const void*,int);  /* The hash function */
+
+  assert( pH!=0 );
+  xHash = hashFunction(pH->keyClass);
+  assert( xHash!=0 );
+  hraw = (*xHash)(pKey, nKey);
+  assert( (pH->htsize & (pH->htsize-1))==0 );
+  h = hraw & (pH->htsize-1);
+  elem = findElementGivenHash(pH,pKey,nKey,h);
+  if( elem ){
+    void *old_data = elem->data;
+    if( data==0 ){
+      removeElementGivenHash(pH,elem,h);
+    }else{
+      elem->data = data;
+    }
+    return old_data;
+  }
+  if( data==0 ) return 0;
+  new_elem = (fts2HashElem*)fts2HashMalloc( sizeof(fts2HashElem) );
+  if( new_elem==0 ) return data;
+  if( pH->copyKey && pKey!=0 ){
+    new_elem->pKey = fts2HashMalloc( nKey );
+    if( new_elem->pKey==0 ){
+      fts2HashFree(new_elem);
+      return data;
+    }
+    memcpy((void*)new_elem->pKey, pKey, nKey);
+  }else{
+    new_elem->pKey = (void*)pKey;
+  }
+  new_elem->nKey = nKey;
+  pH->count++;
+  if( pH->htsize==0 ){
+    rehash(pH,8);
+    if( pH->htsize==0 ){
+      pH->count = 0;
+      fts2HashFree(new_elem);
+      return data;
+    }
+  }
+  if( pH->count > pH->htsize ){
+    rehash(pH,pH->htsize*2);
+  }
+  assert( pH->htsize>0 );
+  assert( (pH->htsize & (pH->htsize-1))==0 );
+  h = hraw & (pH->htsize-1);
+  insertElement(pH, &pH->ht[h], new_elem);
+  new_elem->data = data;
+  return 0;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
@@ -0,0 +1,110 @@
+/*
+** 2001 September 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This is the header file for the generic hash-table implementation
+** used in SQLite.  We've modified it slightly to serve as a standalone
+** hash table implementation for the full-text indexing module.
+**
+*/
+#ifndef _FTS2_HASH_H_
+#define _FTS2_HASH_H_
+
+/* Forward declarations of structures. */
+typedef struct fts2Hash fts2Hash;
+typedef struct fts2HashElem fts2HashElem;
+
+/* A complete hash table is an instance of the following structure.
+** The internals of this structure are intended to be opaque -- client
+** code should not attempt to access or modify the fields of this structure
+** directly.  Change this structure only by using the routines below.
+** However, many of the "procedures" and "functions" for modifying and
+** accessing this structure are really macros, so we can't really make
+** this structure opaque.
+*/
+struct fts2Hash {
+  char keyClass;          /* HASH_INT, _POINTER, _STRING, _BINARY */
+  char copyKey;           /* True if copy of key made on insert */
+  int count;              /* Number of entries in this table */
+  fts2HashElem *first;    /* The first element of the array */
+  int htsize;             /* Number of buckets in the hash table */
+  struct _fts2ht {        /* the hash table */
+    int count;               /* Number of entries with this hash */
+    fts2HashElem *chain;     /* Pointer to first entry with this hash */
+  } *ht;
+};
+
+/* Each element in the hash table is an instance of the following 
+** structure.  All elements are stored on a single doubly-linked list.
+**
+** Again, this structure is intended to be opaque, but it can't really
+** be opaque because it is used by macros.
+*/
+struct fts2HashElem {
+  fts2HashElem *next, *prev; /* Next and previous elements in the table */
+  void *data;                /* Data associated with this element */
+  void *pKey; int nKey;      /* Key associated with this element */
+};
+
+/*
+** There are 2 different modes of operation for a hash table:
+**
+**   FTS2_HASH_STRING        pKey points to a string that is nKey bytes long
+**                           (including the null-terminator, if any).  Case
+**                           is respected in comparisons.
+**
+**   FTS2_HASH_BINARY        pKey points to binary data nKey bytes long. 
+**                           memcmp() is used to compare keys.
+**
+** A copy of the key is made if the copyKey parameter to fts2HashInit is 1.  
+*/
+#define FTS2_HASH_STRING    1
+#define FTS2_HASH_BINARY    2
+
+/*
+** Access routines.  To delete, insert a NULL pointer.
+*/
+void sqlite3Fts2HashInit(fts2Hash*, int keytype, int copyKey);
+void *sqlite3Fts2HashInsert(fts2Hash*, const void *pKey, int nKey, void *pData);
+void *sqlite3Fts2HashFind(const fts2Hash*, const void *pKey, int nKey);
+void sqlite3Fts2HashClear(fts2Hash*);
+
+/*
+** Shorthand for the functions above
+*/
+#define fts2HashInit   sqlite3Fts2HashInit
+#define fts2HashInsert sqlite3Fts2HashInsert
+#define fts2HashFind   sqlite3Fts2HashFind
+#define fts2HashClear  sqlite3Fts2HashClear
+
+/*
+** Macros for looping over all elements of a hash table.  The idiom is
+** like this:
+**
+**   fts2Hash h;
+**   fts2HashElem *p;
+**   ...
+**   for(p=fts2HashFirst(&h); p; p=fts2HashNext(p)){
+**     SomeStructure *pData = fts2HashData(p);
+**     // do something with pData
+**   }
+*/
+#define fts2HashFirst(H)  ((H)->first)
+#define fts2HashNext(E)   ((E)->next)
+#define fts2HashData(E)   ((E)->data)
+#define fts2HashKey(E)    ((E)->pKey)
+#define fts2HashKeysize(E) ((E)->nKey)
+
+/*
+** Number of entries in a hash table
+*/
+#define fts2HashCount(H)  ((H)->count)
+
+#endif /* _FTS2_HASH_H_ */
@@ -0,0 +1,260 @@
+/*
+** 2007 June 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file implements a tokenizer for fts2 based on the ICU library.
+** 
+** $Id: fts2_icu.c,v 1.3 2008/12/18 05:30:26 danielk1977 Exp $
+*/
+
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
+#ifdef SQLITE_ENABLE_ICU
+
+#include <assert.h>
+#include <string.h>
+#include "fts2_tokenizer.h"
+
+#include <unicode/ubrk.h>
+#include <unicode/ucol.h>
+#include <unicode/ustring.h>
+#include <unicode/utf16.h>
+
+typedef struct IcuTokenizer IcuTokenizer;
+typedef struct IcuCursor IcuCursor;
+
+struct IcuTokenizer {
+  sqlite3_tokenizer base;
+  char *zLocale;
+};
+
+struct IcuCursor {
+  sqlite3_tokenizer_cursor base;
+
+  UBreakIterator *pIter;      /* ICU break-iterator object */
+  int nChar;                  /* Number of UChar elements in pInput */
+  UChar *aChar;               /* Copy of input using utf-16 encoding */
+  int *aOffset;               /* Offsets of each character in utf-8 input */
+
+  int nBuffer;
+  char *zBuffer;
+
+  int iToken;
+};
+
+/*
+** Create a new tokenizer instance.
+*/
+static int icuCreate(
+  int argc,                            /* Number of entries in argv[] */
+  const char * const *argv,            /* Tokenizer creation arguments */
+  sqlite3_tokenizer **ppTokenizer      /* OUT: Created tokenizer */
+){
+  IcuTokenizer *p;
+  int n = 0;
+
+  if( argc>0 ){
+    n = strlen(argv[0])+1;
+  }
+  p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n);
+  if( !p ){
+    return SQLITE_NOMEM;
+  }
+  memset(p, 0, sizeof(IcuTokenizer));
+
+  if( n ){
+    p->zLocale = (char *)&p[1];
+    memcpy(p->zLocale, argv[0], n);
+  }
+
+  *ppTokenizer = (sqlite3_tokenizer *)p;
+
+  return SQLITE_OK;
+}
+
+/*
+** Destroy a tokenizer
+*/
+static int icuDestroy(sqlite3_tokenizer *pTokenizer){
+  IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
+  sqlite3_free(p);
+  return SQLITE_OK;
+}
+
+/*
+** Prepare to begin tokenizing a particular string.  The input
+** string to be tokenized is pInput[0..nBytes-1].  A cursor
+** used to incrementally tokenize this string is returned in 
+** *ppCursor.
+*/
+static int icuOpen(
+  sqlite3_tokenizer *pTokenizer,         /* The tokenizer */
+  const char *zInput,                    /* Input string */
+  int nInput,                            /* Length of zInput in bytes */
+  sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */
+){
+  IcuTokenizer *p = (IcuTokenizer *)pTokenizer;
+  IcuCursor *pCsr;
+
+  const int32_t opt = U_FOLD_CASE_DEFAULT;
+  UErrorCode status = U_ZERO_ERROR;
+  int nChar;
+
+  UChar32 c;
+  int iInput = 0;
+  int iOut = 0;
+
+  *ppCursor = 0;
+
+  if( nInput<0 ){
+    nInput = strlen(zInput);
+  }
+  nChar = nInput+1;
+  pCsr = (IcuCursor *)sqlite3_malloc(
+      sizeof(IcuCursor) +                /* IcuCursor */
+      ((nChar+3)&~3) * sizeof(UChar) +   /* IcuCursor.aChar[] */
+      (nChar+1) * sizeof(int)            /* IcuCursor.aOffset[] */
+  );
+  if( !pCsr ){
+    return SQLITE_NOMEM;
+  }
+  memset(pCsr, 0, sizeof(IcuCursor));
+  pCsr->aChar = (UChar *)&pCsr[1];
+  pCsr->aOffset = (int *)&pCsr->aChar[(nChar+3)&~3];
+
+  pCsr->aOffset[iOut] = iInput;
+  U8_NEXT(zInput, iInput, nInput, c); 
+  while( c>0 ){
+    int isError = 0;
+    c = u_foldCase(c, opt);
+    U16_APPEND(pCsr->aChar, iOut, nChar, c, isError);
+    if( isError ){
+      sqlite3_free(pCsr);
+      return SQLITE_ERROR;
+    }
+    pCsr->aOffset[iOut] = iInput;
+
+    if( iInput<nInput ){
+      U8_NEXT(zInput, iInput, nInput, c);
+    }else{
+      c = 0;
+    }
+  }
+
+  pCsr->pIter = ubrk_open(UBRK_WORD, p->zLocale, pCsr->aChar, iOut, &status);
+  if( !U_SUCCESS(status) ){
+    sqlite3_free(pCsr);
+    return SQLITE_ERROR;
+  }
+  pCsr->nChar = iOut;
+
+  ubrk_first(pCsr->pIter);
+  *ppCursor = (sqlite3_tokenizer_cursor *)pCsr;
+  return SQLITE_OK;
+}
+
+/*
+** Close a tokenization cursor previously opened by a call to icuOpen().
+*/
+static int icuClose(sqlite3_tokenizer_cursor *pCursor){
+  IcuCursor *pCsr = (IcuCursor *)pCursor;
+  ubrk_close(pCsr->pIter);
+  sqlite3_free(pCsr->zBuffer);
+  sqlite3_free(pCsr);
+  return SQLITE_OK;
+}
+
+/*
+** Extract the next token from a tokenization cursor.
+*/
+static int icuNext(
+  sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by simpleOpen */
+  const char **ppToken,               /* OUT: *ppToken is the token text */
+  int *pnBytes,                       /* OUT: Number of bytes in token */
+  int *piStartOffset,                 /* OUT: Starting offset of token */
+  int *piEndOffset,                   /* OUT: Ending offset of token */
+  int *piPosition                     /* OUT: Position integer of token */
+){
+  IcuCursor *pCsr = (IcuCursor *)pCursor;
+
+  int iStart = 0;
+  int iEnd = 0;
+  int nByte = 0;
+
+  while( iStart==iEnd ){
+    UChar32 c;
+
+    iStart = ubrk_current(pCsr->pIter);
+    iEnd = ubrk_next(pCsr->pIter);
+    if( iEnd==UBRK_DONE ){
+      return SQLITE_DONE;
+    }
+
+    while( iStart<iEnd ){
+      int iWhite = iStart;
+      U8_NEXT(pCsr->aChar, iWhite, pCsr->nChar, c);
+      if( u_isspace(c) ){
+        iStart = iWhite;
+      }else{
+        break;
+      }
+    }
+    assert(iStart<=iEnd);
+  }
+
+  do {
+    UErrorCode status = U_ZERO_ERROR;
+    if( nByte ){
+      char *zNew = sqlite3_realloc(pCsr->zBuffer, nByte);
+      if( !zNew ){
+        return SQLITE_NOMEM;
+      }
+      pCsr->zBuffer = zNew;
+      pCsr->nBuffer = nByte;
+    }
+
+    u_strToUTF8(
+        pCsr->zBuffer, pCsr->nBuffer, &nByte,    /* Output vars */
+        &pCsr->aChar[iStart], iEnd-iStart,       /* Input vars */
+        &status                                  /* Output success/failure */
+    );
+  } while( nByte>pCsr->nBuffer );
+
+  *ppToken = pCsr->zBuffer;
+  *pnBytes = nByte;
+  *piStartOffset = pCsr->aOffset[iStart];
+  *piEndOffset = pCsr->aOffset[iEnd];
+  *piPosition = pCsr->iToken++;
+
+  return SQLITE_OK;
+}
+
+/*
+** The set of routines that implement the simple tokenizer
+*/
+static const sqlite3_tokenizer_module icuTokenizerModule = {
+  0,                           /* iVersion */
+  icuCreate,                   /* xCreate  */
+  icuDestroy,                  /* xCreate  */
+  icuOpen,                     /* xOpen    */
+  icuClose,                    /* xClose   */
+  icuNext,                     /* xNext    */
+};
+
+/*
+** Set *ppModule to point at the implementation of the ICU tokenizer.
+*/
+void sqlite3Fts2IcuTokenizerModule(
+  sqlite3_tokenizer_module const**ppModule
+){
+  *ppModule = &icuTokenizerModule;
+}
+
+#endif /* defined(SQLITE_ENABLE_ICU) */
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
@@ -0,0 +1,644 @@
+/*
+** 2006 September 30
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** Implementation of the full-text-search tokenizer that implements
+** a Porter stemmer.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+**     * The FTS2 module is being built as an extension
+**       (in which case SQLITE_CORE is not defined), or
+**
+**     * The FTS2 module is being built into the core of
+**       SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
+
+
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "sqlite3.h"
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT3
+#include "fts2_tokenizer.h"
+
+/*
+** Class derived from sqlite3_tokenizer
+*/
+typedef struct porter_tokenizer {
+  sqlite3_tokenizer base;      /* Base class */
+} porter_tokenizer;
+
+/*
+** Class derived from sqlit3_tokenizer_cursor
+*/
+typedef struct porter_tokenizer_cursor {
+  sqlite3_tokenizer_cursor base;
+  const char *zInput;          /* input we are tokenizing */
+  int nInput;                  /* size of the input */
+  int iOffset;                 /* current position in zInput */
+  int iToken;                  /* index of next token to be returned */
+  char *zToken;                /* storage for current token */
+  int nAllocated;              /* space allocated to zToken buffer */
+} porter_tokenizer_cursor;
+
+
+/* Forward declaration */
+static const sqlite3_tokenizer_module porterTokenizerModule;
+
+
+/*
+** Create a new tokenizer instance.
+*/
+static int porterCreate(
+  int argc, const char * const *argv,
+  sqlite3_tokenizer **ppTokenizer
+){
+  porter_tokenizer *t;
+  t = (porter_tokenizer *) sqlite3_malloc(sizeof(*t));
+  if( t==NULL ) return SQLITE_NOMEM;
+  memset(t, 0, sizeof(*t));
+  *ppTokenizer = &t->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destroy a tokenizer
+*/
+static int porterDestroy(sqlite3_tokenizer *pTokenizer){
+  sqlite3_free(pTokenizer);
+  return SQLITE_OK;
+}
+
+/*
+** Prepare to begin tokenizing a particular string.  The input
+** string to be tokenized is zInput[0..nInput-1].  A cursor
+** used to incrementally tokenize this string is returned in 
+** *ppCursor.
+*/
+static int porterOpen(
+  sqlite3_tokenizer *pTokenizer,         /* The tokenizer */
+  const char *zInput, int nInput,        /* String to be tokenized */
+  sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */
+){
+  porter_tokenizer_cursor *c;
+
+  c = (porter_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
+  if( c==NULL ) return SQLITE_NOMEM;
+
+  c->zInput = zInput;
+  if( zInput==0 ){
+    c->nInput = 0;
+  }else if( nInput<0 ){
+    c->nInput = (int)strlen(zInput);
+  }else{
+    c->nInput = nInput;
+  }
+  c->iOffset = 0;                 /* start tokenizing at the beginning */
+  c->iToken = 0;
+  c->zToken = NULL;               /* no space allocated, yet. */
+  c->nAllocated = 0;
+
+  *ppCursor = &c->base;
+  return SQLITE_OK;
+}
+
+/*
+** Close a tokenization cursor previously opened by a call to
+** porterOpen() above.
+*/
+static int porterClose(sqlite3_tokenizer_cursor *pCursor){
+  porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
+  sqlite3_free(c->zToken);
+  sqlite3_free(c);
+  return SQLITE_OK;
+}
+/*
+** Vowel or consonant
+*/
+static const char cType[] = {
+   0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0,
+   1, 1, 1, 2, 1
+};
+
+/*
+** isConsonant() and isVowel() determine if their first character in
+** the string they point to is a consonant or a vowel, according
+** to Porter ruls.  
+**
+** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'.
+** 'Y' is a consonant unless it follows another consonant,
+** in which case it is a vowel.
+**
+** In these routine, the letters are in reverse order.  So the 'y' rule
+** is that 'y' is a consonant unless it is followed by another
+** consonent.
+*/
+static int isVowel(const char*);
+static int isConsonant(const char *z){
+  int j;
+  char x = *z;
+  if( x==0 ) return 0;
+  assert( x>='a' && x<='z' );
+  j = cType[x-'a'];
+  if( j<2 ) return j;
+  return z[1]==0 || isVowel(z + 1);
+}
+static int isVowel(const char *z){
+  int j;
+  char x = *z;
+  if( x==0 ) return 0;
+  assert( x>='a' && x<='z' );
+  j = cType[x-'a'];
+  if( j<2 ) return 1-j;
+  return isConsonant(z + 1);
+}
+
+/*
+** Let any sequence of one or more vowels be represented by V and let
+** C be sequence of one or more consonants.  Then every word can be
+** represented as:
+**
+**           [C] (VC){m} [V]
+**
+** In prose:  A word is an optional consonant followed by zero or
+** vowel-consonant pairs followed by an optional vowel.  "m" is the
+** number of vowel consonant pairs.  This routine computes the value
+** of m for the first i bytes of a word.
+**
+** Return true if the m-value for z is 1 or more.  In other words,
+** return true if z contains at least one vowel that is followed
+** by a consonant.
+**
+** In this routine z[] is in reverse order.  So we are really looking
+** for an instance of of a consonant followed by a vowel.
+*/
+static int m_gt_0(const char *z){
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isConsonant(z) ){ z++; }
+  return *z!=0;
+}
+
+/* Like mgt0 above except we are looking for a value of m which is
+** exactly 1
+*/
+static int m_eq_1(const char *z){
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isConsonant(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 1;
+  while( isConsonant(z) ){ z++; }
+  return *z==0;
+}
+
+/* Like mgt0 above except we are looking for a value of m>1 instead
+** or m>0
+*/
+static int m_gt_1(const char *z){
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isConsonant(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isVowel(z) ){ z++; }
+  if( *z==0 ) return 0;
+  while( isConsonant(z) ){ z++; }
+  return *z!=0;
+}
+
+/*
+** Return TRUE if there is a vowel anywhere within z[0..n-1]
+*/
+static int hasVowel(const char *z){
+  while( isConsonant(z) ){ z++; }
+  return *z!=0;
+}
+
+/*
+** Return TRUE if the word ends in a double consonant.
+**
+** The text is reversed here. So we are really looking at
+** the first two characters of z[].
+*/
+static int doubleConsonant(const char *z){
+  return isConsonant(z) && z[0]==z[1] && isConsonant(z+1);
+}
+
+/*
+** Return TRUE if the word ends with three letters which
+** are consonant-vowel-consonent and where the final consonant
+** is not 'w', 'x', or 'y'.
+**
+** The word is reversed here.  So we are really checking the
+** first three letters and the first one cannot be in [wxy].
+*/
+static int star_oh(const char *z){
+  return
+    z[0]!=0 && isConsonant(z) &&
+    z[0]!='w' && z[0]!='x' && z[0]!='y' &&
+    z[1]!=0 && isVowel(z+1) &&
+    z[2]!=0 && isConsonant(z+2);
+}
+
+/*
+** If the word ends with zFrom and xCond() is true for the stem
+** of the word that preceeds the zFrom ending, then change the 
+** ending to zTo.
+**
+** The input word *pz and zFrom are both in reverse order.  zTo
+** is in normal order. 
+**
+** Return TRUE if zFrom matches.  Return FALSE if zFrom does not
+** match.  Not that TRUE is returned even if xCond() fails and
+** no substitution occurs.
+*/
+static int stem(
+  char **pz,             /* The word being stemmed (Reversed) */
+  const char *zFrom,     /* If the ending matches this... (Reversed) */
+  const char *zTo,       /* ... change the ending to this (not reversed) */
+  int (*xCond)(const char*)   /* Condition that must be true */
+){
+  char *z = *pz;
+  while( *zFrom && *zFrom==*z ){ z++; zFrom++; }
+  if( *zFrom!=0 ) return 0;
+  if( xCond && !xCond(z) ) return 1;
+  while( *zTo ){
+    *(--z) = *(zTo++);
+  }
+  *pz = z;
+  return 1;
+}
+
+/*
+** This is the fallback stemmer used when the porter stemmer is
+** inappropriate.  The input word is copied into the output with
+** US-ASCII case folding.  If the input word is too long (more
+** than 20 bytes if it contains no digits or more than 6 bytes if
+** it contains digits) then word is truncated to 20 or 6 bytes
+** by taking 10 or 3 bytes from the beginning and end.
+*/
+static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
+  int i, mx, j;
+  int hasDigit = 0;
+  for(i=0; i<nIn; i++){
+    int c = zIn[i];
+    if( c>='A' && c<='Z' ){
+      zOut[i] = c - 'A' + 'a';
+    }else{
+      if( c>='0' && c<='9' ) hasDigit = 1;
+      zOut[i] = c;
+    }
+  }
+  mx = hasDigit ? 3 : 10;
+  if( nIn>mx*2 ){
+    for(j=mx, i=nIn-mx; i<nIn; i++, j++){
+      zOut[j] = zOut[i];
+    }
+    i = j;
+  }
+  zOut[i] = 0;
+  *pnOut = i;
+}
+
+
+/*
+** Stem the input word zIn[0..nIn-1].  Store the output in zOut.
+** zOut is at least big enough to hold nIn bytes.  Write the actual
+** size of the output word (exclusive of the '\0' terminator) into *pnOut.
+**
+** Any upper-case characters in the US-ASCII character set ([A-Z])
+** are converted to lower case.  Upper-case UTF characters are
+** unchanged.
+**
+** Words that are longer than about 20 bytes are stemmed by retaining
+** a few bytes from the beginning and the end of the word.  If the
+** word contains digits, 3 bytes are taken from the beginning and
+** 3 bytes from the end.  For long words without digits, 10 bytes
+** are taken from each end.  US-ASCII case folding still applies.
+** 
+** If the input word contains not digits but does characters not 
+** in [a-zA-Z] then no stemming is attempted and this routine just 
+** copies the input into the input into the output with US-ASCII
+** case folding.
+**
+** Stemming never increases the length of the word.  So there is
+** no chance of overflowing the zOut buffer.
+*/
+static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){
+  int i, j, c;
+  char zReverse[28];
+  char *z, *z2;
+  if( nIn<3 || nIn>=sizeof(zReverse)-7 ){
+    /* The word is too big or too small for the porter stemmer.
+    ** Fallback to the copy stemmer */
+    copy_stemmer(zIn, nIn, zOut, pnOut);
+    return;
+  }
+  for(i=0, j=sizeof(zReverse)-6; i<nIn; i++, j--){
+    c = zIn[i];
+    if( c>='A' && c<='Z' ){
+      zReverse[j] = c + 'a' - 'A';
+    }else if( c>='a' && c<='z' ){
+      zReverse[j] = c;
+    }else{
+      /* The use of a character not in [a-zA-Z] means that we fallback
+      ** to the copy stemmer */
+      copy_stemmer(zIn, nIn, zOut, pnOut);
+      return;
+    }
+  }
+  memset(&zReverse[sizeof(zReverse)-5], 0, 5);
+  z = &zReverse[j+1];
+
+
+  /* Step 1a */
+  if( z[0]=='s' ){
+    if(
+     !stem(&z, "sess", "ss", 0) &&
+     !stem(&z, "sei", "i", 0)  &&
+     !stem(&z, "ss", "ss", 0)
+    ){
+      z++;
+    }
+  }
+
+  /* Step 1b */  
+  z2 = z;
+  if( stem(&z, "dee", "ee", m_gt_0) ){
+    /* Do nothing.  The work was all in the test */
+  }else if( 
+     (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel))
+      && z!=z2
+  ){
+     if( stem(&z, "ta", "ate", 0) ||
+         stem(&z, "lb", "ble", 0) ||
+         stem(&z, "zi", "ize", 0) ){
+       /* Do nothing.  The work was all in the test */
+     }else if( doubleConsonant(z) && (*z!='l' && *z!='s' && *z!='z') ){
+       z++;
+     }else if( m_eq_1(z) && star_oh(z) ){
+       *(--z) = 'e';
+     }
+  }
+
+  /* Step 1c */
+  if( z[0]=='y' && hasVowel(z+1) ){
+    z[0] = 'i';
+  }
+
+  /* Step 2 */
+  switch( z[1] ){
+   case 'a':
+     stem(&z, "lanoita", "ate", m_gt_0) ||
+     stem(&z, "lanoit", "tion", m_gt_0);
+     break;
+   case 'c':
+     stem(&z, "icne", "ence", m_gt_0) ||
+     stem(&z, "icna", "ance", m_gt_0);
+     break;
+   case 'e':
+     stem(&z, "rezi", "ize", m_gt_0);
+     break;
+   case 'g':
+     stem(&z, "igol", "log", m_gt_0);
+     break;
+   case 'l':
+     stem(&z, "ilb", "ble", m_gt_0) ||
+     stem(&z, "illa", "al", m_gt_0) ||
+     stem(&z, "iltne", "ent", m_gt_0) ||
+     stem(&z, "ile", "e", m_gt_0) ||
+     stem(&z, "ilsuo", "ous", m_gt_0);
+     break;
+   case 'o':
+     stem(&z, "noitazi", "ize", m_gt_0) ||
+     stem(&z, "noita", "ate", m_gt_0) ||
+     stem(&z, "rota", "ate", m_gt_0);
+     break;
+   case 's':
+     stem(&z, "msila", "al", m_gt_0) ||
+     stem(&z, "ssenevi", "ive", m_gt_0) ||
+     stem(&z, "ssenluf", "ful", m_gt_0) ||
+     stem(&z, "ssensuo", "ous", m_gt_0);
+     break;
+   case 't':
+     stem(&z, "itila", "al", m_gt_0) ||
+     stem(&z, "itivi", "ive", m_gt_0) ||
+     stem(&z, "itilib", "ble", m_gt_0);
+     break;
+  }
+
+  /* Step 3 */
+  switch( z[0] ){
+   case 'e':
+     stem(&z, "etaci", "ic", m_gt_0) ||
+     stem(&z, "evita", "", m_gt_0)   ||
+     stem(&z, "ezila", "al", m_gt_0);
+     break;
+   case 'i':
+     stem(&z, "itici", "ic", m_gt_0);
+     break;
+   case 'l':
+     stem(&z, "laci", "ic", m_gt_0) ||
+     stem(&z, "luf", "", m_gt_0);
+     break;
+   case 's':
+     stem(&z, "ssen", "", m_gt_0);
+     break;
+  }
+
+  /* Step 4 */
+  switch( z[1] ){
+   case 'a':
+     if( z[0]=='l' && m_gt_1(z+2) ){
+       z += 2;
+     }
+     break;
+   case 'c':
+     if( z[0]=='e' && z[2]=='n' && (z[3]=='a' || z[3]=='e')  && m_gt_1(z+4)  ){
+       z += 4;
+     }
+     break;
+   case 'e':
+     if( z[0]=='r' && m_gt_1(z+2) ){
+       z += 2;
+     }
+     break;
+   case 'i':
+     if( z[0]=='c' && m_gt_1(z+2) ){
+       z += 2;
+     }
+     break;
+   case 'l':
+     if( z[0]=='e' && z[2]=='b' && (z[3]=='a' || z[3]=='i') && m_gt_1(z+4) ){
+       z += 4;
+     }
+     break;
+   case 'n':
+     if( z[0]=='t' ){
+       if( z[2]=='a' ){
+         if( m_gt_1(z+3) ){
+           z += 3;
+         }
+       }else if( z[2]=='e' ){
+         stem(&z, "tneme", "", m_gt_1) ||
+         stem(&z, "tnem", "", m_gt_1) ||
+         stem(&z, "tne", "", m_gt_1);
+       }
+     }
+     break;
+   case 'o':
+     if( z[0]=='u' ){
+       if( m_gt_1(z+2) ){
+         z += 2;
+       }
+     }else if( z[3]=='s' || z[3]=='t' ){
+       stem(&z, "noi", "", m_gt_1);
+     }
+     break;
+   case 's':
+     if( z[0]=='m' && z[2]=='i' && m_gt_1(z+3) ){
+       z += 3;
+     }
+     break;
+   case 't':
+     stem(&z, "eta", "", m_gt_1) ||
+     stem(&z, "iti", "", m_gt_1);
+     break;
+   case 'u':
+     if( z[0]=='s' && z[2]=='o' && m_gt_1(z+3) ){
+       z += 3;
+     }
+     break;
+   case 'v':
+   case 'z':
+     if( z[0]=='e' && z[2]=='i' && m_gt_1(z+3) ){
+       z += 3;
+     }
+     break;
+  }
+
+  /* Step 5a */
+  if( z[0]=='e' ){
+    if( m_gt_1(z+1) ){
+      z++;
+    }else if( m_eq_1(z+1) && !star_oh(z+1) ){
+      z++;
+    }
+  }
+
+  /* Step 5b */
+  if( m_gt_1(z) && z[0]=='l' && z[1]=='l' ){
+    z++;
+  }
+
+  /* z[] is now the stemmed word in reverse order.  Flip it back
+  ** around into forward order and return.
+  */
+  *pnOut = i = strlen(z);
+  zOut[i] = 0;
+  while( *z ){
+    zOut[--i] = *(z++);
+  }
+}
+
+/*
+** Characters that can be part of a token.  We assume any character
+** whose value is greater than 0x80 (any UTF character) can be
+** part of a token.  In other words, delimiters all must have
+** values of 0x7f or lower.
+*/
+static const char porterIdChar[] = {
+/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,  /* 3x */
+    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 4x */
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,  /* 5x */
+    0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 6x */
+    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,  /* 7x */
+};
+#define isDelim(C) (((ch=C)&0x80)==0 && (ch<0x30 || !porterIdChar[ch-0x30]))
+
+/*
+** Extract the next token from a tokenization cursor.  The cursor must
+** have been opened by a prior call to porterOpen().
+*/
+static int porterNext(
+  sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by porterOpen */
+  const char **pzToken,               /* OUT: *pzToken is the token text */
+  int *pnBytes,                       /* OUT: Number of bytes in token */
+  int *piStartOffset,                 /* OUT: Starting offset of token */
+  int *piEndOffset,                   /* OUT: Ending offset of token */
+  int *piPosition                     /* OUT: Position integer of token */
+){
+  porter_tokenizer_cursor *c = (porter_tokenizer_cursor *) pCursor;
+  const char *z = c->zInput;
+
+  while( c->iOffset<c->nInput ){
+    int iStartOffset, ch;
+
+    /* Scan past delimiter characters */
+    while( c->iOffset<c->nInput && isDelim(z[c->iOffset]) ){
+      c->iOffset++;
+    }
+
+    /* Count non-delimiter characters. */
+    iStartOffset = c->iOffset;
+    while( c->iOffset<c->nInput && !isDelim(z[c->iOffset]) ){
+      c->iOffset++;
+    }
+
+    if( c->iOffset>iStartOffset ){
+      int n = c->iOffset-iStartOffset;
+      if( n>c->nAllocated ){
+        c->nAllocated = n+20;
+        c->zToken = sqlite3_realloc(c->zToken, c->nAllocated);
+        if( c->zToken==NULL ) return SQLITE_NOMEM;
+      }
+      porter_stemmer(&z[iStartOffset], n, c->zToken, pnBytes);
+      *pzToken = c->zToken;
+      *piStartOffset = iStartOffset;
+      *piEndOffset = c->iOffset;
+      *piPosition = c->iToken++;
+      return SQLITE_OK;
+    }
+  }
+  return SQLITE_DONE;
+}
+
+/*
+** The set of routines that implement the porter-stemmer tokenizer
+*/
+static const sqlite3_tokenizer_module porterTokenizerModule = {
+  0,
+  porterCreate,
+  porterDestroy,
+  porterOpen,
+  porterClose,
+  porterNext,
+};
+
+/*
+** Allocate a new porter tokenizer.  Return a pointer to the new
+** tokenizer in *ppModule
+*/
+void sqlite3Fts2PorterTokenizerModule(
+  sqlite3_tokenizer_module const**ppModule
+){
+  *ppModule = &porterTokenizerModule;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
@@ -0,0 +1,375 @@
+/*
+** 2007 June 22
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This is part of an SQLite module implementing full-text search.
+** This particular file implements the generic tokenizer interface.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+**     * The FTS2 module is being built as an extension
+**       (in which case SQLITE_CORE is not defined), or
+**
+**     * The FTS2 module is being built into the core of
+**       SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
+
+
+#include "sqlite3.h"
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT3
+
+#include "fts2_hash.h"
+#include "fts2_tokenizer.h"
+#include <assert.h>
+
+/*
+** Implementation of the SQL scalar function for accessing the underlying 
+** hash table. This function may be called as follows:
+**
+**   SELECT <function-name>(<key-name>);
+**   SELECT <function-name>(<key-name>, <pointer>);
+**
+** where <function-name> is the name passed as the second argument
+** to the sqlite3Fts2InitHashTable() function (e.g. 'fts2_tokenizer').
+**
+** If the <pointer> argument is specified, it must be a blob value
+** containing a pointer to be stored as the hash data corresponding
+** to the string <key-name>. If <pointer> is not specified, then
+** the string <key-name> must already exist in the has table. Otherwise,
+** an error is returned.
+**
+** Whether or not the <pointer> argument is specified, the value returned
+** is a blob containing the pointer stored as the hash data corresponding
+** to string <key-name> (after the hash-table is updated, if applicable).
+*/
+static void scalarFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  fts2Hash *pHash;
+  void *pPtr = 0;
+  const unsigned char *zName;
+  int nName;
+
+  assert( argc==1 || argc==2 );
+
+  pHash = (fts2Hash *)sqlite3_user_data(context);
+
+  zName = sqlite3_value_text(argv[0]);
+  nName = sqlite3_value_bytes(argv[0])+1;
+
+  if( argc==2 ){
+    void *pOld;
+    int n = sqlite3_value_bytes(argv[1]);
+    if( n!=sizeof(pPtr) ){
+      sqlite3_result_error(context, "argument type mismatch", -1);
+      return;
+    }
+    pPtr = *(void **)sqlite3_value_blob(argv[1]);
+    pOld = sqlite3Fts2HashInsert(pHash, (void *)zName, nName, pPtr);
+    if( pOld==pPtr ){
+      sqlite3_result_error(context, "out of memory", -1);
+      return;
+    }
+  }else{
+    pPtr = sqlite3Fts2HashFind(pHash, zName, nName);
+    if( !pPtr ){
+      char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
+      sqlite3_result_error(context, zErr, -1);
+      sqlite3_free(zErr);
+      return;
+    }
+  }
+
+  sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT);
+}
+
+#ifdef SQLITE_TEST
+
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
+#include <string.h>
+
+/*
+** Implementation of a special SQL scalar function for testing tokenizers 
+** designed to be used in concert with the Tcl testing framework. This
+** function must be called with two arguments:
+**
+**   SELECT <function-name>(<key-name>, <input-string>);
+**   SELECT <function-name>(<key-name>, <pointer>);
+**
+** where <function-name> is the name passed as the second argument
+** to the sqlite3Fts2InitHashTable() function (e.g. 'fts2_tokenizer')
+** concatenated with the string '_test' (e.g. 'fts2_tokenizer_test').
+**
+** The return value is a string that may be interpreted as a Tcl
+** list. For each token in the <input-string>, three elements are
+** added to the returned list. The first is the token position, the 
+** second is the token text (folded, stemmed, etc.) and the third is the
+** substring of <input-string> associated with the token. For example, 
+** using the built-in "simple" tokenizer:
+**
+**   SELECT fts_tokenizer_test('simple', 'I don't see how');
+**
+** will return the string:
+**
+**   "{0 i I 1 dont don't 2 see see 3 how how}"
+**   
+*/
+static void testFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  fts2Hash *pHash;
+  sqlite3_tokenizer_module *p;
+  sqlite3_tokenizer *pTokenizer = 0;
+  sqlite3_tokenizer_cursor *pCsr = 0;
+
+  const char *zErr = 0;
+
+  const char *zName;
+  int nName;
+  const char *zInput;
+  int nInput;
+
+  const char *zArg = 0;
+
+  const char *zToken;
+  int nToken;
+  int iStart;
+  int iEnd;
+  int iPos;
+
+  Tcl_Obj *pRet;
+
+  assert( argc==2 || argc==3 );
+
+  nName = sqlite3_value_bytes(argv[0]);
+  zName = (const char *)sqlite3_value_text(argv[0]);
+  nInput = sqlite3_value_bytes(argv[argc-1]);
+  zInput = (const char *)sqlite3_value_text(argv[argc-1]);
+
+  if( argc==3 ){
+    zArg = (const char *)sqlite3_value_text(argv[1]);
+  }
+
+  pHash = (fts2Hash *)sqlite3_user_data(context);
+  p = (sqlite3_tokenizer_module *)sqlite3Fts2HashFind(pHash, zName, nName+1);
+
+  if( !p ){
+    char *zErr = sqlite3_mprintf("unknown tokenizer: %s", zName);
+    sqlite3_result_error(context, zErr, -1);
+    sqlite3_free(zErr);
+    return;
+  }
+
+  pRet = Tcl_NewObj();
+  Tcl_IncrRefCount(pRet);
+
+  if( SQLITE_OK!=p->xCreate(zArg ? 1 : 0, &zArg, &pTokenizer) ){
+    zErr = "error in xCreate()";
+    goto finish;
+  }
+  pTokenizer->pModule = p;
+  if( SQLITE_OK!=p->xOpen(pTokenizer, zInput, nInput, &pCsr) ){
+    zErr = "error in xOpen()";
+    goto finish;
+  }
+  pCsr->pTokenizer = pTokenizer;
+
+  while( SQLITE_OK==p->xNext(pCsr, &zToken, &nToken, &iStart, &iEnd, &iPos) ){
+    Tcl_ListObjAppendElement(0, pRet, Tcl_NewIntObj(iPos));
+    Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
+    zToken = &zInput[iStart];
+    nToken = iEnd-iStart;
+    Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zToken, nToken));
+  }
+
+  if( SQLITE_OK!=p->xClose(pCsr) ){
+    zErr = "error in xClose()";
+    goto finish;
+  }
+  if( SQLITE_OK!=p->xDestroy(pTokenizer) ){
+    zErr = "error in xDestroy()";
+    goto finish;
+  }
+
+finish:
+  if( zErr ){
+    sqlite3_result_error(context, zErr, -1);
+  }else{
+    sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT);
+  }
+  Tcl_DecrRefCount(pRet);
+}
+
+static
+int registerTokenizer(
+  sqlite3 *db, 
+  char *zName, 
+  const sqlite3_tokenizer_module *p
+){
+  int rc;
+  sqlite3_stmt *pStmt;
+  const char zSql[] = "SELECT fts2_tokenizer(?, ?)";
+
+  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+  if( rc!=SQLITE_OK ){
+    return rc;
+  }
+
+  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+  sqlite3_bind_blob(pStmt, 2, &p, sizeof(p), SQLITE_STATIC);
+  sqlite3_step(pStmt);
+
+  return sqlite3_finalize(pStmt);
+}
+
+static
+int queryFts2Tokenizer(
+  sqlite3 *db, 
+  char *zName,  
+  const sqlite3_tokenizer_module **pp
+){
+  int rc;
+  sqlite3_stmt *pStmt;
+  const char zSql[] = "SELECT fts2_tokenizer(?)";
+
+  *pp = 0;
+  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+  if( rc!=SQLITE_OK ){
+    return rc;
+  }
+
+  sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC);
+  if( SQLITE_ROW==sqlite3_step(pStmt) ){
+    if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){
+      memcpy(pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp));
+    }
+  }
+
+  return sqlite3_finalize(pStmt);
+}
+
+void sqlite3Fts2SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule);
+
+/*
+** Implementation of the scalar function fts2_tokenizer_internal_test().
+** This function is used for testing only, it is not included in the
+** build unless SQLITE_TEST is defined.
+**
+** The purpose of this is to test that the fts2_tokenizer() function
+** can be used as designed by the C-code in the queryFts2Tokenizer and
+** registerTokenizer() functions above. These two functions are repeated
+** in the README.tokenizer file as an example, so it is important to
+** test them.
+**
+** To run the tests, evaluate the fts2_tokenizer_internal_test() scalar
+** function with no arguments. An assert() will fail if a problem is
+** detected. i.e.:
+**
+**     SELECT fts2_tokenizer_internal_test();
+**
+*/
+static void intTestFunc(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  int rc;
+  const sqlite3_tokenizer_module *p1;
+  const sqlite3_tokenizer_module *p2;
+  sqlite3 *db = (sqlite3 *)sqlite3_user_data(context);
+
+  /* Test the query function */
+  sqlite3Fts2SimpleTokenizerModule(&p1);
+  rc = queryFts2Tokenizer(db, "simple", &p2);
+  assert( rc==SQLITE_OK );
+  assert( p1==p2 );
+  rc = queryFts2Tokenizer(db, "nosuchtokenizer", &p2);
+  assert( rc==SQLITE_ERROR );
+  assert( p2==0 );
+  assert( 0==strcmp(sqlite3_errmsg(db), "unknown tokenizer: nosuchtokenizer") );
+
+  /* Test the storage function */
+  rc = registerTokenizer(db, "nosuchtokenizer", p1);
+  assert( rc==SQLITE_OK );
+  rc = queryFts2Tokenizer(db, "nosuchtokenizer", &p2);
+  assert( rc==SQLITE_OK );
+  assert( p2==p1 );
+
+  sqlite3_result_text(context, "ok", -1, SQLITE_STATIC);
+}
+
+#endif
+
+/*
+** Set up SQL objects in database db used to access the contents of
+** the hash table pointed to by argument pHash. The hash table must
+** been initialized to use string keys, and to take a private copy 
+** of the key when a value is inserted. i.e. by a call similar to:
+**
+**    sqlite3Fts2HashInit(pHash, FTS2_HASH_STRING, 1);
+**
+** This function adds a scalar function (see header comment above
+** scalarFunc() in this file for details) and, if ENABLE_TABLE is
+** defined at compilation time, a temporary virtual table (see header 
+** comment above struct HashTableVtab) to the database schema. Both 
+** provide read/write access to the contents of *pHash.
+**
+** The third argument to this function, zName, is used as the name
+** of both the scalar and, if created, the virtual table.
+*/
+int sqlite3Fts2InitHashTable(
+  sqlite3 *db, 
+  fts2Hash *pHash, 
+  const char *zName
+){
+  int rc = SQLITE_OK;
+  void *p = (void *)pHash;
+  const int any = SQLITE_ANY;
+  char *zTest = 0;
+  char *zTest2 = 0;
+
+#ifdef SQLITE_TEST
+  void *pdb = (void *)db;
+  zTest = sqlite3_mprintf("%s_test", zName);
+  zTest2 = sqlite3_mprintf("%s_internal_test", zName);
+  if( !zTest || !zTest2 ){
+    rc = SQLITE_NOMEM;
+  }
+#endif
+
+  if( rc!=SQLITE_OK
+   || (rc = sqlite3_create_function(db, zName, 1, any, p, scalarFunc, 0, 0))
+   || (rc = sqlite3_create_function(db, zName, 2, any, p, scalarFunc, 0, 0))
+#ifdef SQLITE_TEST
+   || (rc = sqlite3_create_function(db, zTest, 2, any, p, testFunc, 0, 0))
+   || (rc = sqlite3_create_function(db, zTest, 3, any, p, testFunc, 0, 0))
+   || (rc = sqlite3_create_function(db, zTest2, 0, any, pdb, intTestFunc, 0, 0))
+#endif
+  );
+
+  sqlite3_free(zTest);
+  sqlite3_free(zTest2);
+  return rc;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
@@ -0,0 +1,145 @@
+/*
+** 2006 July 10
+**
+** The author disclaims copyright to this source code.
+**
+*************************************************************************
+** Defines the interface to tokenizers used by fulltext-search.  There
+** are three basic components:
+**
+** sqlite3_tokenizer_module is a singleton defining the tokenizer
+** interface functions.  This is essentially the class structure for
+** tokenizers.
+**
+** sqlite3_tokenizer is used to define a particular tokenizer, perhaps
+** including customization information defined at creation time.
+**
+** sqlite3_tokenizer_cursor is generated by a tokenizer to generate
+** tokens from a particular input.
+*/
+#ifndef _FTS2_TOKENIZER_H_
+#define _FTS2_TOKENIZER_H_
+
+/* TODO(shess) Only used for SQLITE_OK and SQLITE_DONE at this time.
+** If tokenizers are to be allowed to call sqlite3_*() functions, then
+** we will need a way to register the API consistently.
+*/
+#include "sqlite3.h"
+
+/*
+** Structures used by the tokenizer interface. When a new tokenizer
+** implementation is registered, the caller provides a pointer to
+** an sqlite3_tokenizer_module containing pointers to the callback
+** functions that make up an implementation.
+**
+** When an fts2 table is created, it passes any arguments passed to
+** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the
+** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer
+** implementation. The xCreate() function in turn returns an 
+** sqlite3_tokenizer structure representing the specific tokenizer to
+** be used for the fts2 table (customized by the tokenizer clause arguments).
+**
+** To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen()
+** method is called. It returns an sqlite3_tokenizer_cursor object
+** that may be used to tokenize a specific input buffer based on
+** the tokenization rules supplied by a specific sqlite3_tokenizer
+** object.
+*/
+typedef struct sqlite3_tokenizer_module sqlite3_tokenizer_module;
+typedef struct sqlite3_tokenizer sqlite3_tokenizer;
+typedef struct sqlite3_tokenizer_cursor sqlite3_tokenizer_cursor;
+
+struct sqlite3_tokenizer_module {
+
+  /*
+  ** Structure version. Should always be set to 0.
+  */
+  int iVersion;
+
+  /*
+  ** Create a new tokenizer. The values in the argv[] array are the
+  ** arguments passed to the "tokenizer" clause of the CREATE VIRTUAL
+  ** TABLE statement that created the fts2 table. For example, if
+  ** the following SQL is executed:
+  **
+  **   CREATE .. USING fts2( ... , tokenizer <tokenizer-name> arg1 arg2)
+  **
+  ** then argc is set to 2, and the argv[] array contains pointers
+  ** to the strings "arg1" and "arg2".
+  **
+  ** This method should return either SQLITE_OK (0), or an SQLite error 
+  ** code. If SQLITE_OK is returned, then *ppTokenizer should be set
+  ** to point at the newly created tokenizer structure. The generic
+  ** sqlite3_tokenizer.pModule variable should not be initialized by
+  ** this callback. The caller will do so.
+  */
+  int (*xCreate)(
+    int argc,                           /* Size of argv array */
+    const char *const*argv,             /* Tokenizer argument strings */
+    sqlite3_tokenizer **ppTokenizer     /* OUT: Created tokenizer */
+  );
+
+  /*
+  ** Destroy an existing tokenizer. The fts2 module calls this method
+  ** exactly once for each successful call to xCreate().
+  */
+  int (*xDestroy)(sqlite3_tokenizer *pTokenizer);
+
+  /*
+  ** Create a tokenizer cursor to tokenize an input buffer. The caller
+  ** is responsible for ensuring that the input buffer remains valid
+  ** until the cursor is closed (using the xClose() method). 
+  */
+  int (*xOpen)(
+    sqlite3_tokenizer *pTokenizer,       /* Tokenizer object */
+    const char *pInput, int nBytes,      /* Input buffer */
+    sqlite3_tokenizer_cursor **ppCursor  /* OUT: Created tokenizer cursor */
+  );
+
+  /*
+  ** Destroy an existing tokenizer cursor. The fts2 module calls this 
+  ** method exactly once for each successful call to xOpen().
+  */
+  int (*xClose)(sqlite3_tokenizer_cursor *pCursor);
+
+  /*
+  ** Retrieve the next token from the tokenizer cursor pCursor. This
+  ** method should either return SQLITE_OK and set the values of the
+  ** "OUT" variables identified below, or SQLITE_DONE to indicate that
+  ** the end of the buffer has been reached, or an SQLite error code.
+  **
+  ** *ppToken should be set to point at a buffer containing the 
+  ** normalized version of the token (i.e. after any case-folding and/or
+  ** stemming has been performed). *pnBytes should be set to the length
+  ** of this buffer in bytes. The input text that generated the token is
+  ** identified by the byte offsets returned in *piStartOffset and
+  ** *piEndOffset.
+  **
+  ** The buffer *ppToken is set to point at is managed by the tokenizer
+  ** implementation. It is only required to be valid until the next call
+  ** to xNext() or xClose(). 
+  */
+  /* TODO(shess) current implementation requires pInput to be
+  ** nul-terminated.  This should either be fixed, or pInput/nBytes
+  ** should be converted to zInput.
+  */
+  int (*xNext)(
+    sqlite3_tokenizer_cursor *pCursor,   /* Tokenizer cursor */
+    const char **ppToken, int *pnBytes,  /* OUT: Normalized text for token */
+    int *piStartOffset,  /* OUT: Byte offset of token in input buffer */
+    int *piEndOffset,    /* OUT: Byte offset of end of token in input buffer */
+    int *piPosition      /* OUT: Number of tokens returned before this one */
+  );
+};
+
+struct sqlite3_tokenizer {
+  const sqlite3_tokenizer_module *pModule;  /* The module for this tokenizer */
+  /* Tokenizer implementations will typically add additional fields */
+};
+
+struct sqlite3_tokenizer_cursor {
+  sqlite3_tokenizer *pTokenizer;       /* Tokenizer for this cursor. */
+  /* Tokenizer implementations will typically add additional fields */
+};
+
+#endif /* _FTS2_TOKENIZER_H_ */
@@ -0,0 +1,233 @@
+/*
+** 2006 Oct 10
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** Implementation of the "simple" full-text-search tokenizer.
+*/
+
+/*
+** The code in this file is only compiled if:
+**
+**     * The FTS2 module is being built as an extension
+**       (in which case SQLITE_CORE is not defined), or
+**
+**     * The FTS2 module is being built into the core of
+**       SQLite (in which case SQLITE_ENABLE_FTS2 is defined).
+*/
+#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2)
+
+
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "sqlite3.h"
+#include "sqlite3ext.h"
+SQLITE_EXTENSION_INIT3
+#include "fts2_tokenizer.h"
+
+typedef struct simple_tokenizer {
+  sqlite3_tokenizer base;
+  char delim[128];             /* flag ASCII delimiters */
+} simple_tokenizer;
+
+typedef struct simple_tokenizer_cursor {
+  sqlite3_tokenizer_cursor base;
+  const char *pInput;          /* input we are tokenizing */
+  int nBytes;                  /* size of the input */
+  int iOffset;                 /* current position in pInput */
+  int iToken;                  /* index of next token to be returned */
+  char *pToken;                /* storage for current token */
+  int nTokenAllocated;         /* space allocated to zToken buffer */
+} simple_tokenizer_cursor;
+
+
+/* Forward declaration */
+static const sqlite3_tokenizer_module simpleTokenizerModule;
+
+static int simpleDelim(simple_tokenizer *t, unsigned char c){
+  return c<0x80 && t->delim[c];
+}
+
+/*
+** Create a new tokenizer instance.
+*/
+static int simpleCreate(
+  int argc, const char * const *argv,
+  sqlite3_tokenizer **ppTokenizer
+){
+  simple_tokenizer *t;
+
+  t = (simple_tokenizer *) sqlite3_malloc(sizeof(*t));
+  if( t==NULL ) return SQLITE_NOMEM;
+  memset(t, 0, sizeof(*t));
+
+  /* TODO(shess) Delimiters need to remain the same from run to run,
+  ** else we need to reindex.  One solution would be a meta-table to
+  ** track such information in the database, then we'd only want this
+  ** information on the initial create.
+  */
+  if( argc>1 ){
+    int i, n = strlen(argv[1]);
+    for(i=0; i<n; i++){
+      unsigned char ch = argv[1][i];
+      /* We explicitly don't support UTF-8 delimiters for now. */
+      if( ch>=0x80 ){
+        sqlite3_free(t);
+        return SQLITE_ERROR;
+      }
+      t->delim[ch] = 1;
+    }
+  } else {
+    /* Mark non-alphanumeric ASCII characters as delimiters */
+    int i;
+    for(i=1; i<0x80; i++){
+      t->delim[i] = !((i>='0' && i<='9') || (i>='A' && i<='Z') ||
+                      (i>='a' && i<='z'));
+    }
+  }
+
+  *ppTokenizer = &t->base;
+  return SQLITE_OK;
+}
+
+/*
+** Destroy a tokenizer
+*/
+static int simpleDestroy(sqlite3_tokenizer *pTokenizer){
+  sqlite3_free(pTokenizer);
+  return SQLITE_OK;
+}
+
+/*
+** Prepare to begin tokenizing a particular string.  The input
+** string to be tokenized is pInput[0..nBytes-1].  A cursor
+** used to incrementally tokenize this string is returned in 
+** *ppCursor.
+*/
+static int simpleOpen(
+  sqlite3_tokenizer *pTokenizer,         /* The tokenizer */
+  const char *pInput, int nBytes,        /* String to be tokenized */
+  sqlite3_tokenizer_cursor **ppCursor    /* OUT: Tokenization cursor */
+){
+  simple_tokenizer_cursor *c;
+
+  c = (simple_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
+  if( c==NULL ) return SQLITE_NOMEM;
+
+  c->pInput = pInput;
+  if( pInput==0 ){
+    c->nBytes = 0;
+  }else if( nBytes<0 ){
+    c->nBytes = (int)strlen(pInput);
+  }else{
+    c->nBytes = nBytes;
+  }
+  c->iOffset = 0;                 /* start tokenizing at the beginning */
+  c->iToken = 0;
+  c->pToken = NULL;               /* no space allocated, yet. */
+  c->nTokenAllocated = 0;
+
+  *ppCursor = &c->base;
+  return SQLITE_OK;
+}
+
+/*
+** Close a tokenization cursor previously opened by a call to
+** simpleOpen() above.
+*/
+static int simpleClose(sqlite3_tokenizer_cursor *pCursor){
+  simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
+  sqlite3_free(c->pToken);
+  sqlite3_free(c);
+  return SQLITE_OK;
+}
+
+/*
+** Extract the next token from a tokenization cursor.  The cursor must
+** have been opened by a prior call to simpleOpen().
+*/
+static int simpleNext(
+  sqlite3_tokenizer_cursor *pCursor,  /* Cursor returned by simpleOpen */
+  const char **ppToken,               /* OUT: *ppToken is the token text */
+  int *pnBytes,                       /* OUT: Number of bytes in token */
+  int *piStartOffset,                 /* OUT: Starting offset of token */
+  int *piEndOffset,                   /* OUT: Ending offset of token */
+  int *piPosition                     /* OUT: Position integer of token */
+){
+  simple_tokenizer_cursor *c = (simple_tokenizer_cursor *) pCursor;
+  simple_tokenizer *t = (simple_tokenizer *) pCursor->pTokenizer;
+  unsigned char *p = (unsigned char *)c->pInput;
+
+  while( c->iOffset<c->nBytes ){
+    int iStartOffset;
+
+    /* Scan past delimiter characters */
+    while( c->iOffset<c->nBytes && simpleDelim(t, p[c->iOffset]) ){
+      c->iOffset++;
+    }
+
+    /* Count non-delimiter characters. */
+    iStartOffset = c->iOffset;
+    while( c->iOffset<c->nBytes && !simpleDelim(t, p[c->iOffset]) ){
+      c->iOffset++;
+    }
+
+    if( c->iOffset>iStartOffset ){
+      int i, n = c->iOffset-iStartOffset;
+      if( n>c->nTokenAllocated ){
+        c->nTokenAllocated = n+20;
+        c->pToken = sqlite3_realloc(c->pToken, c->nTokenAllocated);
+        if( c->pToken==NULL ) return SQLITE_NOMEM;
+      }
+      for(i=0; i<n; i++){
+        /* TODO(shess) This needs expansion to handle UTF-8
+        ** case-insensitivity.
+        */
+        unsigned char ch = p[iStartOffset+i];
+        c->pToken[i] = (ch>='A' && ch<='Z') ? (ch - 'A' + 'a') : ch;
+      }
+      *ppToken = c->pToken;
+      *pnBytes = n;
+      *piStartOffset = iStartOffset;
+      *piEndOffset = c->iOffset;
+      *piPosition = c->iToken++;
+
+      return SQLITE_OK;
+    }
+  }
+  return SQLITE_DONE;
+}
+
+/*
+** The set of routines that implement the simple tokenizer
+*/
+static const sqlite3_tokenizer_module simpleTokenizerModule = {
+  0,
+  simpleCreate,
+  simpleDestroy,
+  simpleOpen,
+  simpleClose,
+  simpleNext,
+};
+
+/*
+** Allocate a new simple tokenizer.  Return a pointer to the new
+** tokenizer in *ppModule
+*/
+void sqlite3Fts2SimpleTokenizerModule(
+  sqlite3_tokenizer_module const**ppModule
+){
+  *ppModule = &simpleTokenizerModule;
+}
+
+#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS2) */
@@ -0,0 +1,116 @@
+#!/usr/bin/tclsh
+#
+# This script builds a single C code file holding all of FTS2 code.
+# The name of the output file is fts2amal.c.  To build this file,
+# first do:
+#
+#      make target_source
+#
+# The make target above moves all of the source code files into
+# a subdirectory named "tsrc".  (This script expects to find the files
+# there and will not work if they are not found.)
+#
+# After the "tsrc" directory has been created and populated, run
+# this script:
+#
+#      tclsh mkfts2amal.tcl
+#
+# The amalgamated FTS2 code will be written into fts2amal.c
+#
+
+# Open the output file and write a header comment at the beginning
+# of the file.
+#
+set out [open fts2amal.c w]
+set today [clock format [clock seconds] -format "%Y-%m-%d %H:%M:%S UTC" -gmt 1]
+puts $out [subst \
+{/******************************************************************************
+** This file is an amalgamation of separate C source files from the SQLite
+** Full Text Search extension 2 (fts2).  By combining all the individual C 
+** code  files into this single large file, the entire code can be compiled 
+** as a one translation unit.  This allows many compilers to do optimizations
+** that would not be possible if the files were compiled separately.  It also
+** makes the code easier to import into other projects.
+**
+** This amalgamation was generated on $today.
+*/}]
+
+# These are the header files used by FTS2.  The first time any of these 
+# files are seen in a #include statement in the C code, include the complete
+# text of the file in-line.  The file only needs to be included once.
+#
+foreach hdr {
+   fts2.h
+   fts2_hash.h
+   fts2_tokenizer.h
+   sqlite3.h
+   sqlite3ext.h
+} {
+  set available_hdr($hdr) 1
+}
+
+# 78 stars used for comment formatting.
+set s78 \
+{*****************************************************************************}
+
+# Insert a comment into the code
+#
+proc section_comment {text} {
+  global out s78
+  set n [string length $text]
+  set nstar [expr {60 - $n}]
+  set stars [string range $s78 0 $nstar]
+  puts $out "/************** $text $stars/"
+}
+
+# Read the source file named $filename and write it into the
+# sqlite3.c output file.  If any #include statements are seen,
+# process them approprately.
+#
+proc copy_file {filename} {
+  global seen_hdr available_hdr out
+  set tail [file tail $filename]
+  section_comment "Begin file $tail"
+  set in [open $filename r]
+  while {![eof $in]} {
+    set line [gets $in]
+    if {[regexp {^#\s*include\s+["<]([^">]+)[">]} $line all hdr]} {
+      if {[info exists available_hdr($hdr)]} {
+        if {$available_hdr($hdr)} {
+          section_comment "Include $hdr in the middle of $tail"
+          copy_file tsrc/$hdr
+          section_comment "Continuing where we left off in $tail"
+        }
+      } elseif {![info exists seen_hdr($hdr)]} {
+        set seen_hdr($hdr) 1
+        puts $out $line
+      }
+    } elseif {[regexp {^#ifdef __cplusplus} $line]} {
+      puts $out "#if 0"
+    } elseif {[regexp {^#line} $line]} {
+      # Skip #line directives.
+    } else {
+      puts $out $line
+    }
+  }
+  close $in
+  section_comment "End of $tail"
+}
+
+
+# Process the source files.  Process files containing commonly
+# used subroutines first in order to help the compiler find
+# inlining opportunities.
+#
+foreach file {
+   fts2.c
+   fts2_hash.c
+   fts2_porter.c
+   fts2_tokenizer.c
+   fts2_tokenizer1.c
+   fts2_icu.c
+} {
+  copy_file tsrc/$file
+}
+
+close $out
@@ -5287,8 +5287,9 @@ static void fts3EvalNextRow(
  Fts3Expr *pExpr,                /* Expr. to advance to next matching row */
  int *pRc                        /* IN/OUT: Error code */
 ){
-  if( *pRc==SQLITE_OK && pExpr->bEof==0 ){
+  if( *pRc==SQLITE_OK ){
    int bDescDoclist = pCsr->bDesc;         /* Used by DOCID_CMP() macro */
+    assert( pExpr->bEof==0 );
    pExpr->bStart = 1;

    switch( pExpr->eType ){
@@ -5764,22 +5765,6 @@ static void fts3EvalUpdateCounts(Fts3Expr *pExpr, int nCol){
  }
 }

-/*
-** This is an sqlite3Fts3ExprIterate() callback. If the Fts3Expr.aMI[] array
-** has not yet been allocated, allocate and zero it. Otherwise, just zero
-** it.
-*/
-static int fts3AllocateMSI(Fts3Expr *pExpr, int iPhrase, void *pCtx){
-  Fts3Table *pTab = (Fts3Table*)pCtx;
-  UNUSED_PARAMETER(iPhrase);
-  if( pExpr->aMI==0 ){
-    pExpr->aMI = (u32 *)sqlite3_malloc64(pTab->nColumn * 3 * sizeof(u32));
-    if( pExpr->aMI==0 ) return SQLITE_NOMEM;
-  }
-  memset(pExpr->aMI, 0, pTab->nColumn * 3 * sizeof(u32));
-  return SQLITE_OK;
-}
-
 /*
 ** Expression pExpr must be of type FTSQUERY_PHRASE.
 **
@@ -5801,6 +5786,7 @@ static int fts3EvalGatherStats(
  if( pExpr->aMI==0 ){
    Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab;
    Fts3Expr *pRoot;                /* Root of NEAR expression */
+    Fts3Expr *p;                    /* Iterator used for several purposes */

    sqlite3_int64 iPrevId = pCsr->iPrevId;
    sqlite3_int64 iDocid;
@@ -5808,9 +5794,7 @@ static int fts3EvalGatherStats(

    /* Find the root of the NEAR expression */
    pRoot = pExpr;
-    while( pRoot->pParent 
-        && (pRoot->pParent->eType==FTSQUERY_NEAR || pRoot->bDeferred) 
-    ){
+    while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){
      pRoot = pRoot->pParent;
    }
    iDocid = pRoot->iDocid;
@@ -5818,8 +5802,14 @@ static int fts3EvalGatherStats(
    assert( pRoot->bStart );

    /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */
-    rc = sqlite3Fts3ExprIterate(pRoot, fts3AllocateMSI, (void*)pTab);
-    if( rc!=SQLITE_OK ) return rc;
+    for(p=pRoot; p; p=p->pLeft){
+      Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight);
+      assert( pE->aMI==0 );
+      pE->aMI = (u32 *)sqlite3_malloc64(pTab->nColumn * 3 * sizeof(u32));
+      if( !pE->aMI ) return SQLITE_NOMEM;
+      memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32));
+    }
+
    fts3EvalRestart(pCsr, pRoot, &rc);

    while( pCsr->isEof==0 && rc==SQLITE_OK ){
@@ -5975,7 +5965,6 @@ int sqlite3Fts3EvalPhrasePoslist(
    u8 bTreeEof = 0;
    Fts3Expr *p;                  /* Used to iterate from pExpr to root */
    Fts3Expr *pNear;              /* Most senior NEAR ancestor (or pExpr) */
-    Fts3Expr *pRun;               /* Closest non-deferred ancestor of pNear */
    int bMatch;

    /* Check if this phrase descends from an OR expression node. If not, 
@@ -5990,30 +5979,25 @@ int sqlite3Fts3EvalPhrasePoslist(
      if( p->bEof ) bTreeEof = 1;
    }
    if( bOr==0 ) return SQLITE_OK;
-    pRun = pNear;
-    while( pRun->bDeferred ){
-      assert( pRun->pParent );
-      pRun = pRun->pParent;
-    }

    /* This is the descendent of an OR node. In this case we cannot use
    ** an incremental phrase. Load the entire doclist for the phrase
    ** into memory in this case.  */
    if( pPhrase->bIncr ){
-      int bEofSave = pRun->bEof;
-      fts3EvalRestart(pCsr, pRun, &rc);
-      while( rc==SQLITE_OK && !pRun->bEof ){
-        fts3EvalNextRow(pCsr, pRun, &rc);
-        if( bEofSave==0 && pRun->iDocid==iDocid ) break;
+      int bEofSave = pNear->bEof;
+      fts3EvalRestart(pCsr, pNear, &rc);
+      while( rc==SQLITE_OK && !pNear->bEof ){
+        fts3EvalNextRow(pCsr, pNear, &rc);
+        if( bEofSave==0 && pNear->iDocid==iDocid ) break;
      }
      assert( rc!=SQLITE_OK || pPhrase->bIncr==0 );
-      if( rc==SQLITE_OK && pRun->bEof!=bEofSave ){
+      if( rc==SQLITE_OK && pNear->bEof!=bEofSave ){
        rc = FTS_CORRUPT_VTAB;
      }
    }
    if( bTreeEof ){
-      while( rc==SQLITE_OK && !pRun->bEof ){
-        fts3EvalNextRow(pCsr, pRun, &rc);
+      while( rc==SQLITE_OK && !pNear->bEof ){
+        fts3EvalNextRow(pCsr, pNear, &rc);
      }
    }
    if( rc!=SQLITE_OK ) return rc;
@@ -650,7 +650,5 @@ int sqlite3FtsUnicodeIsalnum(int);
 int sqlite3FtsUnicodeIsdiacritic(int);
 #endif

-int sqlite3Fts3ExprIterate(Fts3Expr*, int (*x)(Fts3Expr*,int,void*), void*);
-
 #endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */
 #endif /* _FTSINT_H */
@@ -41,7 +41,7 @@ typedef sqlite3_int64 i64;


 /*
-** Used as an sqlite3Fts3ExprIterate() context when loading phrase doclists to
+** Used as an fts3ExprIterate() context when loading phrase doclists to
 ** Fts3Expr.aDoclist[]/nDoclist.
 */
 typedef struct LoadDoclistCtx LoadDoclistCtx;
@@ -85,7 +85,7 @@ struct SnippetFragment {
 };

 /*
-** This type is used as an sqlite3Fts3ExprIterate() context object while 
+** This type is used as an fts3ExprIterate() context object while 
 ** accumulating the data returned by the matchinfo() function.
 */
 typedef struct MatchInfo MatchInfo;
@@ -244,7 +244,7 @@ static void fts3GetDeltaPosition(char **pp, i64 *piPos){
 }

 /*
-** Helper function for sqlite3Fts3ExprIterate() (see below).
+** Helper function for fts3ExprIterate() (see below).
 */
 static int fts3ExprIterate2(
  Fts3Expr *pExpr,                /* Expression to iterate phrases of */
@@ -278,7 +278,7 @@ static int fts3ExprIterate2(
 ** Otherwise, SQLITE_OK is returned after a callback has been made for
 ** all eligible phrase nodes.
 */
-int sqlite3Fts3ExprIterate(
+static int fts3ExprIterate(
  Fts3Expr *pExpr,                /* Expression to iterate phrases of */
  int (*x)(Fts3Expr*,int,void*),  /* Callback function to invoke for phrases */
  void *pCtx                      /* Second argument to pass to callback */
@@ -287,9 +287,10 @@ int sqlite3Fts3ExprIterate(
  return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx);
 }

+
 /*
-** This is an sqlite3Fts3ExprIterate() callback used while loading the 
-** doclists for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
+** This is an fts3ExprIterate() callback used while loading the doclists
+** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also
 ** fts3ExprLoadDoclists().
 */
 static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
@@ -321,9 +322,9 @@ static int fts3ExprLoadDoclists(
  int *pnToken                    /* OUT: Number of tokens in query */
 ){
  int rc;                         /* Return Code */
-  LoadDoclistCtx sCtx = {0,0,0};  /* Context for sqlite3Fts3ExprIterate() */
+  LoadDoclistCtx sCtx = {0,0,0};  /* Context for fts3ExprIterate() */
  sCtx.pCsr = pCsr;
-  rc = sqlite3Fts3ExprIterate(pCsr->pExpr,fts3ExprLoadDoclistsCb,(void*)&sCtx);
+  rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx);
  if( pnPhrase ) *pnPhrase = sCtx.nPhrase;
  if( pnToken ) *pnToken = sCtx.nToken;
  return rc;
@@ -336,7 +337,7 @@ static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){
 }
 static int fts3ExprPhraseCount(Fts3Expr *pExpr){
  int nPhrase = 0;
-  (void)sqlite3Fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase);
+  (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase);
  return nPhrase;
 }

@@ -464,9 +465,8 @@ static void fts3SnippetDetails(
 }

 /*
-** This function is an sqlite3Fts3ExprIterate() callback used by
-** fts3BestSnippet().  Each invocation populates an element of the
-** SnippetIter.aPhrase[] array.
+** This function is an fts3ExprIterate() callback used by fts3BestSnippet().
+** Each invocation populates an element of the SnippetIter.aPhrase[] array.
 */
 static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){
  SnippetIter *p = (SnippetIter *)ctx;
@@ -556,9 +556,7 @@ static int fts3BestSnippet(
  sIter.nSnippet = nSnippet;
  sIter.nPhrase = nList;
  sIter.iCurrent = -1;
-  rc = sqlite3Fts3ExprIterate(
-      pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter
-  );
+  rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter);
  if( rc==SQLITE_OK ){

    /* Set the *pmSeen output variable. */
@@ -919,10 +917,10 @@ static int fts3ExprLHitGather(
 }

 /*
-** sqlite3Fts3ExprIterate() callback used to collect the "global" matchinfo
-** stats for a single query. 
+** fts3ExprIterate() callback used to collect the "global" matchinfo stats
+** for a single query. 
 **
-** sqlite3Fts3ExprIterate() callback to load the 'global' elements of a
+** fts3ExprIterate() callback to load the 'global' elements of a
 ** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements 
 ** of the matchinfo array that are constant for all rows returned by the 
 ** current query.
@@ -957,7 +955,7 @@ static int fts3ExprGlobalHitsCb(
 }

 /*
-** sqlite3Fts3ExprIterate() callback used to collect the "local" part of the
+** fts3ExprIterate() callback used to collect the "local" part of the
 ** FTS3_MATCHINFO_HITS array. The local stats are those elements of the 
 ** array that are different for each row returned by the query.
 */
@@ -1153,7 +1151,7 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){
  **/
  aIter = sqlite3Fts3MallocZero(sizeof(LcsIterator) * pCsr->nPhrase);
  if( !aIter ) return SQLITE_NOMEM;
-  (void)sqlite3Fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);
+  (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter);

  for(i=0; i<pInfo->nPhrase; i++){
    LcsIterator *pIter = &aIter[i];
@@ -1330,11 +1328,11 @@ static int fts3MatchinfoValues(
            rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc,0,0);
            if( rc!=SQLITE_OK ) break;
          }
-          rc = sqlite3Fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
+          rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo);
          sqlite3Fts3EvalTestDeferred(pCsr, &rc);
          if( rc!=SQLITE_OK ) break;
        }
-        (void)sqlite3Fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);
+        (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo);
        break;
      }
    }
@@ -1557,7 +1555,7 @@ struct TermOffsetCtx {
 };

 /*
-** This function is an sqlite3Fts3ExprIterate() callback used by sqlite3Fts3Offsets().
+** This function is an fts3ExprIterate() callback used by sqlite3Fts3Offsets().
 */
 static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){
  TermOffsetCtx *p = (TermOffsetCtx *)ctx;
@@ -1639,9 +1637,7 @@ void sqlite3Fts3Offsets(
    */
    sCtx.iCol = iCol;
    sCtx.iTerm = 0;
-    rc = sqlite3Fts3ExprIterate(
-        pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx
-    );
+    rc = fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx);
    if( rc!=SQLITE_OK ) goto offsets_out;

    /* Retreive the text stored in column iCol. If an SQL NULL is stored 
@@ -2667,18 +2667,16 @@ static int fts3MsrBufferData(
  char *pList,
  i64 nList
 ){
-  if( (nList+FTS3_NODE_PADDING)>pMsr->nBuffer ){
+  if( nList>pMsr->nBuffer ){
    char *pNew;
-    int nNew = nList*2 + FTS3_NODE_PADDING;
-    pNew = (char *)sqlite3_realloc64(pMsr->aBuffer, nNew);
+    pMsr->nBuffer = nList*2;
+    pNew = (char *)sqlite3_realloc64(pMsr->aBuffer, pMsr->nBuffer);
    if( !pNew ) return SQLITE_NOMEM;
    pMsr->aBuffer = pNew;
-    pMsr->nBuffer = nNew;
  }

  assert( nList>0 );
  memcpy(pMsr->aBuffer, pList, nList);
-  memset(&pMsr->aBuffer[nList], 0, FTS3_NODE_PADDING);
  return SQLITE_OK;
 }

@@ -163,7 +163,7 @@ static int fts5HighlightCb(
  if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK;
  iPos = p->iPos++;

-  if( p->iRangeEnd>=0 ){
+  if( p->iRangeEnd>0 ){
    if( iPos<p->iRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK;
    if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff;
  }
@@ -175,7 +175,7 @@ static int fts5HighlightCb(
  }

  if( iPos==p->iter.iEnd ){
-    if( p->iRangeEnd>=0 && p->iter.iStart<p->iRangeStart ){
+    if( p->iRangeEnd && p->iter.iStart<p->iRangeStart ){
      fts5HighlightAppend(&rc, p, p->zOpen, -1);
    }
    fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
@@ -186,7 +186,7 @@ static int fts5HighlightCb(
    }
  }

-  if( p->iRangeEnd>=0 && iPos==p->iRangeEnd ){
+  if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){
    fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
    p->iOff = iEndOff;
    if( iPos>=p->iter.iStart && iPos<p->iter.iEnd ){
@@ -221,7 +221,6 @@ static void fts5HighlightFunction(
  memset(&ctx, 0, sizeof(HighlightContext));
  ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
  ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
-  ctx.iRangeEnd = -1;
  rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn);

  if( ctx.zIn ){
@@ -407,7 +406,6 @@ static void fts5SnippetFunction(
  iCol = sqlite3_value_int(apVal[0]);
  ctx.zOpen = fts5ValueToText(apVal[1]);
  ctx.zClose = fts5ValueToText(apVal[2]);
-  ctx.iRangeEnd = -1;
  zEllips = fts5ValueToText(apVal[3]);
  nToken = sqlite3_value_int(apVal[4]);

@@ -289,19 +289,6 @@ int sqlite3Fts5ExprNew(
  return sParse.rc;
 }

-/*
-** Assuming that buffer z is at least nByte bytes in size and contains a
-** valid utf-8 string, return the number of characters in the string.
-*/
-static int fts5ExprCountChar(const char *z, int nByte){
-  int nRet = 0;
-  int ii;
-  for(ii=0; ii<nByte; ii++){
-    if( (z[ii] & 0xC0)!=0x80 ) nRet++;
-  }
-  return nRet;
-}
-
 /*
 ** This function is only called when using the special 'trigram' tokenizer.
 ** Argument zText contains the text of a LIKE or GLOB pattern matched
@@ -339,8 +326,7 @@ int sqlite3Fts5ExprPattern(
      if( i==nText 
       || zText[i]==aSpec[0] || zText[i]==aSpec[1] || zText[i]==aSpec[2] 
      ){
-
-        if( fts5ExprCountChar(&zText[iFirst], i-iFirst)>=3 ){
+        if( i-iFirst>=3 ){
          int jj;
          zExpr[iOut++] = '"';
          for(jj=iFirst; jj<i; jj++){
@@ -54,8 +54,6 @@
 # error "FTS5_MAX_PREFIX_INDEXES is too large"
 #endif

-#define FTS5_MAX_LEVEL 64
-
 /*
 ** Details:
 **
@@ -4770,10 +4768,10 @@ static Fts5Structure *fts5IndexOptimizeStruct(
  if( pNew ){
    Fts5StructureLevel *pLvl;
    nByte = nSeg * sizeof(Fts5StructureSegment);
-    pNew->nLevel = MIN(pStruct->nLevel+1, FTS5_MAX_LEVEL);
+    pNew->nLevel = pStruct->nLevel+1;
    pNew->nRef = 1;
    pNew->nWriteCounter = pStruct->nWriteCounter;
-    pLvl = &pNew->aLevel[pNew->nLevel-1];
+    pLvl = &pNew->aLevel[pStruct->nLevel];
    pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte);
    if( pLvl->aSeg ){
      int iLvl, iSeg;
@@ -5075,7 +5073,7 @@ static void fts5MergePrefixLists(
  /* Initialize a doclist-iterator for each input buffer. Arrange them in
  ** a linked-list starting at pHead in ascending order of rowid. Avoid
  ** linking any iterators already at EOF into the linked list at all. */ 
-  assert( nBuf+1<=(int)(sizeof(aMerger)/sizeof(aMerger[0])) );
+  assert( nBuf+1<=sizeof(aMerger)/sizeof(aMerger[0]) );
  memset(aMerger, 0, sizeof(PrefixMerger)*(nBuf+1));
  pHead = &aMerger[nBuf];
  fts5DoclistIterInit(p1, &pHead->iter);
@@ -6280,7 +6278,6 @@ int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){

    /* If this is a new term, query for it. Update cksum3 with the results. */
    fts5TestTerm(p, &term, z, n, cksum2, &cksum3);
-    if( p->rc ) break;

    if( eDetail==FTS5_DETAIL_NONE ){
      if( 0==fts5MultiIterIsEmpty(p, pIter) ){
@@ -260,7 +260,7 @@ static void fts5CheckTransactionState(Fts5FullTable *p, int op, int iSavepoint){
      break;

    case FTS5_SYNC:
-      assert( p->ts.eState==1 || p->ts.eState==2 );
+      assert( p->ts.eState==1 );
      p->ts.eState = 2;
      break;

@@ -275,21 +275,21 @@ static void fts5CheckTransactionState(Fts5FullTable *p, int op, int iSavepoint){
      break;

    case FTS5_SAVEPOINT:
-      assert( p->ts.eState>=1 );
+      assert( p->ts.eState==1 );
      assert( iSavepoint>=0 );
      assert( iSavepoint>=p->ts.iSavepoint );
      p->ts.iSavepoint = iSavepoint;
      break;
      
    case FTS5_RELEASE:
-      assert( p->ts.eState>=1 );
+      assert( p->ts.eState==1 );
      assert( iSavepoint>=0 );
      assert( iSavepoint<=p->ts.iSavepoint );
      p->ts.iSavepoint = iSavepoint-1;
      break;

    case FTS5_ROLLBACKTO:
-      assert( p->ts.eState>=1 );
+      assert( p->ts.eState==1 );
      assert( iSavepoint>=-1 );
      /* The following assert() can fail if another vtab strikes an error
      ** within an xSavepoint() call then SQLite calls xRollbackTo() - without
@@ -1625,7 +1625,7 @@ static int fts5UpdateMethod(
  int rc = SQLITE_OK;             /* Return code */

  /* A transaction must be open when this is called. */
-  assert( pTab->ts.eState==1 || pTab->ts.eState==2 );
+  assert( pTab->ts.eState==1 );

  assert( pVtab->zErrMsg==0 );
  assert( nArg==1 || nArg==(2+pConfig->nCol+2) );
@@ -2866,9 +2866,7 @@ static int fts5Init(sqlite3 *db){
    }
    if( rc==SQLITE_OK ){
      rc = sqlite3_create_function(
-          db, "fts5_source_id", 0, 
-          SQLITE_UTF8|SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS,
-          p, fts5SourceIdFunc, 0, 0
+          db, "fts5_source_id", 0, SQLITE_UTF8, p, fts5SourceIdFunc, 0, 0
      );
    }
  }
@@ -193,34 +193,4 @@ do_execsql_test 5.6 {

 } ;# foreach_detail_mode 

-reset_db
-do_execsql_test 6.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(colA, colB);
-  INSERT INTO t1 VALUES('A B C', 'D E F');
-}
-
-do_execsql_test 6.1 {
-  SELECT colA, colB, snippet(t1,0,'[', ']','...',1) FROM t1 WHERE t1 MATCH 'B';
-} {{A B C} {D E F} ...[B]...}
-breakpoint
-do_execsql_test 6.2 {
-  SELECT colA, colB, snippet(t1, 1,'[',']','...',2) FROM t1 WHERE t1 MATCH 'B';
-} {{A B C} {D E F} {D E...}}
-do_execsql_test 6.3 {
-  SELECT colA, colB, snippet(t1, 1,'[',']','...',1) FROM t1 WHERE t1 MATCH 'B';
-} {{A B C} {D E F} {D...}}
-
-do_execsql_test 6.1 {
-  SELECT colA, colB, snippet(t1,0,'[', ']','...',1) FROM t1 WHERE t1 MATCH 'A';
-} {{A B C} {D E F} [A]...}
-breakpoint
-do_execsql_test 6.2 {
-  SELECT colA, colB, snippet(t1, 1,'[',']','...',2) FROM t1 WHERE t1 MATCH 'A';
-} {{A B C} {D E F} {D E...}}
-do_execsql_test 6.3 {
-  SELECT colA, colB, snippet(t1, 1,'[',']','...',1) FROM t1 WHERE t1 MATCH 'A';
-} {{A B C} {D E F} {D...}}
-
-
-
 finish_test
@@ -154,30 +154,4 @@ do_execsql_test 3.2 {

 }

-# 2023-04-06 https://sqlite.org/forum/forumpost/cae4367d9b
-#
-# This is not a test of FTS5, but rather a test of the of what happens to
-# prepared statements that encounter SQLITE_SCHEMA while other prepared
-# statements are running.  The original problem POC used FTS5, and so
-# is seems reasonable to put the test here.
-#
-# The vdbeaux24.test module in TH3 also tests this same behavior but
-# without requiring FTS5 or an other extension.
-#
-reset_db
-db null NULL
-do_execsql_test 4.0 {
-  CREATE TABLE t5(a PRIMARY KEY);
-  INSERT INTO t5 VALUES(0);
-  CREATE VIRTUAL TABLE t6 USING fts5(0);
-  DELETE FROM t6;
-  CREATE TABLE t7(x);
-  WITH cte(a) AS (
-     SELECT a FROM t5
-      WHERE ((0,0) IN (SELECT 0, LAG(0) OVER (PARTITION BY 0) FROM t6), 0)
-               < (a,0)
-  )
-  SELECT max(a) FROM cte;
-} NULL
-
 finish_test
@@ -352,96 +352,5 @@ do_test 13.3 {
  sqlite3_errmsg db
 } {not an error}

-#-------------------------------------------------------------------------
-reset_db
-db close
-sqlite3 db test.db -uri 1
-
-do_execsql_test 14.0 {
-  PRAGMA locking_mode=EXCLUSIVE;
-  BEGIN;
-  ATTACH 'file:/one?vfs=memdb' AS aux1;
-  ATTACH 'file:/one?vfs=memdb' AS aux2;
-  CREATE VIRTUAL TABLE t1 USING fts5(x);
-} {exclusive}
-do_catchsql_test 14.1 {
-  ANALYZE;
-} {1 {database is locked}}
-do_catchsql_test 14.2 {
-  COMMIT;
-} {1 {database is locked}}
-do_catchsql_test 14.3 {
-  COMMIT;
-} {1 {database is locked}}
-do_catchsql_test 14.4 {
-  ROLLBACK;
-} {0 {}}
-
-#-------------------------------------------------------------------------
-reset_db
-sqlite3 db2 test.db
-
-do_execsql_test 15.0 {
-  CREATE TABLE t1(a, b);
-  BEGIN;
-    SELECT * FROM t1;
-}
-
-do_execsql_test -db db2 15.1 {
-  BEGIN; 
-    CREATE VIRTUAL TABLE x1 USING fts5(y);
-}
-do_test 15.2 {
-  list [catch { db2 eval COMMIT } msg] $msg
-} {1 {database is locked}}
-do_execsql_test -db db2 15.3 {
-  SAVEPOINT one;
-} {}
-do_execsql_test 15.4 END
-do_test 15.4 {
-  list [catch { db2 eval COMMIT } msg] $msg
-} {0 {}}
-
-#-------------------------------------------------------------------------
-reset_db
-forcedelete test.db2
-sqlite3 db2 test.db
-do_execsql_test 16.0 {
-
-  ATTACH 'test.db2' AS aux;
-  CREATE TABLE aux.t2(x,y);
-  INSERT INTO t2 VALUES(1, 2);
-  CREATE VIRTUAL TABLE x1 USING fts5(a);
-  BEGIN;
-    INSERT INTO x1 VALUES('abc');
-    INSERT INTO t2 VALUES(3, 4);
-}
-
-do_execsql_test -db db2 16.1 {
-  ATTACH 'test.db2' AS aux;
-  BEGIN;
-    SELECT * FROM t2
-} {1 2}
-
-do_catchsql_test 16.2 {
-  COMMIT;
-} {1 {database is locked}}
-
-do_execsql_test 16.3 {
-  INSERT INTO x1 VALUES('def');
-}
-
-do_execsql_test -db db2 16.4 { 
-  END 
-}
-
-do_execsql_test 16.5 {
-  COMMIT
-}
-
-do_execsql_test -db db2 16.6 {
-  SELECT * FROM x1
-} {abc def}
-
 finish_test

@@ -1,64 +0,0 @@
-# 2014 Dec 20
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-#
-# TESTRUNNER: slow
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5optimize2
-
-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-
-set nLoop 2500
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x);
-  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
-}
-
-do_test 1.1 {
-  for {set ii 0} {$ii < $nLoop} {incr ii} {
-    execsql {
-      INSERT INTO t1 VALUES('abc def ghi');
-      INSERT INTO t1 VALUES('jkl mno pqr');
-      INSERT INTO t1(t1) VALUES('optimize');
-    }
-  }
-} {}
-
-do_execsql_test 1.2 {
-  SELECT count(*) FROM t1('mno')
-} $nLoop
-
-do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE t2 USING fts5(x);
-  INSERT INTO t2(t2, rank) VALUES('pgsz', 32);
-}
-
-do_test 2.1 {
-  for {set ii 0} {$ii < $nLoop} {incr ii} {
-    execsql {
-      INSERT INTO t2 VALUES('abc def ghi');
-      INSERT INTO t2 VALUES('jkl mno pqr');
-      INSERT INTO t2(t2, rank) VALUES('merge', -1);
-    }
-  }
-} {}
-
-do_execsql_test 2.2 {
-  SELECT count(*) FROM t2('mno')
-} $nLoop
-
-finish_test
@@ -55,7 +55,6 @@ foreach {tn like res} {
  6 {abc%klm}  1
  7 {ABCDEFG%} 1
  8 {%รุงเ%}    2
-  9 {%งเ%}     2
 } {
  do_execsql_test 1.3.$tn {
    SELECT rowid FROM t1 WHERE y LIKE $like
@@ -198,21 +197,4 @@ do_eqp_test 6.4 {
  SELECT * FROM ci1 WHERE x GLOB ?
 } {VIRTUAL TABLE INDEX 0:G0}

-reset_db
-do_execsql_test 7.0 {
-  CREATE VIRTUAL TABLE f USING FTS5(filename, tokenize="trigram");
-  INSERT INTO f (rowid, filename) VALUES 
-      (10, "giraffe.png"), 
-      (20, "жираф.png"), 
-      (30, "cat.png"), 
-      (40, "кот.png"), 
-      (50, "misic-🎵-.mp3");
-}
-do_execsql_test 7.1 {
-  SELECT rowid FROM f WHERE +filename GLOB '*ир*';
-} {20}
-do_execsql_test 7.2 {
-  SELECT rowid FROM f WHERE filename GLOB '*ир*';
-} {20}
-
 finish_test
@@ -299,9 +299,8 @@ static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){

    if( U_SUCCESS(status) ){
      sqlite3_set_auxdata(p, 0, pExpr, icuRegexpDelete);
-      pExpr = sqlite3_get_auxdata(p, 0);
-    }
-    if( !pExpr ){
+    }else{
+      assert(!pExpr);
      icuFunctionError(p, "uregex_open", status);
      return;
    }
@@ -44,8 +44,6 @@
 # endif
 #endif

-/* #define LSM_DEBUG_EXPENSIVE 1 */
-
 /*
 ** Default values for various data structure parameters. These may be
 ** overridden by calls to lsm_config().
@@ -407,7 +405,7 @@ struct Segment {
  LsmPgno iFirst;                  /* First page of this run */
  LsmPgno iLastPg;                 /* Last page of this run */
  LsmPgno iRoot;                   /* Root page number (if any) */
-  LsmPgno nSize;                   /* Size of this run in pages */
+  int nSize;                       /* Size of this run in pages */

  Redirect *pRedirect;             /* Block redirects (or NULL) */
 };
@@ -855,8 +853,6 @@ int lsmVarintGet32(u8 *, int *);
 int lsmVarintPut64(u8 *aData, i64 iVal);
 int lsmVarintGet64(const u8 *aData, i64 *piVal);

-int lsmVarintLen64(i64);
-
 int lsmVarintLen32(int);
 int lsmVarintSize(u8 c);

@@ -511,7 +511,7 @@ static void ckptNewSegment(
  pSegment->iFirst = ckptGobble64(aIn, piIn);
  pSegment->iLastPg = ckptGobble64(aIn, piIn);
  pSegment->iRoot = ckptGobble64(aIn, piIn);
-  pSegment->nSize = ckptGobble64(aIn, piIn);
+  pSegment->nSize = (int)ckptGobble64(aIn, piIn);
  assert( pSegment->iFirst );
 }

@@ -215,8 +215,8 @@ struct FileSystem {
  char *zLog;                     /* Database file name */
  int nMetasize;                  /* Size of meta pages in bytes */
  int nMetaRwSize;                /* Read/written size of meta pages in bytes */
-  i64 nPagesize;                  /* Database page-size in bytes */
-  i64 nBlocksize;                 /* Database block-size in bytes */
+  int nPagesize;                  /* Database page-size in bytes */
+  int nBlocksize;                 /* Database block-size in bytes */

  /* r/w file descriptors for both files. */
  LsmFile *pLsmFile;              /* Used after lsm_close() to link into list */
@@ -889,7 +889,7 @@ static LsmPgno fsFirstPageOnBlock(FileSystem *pFS, int iBlock){
      iPg = pFS->nBlocksize * (LsmPgno)(iBlock-1) + 4;
    }
  }else{
-    const i64 nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
+    const int nPagePerBlock = (pFS->nBlocksize / pFS->nPagesize);
    if( iBlock==1 ){
      iPg = 1 + ((pFS->nMetasize*2 + pFS->nPagesize - 1) / pFS->nPagesize);
    }else{
@@ -1131,6 +1131,7 @@ static void fsGrowMapping(
  i64 iSz,                        /* Minimum size to extend mapping to */
  int *pRc                        /* IN/OUT: Error code */
 ){
+  assert( pFS->pCompress==0 );
  assert( PAGE_HASPREV==4 );

  if( *pRc==LSM_OK && iSz>pFS->nMap ){
@@ -1871,7 +1872,7 @@ void lsmFsGobble(
  assert( nPgno>0 && 0==fsPageRedirects(pFS, pRun, aPgno[0]) );

  iBlk = fsPageToBlock(pFS, pRun->iFirst);
-  pRun->nSize += (pRun->iFirst - fsFirstPageOnBlock(pFS, iBlk));
+  pRun->nSize += (int)(pRun->iFirst - fsFirstPageOnBlock(pFS, iBlk));

  while( rc==LSM_OK ){
    int iNext = 0;
@@ -1882,13 +1883,13 @@ void lsmFsGobble(
    }
    rc = fsBlockNext(pFS, pRun, iBlk, &iNext);
    if( rc==LSM_OK ) rc = fsFreeBlock(pFS, pSnapshot, pRun, iBlk);
-    pRun->nSize -= (
+    pRun->nSize -= (int)(
        1 + fsLastPageOnBlock(pFS, iBlk) - fsFirstPageOnBlock(pFS, iBlk)
    );
    iBlk = iNext;
  }

-  pRun->nSize -= (pRun->iFirst - fsFirstPageOnBlock(pFS, iBlk));
+  pRun->nSize -= (int)(pRun->iFirst - fsFirstPageOnBlock(pFS, iBlk));
  assert( pRun->nSize>0 );
 }

@@ -432,7 +432,7 @@ int lsm_config(lsm_db *pDb, int eParam, ...){
 }

 void lsmAppendSegmentList(LsmString *pStr, char *zPre, Segment *pSeg){
-  lsmStringAppendf(pStr, "%s{%lld %lld %lld %lld}", zPre, 
+  lsmStringAppendf(pStr, "%s{%d %d %d %d}", zPre, 
        pSeg->iFirst, pSeg->iLastPg, pSeg->iRoot, pSeg->nSize
  );
 }
@@ -1306,24 +1306,6 @@ int lsmBeginRoTrans(lsm_db *db){
      }
    }

-    /* In 'lsm_open()' we don't update the page and block sizes in the
-    ** Filesystem for 'readonly' connection. Because member 'db->pShmhdr' is a
-    ** nullpointer, this prevents loading a checkpoint. Now that the system is 
-    ** live this member should be set. So we can update both values in 
-    ** the Filesystem.
-    **
-    ** Configure the file-system connection with the page-size and block-size
-    ** of this database. Even if the database file is zero bytes in size
-    ** on disk, these values have been set in shared-memory by now, and so
-    ** are guaranteed not to change during the lifetime of this connection. */
-    if( LSM_OK==rc
-     && 0==lsmCheckpointClientCacheOk(db)
-     && LSM_OK==(rc=lsmCheckpointLoad(db, 0)) 
-    ){
-      lsmFsSetPageSize(db->pFS, lsmCheckpointPgsz(db->aSnapshot));
-      lsmFsSetBlockSize(db->pFS, lsmCheckpointBlksz(db->aSnapshot));
-    }
-
    if( rc==LSM_OK ){
      rc = lsmBeginReadTrans(db);
    }
@@ -525,7 +525,7 @@ static u8 *pageGetKey(
  LsmBlob *pBlob                  /* If required, use this for dynamic memory */
 ){
  u8 *pKey;
-  i64 nDummy;
+  int nDummy;
  int eType;
  u8 *aData;
  int nData;
@@ -537,10 +537,10 @@ static u8 *pageGetKey(

  pKey = pageGetCell(aData, nData, iCell);
  eType = *pKey++;
-  pKey += lsmVarintGet64(pKey, &nDummy);
+  pKey += lsmVarintGet32(pKey, &nDummy);
  pKey += lsmVarintGet32(pKey, pnKey);
  if( rtIsWrite(eType) ){
-    pKey += lsmVarintGet64(pKey, &nDummy);
+    pKey += lsmVarintGet32(pKey, &nDummy);
  }
  *piTopic = rtTopic(eType);

@@ -657,14 +657,14 @@ static int btreeCursorLoadKey(BtreeCursor *pCsr){
  return rc;
 }

-static LsmPgno btreeCursorPtr(u8 *aData, int nData, int iCell){
+static int btreeCursorPtr(u8 *aData, int nData, int iCell){
  int nCell;

  nCell = pageGetNRec(aData, nData);
  if( iCell>=nCell ){
-    return pageGetPtr(aData, nData);
+    return (int)pageGetPtr(aData, nData);
  }
-  return pageGetRecordPtr(aData, nData, iCell);
+  return (int)pageGetRecordPtr(aData, nData, iCell);
 }

 static int btreeCursorNext(BtreeCursor *pCsr){
@@ -751,7 +751,7 @@ static int btreeCursorFirst(BtreeCursor *pCsr){

  Page *pPg = 0;
  FileSystem *pFS = pCsr->pFS;
-  LsmPgno iPg = pCsr->pSeg->iRoot;
+  int iPg = (int)pCsr->pSeg->iRoot;

  do {
    rc = lsmFsDbPageGet(pFS, pCsr->pSeg, iPg, &pPg);
@@ -779,7 +779,7 @@ static int btreeCursorFirst(BtreeCursor *pCsr){
        assert( pCsr->aPg[pCsr->nDepth].iCell==0 );
        pCsr->aPg[pCsr->nDepth].pPage = pPg;
        pCsr->nDepth++;
-        iPg = pageGetRecordPtr(aData, nData, 0);
+        iPg = (int)pageGetRecordPtr(aData, nData, 0);
      }
    }
  }while( rc==LSM_OK );
@@ -871,7 +871,7 @@ static int btreeCursorRestore(
      int nSeek;
      int iTopicSeek;
      int iPg = 0;
-      LsmPgno iLoad = pSeg->iRoot;
+      int iLoad = (int)pSeg->iRoot;
      Page *pPg = pCsr->aPg[nDepth-1].pPage;
 
      if( pageObjGetNRec(pPg)==0 ){
@@ -903,7 +903,7 @@ static int btreeCursorRestore(
          aData = fsPageData(pPg2, &nData);
          assert( (pageGetFlags(aData, nData) & SEGMENT_BTREE_FLAG) );

-          iLoad = pageGetPtr(aData, nData);
+          iLoad = (int)pageGetPtr(aData, nData);
          iCell2 = pageGetNRec(aData, nData); 
          iMax = iCell2-1;
          iMin = 0;
@@ -926,7 +926,7 @@ static int btreeCursorRestore(
            assert( res!=0 );

            if( res<0 ){
-              iLoad = iPtr;
+              iLoad = (int)iPtr;
              iCell2 = iTry;
              iMax = iTry-1;
            }else{
@@ -1013,7 +1013,7 @@ static void segmentPtrSetPage(SegmentPtr *pPtr, Page *pNext){
 static int segmentPtrLoadPage(
  FileSystem *pFS,
  SegmentPtr *pPtr,              /* Load page into this SegmentPtr object */
-  LsmPgno iNew                       /* Page number of new page */
+  int iNew                       /* Page number of new page */
 ){
  Page *pPg = 0;                 /* The new page */
  int rc;                        /* Return Code */
@@ -1633,7 +1633,7 @@ static int segmentPtrSeek(
  int iTopic,                     /* Key topic to seek to */
  void *pKey, int nKey,           /* Key to seek to */
  int eSeek,                      /* Search bias - see above */
-  LsmPgno *piPtr,                 /* OUT: FC pointer */
+  int *piPtr,                     /* OUT: FC pointer */
  int *pbStop
 ){
  int (*xCmp)(void *, int, void *, int) = pCsr->pDb->xCmp;
@@ -1759,7 +1759,7 @@ static int segmentPtrSeek(
  }

  assert( rc!=LSM_OK || assertSeekResult(pCsr,pPtr,iTopic,pKey,nKey,eSeek) );
-  *piPtr = iPtrOut;
+  *piPtr = (int)iPtrOut;
  return rc;
 }

@@ -1773,11 +1773,11 @@ static int seekInBtree(
 ){
  int i = 0;
  int rc;
-  LsmPgno iPg;
+  int iPg;
  Page *pPg = 0;
  LsmBlob blob = {0, 0, 0};

-  iPg = pSeg->iRoot;
+  iPg = (int)pSeg->iRoot;
  do {
    LsmPgno *piFirst = 0;
    if( aPg ){
@@ -1799,7 +1799,7 @@ static int seekInBtree(
      flags = pageGetFlags(aData, nData);
      if( (flags & SEGMENT_BTREE_FLAG)==0 ) break;

-      iPg = pageGetPtr(aData, nData);
+      iPg = (int)pageGetPtr(aData, nData);
      nRec = pageGetNRec(aData, nData);

      iMin = 0;
@@ -1825,7 +1825,7 @@ static int seekInBtree(
            pCsr->pDb->xCmp, iTopic, pKey, nKey, iTopicT, pKeyT, nKeyT
        );
        if( res<0 ){
-          iPg = iPtr;
+          iPg = (int)iPtr;
          iMax = iTry-1;
        }else{
          iMin = iTry+1;
@@ -1851,12 +1851,12 @@ static int seekInSegment(
  SegmentPtr *pPtr,
  int iTopic,
  void *pKey, int nKey,
-  LsmPgno iPg,                    /* Page to search */
+  int iPg,                        /* Page to search */
  int eSeek,                      /* Search bias - see above */
-  LsmPgno *piPtr,                 /* OUT: FC pointer */
+  int *piPtr,                     /* OUT: FC pointer */
  int *pbStop                     /* OUT: Stop search flag */
 ){
-  LsmPgno iPtr = iPg;
+  int iPtr = iPg;
  int rc = LSM_OK;

  if( pPtr->pSeg->iRoot ){
@@ -1866,7 +1866,7 @@ static int seekInSegment(
    if( rc==LSM_OK ) segmentPtrSetPage(pPtr, pPg);
  }else{
    if( iPtr==0 ){
-      iPtr = pPtr->pSeg->iFirst;
+      iPtr = (int)pPtr->pSeg->iFirst;
    }
    if( rc==LSM_OK ){
      rc = segmentPtrLoadPage(pCsr->pDb->pFS, pPtr, iPtr);
@@ -1904,7 +1904,7 @@ static int seekInLevel(
 ){
  Level *pLvl = aPtr[0].pLevel;   /* Level to seek within */
  int rc = LSM_OK;                /* Return code */
-  LsmPgno iOut = 0;               /* Pointer to return to caller */
+  int iOut = 0;                   /* Pointer to return to caller */
  int res = -1;                   /* Result of xCmp(pKey, split) */
  int nRhs = pLvl->nRight;        /* Number of right-hand-side segments */
  int bStop = 0;
@@ -1923,8 +1923,8 @@ static int seekInLevel(
  ** left-hand-side of the level in this case.  */
  if( res<0 ){
    int i;
-    LsmPgno iPtr = 0;
-    if( nRhs==0 ) iPtr = *piPgno;
+    int iPtr = 0;
+    if( nRhs==0 ) iPtr = (int)*piPgno;

    rc = seekInSegment(
        pCsr, &aPtr[0], iTopic, pKey, nKey, iPtr, eSeek, &iOut, &bStop
@@ -1939,7 +1939,7 @@ static int seekInLevel(
  
  if( res>=0 ){
    int bHit = 0;                 /* True if at least one rhs is not EOF */
-    LsmPgno iPtr = *piPgno;
+    int iPtr = (int)*piPgno;
    int i;
    segmentPtrReset(&aPtr[0], LSM_SEGMENTPTR_FREE_THRESHOLD);
    for(i=1; rc==LSM_OK && i<=nRhs && bStop==0; i++){
@@ -3361,8 +3361,6 @@ static int mergeWorkerPageOffset(u8 *aData, int nData){
  int iOff;
  int nKey;
  int eType;
-  i64 nDummy;
-

  nRec = lsmGetU16(&aData[SEGMENT_NRECORD_OFFSET(nData)]);
  iOff = lsmGetU16(&aData[SEGMENT_CELLPTR_OFFSET(nData, nRec-1)]);
@@ -3372,7 +3370,7 @@ static int mergeWorkerPageOffset(u8 *aData, int nData){
       || eType==(LSM_SEPARATOR)
  );

-  iOff += lsmVarintGet64(&aData[iOff], &nDummy);
+  iOff += lsmVarintGet32(&aData[iOff], &nKey);
  iOff += lsmVarintGet32(&aData[iOff], &nKey);

  return iOff + (eType ? nKey : 0);
@@ -3482,7 +3480,7 @@ static int mergeWorkerLoadHierarchy(MergeWorker *pMW){
    lsm_env *pEnv = pMW->pDb->pEnv;
    Page **apHier = 0;
    int nHier = 0;
-    LsmPgno iPg = pSeg->iRoot;
+    int iPg = (int)pSeg->iRoot;

    do {
      Page *pPg = 0;
@@ -3508,7 +3506,7 @@ static int mergeWorkerLoadHierarchy(MergeWorker *pMW){
        nHier++;

        apHier[0] = pPg;
-        iPg = pageGetPtr(aData, nData);
+        iPg = (int)pageGetPtr(aData, nData);
      }else{
        lsmFsPageRelease(pPg);
        break;
@@ -3627,11 +3625,10 @@ static int mergeWorkerBtreeWrite(
      assert( lsmFsPageWritable(pOld) );
      aData = fsPageData(pOld, &nData);
      if( eType==0 ){
-        nByte = 2 + 1 + lsmVarintLen64(iPtr) + lsmVarintLen64(iKeyPg);
+        nByte = 2 + 1 + lsmVarintLen32((int)iPtr) + lsmVarintLen32((int)iKeyPg);
      }else{
-        nByte = 2 + 1 + lsmVarintLen64(iPtr) + lsmVarintLen32(nKey) + nKey;
+        nByte = 2 + 1 + lsmVarintLen32((int)iPtr) + lsmVarintLen32(nKey) + nKey;
      }
-
      nRec = pageGetNRec(aData, nData);
      nFree = SEGMENT_EOF(nData, nRec) - mergeWorkerPageOffset(aData, nData);
      if( nByte<=nFree ) break;
@@ -3675,11 +3672,11 @@ static int mergeWorkerBtreeWrite(
  lsmPutU16(&aData[SEGMENT_NRECORD_OFFSET(nData)], (u16)(nRec+1));
  if( eType==0 ){
    aData[iOff++] = 0x00;
-    iOff += lsmVarintPut64(&aData[iOff], iPtr);
-    iOff += lsmVarintPut64(&aData[iOff], iKeyPg);
+    iOff += lsmVarintPut32(&aData[iOff], (int)iPtr);
+    iOff += lsmVarintPut32(&aData[iOff], (int)iKeyPg);
  }else{
    aData[iOff++] = eType;
-    iOff += lsmVarintPut64(&aData[iOff], iPtr);
+    iOff += lsmVarintPut32(&aData[iOff], (int)iPtr);
    iOff += lsmVarintPut32(&aData[iOff], nKey);
    memcpy(&aData[iOff], pKey, nKey);
  }
@@ -3875,7 +3872,7 @@ static int mergeWorkerNextPage(
 static int mergeWorkerData(
  MergeWorker *pMW,               /* Merge worker object */
  int bSep,                       /* True to write to separators run */
-  LsmPgno iFPtr,                  /* Footer ptr for new pages */
+  int iFPtr,                      /* Footer ptr for new pages */
  u8 *aWrite,                     /* Write data from this buffer */
  int nWrite                      /* Size of aWrite[] in bytes */
 ){
@@ -3919,14 +3916,14 @@ static int mergeWorkerData(
 static int mergeWorkerFirstPage(MergeWorker *pMW){
  int rc = LSM_OK;                /* Return code */
  Page *pPg = 0;                  /* First page of run pSeg */
-  LsmPgno iFPtr = 0;              /* Pointer value read from footer of pPg */
+  int iFPtr = 0;                  /* Pointer value read from footer of pPg */
  MultiCursor *pCsr = pMW->pCsr;

  assert( pMW->pPage==0 );

  if( pCsr->pBtCsr ){
    rc = LSM_OK;
-    iFPtr = pMW->pLevel->pNext->lhs.iFirst;
+    iFPtr = (int)pMW->pLevel->pNext->lhs.iFirst;
  }else if( pCsr->nPtr>0 ){
    Segment *pSeg;
    pSeg = pCsr->aPtr[pCsr->nPtr-1].pSeg;
@@ -3935,7 +3932,7 @@ static int mergeWorkerFirstPage(MergeWorker *pMW){
      u8 *aData;                    /* Buffer for page pPg */
      int nData;                    /* Size of aData[] in bytes */
      aData = fsPageData(pPg, &nData);
-      iFPtr = pageGetPtr(aData, nData);
+      iFPtr = (int)pageGetPtr(aData, nData);
      lsmFsPageRelease(pPg);
    }
  }
@@ -3954,7 +3951,7 @@ static int mergeWorkerWrite(
  int eType,                      /* One of SORTED_SEPARATOR, WRITE or DELETE */
  void *pKey, int nKey,           /* Key value */
  void *pVal, int nVal,           /* Value value */
-  LsmPgno iPtr                    /* Absolute value of page pointer, or 0 */
+  int iPtr                        /* Absolute value of page pointer, or 0 */
 ){
  int rc = LSM_OK;                /* Return code */
  Merge *pMerge;                  /* Persistent part of level merge state */
@@ -3963,8 +3960,8 @@ static int mergeWorkerWrite(
  u8 *aData;                      /* Data buffer for page pWriter->pPage */
  int nData = 0;                  /* Size of buffer aData[] in bytes */
  int nRec = 0;                   /* Number of records on page pPg */
-  LsmPgno iFPtr = 0;              /* Value of pointer in footer of pPg */
-  LsmPgno iRPtr = 0;              /* Value of pointer written into record */
+  int iFPtr = 0;                  /* Value of pointer in footer of pPg */
+  int iRPtr = 0;                  /* Value of pointer written into record */
  int iOff = 0;                   /* Current write offset within page pPg */
  Segment *pSeg;                  /* Segment being written */
  int flags = 0;                  /* If != 0, flags value for page footer */
@@ -3981,8 +3978,8 @@ static int mergeWorkerWrite(
  if( pPg ){
    aData = fsPageData(pPg, &nData);
    nRec = pageGetNRec(aData, nData);
-    iFPtr = pageGetPtr(aData, nData);
-    iRPtr = iPtr ? (iPtr - iFPtr) : 0;
+    iFPtr = (int)pageGetPtr(aData, nData);
+    iRPtr = iPtr - iFPtr;
  }
     
  /* Figure out how much space is required by the new record. The space
@@ -4000,7 +3997,7 @@ static int mergeWorkerWrite(
  **     4) Value size - 1 varint (only if LSM_INSERT flag is set)
  */
  if( rc==LSM_OK ){
-    nHdr = 1 + lsmVarintLen64(iRPtr) + lsmVarintLen32(nKey);
+    nHdr = 1 + lsmVarintLen32(iRPtr) + lsmVarintLen32(nKey);
    if( rtIsWrite(eType) ) nHdr += lsmVarintLen32(nVal);

    /* If the entire header will not fit on page pPg, or if page pPg is 
@@ -4012,8 +4009,8 @@ static int mergeWorkerWrite(
        assert( aData );
        memset(&aData[iOff], 0, SEGMENT_EOF(nData, nRec)-iOff);
      }
-      iFPtr = *pMW->pCsr->pPrevMergePtr;
-      iRPtr = iPtr ? (iPtr - iFPtr) : 0;
+      iFPtr = (int)*pMW->pCsr->pPrevMergePtr;
+      iRPtr = iPtr - iFPtr;
      iOff = 0;
      nRec = 0;
      rc = mergeWorkerNextPage(pMW, iFPtr);
@@ -4052,7 +4049,7 @@ static int mergeWorkerWrite(

    /* Write the entry header into the current page. */
    aData[iOff++] = (u8)eType;                                           /* 1 */
-    iOff += lsmVarintPut64(&aData[iOff], iRPtr);                         /* 2 */
+    iOff += lsmVarintPut32(&aData[iOff], iRPtr);                         /* 2 */
    iOff += lsmVarintPut32(&aData[iOff], nKey);                          /* 3 */
    if( rtIsWrite(eType) ) iOff += lsmVarintPut32(&aData[iOff], nVal);   /* 4 */
    pMerge->iOutputOff = iOff;
@@ -4286,7 +4283,7 @@ static int mergeWorkerStep(MergeWorker *pMW){
        pVal = pCsr->val.pData;
      }
      if( rc==LSM_OK ){
-        rc = mergeWorkerWrite(pMW, eType, pKey, nKey, pVal, nVal, iPtr);
+        rc = mergeWorkerWrite(pMW, eType, pKey, nKey, pVal, nVal, (int)iPtr);
      }
    }
  }
@@ -4607,7 +4604,7 @@ static int mergeWorkerInit(
          SegmentPtr *pPtr;
          assert( pCsr->aPtr[i].pPg==0 );
          pPtr = &pCsr->aPtr[i];
-          rc = segmentPtrLoadPage(pDb->pFS, pPtr, pInput->iPg);
+          rc = segmentPtrLoadPage(pDb->pFS, pPtr, (int)pInput->iPg);
          if( rc==LSM_OK && pPtr->nCell>0 ){
            rc = segmentPtrLoadCell(pPtr, pInput->iCell);
          }
@@ -5472,7 +5469,7 @@ int lsmFlushTreeToDisk(lsm_db *pDb){
 ** be freed by the caller using lsmFree().
 */
 static char *segToString(lsm_env *pEnv, Segment *pSeg, int nMin){
-  LsmPgno nSize = pSeg->nSize;
+  int nSize = pSeg->nSize;
  LsmPgno iRoot = pSeg->iRoot;
  LsmPgno iFirst = pSeg->iFirst;
  LsmPgno iLast = pSeg->iLastPg;
@@ -5484,9 +5481,9 @@ static char *segToString(lsm_env *pEnv, Segment *pSeg, int nMin){

  z1 = lsmMallocPrintf(pEnv, "%d.%d", iFirst, iLast);
  if( iRoot ){
-    z2 = lsmMallocPrintf(pEnv, "root=%lld", iRoot);
+    z2 = lsmMallocPrintf(pEnv, "root=%d", iRoot);
  }else{
-    z2 = lsmMallocPrintf(pEnv, "size=%lld", nSize);
+    z2 = lsmMallocPrintf(pEnv, "size=%d", nSize);
  }

  nPad = nMin - 2 - strlen(z1) - 1 - strlen(z2);
@@ -5539,7 +5536,7 @@ void sortedDumpPage(lsm_db *pDb, Segment *pRun, Page *pPg, int bVals){
  int i;

  int nRec;
-  LsmPgno iPtr;
+  int iPtr;
  int flags;
  u8 *aData;
  int nData;
@@ -5547,11 +5544,11 @@ void sortedDumpPage(lsm_db *pDb, Segment *pRun, Page *pPg, int bVals){
  aData = fsPageData(pPg, &nData);

  nRec = pageGetNRec(aData, nData);
-  iPtr = pageGetPtr(aData, nData);
+  iPtr = (int)pageGetPtr(aData, nData);
  flags = pageGetFlags(aData, nData);

  lsmStringInit(&s, pDb->pEnv);
-  lsmStringAppendf(&s,"nCell=%d iPtr=%lld flags=%d {", nRec, iPtr, flags);
+  lsmStringAppendf(&s,"nCell=%d iPtr=%d flags=%d {", nRec, iPtr, flags);
  if( flags&SEGMENT_BTREE_FLAG ) iPtr = 0;

  for(i=0; i<nRec; i++){
@@ -5561,13 +5558,13 @@ void sortedDumpPage(lsm_db *pDb, Segment *pRun, Page *pPg, int bVals){
    u8 *aVal = 0; int nVal = 0;   /* Value */
    int iTopic;
    u8 *aCell;
-    i64 iPgPtr;
+    int iPgPtr;
    int eType;

    aCell = pageGetCell(aData, nData, i);
    eType = *aCell++;
    assert( (flags & SEGMENT_BTREE_FLAG) || eType!=0 );
-    aCell += lsmVarintGet64(aCell, &iPgPtr);
+    aCell += lsmVarintGet32(aCell, &iPgPtr);

    if( eType==0 ){
      LsmPgno iRef;               /* Page number of referenced page */
@@ -5593,7 +5590,7 @@ void sortedDumpPage(lsm_db *pDb, Segment *pRun, Page *pPg, int bVals){
      }
    }

-    lsmStringAppendf(&s, " %lld", iPgPtr+iPtr);
+    lsmStringAppendf(&s, " %d", iPgPtr+iPtr);
    lsmFsPageRelease(pRef);
  }
  lsmStringAppend(&s, "}", 1);
@@ -5723,20 +5720,20 @@ static int infoPageDump(
    int nKeyWidth = 0;
    LsmString str;
    int nRec;
-    LsmPgno iPtr;
+    int iPtr;
    int flags2;
    int iCell;
    u8 *aData; int nData;         /* Page data and size thereof */

    aData = fsPageData(pPg, &nData);
    nRec = pageGetNRec(aData, nData);
-    iPtr = pageGetPtr(aData, nData);
+    iPtr = (int)pageGetPtr(aData, nData);
    flags2 = pageGetFlags(aData, nData);

    lsmStringInit(&str, pDb->pEnv);
    lsmStringAppendf(&str, "Page : %lld  (%d bytes)\n", iPg, nData);
    lsmStringAppendf(&str, "nRec : %d\n", nRec);
-    lsmStringAppendf(&str, "iPtr : %lld\n", iPtr);
+    lsmStringAppendf(&str, "iPtr : %d\n", iPtr);
    lsmStringAppendf(&str, "flags: %04x\n", flags2);
    lsmStringAppendf(&str, "\n");

@@ -185,11 +185,6 @@ int lsmVarintLen32(int n){
  return lsmVarintPut32(aData, n);
 }

-int lsmVarintLen64(i64 n){
-  u8 aData[9];
-  return lsmVarintPut64(aData, n);
-}
-
 /*
 ** The argument is the first byte of a varint. This function returns the
 ** total number of bytes in the entire varint (including the first byte).
@@ -1,278 +0,0 @@
-/*
-** 2022-11-18
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-**
-** This is a SQLite extension for converting in either direction
-** between a (binary) blob and base64 text. Base64 can transit a
-** sane USASCII channel unmolested. It also plays nicely in CSV or
-** written as TCL brace-enclosed literals or SQL string literals,
-** and can be used unmodified in XML-like documents.
-**
-** This is an independent implementation of conversions specified in
-** RFC 4648, done on the above date by the author (Larry Brasfield)
-** who thereby has the right to put this into the public domain.
-**
-** The conversions meet RFC 4648 requirements, provided that this
-** C source specifies that line-feeds are included in the encoded
-** data to limit visible line lengths to 72 characters and to
-** terminate any encoded blob having non-zero length.
-**
-** Length limitations are not imposed except that the runtime
-** SQLite string or blob length limits are respected. Otherwise,
-** any length binary sequence can be represented and recovered.
-** Generated base64 sequences, with their line-feeds included,
-** can be concatenated; the result converted back to binary will
-** be the concatenation of the represented binary sequences.
-**
-** This SQLite3 extension creates a function, base64(x), which
-** either: converts text x containing base64 to a returned blob;
-** or converts a blob x to returned text containing base64. An
-** error will be thrown for other input argument types.
-**
-** This code relies on UTF-8 encoding only with respect to the
-** meaning of the first 128 (7-bit) codes matching that of USASCII.
-** It will fail miserably if somehow made to try to convert EBCDIC.
-** Because it is table-driven, it could be enhanced to handle that,
-** but the world and SQLite have moved on from that anachronism.
-**
-** To build the extension:
-** Set shell variable SQDIR=<your favorite SQLite checkout directory>
-** *Nix: gcc -O2 -shared -I$SQDIR -fPIC -o base64.so base64.c
-** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR -o base64.dylib base64.c
-** Win32: gcc -O2 -shared -I%SQDIR% -o base64.dll base64.c
-** Win32: cl /Os -I%SQDIR% base64.c -link -dll -out:base64.dll
-*/
-
-#include <assert.h>
-
-#include "sqlite3ext.h"
-
-#ifndef deliberate_fall_through
-/* Quiet some compilers about some of our intentional code. */
-# if GCC_VERSION>=7000000
-#  define deliberate_fall_through __attribute__((fallthrough));
-# else
-#  define deliberate_fall_through
-# endif
-#endif
-
-SQLITE_EXTENSION_INIT1;
-
-#define PC 0x80 /* pad character */
-#define WS 0x81 /* whitespace */
-#define ND 0x82 /* Not above or digit-value */
-#define PAD_CHAR '='
-
-#ifndef U8_TYPEDEF
-typedef unsigned char u8;
-#define U8_TYPEDEF
-#endif
-
-/* Decoding table, ASCII (7-bit) value to base 64 digit value or other */
-static const u8 b64DigitValues[128] = {
-  /*                             HT LF VT  FF CR       */
-    ND,ND,ND,ND, ND,ND,ND,ND, ND,WS,WS,WS, WS,WS,ND,ND,
-  /*                                                US */
-    ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND,
-  /*sp                                  +            / */
-    WS,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,62, ND,ND,ND,63,
-  /* 0  1            5            9            =       */
-    52,53,54,55, 56,57,58,59, 60,61,ND,ND, ND,PC,ND,ND,
-  /*    A                                            O */
-    ND, 0, 1, 2,  3, 4, 5, 6,  7, 8, 9,10, 11,12,13,14,
-  /* P                               Z                 */
-    15,16,17,18, 19,20,21,22, 23,24,25,ND, ND,ND,ND,ND,
-  /*    a                                            o */
-    ND,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40,
-  /* p                               z                 */
-    41,42,43,44, 45,46,47,48, 49,50,51,ND, ND,ND,ND,ND
-};
-
-static const char b64Numerals[64+1]
-= "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
-
-#define BX_DV_PROTO(c) \
-  ((((u8)(c))<0x80)? (u8)(b64DigitValues[(u8)(c)]) : 0x80)
-#define IS_BX_DIGIT(bdp) (((u8)(bdp))<0x80)
-#define IS_BX_WS(bdp) ((bdp)==WS)
-#define IS_BX_PAD(bdp) ((bdp)==PC)
-#define BX_NUMERAL(dv) (b64Numerals[(u8)(dv)])
-/* Width of base64 lines. Should be an integer multiple of 4. */
-#define B64_DARK_MAX 72
-
-/* Encode a byte buffer into base64 text with linefeeds appended to limit
-** encoded group lengths to B64_DARK_MAX or to terminate the last group.
-*/
-static char* toBase64( u8 *pIn, int nbIn, char *pOut ){
-  int nCol = 0;
-  while( nbIn >= 3 ){
-    /* Do the bit-shuffle, exploiting unsigned input to avoid masking. */
-    pOut[0] = BX_NUMERAL(pIn[0]>>2);
-    pOut[1] = BX_NUMERAL(((pIn[0]<<4)|(pIn[1]>>4))&0x3f);
-    pOut[2] = BX_NUMERAL(((pIn[1]&0xf)<<2)|(pIn[2]>>6));
-    pOut[3] = BX_NUMERAL(pIn[2]&0x3f);
-    pOut += 4;
-    nbIn -= 3;
-    pIn += 3;
-    if( (nCol += 4)>=B64_DARK_MAX || nbIn<=0 ){
-      *pOut++ = '\n';
-      nCol = 0;
-    }
-  }
-  if( nbIn > 0 ){
-    signed char nco = nbIn+1;
-    int nbe;
-    unsigned long qv = *pIn++;
-    for( nbe=1; nbe<3; ++nbe ){
-      qv <<= 8;
-      if( nbe<nbIn ) qv |= *pIn++;
-    }
-    for( nbe=3; nbe>=0; --nbe ){
-      char ce = (nbe<nco)? BX_NUMERAL((u8)(qv & 0x3f)) : PAD_CHAR;
-      qv >>= 6;
-      pOut[nbe] = ce;
-    }
-    pOut += 4;
-    *pOut++ = '\n';
-  }
-  *pOut = 0;
-  return pOut;
-}
-
-/* Skip over text which is not base64 numeral(s). */
-static char * skipNonB64( char *s ){
-  char c;
-  while( (c = *s) && !IS_BX_DIGIT(BX_DV_PROTO(c)) ) ++s;
-  return s;
-}
-
-/* Decode base64 text into a byte buffer. */
-static u8* fromBase64( char *pIn, int ncIn, u8 *pOut ){
-  if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn;
-  while( ncIn>0 && *pIn!=PAD_CHAR ){
-    static signed char nboi[] = { 0, 0, 1, 2, 3 };
-    char *pUse = skipNonB64(pIn);
-    unsigned long qv = 0L;
-    int nti, nbo, nac;
-    ncIn -= (pUse - pIn);
-    pIn = pUse;
-    nti = (ncIn>4)? 4 : ncIn;
-    ncIn -= nti;
-    nbo = nboi[nti];
-    if( nbo==0 ) break;
-    for( nac=0; nac<4; ++nac ){
-      char c = (nac<nti)? *pIn++ : b64Numerals[0];
-      u8 bdp = BX_DV_PROTO(c);
-      switch( bdp ){
-      case ND:
-        /*  Treat dark non-digits as pad, but they terminate decode too. */
-        ncIn = 0;
-        deliberate_fall_through;
-      case WS:
-        /* Treat whitespace as pad and terminate this group.*/
-        nti = nac;
-        deliberate_fall_through;
-      case PC:
-        bdp = 0;
-        --nbo;
-        deliberate_fall_through;
-      default: /* bdp is the digit value. */
-        qv = qv<<6 | bdp;
-        break;
-      }
-    }
-    switch( nbo ){
-    case 3:
-      pOut[2] = (qv) & 0xff;
-    case 2:
-      pOut[1] = (qv>>8) & 0xff;
-    case 1:
-      pOut[0] = (qv>>16) & 0xff;
-    }
-    pOut += nbo;
-  }
-  return pOut;
-}
-
-/* This function does the work for the SQLite base64(x) UDF. */
-static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){
-  int nb, nc, nv = sqlite3_value_bytes(av[0]);
-  int nvMax = sqlite3_limit(sqlite3_context_db_handle(context),
-                            SQLITE_LIMIT_LENGTH, -1);
-  char *cBuf;
-  u8 *bBuf;
-  assert(na==1);
-  switch( sqlite3_value_type(av[0]) ){
-  case SQLITE_BLOB:
-    nb = nv;
-    nc = 4*(nv+2/3); /* quads needed */
-    nc += (nc+(B64_DARK_MAX-1))/B64_DARK_MAX + 1; /* LFs and a 0-terminator */
-    if( nvMax < nc ){
-      sqlite3_result_error(context, "blob expanded to base64 too big", -1);
-      return;
-    }
-    cBuf = sqlite3_malloc(nc);
-    if( !cBuf ) goto memFail;
-    bBuf = (u8*)sqlite3_value_blob(av[0]);
-    nc = (int)(toBase64(bBuf, nb, cBuf) - cBuf);
-    sqlite3_result_text(context, cBuf, nc, sqlite3_free);
-    break;
-  case SQLITE_TEXT:
-    nc = nv;
-    nb = 3*((nv+3)/4); /* may overestimate due to LF and padding */
-    if( nvMax < nb ){
-      sqlite3_result_error(context, "blob from base64 may be too big", -1);
-      return;
-    }else if( nb<1 ){
-      nb = 1;
-    }
-    bBuf = sqlite3_malloc(nb);
-    if( !bBuf ) goto memFail;
-    cBuf = (char *)sqlite3_value_text(av[0]);
-    nb = (int)(fromBase64(cBuf, nc, bBuf) - bBuf);
-    sqlite3_result_blob(context, bBuf, nb, sqlite3_free);
-    break;
-  default:
-    sqlite3_result_error(context, "base64 accepts only blob or text", -1);
-    return;
-  }
-  return;
- memFail:
-  sqlite3_result_error(context, "base64 OOM", -1);
-}
-
-/*
-** Establish linkage to running SQLite library.
-*/
-#ifndef SQLITE_SHELL_EXTFUNCS
-#ifdef _WIN32
-__declspec(dllexport)
-#endif
-int sqlite3_base_init
-#else
-static int sqlite3_base64_init
-#endif
-(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){
-  SQLITE_EXTENSION_INIT2(pApi);
-  (void)pzErr;
-  return sqlite3_create_function
-    (db, "base64", 1,
-     SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8,
-     0, base64, 0, 0);
-}
-
-/*
-** Define some macros to allow this extension to be built into the shell
-** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This
-** allows shell.c, as distributed, to have this extension built in.
-*/
-#define BASE64_INIT(db) sqlite3_base64_init(db, 0, 0)
-#define BASE64_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */
@@ -1,436 +0,0 @@
-/*
-** 2022-11-16
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-**
-** This is a utility for converting binary to base85 or vice-versa.
-** It can be built as a standalone program or an SQLite3 extension.
-**
-** Much like base64 representations, base85 can be sent through a
-** sane USASCII channel unmolested. It also plays nicely in CSV or
-** written as TCL brace-enclosed literals or SQL string literals.
-** It is not suited for unmodified use in XML-like documents.
-**
-** The encoding used resembles Ascii85, but was devised by the author
-** (Larry Brasfield) before Mozilla, Adobe, ZMODEM or other Ascii85
-** variant sources existed, in the 1984 timeframe on a VAX mainframe.
-** Further, this is an independent implementation of a base85 system.
-** Hence, the author has rightfully put this into the public domain.
-**
-** Base85 numerals are taken from the set of 7-bit USASCII codes,
-** excluding control characters and Space ! " ' ( ) { | } ~ Del
-** in code order representing digit values 0 to 84 (base 10.)
-**
-** Groups of 4 bytes, interpreted as big-endian 32-bit values,
-** are represented as 5-digit base85 numbers with MS to LS digit
-** order. Groups of 1-3 bytes are represented with 2-4 digits,
-** still big-endian but 8-24 bit values. (Using big-endian yields
-** the simplest transition to byte groups smaller than 4 bytes.
-** These byte groups can also be considered base-256 numbers.)
-** Groups of 0 bytes are represented with 0 digits and vice-versa.
-** No pad characters are used; Encoded base85 numeral sequence
-** (aka "group") length maps 1-to-1 to the decoded binary length.
-**
-** Any character not in the base85 numeral set delimits groups.
-** When base85 is streamed or stored in containers of indefinite
-** size, newline is used to separate it into sub-sequences of no
-** more than 80 digits so that fgets() can be used to read it.
-**
-** Length limitations are not imposed except that the runtime
-** SQLite string or blob length limits are respected. Otherwise,
-** any length binary sequence can be represented and recovered.
-** Base85 sequences can be concatenated by separating them with
-** a non-base85 character; the conversion to binary will then
-** be the concatenation of the represented binary sequences.
-
-** The standalone program either converts base85 on stdin to create
-** a binary file or converts a binary file to base85 on stdout.
-** Read or make it blurt its help for invocation details.
-**
-** The SQLite3 extension creates a function, base85(x), which will
-** either convert text base85 to a blob or a blob to text base85
-** and return the result (or throw an error for other types.)
-** Unless built with OMIT_BASE85_CHECKER defined, it also creates a
-** function, is_base85(t), which returns 1 iff the text t contains
-** nothing other than base85 numerals and whitespace, or 0 otherwise.
-**
-** To build the extension:
-** Set shell variable SQDIR=<your favorite SQLite checkout directory>
-** and variable OPTS to -DOMIT_BASE85_CHECKER if is_base85() unwanted.
-** *Nix: gcc -O2 -shared -I$SQDIR $OPTS -fPIC -o base85.so base85.c
-** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR $OPTS -o base85.dylib base85.c
-** Win32: gcc -O2 -shared -I%SQDIR% %OPTS% -o base85.dll base85.c
-** Win32: cl /Os -I%SQDIR% %OPTS% base85.c -link -dll -out:base85.dll
-**
-** To build the standalone program, define PP symbol BASE85_STANDALONE. Eg.
-** *Nix or OSX: gcc -O2 -DBASE85_STANDALONE base85.c -o base85
-** Win32: gcc -O2 -DBASE85_STANDALONE -o base85.exe base85.c
-** Win32: cl /Os /MD -DBASE85_STANDALONE base85.c
-*/
-
-#include <stdio.h>
-#include <memory.h>
-#include <string.h>
-#include <assert.h>
-#ifndef OMIT_BASE85_CHECKER
-# include <ctype.h>
-#endif
-
-#ifndef BASE85_STANDALONE
-
-# include "sqlite3ext.h"
-
-SQLITE_EXTENSION_INIT1;
-
-#else
-
-# ifdef _WIN32
-#  include <io.h>
-#  include <fcntl.h>
-# else
-#  define setmode(fd,m)
-# endif
-
-static char *zHelp =
-  "Usage: base85 <dirFlag> <binFile>\n"
-  " <dirFlag> is either -r to read or -w to write <binFile>,\n"
-  "   content to be converted to/from base85 on stdout/stdin.\n"
-  " <binFile> names a binary file to be rendered or created.\n"
-  "   Or, the name '-' refers to the stdin or stdout stream.\n"
-  ;
-
-static void sayHelp(){
-  printf("%s", zHelp);
-}
-#endif
-
-#ifndef U8_TYPEDEF
-typedef unsigned char u8;
-#define U8_TYPEDEF
-#endif
-
-/* Classify c according to interval within USASCII set w.r.t. base85
- * Values of 1 and 3 are base85 numerals. Values of 0, 2, or 4 are not.
- */
-#define B85_CLASS( c ) (((c)>='#')+((c)>'&')+((c)>='*')+((c)>'z'))
-
-/* Provide digitValue to b85Numeral offset as a function of above class. */
-static u8 b85_cOffset[] = { 0, '#', 0, '*'-4, 0 };
-#define B85_DNOS( c ) b85_cOffset[B85_CLASS(c)]
-
-/* Say whether c is a base85 numeral. */
-#define IS_B85( c ) (B85_CLASS(c) & 1)
-
-#if 0 /* Not used, */
-static u8 base85DigitValue( char c ){
-  u8 dv = (u8)(c - '#');
-  if( dv>87 ) return 0xff;
-  return (dv > 3)? dv-3 : dv;
-}
-#endif
-
-/* Width of base64 lines. Should be an integer multiple of 5. */
-#define B85_DARK_MAX 80
-
-
-static char * skipNonB85( char *s ){
-  char c;
-  while( (c = *s) && !IS_B85(c) ) ++s;
-  return s;
-}
-
-/* Convert small integer, known to be in 0..84 inclusive, to base85 numeral.
- * Do not use the macro form with argument expression having a side-effect.*/
-#if 0
-static char base85Numeral( u8 b ){
-  return (b < 4)? (char)(b + '#') : (char)(b - 4 + '*');
-}
-#else
-# define base85Numeral( dn )\
-  ((char)(((dn) < 4)? (char)((dn) + '#') : (char)((dn) - 4 + '*')))
-#endif
-
-static char *putcs(char *pc, char *s){
-  char c;
-  while( (c = *s++)!=0 ) *pc++ = c;
-  return pc;
-}
-
-/* Encode a byte buffer into base85 text. If pSep!=0, it's a C string
-** to be appended to encoded groups to limit their length to B85_DARK_MAX
-** or to terminate the last group (to aid concatenation.)
-*/
-static char* toBase85( u8 *pIn, int nbIn, char *pOut, char *pSep ){
-  int nCol = 0;
-  while( nbIn >= 4 ){
-    int nco = 5;
-    unsigned long qbv = (((unsigned long)pIn[0])<<24) |
-                        (pIn[1]<<16) | (pIn[2]<<8) | pIn[3];
-    while( nco > 0 ){
-      unsigned nqv = (unsigned)(qbv/85UL);
-      unsigned char dv = qbv - 85UL*nqv;
-      qbv = nqv;
-      pOut[--nco] = base85Numeral(dv);
-    }
-    nbIn -= 4;
-    pIn += 4;
-    pOut += 5;
-    if( pSep && (nCol += 5)>=B85_DARK_MAX ){
-      pOut = putcs(pOut, pSep);
-      nCol = 0;
-    }
-  }
-  if( nbIn > 0 ){
-    int nco = nbIn + 1;
-    unsigned long qv = *pIn++;
-    int nbe = 1;
-    while( nbe++ < nbIn ){
-      qv = (qv<<8) | *pIn++;
-    }
-    nCol += nco;
-    while( nco > 0 ){
-      u8 dv = (u8)(qv % 85);
-      qv /= 85;
-      pOut[--nco] = base85Numeral(dv);
-    }
-    pOut += (nbIn+1);
-  }
-  if( pSep && nCol>0 ) pOut = putcs(pOut, pSep);
-  *pOut = 0;
-  return pOut;
-}
-
-/* Decode base85 text into a byte buffer. */
-static u8* fromBase85( char *pIn, int ncIn, u8 *pOut ){
-  if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn;
-  while( ncIn>0 ){
-    static signed char nboi[] = { 0, 0, 1, 2, 3, 4 };
-    char *pUse = skipNonB85(pIn);
-    unsigned long qv = 0L;
-    int nti, nbo;
-    ncIn -= (pUse - pIn);
-    pIn = pUse;
-    nti = (ncIn>5)? 5 : ncIn;
-    nbo = nboi[nti];
-    if( nbo==0 ) break;
-    while( nti>0 ){
-      char c = *pIn++;
-      u8 cdo = B85_DNOS(c);
-      --ncIn;
-      if( cdo==0 ) break;
-      qv = 85 * qv + (c - cdo);
-      --nti;
-    }
-    nbo -= nti; /* Adjust for early (non-digit) end of group. */
-    switch( nbo ){
-    case 4:
-      *pOut++ = (qv >> 24)&0xff;
-    case 3:
-      *pOut++ = (qv >> 16)&0xff;
-    case 2:
-      *pOut++ = (qv >> 8)&0xff;
-    case 1:
-      *pOut++ = qv&0xff;
-    case 0:
-      break;
-    }
-  }
-  return pOut;
-}
-
-#ifndef OMIT_BASE85_CHECKER
-/* Say whether input char sequence is all (base85 and/or whitespace).*/
-static int allBase85( char *p, int len ){
-  char c;
-  while( len-- > 0 && (c = *p++) != 0 ){
-    if( !IS_B85(c) && !isspace(c) ) return 0;
-  }
-  return 1;
-}
-#endif
-
-#ifndef BASE85_STANDALONE
-
-# ifndef OMIT_BASE85_CHECKER
-/* This function does the work for the SQLite is_base85(t) UDF. */
-static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){
-  assert(na==1);
-  switch( sqlite3_value_type(av[0]) ){
-  case SQLITE_TEXT:
-    {
-      int rv = allBase85( (char *)sqlite3_value_text(av[0]),
-                          sqlite3_value_bytes(av[0]) );
-      sqlite3_result_int(context, rv);
-    }
-    break;
-  case SQLITE_NULL:
-    sqlite3_result_null(context);
-    break;
-  default:
-    sqlite3_result_error(context, "is_base85 accepts only text or NULL", -1);
-    return;
-  }
-}
-# endif
-
-/* This function does the work for the SQLite base85(x) UDF. */
-static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){
-  int nb, nc, nv = sqlite3_value_bytes(av[0]);
-  int nvMax = sqlite3_limit(sqlite3_context_db_handle(context),
-                            SQLITE_LIMIT_LENGTH, -1);
-  char *cBuf;
-  u8 *bBuf;
-  assert(na==1);
-  switch( sqlite3_value_type(av[0]) ){
-  case SQLITE_BLOB:
-    nb = nv;
-    /*    ulongs    tail   newlines  tailenc+nul*/
-    nc = 5*(nv/4) + nv%4 + nv/64+1 + 2;
-    if( nvMax < nc ){
-      sqlite3_result_error(context, "blob expanded to base85 too big", -1);
-      return;
-    }
-    cBuf = sqlite3_malloc(nc);
-    if( !cBuf ) goto memFail;
-    bBuf = (u8*)sqlite3_value_blob(av[0]);
-    nc = (int)(toBase85(bBuf, nb, cBuf, "\n") - cBuf);
-    sqlite3_result_text(context, cBuf, nc, sqlite3_free);
-    break;
-  case SQLITE_TEXT:
-    nc = nv;
-    nb = 4*(nv/5) + nv%5; /* may overestimate */
-    if( nvMax < nb ){
-      sqlite3_result_error(context, "blob from base85 may be too big", -1);
-      return;
-    }else if( nb<1 ){
-      nb = 1;
-    }
-    bBuf = sqlite3_malloc(nb);
-    if( !bBuf ) goto memFail;
-    cBuf = (char *)sqlite3_value_text(av[0]);
-    nb = (int)(fromBase85(cBuf, nc, bBuf) - bBuf);
-    sqlite3_result_blob(context, bBuf, nb, sqlite3_free);
-    break;
-  default:
-    sqlite3_result_error(context, "base85 accepts only blob or text.", -1);
-    return;
-  }
-  return;
- memFail:
-  sqlite3_result_error(context, "base85 OOM", -1);
-}
-
-/*
-** Establish linkage to running SQLite library.
-*/
-#ifndef SQLITE_SHELL_EXTFUNCS
-#ifdef _WIN32
-__declspec(dllexport)
-#endif
-int sqlite3_base_init
-#else
-static int sqlite3_base85_init
-#endif
-(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){
-  SQLITE_EXTENSION_INIT2(pApi);
-  (void)pzErr;
-# ifndef OMIT_BASE85_CHECKER
-  {
-    int rc = sqlite3_create_function
-      (db, "is_base85", 1,
-       SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_UTF8,
-       0, is_base85, 0, 0);
-    if( rc!=SQLITE_OK ) return rc;
-  }
-# endif
-  return sqlite3_create_function
-    (db, "base85", 1,
-     SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8,
-     0, base85, 0, 0);
-}
-
-/*
-** Define some macros to allow this extension to be built into the shell
-** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This
-** allows shell.c, as distributed, to have this extension built in.
-*/
-# define BASE85_INIT(db) sqlite3_base85_init(db, 0, 0)
-# define BASE85_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */
-
-#else /* standalone program */
-
-int main(int na, char *av[]){
-  int cin;
-  int rc = 0;
-  u8 bBuf[4*(B85_DARK_MAX/5)];
-  char cBuf[5*(sizeof(bBuf)/4)+2];
-  size_t nio;
-# ifndef OMIT_BASE85_CHECKER
-  int b85Clean = 1;
-# endif
-  char rw;
-  FILE *fb = 0, *foc = 0;
-  char fmode[3] = "xb";
-  if( na < 3 || av[1][0]!='-' || (rw = av[1][1])==0 || (rw!='r' && rw!='w') ){
-    sayHelp();
-    return 0;
-  }
-  fmode[0] = rw;
-  if( av[2][0]=='-' && av[2][1]==0 ){
-    switch( rw ){
-    case 'r':
-      fb = stdin;
-      setmode(fileno(stdin), O_BINARY);
-      break;
-    case 'w':
-      fb = stdout;
-      setmode(fileno(stdout), O_BINARY);
-      break;
-    }
-  }else{
-    fb = fopen(av[2], fmode);
-    foc = fb;
-  }
-  if( !fb ){
-    fprintf(stderr, "Cannot open %s for %c\n", av[2], rw);
-    rc = 1;
-  }else{
-    switch( rw ){
-    case 'r':
-      while( (nio = fread( bBuf, 1, sizeof(bBuf), fb))>0 ){
-        toBase85( bBuf, (int)nio, cBuf, 0 );
-        fprintf(stdout, "%s\n", cBuf);
-      }
-      break;
-    case 'w':
-      while( 0 != fgets(cBuf, sizeof(cBuf), stdin) ){
-        int nc = strlen(cBuf);
-        size_t nbo = fromBase85( cBuf, nc, bBuf ) - bBuf;
-        if( 1 != fwrite(bBuf, nbo, 1, fb) ) rc = 1;
-# ifndef OMIT_BASE85_CHECKER
-        b85Clean &= allBase85( cBuf, nc );
-# endif
-      }
-      break;
-    default:
-      sayHelp();
-      rc = 1;
-    }
-    if( foc ) fclose(foc);
-  }
-# ifndef OMIT_BASE85_CHECKER
-  if( !b85Clean ){
-    fprintf(stderr, "Base85 input had non-base85 dark or control content.\n");
-  }
-# endif
-  return rc;
-}
-
-#endif
@@ -1,86 +0,0 @@
-/*
-** 2022-11-20
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-**
-** This source allows multiple SQLite extensions to be either: combined
-** into a single runtime-loadable library; or built into the SQLite shell
-** using a preprocessing convention set by src/shell.c.in (and shell.c).
-**
-** Presently, it combines the base64.c and base85.c extensions. However,
-** it can be used as a template for other combinations.
-**
-** Example usages:
-**
-**  - Build a runtime-loadable extension from SQLite checkout directory:
-** *Nix, OSX: gcc -O2 -shared -I. -fPIC -o basexx.so ext/misc/basexx.c
-** Win32: cl /Os -I. ext/misc/basexx.c -link -dll -out:basexx.dll
-**
-**  - Incorporate as built-in in sqlite3 shell:
-** *Nix, OSX with gcc on a like platform:
-**  export mop1=-DSQLITE_SHELL_EXTSRC=ext/misc/basexx.c
-**  export mop2=-DSQLITE_SHELL_EXTFUNCS=BASEXX
-**  make sqlite3 "OPTS=$mop1 $mop2"
-** Win32 with Microsoft toolset on Windows:
-**  set mop1=-DSQLITE_SHELL_EXTSRC=ext/misc/basexx.c
-**  set mop2=-DSQLITE_SHELL_EXTFUNCS=BASEXX
-**  set mops="OPTS=%mop1% %mop2%"
-**  nmake -f Makefile.msc sqlite3.exe %mops%
-*/
-
-#ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */
-# include "sqlite3ext.h"
-SQLITE_EXTENSION_INIT1;
-#endif
-
-static void init_api_ptr(const sqlite3_api_routines *pApi){
-  SQLITE_EXTENSION_INIT2(pApi);
-}
-
-#undef SQLITE_EXTENSION_INIT1
-#define SQLITE_EXTENSION_INIT1 /* */
-#undef SQLITE_EXTENSION_INIT2
-#define SQLITE_EXTENSION_INIT2(v) (void)v
-
-typedef unsigned char u8;
-#define U8_TYPEDEF
-
-/* These next 2 undef's are only needed because the entry point names
- * collide when formulated per the rules stated for loadable extension
- * entry point names that will be deduced from the file basenames.
- */
-#undef sqlite3_base_init
-#define sqlite3_base_init sqlite3_base64_init
-#include "base64.c"
-
-#undef sqlite3_base_init
-#define sqlite3_base_init sqlite3_base85_init
-#include "base85.c"
-
-#ifdef _WIN32
-__declspec(dllexport)
-#endif
-int sqlite3_basexx_init(sqlite3 *db, char **pzErr,
-                               const sqlite3_api_routines *pApi){
-  init_api_ptr(pApi);
-  int rc1 = BASE64_INIT(db);
-  int rc2 = BASE85_INIT(db);
-
-  if( rc1==SQLITE_OK && rc2==SQLITE_OK ){
-    BASE64_EXPOSE(db, pzErr);
-    BASE64_EXPOSE(db, pzErr);
-    return SQLITE_OK;
-  }else{
-    return SQLITE_ERROR;
-  }
-}
-
-# define BASEXX_INIT(db) sqlite3_basexx_init(db, 0, 0)
-# define BASEXX_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */
@@ -28,7 +28,7 @@
 **
 ** There is an optional third parameter to determine the datatype of
 ** the C-language array.  Allowed values of the third parameter are
-** 'int32', 'int64', 'double', 'char*', 'struct iovec'.  Example:
+** 'int32', 'int64', 'double', 'char*'.  Example:
 **
 **      SELECT * FROM carray($ptr,10,'char*');
 **
@@ -56,14 +56,6 @@
 SQLITE_EXTENSION_INIT1
 #include <assert.h>
 #include <string.h>
-#ifdef _WIN32
-  struct iovec {
-    void *iov_base;
-    size_t iov_len;
-  };
-#else
-# include <sys/uio.h>
-#endif
 
 /* Allowed values for the mFlags parameter to sqlite3_carray_bind().
 ** Must exactly match the definitions in carray.h.
@@ -73,7 +65,6 @@ SQLITE_EXTENSION_INIT1
 # define CARRAY_INT64     1      /* Data is 64-bit signed integers */
 # define CARRAY_DOUBLE    2      /* Data is doubles */
 # define CARRAY_TEXT      3      /* Data is char* */
-# define CARRAY_BLOB      4      /* Data is struct iovec* */
 #endif

 #ifndef SQLITE_API
@@ -89,8 +80,7 @@ SQLITE_EXTENSION_INIT1
 /*
 ** Names of allowed datatypes
 */
-static const char *azType[] = { "int32", "int64", "double", "char*",
-                                "struct iovec" };
+static const char *azType[] = { "int32", "int64", "double", "char*" };

 /*
 ** Structure used to hold the sqlite3_carray_bind() information
@@ -234,12 +224,6 @@ static int carrayColumn(
          sqlite3_result_text(ctx, p[pCur->iRowid-1], -1, SQLITE_TRANSIENT);
          return SQLITE_OK;
        }
-        case CARRAY_BLOB: {
-          const struct iovec *p = (struct iovec*)pCur->pPtr;
-          sqlite3_result_blob(ctx, p[pCur->iRowid-1].iov_base,
-                               (int)p[pCur->iRowid-1].iov_len, SQLITE_TRANSIENT);
-          return SQLITE_OK;
-        }
      }
    }
  }
@@ -284,7 +268,7 @@ static int carrayFilter(
      if( pBind==0 ) break;
      pCur->pPtr = pBind->aData;
      pCur->iCnt = pBind->nData;
-      pCur->eType = pBind->mFlags & 0x07;
+      pCur->eType = pBind->mFlags & 0x03;
      break;
    }
    case 2:
@@ -447,29 +431,24 @@ SQLITE_API int sqlite3_carray_bind(
  pNew->mFlags = mFlags;
  if( xDestroy==SQLITE_TRANSIENT ){
    sqlite3_int64 sz = nData;
-    switch( mFlags & 0x07 ){
-      case CARRAY_INT32:   sz *= 4;                     break;
-      case CARRAY_INT64:   sz *= 8;                     break;
-      case CARRAY_DOUBLE:  sz *= 8;                     break;
-      case CARRAY_TEXT:    sz *= sizeof(char*);         break;
-      case CARRAY_BLOB:    sz *= sizeof(struct iovec);  break;
+    switch( mFlags & 0x03 ){
+      case CARRAY_INT32:   sz *= 4;              break;
+      case CARRAY_INT64:   sz *= 8;              break;
+      case CARRAY_DOUBLE:  sz *= 8;              break;
+      case CARRAY_TEXT:    sz *= sizeof(char*);  break;
    }
-    if( (mFlags & 0x07)==CARRAY_TEXT ){
+    if( (mFlags & 0x03)==CARRAY_TEXT ){
      for(i=0; i<nData; i++){
        const char *z = ((char**)aData)[i];
        if( z ) sz += strlen(z) + 1;
      }
-    }else if( (mFlags & 0x07)==CARRAY_BLOB ){
-      for(i=0; i<nData; i++){
-        sz += ((struct iovec*)aData)[i].iov_len;
-      }
    } 
    pNew->aData = sqlite3_malloc64( sz );
    if( pNew->aData==0 ){
      sqlite3_free(pNew);
      return SQLITE_NOMEM;
    }
-    if( (mFlags & 0x07)==CARRAY_TEXT ){
+    if( (mFlags & 0x03)==CARRAY_TEXT ){
      char **az = (char**)pNew->aData;
      char *z = (char*)&az[nData];
      for(i=0; i<nData; i++){
@@ -484,16 +463,6 @@ SQLITE_API int sqlite3_carray_bind(
        memcpy(z, zData, n+1);
        z += n+1;
      }
-    }else if( (mFlags & 0x07)==CARRAY_BLOB ){
-      struct iovec *p = (struct iovec*)pNew->aData;
-      unsigned char *z = (unsigned char*)&p[nData];
-      for(i=0; i<nData; i++){
-        size_t n = ((struct iovec*)aData)[i].iov_len;
-        p[i].iov_len = n;
-        p[i].iov_base = z;
-        z += n;
-        memcpy(p[i].iov_base, ((struct iovec*)aData)[i].iov_base, n);
-      }
    }else{
      memcpy(pNew->aData, aData, sz);
    }
@@ -41,10 +41,10 @@ SQLITE_API int sqlite3_carray_bind(
 #define CARRAY_INT64     1    /* Data is 64-bit signed integers */
 #define CARRAY_DOUBLE    2    /* Data is doubles */
 #define CARRAY_TEXT      3    /* Data is char* */
-#define CARRAY_BLOB      4    /* Data is struct iovec */

 #ifdef __cplusplus
 }  /* end of the 'extern "C"' block */
 #endif

 #endif /* ifndef _CARRAY_H */
+
@@ -47,7 +47,7 @@
 **
 **     sqlite3 *db;
 **     sqlite3_open(":memory:", &db);
-**     sqlite3_load_extension(db, "./cksumvfs");
+**     sqlite3_load_extention(db, "./cksumvfs");
 **     sqlite3_close(db);
 **
 ** If this extension is compiled with -DSQLITE_CKSUMVFS_STATIC and
@@ -71,20 +71,16 @@
 **   It contains one entry for each b-tree pointer between a parent and
 **   child page in the database.
 */
-
 #if !defined(SQLITEINT_H) 
 #include "sqlite3ext.h"

 typedef unsigned char u8;
-typedef unsigned int u32;

 #endif
 SQLITE_EXTENSION_INIT1
 #include <string.h>
 #include <assert.h>

-#ifndef SQLITE_OMIT_VIRTUALTABLE
-
 #define DBDATA_PADDING_BYTES 100 

 typedef struct DbdataTable DbdataTable;
@@ -106,12 +102,11 @@ struct DbdataCursor {

  /* Only for the sqlite_dbdata table */
  u8 *pRec;                       /* Buffer containing current record */
-  sqlite3_int64 nRec;             /* Size of pRec[] in bytes */
-  sqlite3_int64 nHdr;             /* Size of header in bytes */
+  int nRec;                       /* Size of pRec[] in bytes */
+  int nHdr;                       /* Size of header in bytes */
  int iField;                     /* Current field number */
  u8 *pHdrPtr;
  u8 *pPtr;
-  u32 enc;                        /* Text encoding */
  
  sqlite3_int64 iIntkey;          /* Integer key value */
 };
@@ -164,10 +159,6 @@ static int dbdataConnect(
  DbdataTable *pTab = 0;
  int rc = sqlite3_declare_vtab(db, pAux ? DBPTR_SCHEMA : DBDATA_SCHEMA);

-  (void)argc;
-  (void)argv;
-  (void)pzErr;
-  sqlite3_vtab_config(db, SQLITE_VTAB_USES_ALL_SCHEMAS);
  if( rc==SQLITE_OK ){
    pTab = (DbdataTable*)sqlite3_malloc64(sizeof(DbdataTable));
    if( pTab==0 ){
@@ -308,14 +299,14 @@ static int dbdataClose(sqlite3_vtab_cursor *pCursor){
 /* 
 ** Utility methods to decode 16 and 32-bit big-endian unsigned integers. 
 */
-static u32 get_uint16(unsigned char *a){
+static unsigned int get_uint16(unsigned char *a){
  return (a[0]<<8)|a[1];
 }
-static u32 get_uint32(unsigned char *a){
-  return ((u32)a[0]<<24)
-       | ((u32)a[1]<<16)
-       | ((u32)a[2]<<8)
-       | ((u32)a[3]);
+static unsigned int get_uint32(unsigned char *a){
+  return ((unsigned int)a[0]<<24)
+       | ((unsigned int)a[1]<<16)
+       | ((unsigned int)a[2]<<8)
+       | ((unsigned int)a[3]);
 }

 /*
@@ -330,7 +321,7 @@ static u32 get_uint32(unsigned char *a){
 */
 static int dbdataLoadPage(
  DbdataCursor *pCsr,             /* Cursor object */
-  u32 pgno,                       /* Page number of page to load */
+  unsigned int pgno,              /* Page number of page to load */
  u8 **ppPage,                    /* OUT: pointer to page buffer */
  int *pnPage                     /* OUT: Size of (*ppPage) in bytes */
 ){
@@ -340,27 +331,25 @@ static int dbdataLoadPage(

  *ppPage = 0;
  *pnPage = 0;
-  if( pgno>0 ){
-    sqlite3_bind_int64(pStmt, 2, pgno);
-    if( SQLITE_ROW==sqlite3_step(pStmt) ){
-      int nCopy = sqlite3_column_bytes(pStmt, 0);
-      if( nCopy>0 ){
-        u8 *pPage;
-        pPage = (u8*)sqlite3_malloc64(nCopy + DBDATA_PADDING_BYTES);
-        if( pPage==0 ){
-          rc = SQLITE_NOMEM;
-        }else{
-          const u8 *pCopy = sqlite3_column_blob(pStmt, 0);
-          memcpy(pPage, pCopy, nCopy);
-          memset(&pPage[nCopy], 0, DBDATA_PADDING_BYTES);
-        }
-        *ppPage = pPage;
-        *pnPage = nCopy;
+  sqlite3_bind_int64(pStmt, 2, pgno);
+  if( SQLITE_ROW==sqlite3_step(pStmt) ){
+    int nCopy = sqlite3_column_bytes(pStmt, 0);
+    if( nCopy>0 ){
+      u8 *pPage;
+      pPage = (u8*)sqlite3_malloc64(nCopy + DBDATA_PADDING_BYTES);
+      if( pPage==0 ){
+        rc = SQLITE_NOMEM;
+      }else{
+        const u8 *pCopy = sqlite3_column_blob(pStmt, 0);
+        memcpy(pPage, pCopy, nCopy);
+        memset(&pPage[nCopy], 0, DBDATA_PADDING_BYTES);
      }
+      *ppPage = pPage;
+      *pnPage = nCopy;
    }
-    rc2 = sqlite3_reset(pStmt);
-    if( rc==SQLITE_OK ) rc = rc2;
  }
+  rc2 = sqlite3_reset(pStmt);
+  if( rc==SQLITE_OK ) rc = rc2;

  return rc;
 }
@@ -369,30 +358,17 @@ static int dbdataLoadPage(
 ** Read a varint.  Put the value in *pVal and return the number of bytes.
 */
 static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){
-  sqlite3_uint64 u = 0;
+  sqlite3_int64 v = 0;
  int i;
  for(i=0; i<8; i++){
-    u = (u<<7) + (z[i]&0x7f);
-    if( (z[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; }
+    v = (v<<7) + (z[i]&0x7f);
+    if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; }
  }
-  u = (u<<8) + (z[i]&0xff);
-  *pVal = (sqlite3_int64)u;
+  v = (v<<8) + (z[i]&0xff);
+  *pVal = v;
  return 9;
 }

-/*
-** Like dbdataGetVarint(), but set the output to 0 if it is less than 0
-** or greater than 0xFFFFFFFF. This can be used for all varints in an
-** SQLite database except for key values in intkey tables.
-*/
-static int dbdataGetVarintU32(const u8 *z, sqlite3_int64 *pVal){
-  sqlite3_int64 val;
-  int nRet = dbdataGetVarint(z, &val);
-  if( val<0 || val>0xFFFFFFFF ) val = 0;
-  *pVal = val;
-  return nRet;
-}
-
 /*
 ** Return the number of bytes of space used by an SQLite value of type
 ** eType.
@@ -429,10 +405,9 @@ static int dbdataValueBytes(int eType){
 */
 static void dbdataValue(
  sqlite3_context *pCtx, 
-  u32 enc,
  int eType, 
  u8 *pData,
-  sqlite3_int64 nData
+  int nData
 ){
  if( eType>=0 && dbdataValueBytes(eType)<=nData ){
    switch( eType ){
@@ -474,19 +449,7 @@ static void dbdataValue(
      default: {
        int n = ((eType-12) / 2);
        if( eType % 2 ){
-          switch( enc ){
-#ifndef SQLITE_OMIT_UTF16
-            case SQLITE_UTF16BE:
-              sqlite3_result_text16be(pCtx, (void*)pData, n, SQLITE_TRANSIENT);
-              break;
-            case SQLITE_UTF16LE:
-              sqlite3_result_text16le(pCtx, (void*)pData, n, SQLITE_TRANSIENT);
-              break;
-#endif
-            default:
-              sqlite3_result_text(pCtx, (char*)pData, n, SQLITE_TRANSIENT);
-              break;
-          }
+          sqlite3_result_text(pCtx, (const char*)pData, n, SQLITE_TRANSIENT);
        }else{
          sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT);
        }
@@ -513,14 +476,9 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){
        if( pCsr->bOnePage==0 && pCsr->iPgno>pCsr->szDb ) return SQLITE_OK;
        rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage);
        if( rc!=SQLITE_OK ) return rc;
-        if( pCsr->aPage && pCsr->nPage>=256 ) break;
-        sqlite3_free(pCsr->aPage);
-        pCsr->aPage = 0;
-        if( pCsr->bOnePage ) return SQLITE_OK;
+        if( pCsr->aPage ) break;
        pCsr->iPgno++;
      }
-
-      assert( iOff+3+2<=pCsr->nPage );
      pCsr->iCell = pTab->bPtr ? -2 : 0;
      pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]);
    }
@@ -582,7 +540,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){
          if( bNextPage || iOff>pCsr->nPage ){
            bNextPage = 1;
          }else{
-            iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload);
+            iOff += dbdataGetVarint(&pCsr->aPage[iOff], &nPayload);
          }
    
          /* If this is a leaf intkey cell, load the rowid */
@@ -629,7 +587,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){
            /* Load content from overflow pages */
            if( nPayload>nLocal ){
              sqlite3_int64 nRem = nPayload - nLocal;
-              u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]);
+              unsigned int pgnoOvfl = get_uint32(&pCsr->aPage[iOff]);
              while( nRem>0 ){
                u8 *aOvfl = 0;
                int nOvfl = 0;
@@ -649,8 +607,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){
              }
            }
    
-            iHdr = dbdataGetVarintU32(pCsr->pRec, &nHdr);
-            if( nHdr>nPayload ) nHdr = 0;
+            iHdr = dbdataGetVarint(pCsr->pRec, &nHdr);
            pCsr->nHdr = nHdr;
            pCsr->pHdrPtr = &pCsr->pRec[iHdr];
            pCsr->pPtr = &pCsr->pRec[pCsr->nHdr];
@@ -664,7 +621,7 @@ static int dbdataNext(sqlite3_vtab_cursor *pCursor){
          if( pCsr->pHdrPtr>&pCsr->pRec[pCsr->nRec] ){
            bNextPage = 1;
          }else{
-            pCsr->pHdrPtr += dbdataGetVarintU32(pCsr->pHdrPtr, &iType);
+            pCsr->pHdrPtr += dbdataGetVarint(pCsr->pHdrPtr, &iType);
            pCsr->pPtr += dbdataValueBytes(iType);
          }
        }
@@ -703,18 +660,6 @@ static int dbdataEof(sqlite3_vtab_cursor *pCursor){
  return pCsr->aPage==0;
 }

-/*
-** Return true if nul-terminated string zSchema ends in "()". Or false
-** otherwise.
-*/
-static int dbdataIsFunction(const char *zSchema){
-  size_t n = strlen(zSchema);
-  if( n>2 && zSchema[n-2]=='(' && zSchema[n-1]==')' ){
-    return (int)n-2;
-  }
-  return 0;
-}
-
 /* 
 ** Determine the size in pages of database zSchema (where zSchema is
 ** "main", "temp" or the name of an attached database) and set 
@@ -725,16 +670,10 @@ static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){
  DbdataTable *pTab = (DbdataTable*)pCsr->base.pVtab;
  char *zSql = 0;
  int rc, rc2;
-  int nFunc = 0;
  sqlite3_stmt *pStmt = 0;

-  if( (nFunc = dbdataIsFunction(zSchema))>0 ){
-    zSql = sqlite3_mprintf("SELECT %.*s(0)", nFunc, zSchema);
-  }else{
-    zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema);
-  }
+  zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema);
  if( zSql==0 ) return SQLITE_NOMEM;
-
  rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0);
  sqlite3_free(zSql);
  if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
@@ -745,24 +684,6 @@ static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){
  return rc;
 }

-/*
-** Attempt to figure out the encoding of the database by retrieving page 1
-** and inspecting the header field. If successful, set the pCsr->enc variable
-** and return SQLITE_OK. Otherwise, return an SQLite error code.
-*/
-static int dbdataGetEncoding(DbdataCursor *pCsr){
-  int rc = SQLITE_OK;
-  int nPg1 = 0;
-  u8 *aPg1 = 0;
-  rc = dbdataLoadPage(pCsr, 1, &aPg1, &nPg1);
-  if( rc==SQLITE_OK && nPg1>=(56+4) ){
-    pCsr->enc = get_uint32(&aPg1[56]);
-  }
-  sqlite3_free(aPg1);
-  return rc;
-}
-
-
 /* 
 ** xFilter method for sqlite_dbdata and sqlite_dbptr.
 */
@@ -775,35 +696,24 @@ static int dbdataFilter(
  DbdataTable *pTab = (DbdataTable*)pCursor->pVtab;
  int rc = SQLITE_OK;
  const char *zSchema = "main";
-  (void)idxStr;
-  (void)argc;

  dbdataResetCursor(pCsr);
  assert( pCsr->iPgno==1 );
  if( idxNum & 0x01 ){
    zSchema = (const char*)sqlite3_value_text(argv[0]);
-    if( zSchema==0 ) zSchema = "";
  }
  if( idxNum & 0x02 ){
    pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]);
    pCsr->bOnePage = 1;
  }else{
+    pCsr->nPage = dbdataDbsize(pCsr, zSchema);
    rc = dbdataDbsize(pCsr, zSchema);
  }

  if( rc==SQLITE_OK ){
-    int nFunc = 0;
    if( pTab->pStmt ){
      pCsr->pStmt = pTab->pStmt;
      pTab->pStmt = 0;
-    }else if( (nFunc = dbdataIsFunction(zSchema))>0 ){
-      char *zSql = sqlite3_mprintf("SELECT %.*s(?2)", nFunc, zSchema);
-      if( zSql==0 ){
-        rc = SQLITE_NOMEM;
-      }else{
-        rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0);
-        sqlite3_free(zSql);
-      }
    }else{
      rc = sqlite3_prepare_v2(pTab->db, 
          "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1,
@@ -813,25 +723,16 @@ static int dbdataFilter(
  }
  if( rc==SQLITE_OK ){
    rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT);
-  }
-
-  /* Try to determine the encoding of the db by inspecting the header
-  ** field on page 1. */
-  if( rc==SQLITE_OK ){
-    rc = dbdataGetEncoding(pCsr);
-  }
-
-  if( rc!=SQLITE_OK ){
+  }else{
    pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db));
  }
-
  if( rc==SQLITE_OK ){
    rc = dbdataNext(pCursor);
  }
  return rc;
 }

-/*
+/* 
 ** Return a column for the sqlite_dbdata or sqlite_dbptr table.
 */
 static int dbdataColumn(
@@ -875,12 +776,11 @@ static int dbdataColumn(
      case DBDATA_COLUMN_VALUE: {
        if( pCsr->iField<0 ){
          sqlite3_result_int64(ctx, pCsr->iIntkey);
-        }else if( &pCsr->pRec[pCsr->nRec] >= pCsr->pPtr ){
+        }else{
          sqlite3_int64 iType;
-          dbdataGetVarintU32(pCsr->pHdrPtr, &iType);
+          dbdataGetVarint(pCsr->pHdrPtr, &iType);
          dbdataValue(
-              ctx, pCsr->enc, iType, pCsr->pPtr, 
-              &pCsr->pRec[pCsr->nRec] - pCsr->pPtr
+              ctx, iType, pCsr->pPtr, &pCsr->pRec[pCsr->nRec] - pCsr->pPtr
          );
        }
        break;
@@ -947,8 +847,5 @@ int sqlite3_dbdata_init(
  const sqlite3_api_routines *pApi
 ){
  SQLITE_EXTENSION_INIT2(pApi);
-  (void)pzErrMsg;
  return sqlite3DbdataRegister(db);
 }
-
-#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */
@@ -616,7 +616,7 @@ int sqlite3_decimal_init(

  SQLITE_EXTENSION_INIT2(pApi);

-  for(i=0; i<(int)(sizeof(aFunc)/sizeof(aFunc[0])) && rc==SQLITE_OK; i++){
+  for(i=0; i<sizeof(aFunc)/sizeof(aFunc[0]) && rc==SQLITE_OK; i++){
    rc = sqlite3_create_function(db, aFunc[i].zFuncName, aFunc[i].nArg,
                   SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC,
                   0, aFunc[i].xFunc, 0, 0);
@@ -185,7 +185,7 @@ static unsigned re_next_char(ReInput *p){
      c = (c&0x0f)<<12 | ((p->z[p->i]&0x3f)<<6) | (p->z[p->i+1]&0x3f);
      p->i += 2;
      if( c<=0x7ff || (c>=0xd800 && c<=0xdfff) ) c = 0xfffd;
-    }else if( (c&0xf8)==0xf0 && p->i+2<p->mx && (p->z[p->i]&0xc0)==0x80
+    }else if( (c&0xf8)==0xf0 && p->i+3<p->mx && (p->z[p->i]&0xc0)==0x80
           && (p->z[p->i+1]&0xc0)==0x80 && (p->z[p->i+2]&0xc0)==0x80 ){
      c = (c&0x07)<<18 | ((p->z[p->i]&0x3f)<<12) | ((p->z[p->i+1]&0x3f)<<6)
                       | (p->z[p->i+2]&0x3f);
@@ -712,15 +712,15 @@ static const char *re_compile(ReCompiled **ppRe, const char *zIn, int noCase){
  ** one or more matching characters, enter those matching characters into
  ** zInit[].  The re_match() routine can then search ahead in the input 
  ** string looking for the initial match without having to run the whole
-  ** regex engine over the string.  Do not worry about trying to match
+  ** regex engine over the string.  Do not worry able trying to match
  ** unicode characters beyond plane 0 - those are very rare and this is
  ** just an optimization. */
  if( pRe->aOp[0]==RE_OP_ANYSTAR && !noCase ){
    for(j=0, i=1; j<(int)sizeof(pRe->zInit)-2 && pRe->aOp[i]==RE_OP_MATCH; i++){
      unsigned x = pRe->aArg[i];
-      if( x<=0x7f ){
+      if( x<=127 ){
        pRe->zInit[j++] = (unsigned char)x;
-      }else if( x<=0x7ff ){
+      }else if( x<=0xfff ){
        pRe->zInit[j++] = (unsigned char)(0xc0 | (x>>6));
        pRe->zInit[j++] = 0x80 | (x&0x3f);
      }else if( x<=0xffff ){
@@ -803,7 +803,6 @@ static void re_bytecode_func(
  int i;
  int n;
  char *z;
-  (void)argc;

  zPattern = (const char*)sqlite3_value_text(argv[0]);
  if( zPattern==0 ) return;
@@ -531,7 +531,7 @@ static void sha3Func(
 /* Compute a string using sqlite3_vsnprintf() with a maximum length
 ** of 50 bytes and add it to the hash.
 */
-static void sha3_step_vformat(
+static void hash_step_vformat(
  SHA3Context *p,                 /* Add content to this context */
  const char *zFormat,
  ...
@@ -627,7 +627,7 @@ static void sha3QueryFunc(
    z = sqlite3_sql(pStmt);
    if( z ){
      n = (int)strlen(z);
-      sha3_step_vformat(&cx,"S%d:",n);
+      hash_step_vformat(&cx,"S%d:",n);
      SHA3Update(&cx,(unsigned char*)z,n);
    }

@@ -671,14 +671,14 @@ static void sha3QueryFunc(
          case SQLITE_TEXT: {
            int n2 = sqlite3_column_bytes(pStmt, i);
            const unsigned char *z2 = sqlite3_column_text(pStmt, i);
-            sha3_step_vformat(&cx,"T%d:",n2);
+            hash_step_vformat(&cx,"T%d:",n2);
            SHA3Update(&cx, z2, n2);
            break;
          }
          case SQLITE_BLOB: {
            int n2 = sqlite3_column_bytes(pStmt, i);
            const unsigned char *z2 = sqlite3_column_blob(pStmt, i);
-            sha3_step_vformat(&cx,"B%d:",n2);
+            hash_step_vformat(&cx,"B%d:",n2);
            SHA3Update(&cx, z2, n2);
            break;
          }
@@ -91,9 +91,7 @@ static void sqlarUncompressFunc(
  }else{
    const Bytef *pData= sqlite3_value_blob(argv[0]);
    Bytef *pOut = sqlite3_malloc(sz);
-    if( pOut==0 ){
-      sqlite3_result_error_nomem(context);
-    }else if( Z_OK!=uncompress(pOut, &sz, pData, nData) ){
+    if( Z_OK!=uncompress(pOut, &sz, pData, nData) ){
      sqlite3_result_error(context, "error in uncompress()", -1);
    }else{
      sqlite3_result_blob(context, pOut, sz, SQLITE_TRANSIENT);
@@ -102,6 +100,7 @@ static void sqlarUncompressFunc(
  }
 }

+
 #ifdef _WIN32
 __declspec(dllexport)
 #endif
@@ -97,10 +97,6 @@ static int stmtConnect(
 #define STMT_COLUMN_MEM    10   /* SQLITE_STMTSTATUS_MEMUSED */


-  (void)pAux;
-  (void)argc;
-  (void)argv;
-  (void)pzErr;
  rc = sqlite3_declare_vtab(db,
     "CREATE TABLE x(sql,ncol,ro,busy,nscan,nsort,naidx,nstep,"
                    "reprep,run,mem)");
@@ -220,10 +216,6 @@ static int stmtFilter(
  sqlite3_int64 iRowid = 1;
  StmtRow **ppRow = 0;

-  (void)idxNum;
-  (void)idxStr;
-  (void)argc;
-  (void)argv;
  stmtCsrReset(pCur);
  ppRow = &pCur->pRow;
  for(p=sqlite3_next_stmt(pCur->db, 0); p; p=sqlite3_next_stmt(pCur->db, p)){
@@ -279,7 +271,6 @@ static int stmtBestIndex(
  sqlite3_vtab *tab,
  sqlite3_index_info *pIdxInfo
 ){
-  (void)tab;
  pIdxInfo->estimatedCost = (double)500;
  pIdxInfo->estimatedRows = 500;
  return SQLITE_OK;
@@ -352,7 +352,6 @@ static int zipfileConnect(
  const char *zFile = 0;
  ZipfileTab *pNew = 0;
  int rc;
-  (void)pAux;

  /* If the table name is not "zipfile", require that the argument be
  ** specified. This stops zipfile tables from being created as:
@@ -809,7 +808,6 @@ static int zipfileGetEntry(
  u8 *aRead;
  char **pzErr = &pTab->base.zErrMsg;
  int rc = SQLITE_OK;
-  (void)nBlob;

  if( aBlob==0 ){
    aRead = pTab->aBuffer;
@@ -1097,10 +1095,7 @@ static int zipfileColumn(
          ** it to be a directory either if the mode suggests so, or if
          ** the final character in the name is '/'.  */
          u32 mode = pCDS->iExternalAttr >> 16;
-          if( !(mode & S_IFDIR)
-           && pCDS->nFile>=1
-           && pCDS->zFile[pCDS->nFile-1]!='/'
-          ){
+          if( !(mode & S_IFDIR) && pCDS->zFile[pCDS->nFile-1]!='/' ){
            sqlite3_result_blob(ctx, "", 0, SQLITE_STATIC);
          }
        }
@@ -1259,9 +1254,6 @@ static int zipfileFilter(
  int rc = SQLITE_OK;             /* Return Code */
  int bInMemory = 0;              /* True for an in-memory zipfile */

-  (void)idxStr;
-  (void)argc;
-
  zipfileResetCursor(pCsr);

  if( pTab->zFile ){
@@ -1288,7 +1280,7 @@ static int zipfileFilter(
  }

  if( 0==pTab->pWriteFd && 0==bInMemory ){
-    pCsr->pFile = zFile ? fopen(zFile, "rb") : 0;
+    pCsr->pFile = fopen(zFile, "rb");
    if( pCsr->pFile==0 ){
      zipfileCursorErr(pCsr, "cannot open file: %s", zFile);
      rc = SQLITE_ERROR;
@@ -1322,7 +1314,6 @@ static int zipfileBestIndex(
  int i;
  int idx = -1;
  int unusable = 0;
-  (void)tab;

  for(i=0; i<pIdxInfo->nConstraint; i++){
    const struct sqlite3_index_constraint *pCons = &pIdxInfo->aConstraint[i];
@@ -1573,8 +1564,6 @@ static int zipfileUpdate(
  int bIsDir = 0;
  u32 iCrc32 = 0;

-  (void)pRowid;
-
  if( pTab->pWriteFd==0 ){
    rc = zipfileBegin(pVtab);
    if( rc!=SQLITE_OK ) return rc;
@@ -1909,7 +1898,6 @@ static int zipfileFindFunction(
  void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */
  void **ppArg                    /* OUT: User data for *pxFunc */
 ){
-  (void)nArg;
  if( sqlite3_stricmp("zipfile_cds", zName)==0 ){
    *pxFunc = zipfileFunctionCds;
    *ppArg = (void*)pVtab;
@@ -11,10 +11,11 @@
 #

 source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
 set ::testprefix rbu1

 db close
+sqlite3_shutdown
+sqlite3_config_uri 1

 # Create a simple RBU database. That expects to write to a table:
 #
@@ -10,10 +10,13 @@
 #***********************************************************************
 #

-source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
 set ::testprefix rbu10

+
 #--------------------------------------------------------------------
 # Test that UPDATE commands work even if the input columns are in a 
 # different order to the output columns. 
@@ -10,8 +10,10 @@
 #***********************************************************************
 #

-source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
 set ::testprefix rbu11


@@ -10,8 +10,10 @@
 #***********************************************************************
 #

-source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
 source $testdir/lock_common.tcl
 set ::testprefix rbu12

@@ -13,8 +13,10 @@
 # for UPDATE statements. This tests RBU's internal UPDATE statement cache.
 #

-source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
 source $testdir/lock_common.tcl
 set ::testprefix rbu13

@@ -13,8 +13,10 @@
 # table.
 #

-source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
 source $testdir/lock_common.tcl
 set ::testprefix rbu14

@@ -10,8 +10,10 @@
 #***********************************************************************
 #

-source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
 set ::testprefix rbu3


@@ -24,6 +26,9 @@ proc run_rbu {target rbu} {
 }

 forcedelete test.db-oal rbu.db
+db close
+sqlite3_shutdown
+sqlite3_config_uri 1
 reset_db

 #--------------------------------------------------------------------
@@ -13,7 +13,6 @@
 #

 source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
 set ::testprefix rbu5


@@ -13,8 +13,10 @@
 # outcome of some other client writing to the database while an RBU update
 # is being applied.

-source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
 set ::testprefix rbu6

 proc setup_test {} {
@@ -13,8 +13,10 @@
 #


-source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
 set ::testprefix rbu7

 # Test index:
@@ -12,8 +12,10 @@
 # Test the rbu_delta() feature.
 #

-source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
 set ::testprefix rbu8

 do_execsql_test 1.0 {
@@ -12,8 +12,10 @@
 # Test RBU with virtual tables. And tables with no PRIMARY KEY declarations.
 #

-source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
 set ::testprefix rbu9

 ifcapable !fts3 {
@@ -14,8 +14,10 @@
 # a wal mode database via RBU.
 #

-source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
 set ::testprefix rbuA

 set db_sql {
@@ -12,7 +12,6 @@
 #

 source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
 set ::testprefix rbuB

 db close
@@ -49,7 +48,7 @@ do_test 1.2 {

 do_test 1.3 {
  set ::errlog
-} {SQLITE_NOTICE_RECOVER_WAL SQLITE_NOTICE_RBU}
+} {SQLITE_NOTICE_RECOVER_WAL SQLITE_INTERNAL}

 do_execsql_test 1.4 {
  SELECT * FROM t1
@@ -13,7 +13,6 @@
 #

 source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
 set ::testprefix rbuC

 #-------------------------------------------------------------------------
@@ -15,16 +15,6 @@ if {![info exists testdir]} {
 }
 source $testdir/tester.tcl

-proc if_no_rbu_support {tcl} {
-  set bOk 1
-  ifcapable !rbu { set bOk 0 }
-  if {[permutation]=="journaltest"} { set bOk 0 }
-  if {$bOk==0} {
-    set c [catch {uplevel 1 $tcl} r]
-    return -code $c $r
-  } 
-}
-
 proc check_prestep_state {target state} {
  set oal_exists [file exists $target-oal]
  set wal_exists [file exists $target-wal]
@@ -12,8 +12,6 @@
 #

 source [file join [file dirname [info script]] rbu_common.tcl]
-ifcapable !rbu { finish_test ; return }
-if_no_rbu_support { finish_test ; return }
 set ::testprefix rbubusy

 db close
@@ -11,7 +11,6 @@
 #

 source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
 set ::testprefix rbucollate

 ifcapable !icu_collations {
@@ -19,6 +18,11 @@ ifcapable !icu_collations {
  return
 }

+db close
+sqlite3_shutdown
+sqlite3_config_uri 1
+reset_db
+ 
 # Create a simple RBU database. That expects to write to a table:
 #
 #   CREATE TABLE t1(a PRIMARY KEY, b, c);
@@ -10,10 +10,18 @@
 #***********************************************************************
 #

-source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
 set ::testprefix rbucrash

+db close
+forcedelete test.db-oal rbu.db
+sqlite3_shutdown
+sqlite3_config_uri 1
+reset_db
+
 # Set up a target database and an rbu update database. The target
 # db is the usual "test.db", the rbu db is "test.db2".
 #
@@ -10,10 +10,18 @@
 #***********************************************************************
 #

-source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
 set ::testprefix rbucrash2

+db close
+forcedelete test.db-oal rbu.db
+sqlite3_shutdown
+sqlite3_config_uri 1
+reset_db
+
 # Set up a target database and an rbu update database. The target
 # db is the usual "test.db", the rbu db is "test.db2".
 #
@@ -12,9 +12,10 @@
 # Tests for the [sqldiff --rbu] command.
 #
 #
-
-source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
 set testprefix rbudiff

 set PROG [test_find_sqldiff]
@@ -13,8 +13,10 @@
 # enabled by SQLITE_DIRECT_OVERFLOW_READ - Direct Overflow Read.
 #

-source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
 set ::testprefix rbudor

 set bigA [string repeat a 5000]
@@ -11,15 +11,11 @@
 #

 source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
 set ::testprefix rbuexlock

 db close
-
-set journalmode delete
-if {[permutation]=="inmemory_journal"} {
-  set journalmode memory
-}
+sqlite3_shutdown
+sqlite3_config_uri 1

 # Create a simple RBU database. That expects to write to a table:
 #
@@ -68,7 +64,7 @@ do_test 1.2.0 {
 do_catchsql_test 1.2.1 { SELECT * FROM t1 } {0 {1 2 3}}
 do_test 1.2.2 {
  db eval {PRAGMA journal_mode}
-} $journalmode
+} {delete}

 do_test 1.3.0 {
  while {[file exists test.db-wal]==0} {
@@ -78,7 +74,7 @@ do_test 1.3.0 {
 do_catchsql_test 1.3.1 { SELECT * FROM t1 } {1 {database is locked}}
 do_test 1.3.2 {
  db eval {PRAGMA journal_mode}
-} $journalmode
+} {delete}


 do_test 1.4.0 {
@@ -87,7 +83,7 @@ do_test 1.4.0 {
 do_catchsql_test 1.4.1 { SELECT * FROM t1 } {1 {database is locked}}
 do_test 1.4.2 {
  db eval {PRAGMA journal_mode}
-} $journalmode
+} {delete}


 rbu close
@@ -102,7 +98,7 @@ do_test 1.5.1 {
 do_catchsql_test 1.5.2 { SELECT * FROM t1 } {1 {database is locked}}
 do_test 1.5.2 {
  db eval {PRAGMA journal_mode}
-} $journalmode
+} {delete}


 do_test 1.6.0 {
@@ -111,7 +107,7 @@ do_test 1.6.0 {
 do_catchsql_test 1.6.1 { SELECT * FROM t1 } {1 {database is locked}}
 do_test 1.6.2 {
  db eval {PRAGMA journal_mode}
-} $journalmode
+} {delete}

 do_test 1.7.0 {
  while {[rbu step]=="SQLITE_OK"} {}
@@ -120,7 +116,7 @@ do_test 1.7.0 {
 do_catchsql_test 1.7.2 { SELECT count(*) FROM t1 } {0 9}
 do_test 1.7.2 {
  db eval {PRAGMA journal_mode}
-} $journalmode
+} {delete}

 reset_db
 do_execsql_test 2.0 {
@@ -11,9 +11,14 @@
 #

 source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
 set ::testprefix rbuexpr

+db close
+sqlite3_shutdown
+sqlite3_config_uri 1
+
+sqlite3 db test.db
+
 do_execsql_test 1.0 {
  CREATE TABLE t1(a, b, c PRIMARY KEY);
  CREATE INDEX i1 ON t1(a, null, b+1);
@@ -10,8 +10,10 @@
 #***********************************************************************
 #

-source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
 source $testdir/malloc_common.tcl
 set ::testprefix rbufault

@@ -10,8 +10,10 @@
 #***********************************************************************
 #

-source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
 source $testdir/malloc_common.tcl
 set ::testprefix rbufault2

@@ -13,7 +13,6 @@
 #

 source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
 source $testdir/malloc_common.tcl
 set ::testprefix rbufault3

@@ -10,8 +10,10 @@
 #***********************************************************************
 #

-source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
 source $testdir/malloc_common.tcl
 set ::testprefix rbufault4

@@ -24,7 +26,6 @@ for {set tn 1} {1} {incr tn} {
    INSERT INTO t1 VALUES(1, 2, 3);
    INSERT INTO t1 VALUES(4, 5, 6);
  }
-  db close

  forcedelete test.db2
  sqlite3rbu_vacuum rbu test.db test.db2
@@ -54,10 +55,8 @@ for {set tn 1} {1} {incr tn} {
    set trc [rbu close]
    if {$trc!="SQLITE_DONE"} { error "Got $trc instead of SQLITE_DONE!" }

-    sqlite3 db test.db
    set rc [db one {PRAGMA integrity_check}]
    if {$rc!="ok"} { error "Got $rc instead of ok!" }
-    db close
  }
 }

@@ -13,8 +13,10 @@
 # contains tests to ensure that RBU works with FTS tables.
 #

-source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
+if {![info exists testdir]} {
+  set testdir [file join [file dirname [info script]] .. .. test]
+}
+source $testdir/tester.tcl
 set ::testprefix rbufts

 ifcapable !fts3 {
@@ -11,9 +11,13 @@
 #

 source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
 set ::testprefix rbumisc

+db close
+sqlite3_shutdown
+sqlite3_config_uri 1
+reset_db
+
 proc populate_rbu_db {} {
  forcedelete rbu.db
  sqlite3 rbu rbu.db
@@ -9,17 +9,16 @@
 #
 #***********************************************************************
 #
-# TESTRUNNER: slow
-#
 # This file contains tests of multiple RBU operations running
 # concurrently within the same process. 
 #

 source [file join [file dirname [info script]] rbu_common.tcl]
-if_no_rbu_support { finish_test ; return }
 set ::testprefix rbumulti

 db close
+sqlite3_shutdown
+sqlite3_config_uri 1

 autoinstall_test_functions

--- a/Show More
+++ b/Show More
@@ -1 +1 @@
 .42.0
 .40.0