Version 3.47.0

FossilOrigin-Name: 03a9703e27c44437c39363d0baf82db4ebc94538a0f28411c85dda156f82636e
Add a case to testrunner.tcl so that mdevtest can run on openbsd.
2024-10-21 16:30:22 +00:00 · 2024-10-21 14:24:27 +00:00 · 2024-10-21 10:47:24 +00:00 · 2024-10-20 07:19:38 +00:00 · 2024-10-19 22:45:50 +00:00 · 2024-10-19 13:38:51 +00:00
425 changed files with 22588 additions and 8222 deletions
@@ -83,9 +83,13 @@ TEMP_STORE = -DSQLITE_TEMP_STORE=@TEMP_STORE@
 # based on configuration. (-DSQLITE_OMIT*, -DSQLITE_ENABLE*).
 # The same set of OMIT and ENABLE flags should be passed to the
 # LEMON parser generator and the mkkeywordhash tool as well.
-OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@
+#
+# Add OPTIONS=... on the command line to append additional options
+# to the OPT_FEATURE_FLAGS.
+#
+OPT_FEATURE_FLAGS = @OPT_FEATURE_FLAGS@ $(OPTIONS)

-TCC += $(OPT_FEATURE_FLAGS)
+TCC += $(OPT_FEATURE_FLAGS) 

 # Add in any optional parameters specified on the make commane line
 # ie.  make "OPTS=-DSQLITE_ENABLE_FOO=1 -DSQLITE_OMIT_FOO=1".
@@ -117,14 +121,15 @@ HAVE_TCL = @HAVE_TCL@
 #
 TCLSH_CMD = @TCLSH_CMD@

+# Additional options when running tests using testrunner.tcl
+# This is usually either blank, or else --status
+#
+TSTRNNR_OPTS = @TSTRNNR_OPTS@
+
 # Where do we want to install the tcl plugin
 #
 TCLLIBDIR = @TCLLIBDIR@

-# The suffix used on shared libraries.  Ex:  ".dll", ".so", ".dylib"
-#
-SHLIB_SUFFIX = @TCL_SHLIB_SUFFIX@
-
 # If gcov support was enabled by the configure script, add the appropriate
 # flags here.  It's not always as easy as just having the user add the right
 # CFLAGS / LDFLAGS, because libtool wants to use CFLAGS when linking, which
@@ -454,6 +459,7 @@ TESTSRC += \
  $(TOP)/ext/misc/remember.c \
  $(TOP)/ext/misc/series.c \
  $(TOP)/ext/misc/spellfix.c \
+  $(TOP)/ext/misc/stmtrand.c \
  $(TOP)/ext/misc/totype.c \
  $(TOP)/ext/misc/unionvtab.c \
  $(TOP)/ext/misc/wholenumber.c \
@@ -644,6 +650,7 @@ FUZZCHECK_SRC += $(TOP)/test/fuzzinvariants.c
 FUZZCHECK_SRC += $(TOP)/ext/recover/dbdata.c
 FUZZCHECK_SRC += $(TOP)/ext/recover/sqlite3recover.c
 FUZZCHECK_SRC += $(TOP)/test/vt02.c
+FUZZCHECK_SRC += $(TOP)/ext/misc/percentile.c
 FUZZCHECK_SRC += $(TOP)/ext/misc/randomjson.c
 DBFUZZ_OPT =
 ST_OPT = -DSQLITE_OS_KV_OPTIONAL
@@ -654,11 +661,16 @@ SQLITE3_SHELL_TARGET_  = sqlite3$(TEXE)
 SQLITE3_SHELL_TARGET_1 =
 SQLITE3_SHELL_TARGET   = $(SQLITE3_SHELL_TARGET_@HAVE_WASI_SDK@)

+# Use $(libtclsqlite3.la_$(HAVE_TCL)) to resolve to either
+# libtclsqlite3.la or an empty value.
+libtclsqlite3.la_0 =
+libtclsqlite3.la_1 = libtclsqlite3.la
+
 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
 #
 all:	sqlite3.h libsqlite3.la $(SQLITE3_SHELL_TARGET) \
-  $(HAVE_TCL:1=libtclsqlite3.la)
+  $(libtclsqlite3.la_$(HAVE_TCL))

 Makefile: $(TOP)/Makefile.in
 	./config.status
@@ -682,12 +694,26 @@ sqlite3$(TEXE):	shell.c sqlite3.c
 		shell.c sqlite3.c \
 		$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"

-sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c sqlite3.lo sqlite3.h
-	$(LTLINK) -o $@ $(TOP)/tool/sqldiff.c sqlite3.lo $(TLIBS)
+sqldiff$(TEXE):	$(TOP)/tool/sqldiff.c $(TOP)/ext/misc/sqlite3_stdio.h sqlite3.lo sqlite3.h
+	$(LTLINK) -I$(TOP)/ext/misc -o $@ $(TOP)/tool/sqldiff.c sqlite3.lo $(TLIBS)

 dbhash$(TEXE):	$(TOP)/tool/dbhash.c sqlite3.lo sqlite3.h
 	$(LTLINK) -o $@ $(TOP)/tool/dbhash.c sqlite3.lo $(TLIBS)

+RSYNC_SRC = \
+  $(TOP)/tool/sqlite3_rsync.c \
+  sqlite3.c
+
+RSYNC_OPT = \
+  -DSQLITE_ENABLE_DBPAGE_VTAB \
+  -USQLITE_THREADSAFE \
+  -DSQLITE_THREADSAFE=0 \
+  -DSQLITE_OMIT_LOAD_EXTENSION \
+  -DSQLITE_OMIT_DEPRECATED
+
+sqlite3_rsync$(TEXE):	$(RSYNC_SRC)
+	$(TCC) -o $@ $(RSYNC_OPT) $(RSYNC_SRC) $(TLIBS)
+
 scrub$(TEXE):	$(TOP)/ext/misc/scrub.c sqlite3.lo
 	$(LTLINK) -o $@ -I. -DSCRUB_STANDALONE \
 		$(TOP)/ext/misc/scrub.c sqlite3.lo $(TLIBS)
@@ -696,13 +722,13 @@ srcck1$(BEXE):	$(TOP)/tool/srcck1.c
 	$(BCC) -o srcck1$(BEXE) $(TOP)/tool/srcck1.c

 sourcetest:	srcck1$(BEXE) sqlite3.c
-	./srcck1 sqlite3.c
+	./srcck1$(BEXE) sqlite3.c

-src-verify:	$(TOP)/tool/src-verify.c
+src-verify$(BEXE):	$(TOP)/tool/src-verify.c
 	$(BCC) -o src-verify$(BEXE) $(TOP)/tool/src-verify.c

-verify-source:	./src-verify
-	./src-verify $(TOP)
+verify-source:	./src-verify$(BEXE)
+	./src-verify$(BEXE) $(TOP)

 fuzzershell$(TEXE):	$(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(FUZZERSHELL_OPT) \
@@ -759,20 +785,6 @@ dbfuzz2$(TEXE):	$(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h
 	mkdir -p dbfuzz2-dir
 	cp $(TOP)/test/dbfuzz2-seed* dbfuzz2-dir

-dbfuzz2-asan:	$(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h
-	clang-6.0 $(OPT_FEATURE_FLAGS) $(OPTS) -I. -g -O0 \
-		-fsanitize=fuzzer,undefined,address -o dbfuzz2-asan \
-		$(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c $(TLIBS)
-	mkdir -p dbfuzz2-dir
-	cp $(TOP)/test/dbfuzz2-seed* dbfuzz2-dir
-
-dbfuzz2-msan:	$(TOP)/test/dbfuzz2.c sqlite3.c sqlite3.h
-	clang-6.0 $(OPT_FEATURE_FLAGS) $(OPTS) -I. -g -O0 \
-		-fsanitize=fuzzer,undefined,memory -o dbfuzz2-msan \
-		$(DBFUZZ2_OPTS) $(TOP)/test/dbfuzz2.c sqlite3.c $(TLIBS)
-	mkdir -p dbfuzz2-dir
-	cp $(TOP)/test/dbfuzz2-seed* dbfuzz2-dir
-
 mptester$(TEXE):	sqlite3.lo $(TOP)/mptest/mptest.c
 	$(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.lo \
 		$(TLIBS) -rpath "$(libdir)"
@@ -799,6 +811,10 @@ has_tclsh85:
 	sh $(TOP)/tool/cktclsh.sh 8.5 $(TCLSH_CMD)
 	touch has_tclsh85

+has_tclconfig:
+	@ if test x"$(HAVE_TCL)" != "x1"; then	echo 'ERROR: Requires access to "tclConfig.sh" which "configure" was not able to locate'; exit 1; fi
+	touch has_tclconfig
+

 # This target creates a directory named "tsrc" and fills it with
 # copies of all of the C source code and header files needed to
@@ -816,7 +832,7 @@ has_tclsh85:
 	cp fts5.c fts5.h tsrc
 	touch .target_source

-sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl src-verify has_tclsh84
+sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl src-verify$(BEXE) has_tclsh84
 	$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl $(AMALGAMATION_LINE_MACROS) $(EXTRA_SRC)
 	cp tsrc/sqlite3ext.h .
 	cp $(TOP)/ext/session/sqlite3session.h .
@@ -1121,7 +1137,7 @@ tclsqlite-shell.lo:	$(TOP)/src/tclsqlite.c $(HDR)
 tclsqlite-stubs.lo:	$(TOP)/src/tclsqlite.c $(HDR)
 	$(LTCOMPILE) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c

-tclsqlite3$(TEXE):	tclsqlite-shell.lo libsqlite3.la
+tclsqlite3$(TEXE):	has_tclconfig tclsqlite-shell.lo libsqlite3.la
 	$(LTLINK) -o $@ tclsqlite-shell.lo \
 		 libsqlite3.la $(LIBTCL)

@@ -1157,8 +1173,6 @@ keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
 # Source and header files that shell.c depends on
 SHELL_DEP = \
    $(TOP)/src/shell.c.in \
-    $(TOP)/ext/consio/console_io.c \
-    $(TOP)/ext/consio/console_io.h \
    $(TOP)/ext/expert/sqlite3expert.c \
    $(TOP)/ext/expert/sqlite3expert.h \
    $(TOP)/ext/intck/sqlite3intck.c \
@@ -1172,11 +1186,16 @@ SHELL_DEP = \
    $(TOP)/ext/misc/ieee754.c \
    $(TOP)/ext/misc/memtrace.c \
    $(TOP)/ext/misc/pcachetrace.c \
+    $(TOP)/ext/misc/percentile.c \
    $(TOP)/ext/misc/regexp.c \
    $(TOP)/ext/misc/series.c \
+    $(TOP)/ext/misc/sha1.c \
    $(TOP)/ext/misc/shathree.c \
    $(TOP)/ext/misc/sqlar.c \
+    $(TOP)/ext/misc/sqlite3_stdio.c \
+    $(TOP)/ext/misc/sqlite3_stdio.h \
    $(TOP)/ext/misc/uint.c \
+    $(TOP)/ext/misc/vfstrace.c \
    $(TOP)/ext/misc/zipfile.c \
    $(TOP)/ext/recover/dbdata.c \
    $(TOP)/ext/recover/sqlite3recover.c \
@@ -1308,7 +1327,7 @@ TESTFIXTURE_SRC1 = sqlite3.c
 TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c
 TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION))

-testfixture$(TEXE):	has_tclsh85 $(TESTFIXTURE_SRC)
+testfixture$(TEXE):	has_tclconfig has_tclsh85 $(TESTFIXTURE_SRC)
 	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \
 		-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)

@@ -1357,15 +1376,20 @@ tcltest:	./testfixture$(TEXE)
 testrunner:	testfixture$(TEXE)
 	./testfixture$(TEXE) $(TOP)/test/testrunner.tcl

-# Runs both fuzztest and testrunner, consecutively.
+# This is the testing target preferred by the core SQLite developers.
+# It runs tests under a standard configuration, regardless of how
+# ./configure was run.  The devs run "make devtest" prior to each
+# check-in, at a minimum.  Probably other tests too, but at least this
+# one.
 #
-devtest:	srctree-check testfixture$(TEXE) fuzztest testrunner
+devtest:	srctree-check sourcetest
+	$(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest $(TSTRNNR_OPTS)

 mdevtest: srctree-check has_tclsh85
-	$(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest
+	$(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest $(TSTRNNR_OPTS)

 sdevtest: has_tclsh85
-	$(TCLSH_CMD) $(TOP)/test/testrunner.tcl sdevtest
+	$(TCLSH_CMD) $(TOP)/test/testrunner.tcl sdevtest $(TSTRNNR_OPTS)

 # Validate that various generated files in the source tree
 # are up-to-date.
@@ -1375,16 +1399,19 @@ srctree-check:	$(TOP)/tool/srctree-check.tcl

 # Testing for a release
 #
-releasetest: srctree-check testfixture$(TEXE)
-	./testfixture$(TEXE) $(TOP)/test/testrunner.tcl release
+releasetest: srctree-check has_tclsh85 verify-source
+	$(TCLSH_CMD) $(TOP)/test/testrunner.tcl release $(TSTRNNR_OPTS)

 # Minimal testing that runs in less than 3 minutes
 #
 quicktest:	./testfixture$(TEXE)
 	./testfixture$(TEXE) $(TOP)/test/extraquick.test $(TESTOPTS)

-# This is the common case.  Run many tests that do not take too long,
-# including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
+# Try to run tests on whatever options are specified by the
+# ./configure.  The developers seldom use this target.  Instead
+# they use "make devtest" which runs tests on a standard set of
+# options regardless of how SQLite is configured.  This "test"
+# target is provided for legacy only.
 #
 test:	srctree-check fuzztest sourcetest $(TESTPROGS) tcltest

@@ -1401,19 +1428,19 @@ valgrindtest:	$(TESTPROGS) valgrindfuzz
 smoketest:	$(TESTPROGS) fuzzcheck$(TEXE)
 	./testfixture$(TEXE) $(TOP)/test/main.test $(TESTOPTS)

-shelltest: $(TESTPROGS)
-	./testfixture$(TEXT) $(TOP)/test/permutations.test shell
+shelltest:
+	$(TCLSH_CMD) $(TOP)/test/testrunner.tcl release shell

 sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in has_tclsh85
 	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c

-sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
+sqlite3_analyzer$(TEXE): has_tclconfig sqlite3_analyzer.c
 	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)

 sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in has_tclsh85
 	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c

-sqltclsh$(TEXE): sqltclsh.c
+sqltclsh$(TEXE): has_tclconfig sqltclsh.c
 	$(LTLINK) sqltclsh.c -o $@ $(LIBTCL) $(TLIBS)

 sqlite3_expert$(TEXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
@@ -1432,7 +1459,7 @@ CHECKER_DEPS =\
 sqlite3_checker.c:	$(CHECKER_DEPS) has_tclsh85
 	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@

-sqlite3_checker$(TEXE):	sqlite3_checker.c
+sqlite3_checker$(TEXE):	has_tclconfig sqlite3_checker.c
 	$(LTLINK) sqlite3_checker.c -o $@ $(LIBTCL) $(TLIBS)

 dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
@@ -1517,8 +1544,10 @@ snapshot-tarball: sqlite3.c sqlite3rc.h

 # Build a ZIP archive containing various command-line tools.
 #
-tool-zip:	testfixture sqlite3 sqldiff sqlite3_analyzer $(TOP)/tool/mktoolzip.tcl
-	./testfixture $(TOP)/tool/mktoolzip.tcl
+tool-zip: testfixture$(TEXE) sqlite3$(TEXE) sqldiff$(TEXE) \
+          sqlite3_analyzer$(TEXE) sqlite3_rsync$(TEXE) $(TOP)/tool/mktoolzip.tcl
+	strip sqlite3$(TEXE) sqldiff$(TEXE) sqlite3_analyzer$(TEXE) sqlite3_rsync$(TEXE)
+	./testfixture$(TEXE) $(TOP)/tool/mktoolzip.tcl

 # The next two rules are used to support the "threadtest" target. Building
 # threadtest runs a few thread-safety tests that are implemented in C. This
@@ -1546,7 +1575,12 @@ lib_install:	libsqlite3.la
 	$(INSTALL) -d $(DESTDIR)$(libdir)
 	$(LTINSTALL) libsqlite3.la $(DESTDIR)$(libdir)

-install:	sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_install}
+# Use $(tcl_install_$(HAVE_TCL)) to resolve to either tclextension-install or
+# an empty value.
+tcl_install_0 =
+tcl_install_1 = tclextension-install
+
+install:	sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc $(tcl_install_$(HAVE_TCL))
 	$(INSTALL) -d $(DESTDIR)$(bindir)
 	$(LTINSTALL) sqlite3$(TEXE) $(DESTDIR)$(bindir)
 	$(INSTALL) -d $(DESTDIR)$(includedir)
@@ -1555,51 +1589,64 @@ install:	sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_instal
 	$(INSTALL) -d $(DESTDIR)$(pkgconfigdir)
 	$(INSTALL) -m 0644 sqlite3.pc $(DESTDIR)$(pkgconfigdir)

-pkgIndex.tcl:
-	echo 'package ifneeded sqlite3 $(RELEASE) [list load [file join $$dir libtclsqlite3[info sharedlibextension]] sqlite3]' > $@
-tcl_install:	lib_install libtclsqlite3.la pkgIndex.tcl
-	$(INSTALL) -d $(DESTDIR)$(TCLLIBDIR)
-	$(LTINSTALL) libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)
-	rm -f $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.a
-	$(INSTALL) -m 0644 pkgIndex.tcl $(DESTDIR)$(TCLLIBDIR)
+# Build the SQLite TCL extension in a way that make it compatible
+# with whatever version of TCL is running as $TCLSH_CMD, possibly defined
+# by --with-tclsh=
+#
+tclextension:	tclsqlite3.c
+	$(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --build-only --cc "$(CC)" $(CFLAGS) $(OPT_FEATURE_FLAGS) $(OPTS)

-clean:
-	rm -f *.lo *.la *.o sqlite3$(TEXE) libsqlite3.la
-	rm -f sqlite3.h opcodes.*
-	rm -rf .libs .deps
-	rm -f lemon$(BEXE) lempar.c parse.* sqlite*.tar.gz
-	rm -f mkkeywordhash$(BEXE) keywordhash.h
-	rm -f mksourceid$(BEXE)
-	rm -f *.da *.bb *.bbg gmon.out
-	rm -rf tsrc .target_source
-	rm -f tclsqlite3$(TEXE)
-	rm -f testfixture$(TEXE) test.db
+# Install the SQLite TCL extension in a way that is appropriate for $TCLSH_CMD
+# to find it.
+#
+tclextension-install:	tclsqlite3.c
+	$(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --cc "$(CC)" $(CFLAGS) $(OPT_FEATURE_FLAGS) $(OPTS)
+
+# Install the SQLite TCL extension that is used by $TCLSH_CMD
+#
+tclextension-uninstall: 
+	$(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --uninstall
+
+# List all installed the SQLite TCL extension that is are accessible
+# by $TCLSH_CMD, included prior versions.
+#
+tclextension-list: 
+	$(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --info
+
+
+# Remove build products sufficient so that subsequent makes will recompile
+# everything from scratch.  Do not remove:
+#
+#   *   test results and test logs
+#   *   output from ./configure
+#
+tidy:
+	rm -f *.lo *.la *.o *.c *.da *.bb *.bbg gmon.* *.rws sqlite3$(TEXE)
+	rm -f fts5.h keywordhash.h opcodes.h sqlite3.h sqlite3ext.h sqlite3session.h
+	rm -rf .libs .deps tsrc .target_source
+	rm -f lemon$(BEXE) sqlite*.tar.gz
+	rm -f mkkeywordhash$(BEXE) mksourceid$(BEXE)
+	rm -f parse.* fts5parse.*
+	rm -f tclsqlite3$(TEXE) $(TESTPROGS)
 	rm -f LogEst$(TEXE) fts3view$(TEXE) rollback-test$(TEXE) showdb$(TEXE)
 	rm -f showjournal$(TEXE) showstat4$(TEXE) showwal$(TEXE) speedtest1$(TEXE)
-	rm -f wordcount$(TEXE) changeset$(TEXE)
-	rm -f version-info$(TEXT)
-	rm -f sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def
-	rm -f sqlite3.c
-	rm -f sqlite3rc.h
-	rm -f shell.c sqlite3ext.h
-	rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c
-	rm -f sqlite-*-output.vsix
-	rm -f mptester mptester.exe
-	rm -f rbu rbu.exe
-	rm -f srcck1 srcck1.exe
-	rm -f fuzzershell fuzzershell.exe
-	rm -f fuzzcheck fuzzcheck.exe
-	rm -f sqldiff sqldiff.exe
-	rm -f dbhash dbhash.exe
-	rm -f fts5.* fts5parse.*
-	rm -f threadtest5
-	rm -f src-verify
-	rm -f custom.rws
-	rm -f has_tclsh84 has_tclsh85
+	rm -f wordcount$(TEXE) changeset$(TEXE) version-info$(TEXE)
+	rm -f *.dll *.lib *.exp *.def *.pc *.vsix *.so *.dylib pkgIndex.tcl
+	rm -f sqlite3_analyzer$(TEXE) sqlite3_rsync$(TEXE)
+	rm -f mptester$(TEXE) rbu$(TEXE)	srcck1$(TEXE)
+	rm -f fuzzershell$(TEXE) fuzzcheck$(TEXE) sqldiff$(TEXE) dbhash$(TEXE)
+	rm -f threadtest5$(TEXE)
+	rm -f src-verify$(BEXE) has_tclsh* has_tclconfig

+# Removes build products and test logs.  Retains ./configure outputs.
+#
+clean:	tidy
+	rm -rf omittest* testrunner* testdir*
+
+# Clean up everything.  No exceptions.
+#
 distclean:	clean
-	rm -f sqlite_cfg.h config.log config.status libtool Makefile sqlite3.pc \
-		$(TESTPROGS)
+	rm -f sqlite_cfg.h config.log config.status Makefile $(LIBTOOL)

 #
 # Windows section
@@ -66,6 +66,14 @@ USE_STDCALL = 0
 USE_SEH = 1
 !ENDIF

+# Use STATICALLY_LINK_TCL=1 to statically link against TCL
+#
+!IFNDEF STATICALLY_LINK_TCL
+STATICALLY_LINK_TCL = 0
+!ELSEIF $(STATICALLY_LINK_TCL)!=0
+CCOPTS = $(CCOPTS) -DSTATIC_BUILD
+!ENDIF
+
 # Set this non-0 to have the shell executable link against the core dynamic
 # link library.
 #
@@ -379,6 +387,7 @@ SQLITE_TCL_DEP =
 # the Windows platform.
 #
 !IFNDEF OPT_FEATURE_FLAGS
+OPT_FEATURE_FLAGS = $(OPT_XTRA)
 !IF $(MINIMAL_AMALGAMATION)==0
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS5=1
@@ -392,6 +401,14 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_BYTECODE_VTAB=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
 !ENDIF

+# Additional feature-options above and beyond what are normally used can be
+# be added using OPTIONS=.... on the command-line.  These values are
+# appended to the OPT_FEATURE_FLAGS variable.
+#
+!IFDEF OPTIONS
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) $(OPTIONS)
+!ENDIF
+
 # Should the session extension be enabled?  If so, add compilation options
 # to enable it.
 #
@@ -920,16 +937,28 @@ TCC = $(TCC) /fsanitize=address
 # prior to running nmake in order to match the actual installed location and
 # version on this machine.
 #
-!IFNDEF TCLVERSION
-TCLVERSION = 86
+!IF $(STATICALLY_LINK_TCL)!=0
+TCLSUFFIX = s
 !ENDIF
-
 !IFNDEF TCLSUFFIX
 TCLSUFFIX =
 !ENDIF

 !IFNDEF TCLDIR
-TCLDIR = $(TOP)\compat\tcl
+TCLDIR = C:\Tcl
+!ENDIF
+
+!IFNDEF TCLVERSION
+!IF EXISTS("$(TCLDIR)\lib\tcl90$(TCLSUFFIX).lib")
+TCLVERSION = 90
+!ELSEIF EXISTS("$(TCLDIR)\lib\tcl86$(TCLSUFFIX).lib")
+TCLVERSION = 86
+!ELSEIF EXISTS("$(TCLDIR)\lib\tcl86t.lib")
+TCLSUFFIX = t
+TCLVERSION = 86
+!ELSE
+TCLVERSION = 90
+!ENDIF
 !ENDIF

 !IFNDEF TCLINCDIR
@@ -944,9 +973,21 @@ TCLLIBDIR = $(TCLDIR)\lib
 LIBTCL = tcl$(TCLVERSION)$(TCLSUFFIX).lib
 !ENDIF

+!IFNDEF TCLLIBS
+!IF $(STATICALLY_LINK_TCL)!=0
+TCLLIBS = /NODEFAULTLIB:libucrt.lib netapi32.lib user32.lib ucrt.lib
+!ELSE
+TCLLIBS =
+!ENDIF
+!ENDIF
+
 !IFNDEF LIBTCLSTUB
+!IF EXISTS("$(TCLLIBDIR)\tclstub$(TCLSUFFIX).lib")
+LIBTCLSTUB = tclstub$(TCLSUFFIX).lib
+!ELSE
 LIBTCLSTUB = tclstub$(TCLVERSION)$(TCLSUFFIX).lib
 !ENDIF
+!ENDIF

 !IFNDEF LIBTCLPATH
 LIBTCLPATH = $(TCLDIR)\bin
@@ -1004,10 +1045,18 @@ LIBICU = icuuc.lib icuin.lib
 # specific Tcl shell to use.
 #
 !IFNDEF TCLSH_CMD
-!IF $(USE_TCLSH_IN_PATH)!=0 || !EXIST("$(TCLDIR)\bin\tclsh.exe")
-TCLSH_CMD = tclsh
-!ELSE
+!IF EXISTS("$(TCLDIR)\bin\tclsh$(TCLVERSION).exe")
+TCLSH_CMD = $(TCLDIR)\bin\tclsh$(TCLVERSION).exe
+!ELSEIF EXISTS("$(TCLDIR)\bin\tclsh90.exe")
+TCLSH_CMD = $(TCLDIR)\bin\tclsh90.exe
+!ELSEIF EXISTS("$(TCLDIR)\bin\tclsh86.exe")
+TCLSH_CMD = $(TCLDIR)\bin\tclsh86.exe
+!ELSEIF EXISTS("$(TCLDIR)\bin\tclsh86t.exe")
+TCLSH_CMD = $(TCLDIR)\bin\tclsh86t.exe
+!ELSEIF EXISTS("$(TCLDIR)\bin\tclsh.exe")
 TCLSH_CMD = $(TCLDIR)\bin\tclsh.exe
+!ELSE
+TCLSH_CMD = tclsh
 !ENDIF
 !ENDIF
 # <</mark>>
@@ -1596,6 +1645,7 @@ TESTEXT = \
  $(TOP)\ext\misc\remember.c \
  $(TOP)\ext\misc\series.c \
  $(TOP)\ext\misc\spellfix.c \
+  $(TOP)\ext\misc\stmtrand.c \
  $(TOP)\ext\misc\totype.c \
  $(TOP)\ext\misc\unionvtab.c \
  $(TOP)\ext\misc\wholenumber.c \
@@ -1757,6 +1807,7 @@ FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\fuzzinvariants.c
 FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\vt02.c
 FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\dbdata.c
 FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\sqlite3recover.c
+FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\percentile.c
 FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\randomjson.c

 OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c
@@ -1813,12 +1864,25 @@ tclsqlite3.def:	tclsqlite.lo

 pkgIndex.tcl:	$(TOP)\VERSION
 	for /F %%V in ('type "$(TOP)\VERSION"') do ( \
-		echo package ifneeded sqlite3 @version@ [list load [file join $$dir $(SQLITE3TCLDLL)] sqlite3] \
+		echo package ifneeded sqlite3 @version@ [list load [file join $$dir $(SQLITE3TCLDLL)] Sqlite3] \
 			| $(TCLSH_CMD) $(TOP)\tool\replace.tcl exact @version@ %%V > pkgIndex.tcl \
 	)

 $(SQLITE3TCLDLL):	libtclsqlite3.lib $(LIBRESOBJS) tclsqlite3.def pkgIndex.tcl
 	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /DEF:tclsqlite3.def /OUT:$@ libtclsqlite3.lib $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
+
+tclextension:	$(SQLITE3TCLDLL)
+
+tclextension-install:	$(SQLITE3TCLDLL)
+	$(TCLSH_CMD) $(TOP)\tool\buildtclext.tcl --install-only
+
+tclextension-uninstall:
+	$(TCLSH_CMD) $(TOP)\tool\buildtclext.tcl --uninstall
+
+tclextension-list:
+	$(TCLSH_CMD) $(TOP)\tool\buildtclext.tcl --info
+
+
 # <</mark>>

 $(SQLITE3DLL):	$(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
@@ -1837,12 +1901,25 @@ $(SQLITE3EXE):	shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLIT
 		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)

 # <<mark>>
-sqldiff.exe:	$(TOP)\tool\sqldiff.c $(TOP)\ext\consio\console_io.h $(TOP)\ext\consio\console_io.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS)
-	$(LTLINK) $(NO_WARN) -I$(TOP)\ext\consio $(TOP)\tool\sqldiff.c $(TOP)\ext\consio\console_io.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS)
+sqldiff.exe:	$(TOP)\tool\sqldiff.c $(TOP)\ext\misc\sqlite3_stdio.h $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS)
+	$(LTLINK) $(NO_WARN) -I$(TOP)\ext\misc $(TOP)\tool\sqldiff.c $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS)

 dbhash.exe:	$(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

+RSYNC_SRC = \
+  $(TOP)\tool\sqlite3_rsync.c \
+  $(SQLITE3C)
+
+RSYNC_OPT = \
+  -DSQLITE_ENABLE_DBPAGE_VTAB \
+  -DSQLITE_THREADSAFE=0 \
+  -DSQLITE_OMIT_LOAD_EXTENSION \
+  -DSQLITE_OMIT_DEPRECATED
+
+sqlite3_rsync.exe:	$(RSYNC_SRC) $(LIBRESOBJS)
+	$(LTLINK) $(RSYNC_OPT) $(NO_WARN) $(RSYNC_SRC) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS)
+
 scrub.exe:	$(TOP)\ext\misc\scrub.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) -DSCRUB_STANDALONE=1 $(TOP)\ext\misc\scrub.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

@@ -1870,6 +1947,10 @@ fuzzcheck.exe:	$(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H)
 fuzzcheck-asan.exe:	$(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) /fsanitize=address $(FUZZCHECK_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

+run-fuzzcheck:	fuzzcheck.exe fuzzcheck-asan.exe
+	fuzzcheck --spinner $(FUZZDB)
+	fuzzcheck-asan --spinner $(FUZZDB)
+
 ossshell.exe:	$(OSSSHELL_SRC) $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(FUZZCHECK_OPTS) $(OSSSHELL_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

@@ -2275,8 +2356,6 @@ keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
 # Source and header files that shell.c depends on
 SHELL_DEP = \
    $(TOP)\src\shell.c.in \
-    $(TOP)\ext\consio\console_io.c \
-    $(TOP)\ext\consio\console_io.h \
    $(TOP)\ext\expert\sqlite3expert.c \
    $(TOP)\ext\expert\sqlite3expert.h \
    $(TOP)\ext\intck\sqlite3intck.c \
@@ -2290,11 +2369,16 @@ SHELL_DEP = \
    $(TOP)\ext\misc\ieee754.c \
    $(TOP)\ext\misc\memtrace.c \
    $(TOP)\ext\misc\pcachetrace.c \
+    $(TOP)\ext\misc\percentile.c \
    $(TOP)\ext\misc\regexp.c \
    $(TOP)\ext\misc\series.c \
+    $(TOP)\ext\misc\sha1.c \
    $(TOP)\ext\misc\shathree.c \
    $(TOP)\ext\misc\sqlar.c \
+    $(TOP)\ext\misc\sqlite3_stdio.c \
+    $(TOP)\ext\misc\sqlite3_stdio.h \
    $(TOP)\ext\misc\uint.c \
+    $(TOP)\ext\misc\vfstrace.c \
    $(TOP)\ext\misc\zipfile.c \
    $(TOP)\ext\recover\dbdata.c \
    $(TOP)\ext\recover\sqlite3recover.c \
@@ -2493,7 +2577,7 @@ extensiontest:	testfixture.exe testloadext.dll
 	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS)

-tool-zip:	testfixture.exe sqlite3.exe sqldiff.exe sqlite3_analyzer.exe $(TOP)\tool\mktoolzip.tcl
+tool-zip:	testfixture.exe sqlite3.exe sqldiff.exe sqlite3_analyzer.exe sqlite3_rsync.exe $(TOP)\tool\mktoolzip.tcl
 	.\testfixture.exe $(TOP)\tool\mktoolzip.tcl

 coretestprogs:	testfixture.exe sqlite3.exe
@@ -2521,6 +2605,13 @@ queryplantest:	testfixture.exe shell
 fuzztest:	fuzzcheck.exe
 	.\fuzzcheck.exe $(FUZZDATA)

+# Legacy testing target for third-party integrators.  The SQLite
+# developers seldom use this target themselves.  Instead
+# they use "nmake /f Makefile.msc devtest" which runs tests on
+# a standard set of options
+#
+test:	$(TESTPROGS) sourcetest fuzztest tcltest
+
 # Minimal testing that runs in less than 3 minutes (on a fast machine)
 #
 quicktest:	testfixture.exe sourcetest
@@ -2530,7 +2621,6 @@ quicktest:	testfixture.exe sourcetest
 # This is the common case.  Run many tests that do not take too long,
 # including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
 #
-test:	$(TESTPROGS) sourcetest fuzztest tcltest

 # The veryquick.test TCL tests.
 #
@@ -2544,17 +2634,27 @@ tcltest: testfixture.exe
 testrunner:	testfixture.exe
 	.\testfixture.exe $(TOP)\test\testrunner.tcl

-# Runs both fuzztest and testrunner, consecutively.
+# This is the testing target preferred by the core SQLite developers.
+# It runs tests under a standard configuration.  The devs run 
+# "nmake /f Makefile.msc devtest" prior to each check-in, at a minimum.
+# Probably other tests too, but at least this one.
 #
-devtest:	testfixture.exe fuzztest testrunner
+devtest:	srctree-check sourcetest
+	$(TCLSH_CMD) $(TOP)\test\testrunner.tcl mdevtest

 mdevtest:
 	$(TCLSH_CMD) $(TOP)\test\testrunner.tcl mdevtest

+# Validate that various generated files in the source tree
+# are up-to-date.
+#
+srctree-check:	$(TOP)\tool\srctree-check.tcl
+	$(TCLSH_CMD) $(TOP)\tool\srctree-check.tcl
+
 # Testing for a release
 #
-releasetest: testfixture.exe
-	testfixture.exe $(TOP)\test\testrunner.tcl release
+releasetest:
+	$(TCLSH_CMD) $(TOP)\test\testrunner.tcl release


 smoketest:	$(TESTPROGS)
@@ -2564,7 +2664,7 @@ smoketest:	$(TESTPROGS)
 shelltest: $(TESTPROGS)
 	.\testfixture.exe $(TOP)\test\permutations.test shell

-sqlite3_analyzer.c:	$(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(TOP)\ext\consio\console_io.h $(TOP)\ext\consio\console_io.c $(SQLITE_TCL_DEP)
+sqlite3_analyzer.c:	$(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(TOP)\ext\misc\sqlite3_stdio.h $(TOP)\ext\misc\sqlite3_stdio.c $(SQLITE_TCL_DEP)
 	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in > $@

 sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
@@ -2686,6 +2786,16 @@ THREADTEST3_SRC = \
 threadtest3.exe: $(THREADTEST3_SRC) $(TOP)\src\test_multiplex.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(TOP)\test\threadtest3.c $(TOP)\src\test_multiplex.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)

+# Display key variables that control which version of TCL is to be used.
+#
+tcl-env:
+	@echo TCLDIR = $(TCLDIR)
+	@echo TCLVERSION = $(TCLVERSION)
+	@echo TCLSUFFIX = $(TCLSUFFIX)
+	@echo LIBTCL = $(LIBTCL)
+	@echo LIBTCLSTUB = $(LIBTCLSTUB)
+	@echo TCLSH_CMD = $(TCLSH_CMD)
+
 LSMDIR=$(TOP)\ext\lsm1
 !INCLUDE $(LSMDIR)\Makefile.msc

@@ -2719,7 +2829,7 @@ clean:
 	del /Q sqlite3.c sqlite3-*.c sqlite3.h 2>NUL
 	del /Q sqlite3rc.h 2>NUL
 	del /Q shell.c sqlite3ext.h sqlite3session.h 2>NUL
-	del /Q sqlite3_analyzer.exe sqlite3_analyzer.c 2>NUL
+	del /Q sqlite3_analyzer.exe sqlite3_analyzer.c sqlite3_rsync.exe 2>NUL
 	del /Q sqlite-*-output.vsix 2>NUL
 	del /Q fuzzershell.exe fuzzcheck.exe sqldiff.exe dbhash.exe 2>NUL
 	del /Q sqltclsh.* 2>NUL
@@ -1,14 +1,19 @@
 <h1 align="center">SQLite Source Repository</h1>

 This repository contains the complete source code for the
-[SQLite database engine](https://sqlite.org/).  Some test scripts
-are also included.  However, many other test scripts
+[SQLite database engine](https://sqlite.org/), including
+many test scripts.  However, other test scripts
 and most of the documentation are managed separately.

+See the [on-line documentation](https://sqlite.org/) for more information
+about what SQLite is and how it works from a user's perspective.  This
+README file is about the source code that goes into building SQLite,
+not about how SQLite is used.
+
 ## Version Control

 SQLite sources are managed using
-[Fossil](https://www.fossil-scm.org/), a distributed version control system
+[Fossil](https://fossil-scm.org/), a distributed version control system
 that was specifically designed and written to support SQLite development.
 The [Fossil repository](https://sqlite.org/src/timeline) contains the urtext.

@@ -68,24 +73,26 @@ archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows:
     then click on the "Tarball" or "ZIP Archive" links on the information
     page.

-If you do want to use Fossil to check out the source tree,
+To access sources directly using [Fossil](https://fossil-scm.org/home),
 first install Fossil version 2.0 or later.
-(Source tarballs and precompiled binaries available
-[here](https://www.fossil-scm.org/fossil/uv/download.html).  Fossil is
+Source tarballs and precompiled binaries available at
+<https://fossil-scm.org/home/uv/download.html>.  Fossil is
 a stand-alone program.  To install, simply download or build the single
-executable file and put that file someplace on your $PATH.)
+executable file and put that file someplace on your $PATH.
 Then run commands like this:

-        mkdir -p ~/sqlite ~/Fossils
+        mkdir -p ~/sqlite
        cd ~/sqlite
-        fossil clone https://www.sqlite.org/src ~/Fossils/sqlite.fossil
-        fossil open ~/Fossils/sqlite.fossil
+        fossil open https://sqlite.org/src

-After setting up a repository using the steps above, you can always
-update to the latest version using:
+The "fossil open" command will take two or three minutes.  Afterwards,
+you can do fast, bandwidth-efficient updates to the whatever versions
+of SQLite you like.  Some examples:

-        fossil update trunk   ;# latest trunk check-in
-        fossil update release ;# latest official release
+        fossil update trunk             ;# latest trunk check-in
+        fossil update release           ;# latest official release
+        fossil update trunk:2024-01-01  ;# First trunk check-in after 2024-01-01
+        fossil update version-3.39.0    ;# Version 3.39.0

 Or type "fossil ui" to get a web-based user interface.

@@ -99,15 +106,42 @@ script found at the root of the source tree.  Then run "make".

 For example:

-        tar xzf sqlite.tar.gz    ;#  Unpack the source tree into "sqlite"
-        mkdir bld                ;#  Build will occur in a sibling directory
-        cd bld                   ;#  Change to the build directory
-        ../sqlite/configure      ;#  Run the configure script
-        make                     ;#  Builds the "sqlite3" command-line tool
-        make sqlite3.c           ;#  Build the "amalgamation" source file
-        make devtest             ;#  Run some tests (requires Tcl)
+        apt install gcc make tcl-dev  ;#  Make sure you have all the necessary build tools
+        tar xzf sqlite.tar.gz         ;#  Unpack the source tree into "sqlite"
+        mkdir bld                     ;#  Build will occur in a sibling directory
+        cd bld                        ;#  Change to the build directory
+        ../sqlite/configure           ;#  Run the configure script
+        make sqlite3                  ;#  Builds the "sqlite3" command-line tool
+        make sqlite3.c                ;#  Build the "amalgamation" source file
+        make sqldiff                  ;#  Builds the "sqldiff" command-line tool
+        # Makefile targets below this point require tcl-dev
+        make tclextension-install     ;#  Build and install the SQLite TCL extension
+        make devtest                  ;#  Run development tests
+        make releasetest              ;#  Run full release tests
+        make sqlite3_analyzer         ;#  Builds the "sqlite3_analyzer" tool

-See the makefile for additional targets.
+See the makefile for additional targets.  For debugging builds, the
+core developers typically run "configure" with options like this:
+
+        ../sqlite/configure --enable-all --enable-debug CFLAGS='-O0 -g'
+
+For release builds, the core developers usually do:
+
+        ../sqlite/configure --enable-all
+
+Almost all makefile targets require a "tclsh" TCL interpreter version 8.6 or
+later.  The "tclextension-install" target and the test targets that follow
+all require TCL development libraries too.  ("apt install tcl-dev").  It is
+helpful, but is not required, to install the SQLite TCL extension (the
+"tclextension-install" target) prior to running tests.  The "releasetest"
+target has additional requiremenst, such as "valgrind".
+
+On "make" command-lines, one can add "OPTIONS=..." to specify additional
+compile-time options over and above those set by ./configure.  For example,
+to compile with the SQLITE_OMIT_DEPRECATED compile-time option, one could say:
+
+        ./configure --enable-all
+        make OPTIONS=-DSQLITE_OMIT_DEPRECATED sqlite3

 The configure script uses autoconf 2.61 and libtool.  If the configure
 script does not work out for you, there is a generic makefile named
@@ -117,7 +151,7 @@ show what changes are needed.

 ## Compiling for Windows Using MSVC

-On Windows, all applicable build products can be compiled with MSVC.
+On Windows, everything can be compiled with MSVC.
 You will also need a working installation of TCL.
 See the [compile-for-windows.md](doc/compile-for-windows.md) document for
 additional information about how to install MSVC and TCL and configure your
@@ -128,48 +162,72 @@ TCL library, using a command like this:

        set TCLDIR=c:\Tcl

-SQLite uses "tclsh.exe" as part of the build process, and so that utility
-program will need to be somewhere on your %PATH%.  The finished SQLite library
-does not contain any TCL code, but it does use TCL to help with the build process
-and to run tests.
+SQLite uses "tclsh.exe" as part of the build process, and so that
+program will need to be somewhere on your %PATH%.  SQLite itself
+does not contain any TCL code, but it does use TCL to help with the
+build process and to run tests.  You may need to install TCL development
+libraries in order to successfully complete some makefile targets.
+It is helpful, but is not required, to install the SQLite TCL extension
+(the "tclextension-install" target) prior to running tests.

 Build using Makefile.msc.  Example:

-        nmake /f Makefile.msc
+        nmake /f Makefile.msc sqlite3.exe
        nmake /f Makefile.msc sqlite3.c
+        nmake /f Makefile.msc sqldiff.exe
+        # Makefile targets below this point require TCL development libraries
+        nmake /f Makefile.msc tclextension-install
        nmake /f Makefile.msc devtest
        nmake /f Makefile.msc releasetest
+        nmake /f Makefile.msc sqlite3_analyzer.exe
 
 There are many other makefile targets.  See comments in Makefile.msc for
 details.

-## Source Code Tour
+As with the unix Makefile, the OPTIONS=... argument can be passed on the nmake
+command-line to enable new compile-time options.  For example:

-Most of the core source files are in the **src/** subdirectory.  The
-**src/** folder also contains files used to build the "testfixture" test
-harness. The names of the source files used by "testfixture" all begin
-with "test".
-The **src/** also contains the "shell.c" file
-which is the main program for the "sqlite3.exe"
-[command-line shell](https://sqlite.org/cli.html) and
-the "tclsqlite.c" file which implements the
-[Tcl bindings](https://sqlite.org/tclsqlite.html) for SQLite.
-(Historical note:  SQLite began as a Tcl
-extension and only later escaped to the wild as an independent library.)
+        nmake /f Makefile.msc OPTIONS=-DSQLITE_OMIT_DEPRECATED sqlite3.exe

-Test scripts and programs are found in the **test/** subdirectory.
-Additional test code is found in other source repositories.
-See [How SQLite Is Tested](https://www.sqlite.org/testing.html) for
-additional information.
+## Source Tree Map

-The **ext/** subdirectory contains code for extensions.  The
-Full-text search engine is in **ext/fts3**.  The R-Tree engine is in
-**ext/rtree**.  The **ext/misc** subdirectory contains a number of
-smaller, single-file extensions, such as a REGEXP operator.
+  *  **src/** - This directory contains the primary source code for the
+     SQLite core.  For historical reasons, C-code used for testing is
+     also found here.  Source files intended for testing begin with "`test`".
+     The `tclsqlite3.c` and `tclsqlite3.h` files are the TCL interface
+     for SQLite and are also not part of the core.

-The **tool/** subdirectory contains various scripts and programs used
-for building generated source code files or for testing or for generating
-accessory programs such as "sqlite3_analyzer(.exe)".
+  *  **test/** - This directory and its subdirectories contains code used
+     for testing.  Files that end in "`.test`" are TCL scripts that run
+     tests using an augmented TCL interpreter named "testfixture".  Use
+     a command like "`make testfixture`" (unix) or 
+     "`nmake /f Makefile.msc testfixture.exe`" (windows) to build that
+     augmented TCL interpreter, then run individual tests using commands like
+     "`testfixture test/main.test`".  This test/ subdirectory also contains
+     additional C code modules and scripts for other kinds of testing.
+
+  *  **tool/** - This directory contains programs and scripts used to
+     build some of the machine-generated code that goes into the SQLite
+     core, as well as to build and run tests and perform diagnostics.
+     The source code to [the Lemon parser generator](./doc/lemon.html) is
+     found here.  There are also TCL scripts used to build and/or transform
+     source code files.  For example, the tool/mksqlite3h.tcl script reads
+     the src/sqlite.h.in file and uses it as a template to construct
+     the deliverable "sqlite3.h" file that defines the SQLite interface.
+
+  *  **ext/** - Various extensions to SQLite are found under this
+     directory.  For example, the FTS5 subsystem is in "ext/fts5/".
+     Some of these extensions (ex: FTS3/4, FTS5, RTREE) might get built
+     into the SQLite amalgamation, but not all of them.  The
+     "ext/misc/" subdirectory contains an assortment of one-file extensions,
+     many of which are omitted from the SQLite core, but which are included
+     in the [SQLite CLI](https://sqlite.org/cli.html).
+     
+  *  **doc/** - Some documentation files about SQLite internals are found
+     here.  Note, however, that the primary documentation designed for
+     application developers and users of SQLite is in a completely separate
+     repository.  Note also that the primary API documentation is derived
+     from specially constructed comments in the src/sqlite.h.in file.

 ### Generated Source Code Files

@@ -252,31 +310,37 @@ individual source file exceeds 32K lines in length.
 SQLite is modular in design.
 See the [architectural description](https://www.sqlite.org/arch.html)
 for details. Other documents that are useful in
-(helping to understand how SQLite works include the
+helping to understand how SQLite works include the
 [file format](https://www.sqlite.org/fileformat2.html) description,
 the [virtual machine](https://www.sqlite.org/opcode.html) that runs
 prepared statements, the description of
 [how transactions work](https://www.sqlite.org/atomiccommit.html), and
 the [overview of the query planner](https://www.sqlite.org/optoverview.html).

-Years of effort have gone into optimizing SQLite, both
+Decades of effort have gone into optimizing SQLite, both
 for small size and high performance.  And optimizations tend to result in
 complex code.  So there is a lot of complexity in the current SQLite
 implementation.  It will not be the easiest library in the world to hack.

-Key files:
+### Key source code files

  *  **sqlite.h.in** - This file defines the public interface to the SQLite
     library.  Readers will need to be familiar with this interface before
-     trying to understand how the library works internally.
+     trying to understand how the library works internally.  This file is
+     really a template that is transformed into the "sqlite3.h" deliverable
+     using a script invoked by the makefile.

  *  **sqliteInt.h** - this header file defines many of the data objects
     used internally by SQLite.  In addition to "sqliteInt.h", some
-     subsystems have their own header files.
+     subsystems inside of sQLite have their own header files.  These internal
+     interfaces are not for use by applications.  They can and do change
+     from one release of SQLite to the next.

  *  **parse.y** - This file describes the LALR(1) grammar that SQLite uses
     to parse SQL statements, and the actions that are taken at each step
-     in the parsing process.
+     in the parsing process.  The file is processed by the
+     [Lemon Parser Generator](./doc/lemon.html) to produce the actual C code
+     used for parsing.

  *  **vdbe.c** - This file implements the virtual machine that runs
     prepared statements.  There are various helper files whose names
@@ -319,6 +383,11 @@ Key files:
     (and some other test programs too) is built and run when you type
     "make test".

+  *  **VERSION**, **manifest**, and **manifest.uuid** - These files define
+     the current SQLite version number.  The "VERSION" file is human generated,
+     but the "manifest" and "manifest.uuid" files are automatically generated
+     by the [Fossil version control system](https://fossil-scm.org/).
+
 There are many other source files.  Each has a succinct header comment that
 describes its purpose and role within the larger system.

@@ -336,7 +405,7 @@ The `manifest.uuid` file should contain the SHA3-256 hash of the
 `manifest` file. If all of the above hash comparisons are correct, then
 you can be confident that your source tree is authentic and unadulterated.
 Details on the format for the `manifest` files are available
-[on the Fossil website](https://fossil-scm.org/fossil/doc/trunk/www/fileformat.wiki#manifest).
+[on the Fossil website](https://fossil-scm.org/home/doc/trunk/www/fileformat.wiki#manifest).

 The process of checking source code authenticity is automated by the 
 makefile:
@@ -357,3 +426,7 @@ The main SQLite website is [https://sqlite.org/](https://sqlite.org/)
 with geographically distributed backups at
 [https://www2.sqlite.org/](https://www2.sqlite.org) and
 [https://www3.sqlite.org/](https://www3.sqlite.org).
+
+Contact the SQLite developers through the
+[SQLite Forum](https://sqlite.org/forum/).  In an emergency, you
+can send private email to the lead developer at drh at sqlite dot org.
@@ -1 +1 @@
-3.46.0
+3.47.0
@@ -66,6 +66,14 @@ USE_STDCALL = 0
 USE_SEH = 1
 !ENDIF

+# Use STATICALLY_LINK_TCL=1 to statically link against TCL
+#
+!IFNDEF STATICALLY_LINK_TCL
+STATICALLY_LINK_TCL = 0
+!ELSEIF $(STATICALLY_LINK_TCL)!=0
+CCOPTS = $(CCOPTS) -DSTATIC_BUILD
+!ENDIF
+
 # Set this non-0 to have the shell executable link against the core dynamic
 # link library.
 #
@@ -301,6 +309,7 @@ SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
 # the Windows platform.
 #
 !IFNDEF OPT_FEATURE_FLAGS
+OPT_FEATURE_FLAGS = $(OPT_XTRA)
 !IF $(MINIMAL_AMALGAMATION)==0
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS5=1
@@ -314,6 +323,14 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_BYTECODE_VTAB=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_COLUMN_METADATA=1
 !ENDIF

+# Additional feature-options above and beyond what are normally used can be
+# be added using OPTIONS=.... on the command-line.  These values are
+# appended to the OPT_FEATURE_FLAGS variable.
+#
+!IFDEF OPTIONS
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) $(OPTIONS)
+!ENDIF
+
 # Should the session extension be enabled?  If so, add compilation options
 # to enable it.
 #
@@ -9,8 +9,37 @@ This package contains:
 * a Makefile.msc, sqlite3.rc, and Replace.cs for building with Microsoft
   Visual C++ on Windows

-SUMMARY OF HOW TO BUILD
-=======================
+WHY USE THIS PACKAGE?
+=====================
+
+The canonical make system for SQLite requires TCL as part of the build
+process.  Various TCL scripts are used to generate parts of the code and
+TCL is used to run tests.  But some people would prefer to build SQLite
+using only generic tools and without having to install TCL.  The purpose
+of this package is to provide that capability.
+
+This package contains a pre-build SQLite amalgamation file "sqlite3.c"
+(and its associated header file "sqlite3.h").  Because the amalgamation
+has been pre-built, no TCL is required.
+
+REASONS TO USE THE CANONICAL BUILD SYSTEM RATHER THAN THIS PACKAGE
+==================================================================
+
+ * the cononical build system allows you to run tests to verify that
+   the build worked
+ * the canonical build system supports more compile-time options
+ * the canonical build system works for any arbitrary check-in to
+   the SQLite source tree
+
+Step-by-step instructions on how to build using the canonical make
+system for SQLite can be found at:
+
+  https://sqlite.org/src/doc/trunk/doc/compile-for-unix.md
+  https://sqlite.org/src/doc/trunk/doc/compile-for-windows.md
+
+
+SUMMARY OF HOW TO BUILD USING THIS PACKAGE
+==========================================

  Unix:      ./configure; make
  Windows:   nmake /f Makefile.msc
@@ -53,48 +82,6 @@ Using Microsoft Visual C++ 2005 (or later) is recommended.  Several Windows
 platform variants may be built by adding additional macros to the NMAKE
 command line.

-Building for WinRT 8.0
----------------------
-
-  FOR_WINRT=1
-
-Using Microsoft Visual C++ 2012 (or later) is required.  When using the
-above, something like the following macro will need to be added to the
-NMAKE command line as well:
-
-  "NSDKLIBPATH=%WindowsSdkDir%\..\8.0\lib\win8\um\x86"
-
-Building for WinRT 8.1
----------------------
-
-  FOR_WINRT=1
-
-Using Microsoft Visual C++ 2013 (or later) is required.  When using the
-above, something like the following macro will need to be added to the
-NMAKE command line as well:
-
-  "NSDKLIBPATH=%WindowsSdkDir%\..\8.1\lib\winv6.3\um\x86"
-
-Building for UWP 10.0
---------------------
-
-  FOR_WINRT=1 FOR_UWP=1
-
-Using Microsoft Visual C++ 2015 (or later) is required.  When using the
-above, something like the following macros will need to be added to the
-NMAKE command line as well:
-
-  "NSDKLIBPATH=%WindowsSdkDir%\..\10\lib\10.0.10586.0\um\x86"
-  "PSDKLIBPATH=%WindowsSdkDir%\..\10\lib\10.0.10586.0\um\x86"
-  "NUCRTLIBPATH=%UniversalCRTSdkDir%\..\10\lib\10.0.10586.0\ucrt\x86"
-
-Building for the Windows 10 SDK
-------------------------------
-
-  FOR_WIN10=1
-
-Using Microsoft Visual C++ 2015 (or later) is required.  When using the
-above, no other macros should be needed on the NMAKE command line.

 Other preprocessor defines
 --------------------------
@@ -99,7 +99,6 @@ INSTALL_LIBRARY	= @INSTALL_LIBRARY@
 PACKAGE_NAME	= @PACKAGE_NAME@
 PACKAGE_VERSION	= @PACKAGE_VERSION@
 CC		= @CC@
-CCLD		= @CCLD@
 CFLAGS_DEFAULT	= @CFLAGS_DEFAULT@
 CFLAGS_WARNING	= @CFLAGS_WARNING@
 EXEEXT		= @EXEEXT@
@@ -311,19 +310,6 @@ VPATH = $(srcdir):$(srcdir)/generic:$(srcdir)/unix:$(srcdir)/win:$(srcdir)/macos
 .c.@OBJEXT@:
 	$(COMPILE) -c `@CYGPATH@ $<` -o $@

-tclsample.@OBJEXT@:	sampleUuid.h
-
-$(srcdir)/manifest.uuid:
-	printf "git-" >$(srcdir)/manifest.uuid
-	(cd $(srcdir); git rev-parse HEAD >>$(srcdir)/manifest.uuid || \
-	    (printf "svn-r" >$(srcdir)/manifest.uuid ; \
-	    svn info --show-item last-changed-revision >>$(srcdir)/manifest.uuid) || \
-	    printf "unknown" >$(srcdir)/manifest.uuid)
-
-sampleUuid.h:	$(srcdir)/manifest.uuid
-	echo "#define SAMPLE_VERSION_UUID \\" >$@
-	cat $(srcdir)/manifest.uuid >>$@
-	echo "" >>$@

 #========================================================================
 # Distribution creation
@@ -451,6 +437,8 @@ install-bin-binaries: binaries
 	  fi; \
 	done

+.SUFFIXES: .c .$(OBJEXT)
+
 Makefile: $(srcdir)/Makefile.in  $(top_builddir)/config.status
 	cd $(top_builddir) \
 	  && CONFIG_FILES=$@ CONFIG_HEADERS= $(SHELL) ./config.status
@@ -1,36 +0,0 @@
-This is the SQLite extension for Tcl using the Tcl Extension
-Architecture (TEA).  For additional information on SQLite see
-
-        http://www.sqlite.org/
-
-
-UNIX BUILD
-==========
-
-Building under most UNIX systems is easy, just run the configure script
-and then run make. For more information about the build process, see
-the tcl/unix/README file in the Tcl src dist. The following minimal
-example will install the extension in the /opt/tcl directory.
-
-	$ cd sqlite-*-tea
-	$ ./configure --prefix=/opt/tcl
-	$ make
-	$ make install
-
-WINDOWS BUILD
-=============
-
-The recommended method to build extensions under windows is to use the
-Msys + Mingw build process. This provides a Unix-style build while
-generating native Windows binaries. Using the Msys + Mingw build tools
-means that you can use the same configure script as per the Unix build
-to create a Makefile. See the tcl/win/README file for the URL of
-the Msys + Mingw download.
-
-If you have VC++ then you may wish to use the files in the win
-subdirectory and build the extension using just VC++. These files have
-been designed to be as generic as possible but will require some
-additional maintenance by the project developer to synchronise with
-the TEA configure.in and Makefile.in files. Instructions for using the
-VC++ makefile are written in the first part of the Makefile.vc
-file.
@@ -0,0 +1,78 @@
+This is the SQLite extension for Tcl using the Tcl Extension
+Architecture (TEA). 
+
+----------------------- A BETTER WAY ---------------------------
+
+A better way to build the TCL extension for SQLite is to use the
+canonical source code tarball.  For Unix:
+
+    ./configure --with-tclsh=$(TCLSH)
+    make tclextension-install
+
+For Windows:
+
+    nmake /f Makefile.msc tclextension-install TCLSH_CMD=$(TCLSH)
+
+In both of the above, replace $(TCLSH) with the full pathname of
+of the tclsh that you want the SQLite extension to work with.  See
+step-by-step instructions at the links below for more information:
+
+    https://sqlite.org/src/doc/trunk/doc/compile-for-unix.md
+    https://sqlite.org/src/doc/trunk/doc/compile-for-windows.md
+
+The whole point of the amalgamation-autoconf tarball (in which this
+README.txt file is embedded) is to provide a means of compiling
+SQLite that does not require first installing TCL and/or "tclsh".
+The canonical Makefile in the SQLite source tree provides more
+capabilities (such as the the ability to run test cases to ensure
+that the build worked) and is better maintained.  The only
+downside of the canonical Makfile is that it requires a TCL
+installation.  But if you are wanting to build the TCL extension for
+SQLite, then presumably you already have a TCL installation.  So why
+not just use the more-capable and better-maintained canoncal Makefile?
+
+This TEA builder is derived from code found at
+
+    http://core.tcl-lang.org/tclconfig
+    http://core.tcl-lang.org/sampleextension
+
+The SQLite developers do not understand how it works.  It seems to
+work for us.  It might also work for you.  But we cannot promise that.
+
+If you want to use this TEA builder and it works for you, that's fine.
+But if you have trouble, the first thing you should do is go back
+to using the canonical Makefile in the SQLite source tree.
+
+------------------------------------------------------------------
+
+
+UNIX BUILD
+==========
+
+Building under most UNIX systems is easy, just run the configure script
+and then run make. For more information about the build process, see
+the tcl/unix/README file in the Tcl src dist. The following minimal
+example will install the extension in the /opt/tcl directory.
+
+	$ cd sqlite-*-tea
+	$ ./configure --prefix=/opt/tcl
+	$ make
+	$ make install
+
+WINDOWS BUILD
+=============
+
+The recommended method to build extensions under windows is to use the
+Msys + Mingw build process. This provides a Unix-style build while
+generating native Windows binaries. Using the Msys + Mingw build tools
+means that you can use the same configure script as per the Unix build
+to create a Makefile. See the tcl/win/README file for the URL of
+the Msys + Mingw download.
+
+If you have VC++ then you may wish to use the files in the win
+subdirectory and build the extension using just VC++. These files have
+been designed to be as generic as possible but will require some
+additional maintenance by the project developer to synchronise with
+the TEA configure.in and Makefile.in files. Instructions for using the
+VC++ makefile are written in the first part of the Makefile.vc
+file.
@@ -19,7 +19,7 @@ dnl	to configure the system for the local environment.
 # so that we create the export library with the dll.
 #-----------------------------------------------------------------------

-AC_INIT([sqlite],[3.46.0])
+AC_INIT([sqlite],[3.47.0])

 #--------------------------------------------------------------------
 # Call TEA_INIT as the first TEA_ macro to set up initial vars.
@@ -97,7 +97,7 @@ TEA_ADD_TCL_SOURCES([])
 # Patchs from rmax.
 #--------------------------------------------------------------------
 AC_ARG_WITH([system-sqlite],
- [AC_HELP_STRING([--with-system-sqlite],
+ [AS_HELP_STRING([--with-system-sqlite],
   [use a system-supplied libsqlite3 instead of the bundled one])],
 [], [with_system_sqlite=no])
 if test x$with_system_sqlite != xno; then
@@ -3,8 +3,8 @@
 #
 if {[package vsatisfies [package provide Tcl] 9.0-]} {
    package ifneeded sqlite3 @PACKAGE_VERSION@ \
-	    [list load [file join $dir @PKG_LIB_FILE9@] sqlite3]
+	    [list load [file join $dir @PKG_LIB_FILE9@] Sqlite3]
 } else {
    package ifneeded sqlite3 @PACKAGE_VERSION@ \
-	    [list load [file join $dir @PKG_LIB_FILE8@] sqlite3]
+	    [list load [file join $dir @PKG_LIB_FILE8@] Sqlite3]
 }
@@ -1,7 +1,7 @@
 #!/bin/sh
 # install - install a program, script, or datafile

-scriptversion=2011-04-20.01; # UTC
+scriptversion=2020-11-14.01; # UTC

 # This originates from X11R5 (mit/util/scripts/install.sh), which was
 # later released in X11R6 (xc/config/util/install.sh) with the
@@ -35,25 +35,21 @@ scriptversion=2011-04-20.01; # UTC
 # FSF changes to this file are in the public domain.
 #
 # Calling this script install-sh is preferred over install.sh, to prevent
-# `make' implicit rules from creating a file called install from it
+# 'make' implicit rules from creating a file called install from it
 # when there is no Makefile.
 #
 # This script is compatible with the BSD install script, but was written
 # from scratch.

+tab='	'
 nl='
 '
-IFS=" ""	$nl"
+IFS=" $tab$nl"

-# set DOITPROG to echo to test this script
+# Set DOITPROG to "echo" to test this script.

-# Don't use :- since 4.3BSD and earlier shells don't like it.
 doit=${DOITPROG-}
-if test -z "$doit"; then
-  doit_exec=exec
-else
-  doit_exec=$doit
-fi
+doit_exec=${doit:-exec}

 # Put in absolute file names if you don't have them in your path;
 # or use environment vars.
@@ -68,22 +64,16 @@ mvprog=${MVPROG-mv}
 rmprog=${RMPROG-rm}
 stripprog=${STRIPPROG-strip}

-posix_glob='?'
-initialize_posix_glob='
-  test "$posix_glob" != "?" || {
-    if (set -f) 2>/dev/null; then
-      posix_glob=
-    else
-      posix_glob=:
-    fi
-  }
-'
-
 posix_mkdir=

 # Desired mode of installed file.
 mode=0755

+# Create dirs (including intermediate dirs) using mode 755.
+# This is like GNU 'install' as of coreutils 8.32 (2020).
+mkdir_umask=22
+
+backupsuffix=
 chgrpcmd=
 chmodcmd=$chmodprog
 chowncmd=
@@ -97,7 +87,7 @@ dir_arg=
 dst_arg=

 copy_on_change=false
-no_target_directory=
+is_target_a_directory=possibly

 usage="\
 Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE
@@ -114,19 +104,28 @@ Options:
     --version  display version info and exit.

  -c            (ignored)
-  -C            install only if different (preserve the last data modification time)
+  -C            install only if different (preserve data modification time)
  -d            create directories instead of installing files.
  -g GROUP      $chgrpprog installed files to GROUP.
  -m MODE       $chmodprog installed files to MODE.
  -o USER       $chownprog installed files to USER.
+  -p            pass -p to $cpprog.
  -s            $stripprog installed files.
-  -S            $stripprog installed files.
+  -S SUFFIX     attempt to back up existing files, with suffix SUFFIX.
  -t DIRECTORY  install into DIRECTORY.
  -T            report an error if DSTFILE is a directory.

 Environment variables override the default commands:
  CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG
  RMPROG STRIPPROG
+
+By default, rm is invoked with -f; when overridden with RMPROG,
+it's up to you to specify -f if you want it.
+
+If -S is not specified, no backups are attempted.
+
+Email bug reports to bug-automake@gnu.org.
+Automake home page: https://www.gnu.org/software/automake/
 "

 while test $# -ne 0; do
@@ -138,45 +137,62 @@ while test $# -ne 0; do
    -d) dir_arg=true;;

    -g) chgrpcmd="$chgrpprog $2"
-	shift;;
+        shift;;

    --help) echo "$usage"; exit $?;;

    -m) mode=$2
-	case $mode in
-	  *' '* | *'	'* | *'
-'*	  | *'*'* | *'?'* | *'['*)
-	    echo "$0: invalid mode: $mode" >&2
-	    exit 1;;
-	esac
-	shift;;
+        case $mode in
+          *' '* | *"$tab"* | *"$nl"* | *'*'* | *'?'* | *'['*)
+            echo "$0: invalid mode: $mode" >&2
+            exit 1;;
+        esac
+        shift;;

    -o) chowncmd="$chownprog $2"
-	shift;;
+        shift;;
+
+    -p) cpprog="$cpprog -p";;

    -s) stripcmd=$stripprog;;

-    -S) stripcmd="$stripprog $2"
-	shift;;
+    -S) backupsuffix="$2"
+        shift;;

-    -t) dst_arg=$2
-	shift;;
+    -t)
+        is_target_a_directory=always
+        dst_arg=$2
+        # Protect names problematic for 'test' and other utilities.
+        case $dst_arg in
+          -* | [=\(\)!]) dst_arg=./$dst_arg;;
+        esac
+        shift;;

-    -T) no_target_directory=true;;
+    -T) is_target_a_directory=never;;

    --version) echo "$0 $scriptversion"; exit $?;;

-    --)	shift
-	break;;
+    --) shift
+        break;;

-    -*)	echo "$0: invalid option: $1" >&2
-	exit 1;;
+    -*) echo "$0: invalid option: $1" >&2
+        exit 1;;

    *)  break;;
  esac
  shift
 done

+# We allow the use of options -d and -T together, by making -d
+# take the precedence; this is for compatibility with GNU install.
+
+if test -n "$dir_arg"; then
+  if test -n "$dst_arg"; then
+    echo "$0: target directory not allowed when installing a directory." >&2
+    exit 1
+  fi
+fi
+
 if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then
  # When -d is used, all remaining arguments are directories to create.
  # When -t is used, the destination is already specified.
@@ -190,6 +206,10 @@ if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then
    fi
    shift # arg
    dst_arg=$arg
+    # Protect names problematic for 'test' and other utilities.
+    case $dst_arg in
+      -* | [=\(\)!]) dst_arg=./$dst_arg;;
+    esac
  done
 fi

@@ -198,11 +218,20 @@ if test $# -eq 0; then
    echo "$0: no input file specified." >&2
    exit 1
  fi
-  # It's OK to call `install-sh -d' without argument.
+  # It's OK to call 'install-sh -d' without argument.
  # This can happen when creating conditional directories.
  exit 0
 fi

+if test -z "$dir_arg"; then
+  if test $# -gt 1 || test "$is_target_a_directory" = always; then
+    if test ! -d "$dst_arg"; then
+      echo "$0: $dst_arg: Is not a directory." >&2
+      exit 1
+    fi
+  fi
+fi
+
 if test -z "$dir_arg"; then
  do_exit='(exit $ret); exit $ret'
  trap "ret=129; $do_exit" 1
@@ -219,16 +248,16 @@ if test -z "$dir_arg"; then

    *[0-7])
      if test -z "$stripcmd"; then
-	u_plus_rw=
+        u_plus_rw=
      else
-	u_plus_rw='% 200'
+        u_plus_rw='% 200'
      fi
      cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;;
    *)
      if test -z "$stripcmd"; then
-	u_plus_rw=
+        u_plus_rw=
      else
-	u_plus_rw=,u+rw
+        u_plus_rw=,u+rw
      fi
      cp_umask=$mode$u_plus_rw;;
  esac
@@ -236,9 +265,9 @@ fi

 for src
 do
-  # Protect names starting with `-'.
+  # Protect names problematic for 'test' and other utilities.
  case $src in
-    -*) src=./$src;;
+    -* | [=\(\)!]) src=./$src;;
  esac

  if test -n "$dir_arg"; then
@@ -246,6 +275,10 @@ do
    dstdir=$dst
    test -d "$dstdir"
    dstdir_status=$?
+    # Don't chown directories that already exist.
+    if test $dstdir_status = 0; then
+      chowncmd=""
+    fi
  else

    # Waiting for this to be detected by the "$cpprog $src $dsttmp" command
@@ -260,185 +293,150 @@ do
      echo "$0: no destination specified." >&2
      exit 1
    fi
-
    dst=$dst_arg
-    # Protect names starting with `-'.
-    case $dst in
-      -*) dst=./$dst;;
-    esac

-    # If destination is a directory, append the input filename; won't work
-    # if double slashes aren't ignored.
+    # If destination is a directory, append the input filename.
    if test -d "$dst"; then
-      if test -n "$no_target_directory"; then
-	echo "$0: $dst_arg: Is a directory" >&2
-	exit 1
+      if test "$is_target_a_directory" = never; then
+        echo "$0: $dst_arg: Is a directory" >&2
+        exit 1
      fi
      dstdir=$dst
-      dst=$dstdir/`basename "$src"`
+      dstbase=`basename "$src"`
+      case $dst in
+	*/) dst=$dst$dstbase;;
+	*)  dst=$dst/$dstbase;;
+      esac
      dstdir_status=0
    else
-      # Prefer dirname, but fall back on a substitute if dirname fails.
-      dstdir=`
-	(dirname "$dst") 2>/dev/null ||
-	expr X"$dst" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \
-	     X"$dst" : 'X\(//\)[^/]' \| \
-	     X"$dst" : 'X\(//\)$' \| \
-	     X"$dst" : 'X\(/\)' \| . 2>/dev/null ||
-	echo X"$dst" |
-	    sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{
-		   s//\1/
-		   q
-		 }
-		 /^X\(\/\/\)[^/].*/{
-		   s//\1/
-		   q
-		 }
-		 /^X\(\/\/\)$/{
-		   s//\1/
-		   q
-		 }
-		 /^X\(\/\).*/{
-		   s//\1/
-		   q
-		 }
-		 s/.*/./; q'
-      `
-
+      dstdir=`dirname "$dst"`
      test -d "$dstdir"
      dstdir_status=$?
    fi
  fi

+  case $dstdir in
+    */) dstdirslash=$dstdir;;
+    *)  dstdirslash=$dstdir/;;
+  esac
+
  obsolete_mkdir_used=false

  if test $dstdir_status != 0; then
    case $posix_mkdir in
      '')
-	# Create intermediate dirs using mode 755 as modified by the umask.
-	# This is like FreeBSD 'install' as of 1997-10-28.
-	umask=`umask`
-	case $stripcmd.$umask in
-	  # Optimize common cases.
-	  *[2367][2367]) mkdir_umask=$umask;;
-	  .*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;;
+        # With -d, create the new directory with the user-specified mode.
+        # Otherwise, rely on $mkdir_umask.
+        if test -n "$dir_arg"; then
+          mkdir_mode=-m$mode
+        else
+          mkdir_mode=
+        fi

-	  *[0-7])
-	    mkdir_umask=`expr $umask + 22 \
-	      - $umask % 100 % 40 + $umask % 20 \
-	      - $umask % 10 % 4 + $umask % 2
-	    `;;
-	  *) mkdir_umask=$umask,go-w;;
-	esac
+        posix_mkdir=false
+	# The $RANDOM variable is not portable (e.g., dash).  Use it
+	# here however when possible just to lower collision chance.
+	tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$

-	# With -d, create the new directory with the user-specified mode.
-	# Otherwise, rely on $mkdir_umask.
-	if test -n "$dir_arg"; then
-	  mkdir_mode=-m$mode
+	trap '
+	  ret=$?
+	  rmdir "$tmpdir/a/b" "$tmpdir/a" "$tmpdir" 2>/dev/null
+	  exit $ret
+	' 0
+
+	# Because "mkdir -p" follows existing symlinks and we likely work
+	# directly in world-writeable /tmp, make sure that the '$tmpdir'
+	# directory is successfully created first before we actually test
+	# 'mkdir -p'.
+	if (umask $mkdir_umask &&
+	    $mkdirprog $mkdir_mode "$tmpdir" &&
+	    exec $mkdirprog $mkdir_mode -p -- "$tmpdir/a/b") >/dev/null 2>&1
+	then
+	  if test -z "$dir_arg" || {
+	       # Check for POSIX incompatibilities with -m.
+	       # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or
+	       # other-writable bit of parent directory when it shouldn't.
+	       # FreeBSD 6.1 mkdir -m -p sets mode of existing directory.
+	       test_tmpdir="$tmpdir/a"
+	       ls_ld_tmpdir=`ls -ld "$test_tmpdir"`
+	       case $ls_ld_tmpdir in
+		 d????-?r-*) different_mode=700;;
+		 d????-?--*) different_mode=755;;
+		 *) false;;
+	       esac &&
+	       $mkdirprog -m$different_mode -p -- "$test_tmpdir" && {
+		 ls_ld_tmpdir_1=`ls -ld "$test_tmpdir"`
+		 test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1"
+	       }
+	     }
+	  then posix_mkdir=:
+	  fi
+	  rmdir "$tmpdir/a/b" "$tmpdir/a" "$tmpdir"
 	else
-	  mkdir_mode=
+	  # Remove any dirs left behind by ancient mkdir implementations.
+	  rmdir ./$mkdir_mode ./-p ./-- "$tmpdir" 2>/dev/null
 	fi
-
-	posix_mkdir=false
-	case $umask in
-	  *[123567][0-7][0-7])
-	    # POSIX mkdir -p sets u+wx bits regardless of umask, which
-	    # is incompatible with FreeBSD 'install' when (umask & 300) != 0.
-	    ;;
-	  *)
-	    tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$
-	    trap 'ret=$?; rmdir "$tmpdir/d" "$tmpdir" 2>/dev/null; exit $ret' 0
-
-	    if (umask $mkdir_umask &&
-		exec $mkdirprog $mkdir_mode -p -- "$tmpdir/d") >/dev/null 2>&1
-	    then
-	      if test -z "$dir_arg" || {
-		   # Check for POSIX incompatibilities with -m.
-		   # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or
-		   # other-writeable bit of parent directory when it shouldn't.
-		   # FreeBSD 6.1 mkdir -m -p sets mode of existing directory.
-		   ls_ld_tmpdir=`ls -ld "$tmpdir"`
-		   case $ls_ld_tmpdir in
-		     d????-?r-*) different_mode=700;;
-		     d????-?--*) different_mode=755;;
-		     *) false;;
-		   esac &&
-		   $mkdirprog -m$different_mode -p -- "$tmpdir" && {
-		     ls_ld_tmpdir_1=`ls -ld "$tmpdir"`
-		     test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1"
-		   }
-		 }
-	      then posix_mkdir=:
-	      fi
-	      rmdir "$tmpdir/d" "$tmpdir"
-	    else
-	      # Remove any dirs left behind by ancient mkdir implementations.
-	      rmdir ./$mkdir_mode ./-p ./-- 2>/dev/null
-	    fi
-	    trap '' 0;;
-	esac;;
+	trap '' 0;;
    esac

    if
      $posix_mkdir && (
-	umask $mkdir_umask &&
-	$doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir"
+        umask $mkdir_umask &&
+        $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir"
      )
    then :
    else

-      # The umask is ridiculous, or mkdir does not conform to POSIX,
+      # mkdir does not conform to POSIX,
      # or it failed possibly due to a race condition.  Create the
      # directory the slow way, step by step, checking for races as we go.

      case $dstdir in
-	/*) prefix='/';;
-	-*) prefix='./';;
-	*)  prefix='';;
+        /*) prefix='/';;
+        [-=\(\)!]*) prefix='./';;
+        *)  prefix='';;
      esac

-      eval "$initialize_posix_glob"
-
      oIFS=$IFS
      IFS=/
-      $posix_glob set -f
+      set -f
      set fnord $dstdir
      shift
-      $posix_glob set +f
+      set +f
      IFS=$oIFS

      prefixes=

      for d
      do
-	test -z "$d" && continue
+        test X"$d" = X && continue

-	prefix=$prefix$d
-	if test -d "$prefix"; then
-	  prefixes=
-	else
-	  if $posix_mkdir; then
-	    (umask=$mkdir_umask &&
-	     $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break
-	    # Don't fail if two instances are running concurrently.
-	    test -d "$prefix" || exit 1
-	  else
-	    case $prefix in
-	      *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;;
-	      *) qprefix=$prefix;;
-	    esac
-	    prefixes="$prefixes '$qprefix'"
-	  fi
-	fi
-	prefix=$prefix/
+        prefix=$prefix$d
+        if test -d "$prefix"; then
+          prefixes=
+        else
+          if $posix_mkdir; then
+            (umask $mkdir_umask &&
+             $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break
+            # Don't fail if two instances are running concurrently.
+            test -d "$prefix" || exit 1
+          else
+            case $prefix in
+              *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;;
+              *) qprefix=$prefix;;
+            esac
+            prefixes="$prefixes '$qprefix'"
+          fi
+        fi
+        prefix=$prefix/
      done

      if test -n "$prefixes"; then
-	# Don't fail if two instances are running concurrently.
-	(umask $mkdir_umask &&
-	 eval "\$doit_exec \$mkdirprog $prefixes") ||
-	  test -d "$dstdir" || exit 1
-	obsolete_mkdir_used=true
+        # Don't fail if two instances are running concurrently.
+        (umask $mkdir_umask &&
+         eval "\$doit_exec \$mkdirprog $prefixes") ||
+          test -d "$dstdir" || exit 1
+        obsolete_mkdir_used=true
      fi
    fi
  fi
@@ -451,14 +449,25 @@ do
  else

    # Make a couple of temp file names in the proper directory.
-    dsttmp=$dstdir/_inst.$$_
-    rmtmp=$dstdir/_rm.$$_
+    dsttmp=${dstdirslash}_inst.$$_
+    rmtmp=${dstdirslash}_rm.$$_

    # Trap to clean up those temp files at exit.
    trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0

    # Copy the file name to the temp name.
-    (umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") &&
+    (umask $cp_umask &&
+     { test -z "$stripcmd" || {
+	 # Create $dsttmp read-write so that cp doesn't create it read-only,
+	 # which would cause strip to fail.
+	 if test -z "$doit"; then
+	   : >"$dsttmp" # No need to fork-exec 'touch'.
+	 else
+	   $doit touch "$dsttmp"
+	 fi
+       }
+     } &&
+     $doit_exec $cpprog "$src" "$dsttmp") &&

    # and set any options; do chmod last to preserve setuid bits.
    #
@@ -473,20 +482,24 @@ do

    # If -C, don't bother to copy if it wouldn't change the file.
    if $copy_on_change &&
-       old=`LC_ALL=C ls -dlL "$dst"	2>/dev/null` &&
-       new=`LC_ALL=C ls -dlL "$dsttmp"	2>/dev/null` &&
-
-       eval "$initialize_posix_glob" &&
-       $posix_glob set -f &&
+       old=`LC_ALL=C ls -dlL "$dst"     2>/dev/null` &&
+       new=`LC_ALL=C ls -dlL "$dsttmp"  2>/dev/null` &&
+       set -f &&
       set X $old && old=:$2:$4:$5:$6 &&
       set X $new && new=:$2:$4:$5:$6 &&
-       $posix_glob set +f &&
-
+       set +f &&
       test "$old" = "$new" &&
       $cmpprog "$dst" "$dsttmp" >/dev/null 2>&1
    then
      rm -f "$dsttmp"
    else
+      # If $backupsuffix is set, and the file being installed
+      # already exists, attempt a backup.  Don't worry if it fails,
+      # e.g., if mv doesn't support -f.
+      if test -n "$backupsuffix" && test -f "$dst"; then
+        $doit $mvcmd -f "$dst" "$dst$backupsuffix" 2>/dev/null
+      fi
+
      # Rename the file to the real destination.
      $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null ||

@@ -494,24 +507,24 @@ do
      # to itself, or perhaps because mv is so ancient that it does not
      # support -f.
      {
-	# Now remove or move aside any old file at destination location.
-	# We try this two ways since rm can't unlink itself on some
-	# systems and the destination file might be busy for other
-	# reasons.  In this case, the final cleanup might fail but the new
-	# file should still install successfully.
-	{
-	  test ! -f "$dst" ||
-	  $doit $rmcmd -f "$dst" 2>/dev/null ||
-	  { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null &&
-	    { $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; }
-	  } ||
-	  { echo "$0: cannot unlink or rename $dst" >&2
-	    (exit 1); exit 1
-	  }
-	} &&
+        # Now remove or move aside any old file at destination location.
+        # We try this two ways since rm can't unlink itself on some
+        # systems and the destination file might be busy for other
+        # reasons.  In this case, the final cleanup might fail but the new
+        # file should still install successfully.
+        {
+          test ! -f "$dst" ||
+          $doit $rmcmd "$dst" 2>/dev/null ||
+          { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null &&
+            { $doit $rmcmd "$rmtmp" 2>/dev/null; :; }
+          } ||
+          { echo "$0: cannot unlink or rename $dst" >&2
+            (exit 1); exit 1
+          }
+        } &&

-	# Now rename the file to the real destination.
-	$doit $mvcmd "$dsttmp" "$dst"
+        # Now rename the file to the real destination.
+        $doit $mvcmd "$dsttmp" "$dst"
      }
    fi || exit 1

@@ -520,9 +533,9 @@ do
 done

 # Local variables:
-# eval: (add-hook 'write-file-hooks 'time-stamp)
+# eval: (add-hook 'before-save-hook 'time-stamp)
 # time-stamp-start: "scriptversion="
 # time-stamp-format: "%:y-%02m-%02d.%02H"
-# time-stamp-time-zone: "UTC"
+# time-stamp-time-zone: "UTC0"
 # time-stamp-end: "; # UTC"
 # End:
@@ -53,6 +53,10 @@ AC_DEFUN([TEA_PATH_TCLCONFIG], [
 	    AS_HELP_STRING([--with-tcl],
 		[directory containing tcl configuration (tclConfig.sh)]),
 	    [with_tclconfig="${withval}"])
+	AC_ARG_WITH(tcl8,
+	    AS_HELP_STRING([--with-tcl8],
+		[Compile for Tcl8 in Tcl9 environment]),
+	    [with_tcl8="${withval}"])
 	AC_MSG_CHECKING([for Tcl configuration])
 	AC_CACHE_VAL(ac_cv_c_tclconfig,[

@@ -138,10 +142,16 @@ AC_DEFUN([TEA_PATH_TCLCONFIG], [
 			`ls -d /usr/pkg/lib 2>/dev/null` \
 			`ls -d /usr/lib 2>/dev/null` \
 			`ls -d /usr/lib64 2>/dev/null` \
+			`ls -d /usr/lib/tcl9.0 2>/dev/null` \
+			`ls -d /usr/lib/tcl8.7 2>/dev/null` \
 			`ls -d /usr/lib/tcl8.6 2>/dev/null` \
 			`ls -d /usr/lib/tcl8.5 2>/dev/null` \
+			`ls -d /usr/local/lib/tcl9.0 2>/dev/null` \
+			`ls -d /usr/local/lib/tcl8.7 2>/dev/null` \
 			`ls -d /usr/local/lib/tcl8.6 2>/dev/null` \
 			`ls -d /usr/local/lib/tcl8.5 2>/dev/null` \
+			`ls -d /usr/local/lib/tcl/tcl9.0 2>/dev/null` \
+			`ls -d /usr/local/lib/tcl/tcl8.7 2>/dev/null` \
 			`ls -d /usr/local/lib/tcl/tcl8.6 2>/dev/null` \
 			`ls -d /usr/local/lib/tcl/tcl8.5 2>/dev/null` \
 			; do
@@ -282,12 +292,18 @@ AC_DEFUN([TEA_PATH_TKCONFIG], [
 			`ls -d /usr/local/lib 2>/dev/null` \
 			`ls -d /usr/contrib/lib 2>/dev/null` \
 			`ls -d /usr/pkg/lib 2>/dev/null` \
+			`ls -d /usr/lib/tk9.0 2>/dev/null` \
+			`ls -d /usr/lib/tk8.7 2>/dev/null` \
 			`ls -d /usr/lib/tk8.6 2>/dev/null` \
 			`ls -d /usr/lib/tk8.5 2>/dev/null` \
 			`ls -d /usr/lib 2>/dev/null` \
 			`ls -d /usr/lib64 2>/dev/null` \
+			`ls -d /usr/local/lib/tk9.0 2>/dev/null` \
+			`ls -d /usr/local/lib/tk8.7 2>/dev/null` \
 			`ls -d /usr/local/lib/tk8.6 2>/dev/null` \
 			`ls -d /usr/local/lib/tk8.5 2>/dev/null` \
+			`ls -d /usr/local/lib/tcl/tk9.0 2>/dev/null` \
+			`ls -d /usr/local/lib/tcl/tk8.7 2>/dev/null` \
 			`ls -d /usr/local/lib/tcl/tk8.6 2>/dev/null` \
 			`ls -d /usr/local/lib/tcl/tk8.5 2>/dev/null` \
 			; do
@@ -366,10 +382,10 @@ AC_DEFUN([TEA_LOAD_TCLCONFIG], [
    AC_MSG_CHECKING([for existence of ${TCL_BIN_DIR}/tclConfig.sh])

    if test -f "${TCL_BIN_DIR}/tclConfig.sh" ; then
-        AC_MSG_RESULT([loading])
+	AC_MSG_RESULT([loading])
 	. "${TCL_BIN_DIR}/tclConfig.sh"
    else
-        AC_MSG_RESULT([could not find ${TCL_BIN_DIR}/tclConfig.sh])
+	AC_MSG_RESULT([could not find ${TCL_BIN_DIR}/tclConfig.sh])
    fi

    # If the TCL_BIN_DIR is the build directory (not the install directory),
@@ -379,9 +395,9 @@ AC_DEFUN([TEA_LOAD_TCLCONFIG], [
    # instead of TCL_BUILD_LIB_SPEC since it will work with both an
    # installed and uninstalled version of Tcl.
    if test -f "${TCL_BIN_DIR}/Makefile" ; then
-        TCL_LIB_SPEC="${TCL_BUILD_LIB_SPEC}"
-        TCL_STUB_LIB_SPEC="${TCL_BUILD_STUB_LIB_SPEC}"
-        TCL_STUB_LIB_PATH="${TCL_BUILD_STUB_LIB_PATH}"
+	TCL_LIB_SPEC="${TCL_BUILD_LIB_SPEC}"
+	TCL_STUB_LIB_SPEC="${TCL_BUILD_STUB_LIB_SPEC}"
+	TCL_STUB_LIB_PATH="${TCL_BUILD_STUB_LIB_PATH}"
    elif test "`uname -s`" = "Darwin"; then
 	# If Tcl was built as a framework, attempt to use the libraries
 	# from the framework at the given location so that linking works
@@ -474,10 +490,10 @@ AC_DEFUN([TEA_LOAD_TKCONFIG], [
    AC_MSG_CHECKING([for existence of ${TK_BIN_DIR}/tkConfig.sh])

    if test -f "${TK_BIN_DIR}/tkConfig.sh" ; then
-        AC_MSG_RESULT([loading])
+	AC_MSG_RESULT([loading])
 	. "${TK_BIN_DIR}/tkConfig.sh"
    else
-        AC_MSG_RESULT([could not find ${TK_BIN_DIR}/tkConfig.sh])
+	AC_MSG_RESULT([could not find ${TK_BIN_DIR}/tkConfig.sh])
    fi

    # If the TK_BIN_DIR is the build directory (not the install directory),
@@ -487,9 +503,9 @@ AC_DEFUN([TEA_LOAD_TKCONFIG], [
    # instead of TK_BUILD_LIB_SPEC since it will work with both an
    # installed and uninstalled version of Tcl.
    if test -f "${TK_BIN_DIR}/Makefile" ; then
-        TK_LIB_SPEC="${TK_BUILD_LIB_SPEC}"
-        TK_STUB_LIB_SPEC="${TK_BUILD_STUB_LIB_SPEC}"
-        TK_STUB_LIB_PATH="${TK_BUILD_STUB_LIB_PATH}"
+	TK_LIB_SPEC="${TK_BUILD_LIB_SPEC}"
+	TK_STUB_LIB_SPEC="${TK_BUILD_STUB_LIB_SPEC}"
+	TK_STUB_LIB_PATH="${TK_BUILD_STUB_LIB_PATH}"
    elif test "`uname -s`" = "Darwin"; then
 	# If Tk was built as a framework, attempt to use the libraries
 	# from the framework at the given location so that linking works
@@ -567,37 +583,37 @@ AC_DEFUN([TEA_LOAD_TKCONFIG], [
 AC_DEFUN([TEA_PROG_TCLSH], [
    AC_MSG_CHECKING([for tclsh])
    if test -f "${TCL_BIN_DIR}/Makefile" ; then
-        # tclConfig.sh is in Tcl build directory
-        if test "${TEA_PLATFORM}" = "windows"; then
-          if test -f "${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}${EXEEXT}" ; then
-            TCLSH_PROG="${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}${EXEEXT}"
-          elif test -f "${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}s${EXEEXT}" ; then
-            TCLSH_PROG="${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}s${EXEEXT}"
-          elif test -f "${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}t${EXEEXT}" ; then
-            TCLSH_PROG="${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}t${EXEEXT}"
-          elif test -f "${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}st${EXEEXT}" ; then
-            TCLSH_PROG="${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}st${EXEEXT}"
-          fi
-        else
-            TCLSH_PROG="${TCL_BIN_DIR}/tclsh"
-        fi
+	# tclConfig.sh is in Tcl build directory
+	if test "${TEA_PLATFORM}" = "windows"; then
+	  if test -f "${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}${EXEEXT}" ; then
+	    TCLSH_PROG="${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}${EXEEXT}"
+	  elif test -f "${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}s${EXEEXT}" ; then
+	    TCLSH_PROG="${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}s${EXEEXT}"
+	  elif test -f "${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}t${EXEEXT}" ; then
+	    TCLSH_PROG="${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}t${EXEEXT}"
+	  elif test -f "${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}st${EXEEXT}" ; then
+	    TCLSH_PROG="${TCL_BIN_DIR}/tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}st${EXEEXT}"
+	  fi
+	else
+	    TCLSH_PROG="${TCL_BIN_DIR}/tclsh"
+	fi
    else
-        # tclConfig.sh is in install location
-        if test "${TEA_PLATFORM}" = "windows"; then
-            TCLSH_PROG="tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}${EXEEXT}"
-        else
-            TCLSH_PROG="tclsh${TCL_MAJOR_VERSION}.${TCL_MINOR_VERSION}"
-        fi
-        list="`ls -d ${TCL_BIN_DIR}/../bin 2>/dev/null` \
-              `ls -d ${TCL_BIN_DIR}/..     2>/dev/null` \
-              `ls -d ${TCL_PREFIX}/bin     2>/dev/null`"
-        for i in $list ; do
-            if test -f "$i/${TCLSH_PROG}" ; then
-                REAL_TCL_BIN_DIR="`cd "$i"; pwd`/"
-                break
-            fi
-        done
-        TCLSH_PROG="${REAL_TCL_BIN_DIR}${TCLSH_PROG}"
+	# tclConfig.sh is in install location
+	if test "${TEA_PLATFORM}" = "windows"; then
+	    TCLSH_PROG="tclsh${TCL_MAJOR_VERSION}${TCL_MINOR_VERSION}${EXEEXT}"
+	else
+	    TCLSH_PROG="tclsh${TCL_MAJOR_VERSION}.${TCL_MINOR_VERSION}"
+	fi
+	list="`ls -d ${TCL_BIN_DIR}/../bin 2>/dev/null` \
+	      `ls -d ${TCL_BIN_DIR}/..     2>/dev/null` \
+	      `ls -d ${TCL_PREFIX}/bin     2>/dev/null`"
+	for i in $list ; do
+	    if test -f "$i/${TCLSH_PROG}" ; then
+		REAL_TCL_BIN_DIR="`cd "$i"; pwd`/"
+		break
+	    fi
+	done
+	TCLSH_PROG="${REAL_TCL_BIN_DIR}${TCLSH_PROG}"
    fi
    AC_MSG_RESULT([${TCLSH_PROG}])
    AC_SUBST(TCLSH_PROG)
@@ -625,37 +641,37 @@ AC_DEFUN([TEA_PROG_TCLSH], [
 AC_DEFUN([TEA_PROG_WISH], [
    AC_MSG_CHECKING([for wish])
    if test -f "${TK_BIN_DIR}/Makefile" ; then
-        # tkConfig.sh is in Tk build directory
-        if test "${TEA_PLATFORM}" = "windows"; then
-          if test -f "${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}${EXEEXT}" ; then
-            WISH_PROG="${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}${EXEEXT}"
-          elif test -f "${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}s${EXEEXT}" ; then
-            WISH_PROG="${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}$s{EXEEXT}"
-          elif test -f "${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}t${EXEEXT}" ; then
-            WISH_PROG="${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}t${EXEEXT}"
-          elif test -f "${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}st${EXEEXT}" ; then
-            WISH_PROG="${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}st${EXEEXT}"
-          fi
-        else
-            WISH_PROG="${TK_BIN_DIR}/wish"
-        fi
+	# tkConfig.sh is in Tk build directory
+	if test "${TEA_PLATFORM}" = "windows"; then
+	  if test -f "${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}${EXEEXT}" ; then
+	    WISH_PROG="${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}${EXEEXT}"
+	  elif test -f "${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}s${EXEEXT}" ; then
+	    WISH_PROG="${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}$s{EXEEXT}"
+	  elif test -f "${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}t${EXEEXT}" ; then
+	    WISH_PROG="${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}t${EXEEXT}"
+	  elif test -f "${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}st${EXEEXT}" ; then
+	    WISH_PROG="${TK_BIN_DIR}/wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}st${EXEEXT}"
+	  fi
+	else
+	    WISH_PROG="${TK_BIN_DIR}/wish"
+	fi
    else
-        # tkConfig.sh is in install location
-        if test "${TEA_PLATFORM}" = "windows"; then
-            WISH_PROG="wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}${EXEEXT}"
-        else
-            WISH_PROG="wish${TK_MAJOR_VERSION}.${TK_MINOR_VERSION}"
-        fi
-        list="`ls -d ${TK_BIN_DIR}/../bin 2>/dev/null` \
-              `ls -d ${TK_BIN_DIR}/..     2>/dev/null` \
-              `ls -d ${TK_PREFIX}/bin     2>/dev/null`"
-        for i in $list ; do
-            if test -f "$i/${WISH_PROG}" ; then
-                REAL_TK_BIN_DIR="`cd "$i"; pwd`/"
-                break
-            fi
-        done
-        WISH_PROG="${REAL_TK_BIN_DIR}${WISH_PROG}"
+	# tkConfig.sh is in install location
+	if test "${TEA_PLATFORM}" = "windows"; then
+	    WISH_PROG="wish${TK_MAJOR_VERSION}${TK_MINOR_VERSION}${EXEEXT}"
+	else
+	    WISH_PROG="wish${TK_MAJOR_VERSION}.${TK_MINOR_VERSION}"
+	fi
+	list="`ls -d ${TK_BIN_DIR}/../bin 2>/dev/null` \
+	      `ls -d ${TK_BIN_DIR}/..     2>/dev/null` \
+	      `ls -d ${TK_PREFIX}/bin     2>/dev/null`"
+	for i in $list ; do
+	    if test -f "$i/${WISH_PROG}" ; then
+		REAL_TK_BIN_DIR="`cd "$i"; pwd`/"
+		break
+	    fi
+	done
+	WISH_PROG="${REAL_TK_BIN_DIR}${WISH_PROG}"
    fi
    AC_MSG_RESULT([${WISH_PROG}])
    AC_SUBST(WISH_PROG)
@@ -717,22 +733,22 @@ AC_DEFUN([TEA_ENABLE_SHARED], [
    if test "$shared_ok" = "yes" ; then
 	AC_MSG_RESULT([shared])
 	SHARED_BUILD=1
-        STUBS_BUILD=1
+	STUBS_BUILD=1
    else
 	AC_MSG_RESULT([static])
 	SHARED_BUILD=0
 	AC_DEFINE(STATIC_BUILD, 1, [This a static build])
-        if test "$stubs_ok" = "yes" ; then
-          STUBS_BUILD=1
-        else
-          STUBS_BUILD=0
-        fi
+	if test "$stubs_ok" = "yes" ; then
+	  STUBS_BUILD=1
+	else
+	  STUBS_BUILD=0
+	fi
    fi
    if test "${STUBS_BUILD}" = "1" ; then
      AC_DEFINE(USE_TCL_STUBS, 1, [Use Tcl stubs])
      AC_DEFINE(USE_TCLOO_STUBS, 1, [Use TclOO stubs])
      if test "${TEA_WINDOWINGSYSTEM}" != ""; then
-        AC_DEFINE(USE_TK_STUBS, 1, [Use Tk stubs])
+	AC_DEFINE(USE_TK_STUBS, 1, [Use Tk stubs])
      fi
    fi

@@ -1177,21 +1193,21 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [
 	    fi

 	    if test "$GCC" != "yes" ; then
-	        if test "${SHARED_BUILD}" = "0" ; then
+		if test "${SHARED_BUILD}" = "0" ; then
 		    runtime=-MT
-	        else
+		else
 		    runtime=-MD
-	        fi
-	        case "x`echo \${VisualStudioVersion}`" in
-	            x1[[4-9]]*)
-		        lflags="${lflags} -nodefaultlib:libucrt.lib"
-		        TEA_ADD_LIBS([ucrt.lib])
-	            ;;
-	            *)
-	            ;;
-	        esac
+		fi
+		case "x`echo \${VisualStudioVersion}`" in
+		    x1[[4-9]]*)
+			lflags="${lflags} -nodefaultlib:libucrt.lib"
+			TEA_ADD_LIBS([ucrt.lib])
+		    ;;
+		    *)
+		    ;;
+		esac

-                if test "$do64bit" != "no" ; then
+		if test "$do64bit" != "no" ; then
 		    CC="cl.exe"
 		    RC="rc.exe"
 		    lflags="${lflags} -nologo -MACHINE:${MACHINE} "
@@ -1490,14 +1506,14 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [
 	    # Check to enable 64-bit flags for compiler/linker

 	    AS_IF([test "$do64bit" = yes], [
-	        AS_IF([test "$GCC" = yes], [
-	            AC_MSG_WARN([64bit mode not supported by gcc])
-	        ], [
-	            do64bit_ok=yes
-	            SHLIB_LD="ld -64 -shared -rdata_shared"
-	            CFLAGS="$CFLAGS -64"
-	            LDFLAGS_ARCH="-64"
-	        ])
+		AS_IF([test "$GCC" = yes], [
+		    AC_MSG_WARN([64bit mode not supported by gcc])
+		], [
+		    do64bit_ok=yes
+		    SHLIB_LD="ld -64 -shared -rdata_shared"
+		    CFLAGS="$CFLAGS -64"
+		    LDFLAGS_ARCH="-64"
+		])
 	    ])
 	    ;;
 	Linux*|GNU*|NetBSD-Debian|DragonFly-*|FreeBSD-*)
@@ -1519,7 +1535,7 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [
 		    CFLAGS="$CFLAGS $PTHREAD_CFLAGS"
 		    LDFLAGS="$LDFLAGS $PTHREAD_LIBS"])
 	    ;;
-            esac
+	    esac

 	    AS_IF([test $doRpath = yes], [
 		CC_SEARCH_FLAGS='"-Wl,-rpath,${LIB_RUNTIME_DIR}"'])
@@ -1724,9 +1740,9 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [
 	    # Digital OSF/1
 	    SHLIB_CFLAGS=""
 	    AS_IF([test "$SHARED_BUILD" = 1], [
-	        SHLIB_LD='ld -shared -expect_unresolved "*"'
+		SHLIB_LD='ld -shared -expect_unresolved "*"'
 	    ], [
-	        SHLIB_LD='ld -non_shared -expect_unresolved "*"'
+		SHLIB_LD='ld -non_shared -expect_unresolved "*"'
 	    ])
 	    SHLIB_SUFFIX=".so"
 	    AS_IF([test $doRpath = yes], [
@@ -1896,7 +1912,7 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [
 		LDFLAGS="$LDFLAGS -Wl,-Bexport"
 		AC_LINK_IFELSE([AC_LANG_PROGRAM([[]], [[int i;]])],
 			[tcl_cv_ld_Bexport=yes],[tcl_cv_ld_Bexport=no])
-	        LDFLAGS=$hold_ldflags])
+		LDFLAGS=$hold_ldflags])
 	    AS_IF([test $tcl_cv_ld_Bexport = yes], [
 		LDFLAGS="$LDFLAGS -Wl,-Bexport"
 	    ])
@@ -2015,8 +2031,8 @@ dnl # preprocessing tests use only CPPFLAGS.
 		SHORT s;
 		LONG l;
 	    ]])],
-        [tcl_cv_winnt_ignore_void=yes],
-        [tcl_cv_winnt_ignore_void=no])
+	[tcl_cv_winnt_ignore_void=yes],
+	[tcl_cv_winnt_ignore_void=no])
 	)
 	if test "$tcl_cv_winnt_ignore_void" = "yes" ; then
 	    AC_DEFINE(HAVE_WINNT_IGNORE_VOID, 1,
@@ -2529,20 +2545,19 @@ AC_DEFUN([TEA_TCL_LINK_LIBS], [
 #
 #	Might define the following vars:
 #		_ISOC99_SOURCE
-#		_LARGEFILE64_SOURCE
-#		_LARGEFILE_SOURCE64
+#		_FILE_OFFSET_BITS
 #
 #--------------------------------------------------------------------

 AC_DEFUN([TEA_TCL_EARLY_FLAG],[
    AC_CACHE_VAL([tcl_cv_flag_]translit($1,[A-Z],[a-z]),
 	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[$2]], [[$3]])],
-	    [tcl_cv_flag_]translit($1,[A-Z],[a-z])=no,[AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[[#define ]$1[ 1
+	    [tcl_cv_flag_]translit($1,[A-Z],[a-z])=no,[AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[[#define ]$1[ ]m4_default([$4],[1])[
 ]$2]], [[$3]])],
 	[tcl_cv_flag_]translit($1,[A-Z],[a-z])=yes,
 	[tcl_cv_flag_]translit($1,[A-Z],[a-z])=no)]))
    if test ["x${tcl_cv_flag_]translit($1,[A-Z],[a-z])[}" = "xyes"] ; then
-	AC_DEFINE($1, 1, [Add the ]$1[ flag when building])
+	AC_DEFINE($1, m4_default([$4],[1]), [Add the ]$1[ flag when building])
 	tcl_flags="$tcl_flags $1"
    fi
 ])
@@ -2552,10 +2567,10 @@ AC_DEFUN([TEA_TCL_EARLY_FLAGS],[
    tcl_flags=""
    TEA_TCL_EARLY_FLAG(_ISOC99_SOURCE,[#include <stdlib.h>],
 	[char *p = (char *)strtoll; char *q = (char *)strtoull;])
-    TEA_TCL_EARLY_FLAG(_LARGEFILE64_SOURCE,[#include <sys/stat.h>],
-	[struct stat64 buf; int i = stat64("/", &buf);])
-    TEA_TCL_EARLY_FLAG(_LARGEFILE_SOURCE64,[#include <sys/stat.h>],
-	[char *p = (char *)open64;])
+    if test "${TCL_MAJOR_VERSION}" -ne 8 ; then
+	TEA_TCL_EARLY_FLAG(_FILE_OFFSET_BITS,[#include <sys/stat.h>],
+	    [switch (0) { case 0: case (sizeof(off_t)==sizeof(long long)): ; }],64)
+    fi
    if test "x${tcl_flags}" = "x" ; then
 	AC_MSG_RESULT([none])
    else
@@ -2579,6 +2594,7 @@ AC_DEFUN([TEA_TCL_EARLY_FLAGS],[
 #		HAVE_STRUCT_DIRENT64, HAVE_DIR64
 #		HAVE_STRUCT_STAT64
 #		HAVE_TYPE_OFF64_T
+#		_TIME_BITS
 #
 #--------------------------------------------------------------------

@@ -2592,9 +2608,9 @@ AC_DEFUN([TEA_TCL_64BIT_FLAGS], [
 	# See if we could use long anyway  Note that we substitute in the
 	# type that is our current guess for a 64-bit type inside this check
 	# program, so it should be modified only carefully...
-        AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[switch (0) {
-            case 1: case (sizeof(${tcl_type_64bit})==sizeof(long)): ;
-        }]])],[tcl_cv_type_64bit=${tcl_type_64bit}],[])])
+	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[switch (0) {
+	    case 1: case (sizeof(${tcl_type_64bit})==sizeof(long)): ;
+	}]])],[tcl_cv_type_64bit=${tcl_type_64bit}],[])])
    if test "${tcl_cv_type_64bit}" = none ; then
 	AC_DEFINE(TCL_WIDE_INT_IS_LONG, 1, [Do 'long' and 'long long' have the same size (64-bit)?])
 	AC_MSG_RESULT([yes])
@@ -2609,6 +2625,25 @@ AC_DEFUN([TEA_TCL_64BIT_FLAGS], [
 	AC_MSG_RESULT([${tcl_cv_type_64bit}])

 	# Now check for auxiliary declarations
+    if test "${TCL_MAJOR_VERSION}" -ne 8 ; then
+	    AC_CACHE_CHECK([for 64-bit time_t], tcl_cv_time_t_64,[
+		AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h>]],
+		    [[switch (0) {case 0: case (sizeof(time_t)==sizeof(long long)): ;}]])],
+		    [tcl_cv_time_t_64=yes],[tcl_cv_time_t_64=no])])
+	    if test "x${tcl_cv_time_t_64}" = "xno" ; then
+		# Note that _TIME_BITS=64 requires _FILE_OFFSET_BITS=64
+		# which SC_TCL_EARLY_FLAGS has defined if necessary.
+		AC_CACHE_CHECK([if _TIME_BITS=64 enables 64-bit time_t], tcl_cv__time_bits,[
+		    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#define _TIME_BITS 64
+#include <sys/types.h>]],
+			[[switch (0) {case 0: case (sizeof(time_t)==sizeof(long long)): ;}]])],
+			[tcl_cv__time_bits=yes],[tcl_cv__time_bits=no])])
+		if test "x${tcl_cv__time_bits}" = "xyes" ; then
+		    AC_DEFINE(_TIME_BITS, 64, [_TIME_BITS=64 enables 64-bit time_t.])
+		fi
+	    fi
+	fi
+
 	AC_CACHE_CHECK([for struct dirent64], tcl_cv_struct_dirent64,[
 	    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h>
 #include <dirent.h>]], [[struct dirent64 p;]])],
@@ -2620,7 +2655,7 @@ AC_DEFUN([TEA_TCL_64BIT_FLAGS], [
 	AC_CACHE_CHECK([for DIR64], tcl_cv_DIR64,[
 	    AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <sys/types.h>
 #include <dirent.h>]], [[struct dirent64 *p; DIR64 d = opendir64(".");
-            p = readdir64(d); rewinddir64(d); closedir64(d);]])],
+	    p = readdir64(d); rewinddir64(d); closedir64(d);]])],
 		[tcl_cv_DIR64=yes], [tcl_cv_DIR64=no])])
 	if test "x${tcl_cv_DIR64}" = "xyes" ; then
 	    AC_DEFINE(HAVE_DIR64, 1, [Is 'DIR64' in <sys/types.h>?])
@@ -2643,8 +2678,8 @@ AC_DEFUN([TEA_TCL_64BIT_FLAGS], [
 	dnl Define HAVE_TYPE_OFF64_T only when the off64_t type and the
 	dnl functions lseek64 and open64 are defined.
 	if test "x${tcl_cv_type_off64_t}" = "xyes" && \
-	        test "x${ac_cv_func_lseek64}" = "xyes" && \
-	        test "x${ac_cv_func_open64}" = "xyes" ; then
+		test "x${ac_cv_func_lseek64}" = "xyes" && \
+		test "x${ac_cv_func_open64}" = "xyes" ; then
 	    AC_DEFINE(HAVE_TYPE_OFF64_T, 1, [Is off64_t in <sys/types.h>?])
 	    AC_MSG_RESULT([yes])
 	else
@@ -2747,8 +2782,6 @@ The PACKAGE_NAME variable must be defined by your TEA configure.ac])
    AC_SUBST(PKG_LIB_FILE)
    AC_SUBST(PKG_LIB_FILE8)
    AC_SUBST(PKG_LIB_FILE9)
-    # Substitute STUB_LIB_FILE in case package creates a stub library too.
-    AC_SUBST(PKG_STUB_LIB_FILE)

    # We AC_SUBST these here to ensure they are subst'ed,
    # in case the user doesn't call TEA_ADD_...
@@ -3106,11 +3139,15 @@ AC_DEFUN([TEA_SETUP_COMPILER], [
 	fi
    fi

+    if test "${TCL_MAJOR_VERSION}" -lt 9 -a "${TCL_MINOR_VERSION}" -lt 7; then
+	    AC_DEFINE(Tcl_Size, int, [Is 'Tcl_Size' in <tcl.h>?])
+	fi
+
    #--------------------------------------------------------------------
    # Common compiler flag setup
    #--------------------------------------------------------------------

-    AC_C_BIGENDIAN
+    AC_C_BIGENDIAN(,,,[#])
 ])

 #------------------------------------------------------------------------
@@ -3174,10 +3211,11 @@ print("manifest needed")

    PACKAGE_LIB_PREFIX8="${PACKAGE_LIB_PREFIX}"
    PACKAGE_LIB_PREFIX9="${PACKAGE_LIB_PREFIX}tcl9"
-    if test "${TCL_MAJOR_VERSION}" -gt 8 ; then
+    if test "${TCL_MAJOR_VERSION}" -gt 8 -a x"${with_tcl8}" = x; then
 	PACKAGE_LIB_PREFIX="${PACKAGE_LIB_PREFIX9}"
    else
 	PACKAGE_LIB_PREFIX="${PACKAGE_LIB_PREFIX8}"
+	AC_DEFINE(TCL_MAJOR_VERSION, 8, [Compile for Tcl8?])
    fi
    if test "${TEA_PLATFORM}" = "windows" ; then
 	if test "${SHARED_BUILD}" = "1" ; then
@@ -3202,7 +3240,11 @@ print("manifest needed")
 	    eval eval "PKG_LIB_FILE=${PACKAGE_LIB_PREFIX}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}"
 	fi
 	# Some packages build their own stubs libraries
-	eval eval "PKG_STUB_LIB_FILE=${PACKAGE_LIB_PREFIX}${PACKAGE_NAME}stub${UNSHARED_LIB_SUFFIX}"
+	if test "${TCL_MAJOR_VERSION}" -gt 8 -a x"${with_tcl8}" = x; then
+	    eval eval "PKG_STUB_LIB_FILE=${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}stub.a"
+	else
+	    eval eval "PKG_STUB_LIB_FILE=${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}stub${UNSHARED_LIB_SUFFIX}"
+	fi
 	if test "$GCC" = "yes"; then
 	    PKG_STUB_LIB_FILE=lib${PKG_STUB_LIB_FILE}
 	fi
@@ -3221,12 +3263,16 @@ print("manifest needed")
 	    eval eval "PKG_LIB_FILE=lib${PACKAGE_LIB_PREFIX}${PACKAGE_NAME}${SHARED_LIB_SUFFIX}"
 	    RANLIB=:
 	else
-	    eval eval "PKG_LIB_FILE=lib${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}"
-	    eval eval "PKG_LIB_FILE=lib${PACKAGE_LIB_PREFIX9}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}"
+	    eval eval "PKG_LIB_FILE8=lib${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}"
+	    eval eval "PKG_LIB_FILE9=lib${PACKAGE_LIB_PREFIX9}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}"
 	    eval eval "PKG_LIB_FILE=lib${PACKAGE_LIB_PREFIX}${PACKAGE_NAME}${UNSHARED_LIB_SUFFIX}"
 	fi
 	# Some packages build their own stubs libraries
-	eval eval "PKG_STUB_LIB_FILE=lib${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}stub${UNSHARED_LIB_SUFFIX}"
+	if test "${TCL_MAJOR_VERSION}" -gt 8 -a x"${with_tcl8}" = x; then
+	    eval eval "PKG_STUB_LIB_FILE=lib${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}stub.a"
+	else
+	    eval eval "PKG_STUB_LIB_FILE=lib${PACKAGE_LIB_PREFIX8}${PACKAGE_NAME}stub${UNSHARED_LIB_SUFFIX}"
+	fi
    fi

    # These are escaped so that only CFLAGS is picked up at configure time.
@@ -3240,6 +3286,8 @@ print("manifest needed")
    AC_SUBST(MAKE_SHARED_LIB)
    AC_SUBST(MAKE_STATIC_LIB)
    AC_SUBST(MAKE_STUB_LIB)
+    # Substitute STUB_LIB_FILE in case package creates a stub library too.
+    AC_SUBST(PKG_STUB_LIB_FILE)
    AC_SUBST(RANLIB_STUB)
    AC_SUBST(VC_MANIFEST_EMBED_DLL)
    AC_SUBST(VC_MANIFEST_EMBED_EXE)
@@ -3366,9 +3414,9 @@ AC_DEFUN([TEA_PRIVATE_TCL_HEADERS], [
 	# any *_NATIVE vars be defined in the Makefile
 	TCL_INCLUDES="-I${TCL_GENERIC_DIR_NATIVE} -I${TCL_PLATFORM_DIR_NATIVE}"
 	if test "`uname -s`" = "Darwin"; then
-            # If Tcl was built as a framework, attempt to use
-            # the framework's Headers and PrivateHeaders directories
-            case ${TCL_DEFS} in
+	    # If Tcl was built as a framework, attempt to use
+	    # the framework's Headers and PrivateHeaders directories
+	    case ${TCL_DEFS} in
 	    	*TCL_FRAMEWORK*)
 		    if test -d "${TCL_BIN_DIR}/Headers" -a \
 			    -d "${TCL_BIN_DIR}/PrivateHeaders"; then
@@ -3376,7 +3424,7 @@ AC_DEFUN([TEA_PRIVATE_TCL_HEADERS], [
 		    else
 			TCL_INCLUDES="${TCL_INCLUDES} ${TCL_INCLUDE_SPEC} `echo "${TCL_INCLUDE_SPEC}" | sed -e 's/Headers/PrivateHeaders/'`"
 		    fi
-	            ;;
+		    ;;
 	    esac
 	    result="Using ${TCL_INCLUDES}"
 	else
@@ -3817,10 +3865,10 @@ AC_DEFUN([TEA_LOAD_CONFIG], [
    AC_MSG_CHECKING([for existence of ${$1_BIN_DIR}/$1Config.sh])

    if test -f "${$1_BIN_DIR}/$1Config.sh" ; then
-        AC_MSG_RESULT([loading])
+	AC_MSG_RESULT([loading])
 	. "${$1_BIN_DIR}/$1Config.sh"
    else
-        AC_MSG_RESULT([file not found])
+	AC_MSG_RESULT([file not found])
    fi

    #
@@ -3834,11 +3882,11 @@ AC_DEFUN([TEA_LOAD_CONFIG], [

    if test -f "${$1_BIN_DIR}/Makefile" ; then
 	AC_MSG_WARN([Found Makefile - using build library specs for $1])
-        $1_LIB_SPEC=${$1_BUILD_LIB_SPEC}
-        $1_STUB_LIB_SPEC=${$1_BUILD_STUB_LIB_SPEC}
-        $1_STUB_LIB_PATH=${$1_BUILD_STUB_LIB_PATH}
-        $1_INCLUDE_SPEC=${$1_BUILD_INCLUDE_SPEC}
-        $1_LIBRARY_PATH=${$1_LIBRARY_PATH}
+	$1_LIB_SPEC=${$1_BUILD_LIB_SPEC}
+	$1_STUB_LIB_SPEC=${$1_BUILD_STUB_LIB_SPEC}
+	$1_STUB_LIB_PATH=${$1_BUILD_STUB_LIB_PATH}
+	$1_INCLUDE_SPEC=${$1_BUILD_INCLUDE_SPEC}
+	$1_LIBRARY_PATH=${$1_LIBRARY_PATH}
    fi

    AC_SUBST($1_VERSION)
@@ -3919,6 +3967,10 @@ AC_DEFUN([TEA_EXPORT_CONFIG], [
 	eval $1_LIB_FLAG="-l$1`echo ${PACKAGE_VERSION} | tr -d .`"
 	eval $1_STUB_LIB_FLAG="-l$1stub`echo ${PACKAGE_VERSION} | tr -d .`"
    fi
+    if test "${TCL_MAJOR_VERSION}" -gt 8 -a x"${with_tcl8}" = x; then
+	eval $1_STUB_LIB_FLAG="-l$1stub"
+    fi
+
    $1_BUILD_LIB_SPEC="-L`$CYGPATH $(pwd)` ${$1_LIB_FLAG}"
    $1_LIB_SPEC="-L`$CYGPATH ${pkglibdir}` ${$1_LIB_FLAG}"
    $1_BUILD_STUB_LIB_SPEC="-L`$CYGPATH $(pwd)` [$]{$1_STUB_LIB_FLAG}"
@@ -4008,52 +4060,52 @@ AC_DEFUN([TEA_ZIPFS_SUPPORT], [
    AC_CACHE_VAL(ac_cv_path_macher, [
    search_path=`echo ${PATH} | sed -e 's/:/ /g'`
    for dir in $search_path ; do
-        for j in `ls -r $dir/macher 2> /dev/null` \
-            `ls -r $dir/macher 2> /dev/null` ; do
-        if test x"$ac_cv_path_macher" = x ; then
-            if test -f "$j" ; then
-            ac_cv_path_macher=$j
-            break
-            fi
-        fi
-        done
+	for j in `ls -r $dir/macher 2> /dev/null` \
+	    `ls -r $dir/macher 2> /dev/null` ; do
+	if test x"$ac_cv_path_macher" = x ; then
+	    if test -f "$j" ; then
+	    ac_cv_path_macher=$j
+	    break
+	    fi
+	fi
+	done
    done
    ])
    if test -f "$ac_cv_path_macher" ; then
-        MACHER_PROG="$ac_cv_path_macher"
-        AC_MSG_RESULT([$MACHER_PROG])
-        AC_MSG_RESULT([Found macher in environment])
+	MACHER_PROG="$ac_cv_path_macher"
+	AC_MSG_RESULT([$MACHER_PROG])
+	AC_MSG_RESULT([Found macher in environment])
    fi
    AC_MSG_CHECKING([for zip])
    AC_CACHE_VAL(ac_cv_path_zip, [
    search_path=`echo ${PATH} | sed -e 's/:/ /g'`
    for dir in $search_path ; do
-        for j in `ls -r $dir/zip 2> /dev/null` \
-            `ls -r $dir/zip 2> /dev/null` ; do
-        if test x"$ac_cv_path_zip" = x ; then
-            if test -f "$j" ; then
-            ac_cv_path_zip=$j
-            break
-            fi
-        fi
-        done
+	for j in `ls -r $dir/zip 2> /dev/null` \
+	    `ls -r $dir/zip 2> /dev/null` ; do
+	if test x"$ac_cv_path_zip" = x ; then
+	    if test -f "$j" ; then
+	    ac_cv_path_zip=$j
+	    break
+	    fi
+	fi
+	done
    done
    ])
    if test -f "$ac_cv_path_zip" ; then
-        ZIP_PROG="$ac_cv_path_zip"
-        AC_MSG_RESULT([$ZIP_PROG])
-        ZIP_PROG_OPTIONS="-rq"
-        ZIP_PROG_VFSSEARCH="*"
-        AC_MSG_RESULT([Found INFO Zip in environment])
-        # Use standard arguments for zip
+	ZIP_PROG="$ac_cv_path_zip"
+	AC_MSG_RESULT([$ZIP_PROG])
+	ZIP_PROG_OPTIONS="-rq"
+	ZIP_PROG_VFSSEARCH="*"
+	AC_MSG_RESULT([Found INFO Zip in environment])
+	# Use standard arguments for zip
    else
-        # It is not an error if an installed version of Zip can't be located.
-        # We can use the locally distributed minizip instead
-        ZIP_PROG="./minizip${EXEEXT_FOR_BUILD}"
-        ZIP_PROG_OPTIONS="-o -r"
-        ZIP_PROG_VFSSEARCH="*"
-        ZIP_INSTALL_OBJS="minizip${EXEEXT_FOR_BUILD}"
-        AC_MSG_RESULT([No zip found on PATH. Building minizip])
+	# It is not an error if an installed version of Zip can't be located.
+	# We can use the locally distributed minizip instead
+	ZIP_PROG="./minizip${EXEEXT_FOR_BUILD}"
+	ZIP_PROG_OPTIONS="-o -r"
+	ZIP_PROG_VFSSEARCH="*"
+	ZIP_INSTALL_OBJS="minizip${EXEEXT_FOR_BUILD}"
+	AC_MSG_RESULT([No zip found on PATH. Building minizip])
    fi
    AC_SUBST(MACHER_PROG)
    AC_SUBST(ZIP_PROG)
@@ -4064,4 +4116,4 @@ AC_DEFUN([TEA_ZIPFS_SUPPORT], [

 # Local Variables:
 # mode: autoconf
-# End:
+# End:
@@ -1,430 +1,61 @@
-# makefile.vc --                                               -*- Makefile -*-
+#------------------------------------------------------------- -*- makefile -*-
 #
-# Microsoft Visual C++ makefile for use with nmake.exe v1.62+ (VC++ 5.0+)
+# Sample makefile for building Tcl extensions.
 #
-# This makefile is based upon the Tcl 8.4 Makefile.vc and modified to 
-# make it suitable as a general package makefile. Look for the word EDIT
-# which marks sections that may need modification. As a minumum you will
-# need to change the PROJECT, DOTVERSION and DLLOBJS variables to values
-# relevant to your package.
+# Basic build, test and install
+#   nmake /s /nologo /f makefile.vc INSTALLDIR=c:\path\to\tcl
+#   nmake /s /nologo /f makefile.vc INSTALLDIR=c:\path\to\tcl test
+#   nmake /s /nologo /f makefile.vc INSTALLDIR=c:\path\to\tcl install
+#
+# For other build options (debug, static etc.)
+# See TIP 477 (https://core.tcl.tk/tips/doc/trunk/tip/477.md) for
+# detailed documentation.
 #
 # See the file "license.terms" for information on usage and redistribution
 # of this file, and for a DISCLAIMER OF ALL WARRANTIES.
-# 
-# Copyright (c) 1995-1996 Sun Microsystems, Inc.
-# Copyright (c) 1998-2000 Ajuba Solutions.
-# Copyright (c) 2001 ActiveState Corporation.
-# Copyright (c) 2001-2002 David Gravereaux.
-# Copyright (c) 2003 Pat Thoyts
-#
-#-------------------------------------------------------------------------
-# RCS: @(#)$Id: makefile.vc,v 1.4 2004/07/26 08:22:05 patthoyts Exp $
-#-------------------------------------------------------------------------
-
-!if !defined(MSDEVDIR) && !defined(MSVCDIR) && !defined(VCINSTALLDIR) && !defined(MSSDK) && !defined(WINDOWSSDKDIR)
-MSG = ^
-You will need to run vcvars32.bat from Developer Studio, first, to setup^
-the environment.  Jump to this line to read the new instructions.
-!error $(MSG)
-!endif
-
-#------------------------------------------------------------------------------
-# HOW TO USE this makefile:
-#
-# 1)  It is now necessary to have %MSVCDir% set in the environment.  This is
-#     used  as a check to see if vcvars32.bat had been run prior to running
-#     nmake or during the installation of Microsoft Visual C++, MSVCDir had
-#     been set globally and the PATH adjusted.  Either way is valid.
-#
-#     You'll need to run vcvars32.bat contained in the MsDev's vc(98)/bin
-#     directory to setup the proper environment, if needed, for your current
-#     setup.  This is a needed bootstrap requirement and allows the swapping of
-#     different environments to be easier.
-#
-# 2)  To use the Platform SDK (not expressly needed), run setenv.bat after
-#     vcvars32.bat according to the instructions for it.  This can also turn on
-#     the 64-bit compiler, if your SDK has it.
-#
-# 3)  Targets are:
-#	all       -- Builds everything.
-#       <project> -- Builds the project (eg: nmake sample)
-#	test      -- Builds and runs the test suite.
-#	install   -- Installs the built binaries and libraries to $(INSTALLDIR)
-#		     in an appropriate subdirectory.
-#	clean/realclean/distclean -- varying levels of cleaning.
-#
-# 4)  Macros usable on the commandline:
-#	INSTALLDIR=<path>
-#		Sets where to install Tcl from the built binaries.
-#		C:\Progra~1\Tcl is assumed when not specified.
-#
-#	OPTS=static,msvcrt,staticpkg,threads,symbols,profile,loimpact,none
-#		Sets special options for the core.  The default is for none.
-#		Any combination of the above may be used (comma separated).
-#		'none' will over-ride everything to nothing.
-#
-#		static  =  Builds a static library of the core instead of a
-#			   dll.  The shell will be static (and large), as well.
-#		msvcrt  =  Effects the static option only to switch it from
-#			   using libcmt(d) as the C runtime [by default] to
-#			   msvcrt(d). This is useful for static embedding
-#			   support.
-#		staticpkg = Effects the static option only to switch
-#			   tclshXX.exe to have the dde and reg extension linked
-#			   inside it.
-#		threads =  Turns on full multithreading support.
-#		thrdalloc = Use the thread allocator (shared global free pool).
-#		symbols =  Adds symbols for step debugging.
-#		profile =  Adds profiling hooks.  Map file is assumed.
-#		loimpact =  Adds a flag for how NT treats the heap to keep memory
-#			   in use, low.  This is said to impact alloc performance.
-#
-#	STATS=memdbg,compdbg,none
-#		Sets optional memory and bytecode compiler debugging code added
-#		to the core.  The default is for none.  Any combination of the
-#		above may be used (comma separated).  'none' will over-ride
-#		everything to nothing.
-#
-#		memdbg   = Enables the debugging memory allocator.
-#		compdbg  = Enables byte compilation logging.
-#
-#	MACHINE=(IX86|IA64|ALPHA)
-#		Set the machine type used for the compiler, linker, and
-#		resource compiler.  This hook is needed to tell the tools
-#		when alternate platforms are requested.  IX86 is the default
-#		when not specified.
-#
-#	TMP_DIR=<path>
-#	OUT_DIR=<path>
-#		Hooks to allow the intermediate and output directories to be
-#		changed.  $(OUT_DIR) is assumed to be 
-#		$(BINROOT)\(Release|Debug) based on if symbols are requested.
-#		$(TMP_DIR) will de $(OUT_DIR)\<buildtype> by default.
-#
-#	TESTPAT=<file>
-#		Reads the tests requested to be run from this file.
-#
-#	CFG_ENCODING=encoding
-#		name of encoding for configuration information. Defaults
-#		to cp1252
-#
-# 5)  Examples:
-#
-#	Basic syntax of calling nmake looks like this:
-#	nmake [-nologo] -f makefile.vc [target|macrodef [target|macrodef] [...]]
-#
-#                        Standard (no frills)
-#       c:\tcl_src\win\>c:\progra~1\micros~1\vc98\bin\vcvars32.bat
-#       Setting environment for using Microsoft Visual C++ tools.
-#       c:\tcl_src\win\>nmake -f makefile.vc all
-#       c:\tcl_src\win\>nmake -f makefile.vc install INSTALLDIR=c:\progra~1\tcl
-#
-#                         Building for Win64
-#       c:\tcl_src\win\>c:\progra~1\micros~1\vc98\bin\vcvars32.bat
-#       Setting environment for using Microsoft Visual C++ tools.
-#       c:\tcl_src\win\>c:\progra~1\platfo~1\setenv.bat /pre64 /RETAIL
-#       Targeting Windows pre64 RETAIL
-#       c:\tcl_src\win\>nmake -f makefile.vc MACHINE=IA64
 #
 #------------------------------------------------------------------------------
-#==============================================================================
-###############################################################################
-#------------------------------------------------------------------------------

-!if !exist("makefile.vc")
-MSG = ^
-You must run this makefile only from the directory it is in.^
-Please `cd` to its location first.
-!error $(MSG)
-!endif
+# PROJECT is sqlite, not sqlite3 to match TEA AC_INIT definition.
+# This makes the generated DLL name also consistent between the two
+# except for the "t" suffix which is the convention for nmake builds.
+PROJECT = sqlite
+PRJ_PACKAGE_TCLNAME = sqlite3

-#-------------------------------------------------------------------------
-# Project specific information (EDIT)
-#
-# You should edit this with the name and version of your project. This
-# information is used to generate the name of the package library and
-# it's install location.
-#
-# For example, the sample extension is  going to build sample04.dll and
-# would install it into $(INSTALLDIR)\lib\sample04
-#
-# You need to specify the object files that need to be linked into your
-# binary here.
-#
-#-------------------------------------------------------------------------
+!include "rules-ext.vc"

-PROJECT = sqlite3
-!include "rules.vc"
+PRJ_OBJS = $(TMP_DIR)\tclsqlite3.obj

-# nmakehelp -V <file> <tag> will search the file for tag, skips until a
-#	number and returns all character until a character not in [0-9.ab]
-#	is read.
-
-!if [echo REM = This file is generated from Makefile.vc > versions.vc]
-!endif
-# get project version from row "AC_INIT([sqlite], [3.x.y])"
-!if [echo DOTVERSION = \>> versions.vc] \
-   && [nmakehlp -V ..\configure.ac AC_INIT >> versions.vc]
-!endif
-!include "versions.vc"
-
-VERSION         = $(DOTVERSION:.=)
-STUBPREFIX      = $(PROJECT)stub
-
-#-------------------------------------------------------------------------
-# Target names and paths ( shouldn't need changing )
-#-------------------------------------------------------------------------
-
-BINROOT		= .
-ROOT            = ..
-
-PRJIMPLIB	= $(OUT_DIR)\$(PROJECT)$(VERSION)$(SUFX).lib
-PRJLIBNAME	= $(PROJECT).$(EXT)
-PRJLIB		= $(OUT_DIR)\$(PRJLIBNAME)
-
-PRJSTUBLIBNAME	= $(STUBPREFIX)$(VERSION).lib
-PRJSTUBLIB	= $(OUT_DIR)\$(PRJSTUBLIBNAME)
-
-### Make sure we use backslash only.
-PRJ_INSTALL_DIR         = $(_INSTALLDIR)\$(PROJECT)$(DOTVERSION)
-LIB_INSTALL_DIR		= $(PRJ_INSTALL_DIR)
-BIN_INSTALL_DIR		= $(PRJ_INSTALL_DIR)
-DOC_INSTALL_DIR		= $(PRJ_INSTALL_DIR)
-SCRIPT_INSTALL_DIR	= $(PRJ_INSTALL_DIR)
-INCLUDE_INSTALL_DIR	= $(_TCLDIR)\include
-
-### The following paths CANNOT have spaces in them.
-GENERICDIR	= $(ROOT)\generic
-WINDIR		= $(ROOT)\win
-LIBDIR          = $(ROOT)\library
-DOCDIR		= $(ROOT)\doc
-TOOLSDIR	= $(ROOT)\tools
-COMPATDIR	= $(ROOT)\compat
-
-### Figure out where the primary source code file(s) is/are.
-!if exist("$(ROOT)\..\..\sqlite3.c") && exist("$(ROOT)\..\..\src\tclsqlite.c")
-SQL_INCLUDES = -I"$(ROOT)\..\.."
-SQLITE_SRCDIR = $(ROOT)\..\..
-TCLSQLITE_SRCDIR = $(ROOT)\..\..\src
-DLLOBJS = $(TMP_DIR)\sqlite3.obj $(TMP_DIR)\tclsqlite.obj
-!else
-TCLSQLITE_SRCDIR = $(ROOT)\generic
-DLLOBJS = $(TMP_DIR)\tclsqlite3.obj
-!endif
-
-#---------------------------------------------------------------------
-# Compile flags
-#---------------------------------------------------------------------
-
-!if !$(DEBUG)
-!if $(OPTIMIZING)
-### This cranks the optimization level to maximize speed
-cdebug	= -O2 -Op -Gs
-!else
-cdebug	=
-!endif
-!else if "$(MACHINE)" == "IA64"
-### Warnings are too many, can't support warnings into errors.
-cdebug	= -Z7 -Od -GZ
-!else
-cdebug	= -Z7 -WX -Od -GZ
-!endif
-
-### Declarations common to all compiler options
-cflags = -nologo -c -W3 -D_CRT_SECURE_NO_WARNINGS -YX -Fp$(TMP_DIR)^\
-
-!if $(MSVCRT)
-!if $(DEBUG)
-crt = -MDd
-!else
-crt = -MD
-!endif
-!else
-!if $(DEBUG)
-crt = -MTd
-!else
-crt = -MT
-!endif
-!endif
-
-INCLUDES	= $(SQL_INCLUDES) $(TCL_INCLUDES) -I"$(WINDIR)" \
-                  -I"$(GENERICDIR)" -I"$(ROOT)\.."
-BASE_CLFAGS	= $(cflags) $(cdebug) $(crt) $(INCLUDES) \
+# Preprocessor macros specific to sqlite3.
+PRJ_DEFINES = -I"$(ROOT)\.." -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE \
+                  -DSQLITE_ENABLE_DBPAGE_VTAB=1 -DSQLITE_ENABLE_DBSTAT_VTAB=1 \
+                  -DSQLITE_ENABLE_FTS3_PARENTHESIS=1 -DSQLITE_ENABLE_FTS4=1 \
+                  -DSQLITE_ENABLE_FTS5=1 -DSQLITE_ENABLE_COLUMN_METADATA=1 \
+                  -DSQLITE_ENABLE_JSON1=1 -DSQLITE_LIKE_DOESNT_MATCH_BLOBS=1 \
                  -DSQLITE_3_SUFFIX_ONLY=1 -DSQLITE_ENABLE_RTREE=1 \
-                  -DSQLITE_ENABLE_FTS3=1 -DSQLITE_OMIT_DEPRECATED=1 \
-                  -DSQLITE_ENABLE_FTS4=1 \
-                  -DSQLITE_ENABLE_FTS5=1 \
-                  -DSQLITE_3_SUFFIX_ONLY=1 \
-                  -DSQLITE_ENABLE_RTREE=1 \
-                  -DSQLITE_ENABLE_GEOPOLY=1 \
-                  -DSQLITE_ENABLE_MATH_FUNCTIONS=1 \
-                  -DSQLITE_ENABLE_DESERIALIZE=1 \
-                  -DSQLITE_ENABLE_DBPAGE_VTAB=1 \
-                  -DSQLITE_ENABLE_BYTECODE_VTAB=1 \
-                  -DSQLITE_ENABLE_DBSTAT_VTAB=1 
+                  -DSQLITE_UNTESTABLE=1 -DSQLITE_OMIT_LOOKASIDE=1 \
+                  -DSQLITE_SECURE_DELETE=1 -DSQLITE_SOUNDEX=1 -DSQLITE_ENABLE_GEOPOLY=1 \
+                  -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1 \
+                  -DSQLITE_ENABLE_MATH_FUNCTIONS=1 -DDSQLITE_USE_ALLOCA=1 \
+                  -DSQLITE_ENABLE_STAT4=1 -DSQLITE_OMIT_DEPRECATED=1 \
+                  -DSQLITE_WIN32_GETVERSIONEX=0 -DSQLITE_WIN32_NO_ANSI=1
+PRJ_DEFINES = $(PRJ_DEFINES) -I$(TMP_DIR)

-CON_CFLAGS	= $(cflags) $(cdebug) $(crt) -DCONSOLE -DSQLITE_ENABLE_FTS3=1
-TCL_CFLAGS	= -DBUILD_sqlite -DUSE_TCL_STUBS \
-                  -DPACKAGE_VERSION="\"$(DOTVERSION)\"" $(BASE_CLFAGS) \
-                  $(OPTDEFINES)
+# Standard targets to build, install, test etc.
+!include "$(_RULESDIR)\targets.vc"

-#---------------------------------------------------------------------
-# Link flags
-#---------------------------------------------------------------------
+# The built-in pkgindex does no suffice for our extension as
+# the PROJECT name (sqlite) is not same as init function name (Sqlite3)
+pkgindex:
+	@echo if {[package vsatisfies [package provide Tcl] 9.0-]} { > $(OUT_DIR)\pkgIndex.tcl
+	@echo package ifneeded $(PRJ_PACKAGE_TCLNAME) $(DOTVERSION) \
+	    [list load [file join $$dir $(PRJLIBNAME9)] [string totitle $(PRJ_PACKAGE_TCLNAME)]] >> $(OUT_DIR)\pkgIndex.tcl
+	@echo } else { >> $(OUT_DIR)\pkgIndex.tcl
+	@echo package ifneeded $(PRJ_PACKAGE_TCLNAME) $(DOTVERSION) \
+	    [list load [file join $$dir $(PRJLIBNAME8)] [string totitle $(PRJ_PACKAGE_TCLNAME)]] >> $(OUT_DIR)\pkgIndex.tcl
+	@echo } >> $(OUT_DIR)\pkgIndex.tcl

-!if $(DEBUG)
-ldebug	= -debug:full -debugtype:cv
-!else
-ldebug	= -release -opt:ref -opt:icf,3
-!endif
+# Install the manpage though on Windows, doubt it does much good
+install:    default-install-docs-n

-### Declarations common to all linker options
-lflags	= -nologo -machine:$(MACHINE) $(ldebug)
-
-!if $(PROFILE)
-lflags	= $(lflags) -profile
-!endif
-
-!if $(ALIGN98_HACK) && !$(STATIC_BUILD)
-### Align sections for PE size savings.
-lflags	= $(lflags) -opt:nowin98
-!else if !$(ALIGN98_HACK) && $(STATIC_BUILD)
-### Align sections for speed in loading by choosing the virtual page size.
-lflags	= $(lflags) -align:4096
-!endif
-
-!if $(LOIMPACT)
-lflags	= $(lflags) -ws:aggressive
-!endif
-
-dlllflags = $(lflags) -dll
-conlflags = $(lflags) -subsystem:console
-guilflags = $(lflags) -subsystem:windows
-baselibs   = $(TCLSTUBLIB)
-
-#---------------------------------------------------------------------
-# TclTest flags
-#---------------------------------------------------------------------
-
-!IF "$(TESTPAT)" != ""
-TESTFLAGS = $(TESTFLAGS) -file $(TESTPAT)
-!ENDIF
-
-#---------------------------------------------------------------------
-# Project specific targets (EDIT)
-#---------------------------------------------------------------------
-
-all:	    setup $(PROJECT)
-$(PROJECT): setup $(PRJLIB)
-install:    install-binaries install-libraries install-docs
-
-# Tests need to ensure we load the right dll file we
-# have to handle the output differently on Win9x.
-#
-!if "$(OS)" == "Windows_NT"  || "$(MSVCDIR)" == "IDE"
-test: setup $(PROJECT)
-        set TCL_LIBRARY=$(ROOT)/library
-        $(TCLSH) <<
-load $(PRJLIB:\=/)
-cd "$(ROOT)/tests"
-set argv "$(TESTFLAGS)"
-source all.tcl
-<<
-!else
-test: setup $(PROJECT)
-        echo Please wait while the test results are collected
-        set TCL_LIBRARY=$(ROOT)/library
-        $(TCLSH) << >tests.log
-load $(PRJLIB:\=/)
-cd "$(ROOT)/tests"
-set argv "$(TESTFLAGS)"
-source all.tcl
-<<
-        type tests.log | more
-!endif
-
-setup:
-	@if not exist $(OUT_DIR)\nul mkdir $(OUT_DIR)
-	@if not exist $(TMP_DIR)\nul mkdir $(TMP_DIR)
-
-$(PRJLIB): $(DLLOBJS)
-	$(link32) $(dlllflags) -out:$@ $(baselibs) @<<
-$**
-<<
-	-@del $*.exp
-
-$(PRJSTUBLIB): $(PRJSTUBOBJS)
-	$(lib32) -nologo -out:$@ $(PRJSTUBOBJS)
-
-#---------------------------------------------------------------------
-# Implicit rules
-#---------------------------------------------------------------------
-
-$(TMP_DIR)\sqlite3.obj:		$(SQLITE_SRCDIR)\sqlite3.c
-	$(cc32) $(TCL_CFLAGS) -DBUILD_$(PROJECT) -Fo$(TMP_DIR)\ \
-		-c $(SQLITE_SRCDIR)\sqlite3.c
-
-$(TMP_DIR)\tclsqlite.obj:	$(TCLSQLITE_SRCDIR)\tclsqlite.c
-	$(cc32) $(TCL_CFLAGS) -DBUILD_$(PROJECT) -Fo$(TMP_DIR)\ \
-		-c $(TCLSQLITE_SRCDIR)\tclsqlite.c
-
-$(TMP_DIR)\tclsqlite3.obj:	$(TCLSQLITE_SRCDIR)\tclsqlite3.c
-	$(cc32) $(TCL_CFLAGS) -DBUILD_$(PROJECT) -Fo$(TMP_DIR)\ \
-		-c $(TCLSQLITE_SRCDIR)\tclsqlite3.c
-
-{$(WINDIR)}.rc{$(TMP_DIR)}.res:
-	$(rc32) -fo $@ -r -i "$(GENERICDIR)" -D__WIN32__ \
-!if $(DEBUG)
-	-d DEBUG \
-!endif
-!if $(TCL_THREADS)
-	-d TCL_THREADS \
-!endif
-!if $(STATIC_BUILD)
-	-d STATIC_BUILD \
-!endif
-	$<
-
-.SUFFIXES:
-.SUFFIXES:.c .rc
-
-#---------------------------------------------------------------------
-# Installation. (EDIT)
-#
-# You may need to modify this section to reflect the final distribution
-# of your files and possibly to generate documentation.
-#
-#---------------------------------------------------------------------
-
-install-binaries:
-	@echo Installing binaries to '$(SCRIPT_INSTALL_DIR)'
-	@if not exist "$(SCRIPT_INSTALL_DIR)" mkdir "$(SCRIPT_INSTALL_DIR)"
-	@$(CPY) $(PRJLIB) "$(SCRIPT_INSTALL_DIR)" >NUL
-
-install-libraries:
-        @echo Installing libraries to '$(SCRIPT_INSTALL_DIR)'
-        @if exist $(LIBDIR) $(CPY) $(LIBDIR)\*.tcl "$(SCRIPT_INSTALL_DIR)"
-        @echo Installing package index in '$(SCRIPT_INSTALL_DIR)'
-        @type << >"$(SCRIPT_INSTALL_DIR)\pkgIndex.tcl"
-package ifneeded $(PROJECT) $(DOTVERSION) \
-    [list load [file join $$dir $(PRJLIBNAME)] sqlite3]
-<<
-
-install-docs:
-	@echo Installing documentation files to '$(DOC_INSTALL_DIR)'
-	@if exist $(DOCDIR) $(CPY) $(DOCDIR)\*.n "$(DOC_INSTALL_DIR)"
-
-#---------------------------------------------------------------------
-# Clean up
-#---------------------------------------------------------------------
-
-clean:
-	@if exist $(TMP_DIR)\nul $(RMDIR) $(TMP_DIR)
-	@if exist $(WINDIR)\version.vc del $(WINDIR)\version.vc
-
-realclean: clean
-	@if exist $(OUT_DIR)\nul $(RMDIR) $(OUT_DIR)
-
-distclean: realclean
-	@if exist $(WINDIR)\nmakehlp.exe del $(WINDIR)\nmakehlp.exe
-	@if exist $(WINDIR)\nmakehlp.obj del $(WINDIR)\nmakehlp.obj
+# Explicit dependency rules
@@ -0,0 +1,123 @@
+# This file should only be included in makefiles for Tcl extensions,
+# NOT in the makefile for Tcl itself.
+
+!ifndef _RULES_EXT_VC
+
+# We need to run from the directory the parent makefile is located in.
+# nmake does not tell us what makefile was used to invoke it so parent
+# makefile has to set the MAKEFILEVC macro or we just make a guess and
+# warn if we think that is not the case.
+!if "$(MAKEFILEVC)" == ""
+
+!if exist("$(PROJECT).vc")
+MAKEFILEVC = $(PROJECT).vc
+!elseif exist("makefile.vc")
+MAKEFILEVC = makefile.vc
+!endif
+!endif # "$(MAKEFILEVC)" == ""
+
+!if !exist("$(MAKEFILEVC)")
+MSG = ^
+You must run nmake from the directory containing the project makefile.^
+If you are doing that and getting this message, set the MAKEFILEVC^
+macro to the name of the project makefile.
+!message WARNING: $(MSG)
+!endif
+
+!if "$(PROJECT)" == "tcl"
+!error The rules-ext.vc file is not intended for Tcl itself.
+!endif
+
+# We extract version numbers using the nmakehlp program. For now use
+# the local copy of nmakehlp. Once we locate Tcl, we will use that
+# one if it is newer.
+!if "$(MACHINE)" == "IX86" || "$(MACHINE)" == "$(NATIVE_ARCH)"
+!if [$(CC) -nologo -DNDEBUG "nmakehlp.c" -link -subsystem:console > nul]
+!endif
+!else
+!if [copy x86_64-w64-mingw32-nmakehlp.exe nmakehlp.exe >NUL]
+!endif
+!endif
+
+# First locate the Tcl directory that we are working with.
+!if "$(TCLDIR)" != ""
+
+_RULESDIR = $(TCLDIR:/=\)
+
+!else
+
+# If an installation path is specified, that is also the Tcl directory.
+# Also Tk never builds against an installed Tcl, it needs Tcl sources
+!if defined(INSTALLDIR) && "$(PROJECT)" != "tk"
+_RULESDIR=$(INSTALLDIR:/=\)
+!else
+# Locate Tcl sources
+!if [echo _RULESDIR = \> nmakehlp.out] \
+   || [nmakehlp -L generic\tcl.h >> nmakehlp.out]
+_RULESDIR = ..\..\tcl
+!else
+!include nmakehlp.out
+!endif
+
+!endif # defined(INSTALLDIR)....
+
+!endif # ifndef TCLDIR
+
+# Now look for the targets.vc file under the Tcl root. Note we check this
+# file and not rules.vc because the latter also exists on older systems.
+!if exist("$(_RULESDIR)\lib\nmake\targets.vc") # Building against installed Tcl
+_RULESDIR = $(_RULESDIR)\lib\nmake
+!elseif exist("$(_RULESDIR)\win\targets.vc")   # Building against Tcl sources
+_RULESDIR = $(_RULESDIR)\win
+!else
+# If we have not located Tcl's targets file, most likely we are compiling
+# against an older version of Tcl and so must use our own support files.
+_RULESDIR = .
+!endif
+
+!if "$(_RULESDIR)" != "."
+# Potentially using Tcl's support files. If this extension has its own
+# nmake support files, need to compare the versions and pick newer.
+
+!if exist("rules.vc") # The extension has its own copy
+
+!if [echo TCL_RULES_MAJOR = \> versions.vc] \
+   && [nmakehlp -V "$(_RULESDIR)\rules.vc" RULES_VERSION_MAJOR >> versions.vc]
+!endif
+!if [echo TCL_RULES_MINOR = \>> versions.vc] \
+   && [nmakehlp -V "$(_RULESDIR)\rules.vc" RULES_VERSION_MINOR >> versions.vc]
+!endif
+
+!if [echo OUR_RULES_MAJOR = \>> versions.vc] \
+   && [nmakehlp -V "rules.vc" RULES_VERSION_MAJOR >> versions.vc]
+!endif
+!if [echo OUR_RULES_MINOR = \>> versions.vc] \
+   && [nmakehlp -V "rules.vc" RULES_VERSION_MINOR >> versions.vc]
+!endif
+!include versions.vc
+# We have a newer version of the support files, use them
+!if ($(TCL_RULES_MAJOR) != $(OUR_RULES_MAJOR)) || ($(TCL_RULES_MINOR) < $(OUR_RULES_MINOR))
+_RULESDIR = .
+!endif
+
+!endif # if exist("rules.vc")
+
+!endif # if $(_RULESDIR) != "."
+
+# Let rules.vc know what copy of nmakehlp.c to use.
+NMAKEHLPC = $(_RULESDIR)\nmakehlp.c
+
+# Get rid of our internal defines before calling rules.vc
+!undef TCL_RULES_MAJOR
+!undef TCL_RULES_MINOR
+!undef OUR_RULES_MAJOR
+!undef OUR_RULES_MINOR
+
+!if exist("$(_RULESDIR)\rules.vc")
+!message *** Using $(_RULESDIR)\rules.vc
+!include "$(_RULESDIR)\rules.vc"
+!else
+!error *** Could not locate rules.vc in $(_RULESDIR)
+!endif
+
+!endif # _RULES_EXT_VC
@@ -0,0 +1,98 @@
+#------------------------------------------------------------- -*- makefile -*-
+# targets.vc --
+#
+# Part of the nmake based build system for Tcl and its extensions.
+# This file defines some standard targets for the convenience of extensions
+# and can be optionally included by the extension makefile.
+# See TIP 477 (https://core.tcl-lang.org/tips/doc/main/tip/477.md) for docs.
+
+$(PROJECT): setup pkgindex $(PRJLIB)
+
+!ifdef PRJ_STUBOBJS
+$(PROJECT): $(PRJSTUBLIB)
+$(PRJSTUBLIB): $(PRJ_STUBOBJS)
+	$(LIBCMD) $**
+
+$(PRJ_STUBOBJS):
+	$(CCSTUBSCMD) %s
+!endif # PRJ_STUBOBJS
+
+!ifdef PRJ_MANIFEST
+$(PROJECT): $(PRJLIB).manifest
+$(PRJLIB).manifest: $(PRJ_MANIFEST)
+	@nmakehlp -s << $** >$@
+@MACHINE@	  $(MACHINE:IX86=X86)
+<<
+!endif
+
+!if "$(PROJECT)" != "tcl" && "$(PROJECT)" != "tk"
+$(PRJLIB): $(PRJ_OBJS) $(RESFILE)
+!if $(STATIC_BUILD)
+       $(LIBCMD) $**
+!else
+       $(DLLCMD) $**
+       $(_VC_MANIFEST_EMBED_DLL)
+!endif
+       -@del $*.exp
+!endif
+
+!if "$(PRJ_HEADERS)" != "" && "$(PRJ_OBJS)" != ""
+$(PRJ_OBJS): $(PRJ_HEADERS)
+!endif
+
+# If parent makefile has defined stub objects, add their installation
+# to the default install
+!if "$(PRJ_STUBOBJS)" != ""
+default-install: default-install-stubs
+!endif
+
+# Unlike the other default targets, these cannot be in rules.vc because
+# the executed command depends on existence of macro PRJ_HEADERS_PUBLIC
+# that the parent makefile will not define until after including rules-ext.vc
+!if "$(PRJ_HEADERS_PUBLIC)" != ""
+default-install: default-install-headers
+default-install-headers:
+	@echo Installing headers to '$(INCLUDE_INSTALL_DIR)'
+	@for %f in ($(PRJ_HEADERS_PUBLIC)) do @$(COPY) %f "$(INCLUDE_INSTALL_DIR)"
+!endif
+
+!if "$(DISABLE_STANDARD_TARGETS)" == ""
+DISABLE_STANDARD_TARGETS = 0
+!endif
+
+!if "$(DISABLE_TARGET_setup)" == ""
+DISABLE_TARGET_setup = 0
+!endif
+!if "$(DISABLE_TARGET_install)" == ""
+DISABLE_TARGET_install = 0
+!endif
+!if "$(DISABLE_TARGET_clean)" == ""
+DISABLE_TARGET_clean = 0
+!endif
+!if "$(DISABLE_TARGET_test)" == ""
+DISABLE_TARGET_test = 0
+!endif
+!if "$(DISABLE_TARGET_shell)" == ""
+DISABLE_TARGET_shell = 0
+!endif
+
+!if !$(DISABLE_STANDARD_TARGETS)
+!if !$(DISABLE_TARGET_setup)
+setup: default-setup
+!endif
+!if !$(DISABLE_TARGET_install)
+install: default-install
+!endif
+!if !$(DISABLE_TARGET_clean)
+clean: default-clean
+realclean: hose
+hose: default-hose
+distclean: realclean default-distclean
+!endif
+!if !$(DISABLE_TARGET_test)
+test: default-test
+!endif
+!if !$(DISABLE_TARGET_shell)
+shell: default-shell
+!endif
+!endif # DISABLE_STANDARD_TARGETS
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.46.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.47.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.46.0'
-PACKAGE_STRING='sqlite 3.46.0'
+PACKAGE_VERSION='3.47.0'
+PACKAGE_STRING='sqlite 3.47.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''

@@ -781,18 +781,6 @@ TARGET_HAVE_EDITLINE
 TARGET_HAVE_READLINE
 TARGET_READLINE_INC
 TARGET_READLINE_LIBS
-HAVE_TCL
-TCL_SHLIB_SUFFIX
-TCL_STUB_LIB_SPEC
-TCL_STUB_LIB_FLAG
-TCL_STUB_LIB_FILE
-TCL_LIB_SPEC
-TCL_LIB_FLAG
-TCL_LIB_FILE
-TCL_INCLUDE_SPEC
-TCL_SRC_DIR
-TCL_BIN_DIR
-TCL_VERSION
 TARGET_EXEEXT
 SQLITE_OS_WIN
 SQLITE_OS_UNIX
@@ -805,7 +793,12 @@ HAVE_WASI_SDK
 RELEASE
 VERSION
 program_prefix
+TSTRNNR_OPTS
 TCLLIBDIR
+HAVE_TCL
+TCL_STUB_LIB_SPEC
+TCL_LIB_SPEC
+TCL_INCLUDE_SPEC
 TCLSH_CMD
 INSTALL_DATA
 INSTALL_SCRIPT
@@ -894,12 +887,14 @@ enable_fast_install
 with_gnu_ld
 enable_libtool_lock
 enable_largefile
+with_tclsh
+with_tcl
+enable_tcl
+enable_test_status
 with_wasi_sdk
 enable_threadsafe
 enable_releasemode
 enable_tempstore
-enable_tcl
-with_tcl
 enable_editline
 enable_readline
 with_readline_lib
@@ -1472,7 +1467,7 @@ if test "$ac_init_help" = "long"; then
  # Omit some internal or obsolete options to make the list less imposing.
  # This message is too long to be a string in the A/UX 3.1 sh.
  cat <<_ACEOF
-\`configure' configures sqlite 3.46.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.47.0 to adapt to many kinds of systems.

 Usage: $0 [OPTION]... [VAR=VALUE]...

@@ -1537,7 +1532,7 @@ fi

 if test -n "$ac_init_help"; then
  case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.46.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.47.0:";;
   esac
  cat <<\_ACEOF

@@ -1551,11 +1546,13 @@ Optional Features:
                          optimize for fast installation [default=yes]
  --disable-libtool-lock  avoid locking (might break parallel builds)
  --disable-largefile     omit support for large files
+  --disable-tcl           omit building accessory programs that require
+                          TCL-dev
+  --enable-test-status    Full-screen status of tests
  --disable-threadsafe    Disable mutexing
  --enable-releasemode    Support libtool link to release mode
  --enable-tempstore      Use an in-ram database for temporary tables
                          (never,no,yes,always)
-  --disable-tcl           do not build TCL extension
  --enable-editline       enable BSD editline support
  --disable-readline      disable readline support
  --enable-debug          enable debugging & verbose explain
@@ -1583,10 +1580,10 @@ Optional Packages:
  --with-pic              try to use only PIC/non-PIC objects [default=use
                          both]
  --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
+  --with-tclsh=PATHNAME   full pathname of a tclsh to use
+  --with-tcl=DIR          directory containing (tclConfig.sh)
  --with-wasi-sdk=DIR     directory containing the WASI SDK. Triggers
                          cross-compile to WASM.
-  --with-tcl=DIR          directory containing tcl configuration
-                          (tclConfig.sh)
  --with-readline-lib     specify readline library
  --with-readline-inc     specify readline include paths
  --with-linenoise=DIR    source directory for linenoise library
@@ -1668,7 +1665,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
  cat <<\_ACEOF
-sqlite configure 3.46.0
+sqlite configure 3.47.0
 generated by GNU Autoconf 2.69

 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2087,7 +2084,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.

-It was created by sqlite $as_me 3.46.0, which was
+It was created by sqlite $as_me 3.47.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  $ $0 $@
@@ -3945,13 +3942,13 @@ if ${lt_cv_nm_interface+:} false; then :
 else
  lt_cv_nm_interface="BSD nm"
  echo "int some_variable = 0;" > conftest.$ac_ext
-  (eval echo "\"\$as_me:3948: $ac_compile\"" >&5)
+  (eval echo "\"\$as_me:3945: $ac_compile\"" >&5)
  (eval "$ac_compile" 2>conftest.err)
  cat conftest.err >&5
-  (eval echo "\"\$as_me:3951: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
+  (eval echo "\"\$as_me:3948: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
  (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
  cat conftest.err >&5
-  (eval echo "\"\$as_me:3954: output\"" >&5)
+  (eval echo "\"\$as_me:3951: output\"" >&5)
  cat conftest.out >&5
  if $GREP 'External.*some_variable' conftest.out > /dev/null; then
    lt_cv_nm_interface="MS dumpbin"
@@ -5157,7 +5154,7 @@ ia64-*-hpux*)
  ;;
 *-*-irix6*)
  # Find out which ABI we are using.
-  echo '#line 5160 "configure"' > conftest.$ac_ext
+  echo '#line 5157 "configure"' > conftest.$ac_ext
  if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
  (eval $ac_compile) 2>&5
  ac_status=$?
@@ -6682,11 +6679,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:6685: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:6682: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>conftest.err)
   ac_status=$?
   cat conftest.err >&5
-   echo "$as_me:6689: \$? = $ac_status" >&5
+   echo "$as_me:6686: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s "$ac_outfile"; then
     # The compiler can only warn and ignore the option if not recognized
     # So say no if there are warnings other than the usual output.
@@ -7021,11 +7018,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7024: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7021: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>conftest.err)
   ac_status=$?
   cat conftest.err >&5
-   echo "$as_me:7028: \$? = $ac_status" >&5
+   echo "$as_me:7025: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s "$ac_outfile"; then
     # The compiler can only warn and ignore the option if not recognized
     # So say no if there are warnings other than the usual output.
@@ -7126,11 +7123,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7129: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7126: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>out/conftest.err)
   ac_status=$?
   cat out/conftest.err >&5
-   echo "$as_me:7133: \$? = $ac_status" >&5
+   echo "$as_me:7130: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s out/conftest2.$ac_objext
   then
     # The compiler can only warn and ignore the option if not recognized
@@ -7181,11 +7178,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7184: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7181: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>out/conftest.err)
   ac_status=$?
   cat out/conftest.err >&5
-   echo "$as_me:7188: \$? = $ac_status" >&5
+   echo "$as_me:7185: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s out/conftest2.$ac_objext
   then
     # The compiler can only warn and ignore the option if not recognized
@@ -9561,7 +9558,7 @@ else
  lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  lt_status=$lt_dlunknown
  cat > conftest.$ac_ext <<_LT_EOF
-#line 9564 "configure"
+#line 9561 "configure"
 #include "confdefs.h"

 #if HAVE_DLFCN_H
@@ -9657,7 +9654,7 @@ else
  lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  lt_status=$lt_dlunknown
  cat > conftest.$ac_ext <<_LT_EOF
-#line 9660 "configure"
+#line 9657 "configure"
 #include "confdefs.h"

 #if HAVE_DLFCN_H
@@ -10314,11 +10311,31 @@ done
 USE_AMALGAMATION=1

 #########
-# See whether we can run specific tclsh versions known to work well;
-# if not, then we fall back to plain tclsh.
-# TODO: try other versions before falling back?
+# Figure out all the name of a working tclsh and parameters needed to compile against Tcl.
+# The --with-tcl= and/or --with-tclsh= configuration arguments might be useful for this.
 #
-for ac_prog in tclsh8.7 tclsh8.6 tclsh8.5 tclsh
+
+# Check whether --with-tclsh was given.
+if test "${with_tclsh+set}" = set; then :
+  withval=$with_tclsh;
+fi
+
+
+# Check whether --with-tcl was given.
+if test "${with_tcl+set}" = set; then :
+  withval=$with_tcl;
+fi
+
+# Check whether --enable-tcl was given.
+if test "${enable_tcl+set}" = set; then :
+  enableval=$enable_tcl; use_tcl=$enableval
+else
+  use_tcl=yes
+fi
+
+original_use_tcl=${use_tcl}
+if test x"${with_tclsh}" == x -a x"${with_tcl}" == x; then
+  for ac_prog in tclsh9.0 tclsh8.6 tclsh
 do
  # Extract the first word of "$ac_prog", so it can be a program name with args.
 set dummy $ac_prog; ac_word=$2
@@ -10361,10 +10378,99 @@ fi
 done
 test -n "$TCLSH_CMD" || TCLSH_CMD="none"

+  with_tclsh=${TCLSH_CMD}
+fi
+if test x"${with_tclsh}" != x -a x"${with_tclsh}" != xnone; then
+  TCLSH_CMD=${with_tclsh}
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: using tclsh at \"$TCLSH_CMD\"" >&5
+$as_echo "using tclsh at \"$TCLSH_CMD\"" >&6; }
+  if test x"${use_tcl}" = "xyes"; then
+    with_tcl=`${with_tclsh} <${srcdir}/tool/find_tclconfig.tcl`
+    if test x"${with_tcl}" != x; then
+      { $as_echo "$as_me:${as_lineno-$LINENO}: result: $TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}" >&5
+$as_echo "$TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}" >&6; }
+    else
+      { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $TCLSH_CMD is unable to recommend a tclConfig.sh" >&5
+$as_echo "$as_me: WARNING: $TCLSH_CMD is unable to recommend a tclConfig.sh" >&2;}
+      use_tcl=no
+    fi
+  fi
+fi
+if test x"${use_tcl}" = "xyes"; then
+  if test x"${with_tcl}" != x; then
+    if test -r ${with_tcl}/tclConfig.sh; then
+      tclconfig="${with_tcl}/tclConfig.sh"
+    else
+      for i in tcl8.6 tcl9.0 lib; do
+        if test -r ${with_tcl}/$i/tclConfig.sh; then
+          tclconfig=${with_tcl}/$i/tclConfig.sh
+          break
+        fi
+      done
+    fi
+    if test ! -r "${tclconfig}"; then
+      as_fn_error $? "no tclConfig.sh file found under ${with_tcl}" "$LINENO" 5
+    fi
+  else
+    # If we have not yet found a tclConfig.sh file, look in $libdir whic is
+    # set automatically by autoconf or by the --prefix command-line option.
+    # See https://sqlite.org/forum/forumpost/e04e693439a22457
+    libdir=${prefix}/lib
+    if test -r ${libdir}/tclConfig.sh; then
+      tclconfig=${libdir}/tclConfig.sh
+    else
+      for i in tcl8.6 tcl9.0 lib; do
+        if test -r ${libdir}/$i/tclConfig.sh; then
+          tclconfig=${libdir}/$i/tclConfig.sh
+          break
+        fi
+      done
+    fi
+    if test ! -r "${tclconfig}"; then
+      as_fn_error $? "cannot find a usable tclConfig.sh file.
+        Use --with-tcl=DIR to specify a directory where tclConfig.sh can be found.
+        SQLite does not use TCL internally, but TCL is required to build SQLite
+        from canonical sources and TCL is required for testing." "$LINENO" 5
+    fi
+  fi
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: loading TCL configuration from ${tclconfig}" >&5
+$as_echo "loading TCL configuration from ${tclconfig}" >&6; }
+  . ${tclconfig}
+
+
+
+  # There are lots of other configuration variables that are provided by the
+  # tclConfig.sh file and that could be included here.  But as of right now,
+  # TCL_LIB_SPEC is the only what that the Makefile uses.
+  HAVE_TCL=1
+elif test x"${original_use_tcl}" = "xno"; then
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: unable to run tests because of --disable-tcl" >&5
+$as_echo "unable to run tests because of --disable-tcl" >&6; }
+  HAVE_TCL=0
+else
+  { $as_echo "$as_me:${as_lineno-$LINENO}: result: unable to run tests because no tclConfig.sh file could be located" >&5
+$as_echo "unable to run tests because no tclConfig.sh file could be located" >&6; }
+  HAVE_TCL=0
+fi
+
+if test x"$TCLSH_CMD" == x; then
+  TCLSH_CMD=${TCL_EXEC_PREFIX}/bin/tclsh${TCL_VERSION}
+  if test ! -x ${TCLSH_CMD}; then
+    TCLSH_CMD_2=${TCL_EXEC_PREFIX}/bin/tclsh
+    if test ! -x ${TCLSH_CMD_2}; then
+      { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: cannot find a usable tclsh at either ${TCLSH_CMD} or ${TCLSH_CMD_2}" >&5
+$as_echo "$as_me: WARNING: cannot find a usable tclsh at either ${TCLSH_CMD} or ${TCLSH_CMD_2}" >&2;}
+      TCLSH_CMD=none
+    else
+      TCLSH_CMD=${TCLSH_CMD_2}
+    fi
+  fi
+fi
 if test "$TCLSH_CMD" = "none"; then
  # If we can't find a local tclsh, then building the amalgamation will fail.
  # We act as though --disable-amalgamation has been used.
-  echo "Warning: can't find tclsh - defaulting to non-amalgamation build."
+  { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Warning: can't find tclsh - defaulting to non-amalgamation build." >&5
+$as_echo "$as_me: WARNING: Warning: can't find tclsh - defaulting to non-amalgamation build." >&2;}
  USE_AMALGAMATION=0
  TCLSH_CMD="tclsh"
 fi
@@ -10372,7 +10478,6 @@ fi


 if test "x${TCLLIBDIR+set}" != "xset" ; then
-  TCLLIBDIR='$(libdir)'
  for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` ; do
    if test -d $i ; then
      TCLLIBDIR=$i
@@ -10382,6 +10487,23 @@ if test "x${TCLLIBDIR+set}" != "xset" ; then
  TCLLIBDIR="${TCLLIBDIR}/sqlite3"
 fi

+#########
+# Set up options for running tests.
+#
+# Check whether --enable-test-status was given.
+if test "${enable_test_status+set}" = set; then :
+  enableval=$enable_test_status; use_vt100=$enableval
+else
+  use_vt100=no
+fi
+
+if test $use_vt100 != no; then
+  TSTRNNR_OPTS=--status
+else
+  TSTRNNR_OPTS=
+fi
+
+

 #########
 # Set up an appropriate program prefix
@@ -10769,213 +10891,6 @@ fi



-##########
-# Figure out all the parameters needed to compile against Tcl.
-#
-# This code is derived from the SC_PATH_TCLCONFIG and SC_LOAD_TCLCONFIG
-# macros in the in the tcl.m4 file of the standard TCL distribution.
-# Those macros could not be used directly since we have to make some
-# minor changes to accomodate systems that do not have TCL installed.
-#
-# Check whether --enable-tcl was given.
-if test "${enable_tcl+set}" = set; then :
-  enableval=$enable_tcl; use_tcl=$enableval
-else
-  use_tcl=yes
-fi
-
-if test "${use_tcl}" = "yes" ; then
-
-# Check whether --with-tcl was given.
-if test "${with_tcl+set}" = set; then :
-  withval=$with_tcl; with_tclconfig=${withval}
-fi
-
-  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for Tcl configuration" >&5
-$as_echo_n "checking for Tcl configuration... " >&6; }
-  if ${ac_cv_c_tclconfig+:} false; then :
-  $as_echo_n "(cached) " >&6
-else
-
-    # First check to see if --with-tcl was specified.
-    if test x"${with_tclconfig}" != x ; then
-      if test -f "${with_tclconfig}/tclConfig.sh" ; then
-        ac_cv_c_tclconfig=`(cd ${with_tclconfig}; pwd)`
-      else
-        as_fn_error $? "${with_tclconfig} directory doesn't contain tclConfig.sh" "$LINENO" 5
-      fi
-    fi
-
-    # Start autosearch by asking tclsh
-    if test x"${ac_cv_c_tclconfig}" = x ; then
-      if test x"$cross_compiling" = xno; then
-        for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}`
-        do
-          if test -f "$i/tclConfig.sh" ; then
-            ac_cv_c_tclconfig="$i"
-            break
-          fi
-        done
-      fi
-    fi
-
-    # On ubuntu 14.10, $auto_path on tclsh is not quite correct.
-    # So try again after applying corrections.
-    if test x"${ac_cv_c_tclconfig}" = x ; then
-      if test x"$cross_compiling" = xno; then
-        for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD} | sed 's,/tcltk/tcl,/tcl,g'`
-        do
-          if test -f "$i/tclConfig.sh" ; then
-            ac_cv_c_tclconfig="$i"
-            break
-          fi
-        done
-      fi
-    fi
-
-    # Recent versions of Xcode on Macs hid the tclConfig.sh file
-    # in a strange place.
-    if test x"${ac_cv_c_tclconfig}" = x ; then
-      if test x"$cross_compiling" = xno; then
-        for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib
-        do
-          if test -f "$i/tclConfig.sh" ; then
-            ac_cv_c_tclconfig="$i"
-            break
-          fi
-        done
-      fi
-    fi
-
-    # then check for a private Tcl installation
-    if test x"${ac_cv_c_tclconfig}" = x ; then
-      for i in \
-            ../tcl \
-            `ls -dr ../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \
-            `ls -dr ../tcl[8-9].[0-9] 2>/dev/null` \
-            `ls -dr ../tcl[8-9].[0-9]* 2>/dev/null` \
-            ../../tcl \
-            `ls -dr ../../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \
-            `ls -dr ../../tcl[8-9].[0-9] 2>/dev/null` \
-            `ls -dr ../../tcl[8-9].[0-9]* 2>/dev/null` \
-            ../../../tcl \
-            `ls -dr ../../../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \
-            `ls -dr ../../../tcl[8-9].[0-9] 2>/dev/null` \
-            `ls -dr ../../../tcl[8-9].[0-9]* 2>/dev/null`
-      do
-        if test -f "$i/unix/tclConfig.sh" ; then
-          ac_cv_c_tclconfig=`(cd $i/unix; pwd)`
-          break
-        fi
-      done
-    fi
-
-    # check in a few common install locations
-    if test x"${ac_cv_c_tclconfig}" = x ; then
-      for i in \
-            `ls -d ${libdir} 2>/dev/null` \
-            `ls -d /usr/local/lib 2>/dev/null` \
-            `ls -d /usr/contrib/lib 2>/dev/null` \
-            `ls -d /usr/lib 2>/dev/null`
-      do
-        if test -f "$i/tclConfig.sh" ; then
-           ac_cv_c_tclconfig=`(cd $i; pwd)`
-           break
-        fi
-      done
-    fi
-
-    # check in a few other private locations
-    if test x"${ac_cv_c_tclconfig}" = x ; then
-      for i in \
-         ${srcdir}/../tcl \
-         `ls -dr ${srcdir}/../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \
-         `ls -dr ${srcdir}/../tcl[8-9].[0-9] 2>/dev/null` \
-         `ls -dr ${srcdir}/../tcl[8-9].[0-9]* 2>/dev/null`
-      do
-        if test -f "$i/unix/tclConfig.sh" ; then
-          ac_cv_c_tclconfig=`(cd $i/unix; pwd)`
-          break
-        fi
-      done
-    fi
-
-fi
-
-
-  if test x"${ac_cv_c_tclconfig}" = x ; then
-    use_tcl=no
-    { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Can't find Tcl configuration definitions" >&5
-$as_echo "$as_me: WARNING: Can't find Tcl configuration definitions" >&2;}
-    { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: *** Without Tcl the regression tests cannot be executed ***" >&5
-$as_echo "$as_me: WARNING: *** Without Tcl the regression tests cannot be executed ***" >&2;}
-    { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: *** Consider using --with-tcl=... to define location of Tcl ***" >&5
-$as_echo "$as_me: WARNING: *** Consider using --with-tcl=... to define location of Tcl ***" >&2;}
-  else
-    TCL_BIN_DIR=${ac_cv_c_tclconfig}
-    { $as_echo "$as_me:${as_lineno-$LINENO}: result: found $TCL_BIN_DIR/tclConfig.sh" >&5
-$as_echo "found $TCL_BIN_DIR/tclConfig.sh" >&6; }
-
-    { $as_echo "$as_me:${as_lineno-$LINENO}: checking for existence of $TCL_BIN_DIR/tclConfig.sh" >&5
-$as_echo_n "checking for existence of $TCL_BIN_DIR/tclConfig.sh... " >&6; }
-    if test -f "$TCL_BIN_DIR/tclConfig.sh" ; then
-      { $as_echo "$as_me:${as_lineno-$LINENO}: result: loading" >&5
-$as_echo "loading" >&6; }
-      . $TCL_BIN_DIR/tclConfig.sh
-    else
-      { $as_echo "$as_me:${as_lineno-$LINENO}: result: file not found" >&5
-$as_echo "file not found" >&6; }
-    fi
-
-    #
-    # If the TCL_BIN_DIR is the build directory (not the install directory),
-    # then set the common variable name to the value of the build variables.
-    # For example, the variable TCL_LIB_SPEC will be set to the value
-    # of TCL_BUILD_LIB_SPEC. An extension should make use of TCL_LIB_SPEC
-    # instead of TCL_BUILD_LIB_SPEC since it will work with both an
-    # installed and uninstalled version of Tcl.
-    #
-
-    if test -f $TCL_BIN_DIR/Makefile ; then
-      TCL_LIB_SPEC=${TCL_BUILD_LIB_SPEC}
-      TCL_STUB_LIB_SPEC=${TCL_BUILD_STUB_LIB_SPEC}
-      TCL_STUB_LIB_PATH=${TCL_BUILD_STUB_LIB_PATH}
-    fi
-
-    #
-    # eval is required to do the TCL_DBGX substitution
-    #
-
-    eval "TCL_LIB_FILE=\"${TCL_LIB_FILE}\""
-    eval "TCL_LIB_FLAG=\"${TCL_LIB_FLAG}\""
-    eval "TCL_LIB_SPEC=\"${TCL_LIB_SPEC}\""
-
-    eval "TCL_STUB_LIB_FILE=\"${TCL_STUB_LIB_FILE}\""
-    eval "TCL_STUB_LIB_FLAG=\"${TCL_STUB_LIB_FLAG}\""
-    eval "TCL_STUB_LIB_SPEC=\"${TCL_STUB_LIB_SPEC}\""
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-  fi
-fi
-if test "${use_tcl}" = "no" ; then
-  HAVE_TCL=""
-else
-  HAVE_TCL=1
-fi
-
-
 ##########
 # Figure out what C libraries are required to compile programs
 # that use "readline()" library.
@@ -12481,7 +12396,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.46.0, which was
+This file was extended by sqlite $as_me 3.47.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  CONFIG_FILES    = $CONFIG_FILES
@@ -12547,7 +12462,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.46.0
+sqlite config.status 3.47.0
 configured by $0, generated by GNU Autoconf 2.69,
  with options \\"\$ac_cs_config\\"

@@ -116,15 +116,101 @@ AC_CHECK_FUNCS([fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_s
 USE_AMALGAMATION=1

 #########
-# See whether we can run specific tclsh versions known to work well;
-# if not, then we fall back to plain tclsh.
-# TODO: try other versions before falling back?
-# 
-AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.7 tclsh8.6 tclsh8.5 tclsh], none)
+# Figure out all the name of a working tclsh and parameters needed to compile against Tcl.
+# The --with-tcl= and/or --with-tclsh= configuration arguments might be useful for this.
+#
+AC_ARG_WITH(tclsh, AS_HELP_STRING([--with-tclsh=PATHNAME],[full pathname of a tclsh to use]))
+AC_ARG_WITH(tcl, AS_HELP_STRING([--with-tcl=DIR],[directory containing (tclConfig.sh)]))
+AC_ARG_ENABLE(tcl, AS_HELP_STRING([--disable-tcl],[omit building accessory programs that require TCL-dev]),
+      [use_tcl=$enableval],[use_tcl=yes])
+original_use_tcl=${use_tcl}
+if test x"${with_tclsh}" == x -a x"${with_tcl}" == x; then
+  AC_CHECK_PROGS(TCLSH_CMD, [tclsh9.0 tclsh8.6 tclsh],none)
+  with_tclsh=${TCLSH_CMD}
+fi
+if test x"${with_tclsh}" != x -a x"${with_tclsh}" != xnone; then
+  TCLSH_CMD=${with_tclsh}
+  AC_MSG_RESULT([using tclsh at "$TCLSH_CMD"])
+  if test x"${use_tcl}" = "xyes"; then
+    with_tcl=`${with_tclsh} <${srcdir}/tool/find_tclconfig.tcl`
+    if test x"${with_tcl}" != x; then
+      AC_MSG_RESULT([$TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}])
+    else
+      AC_MSG_WARN([$TCLSH_CMD is unable to recommend a tclConfig.sh])
+      use_tcl=no
+    fi
+  fi
+fi
+if test x"${use_tcl}" = "xyes"; then
+  if test x"${with_tcl}" != x; then
+    if test -r ${with_tcl}/tclConfig.sh; then
+      tclconfig="${with_tcl}/tclConfig.sh"
+    else
+      for i in tcl8.6 tcl9.0 lib; do
+        if test -r ${with_tcl}/$i/tclConfig.sh; then
+          tclconfig=${with_tcl}/$i/tclConfig.sh
+          break
+        fi
+      done
+    fi
+    if test ! -r "${tclconfig}"; then
+      AC_MSG_ERROR([no tclConfig.sh file found under ${with_tcl}])
+    fi
+  else
+    # If we have not yet found a tclConfig.sh file, look in $libdir whic is
+    # set automatically by autoconf or by the --prefix command-line option.
+    # See https://sqlite.org/forum/forumpost/e04e693439a22457
+    libdir=${prefix}/lib
+    if test -r ${libdir}/tclConfig.sh; then
+      tclconfig=${libdir}/tclConfig.sh
+    else
+      for i in tcl8.6 tcl9.0 lib; do
+        if test -r ${libdir}/$i/tclConfig.sh; then
+          tclconfig=${libdir}/$i/tclConfig.sh
+          break
+        fi
+      done
+    fi
+    if test ! -r "${tclconfig}"; then
+      AC_MSG_ERROR([cannot find a usable tclConfig.sh file. 
+        Use --with-tcl=DIR to specify a directory where tclConfig.sh can be found.
+        SQLite does not use TCL internally, but TCL is required to build SQLite
+        from canonical sources and TCL is required for testing.])
+    fi
+  fi
+  AC_MSG_RESULT([loading TCL configuration from ${tclconfig}])
+  . ${tclconfig}
+  AC_SUBST(TCL_INCLUDE_SPEC)
+  AC_SUBST(TCL_LIB_SPEC)
+  AC_SUBST(TCL_STUB_LIB_SPEC)
+  # There are lots of other configuration variables that are provided by the
+  # tclConfig.sh file and that could be included here.  But as of right now,
+  # TCL_LIB_SPEC is the only what that the Makefile uses.
+  HAVE_TCL=1
+elif test x"${original_use_tcl}" = "xno"; then
+  AC_MSG_RESULT([unable to run tests because of --disable-tcl])
+  HAVE_TCL=0
+else
+  AC_MSG_RESULT([unable to run tests because no tclConfig.sh file could be located])
+  HAVE_TCL=0
+fi
+AC_SUBST(HAVE_TCL)
+if test x"$TCLSH_CMD" == x; then
+  TCLSH_CMD=${TCL_EXEC_PREFIX}/bin/tclsh${TCL_VERSION}
+  if test ! -x ${TCLSH_CMD}; then
+    TCLSH_CMD_2=${TCL_EXEC_PREFIX}/bin/tclsh
+    if test ! -x ${TCLSH_CMD_2}; then
+      AC_MSG_WARN([cannot find a usable tclsh at either ${TCLSH_CMD} or ${TCLSH_CMD_2}])
+      TCLSH_CMD=none
+    else
+      TCLSH_CMD=${TCLSH_CMD_2}
+    fi
+  fi
+fi
 if test "$TCLSH_CMD" = "none"; then
  # If we can't find a local tclsh, then building the amalgamation will fail.
  # We act as though --disable-amalgamation has been used.
-  echo "Warning: can't find tclsh - defaulting to non-amalgamation build."
+  AC_MSG_WARN([Warning: can't find tclsh - defaulting to non-amalgamation build.])
  USE_AMALGAMATION=0
  TCLSH_CMD="tclsh"
 fi
@@ -132,7 +218,6 @@ AC_SUBST(TCLSH_CMD)

 AC_ARG_VAR([TCLLIBDIR], [Where to install tcl plugin])
 if test "x${TCLLIBDIR+set}" != "xset" ; then
-  TCLLIBDIR='$(libdir)'
  for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` ; do
    if test -d $i ; then
      TCLLIBDIR=$i
@@ -142,6 +227,18 @@ if test "x${TCLLIBDIR+set}" != "xset" ; then
  TCLLIBDIR="${TCLLIBDIR}/sqlite3"
 fi

+#########
+# Set up options for running tests.
+#
+AC_ARG_ENABLE(test-status, AS_HELP_STRING([--enable-test-status],[Full-screen status of tests]),
+      [use_vt100=$enableval],[use_vt100=no])
+if test $use_vt100 != no; then
+  TSTRNNR_OPTS=--status
+else
+  TSTRNNR_OPTS=
+fi
+AC_SUBST(TSTRNNR_OPTS)
+

 #########
 # Set up an appropriate program prefix
@@ -339,190 +436,6 @@ AC_SUBST(SQLITE_OS_UNIX)
 AC_SUBST(SQLITE_OS_WIN)
 AC_SUBST(TARGET_EXEEXT)

-##########
-# Figure out all the parameters needed to compile against Tcl.
-#
-# This code is derived from the SC_PATH_TCLCONFIG and SC_LOAD_TCLCONFIG
-# macros in the in the tcl.m4 file of the standard TCL distribution.
-# Those macros could not be used directly since we have to make some
-# minor changes to accomodate systems that do not have TCL installed.
-#
-AC_ARG_ENABLE(tcl, AS_HELP_STRING([--disable-tcl],[do not build TCL extension]),
-      [use_tcl=$enableval],[use_tcl=yes])
-if test "${use_tcl}" = "yes" ; then
-  AC_ARG_WITH(tcl, AS_HELP_STRING([--with-tcl=DIR],[directory containing tcl configuration (tclConfig.sh)]), with_tclconfig=${withval})
-  AC_MSG_CHECKING([for Tcl configuration])
-  AC_CACHE_VAL(ac_cv_c_tclconfig,[
-    # First check to see if --with-tcl was specified.
-    if test x"${with_tclconfig}" != x ; then
-      if test -f "${with_tclconfig}/tclConfig.sh" ; then
-        ac_cv_c_tclconfig=`(cd ${with_tclconfig}; pwd)`
-      else
-        AC_MSG_ERROR([${with_tclconfig} directory doesn't contain tclConfig.sh])
-      fi
-    fi
-
-    # Start autosearch by asking tclsh
-    if test x"${ac_cv_c_tclconfig}" = x ; then
-      if test x"$cross_compiling" = xno; then
-        for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}`
-        do
-          if test -f "$i/tclConfig.sh" ; then
-            ac_cv_c_tclconfig="$i"
-            break
-          fi
-        done
-      fi
-    fi
-
-    # On ubuntu 14.10, $auto_path on tclsh is not quite correct.
-    # So try again after applying corrections.
-    if test x"${ac_cv_c_tclconfig}" = x ; then
-      if test x"$cross_compiling" = xno; then
-        for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD} | sed 's,/tcltk/tcl,/tcl,g'`
-        do
-          if test -f "$i/tclConfig.sh" ; then
-            ac_cv_c_tclconfig="$i"
-            break
-          fi
-        done
-      fi
-    fi
-
-    # Recent versions of Xcode on Macs hid the tclConfig.sh file
-    # in a strange place.
-    if test x"${ac_cv_c_tclconfig}" = x ; then
-      if test x"$cross_compiling" = xno; then
-        for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib
-        do
-          if test -f "$i/tclConfig.sh" ; then
-            ac_cv_c_tclconfig="$i"
-            break
-          fi
-        done
-      fi
-    fi
-
-    # then check for a private Tcl installation
-    if test x"${ac_cv_c_tclconfig}" = x ; then
-      for i in \
-            ../tcl \
-            `ls -dr ../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \
-            `ls -dr ../tcl[[8-9]].[[0-9]] 2>/dev/null` \
-            `ls -dr ../tcl[[8-9]].[[0-9]]* 2>/dev/null` \
-            ../../tcl \
-            `ls -dr ../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \
-            `ls -dr ../../tcl[[8-9]].[[0-9]] 2>/dev/null` \
-            `ls -dr ../../tcl[[8-9]].[[0-9]]* 2>/dev/null` \
-            ../../../tcl \
-            `ls -dr ../../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \
-            `ls -dr ../../../tcl[[8-9]].[[0-9]] 2>/dev/null` \
-            `ls -dr ../../../tcl[[8-9]].[[0-9]]* 2>/dev/null`
-      do
-        if test -f "$i/unix/tclConfig.sh" ; then
-          ac_cv_c_tclconfig=`(cd $i/unix; pwd)`
-          break
-        fi
-      done
-    fi
-
-    # check in a few common install locations
-    if test x"${ac_cv_c_tclconfig}" = x ; then
-      for i in \
-            `ls -d ${libdir} 2>/dev/null` \
-            `ls -d /usr/local/lib 2>/dev/null` \
-            `ls -d /usr/contrib/lib 2>/dev/null` \
-            `ls -d /usr/lib 2>/dev/null`
-      do
-        if test -f "$i/tclConfig.sh" ; then
-           ac_cv_c_tclconfig=`(cd $i; pwd)`
-           break
-        fi
-      done
-    fi
-
-    # check in a few other private locations
-    if test x"${ac_cv_c_tclconfig}" = x ; then
-      for i in \
-         ${srcdir}/../tcl \
-         `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \
-         `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]] 2>/dev/null` \
-         `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]]* 2>/dev/null`
-      do
-        if test -f "$i/unix/tclConfig.sh" ; then
-          ac_cv_c_tclconfig=`(cd $i/unix; pwd)`
-          break
-        fi
-      done
-    fi
-  ])
-
-  if test x"${ac_cv_c_tclconfig}" = x ; then
-    use_tcl=no
-    AC_MSG_WARN(Can't find Tcl configuration definitions)
-    AC_MSG_WARN(*** Without Tcl the regression tests cannot be executed ***)
-    AC_MSG_WARN(*** Consider using --with-tcl=... to define location of Tcl ***)
-  else
-    TCL_BIN_DIR=${ac_cv_c_tclconfig}
-    AC_MSG_RESULT(found $TCL_BIN_DIR/tclConfig.sh)
-
-    AC_MSG_CHECKING([for existence of $TCL_BIN_DIR/tclConfig.sh])
-    if test -f "$TCL_BIN_DIR/tclConfig.sh" ; then
-      AC_MSG_RESULT([loading])
-      . $TCL_BIN_DIR/tclConfig.sh
-    else
-      AC_MSG_RESULT([file not found])
-    fi
-    
-    #
-    # If the TCL_BIN_DIR is the build directory (not the install directory),
-    # then set the common variable name to the value of the build variables.
-    # For example, the variable TCL_LIB_SPEC will be set to the value
-    # of TCL_BUILD_LIB_SPEC. An extension should make use of TCL_LIB_SPEC
-    # instead of TCL_BUILD_LIB_SPEC since it will work with both an
-    # installed and uninstalled version of Tcl.
-    #
-    
-    if test -f $TCL_BIN_DIR/Makefile ; then
-      TCL_LIB_SPEC=${TCL_BUILD_LIB_SPEC}
-      TCL_STUB_LIB_SPEC=${TCL_BUILD_STUB_LIB_SPEC}
-      TCL_STUB_LIB_PATH=${TCL_BUILD_STUB_LIB_PATH}
-    fi
-    
-    #
-    # eval is required to do the TCL_DBGX substitution
-    #
-    
-    eval "TCL_LIB_FILE=\"${TCL_LIB_FILE}\""
-    eval "TCL_LIB_FLAG=\"${TCL_LIB_FLAG}\""
-    eval "TCL_LIB_SPEC=\"${TCL_LIB_SPEC}\""
-    
-    eval "TCL_STUB_LIB_FILE=\"${TCL_STUB_LIB_FILE}\""
-    eval "TCL_STUB_LIB_FLAG=\"${TCL_STUB_LIB_FLAG}\""
-    eval "TCL_STUB_LIB_SPEC=\"${TCL_STUB_LIB_SPEC}\""
-    
-    AC_SUBST(TCL_VERSION)
-    AC_SUBST(TCL_BIN_DIR)
-    AC_SUBST(TCL_SRC_DIR)
-    AC_SUBST(TCL_INCLUDE_SPEC)
-    
-    AC_SUBST(TCL_LIB_FILE)
-    AC_SUBST(TCL_LIB_FLAG)
-    AC_SUBST(TCL_LIB_SPEC)
-    
-    AC_SUBST(TCL_STUB_LIB_FILE)
-    AC_SUBST(TCL_STUB_LIB_FLAG)
-    AC_SUBST(TCL_STUB_LIB_SPEC)
-    AC_SUBST(TCL_SHLIB_SUFFIX)
-  fi
-fi
-if test "${use_tcl}" = "no" ; then
-  HAVE_TCL=""
-else
-  HAVE_TCL=1
-fi
-AC_SUBST(HAVE_TCL)
-
 ##########
 # Figure out what C libraries are required to compile programs
 # that use "readline()" library.
@@ -0,0 +1,57 @@
+# Notes On Compiling SQLite On All Kinds Of Unix
+
+Here are step-by-step instructions on how to build SQLite from
+canonical source on any modern machine that isn't Windows.  These
+notes are tested (on 2024-10-11) on Ubuntu and on MacOS, but they
+are general and should work on most any modern unix platform.
+
+  1.  Install a C-compiler.  GCC or Clang both work fine.  If you are
+      reading this document, you've probably already done that.
+
+  2.  Install TCL9 development libraries.  In this note, we'll do a
+      private install in the $HOME/local directory, but you can make
+      adjustments to install TCL9 wherever you like.
+      <p>
+      This document assumes you are working with <b>TCL version 9.0</b>.
+      <ol type="a">
+      <li>Get the TCL source archive, perhaps from
+      <https://www.tcl.tk/software/tcltk/download.html>
+      or <https://sqlite.org/tmp/tcl9.0.0.tar.gz>.
+      <li>Untar the source archive.  CD into the "unix/" subfolder
+          of the source tree.
+      <li>Run: `mkdir $HOME/local`
+      <li>Run: `./configure --prefix=$HOME/local`
+      <li>Run: `make install`
+      </ol>
+
+  4.  Download the SQLite source tree and unpack it. CD into the
+      toplevel directory of the source tree.
+
+  5.  Run: `./configure --enable-all --with-tclsh=$HOME/local/bin/tclsh9.0`
+
+      You do not need to use --with-tclsh if the tclsh you want to use is the
+      first one on your PATH.
+
+  6.  Run the "`Makefile`" makefile with an appropriate target.
+      Examples:
+      <ul>
+      <li>  `make sqlite3.c`
+      <li>  `make sqlite3`
+      <li>  `make sqldiff`
+      <li>  `make sqlite3_rsync`
+      <li>  `make tclextension-install`
+      <li>  `make devtest`
+      <li>  `make releasetest`
+      <li>  `make sqlite3_analyzer`
+      </ul>
+
+      It is not required that you run the "tclextension-install" target prior to
+      running tests.  However, the tests will run more smoothly if you do.
+      The version of SQLite used for the TCL extension does *not* need to
+      correspond to the version of SQLite under test.  So you can install the
+      SQLite TCL extension once, and then use it to test many different versions
+      of SQLite.
+
+
+  7.  For a debugging build of the CLI, where the ".treetrace" and ".wheretrace"
+      commands work, add the the --enable-debug argument to configure.
@@ -1,7 +1,7 @@
 # Notes On Compiling SQLite On Windows 11

 Here are step-by-step instructions on how to build SQLite from
-canonical source on a new Windows 11 PC, as of 2023-11-01:
+canonical source on a new Windows 11 PC, as of 2024-10-09:

  1.  Install Microsoft Visual Studio. The free "community edition" 
      will work fine.  Do a standard install for C++ development.
@@ -20,23 +20,23 @@ canonical source on a new Windows 11 PC, as of 2023-11-01:
      install the TCL development libraries in the "`c:\Tcl`" directory.
      Make adjustments
      if you want TCL installed somewhere else.  SQLite needs both the
-      "tclsh.exe" command-line tool as part of the build process, and
-      the "tcl86.lib" library in order to run tests.  You will need
-      TCL version 8.6 or later.
+      "tclsh90.exe" command-line tool as part of the build process, and
+      the "tcl90.lib" and "tclstub.lib" libraries in order to run tests.
+      This document assumes you are working with <b>TCL version 9.0</b>.
+      See versions of this document from prior to 2024-10-10 for
+      instructions on how to build using TCL version 8.6.
      <ol type="a">
      <li>Get the TCL source archive, perhaps from
-      [https://www.tcl.tk/software/tcltk/download.html](https://www.tcl.tk/software/tcltk/download.html).
+      <https://www.tcl.tk/software/tcltk/download.html>
+      or <https://sqlite.org/tmp/tcl9.0.0.tar.gz>.
      <li>Untar or unzip the source archive.  CD into the "win/" subfolder
          of the source tree.
      <li>Run: `nmake /f makefile.vc release`
      <li>Run: `nmake /f makefile.vc INSTALLDIR=c:\Tcl install`
-      <li>CD to `c:\Tcl\lib`.  In that subfolder make a copy of the
-          "`tcl86t.lib`" file to the alternative name "`tcl86.lib`"
-          (omitting the second 't').  Leave the copy in the same directory
-          as the original.
-      <li>CD to `c:\Tcl\bin`.  Make a copy of the "`tclsh86t.exe`"
-          file into "`tclsh.exe`" (without the "86t") in the same directory.
-      <li>Add `c:\Tcl\bin` to your %PATH%.  To do this, go to Settings
+      <li><i>Optional:</i> CD to `c:\Tcl\bin` and make a copy of
+          `tclsh90.exe` over into just `tclsh.exe`.
+      <li><i>Optional:</i>
+          Add `c:\Tcl\bin` to your %PATH%.  To do this, go to Settings
          and search for "path".  Select "edit environment variables for
          your account" and modify your default PATH accordingly.
          You will need to close and reopen your command prompts after
@@ -52,15 +52,41 @@ canonical source on a new Windows 11 PC, as of 2023-11-01:
      <li> `set TCLDIR=c:\Tcl`
      </ul>

+      If you install TCL in the "`c:\Tcl`" directory (as recommended
+      in step 3 above), then this step is optional because
+      "`c:\Tcl`" is the default value for TCLDIR.  You can also skip this
+      step by specifying "`TCLDIR=c:\Tcl`" as an argument to the nmake
+      commands in step 6 below.
+
  6.  Run the "`Makefile.msc`" makefile with an appropriate target.
      Examples:
      <ul>
      <li>  `nmake /f makefile.msc`
      <li>  `nmake /f makefile.msc sqlite3.c`
+      <li>  `nmake /f makefile.msc sqlite3.exe`
+      <li>  `nmake /f makefile.msc sqldiff.exe`
+      <li>  `nmake /f makefile.msc sqlite3_rsync.exe`
+      <li>  `nmake /f makefile.msc tclextension-install`
      <li>  `nmake /f makefile.msc devtest`
      <li>  `nmake /f makefile.msc releasetest`
+      <li>  `nmake /f makefile.msc sqlite3_analyzer.exe`
      </ul>

+      It is not required that you run the "tclextension-install" target prior to
+      running tests.  However, the tests will run more smoothly if you do.
+      The version of SQLite used for the TCL extension does *not* need to
+      correspond to the version of SQLite under test.  So you can install the
+      SQLite TCL extension once, and then use it to test many different versions
+      of SQLite.
+
+
+  7.  For a debugging build of the CLI, where the ".treetrace" and ".wheretrace"
+      commands work, add the DEBUG=3 argument to nmake.  Like this:
+      <ul>
+      <li> `nmake /f makefile.msc DEBUG=3 clean sqlite3.exe`
+      </ul>
+   
+
 ## 32-bit Builds

 Doing a 32-bit build is just like doing a 64-bit build with the
@@ -108,7 +134,7 @@ nmake /f Makefile.msc sqlite3_analyzer.exe
 ~~~~

 And you will end up with a working executable.  However, that executable
-will depend on having the "tcl86.dll" library somewhere on your %PATH%.
+will depend on having the "tcl98.dll" library somewhere on your %PATH%.
 Use the following steps to build an executable that has the TCL library
 statically linked so that it does not depend on separate DLL:

@@ -118,27 +144,26 @@ statically linked so that it does not depend on separate DLL:
  2.  Untar the TCL source tarball into a fresh directory.  CD into
      the "win/" subfolder.

-  3.  Run: `nmake /f makefile.vc OPTS=nothreads,static shell`
-
+  3.  Run: `nmake /f makefile.vc OPTS=static shell`

  4.  CD into the "Release*" subfolder that is created (note the
      wildcard - the full name of the directory might vary).  There
-      you will find the "tcl86s.lib" file.  Copy this file into the
-      same directory that you put the "tcl86.lib" on your initial
+      you will find the "tcl90s.lib" file.  Copy this file into the
+      same directory that you put the "tcl90.lib" on your initial
      installation.  (In this document, that directory is
      "`C:\Tcl32\lib`" for 32-bit builds and
      "`C:\Tcl\lib`" for 64-bit builds.)

  5.  CD into your SQLite source code directory and build the desired
-      utility program, but add the following extra arguments to the
+      utility program, but add the following extra argument to the
      nmake command line:
      <blockquote><pre>
-      CCOPTS="-DSTATIC_BUILD" LIBTCL="tcl86s.lib netapi32.lib user32.lib"
+      STATICALLY_LINK_TCL=1
      </pre></blockquote>
      <p>So, for example, to build a statically linked version of
      sqlite3_analyzer.exe, you might type:
      <blockquote><pre>
-      nmake /f Makefile.msc CCOPTS="-DSTATIC_BUILD" LIBTCL="tcl86s.lib netapi32.lib user32.lib" sqlite3_analyzer.exe
+      nmake /f Makefile.msc STATICALLY_LINK_TCL=1 sqlite3_analyzer.exe
      </pre></blockquote>

  6.  After your executable is built, you can verify that it does not
@@ -53,11 +53,6 @@
 # define CIO_WIN_WC_XLATE 0 /* Not exposing translation routines at all */
 #endif

-#if CIO_WIN_WC_XLATE
-/* Character used to represent a known-incomplete UTF-8 char group (�) */
-static WCHAR cBadGroup = 0xfffd;
-#endif
-
 #if CIO_WIN_WC_XLATE
 static HANDLE handleOfFile(FILE *pf){
  int fileDesc = _fileno(pf);
@@ -600,6 +595,93 @@ oPutbUtf8(const char *cBuf, int nAccept){
 # endif
 }

+/*
+** Flush the given output stream. Return non-zero for success, else 0.
+*/
+#if !defined(SQLITE_CIO_NO_FLUSH) && !defined(SQLITE_CIO_NO_SETMODE)
+SQLITE_INTERNAL_LINKAGE int
+fFlushBuffer(FILE *pfOut){
+# if CIO_WIN_WC_XLATE && !defined(SHELL_OMIT_FIO_DUPE)
+  return FlushFileBuffers(handleOfFile(pfOut))? 1 : 0;
+# else
+  return fflush(pfOut);
+# endif
+}
+#endif
+
+#if CIO_WIN_WC_XLATE \
+   && !defined(SHELL_OMIT_FIO_DUPE) \
+   && defined(SQLITE_USE_ONLY_WIN32)
+static struct FileAltIds {
+  int fd;
+  HANDLE fh;
+} altIdsOfFile(FILE *pf){
+  struct FileAltIds rv = { _fileno(pf) };
+  union { intptr_t osfh; HANDLE fh; } fid = {
+    (rv.fd>=0)? _get_osfhandle(rv.fd) : (intptr_t)INVALID_HANDLE_VALUE
+  };
+  rv.fh = fid.fh;
+  return rv;
+}
+
+SQLITE_INTERNAL_LINKAGE size_t
+cfWrite(const void *buf, size_t osz, size_t ocnt, FILE *pf){
+  size_t rv = 0;
+  struct FileAltIds fai = altIdsOfFile(pf);
+  int fmode = _setmode(fai.fd, _O_BINARY);
+  _setmode(fai.fd, fmode);
+  while( rv < ocnt ){
+    size_t nbo = osz;
+    while( nbo > 0 ){
+      DWORD dwno = (nbo>(1L<<24))? 1L<<24 : (DWORD)nbo;
+      BOOL wrc = TRUE;
+      BOOL genCR = (fmode & _O_TEXT)!=0;
+      if( genCR ){
+        const char *pnl = (const char*)memchr(buf, '\n', nbo);
+        if( pnl ) nbo = pnl - (const char*)buf;
+        else genCR = 0;
+      }
+      if( dwno>0 ) wrc = WriteFile(fai.fh, buf, dwno, 0,0);
+      if( genCR && wrc ){
+        wrc = WriteFile(fai.fh, "\r\n", 2, 0,0);
+        ++dwno; /* Skip over the LF */
+      }
+      if( !wrc ) return rv;
+      buf = (const char*)buf + dwno;
+      nbo += dwno;
+    }
+    ++rv;
+  }
+  return rv;
+}
+
+/* An fgets() equivalent, using Win32 file API for actual input.
+** Input ends when given buffer is filled or a newline is read.
+** If the FILE object is in text mode, swallows 0x0D. (ASCII CR)
+*/
+SQLITE_INTERNAL_LINKAGE char *
+cfGets(char *cBuf, int n, FILE *pf){
+  int nci = 0;
+  struct FileAltIds fai = altIdsOfFile(pf);
+  int fmode = _setmode(fai.fd, _O_BINARY);
+  BOOL eatCR = (fmode & _O_TEXT)!=0;
+  _setmode(fai.fd, fmode);
+  while( nci < n-1 ){
+    DWORD nr;
+    if( !ReadFile(fai.fh, cBuf+nci, 1, &nr, 0) || nr==0 ) break;
+    if( nr>0 && (!eatCR || cBuf[nci]!='\r') ){
+      nci += nr;
+      if( cBuf[nci-nr]=='\n' ) break;
+    }
+  }
+  if( nci < n ) cBuf[nci] = 0;
+  return (nci>0)? cBuf : 0;
+}
+# else
+#  define cfWrite(b,os,no,f) fwrite(b,os,no,f)
+#  define cfGets(b,n,f) fgets(b,n,f)
+# endif
+
 # ifdef CONSIO_EPUTB
 SQLITE_INTERNAL_LINKAGE int
 ePutbUtf8(const char *cBuf, int nAccept){
@@ -611,7 +693,7 @@ ePutbUtf8(const char *cBuf, int nAccept){
    return conZstrEmit(ppst, cBuf, nAccept);
  }else {
 #  endif
-    return (int)fwrite(cBuf, 1, nAccept, pfErr);
+    return (int)cfWrite(cBuf, 1, nAccept, pfErr);
 #  if CIO_WIN_WC_XLATE
  }
 #  endif
@@ -677,7 +759,7 @@ SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){
 #  endif
  }else{
 # endif
-    return fgets(cBuf, ncMax, pfIn);
+    return cfGets(cBuf, ncMax, pfIn);
 # if CIO_WIN_WC_XLATE
  }
 # endif
@@ -176,12 +176,19 @@ SQLITE_INTERNAL_LINKAGE int
 ePutbUtf8(const char *cBuf, int nAccept);
 #endif

+/*
+** Flush the given output stream. Return non-zero for success, else 0.
+*/
+#if !defined(SQLITE_CIO_NO_FLUSH) && !defined(SQLITE_CIO_NO_SETMODE)
+SQLITE_INTERNAL_LINKAGE int
+fFlushBuffer(FILE *pfOut);
+#endif
+
 /*
 ** Collect input like fgets(...) with special provisions for input
-** from the console on platforms that require same. Defers to the
-** C library fgets() when input is not from the console. Newline
-** translation may be done as set by set{Binary,Text}Mode(). As a
-** convenience, pfIn==NULL is treated as stdin.
+** from the console on such platforms as require same. Newline
+** translation may be done as set by set{Binary,Text}Mode().
+** As a convenience, pfIn==NULL is treated as stdin.
 */
 SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn);
 /* Like fGetsUtf8 except stream is always the designated input. */
@@ -402,6 +402,19 @@ do_setup_rec_test $tn.19.0 {
  SCAN t1 USING COVERING INDEX t1_idx_01a7214e
 }

+ifcapable fts5 {
+  do_setup_rec_test $tn.20.0 {
+    CREATE VIRTUAL TABLE ft USING fts5(a);
+    CREATE TABLE t1(x, y);
+  } {
+    SELECT * FROM ft, t1 WHERE a=x
+  } {
+    CREATE INDEX t1_idx_00000078 ON t1(x);
+    SCAN ft VIRTUAL TABLE INDEX 0:
+    SEARCH t1 USING INDEX t1_idx_00000078 (x=?)
+  }
+}
+
 }

 proc do_candidates_test {tn sql res} {
@@ -428,6 +441,8 @@ do_execsql_test 5.0 {

  WITH s(i) AS ( VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100)
  INSERT INTO t2 SELECT (i-1)/20, (i-1)/5 FROM s;
+
+  CREATE INDEX i1 ON t1( lower(a) );
 }
 do_candidates_test 5.1 {
  SELECT * FROM t1,t2 WHERE (b=? OR a=?) AND (c=? OR d=?)
@@ -457,6 +472,7 @@ do_execsql_test 5.3 {
  ANALYZE;
  SELECT * FROM sqlite_stat1 ORDER BY 1, 2;
 } {
+  t1 i1 {100 50} 
  t1 t1_idx_00000061 {100 50} 
  t1 t1_idx_00000062 {100 20}
  t1 t1_idx_000123a7 {100 50 17}
@@ -465,6 +481,13 @@ do_execsql_test 5.3 {
  t2 t2_idx_0001295b {100 20 5}
 }

+do_catchsql_test 5.4 {
+  SELECT sqlite_expert_rem(123, 123);
+} {1 {no such function: sqlite_expert_rem}}
+do_catchsql_test 5.5 {
+  SELECT sqlite_expert_sample();
+} {1 {no such function: sqlite_expert_sample}}
+
 if 0 {
 do_test expert1-6.0 {
  catchcmd :memory: {
@@ -484,4 +507,103 @@ USE TEMP B-TREE FOR ORDER BY
 }}
 }

+do_execsql_test 6.0 {
+  CREATE TABLE x1(a, b, c, d);
+  CREATE INDEX x1ab ON x1(a, lower(b));
+  CREATE INDEX x1dcba ON x1(d, b+c, a);
+}
+
+do_candidates_test 6.1 {
+  SELECT * FROM x1 WHERE b=? ORDER BY a;
+} {
+  CREATE INDEX x1_idx_0001267f ON x1(b, a); 
+  CREATE INDEX x1_idx_00000062 ON x1(b);
+}
+
+#-------------------------------------------------------------------------
+ifcapable fts5 {
+  reset_db
+  do_execsql_test 7.0 {
+    CREATE VIRTUAL TABLE ft USING fts5(a);
+    CREATE TABLE t1(x, y);
+  }
+  
+  do_candidates_test 7.1 {
+    SELECT * FROM ft, t1 WHERE a=x
+  } {
+    CREATE INDEX t1_idx_00000078 ON t1(x);
+  }
+  
+  register_tcl_module db
+  proc vtab_command {method args} {
+    global G
+  
+    switch -- $method {
+      xConnect {
+        return "CREATE TABLE t1(a, b, c);"
+      }
+  
+      xBestIndex {
+        return [list]
+      }
+  
+      xFilter {
+        return [list sql "SELECT rowid, * FROM t0"]
+      }
+    }
+  
+    return {}
+  }
+  
+  do_execsql_test 7.2 {
+    CREATE TABLE t0(a, b, c);
+    INSERT INTO t0 VALUES(1, 2, 3), (11, 22, 33);
+    CREATE VIRTUAL TABLE t2 USING tcl(vtab_command);
+  }
+  
+  do_execsql_test 7.3 {
+    SELECT * FROM t2
+  } {
+    1 2 3
+    11 22 33
+  }
+  
+  do_candidates_test 7.4 {
+    SELECT * FROM ft, t1 WHERE a=x
+  } {
+    CREATE INDEX t1_idx_00000078 ON t1(x);
+  }
+  
+  do_test 7.5 {
+    set expert [sqlite3_expert_new db]
+    list [catch { $expert sql "SELECT * FROM ft, t2 WHERE b=1" } msg] $msg
+  } {1 {no such table: t2}}
+  $expert destroy
+
+  reset_db
+  do_execsql_test 7.6 {
+    BEGIN TRANSACTION;
+      CREATE TABLE IF NOT EXISTS 'bfts_idx_data'(id INTEGER PRIMARY KEY, block BLOB);
+      CREATE TABLE IF NOT EXISTS 'fts_idx_data'(id INTEGER PRIMARY KEY, block BLOB);
+      INSERT INTO fts_idx_data VALUES(1,X'');
+      INSERT INTO fts_idx_data VALUES(10,X'00000000ff000001000000');
+      CREATE TABLE IF NOT EXISTS 'fts_idx_idx'(segid, term, pgno, PRIMARY KEY(segid, term)) WITHOUT ROWID;
+      CREATE TABLE IF NOT EXISTS 'fts_idx_docsize'(id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER);
+      CREATE TABLE IF NOT EXISTS 'fts_idx_config'(k PRIMARY KEY, v) WITHOUT ROWID;
+      INSERT INTO fts_idx_config VALUES('version',4);
+      PRAGMA writable_schema=ON;
+      INSERT INTO sqlite_schema(type,name,tbl_name,rootpage,sql)VALUES('table','fts_idx','fts_idx',0,'CREATE VIRTUAL TABLE fts_idx USING fts5(Title, Description, Channel, Tags, content='''', contentless_delete=1)');
+      
+      CREATE TABLE f(x BLOB, y);
+    COMMIT;
+    PRAGMA writable_schema = RESET;
+  }
+
+  do_candidates_test 7.4 {
+    SELECT * FROM fts_idx, f WHERE x = fts_idx.Channel
+  } {
+    CREATE INDEX f_idx_00000078 ON f(x);
+  }
+}
+
 finish_test
@@ -626,7 +626,7 @@ static int expertFilter(
  pCsr->pData = 0;
  if( rc==SQLITE_OK ){
    rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg,
-        "SELECT * FROM main.%Q WHERE sample()", pVtab->pTab->zName
+        "SELECT * FROM main.%Q WHERE sqlite_expert_sample()", pVtab->pTab->zName
    );
  }

@@ -1392,6 +1392,66 @@ static int idxProcessTriggers(sqlite3expert *p, char **pzErr){
  return rc;
 }

+/*
+** This function tests if the schema of the main database of database handle
+** db contains an object named zTab. Assuming no error occurs, output parameter
+** (*pbContains) is set to true if zTab exists, or false if it does not.
+**
+** Or, if an error occurs, an SQLite error code is returned. The final value
+** of (*pbContains) is undefined in this case.
+*/
+static int expertDbContainsObject(
+  sqlite3 *db, 
+  const char *zTab, 
+  int *pbContains                 /* OUT: True if object exists */
+){
+  const char *zSql = "SELECT 1 FROM sqlite_schema WHERE name = ?";
+  sqlite3_stmt *pSql = 0;
+  int rc = SQLITE_OK;
+  int ret = 0;
+
+  rc = sqlite3_prepare_v2(db, zSql, -1, &pSql, 0);
+  if( rc==SQLITE_OK ){
+    sqlite3_bind_text(pSql, 1, zTab, -1, SQLITE_STATIC);
+    if( SQLITE_ROW==sqlite3_step(pSql) ){
+      ret = 1;
+    }
+    rc = sqlite3_finalize(pSql);
+  }
+
+  *pbContains = ret;
+  return rc;
+}
+
+/*
+** Execute SQL command zSql using database handle db. If no error occurs,
+** set (*pzErr) to NULL and return SQLITE_OK. 
+**
+** If an error does occur, return an SQLite error code and set (*pzErr) to
+** point to a buffer containing an English language error message. Except,
+** if the error message begins with "no such module:", then ignore the
+** error and return as if the SQL statement had succeeded.
+**
+** This is used to copy as much of the database schema as possible while 
+** ignoring any errors related to missing virtual table modules.
+*/
+static int expertSchemaSql(sqlite3 *db, const char *zSql, char **pzErr){
+  int rc = SQLITE_OK;
+  char *zErr = 0;
+
+  rc = sqlite3_exec(db, zSql, 0, 0, &zErr);
+  if( rc!=SQLITE_OK && zErr ){
+    int nErr = STRLEN(zErr);
+    if( nErr>=15 && memcmp(zErr, "no such module:", 15)==0 ){
+      sqlite3_free(zErr);
+      rc = SQLITE_OK;
+      zErr = 0;
+    }
+  }
+
+  *pzErr = zErr;
+  return rc;
+}

 static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
  int rc = idxRegisterVtab(p);
@@ -1403,22 +1463,31 @@ static int idxCreateVtabSchema(sqlite3expert *p, char **pzErrmsg){
  **   2) Create the equivalent virtual table in dbv.
  */
  rc = idxPrepareStmt(p->db, &pSchema, pzErrmsg,
-      "SELECT type, name, sql, 1 FROM sqlite_schema "
-      "WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%%' "
+      "SELECT type, name, sql, 1, "
+      "       substr(sql,1,14)=='create virtual' COLLATE nocase "
+      "FROM sqlite_schema "
+      "WHERE type IN ('table','view') AND "
+      "      substr(name,1,7)!='sqlite_' COLLATE nocase "
      " UNION ALL "
-      "SELECT type, name, sql, 2 FROM sqlite_schema "
+      "SELECT type, name, sql, 2, 0 FROM sqlite_schema "
      "WHERE type = 'trigger'"
      "  AND tbl_name IN(SELECT name FROM sqlite_schema WHERE type = 'view') "
-      "ORDER BY 4, 1"
+      "ORDER BY 4, 5 DESC, 1"
  );
  while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSchema) ){
    const char *zType = (const char*)sqlite3_column_text(pSchema, 0);
    const char *zName = (const char*)sqlite3_column_text(pSchema, 1);
    const char *zSql = (const char*)sqlite3_column_text(pSchema, 2);
+    int bVirtual = sqlite3_column_int(pSchema, 4);
+    int bExists = 0;

    if( zType==0 || zName==0 ) continue;
-    if( zType[0]=='v' || zType[1]=='r' ){
-      if( zSql ) rc = sqlite3_exec(p->dbv, zSql, 0, 0, pzErrmsg);
+    rc = expertDbContainsObject(p->dbv, zName, &bExists);
+    if( rc || bExists ) continue;
+
+    if( zType[0]=='v' || zType[1]=='r' || bVirtual ){
+      /* A view. Or a trigger on a view. */
+      if( zSql ) rc = expertSchemaSql(p->dbv, zSql, pzErrmsg);
    }else{
      IdxTable *pTab;
      rc = idxGetTableInfo(p->db, zName, &pTab, pzErrmsg);
@@ -1500,7 +1569,7 @@ struct IdxRemCtx {
 };

 /*
-** Implementation of scalar function rem().
+** Implementation of scalar function sqlite_expert_rem().
 */
 static void idxRemFunc(
  sqlite3_context *pCtx,
@@ -1513,7 +1582,7 @@ static void idxRemFunc(
  assert( argc==2 );

  iSlot = sqlite3_value_int(argv[0]);
-  assert( iSlot<=p->nSlot );
+  assert( iSlot<p->nSlot );
  pSlot = &p->aSlot[iSlot];

  switch( pSlot->eType ){
@@ -1623,8 +1692,15 @@ static int idxPopulateOneStat1(
    const char *zComma = zCols==0 ? "" : ", ";
    const char *zName = (const char*)sqlite3_column_text(pIndexXInfo, 0);
    const char *zColl = (const char*)sqlite3_column_text(pIndexXInfo, 1);
+    if( zName==0 ){
+      /* This index contains an expression. Ignore it. */
+      sqlite3_free(zCols);
+      sqlite3_free(zOrder);
+      return sqlite3_reset(pIndexXInfo);
+    }
    zCols = idxAppendText(&rc, zCols, 
-        "%sx.%Q IS rem(%d, x.%Q) COLLATE %s", zComma, zName, nCol, zName, zColl
+        "%sx.%Q IS sqlite_expert_rem(%d, x.%Q) COLLATE %s", 
+        zComma, zName, nCol, zName, zColl
    );
    zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol);
  }
@@ -1757,13 +1833,13 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){

  if( rc==SQLITE_OK ){
    sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv);
-    rc = sqlite3_create_function(
-        dbrem, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
+    rc = sqlite3_create_function(dbrem, "sqlite_expert_rem", 
+        2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
    );
  }
  if( rc==SQLITE_OK ){
-    rc = sqlite3_create_function(
-        p->db, "sample", 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0
+    rc = sqlite3_create_function(p->db, "sqlite_expert_sample", 
+        0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0
    );
  }

@@ -1815,6 +1891,9 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
    rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_schema", 0, 0, 0);
  }

+  sqlite3_create_function(p->db, "sqlite_expert_rem", 2, SQLITE_UTF8, 0,0,0,0);
+  sqlite3_create_function(p->db, "sqlite_expert_sample", 0,SQLITE_UTF8,0,0,0,0);
+
  sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
  return rc;
 }
@@ -1947,12 +2026,18 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
  if( rc==SQLITE_OK ){
    sqlite3_stmt *pSql = 0;
    rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg, 
-        "SELECT sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%%'"
-        " AND sql NOT LIKE 'CREATE VIRTUAL %%' ORDER BY rowid"
+        "SELECT sql, name, substr(sql,1,14)=='create virtual' COLLATE nocase"
+        " FROM sqlite_schema WHERE substr(name,1,7)!='sqlite_' COLLATE nocase"
+        " ORDER BY 3 DESC, rowid"
    );
    while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
      const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
-      if( zSql ) rc = sqlite3_exec(pNew->dbm, zSql, 0, 0, pzErrmsg);
+      const char *zName = (const char*)sqlite3_column_text(pSql, 1);
+      int bExists = 0;
+      rc = expertDbContainsObject(pNew->dbm, zName, &bExists);
+      if( rc==SQLITE_OK && zSql && bExists==0 ){
+        rc = expertSchemaSql(pNew->dbm, zSql, pzErrmsg);
+      }
    }
    idxFinalize(&rc, pSql);
  }
@@ -16,15 +16,7 @@
 #include "sqlite3expert.h"
 #include <assert.h>
 #include <string.h>
-
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#  ifndef SQLITE_TCLAPI
-#    define SQLITE_TCLAPI
-#  endif
-#endif
+#include "tclsqlite.h"

 #ifndef SQLITE_OMIT_VIRTUALTABLE

@@ -398,6 +398,7 @@ static int fts3SnippetNextCandidate(SnippetIter *pIter){
      return 1;
    }

+    assert( pIter->nSnippet>=0 );
    pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1;
    for(i=0; i<pIter->nPhrase; i++){
      SnippetPhrase *pPhrase = &pIter->aPhrase[i];
@@ -78,6 +78,8 @@ static int fts3termConnectMethod(
    iIndex = atoi(argv[4]);
    argc--;
  }
+  
+  *ppVtab = 0;

  /* The user should specify a single argument - the name of an fts3 table. */
  if( argc!=4 ){
@@ -95,12 +97,17 @@ static int fts3termConnectMethod(
  rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA);
  if( rc!=SQLITE_OK ) return rc;

-  nByte = sizeof(Fts3termTable) + sizeof(Fts3Table) + nDb + nFts3 + 2;
-  p = (Fts3termTable *)sqlite3_malloc64(nByte);
+  nByte = sizeof(Fts3termTable);
+  p = (Fts3termTable *)sqlite3Fts3MallocZero(nByte);
  if( !p ) return SQLITE_NOMEM;
-  memset(p, 0, (size_t)nByte);

-  p->pFts3Tab = (Fts3Table *)&p[1];
+  p->pFts3Tab = (Fts3Table*)sqlite3Fts3MallocZero(
+      sizeof(Fts3Table) + nDb + nFts3 + 2
+  );
+  if( p->pFts3Tab==0 ){
+    sqlite3_free(p);
+    return SQLITE_NOMEM;
+  }
  p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
  p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
  p->pFts3Tab->db = db;
@@ -130,6 +137,7 @@ static int fts3termDisconnectMethod(sqlite3_vtab *pVtab){
    sqlite3_finalize(pFts3->aStmt[i]);
  }
  sqlite3_free(pFts3->zSegmentsTbl);
+  sqlite3_free(pFts3);
  sqlite3_free(p);
  return SQLITE_OK;
 }
@@ -18,14 +18,7 @@
 ** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly.
 */

-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#  ifndef SQLITE_TCLAPI
-#    define SQLITE_TCLAPI
-#  endif
-#endif
+#include "tclsqlite.h"
 #include <string.h>
 #include <assert.h>

@@ -167,7 +160,8 @@ static int SQLITE_TCLAPI fts3_near_match_cmd(
  Tcl_Obj *pPhrasecount = 0;
  
  Tcl_Obj **apExprToken;
-  int nExprToken;
+  Tcl_Size nExprToken;
+  Tcl_Size nn;

  UNUSED_PARAMETER(clientData);

@@ -201,23 +195,25 @@ static int SQLITE_TCLAPI fts3_near_match_cmd(
    }
  }

-  rc = Tcl_ListObjGetElements(interp, objv[1], &doc.nToken, &apDocToken);
+  rc = Tcl_ListObjGetElements(interp, objv[1], &nn, &apDocToken);
+  doc.nToken = (int)nn;
  if( rc!=TCL_OK ) goto near_match_out;
  doc.aToken = (NearToken *)ckalloc(doc.nToken*sizeof(NearToken));
  for(ii=0; ii<doc.nToken; ii++){
-    doc.aToken[ii].z = Tcl_GetStringFromObj(apDocToken[ii], &doc.aToken[ii].n);
+    doc.aToken[ii].z = Tcl_GetStringFromObj(apDocToken[ii], &nn);
+    doc.aToken[ii].n = (int)nn;
  }

  rc = Tcl_ListObjGetElements(interp, objv[2], &nExprToken, &apExprToken);
  if( rc!=TCL_OK ) goto near_match_out;

-  nPhrase = (nExprToken + 1) / 2;
+  nPhrase = (int)(nExprToken + 1) / 2;
  aPhrase = (NearPhrase *)ckalloc(nPhrase * sizeof(NearPhrase));
  memset(aPhrase, 0, nPhrase * sizeof(NearPhrase));
  for(ii=0; ii<nPhrase; ii++){
    Tcl_Obj *pPhrase = apExprToken[ii*2];
    Tcl_Obj **apToken;
-    int nToken;
+    Tcl_Size nToken;
    int jj;

    rc = Tcl_ListObjGetElements(interp, pPhrase, &nToken, &apToken);
@@ -227,11 +223,12 @@ static int SQLITE_TCLAPI fts3_near_match_cmd(
      rc = TCL_ERROR;
      goto near_match_out;
    }
-    for(jj=0; jj<nToken; jj++){
+    for(jj=0; jj<(int)nToken; jj++){
      NearToken *pT = &aPhrase[ii].aToken[jj];
-      pT->z = Tcl_GetStringFromObj(apToken[jj], &pT->n);
+      pT->z = Tcl_GetStringFromObj(apToken[jj], &nn);
+      pT->n = (int)nn;
    }
-    aPhrase[ii].nToken = nToken;
+    aPhrase[ii].nToken = (int)nToken;
  }
  for(ii=1; ii<nPhrase; ii++){
    Tcl_Obj *pNear = apExprToken[2*ii-1];
@@ -226,11 +226,7 @@ int sqlite3Fts3InitTokenizer(

 #ifdef SQLITE_TEST

-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#endif
+#include "tclsqlite.h"
 #include <string.h>

 /*
@@ -628,6 +628,9 @@ proc print_categories {lMap} {
        $caseP
        $caseS
        $caseZ
+
+        default:
+          return 1;
      }
      return 0;
    }
@@ -82,7 +82,7 @@ proc get_struct_docs {data names} {
        set current_doc ""
      }
      set subject n/a
-      regexp {^ *([[:alpha:]]*)} $line -> subject
+      regexp {^ *([[:alnum:]_]*)} $line -> subject
      if {[lsearch $names $subject]>=0} {
        set current_header $subject
      } else {
@@ -108,8 +108,11 @@ proc get_tokenizer_docs {data} {
      append res "<dt><b>$line</b></dt><dd><p style=margin-top:0>\n"
      continue
    }
+    if {[regexp {FTS5_TOKENIZER} $line]} {
+      set line </dl><p>
+    }
    if {[regexp {SYNONYM SUPPORT} $line]} {
-      set line "</dl><h3>Synonym Support</h3>"
+      set line "<h3>Synonym Support</h3>"
    }
    if {[string trim $line] == ""} {
      append res "<p>\n"
@@ -238,6 +238,10 @@ struct Fts5PhraseIter {
 **   (i.e. if it is a contentless table), then this API always iterates
 **   through an empty set (all calls to xPhraseFirst() set iCol to -1).
 **
+**   In all cases, matches are visited in (column ASC, offset ASC) order.
+**   i.e. all those in column 0, sorted by offset, followed by those in 
+**   column 1, etc.
+**
 ** xPhraseNext()
 **   See xPhraseFirst above.
 **
@@ -304,9 +308,32 @@ struct Fts5PhraseIter {
 **
 **   This API can be quite slow if used with an FTS5 table created with the
 **   "detail=none" or "detail=column" option.
+**
+** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale)
+**   If parameter iCol is less than zero, or greater than or equal to the
+**   number of columns in the table, SQLITE_RANGE is returned.
+**
+**   Otherwise, this function attempts to retrieve the locale associated
+**   with column iCol of the current row. Usually, there is no associated
+**   locale, and output parameters (*pzLocale) and (*pnLocale) are set
+**   to NULL and 0, respectively. However, if the fts5_locale() function
+**   was used to associate a locale with the value when it was inserted
+**   into the fts5 table, then (*pzLocale) is set to point to a nul-terminated
+**   buffer containing the name of the locale in utf-8 encoding. (*pnLocale) 
+**   is set to the size in bytes of the buffer, not including the 
+**   nul-terminator.
+**
+**   If successful, SQLITE_OK is returned. Or, if an error occurs, an
+**   SQLite error code is returned. The final value of the output parameters
+**   is undefined in this case.
+**
+** xTokenize_v2:
+**   Tokenize text using the tokenizer belonging to the FTS5 table. This
+**   API is the same as the xTokenize() API, except that it allows a tokenizer
+**   locale to be specified.
 */
 struct Fts5ExtensionApi {
-  int iVersion;                   /* Currently always set to 3 */
+  int iVersion;                   /* Currently always set to 4 */

  void *(*xUserData)(Fts5Context*);

@@ -348,6 +375,15 @@ struct Fts5ExtensionApi {
      const char **ppToken, int *pnToken
  );
  int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*);
+
+  /* Below this point are iVersion>=4 only */
+  int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn);
+  int (*xTokenize_v2)(Fts5Context*,
+    const char *pText, int nText,      /* Text to tokenize */
+    const char *pLocale, int nLocale,  /* Locale to pass to tokenizer */
+    void *pCtx,                        /* Context passed to xToken() */
+    int (*xToken)(void*, int, const char*, int, int, int)       /* Callback */
+  );
 };

 /* 
@@ -368,7 +404,7 @@ struct Fts5ExtensionApi {
 **   A tokenizer instance is required to actually tokenize text.
 **
 **   The first argument passed to this function is a copy of the (void*)
-**   pointer provided by the application when the fts5_tokenizer object
+**   pointer provided by the application when the fts5_tokenizer_v2 object
 **   was registered with FTS5 (the third argument to xCreateTokenizer()). 
 **   The second and third arguments are an array of nul-terminated strings
 **   containing the tokenizer arguments, if any, specified following the
@@ -392,7 +428,7 @@ struct Fts5ExtensionApi {
 **   argument passed to this function is a pointer to an Fts5Tokenizer object
 **   returned by an earlier call to xCreate().
 **
-**   The second argument indicates the reason that FTS5 is requesting
+**   The third argument indicates the reason that FTS5 is requesting
 **   tokenization of the supplied text. This is always one of the following
 **   four values:
 **
@@ -416,6 +452,13 @@ struct Fts5ExtensionApi {
 **            on a columnsize=0 database.  
 **   </ul>
 **
+**   The sixth and seventh arguments passed to xTokenize() - pLocale and
+**   nLocale - are a pointer to a buffer containing the locale to use for
+**   tokenization (e.g. "en_US") and its size in bytes, respectively. The
+**   pLocale buffer is not nul-terminated. pLocale may be passed NULL (in
+**   which case nLocale is always 0) to indicate that the tokenizer should
+**   use its default locale.
+**
 **   For each token in the input string, the supplied callback xToken() must
 **   be invoked. The first argument to it should be a copy of the pointer
 **   passed as the second argument to xTokenize(). The third and fourth
@@ -439,6 +482,30 @@ struct Fts5ExtensionApi {
 **   may abandon the tokenization and return any error code other than
 **   SQLITE_OK or SQLITE_DONE.
 **
+**   If the tokenizer is registered using an fts5_tokenizer_v2 object,
+**   then the xTokenize() method has two additional arguments - pLocale
+**   and nLocale. These specify the locale that the tokenizer should use
+**   for the current request. If pLocale and nLocale are both 0, then the
+**   tokenizer should use its default locale. Otherwise, pLocale points to
+**   an nLocale byte buffer containing the name of the locale to use as utf-8 
+**   text. pLocale is not nul-terminated.
+**
+** FTS5_TOKENIZER
+**
+** There is also an fts5_tokenizer object. This is an older, deprecated,
+** version of fts5_tokenizer_v2. It is similar except that:
+**
+**  <ul>
+**    <li> There is no "iVersion" field, and
+**    <li> The xTokenize() method does not take a locale argument.
+**  </ul>
+**
+** Legacy fts5_tokenizer tokenizers must be registered using the
+** legacy xCreateTokenizer() function, instead of xCreateTokenizer_v2().
+**
+** Tokenizer implementations registered using either API may be retrieved
+** using both xFindTokenizer() and xFindTokenizer_v2().
+**
 ** SYNONYM SUPPORT
 **
 **   Custom tokenizers may also support synonyms. Consider a case in which a
@@ -547,6 +614,33 @@ struct Fts5ExtensionApi {
 **   inefficient.
 */
 typedef struct Fts5Tokenizer Fts5Tokenizer;
+typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2;
+struct fts5_tokenizer_v2 {
+  int iVersion;             /* Currently always 2 */
+
+  int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
+  void (*xDelete)(Fts5Tokenizer*);
+  int (*xTokenize)(Fts5Tokenizer*, 
+      void *pCtx,
+      int flags,            /* Mask of FTS5_TOKENIZE_* flags */
+      const char *pText, int nText, 
+      const char *pLocale, int nLocale,
+      int (*xToken)(
+        void *pCtx,         /* Copy of 2nd argument to xTokenize() */
+        int tflags,         /* Mask of FTS5_TOKEN_* flags */
+        const char *pToken, /* Pointer to buffer containing token */
+        int nToken,         /* Size of token in bytes */
+        int iStart,         /* Byte offset of token within input text */
+        int iEnd            /* Byte offset of end of token within input text */
+      )
+  );
+};
+
+/*
+** New code should use the fts5_tokenizer_v2 type to define tokenizer
+** implementations. The following type is included for legacy applications
+** that still use it.
+*/
 typedef struct fts5_tokenizer fts5_tokenizer;
 struct fts5_tokenizer {
  int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
@@ -566,6 +660,7 @@ struct fts5_tokenizer {
  );
 };

+
 /* Flags that may be passed as the third argument to xTokenize() */
 #define FTS5_TOKENIZE_QUERY     0x0001
 #define FTS5_TOKENIZE_PREFIX    0x0002
@@ -585,7 +680,7 @@ struct fts5_tokenizer {
 */
 typedef struct fts5_api fts5_api;
 struct fts5_api {
-  int iVersion;                   /* Currently always set to 2 */
+  int iVersion;                   /* Currently always set to 3 */

  /* Create a new tokenizer */
  int (*xCreateTokenizer)(
@@ -612,6 +707,25 @@ struct fts5_api {
    fts5_extension_function xFunction,
    void (*xDestroy)(void*)
  );
+
+  /* APIs below this point are only available if iVersion>=3 */
+
+  /* Create a new tokenizer */
+  int (*xCreateTokenizer_v2)(
+    fts5_api *pApi,
+    const char *zName,
+    void *pUserData,
+    fts5_tokenizer_v2 *pTokenizer,
+    void (*xDestroy)(void*)
+  );
+
+  /* Find an existing tokenizer */
+  int (*xFindTokenizer_v2)(
+    fts5_api *pApi,
+    const char *zName,
+    void **ppUserData,
+    fts5_tokenizer_v2 **ppTokenizer
+  );
 };

 /*
@@ -59,6 +59,22 @@ typedef sqlite3_uint64 u64;
 # define LARGEST_INT64  (0xffffffff|(((i64)0x7fffffff)<<32))
 # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)

+/* The uptr type is an unsigned integer large enough to hold a pointer
+*/
+#if defined(HAVE_STDINT_H)
+  typedef uintptr_t uptr;
+#elif SQLITE_PTRSIZE==4
+  typedef u32 uptr;
+#else
+  typedef u64 uptr;
+#endif
+
+#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC
+# define EIGHT_BYTE_ALIGNMENT(X)   ((((uptr)(X) - (uptr)0)&3)==0)
+#else
+# define EIGHT_BYTE_ALIGNMENT(X)   ((((uptr)(X) - (uptr)0)&7)==0)
+#endif
+
 #endif

 /* Truncate very long tokens to this many bytes. Hard limit is 
@@ -142,6 +158,18 @@ struct Fts5Colset {
 */

 typedef struct Fts5Config Fts5Config;
+typedef struct Fts5TokenizerConfig Fts5TokenizerConfig;
+
+struct Fts5TokenizerConfig {
+  Fts5Tokenizer *pTok;
+  fts5_tokenizer_v2 *pApi2;
+  fts5_tokenizer *pApi1;
+  const char **azArg;
+  int nArg;
+  int ePattern;                   /* FTS_PATTERN_XXX constant */
+  const char *pLocale;            /* Current locale to use */
+  int nLocale;                    /* Size of pLocale in bytes */
+};

 /*
 ** An instance of the following structure encodes all information that can
@@ -181,9 +209,12 @@ typedef struct Fts5Config Fts5Config;
 **
 **       INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex);
 **
+** bLocale:
+**   Set to true if locale=1 was specified when the table was created.
 */
 struct Fts5Config {
  sqlite3 *db;                    /* Database handle */
+  Fts5Global *pGlobal;            /* Global fts5 object for handle db */
  char *zDb;                      /* Database holding FTS index (e.g. "main") */
  char *zName;                    /* Name of FTS index */
  int nCol;                       /* Number of columns */
@@ -193,16 +224,17 @@ struct Fts5Config {
  int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
  int eContent;                   /* An FTS5_CONTENT value */
  int bContentlessDelete;         /* "contentless_delete=" option (dflt==0) */
+  int bContentlessUnindexed;      /* "contentless_unindexed=" option (dflt=0) */
  char *zContent;                 /* content table */ 
  char *zContentRowid;            /* "content_rowid=" option value */ 
  int bColumnsize;                /* "columnsize=" option value (dflt==1) */
  int bTokendata;                 /* "tokendata=" option value (dflt==0) */
+  int bLocale;                    /* "locale=" option value (dflt==0) */
  int eDetail;                    /* FTS5_DETAIL_XXX value */
  char *zContentExprlist;
-  Fts5Tokenizer *pTok;
-  fts5_tokenizer *pTokApi;
+  Fts5TokenizerConfig t;
  int bLock;                      /* True when table is preparing statement */
-  int ePattern;                   /* FTS_PATTERN_XXX constant */
+  

  /* Values loaded from the %_config table */
  int iVersion;                   /* fts5 file format 'version' */
@@ -231,9 +263,10 @@ struct Fts5Config {
 #define FTS5_CURRENT_VERSION               4
 #define FTS5_CURRENT_VERSION_SECUREDELETE  5

-#define FTS5_CONTENT_NORMAL   0
-#define FTS5_CONTENT_NONE     1
-#define FTS5_CONTENT_EXTERNAL 2
+#define FTS5_CONTENT_NORMAL    0
+#define FTS5_CONTENT_NONE      1
+#define FTS5_CONTENT_EXTERNAL  2
+#define FTS5_CONTENT_UNINDEXED 3

 #define FTS5_DETAIL_FULL      0
 #define FTS5_DETAIL_NONE      1
@@ -268,6 +301,8 @@ int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);

 int sqlite3Fts5ConfigParseRank(const char*, char**, char**);

+void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...);
+
 /*
 ** End of interface to code in fts5_config.c.
 **************************************************************************/
@@ -312,7 +347,7 @@ char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
 void sqlite3Fts5Put32(u8*, int);
 int sqlite3Fts5Get32(const u8*);

-#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
+#define FTS5_POS2COLUMN(iPos) (int)((iPos >> 32) & 0x7FFFFFFF)
 #define FTS5_POS2OFFSET(iPos) (int)(iPos & 0x7FFFFFFF)

 typedef struct Fts5PoslistReader Fts5PoslistReader;
@@ -597,18 +632,20 @@ struct Fts5Table {
  Fts5Index *pIndex;              /* Full-text index */
 };

-int sqlite3Fts5GetTokenizer(
-  Fts5Global*, 
-  const char **azArg,
-  int nArg,
-  Fts5Config*,
-  char **pzErr
-);
+int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig);

 Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64);

 int sqlite3Fts5FlushToDisk(Fts5Table*);

+void sqlite3Fts5ClearLocale(Fts5Config *pConfig);
+void sqlite3Fts5SetLocale(Fts5Config *pConfig, const char *pLoc, int nLoc);
+
+int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal);
+int sqlite3Fts5DecodeLocaleValue(sqlite3_value *pVal, 
+    const char **ppText, int *pnText, const char **ppLoc, int *pnLoc
+);
+
 /*
 ** End of interface to code in fts5.c.
 **************************************************************************/
@@ -688,8 +725,8 @@ int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
 int sqlite3Fts5DropAll(Fts5Config*);
 int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);

-int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**);
-int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
+int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**, int);
+int sqlite3Fts5StorageContentInsert(Fts5Storage *p, int, sqlite3_value**, i64*);
 int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);

 int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg);
@@ -714,6 +751,9 @@ int sqlite3Fts5StorageOptimize(Fts5Storage *p);
 int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
 int sqlite3Fts5StorageReset(Fts5Storage *p);

+void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage*);
+int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel);
+
 /*
 ** End of interface to code in fts5_storage.c.
 **************************************************************************/
@@ -866,6 +906,7 @@ int sqlite3Fts5TokenizerPattern(
    int (*xCreate)(void*, const char**, int, Fts5Tokenizer**),
    Fts5Tokenizer *pTok
 );
+int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig*);
 /*
 ** End of interface to code in fts5_tokenizer.c.
 **************************************************************************/
@@ -226,6 +226,7 @@ static int fts5HighlightCb(
  return rc;
 }

+
 /*
 ** Implementation of highlight() function.
 */
@@ -256,12 +257,19 @@ static void fts5HighlightFunction(
    sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC);
    rc = SQLITE_OK;
  }else if( ctx.zIn ){
+    const char *pLoc = 0;         /* Locale of column iCol */
+    int nLoc = 0;                 /* Size of pLoc in bytes */
    if( rc==SQLITE_OK ){
      rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter);
    }

    if( rc==SQLITE_OK ){
-      rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
+      rc = pApi->xColumnLocale(pFts, iCol, &pLoc, &nLoc);
+    }
+    if( rc==SQLITE_OK ){
+      rc = pApi->xTokenize_v2(
+          pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx, fts5HighlightCb
+      );
    }
    if( ctx.bOpen ){
      fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1);
@@ -458,6 +466,8 @@ static void fts5SnippetFunction(
  memset(&sFinder, 0, sizeof(Fts5SFinder));
  for(i=0; i<nCol; i++){
    if( iCol<0 || iCol==i ){
+      const char *pLoc = 0;       /* Locale of column iCol */
+      int nLoc = 0;               /* Size of pLoc in bytes */
      int nDoc;
      int nDocsize;
      int ii;
@@ -465,8 +475,10 @@ static void fts5SnippetFunction(
      sFinder.nFirst = 0;
      rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc);
      if( rc!=SQLITE_OK ) break;
-      rc = pApi->xTokenize(pFts, 
-          sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb
+      rc = pApi->xColumnLocale(pFts, i, &pLoc, &nLoc);
+      if( rc!=SQLITE_OK ) break;
+      rc = pApi->xTokenize_v2(pFts, 
+          sFinder.zDoc, nDoc, pLoc, nLoc, (void*)&sFinder, fts5SentenceFinderCb
      );
      if( rc!=SQLITE_OK ) break;
      rc = pApi->xColumnSize(pFts, i, &nDocsize);
@@ -524,6 +536,9 @@ static void fts5SnippetFunction(
    rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
  }
  if( ctx.zIn ){
+    const char *pLoc = 0;         /* Locale of column iBestCol */
+    int nLoc = 0;                 /* Bytes in pLoc */
+
    if( rc==SQLITE_OK ){
      rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
    }
@@ -542,7 +557,12 @@ static void fts5SnippetFunction(
    }

    if( rc==SQLITE_OK ){
-      rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
+      rc = pApi->xColumnLocale(pFts, iBestCol, &pLoc, &nLoc);
+    }
+    if( rc==SQLITE_OK ){
+      rc = pApi->xTokenize_v2(
+          pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx,fts5HighlightCb
+      );
    }
    if( ctx.bOpen ){
      fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1);
@@ -726,6 +746,53 @@ static void fts5Bm25Function(
  }
 }

+/*
+** Implementation of fts5_get_locale() function.
+*/
+static void fts5GetLocaleFunction(
+  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
+  Fts5Context *pFts,              /* First arg to pass to pApi functions */
+  sqlite3_context *pCtx,          /* Context for returning result/error */
+  int nVal,                       /* Number of values in apVal[] array */
+  sqlite3_value **apVal           /* Array of trailing arguments */
+){
+  int iCol = 0;
+  int eType = 0;
+  int rc = SQLITE_OK;
+  const char *zLocale = 0;
+  int nLocale = 0;
+
+  /* xColumnLocale() must be available */
+  assert( pApi->iVersion>=4 );
+
+  if( nVal!=1 ){
+    const char *z = "wrong number of arguments to function fts5_get_locale()";
+    sqlite3_result_error(pCtx, z, -1);
+    return;
+  }
+
+  eType = sqlite3_value_numeric_type(apVal[0]);
+  if( eType!=SQLITE_INTEGER ){
+    const char *z = "non-integer argument passed to function fts5_get_locale()";
+    sqlite3_result_error(pCtx, z, -1);
+    return;
+  }
+
+  iCol = sqlite3_value_int(apVal[0]);
+  if( iCol<0 || iCol>=pApi->xColumnCount(pFts) ){
+    sqlite3_result_error_code(pCtx, SQLITE_RANGE);
+    return;
+  }
+
+  rc = pApi->xColumnLocale(pFts, iCol, &zLocale, &nLocale);
+  if( rc!=SQLITE_OK ){
+    sqlite3_result_error_code(pCtx, rc);
+    return;
+  }
+
+  sqlite3_result_text(pCtx, zLocale, nLocale, SQLITE_TRANSIENT);
+}
+
 int sqlite3Fts5AuxInit(fts5_api *pApi){
  struct Builtin {
    const char *zFunc;            /* Function name (nul-terminated) */
@@ -733,9 +800,10 @@ int sqlite3Fts5AuxInit(fts5_api *pApi){
    fts5_extension_function xFunc;/* Callback function */
    void (*xDestroy)(void*);      /* Destructor function */
  } aBuiltin [] = {
-    { "snippet",   0, fts5SnippetFunction, 0 },
-    { "highlight", 0, fts5HighlightFunction, 0 },
-    { "bm25",      0, fts5Bm25Function,    0 },
+    { "snippet",         0, fts5SnippetFunction,   0 },
+    { "highlight",       0, fts5HighlightFunction, 0 },
+    { "bm25",            0, fts5Bm25Function,      0 },
+    { "fts5_get_locale", 0, fts5GetLocaleFunction, 0 },
  };
  int rc = SQLITE_OK;             /* Return code */
  int i;                          /* To iterate through builtin functions */
@@ -234,7 +234,6 @@ static int fts5ConfigSetEnum(
 ** eventually free any such error message using sqlite3_free().
 */
 static int fts5ConfigParseSpecial(
-  Fts5Global *pGlobal,
  Fts5Config *pConfig,            /* Configuration object to update */
  const char *zCmd,               /* Special command to parse */
  const char *zArg,               /* Argument to parse */
@@ -242,6 +241,7 @@ static int fts5ConfigParseSpecial(
 ){
  int rc = SQLITE_OK;
  int nCmd = (int)strlen(zCmd);
+
  if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){
    const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES;
    const char *p;
@@ -298,12 +298,11 @@ static int fts5ConfigParseSpecial(
  if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
    const char *p = (const char*)zArg;
    sqlite3_int64 nArg = strlen(zArg) + 1;
-    char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
-    char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2);
-    char *pSpace = pDel;
+    char **azArg = sqlite3Fts5MallocZero(&rc, (sizeof(char*) + 2) * nArg);

-    if( azArg && pSpace ){
-      if( pConfig->pTok ){
+    if( azArg ){
+      char *pSpace = (char*)&azArg[nArg];
+      if( pConfig->t.azArg ){
        *pzErr = sqlite3_mprintf("multiple tokenize=... directives");
        rc = SQLITE_ERROR;
      }else{
@@ -326,16 +325,14 @@ static int fts5ConfigParseSpecial(
          *pzErr = sqlite3_mprintf("parse error in tokenize directive");
          rc = SQLITE_ERROR;
        }else{
-          rc = sqlite3Fts5GetTokenizer(pGlobal, 
-              (const char**)azArg, (int)nArg, pConfig,
-              pzErr
-          );
+          pConfig->t.azArg = (const char**)azArg;
+          pConfig->t.nArg = nArg;
+          azArg = 0;
        }
      }
    }
-
    sqlite3_free(azArg);
-    sqlite3_free(pDel);
+
    return rc;
  }

@@ -364,6 +361,16 @@ static int fts5ConfigParseSpecial(
    return rc;
  }

+  if( sqlite3_strnicmp("contentless_unindexed", zCmd, nCmd)==0 ){
+    if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
+      *pzErr = sqlite3_mprintf("malformed contentless_delete=... directive");
+      rc = SQLITE_ERROR;
+    }else{
+      pConfig->bContentlessUnindexed = (zArg[0]=='1');
+    }
+    return rc;
+  }
+
  if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){
    if( pConfig->zContentRowid ){
      *pzErr = sqlite3_mprintf("multiple content_rowid=... directives");
@@ -384,6 +391,16 @@ static int fts5ConfigParseSpecial(
    return rc;
  }

+  if( sqlite3_strnicmp("locale", zCmd, nCmd)==0 ){
+    if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
+      *pzErr = sqlite3_mprintf("malformed locale=... directive");
+      rc = SQLITE_ERROR;
+    }else{
+      pConfig->bLocale = (zArg[0]=='1');
+    }
+    return rc;
+  }
+
  if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){
    const Fts5Enum aDetail[] = {
      { "none", FTS5_DETAIL_NONE },
@@ -412,16 +429,6 @@ static int fts5ConfigParseSpecial(
  return SQLITE_ERROR;
 }

-/*
-** Allocate an instance of the default tokenizer ("simple") at 
-** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
-** code if an error occurs.
-*/
-static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
-  assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
-  return sqlite3Fts5GetTokenizer(pGlobal, 0, 0, pConfig, 0);
-}
-
 /*
 ** Gobble up the first bareword or quoted word from the input buffer zIn.
 ** Return a pointer to the character immediately following the last in
@@ -481,7 +488,8 @@ static int fts5ConfigParseColumn(
  Fts5Config *p, 
  char *zCol, 
  char *zArg, 
-  char **pzErr
+  char **pzErr,
+  int *pbUnindexed
 ){
  int rc = SQLITE_OK;
  if( 0==sqlite3_stricmp(zCol, FTS5_RANK_NAME) 
@@ -492,6 +500,7 @@ static int fts5ConfigParseColumn(
  }else if( zArg ){
    if( 0==sqlite3_stricmp(zArg, "unindexed") ){
      p->abUnindexed[p->nCol] = 1;
+      *pbUnindexed = 1;
    }else{
      *pzErr = sqlite3_mprintf("unrecognized column option: %s", zArg);
      rc = SQLITE_ERROR;
@@ -512,11 +521,26 @@ static int fts5ConfigMakeExprlist(Fts5Config *p){

  sqlite3Fts5BufferAppendPrintf(&rc, &buf, "T.%Q", p->zContentRowid);
  if( p->eContent!=FTS5_CONTENT_NONE ){
+    assert( p->eContent==FTS5_CONTENT_EXTERNAL
+         || p->eContent==FTS5_CONTENT_NORMAL
+         || p->eContent==FTS5_CONTENT_UNINDEXED
+    );
    for(i=0; i<p->nCol; i++){
      if( p->eContent==FTS5_CONTENT_EXTERNAL ){
        sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.%Q", p->azCol[i]);
-      }else{
+      }else if( p->eContent==FTS5_CONTENT_NORMAL || p->abUnindexed[i] ){
        sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.c%d", i);
+      }else{
+        sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL");
+      }
+    }
+  }
+  if( p->eContent==FTS5_CONTENT_NORMAL && p->bLocale ){
+    for(i=0; i<p->nCol; i++){
+      if( p->abUnindexed[i]==0 ){
+        sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.l%d", i);
+      }else{
+        sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL");
      }
    }
  }
@@ -550,10 +574,12 @@ int sqlite3Fts5ConfigParse(
  Fts5Config *pRet;               /* New object to return */
  int i;
  sqlite3_int64 nByte;
+  int bUnindexed = 0;             /* True if there are one or more UNINDEXED */

  *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config));
  if( pRet==0 ) return SQLITE_NOMEM;
  memset(pRet, 0, sizeof(Fts5Config));
+  pRet->pGlobal = pGlobal;
  pRet->db = db;
  pRet->iCookie = -1;

@@ -602,13 +628,13 @@ int sqlite3Fts5ConfigParse(
        rc = SQLITE_ERROR;
      }else{
        if( bOption ){
-          rc = fts5ConfigParseSpecial(pGlobal, pRet, 
+          rc = fts5ConfigParseSpecial(pRet, 
            ALWAYS(zOne)?zOne:"",
            zTwo?zTwo:"",
            pzErr
          );
        }else{
-          rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr);
+          rc = fts5ConfigParseColumn(pRet, zOne, zTwo, pzErr, &bUnindexed);
          zOne = 0;
        }
      }
@@ -640,21 +666,30 @@ int sqlite3Fts5ConfigParse(
    rc = SQLITE_ERROR;
  }

-  /* If a tokenizer= option was successfully parsed, the tokenizer has
-  ** already been allocated. Otherwise, allocate an instance of the default
-  ** tokenizer (unicode61) now.  */
-  if( rc==SQLITE_OK && pRet->pTok==0 ){
-    rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
+  /* We only allow contentless_unindexed=1 if the table is actually a
+  ** contentless one.
+  */
+  if( rc==SQLITE_OK 
+   && pRet->bContentlessUnindexed 
+   && pRet->eContent!=FTS5_CONTENT_NONE
+  ){
+    *pzErr = sqlite3_mprintf(
+        "contentless_unindexed=1 requires a contentless table"
+    );
+    rc = SQLITE_ERROR;
  }

  /* If no zContent option was specified, fill in the default values. */
  if( rc==SQLITE_OK && pRet->zContent==0 ){
    const char *zTail = 0;
-    assert( pRet->eContent==FTS5_CONTENT_NORMAL 
-         || pRet->eContent==FTS5_CONTENT_NONE 
+    assert( pRet->eContent==FTS5_CONTENT_NORMAL
+         || pRet->eContent==FTS5_CONTENT_NONE
    );
    if( pRet->eContent==FTS5_CONTENT_NORMAL ){
      zTail = "content";
+    }else if( bUnindexed && pRet->bContentlessUnindexed ){
+      pRet->eContent = FTS5_CONTENT_UNINDEXED;
+      zTail = "content";
    }else if( pRet->bColumnsize ){
      zTail = "docsize";
    }
@@ -688,9 +723,14 @@ int sqlite3Fts5ConfigParse(
 void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
  if( pConfig ){
    int i;
-    if( pConfig->pTok ){
-      pConfig->pTokApi->xDelete(pConfig->pTok);
+    if( pConfig->t.pTok ){
+      if( pConfig->t.pApi1 ){
+        pConfig->t.pApi1->xDelete(pConfig->t.pTok);
+      }else{
+        pConfig->t.pApi2->xDelete(pConfig->t.pTok);
+      }
    }
+    sqlite3_free((char*)pConfig->t.azArg);
    sqlite3_free(pConfig->zDb);
    sqlite3_free(pConfig->zName);
    for(i=0; i<pConfig->nCol; i++){
@@ -765,10 +805,24 @@ int sqlite3Fts5Tokenize(
  void *pCtx,                     /* Context passed to xToken() */
  int (*xToken)(void*, int, const char*, int, int, int)    /* Callback */
 ){
-  if( pText==0 ) return SQLITE_OK;
-  return pConfig->pTokApi->xTokenize(
-      pConfig->pTok, pCtx, flags, pText, nText, xToken
-  );
+  int rc = SQLITE_OK;
+  if( pText ){
+    if( pConfig->t.pTok==0 ){
+      rc = sqlite3Fts5LoadTokenizer(pConfig);
+    }
+    if( rc==SQLITE_OK ){
+      if( pConfig->t.pApi1 ){
+        rc = pConfig->t.pApi1->xTokenize(
+            pConfig->t.pTok, pCtx, flags, pText, nText, xToken
+        );
+      }else{
+        rc = pConfig->t.pApi2->xTokenize(pConfig->t.pTok, pCtx, flags, 
+            pText, nText, pConfig->t.pLocale, pConfig->t.nLocale, xToken
+        );
+      }
+    }
+  }
+  return rc;
 }

 /*
@@ -1022,13 +1076,10 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
   && iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE
  ){
    rc = SQLITE_ERROR;
-    if( pConfig->pzErrmsg ){
-      assert( 0==*pConfig->pzErrmsg );
-      *pConfig->pzErrmsg = sqlite3_mprintf("invalid fts5 file format "
-          "(found %d, expected %d or %d) - run 'rebuild'",
-          iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE
-      );
-    }
+    sqlite3Fts5ConfigErrmsg(pConfig, "invalid fts5 file format "
+        "(found %d, expected %d or %d) - run 'rebuild'",
+        iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE
+    );
  }else{
    pConfig->iVersion = iVersion;
  }
@@ -1038,3 +1089,26 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
  }
  return rc;
 }
+
+/*
+** Set (*pConfig->pzErrmsg) to point to an sqlite3_malloc()ed buffer 
+** containing the error message created using printf() style formatting
+** string zFmt and its trailing arguments.
+*/
+void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){
+  va_list ap;                     /* ... printf arguments */
+  char *zMsg = 0;
+
+  va_start(ap, zFmt);
+  zMsg = sqlite3_vmprintf(zFmt, ap);
+  if( pConfig->pzErrmsg ){
+    assert( *pConfig->pzErrmsg==0 );
+    *pConfig->pzErrmsg = zMsg;
+  }else{
+    sqlite3_free(zMsg);
+  }
+
+  va_end(ap);
+}
+
+
@@ -54,7 +54,7 @@ struct Fts5Expr {

 /*
 ** eType:
-**   Expression node type. Always one of:
+**   Expression node type. Usually one of:
 **
 **       FTS5_AND                 (nChild, apChild valid)
 **       FTS5_OR                  (nChild, apChild valid)
@@ -62,6 +62,10 @@ struct Fts5Expr {
 **       FTS5_STRING              (pNear valid)
 **       FTS5_TERM                (pNear valid)
 **
+**   An expression node with eType==0 may also exist. It always matches zero
+**   rows. This is created when a phrase containing no tokens is parsed.
+**   e.g. "".
+**
 ** iHeight:
 **   Distance from this node to furthest leaf. This is always 0 for nodes
 **   of type FTS5_STRING and FTS5_TERM. For all other nodes it is one 
@@ -282,11 +286,12 @@ int sqlite3Fts5ExprNew(
  }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF );
  sqlite3Fts5ParserFree(pEngine, fts5ParseFree);

+  assert( sParse.pExpr || sParse.rc!=SQLITE_OK );
  assert_expr_depth_ok(sParse.rc, sParse.pExpr);

  /* If the LHS of the MATCH expression was a user column, apply the
  ** implicit column-filter.  */
-  if( iCol<pConfig->nCol && sParse.pExpr && sParse.rc==SQLITE_OK ){
+  if( sParse.rc==SQLITE_OK && iCol<pConfig->nCol ){
    int n = sizeof(Fts5Colset);
    Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n);
    if( pColset ){
@@ -303,15 +308,7 @@ int sqlite3Fts5ExprNew(
      sParse.rc = SQLITE_NOMEM;
      sqlite3Fts5ParseNodeFree(sParse.pExpr);
    }else{
-      if( !sParse.pExpr ){
-        const int nByte = sizeof(Fts5ExprNode);
-        pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&sParse.rc, nByte);
-        if( pNew->pRoot ){
-          pNew->pRoot->bEof = 1;
-        }
-      }else{
-        pNew->pRoot = sParse.pExpr;
-      }
+      pNew->pRoot = sParse.pExpr;
      pNew->pIndex = 0;
      pNew->pConfig = pConfig;
      pNew->apExprPhrase = sParse.apPhrase;
@@ -324,7 +321,11 @@ int sqlite3Fts5ExprNew(
  }

  sqlite3_free(sParse.apPhrase);
-  *pzErr = sParse.zErr;
+  if( 0==*pzErr ){
+    *pzErr = sParse.zErr;
+  }else{
+    sqlite3_free(sParse.zErr);
+  }
  return sParse.rc;
 }

@@ -1125,7 +1126,7 @@ static int fts5ExprNodeTest_STRING(
          }
        }else{
          Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
-          if( pIter->iRowid==iLast || pIter->bEof ) continue;
+          if( pIter->iRowid==iLast ) continue;
          bMatch = 0;
          if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
            return rc;
@@ -1647,9 +1648,6 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset(
  Fts5ExprNearset *pRet = 0;

  if( pParse->rc==SQLITE_OK ){
-    if( pPhrase==0 ){
-      return pNear;
-    }
    if( pNear==0 ){
      sqlite3_int64 nByte;
      nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*);
@@ -1871,6 +1869,7 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
    }else if( sCtx.pPhrase->nTerm ){
      sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix;
    }
+    assert( pParse->apPhrase!=0 );
    pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
  }

@@ -1890,7 +1889,7 @@ int sqlite3Fts5ExprClonePhrase(
  Fts5ExprPhrase *pOrig = 0;      /* The phrase extracted from pExpr */
  Fts5Expr *pNew = 0;             /* Expression to return via *ppNew */
  TokenCtx sCtx = {0,0,0};        /* Context object for fts5ParseTokenize */
-  if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
+  if( !pExpr || iPhrase<0 || iPhrase>=pExpr->nPhrase ){
    rc = SQLITE_RANGE;
  }else{
    pOrig = pExpr->apExprPhrase[iPhrase];
@@ -2258,6 +2257,9 @@ static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
  }
 }

+/*
+** Add pSub as a child of p.
+*/
 static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){
  int ii = p->nChild;
  if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){
@@ -2402,19 +2404,23 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
                  "fts5: %s queries are not supported (detail!=full)", 
                  pNear->nPhrase==1 ? "phrase": "NEAR"
              );
-              sqlite3_free(pRet);
+              sqlite3Fts5ParseNodeFree(pRet);
              pRet = 0;
+              pNear = 0;
+              assert( pLeft==0 && pRight==0 );
            }
          }
        }else{
+          assert( pNear==0 );
          fts5ExprAddChildren(pRet, pLeft);
          fts5ExprAddChildren(pRet, pRight);
+          pLeft = pRight = 0;
          if( pRet->iHeight>SQLITE_FTS5_MAX_EXPR_DEPTH ){
            sqlite3Fts5ParseError(pParse, 
                "fts5 expression tree is too large (maximum depth %d)", 
                SQLITE_FTS5_MAX_EXPR_DEPTH
            );
-            sqlite3_free(pRet);
+            sqlite3Fts5ParseNodeFree(pRet);
            pRet = 0;
          }
        }
@@ -2452,6 +2458,7 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
    assert( pRight->eType==FTS5_STRING 
        || pRight->eType==FTS5_TERM 
        || pRight->eType==FTS5_EOF 
+        || (pRight->eType==FTS5_AND && pParse->bPhraseToAnd) 
    );

    if( pLeft->eType==FTS5_AND ){
@@ -2465,6 +2472,8 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
        );

    if( pRight->eType==FTS5_EOF ){
+      assert( pParse->apPhrase!=0 );
+      assert( pParse->nPhrase>0 );
      assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] );
      sqlite3Fts5ParseNodeFree(pRight);
      pRet = pLeft;
@@ -3097,6 +3106,7 @@ static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){
  pNode->iRowid = iRowid;
  pNode->bEof = 0;
  switch( pNode->eType ){
+    case 0:
    case FTS5_TERM:
    case FTS5_STRING:
      return (pNode->pNear->apPhrase[0]->poslist.n>0);
@@ -831,11 +831,12 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){
    if( rc==SQLITE_OK ){
      u8 *aOut = 0;               /* Read blob data into this buffer */
      int nByte = sqlite3_blob_bytes(p->pReader);
-      sqlite3_int64 nAlloc = sizeof(Fts5Data) + nByte + FTS5_DATA_PADDING;
+      int szData = (sizeof(Fts5Data) + 7) & ~7;
+      sqlite3_int64 nAlloc = szData + nByte + FTS5_DATA_PADDING;
      pRet = (Fts5Data*)sqlite3_malloc64(nAlloc);
      if( pRet ){
        pRet->nn = nByte;
-        aOut = pRet->p = (u8*)&pRet[1];
+        aOut = pRet->p = (u8*)pRet + szData;
      }else{
        rc = SQLITE_NOMEM;
      }
@@ -858,6 +859,7 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){
  }

  assert( (pRet==0)==(p->rc!=SQLITE_OK) );
+  assert( pRet==0 || EIGHT_BYTE_ALIGNMENT( pRet->p ) );
  return pRet;
 }

@@ -2183,7 +2185,7 @@ static void fts5SegIterNext_None(

  if( iOff<pIter->iEndofDoclist ){
    /* Next entry is on the current page */
-    i64 iDelta;
+    u64 iDelta;
    iOff += sqlite3Fts5GetVarint(&pIter->pLeaf->p[iOff], (u64*)&iDelta);
    pIter->iLeafOffset = iOff;
    pIter->iRowid += iDelta;
@@ -4887,6 +4889,11 @@ static int fts5IndexFindDeleteMerge(Fts5Index *p, Fts5Structure *pStruct){
          nBest = nPercent;
        }
      }
+
+      /* If pLvl is already the input level to an ongoing merge, look no
+      ** further for a merge candidate. The caller should be allowed to
+      ** continue merging from pLvl first.  */
+      if( pLvl->nMerge ) break;
    }
  }
  return iRet;
@@ -8811,7 +8818,7 @@ static int fts5structConnectMethod(

 /*
 ** We must have a single struct=? constraint that will be passed through
-** into the xFilter method.  If there is no valid stmt=? constraint,
+** into the xFilter method.  If there is no valid struct=? constraint,
 ** then return an SQLITE_CONSTRAINT error.
 */
 static int fts5structBestIndexMethod(
@@ -16,13 +16,40 @@

 #include "fts5Int.h"

+/*
+** pSavedRow:
+**   SQL statement FTS5_STMT_LOOKUP2 is a copy of FTS5_STMT_LOOKUP, it 
+**   does a by-rowid lookup to retrieve a single row from the %_content 
+**   table or equivalent external-content table/view.
+**
+**   However, FTS5_STMT_LOOKUP2 is only used when retrieving the original
+**   values for a row being UPDATEd. In that case, the SQL statement is
+**   not reset and pSavedRow is set to point at it. This is so that the
+**   insert operation that follows the delete may access the original 
+**   row values for any new values for which sqlite3_value_nochange() returns
+**   true. i.e. if the user executes:
+**
+**        CREATE VIRTUAL TABLE ft USING fts5(a, b, c, locale=1);
+**        ...
+**        UPDATE fts SET a=?, b=? WHERE rowid=?;
+**
+**   then the value passed to the xUpdate() method of this table as the 
+**   new.c value is an sqlite3_value_nochange() value. So in this case it
+**   must be read from the saved row stored in Fts5Storage.pSavedRow.
+**
+**   This is necessary - using sqlite3_value_nochange() instead of just having
+**   SQLite pass the original value back via xUpdate() - so as not to discard
+**   any locale information associated with such values.
+**
+*/
 struct Fts5Storage {
  Fts5Config *pConfig;
  Fts5Index *pIndex;
  int bTotalsValid;               /* True if nTotalRow/aTotalSize[] are valid */
  i64 nTotalRow;                  /* Total number of rows in FTS table */
  i64 *aTotalSize;                /* Total sizes of each column */ 
-  sqlite3_stmt *aStmt[11];
+  sqlite3_stmt *pSavedRow;
+  sqlite3_stmt *aStmt[12];
 };


@@ -36,14 +63,15 @@ struct Fts5Storage {
 # error "FTS5_STMT_LOOKUP mismatch" 
 #endif

-#define FTS5_STMT_INSERT_CONTENT  3
-#define FTS5_STMT_REPLACE_CONTENT 4
-#define FTS5_STMT_DELETE_CONTENT  5
-#define FTS5_STMT_REPLACE_DOCSIZE  6
-#define FTS5_STMT_DELETE_DOCSIZE  7
-#define FTS5_STMT_LOOKUP_DOCSIZE  8
-#define FTS5_STMT_REPLACE_CONFIG 9
-#define FTS5_STMT_SCAN 10
+#define FTS5_STMT_LOOKUP2         3
+#define FTS5_STMT_INSERT_CONTENT  4
+#define FTS5_STMT_REPLACE_CONTENT 5
+#define FTS5_STMT_DELETE_CONTENT  6
+#define FTS5_STMT_REPLACE_DOCSIZE 7
+#define FTS5_STMT_DELETE_DOCSIZE  8
+#define FTS5_STMT_LOOKUP_DOCSIZE  9
+#define FTS5_STMT_REPLACE_CONFIG 10
+#define FTS5_STMT_SCAN           11

 /*
 ** Prepare the two insert statements - Fts5Storage.pInsertContent and
@@ -73,6 +101,7 @@ static int fts5StorageGetStmt(
      "SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC",
      "SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC",
      "SELECT %s FROM %s T WHERE T.%Q=?",               /* LOOKUP  */
+      "SELECT %s FROM %s T WHERE T.%Q=?",               /* LOOKUP2  */

      "INSERT INTO %Q.'%q_content' VALUES(%s)",         /* INSERT_CONTENT  */
      "REPLACE INTO %Q.'%q_content' VALUES(%s)",        /* REPLACE_CONTENT */
@@ -88,6 +117,8 @@ static int fts5StorageGetStmt(
    Fts5Config *pC = p->pConfig;
    char *zSql = 0;

+    assert( ArraySize(azStmt)==ArraySize(p->aStmt) );
+
    switch( eStmt ){
      case FTS5_STMT_SCAN:
        zSql = sqlite3_mprintf(azStmt[eStmt], 
@@ -104,6 +135,7 @@ static int fts5StorageGetStmt(
        break;

      case FTS5_STMT_LOOKUP:
+      case FTS5_STMT_LOOKUP2:
        zSql = sqlite3_mprintf(azStmt[eStmt], 
            pC->zContentExprlist, pC->zContent, pC->zContentRowid
        );
@@ -111,20 +143,35 @@ static int fts5StorageGetStmt(

      case FTS5_STMT_INSERT_CONTENT: 
      case FTS5_STMT_REPLACE_CONTENT: {
-        int nCol = pC->nCol + 1;
-        char *zBind;
+        char *zBind = 0;
        int i;

-        zBind = sqlite3_malloc64(1 + nCol*2);
-        if( zBind ){
-          for(i=0; i<nCol; i++){
-            zBind[i*2] = '?';
-            zBind[i*2 + 1] = ',';
+        assert( pC->eContent==FTS5_CONTENT_NORMAL 
+             || pC->eContent==FTS5_CONTENT_UNINDEXED 
+        );
+
+        /* Add bindings for the "c*" columns - those that store the actual
+        ** table content. If eContent==NORMAL, then there is one binding
+        ** for each column. Or, if eContent==UNINDEXED, then there are only
+        ** bindings for the UNINDEXED columns. */
+        for(i=0; rc==SQLITE_OK && i<(pC->nCol+1); i++){
+          if( !i || pC->eContent==FTS5_CONTENT_NORMAL || pC->abUnindexed[i-1] ){
+            zBind = sqlite3Fts5Mprintf(&rc, "%z%s?%d", zBind, zBind?",":"",i+1);
          }
-          zBind[i*2-1] = '\0';
-          zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, zBind);
-          sqlite3_free(zBind);
        }
+
+        /* Add bindings for any "l*" columns. Only non-UNINDEXED columns
+        ** require these.  */
+        if( pC->bLocale && pC->eContent==FTS5_CONTENT_NORMAL ){
+          for(i=0; rc==SQLITE_OK && i<pC->nCol; i++){
+            if( pC->abUnindexed[i]==0 ){
+              zBind = sqlite3Fts5Mprintf(&rc, "%z,?%d", zBind, pC->nCol+i+2);
+            }
+          }
+        }
+
+        zSql = sqlite3Fts5Mprintf(&rc, azStmt[eStmt], pC->zDb, pC->zName,zBind);
+        sqlite3_free(zBind);
        break;
      }

@@ -150,7 +197,7 @@ static int fts5StorageGetStmt(
      rc = SQLITE_NOMEM;
    }else{
      int f = SQLITE_PREPARE_PERSISTENT;
-      if( eStmt>FTS5_STMT_LOOKUP ) f |= SQLITE_PREPARE_NO_VTAB;
+      if( eStmt>FTS5_STMT_LOOKUP2 ) f |= SQLITE_PREPARE_NO_VTAB;
      p->pConfig->bLock++;
      rc = sqlite3_prepare_v3(pC->db, zSql, -1, f, &p->aStmt[eStmt], 0);
      p->pConfig->bLock--;
@@ -310,9 +357,11 @@ int sqlite3Fts5StorageOpen(
  p->pIndex = pIndex;

  if( bCreate ){
-    if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
+    if( pConfig->eContent==FTS5_CONTENT_NORMAL 
+     || pConfig->eContent==FTS5_CONTENT_UNINDEXED 
+    ){
      int nDefn = 32 + pConfig->nCol*10;
-      char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 10);
+      char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 20);
      if( zDefn==0 ){
        rc = SQLITE_NOMEM;
      }else{
@@ -321,8 +370,20 @@ int sqlite3Fts5StorageOpen(
        sqlite3_snprintf(nDefn, zDefn, "id INTEGER PRIMARY KEY");
        iOff = (int)strlen(zDefn);
        for(i=0; i<pConfig->nCol; i++){
-          sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i);
-          iOff += (int)strlen(&zDefn[iOff]);
+          if( pConfig->eContent==FTS5_CONTENT_NORMAL 
+           || pConfig->abUnindexed[i] 
+          ){
+            sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i);
+            iOff += (int)strlen(&zDefn[iOff]);
+          }
+        }
+        if( pConfig->bLocale ){
+          for(i=0; i<pConfig->nCol; i++){
+            if( pConfig->abUnindexed[i]==0 ){
+              sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", l%d", i);
+              iOff += (int)strlen(&zDefn[iOff]);
+            }
+          }
        }
        rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr);
      }
@@ -399,15 +460,49 @@ static int fts5StorageInsertCallback(
  return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken);
 }

+/*
+** This function is used as part of an UPDATE statement that modifies the
+** rowid of a row. In that case, this function is called first to set
+** Fts5Storage.pSavedRow to point to a statement that may be used to 
+** access the original values of the row being deleted - iDel.
+**
+** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
+** It is not considered an error if row iDel does not exist. In this case
+** pSavedRow is not set and SQLITE_OK returned.
+*/
+int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel){
+  int rc = SQLITE_OK;
+  sqlite3_stmt *pSeek = 0;
+
+  assert( p->pSavedRow==0 );
+  rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+1, &pSeek, 0);
+  if( rc==SQLITE_OK ){
+    sqlite3_bind_int64(pSeek, 1, iDel);
+    if( sqlite3_step(pSeek)!=SQLITE_ROW ){
+      rc = sqlite3_reset(pSeek);
+    }else{
+      p->pSavedRow = pSeek;
+    }
+  }
+
+  return rc;
+}
+
 /*
 ** If a row with rowid iDel is present in the %_content table, add the
 ** delete-markers to the FTS index necessary to delete it. Do not actually
 ** remove the %_content row at this time though.
+**
+** If parameter bSaveRow is true, then Fts5Storage.pSavedRow is left
+** pointing to a statement (FTS5_STMT_LOOKUP2) that may be used to access
+** the original values of the row being deleted. This is used by UPDATE
+** statements.
 */
 static int fts5StorageDeleteFromIndex(
  Fts5Storage *p, 
  i64 iDel, 
-  sqlite3_value **apVal
+  sqlite3_value **apVal,
+  int bSaveRow                    /* True to set pSavedRow */
 ){
  Fts5Config *pConfig = p->pConfig;
  sqlite3_stmt *pSeek = 0;        /* SELECT to read row iDel from %_data */
@@ -416,12 +511,21 @@ static int fts5StorageDeleteFromIndex(
  int iCol;
  Fts5InsertCtx ctx;

+  assert( bSaveRow==0 || apVal==0 );
+  assert( bSaveRow==0 || bSaveRow==1 );
+  assert( FTS5_STMT_LOOKUP2==FTS5_STMT_LOOKUP+1 );
+
  if( apVal==0 ){
-    rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0);
-    if( rc!=SQLITE_OK ) return rc;
-    sqlite3_bind_int64(pSeek, 1, iDel);
-    if( sqlite3_step(pSeek)!=SQLITE_ROW ){
-      return sqlite3_reset(pSeek);
+    if( p->pSavedRow && bSaveRow ){
+      pSeek = p->pSavedRow;
+      p->pSavedRow = 0;
+    }else{
+      rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+bSaveRow, &pSeek, 0);
+      if( rc!=SQLITE_OK ) return rc;
+      sqlite3_bind_int64(pSeek, 1, iDel);
+      if( sqlite3_step(pSeek)!=SQLITE_ROW ){
+        return sqlite3_reset(pSeek);
+      }
    }
  }

@@ -429,26 +533,42 @@ static int fts5StorageDeleteFromIndex(
  ctx.iCol = -1;
  for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
    if( pConfig->abUnindexed[iCol-1]==0 ){
-      const char *zText;
-      int nText;
+      sqlite3_value *pVal = 0;
+      const char *pText = 0;
+      int nText = 0;
+      const char *pLoc = 0;
+      int nLoc = 0;
+
      assert( pSeek==0 || apVal==0 );
      assert( pSeek!=0 || apVal!=0 );
      if( pSeek ){
-        zText = (const char*)sqlite3_column_text(pSeek, iCol);
-        nText = sqlite3_column_bytes(pSeek, iCol);
-      }else if( ALWAYS(apVal) ){
-        zText = (const char*)sqlite3_value_text(apVal[iCol-1]);
-        nText = sqlite3_value_bytes(apVal[iCol-1]);
+        pVal = sqlite3_column_value(pSeek, iCol);
      }else{
-        continue;
+        pVal = apVal[iCol-1];
      }
-      ctx.szCol = 0;
-      rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, 
-          zText, nText, (void*)&ctx, fts5StorageInsertCallback
-      );
-      p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
-      if( p->aTotalSize[iCol-1]<0 ){
-        rc = FTS5_CORRUPT;
+
+      if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
+        rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
+      }else{
+        pText = (const char*)sqlite3_value_text(pVal);
+        nText = sqlite3_value_bytes(pVal);
+        if( pConfig->bLocale && pSeek ){
+          pLoc = (const char*)sqlite3_column_text(pSeek, iCol + pConfig->nCol);
+          nLoc = sqlite3_column_bytes(pSeek, iCol + pConfig->nCol);
+        }
+      }
+
+      if( rc==SQLITE_OK ){
+        sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
+        ctx.szCol = 0;
+        rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, 
+            pText, nText, (void*)&ctx, fts5StorageInsertCallback
+        );
+        p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
+        if( rc==SQLITE_OK && p->aTotalSize[iCol-1]<0 ){
+          rc = FTS5_CORRUPT;
+        }
+        sqlite3Fts5ClearLocale(pConfig);
      }
    }
  }
@@ -458,11 +578,29 @@ static int fts5StorageDeleteFromIndex(
    p->nTotalRow--;
  }

-  rc2 = sqlite3_reset(pSeek);
-  if( rc==SQLITE_OK ) rc = rc2;
+  if( rc==SQLITE_OK && bSaveRow ){
+    assert( p->pSavedRow==0 );
+    p->pSavedRow = pSeek;
+  }else{
+    rc2 = sqlite3_reset(pSeek);
+    if( rc==SQLITE_OK ) rc = rc2;
+  }
  return rc;
 }

+/*
+** Reset any saved statement pSavedRow. Zero pSavedRow as well. This
+** should be called by the xUpdate() method of the fts5 table before 
+** returning from any operation that may have set Fts5Storage.pSavedRow.
+*/
+void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage *pStorage){
+  assert( pStorage->pSavedRow==0 
+       || pStorage->pSavedRow==pStorage->aStmt[FTS5_STMT_LOOKUP2] 
+  );
+  sqlite3_reset(pStorage->pSavedRow);
+  pStorage->pSavedRow = 0;
+}
+
 /*
 ** This function is called to process a DELETE on a contentless_delete=1
 ** table. It adds the tombstone required to delete the entry with rowid 
@@ -475,7 +613,9 @@ static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){
  int rc = SQLITE_OK;

  assert( p->pConfig->bContentlessDelete );
-  assert( p->pConfig->eContent==FTS5_CONTENT_NONE );
+  assert( p->pConfig->eContent==FTS5_CONTENT_NONE
+       || p->pConfig->eContent==FTS5_CONTENT_UNINDEXED
+  );

  /* Look up the origin of the document in the %_docsize table. Store
  ** this in stack variable iOrigin.  */
@@ -519,12 +659,12 @@ static int fts5StorageInsertDocsize(
        rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin);
        sqlite3_bind_int64(pReplace, 3, iOrigin);
      }
-      if( rc==SQLITE_OK ){
-        sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
-        sqlite3_step(pReplace);
-        rc = sqlite3_reset(pReplace);
-        sqlite3_bind_null(pReplace, 2);
-      }
+    }
+    if( rc==SQLITE_OK ){
+      sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
+      sqlite3_step(pReplace);
+      rc = sqlite3_reset(pReplace);
+      sqlite3_bind_null(pReplace, 2);
    }
  }
  return rc;
@@ -578,7 +718,12 @@ static int fts5StorageSaveTotals(Fts5Storage *p){
 /*
 ** Remove a row from the FTS table.
 */
-int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
+int sqlite3Fts5StorageDelete(
+  Fts5Storage *p,                 /* Storage object */
+  i64 iDel,                       /* Rowid to delete from table */
+  sqlite3_value **apVal,          /* Optional - values to remove from index */
+  int bSaveRow                    /* If true, set pSavedRow for deleted row */
+){
  Fts5Config *pConfig = p->pConfig;
  int rc;
  sqlite3_stmt *pDel = 0;
@@ -594,8 +739,14 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
  if( rc==SQLITE_OK ){
    if( p->pConfig->bContentlessDelete ){
      rc = fts5StorageContentlessDelete(p, iDel);
+      if( rc==SQLITE_OK 
+       && bSaveRow 
+       && p->pConfig->eContent==FTS5_CONTENT_UNINDEXED
+      ){
+        rc = sqlite3Fts5StorageFindDeleteRow(p, iDel);
+      }
    }else{
-      rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
+      rc = fts5StorageDeleteFromIndex(p, iDel, apVal, bSaveRow);
    }
  }

@@ -610,7 +761,9 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
  }

  /* Delete the %_content record */
-  if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
+  if( pConfig->eContent==FTS5_CONTENT_NORMAL 
+   || pConfig->eContent==FTS5_CONTENT_UNINDEXED 
+  ){
    if( rc==SQLITE_OK ){
      rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0);
    }
@@ -642,8 +795,13 @@ int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){
  );
  if( rc==SQLITE_OK && pConfig->bColumnsize ){
    rc = fts5ExecPrintf(pConfig->db, 0,
-        "DELETE FROM %Q.'%q_docsize';",
-        pConfig->zDb, pConfig->zName
+        "DELETE FROM %Q.'%q_docsize';", pConfig->zDb, pConfig->zName
+    );
+  }
+
+  if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_UNINDEXED ){
+    rc = fts5ExecPrintf(pConfig->db, 0,
+        "DELETE FROM %Q.'%q_content';", pConfig->zDb, pConfig->zName
    );
  }

@@ -684,14 +842,36 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){
    for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
      ctx.szCol = 0;
      if( pConfig->abUnindexed[ctx.iCol]==0 ){
-        const char *zText = (const char*)sqlite3_column_text(pScan, ctx.iCol+1);
-        int nText = sqlite3_column_bytes(pScan, ctx.iCol+1);
-        rc = sqlite3Fts5Tokenize(pConfig, 
-            FTS5_TOKENIZE_DOCUMENT,
-            zText, nText,
-            (void*)&ctx,
-            fts5StorageInsertCallback
-        );
+        int nText = 0;            /* Size of pText in bytes */
+        const char *pText = 0;    /* Pointer to buffer containing text value */
+        int nLoc = 0;             /* Size of pLoc in bytes */
+        const char *pLoc = 0;     /* Pointer to buffer containing text value */
+
+        sqlite3_value *pVal = sqlite3_column_value(pScan, ctx.iCol+1);
+        if( pConfig->eContent==FTS5_CONTENT_EXTERNAL
+         && sqlite3Fts5IsLocaleValue(pConfig, pVal)
+        ){
+          rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
+        }else{
+          pText = (const char*)sqlite3_value_text(pVal);
+          nText = sqlite3_value_bytes(pVal);
+          if( pConfig->bLocale ){
+            int iCol = ctx.iCol + 1 + pConfig->nCol;
+            pLoc = (const char*)sqlite3_column_text(pScan, iCol);
+            nLoc = sqlite3_column_bytes(pScan, iCol);
+          }
+        }
+
+        if( rc==SQLITE_OK ){
+          sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
+          rc = sqlite3Fts5Tokenize(pConfig, 
+              FTS5_TOKENIZE_DOCUMENT,
+              pText, nText,
+              (void*)&ctx,
+              fts5StorageInsertCallback
+          );
+          sqlite3Fts5ClearLocale(pConfig);
+        }
      }
      sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
      p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
@@ -757,6 +937,7 @@ static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){
 */
 int sqlite3Fts5StorageContentInsert(
  Fts5Storage *p, 
+  int bReplace,                   /* True to use REPLACE instead of INSERT */
  sqlite3_value **apVal, 
  i64 *piRowid
 ){
@@ -764,7 +945,9 @@ int sqlite3Fts5StorageContentInsert(
  int rc = SQLITE_OK;

  /* Insert the new row into the %_content table. */
-  if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){
+  if( pConfig->eContent!=FTS5_CONTENT_NORMAL 
+   && pConfig->eContent!=FTS5_CONTENT_UNINDEXED
+  ){
    if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){
      *piRowid = sqlite3_value_int64(apVal[1]);
    }else{
@@ -773,9 +956,52 @@ int sqlite3Fts5StorageContentInsert(
  }else{
    sqlite3_stmt *pInsert = 0;    /* Statement to write %_content table */
    int i;                        /* Counter variable */
-    rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0);
-    for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
-      rc = sqlite3_bind_value(pInsert, i, apVal[i]);
+
+    assert( FTS5_STMT_INSERT_CONTENT+1==FTS5_STMT_REPLACE_CONTENT );
+    assert( bReplace==0 || bReplace==1 );
+    rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT+bReplace, &pInsert, 0);
+    if( pInsert ) sqlite3_clear_bindings(pInsert);
+
+    /* Bind the rowid value */
+    sqlite3_bind_value(pInsert, 1, apVal[1]);
+
+    /* Loop through values for user-defined columns. i=2 is the leftmost
+    ** user-defined column. As is column 1 of pSavedRow.  */
+    for(i=2; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
+      int bUnindexed = pConfig->abUnindexed[i-2];
+      if( pConfig->eContent==FTS5_CONTENT_NORMAL || bUnindexed ){
+        sqlite3_value *pVal = apVal[i];
+
+        if( sqlite3_value_nochange(pVal) && p->pSavedRow ){
+          /* This is an UPDATE statement, and user-defined column (i-2) was not
+          ** modified.  Retrieve the value from Fts5Storage.pSavedRow.  */
+          pVal = sqlite3_column_value(p->pSavedRow, i-1);
+          if( pConfig->bLocale && bUnindexed==0 ){
+            sqlite3_bind_value(pInsert, pConfig->nCol + i,
+                sqlite3_column_value(p->pSavedRow, pConfig->nCol + i - 1)
+            );
+          }
+        }else if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
+          const char *pText = 0;
+          const char *pLoc = 0;
+          int nText = 0;
+          int nLoc = 0;
+          assert( pConfig->bLocale );
+
+          rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
+          if( rc==SQLITE_OK ){
+            sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT);
+            if( bUnindexed==0 ){
+              int iLoc = pConfig->nCol + i;
+              sqlite3_bind_text(pInsert, iLoc, pLoc, nLoc, SQLITE_TRANSIENT);
+            }
+          }
+
+          continue;
+        }
+
+        rc = sqlite3_bind_value(pInsert, i, pVal);
+      }
    }
    if( rc==SQLITE_OK ){
      sqlite3_step(pInsert);
@@ -810,14 +1036,38 @@ int sqlite3Fts5StorageIndexInsert(
  for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
    ctx.szCol = 0;
    if( pConfig->abUnindexed[ctx.iCol]==0 ){
-      const char *zText = (const char*)sqlite3_value_text(apVal[ctx.iCol+2]);
-      int nText = sqlite3_value_bytes(apVal[ctx.iCol+2]);
-      rc = sqlite3Fts5Tokenize(pConfig, 
-          FTS5_TOKENIZE_DOCUMENT,
-          zText, nText,
-          (void*)&ctx,
-          fts5StorageInsertCallback
-      );
+      int nText = 0;              /* Size of pText in bytes */
+      const char *pText = 0;      /* Pointer to buffer containing text value */
+      int nLoc = 0;               /* Size of pText in bytes */
+      const char *pLoc = 0;       /* Pointer to buffer containing text value */
+
+      sqlite3_value *pVal = apVal[ctx.iCol+2];
+      if( p->pSavedRow && sqlite3_value_nochange(pVal) ){
+        pVal = sqlite3_column_value(p->pSavedRow, ctx.iCol+1);
+        if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){
+          int iCol = ctx.iCol + 1 + pConfig->nCol;
+          pLoc = (const char*)sqlite3_column_text(p->pSavedRow, iCol);
+          nLoc = sqlite3_column_bytes(p->pSavedRow, iCol);
+        }
+      }else{
+        pVal = apVal[ctx.iCol+2];
+      }
+
+      if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){
+        rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc);
+      }else{
+        pText = (const char*)sqlite3_value_text(pVal);
+        nText = sqlite3_value_bytes(pVal);
+      }
+
+      if( rc==SQLITE_OK ){
+        sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
+        rc = sqlite3Fts5Tokenize(pConfig, 
+            FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx,
+            fts5StorageInsertCallback
+        );
+        sqlite3Fts5ClearLocale(pConfig);
+      }
    }
    sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
    p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
@@ -981,29 +1231,61 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){
          rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
        }
        for(i=0; rc==SQLITE_OK && i<pConfig->nCol; i++){
-          if( pConfig->abUnindexed[i] ) continue;
-          ctx.iCol = i;
-          ctx.szCol = 0;
-          if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
-            rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
-          }
-          if( rc==SQLITE_OK ){
-            const char *zText = (const char*)sqlite3_column_text(pScan, i+1);
-            int nText = sqlite3_column_bytes(pScan, i+1);
-            rc = sqlite3Fts5Tokenize(pConfig, 
-                FTS5_TOKENIZE_DOCUMENT,
-                zText, nText,
-                (void*)&ctx,
-                fts5StorageIntegrityCallback
-            );
-          }
-          if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
-            rc = FTS5_CORRUPT;
-          }
-          aTotalSize[i] += ctx.szCol;
-          if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
-            sqlite3Fts5TermsetFree(ctx.pTermset);
-            ctx.pTermset = 0;
+          if( pConfig->abUnindexed[i]==0 ){
+            const char *pText = 0;
+            int nText = 0;
+            const char *pLoc = 0;
+            int nLoc = 0;
+            sqlite3_value *pVal = sqlite3_column_value(pScan, i+1);
+
+            if( pConfig->eContent==FTS5_CONTENT_EXTERNAL 
+             && sqlite3Fts5IsLocaleValue(pConfig, pVal)
+            ){
+              rc = sqlite3Fts5DecodeLocaleValue(
+                  pVal, &pText, &nText, &pLoc, &nLoc
+              );
+            }else{
+              if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){
+                int iCol = i + 1 + pConfig->nCol;
+                pLoc = (const char*)sqlite3_column_text(pScan, iCol);
+                nLoc = sqlite3_column_bytes(pScan, iCol);
+              }
+              pText = (const char*)sqlite3_value_text(pVal);
+              nText = sqlite3_value_bytes(pVal);
+            }
+
+            ctx.iCol = i;
+            ctx.szCol = 0;
+
+            if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
+              rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
+            }
+
+            if( rc==SQLITE_OK ){
+              sqlite3Fts5SetLocale(pConfig, pLoc, nLoc);
+              rc = sqlite3Fts5Tokenize(pConfig, 
+                  FTS5_TOKENIZE_DOCUMENT,
+                  pText, nText,
+                  (void*)&ctx,
+                  fts5StorageIntegrityCallback
+              );
+              sqlite3Fts5ClearLocale(pConfig);
+            }
+
+            /* If this is not a columnsize=0 database, check that the number
+            ** of tokens in the value matches the aColSize[] value read from
+            ** the %_docsize table.  */
+            if( rc==SQLITE_OK 
+             && pConfig->bColumnsize 
+             && ctx.szCol!=aColSize[i] 
+            ){
+              rc = FTS5_CORRUPT;
+            }
+            aTotalSize[i] += ctx.szCol;
+            if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){
+              sqlite3Fts5TermsetFree(ctx.pTermset);
+              ctx.pTermset = 0;
+            }
          }
        }
        sqlite3Fts5TermsetFree(ctx.pTermset);
@@ -14,14 +14,7 @@


 #ifdef SQLITE_TEST
-#if defined(INCLUDE_SQLITE_TCL_H)
-#  include "sqlite_tcl.h"
-#else
-#  include "tcl.h"
-#  ifndef SQLITE_TCLAPI
-#    define SQLITE_TCLAPI
-#  endif
-#endif
+#include "tclsqlite.h"

 #ifdef SQLITE_ENABLE_FTS5

@@ -103,14 +96,14 @@ static int SQLITE_TCLAPI f5tDbAndApi(

    rc = sqlite3_prepare_v2(db, "SELECT fts5(?1)", -1, &pStmt, 0);
    if( rc!=SQLITE_OK ){
-      Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
+      Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0);
      return TCL_ERROR;
    }
    sqlite3_bind_pointer(pStmt, 1, (void*)&pApi, "fts5_api_ptr", 0);
    sqlite3_step(pStmt);

    if( sqlite3_finalize(pStmt)!=SQLITE_OK ){
-      Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
+      Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0);
      return TCL_ERROR;
    }

@@ -247,6 +240,7 @@ static int SQLITE_TCLAPI xF5tApi(

    { "xQueryToken",       2, "IPHRASE ITERM" },      /* 18 */
    { "xInstToken",        2, "IDX ITERM" },          /* 19 */
+    { "xColumnLocale",     1, "COL" },                /* 20 */
    { 0, 0, 0}
  };

@@ -297,12 +291,12 @@ static int SQLITE_TCLAPI xF5tApi(
      break;
    }
    CASE(3, "xTokenize") {
-      int nText;
+      Tcl_Size nText;
      char *zText = Tcl_GetStringFromObj(objv[2], &nText);
      F5tFunction ctx;
      ctx.interp = interp;
      ctx.pScript = objv[3];
-      rc = p->pApi->xTokenize(p->pFts, zText, nText, &ctx, xTokenizeCb);
+      rc = p->pApi->xTokenize(p->pFts, zText, (int)nText, &ctx, xTokenizeCb);
      if( rc==SQLITE_OK ){
        Tcl_ResetResult(interp);
      }
@@ -398,7 +392,7 @@ static int SQLITE_TCLAPI xF5tApi(
    CASE(12, "xSetAuxdata") {
      F5tAuxData *pData = (F5tAuxData*)sqlite3_malloc(sizeof(F5tAuxData));
      if( pData==0 ){
-        Tcl_AppendResult(interp, "out of memory", 0);
+        Tcl_AppendResult(interp, "out of memory", (char*)0);
        return TCL_ERROR;
      }
      pData->pObj = objv[2];
@@ -458,7 +452,7 @@ static int SQLITE_TCLAPI xF5tApi(

      rc = p->pApi->xPhraseFirst(p->pFts, iPhrase, &iter, &iCol, &iOff);
      if( rc!=SQLITE_OK ){
-        Tcl_AppendResult(interp, sqlite3ErrName(rc), 0);
+        Tcl_AppendResult(interp, sqlite3ErrName(rc), (char*)0);
        return TCL_ERROR;
      }
      for( ;iCol>=0; p->pApi->xPhraseNext(p->pFts, &iter, &iCol, &iOff) ){
@@ -535,6 +529,20 @@ static int SQLITE_TCLAPI xF5tApi(
      break;
    }

+    CASE(20, "xColumnLocale") {
+      const char *z = 0;
+      int n = 0;
+      int iCol;
+      if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){
+        return TCL_ERROR;
+      }
+      rc = p->pApi->xColumnLocale(p->pFts, iCol, &z, &n);
+      if( rc==SQLITE_OK && z ){
+        Tcl_SetObjResult(interp, Tcl_NewStringObj(z, n));
+      }
+      break;
+    }
+
    default: 
      assert( 0 );
      break;
@@ -605,15 +613,16 @@ static void xF5tFunction(
    sqlite3_result_error(pCtx, Tcl_GetStringResult(p->interp), -1);
  }else{
    Tcl_Obj *pVar = Tcl_GetObjResult(p->interp);
-    int n;
    const char *zType = (pVar->typePtr ? pVar->typePtr->name : "");
    char c = zType[0];
    if( c=='b' && strcmp(zType,"bytearray")==0 && pVar->bytes==0 ){
      /* Only return a BLOB type if the Tcl variable is a bytearray and
      ** has no string representation. */
-      unsigned char *data = Tcl_GetByteArrayFromObj(pVar, &n);
-      sqlite3_result_blob(pCtx, data, n, SQLITE_TRANSIENT);
+      Tcl_Size nn;
+      unsigned char *data = Tcl_GetByteArrayFromObj(pVar, &nn);
+      sqlite3_result_blob(pCtx, data, (int)nn, SQLITE_TRANSIENT);
    }else if( c=='b' && strcmp(zType,"boolean")==0 ){
+      int n;
      Tcl_GetIntFromObj(0, pVar, &n);
      sqlite3_result_int(pCtx, n);
    }else if( c=='d' && strcmp(zType,"double")==0 ){
@@ -626,8 +635,9 @@ static void xF5tFunction(
      Tcl_GetWideIntFromObj(0, pVar, &v);
      sqlite3_result_int64(pCtx, v);
    }else{
-      unsigned char *data = (unsigned char *)Tcl_GetStringFromObj(pVar, &n);
-      sqlite3_result_text(pCtx, (char *)data, n, SQLITE_TRANSIENT);
+      Tcl_Size nn;
+      unsigned char *data = (unsigned char *)Tcl_GetStringFromObj(pVar, &nn);
+      sqlite3_result_text(pCtx, (char*)data, (int)nn, SQLITE_TRANSIENT);
    }
  }
 }
@@ -673,7 +683,7 @@ static int SQLITE_TCLAPI f5tCreateFunction(
      pApi, zName, (void*)pCtx, xF5tFunction, xF5tDestroy
  );
  if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
+    Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (char*)0);
    return TCL_ERROR;
  }

@@ -720,7 +730,7 @@ static int SQLITE_TCLAPI f5tTokenize(
  Tcl_Obj *CONST objv[]
 ){
  char *zText;
-  int nText;
+  Tcl_Size nText;
  sqlite3 *db = 0;
  fts5_api *pApi = 0;
  Fts5Tokenizer *pTok = 0;
@@ -729,7 +739,7 @@ static int SQLITE_TCLAPI f5tTokenize(
  void *pUserdata;
  int rc;

-  int nArg;
+  Tcl_Size nArg;
  const char **azArg;
  F5tTokenizeCtx ctx;

@@ -740,7 +750,7 @@ static int SQLITE_TCLAPI f5tTokenize(
  if( objc==5 ){
    char *zOpt = Tcl_GetString(objv[1]);
    if( strcmp("-subst", zOpt) ){
-      Tcl_AppendResult(interp, "unrecognized option: ", zOpt, 0);
+      Tcl_AppendResult(interp, "unrecognized option: ", zOpt, (char*)0);
      return TCL_ERROR;
    }
  }
@@ -749,7 +759,7 @@ static int SQLITE_TCLAPI f5tTokenize(
    return TCL_ERROR;
  }
  if( nArg==0 ){
-    Tcl_AppendResult(interp, "no such tokenizer: ", 0);
+    Tcl_AppendResult(interp, "no such tokenizer: ", (char*)0);
    Tcl_Free((void*)azArg);
    return TCL_ERROR;
  }
@@ -757,13 +767,13 @@ static int SQLITE_TCLAPI f5tTokenize(

  rc = pApi->xFindTokenizer(pApi, azArg[0], &pUserdata, &tokenizer);
  if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, "no such tokenizer: ", azArg[0], 0);
+    Tcl_AppendResult(interp, "no such tokenizer: ", azArg[0], (char*)0);
    return TCL_ERROR;
  }

-  rc = tokenizer.xCreate(pUserdata, &azArg[1], nArg-1, &pTok);
+  rc = tokenizer.xCreate(pUserdata, &azArg[1], (int)(nArg-1), &pTok);
  if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, "error in tokenizer.xCreate()", 0);
+    Tcl_AppendResult(interp, "error in tokenizer.xCreate()", (char*)0);
    return TCL_ERROR;
  }

@@ -773,11 +783,11 @@ static int SQLITE_TCLAPI f5tTokenize(
  ctx.pRet = pRet;
  ctx.zInput = zText;
  rc = tokenizer.xTokenize(
-      pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, zText, nText, xTokenizeCb2
+      pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, zText,(int)nText, xTokenizeCb2
  );
  tokenizer.xDelete(pTok);
  if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", 0);
+    Tcl_AppendResult(interp, "error in tokenizer.xTokenize()", (char*)0);
    Tcl_DecrRefCount(pRet);
    return TCL_ERROR;
  }
@@ -801,18 +811,32 @@ typedef struct F5tTokenizerInstance F5tTokenizerInstance;
 struct F5tTokenizerContext {
  void *pCtx;
  int (*xToken)(void*, int, const char*, int, int, int);
+  F5tTokenizerInstance *pInst;
 };

 struct F5tTokenizerModule {
  Tcl_Interp *interp;
  Tcl_Obj *pScript;
+  void *pParentCtx;
+  fts5_tokenizer_v2 parent_v2;
+  fts5_tokenizer parent;
  F5tTokenizerContext *pContext;
 };

+/*
+** zLocale:
+**   Within a call to xTokenize_v2(), pLocale/nLocale store the locale
+**   passed to the call by fts5. This can be retrieved by a Tcl tokenize 
+**   script using [sqlite3_fts5_locale].
+*/
 struct F5tTokenizerInstance {
  Tcl_Interp *interp;
  Tcl_Obj *pScript;
+  F5tTokenizerModule *pModule;
+  Fts5Tokenizer *pParent;
  F5tTokenizerContext *pContext;
+  const char *pLocale;
+  int nLocale;
 };

 static int f5tTokenizerCreate(
@@ -821,11 +845,20 @@ static int f5tTokenizerCreate(
  int nArg, 
  Fts5Tokenizer **ppOut
 ){
+  Fts5Tokenizer *pParent = 0;
  F5tTokenizerModule *pMod = (F5tTokenizerModule*)pCtx;
  Tcl_Obj *pEval;
  int rc = TCL_OK;
  int i;

+  assert( pMod->parent_v2.xCreate==0 || pMod->parent.xCreate==0 );
+  if( pMod->parent_v2.xCreate ){
+    rc = pMod->parent_v2.xCreate(pMod->pParentCtx, 0, 0, &pParent);
+  }
+  if( pMod->parent.xCreate ){
+    rc = pMod->parent.xCreate(pMod->pParentCtx, 0, 0, &pParent);
+  }
+
  pEval = Tcl_DuplicateObj(pMod->pScript);
  Tcl_IncrRefCount(pEval);
  for(i=0; rc==TCL_OK && i<nArg; i++){
@@ -845,6 +878,8 @@ static int f5tTokenizerCreate(
    pInst->interp = pMod->interp;
    pInst->pScript = Tcl_GetObjResult(pMod->interp);
    pInst->pContext = pMod->pContext;
+    pInst->pParent = pParent;
+    pInst->pModule = pMod;
    Tcl_IncrRefCount(pInst->pScript);
    *ppOut = (Fts5Tokenizer*)pInst;
  }
@@ -855,11 +890,21 @@ static int f5tTokenizerCreate(

 static void f5tTokenizerDelete(Fts5Tokenizer *p){
  F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
-  Tcl_DecrRefCount(pInst->pScript);
-  ckfree((char *)pInst);
+  if( pInst ){
+    if( pInst->pParent ){
+      if( pInst->pModule->parent_v2.xDelete ){
+        pInst->pModule->parent_v2.xDelete(pInst->pParent);
+      }else{
+        pInst->pModule->parent.xDelete(pInst->pParent);
+      }
+    }
+    Tcl_DecrRefCount(pInst->pScript);
+    ckfree((char *)pInst);
+  }
 }

-static int f5tTokenizerTokenize(
+
+static int f5tTokenizerReallyTokenize(
  Fts5Tokenizer *p, 
  void *pCtx,
  int flags,
@@ -867,6 +912,7 @@ static int f5tTokenizerTokenize(
  int (*xToken)(void*, int, const char*, int, int, int)
 ){
  F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
+  F5tTokenizerInstance *pOldInst = 0;
  void *pOldCtx;
  int (*xOldToken)(void*, int, const char*, int, int, int);
  Tcl_Obj *pEval;
@@ -875,9 +921,11 @@ static int f5tTokenizerTokenize(

  pOldCtx = pInst->pContext->pCtx;
  xOldToken = pInst->pContext->xToken;
+  pOldInst = pInst->pContext->pInst;

  pInst->pContext->pCtx = pCtx;
  pInst->pContext->xToken = xToken;
+  pInst->pContext->pInst = pInst;

  assert( 
      flags==FTS5_TOKENIZE_DOCUMENT
@@ -913,9 +961,105 @@ static int f5tTokenizerTokenize(

  pInst->pContext->pCtx = pOldCtx;
  pInst->pContext->xToken = xOldToken;
+  pInst->pContext->pInst = pOldInst;
  return rc;
 }

+typedef struct CallbackCtx CallbackCtx;
+struct CallbackCtx {
+  Fts5Tokenizer *p;
+  void *pCtx;
+  int flags;
+  int (*xToken)(void*, int, const char*, int, int, int);
+};
+
+static int f5tTokenizeCallback(
+  void *pCtx, 
+  int tflags, 
+  const char *z, int n, 
+  int iStart, int iEnd
+){
+  CallbackCtx *p = (CallbackCtx*)pCtx;
+  return f5tTokenizerReallyTokenize(p->p, p->pCtx, p->flags, z, n, p->xToken);
+}
+
+static int f5tTokenizerTokenize_v2(
+  Fts5Tokenizer *p, 
+  void *pCtx,
+  int flags,
+  const char *pText, int nText, 
+  const char *pLoc, int nLoc, 
+  int (*xToken)(void*, int, const char*, int, int, int)
+){
+  int rc = SQLITE_OK;
+  F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
+
+  pInst->pLocale = pLoc;
+  pInst->nLocale = nLoc;
+
+  if( pInst->pParent ){
+    CallbackCtx ctx;
+    ctx.p = p;
+    ctx.pCtx = pCtx;
+    ctx.flags = flags;
+    ctx.xToken = xToken;
+    if( pInst->pModule->parent_v2.xTokenize ){
+      rc = pInst->pModule->parent_v2.xTokenize(
+          pInst->pParent, (void*)&ctx, flags, pText, nText, 
+          pLoc, nLoc, f5tTokenizeCallback
+      );
+    }else{
+      rc = pInst->pModule->parent.xTokenize(
+          pInst->pParent, (void*)&ctx, flags, pText, nText, f5tTokenizeCallback
+      );
+    }
+  }else{
+    rc = f5tTokenizerReallyTokenize(p, pCtx, flags, pText, nText, xToken);
+  }
+
+  pInst->pLocale = 0;
+  pInst->nLocale = 0;
+  return rc;
+}
+static int f5tTokenizerTokenize(
+  Fts5Tokenizer *p, 
+  void *pCtx,
+  int flags,
+  const char *pText, int nText, 
+  int (*xToken)(void*, int, const char*, int, int, int)
+){
+  return f5tTokenizerTokenize_v2(p, pCtx, flags, pText, nText, 0, 0, xToken);
+}
+
+/*
+** sqlite3_fts5_locale 
+*/
+static int SQLITE_TCLAPI f5tTokenizerLocale(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  F5tTokenizerContext *p = (F5tTokenizerContext*)clientData;
+
+  if( objc!=1 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "");
+    return TCL_ERROR;
+  }
+
+  if( p->xToken==0 ){
+    Tcl_AppendResult(interp, 
+        "sqlite3_fts5_locale may only be used by tokenizer callback", (char*)0
+    );
+    return TCL_ERROR;
+  }
+
+  Tcl_SetObjResult(interp, 
+      Tcl_NewStringObj(p->pInst->pLocale, p->pInst->nLocale)
+  );
+  return TCL_OK;
+}
+
 /*
 ** sqlite3_fts5_token ?-colocated? TEXT START END
 */
@@ -928,13 +1072,13 @@ static int SQLITE_TCLAPI f5tTokenizerReturn(
  F5tTokenizerContext *p = (F5tTokenizerContext*)clientData;
  int iStart;
  int iEnd;
-  int nToken;
+  Tcl_Size nToken;
  int tflags = 0;
  char *zToken;
  int rc;

  if( objc==5 ){
-    int nArg;
+    Tcl_Size nArg;
    char *zArg = Tcl_GetStringFromObj(objv[1], &nArg);
    if( nArg<=10 && nArg>=2 && memcmp("-colocated", zArg, nArg)==0 ){
      tflags |= FTS5_TOKEN_COLOCATED;
@@ -954,12 +1098,12 @@ static int SQLITE_TCLAPI f5tTokenizerReturn(

  if( p->xToken==0 ){
    Tcl_AppendResult(interp, 
-        "sqlite3_fts5_token may only be used by tokenizer callback", 0
+        "sqlite3_fts5_token may only be used by tokenizer callback", (char*)0
    );
    return TCL_ERROR;
  }

-  rc = p->xToken(p->pCtx, tflags, zToken, nToken, iStart, iEnd);
+  rc = p->xToken(p->pCtx, tflags, zToken, (int)nToken, iStart, iEnd);
  Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
  return rc==SQLITE_OK ? TCL_OK : TCL_ERROR;

@@ -1001,32 +1145,112 @@ static int SQLITE_TCLAPI f5tCreateTokenizer(
  fts5_api *pApi;
  char *zName;
  Tcl_Obj *pScript;
-  fts5_tokenizer t;
  F5tTokenizerModule *pMod;
-  int rc;
+  int rc = SQLITE_OK;
+  int bV2 = 0;                    /* True to use _v2 API */
+  int iVersion = 2;               /* Value for _v2.iVersion */
+  const char *zParent = 0;        /* Name of parent tokenizer, if any */
+  int ii = 0;

-  if( objc!=4 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "DB NAME SCRIPT");
+  if( objc<4 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "?OPTIONS? DB NAME SCRIPT");
    return TCL_ERROR;
  }
-  if( f5tDbAndApi(interp, objv[1], &db, &pApi) ){
+
+  /* Parse any options. Set stack variables bV2 and zParent. */
+  for(ii=1; ii<objc-3; ii++){
+    int iOpt = 0;
+    const char *azOpt[] = { "-v2", "-parent", "-version", 0 };
+    if( Tcl_GetIndexFromObj(interp, objv[ii], azOpt, "OPTION", 0, &iOpt) ){
+      return TCL_ERROR;
+    }
+    switch( iOpt ){
+      case 0: /* -v2 */ {
+        bV2 = 1;
+        break;
+      }
+      case 1: /* -parent */ {
+        ii++;
+        if( ii==objc-3 ){
+          Tcl_AppendResult(
+              interp, "option requires an argument: -parent", (char*)0
+          );
+          return TCL_ERROR;
+        }
+        zParent = Tcl_GetString(objv[ii]);
+        break;
+      }
+      case 2: /* -version */ {
+        ii++;
+        if( ii==objc-3 ){
+          Tcl_AppendResult(
+              interp, "option requires an argument: -version", (char*)0
+          );
+          return TCL_ERROR;
+        }
+        if( Tcl_GetIntFromObj(interp, objv[ii], &iVersion) ){
+          return TCL_ERROR;
+        }
+        break;
+      }
+      default:
+        assert( 0 );
+        break;
+    }
+  }
+
+  if( f5tDbAndApi(interp, objv[objc-3], &db, &pApi) ){
    return TCL_ERROR;
  }
-  zName = Tcl_GetString(objv[2]);
-  pScript = objv[3];
-
-  t.xCreate = f5tTokenizerCreate;
-  t.xTokenize = f5tTokenizerTokenize;
-  t.xDelete = f5tTokenizerDelete;
+  zName = Tcl_GetString(objv[objc-2]);
+  pScript = objv[objc-1];

  pMod = (F5tTokenizerModule*)ckalloc(sizeof(F5tTokenizerModule));
+  memset(pMod, 0, sizeof(F5tTokenizerModule));
  pMod->interp = interp;
  pMod->pScript = pScript;
-  pMod->pContext = pContext;
  Tcl_IncrRefCount(pScript);
-  rc = pApi->xCreateTokenizer(pApi, zName, (void*)pMod, &t, f5tDelTokenizer);
+  pMod->pContext = pContext;
+  if( zParent ){
+    if( bV2 ){
+      fts5_tokenizer_v2 *pParent = 0;
+      rc = pApi->xFindTokenizer_v2(pApi, zParent, &pMod->pParentCtx, &pParent);
+      if( rc==SQLITE_OK ){
+        memcpy(&pMod->parent_v2, pParent, sizeof(fts5_tokenizer_v2));
+        pMod->parent_v2.xDelete(0);
+      }
+    }else{
+      rc = pApi->xFindTokenizer(pApi, zParent, &pMod->pParentCtx,&pMod->parent);
+      if( rc==SQLITE_OK ){
+        pMod->parent.xDelete(0);
+      }
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    void *pModCtx = (void*)pMod;
+    if( bV2==0 ){
+      fts5_tokenizer t;
+      t.xCreate = f5tTokenizerCreate;
+      t.xTokenize = f5tTokenizerTokenize;
+      t.xDelete = f5tTokenizerDelete;
+      rc = pApi->xCreateTokenizer(pApi, zName, pModCtx, &t, f5tDelTokenizer);
+    }else{
+      fts5_tokenizer_v2 t2;
+      memset(&t2, 0, sizeof(t2));
+      t2.iVersion = iVersion;
+      t2.xCreate = f5tTokenizerCreate;
+      t2.xTokenize = f5tTokenizerTokenize_v2;
+      t2.xDelete = f5tTokenizerDelete;
+      rc = pApi->xCreateTokenizer_v2(pApi, zName, pModCtx, &t2,f5tDelTokenizer);
+    }
+  }
+
  if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, "error in fts5_api.xCreateTokenizer()", 0);
+    Tcl_AppendResult(interp, (
+          bV2 ? "error in fts5_api.xCreateTokenizer_v2()"
+          : "error in fts5_api.xCreateTokenizer()"
+    ), (char*)0);
    return TCL_ERROR;
  }

@@ -1083,7 +1307,7 @@ static int SQLITE_TCLAPI f5tTokenHash(
  Tcl_Obj *CONST objv[]
 ){
  char *z;
-  int n;
+  Tcl_Size n;
  unsigned int iVal;
  int nSlot;

@@ -1096,7 +1320,7 @@ static int SQLITE_TCLAPI f5tTokenHash(
  }
  z = Tcl_GetStringFromObj(objv[2], &n);

-  iVal = f5t_fts5HashKey(nSlot, z, n);
+  iVal = f5t_fts5HashKey(nSlot, z, (int)n);
  Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
  return TCL_OK;
 }
@@ -1316,12 +1540,92 @@ static int SQLITE_TCLAPI f5tRegisterOriginText(

  Tcl_ResetResult(interp);
  if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
+    Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (void*)0);
    return TCL_ERROR;
  }
  return TCL_OK;
 }

+/*
+** This function is used to DROP an fts5 table. It works even if the data
+** structures fts5 stores within the database are corrupt, which sometimes
+** prevents a straight "DROP TABLE" command from succeeding.
+**
+** The first parameter is the database handle to use for the DROP TABLE
+** operation. The second is the name of the database to drop the fts5 table
+** from (i.e. "main", "temp" or the name of an attached database). The
+** third parameter is the name of the fts5 table to drop.
+**
+** SQLITE_OK is returned if the table is successfully dropped. Or, if an
+** error occurs, an SQLite error code.
+*/
+static int sqlite3_fts5_drop_corrupt_table(
+  sqlite3 *db,                    /* Database handle */
+  const char *zDb,                /* Database name ("main", "temp" etc.) */
+  const char *zTab                /* Name of fts5 table to drop */
+){
+  int rc = SQLITE_OK;
+  int bDef = 0;
+
+  rc = sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, -1, &bDef);
+  if( rc==SQLITE_OK ){
+    char *zScript = sqlite3_mprintf(
+        "DELETE FROM %Q.'%q_data';"
+        "DELETE FROM %Q.'%q_config';"
+        "INSERT INTO %Q.'%q_data' VALUES(10, X'0000000000');"
+        "INSERT INTO %Q.'%q_config' VALUES('version', 4);"
+        "DROP TABLE %Q.'%q';",
+        zDb, zTab, zDb, zTab, zDb, zTab, zDb, zTab, zDb, zTab
+    );
+
+    if( zScript==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      if( bDef ) sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, 0, 0);
+      rc = sqlite3_exec(db, zScript, 0, 0, 0);
+      if( bDef ) sqlite3_db_config(db, SQLITE_DBCONFIG_DEFENSIVE, 1, 0);
+      sqlite3_free(zScript);
+    }
+  }
+
+  return rc;
+}
+
+/*
+**      sqlite3_fts5_drop_corrupt_table DB DATABASE TABLE
+**
+** Description...
+*/
+static int SQLITE_TCLAPI f5tDropCorruptTable(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  sqlite3 *db = 0;
+  const char *zDb = 0;
+  const char *zTab = 0;
+  int rc = SQLITE_OK;
+
+  if( objc!=4 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB DATABASE TABLE");
+    return TCL_ERROR;
+  }
+  if( f5tDbPointer(interp, objv[1], &db) ){
+    return TCL_ERROR;
+  }
+  zDb = Tcl_GetString(objv[2]);
+  zTab = Tcl_GetString(objv[3]);
+
+  rc = sqlite3_fts5_drop_corrupt_table(db, zDb, zTab);
+  if( rc!=SQLITE_OK ){
+    Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), (void*)0);
+    return TCL_ERROR;
+  }
+
+  return TCL_OK;
+}
+
 /*
 ** Entry point.
 */
@@ -1333,13 +1637,15 @@ int Fts5tcl_Init(Tcl_Interp *interp){
  } aCmd[] = {
    { "sqlite3_fts5_create_tokenizer",   f5tCreateTokenizer, 1 },
    { "sqlite3_fts5_token",              f5tTokenizerReturn, 1 },
+    { "sqlite3_fts5_locale",             f5tTokenizerLocale, 1 },
    { "sqlite3_fts5_tokenize",           f5tTokenize, 0 },
    { "sqlite3_fts5_create_function",    f5tCreateFunction, 0 },
    { "sqlite3_fts5_may_be_corrupt",     f5tMayBeCorrupt, 0 },
    { "sqlite3_fts5_token_hash",         f5tTokenHash, 0 },
    { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 },
    { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 },
-    { "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 }
+    { "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 },
+    { "sqlite3_fts5_drop_corrupt_table", f5tDropCorruptTable, 0 }
  };
  int i;
  F5tTokenizerContext *pContext;
@@ -198,7 +198,7 @@ static const unsigned char sqlite3Utf8Trans1[] = {
  c = *(zIn++);                                            \
  if( c>=0xc0 ){                                           \
    c = sqlite3Utf8Trans1[c-0xc0];                         \
-    while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){            \
+    while( zIn<zTerm && (*zIn & 0xc0)==0x80 ){             \
      c = (c<<6) + (0x3f & *(zIn++));                      \
    }                                                      \
    if( c<0x80                                             \
@@ -386,7 +386,6 @@ static int fts5UnicodeCreate(
          zCat = azArg[i+1];
        }
      }
-
      if( rc==SQLITE_OK ){
        rc = unicodeSetCategories(p, zCat);
      }
@@ -416,7 +415,6 @@ static int fts5UnicodeCreate(
          rc = SQLITE_ERROR;
        }
      }
-
    }else{
      rc = SQLITE_NOMEM;
    }
@@ -555,7 +553,7 @@ static int fts5UnicodeTokenize(

 typedef struct PorterTokenizer PorterTokenizer;
 struct PorterTokenizer {
-  fts5_tokenizer tokenizer;       /* Parent tokenizer module */
+  fts5_tokenizer_v2 tokenizer_v2; /* Parent tokenizer module */
  Fts5Tokenizer *pTokenizer;      /* Parent tokenizer instance */
  char aBuf[FTS5_PORTER_MAX_TOKEN + 64];
 };
@@ -567,7 +565,7 @@ static void fts5PorterDelete(Fts5Tokenizer *pTok){
  if( pTok ){
    PorterTokenizer *p = (PorterTokenizer*)pTok;
    if( p->pTokenizer ){
-      p->tokenizer.xDelete(p->pTokenizer);
+      p->tokenizer_v2.xDelete(p->pTokenizer);
    }
    sqlite3_free(p);
  }
@@ -586,6 +584,7 @@ static int fts5PorterCreate(
  PorterTokenizer *pRet;
  void *pUserdata = 0;
  const char *zBase = "unicode61";
+  fts5_tokenizer_v2 *pV2 = 0;

  if( nArg>0 ){
    zBase = azArg[0];
@@ -594,14 +593,15 @@ static int fts5PorterCreate(
  pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer));
  if( pRet ){
    memset(pRet, 0, sizeof(PorterTokenizer));
-    rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer);
+    rc = pApi->xFindTokenizer_v2(pApi, zBase, &pUserdata, &pV2);
  }else{
    rc = SQLITE_NOMEM;
  }
  if( rc==SQLITE_OK ){
    int nArg2 = (nArg>0 ? nArg-1 : 0);
-    const char **azArg2 = (nArg2 ? &azArg[1] : 0);
-    rc = pRet->tokenizer.xCreate(pUserdata, azArg2, nArg2, &pRet->pTokenizer);
+    const char **az2 = (nArg2 ? &azArg[1] : 0);
+    memcpy(&pRet->tokenizer_v2, pV2, sizeof(fts5_tokenizer_v2));
+    rc = pRet->tokenizer_v2.xCreate(pUserdata, az2, nArg2, &pRet->pTokenizer);
  }

  if( rc!=SQLITE_OK ){
@@ -1252,6 +1252,7 @@ static int fts5PorterTokenize(
  void *pCtx,
  int flags,
  const char *pText, int nText,
+  const char *pLoc, int nLoc,
  int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
 ){
  PorterTokenizer *p = (PorterTokenizer*)pTokenizer;
@@ -1259,8 +1260,8 @@ static int fts5PorterTokenize(
  sCtx.xToken = xToken;
  sCtx.pCtx = pCtx;
  sCtx.aBuf = p->aBuf;
-  return p->tokenizer.xTokenize(
-      p->pTokenizer, (void*)&sCtx, flags, pText, nText, fts5PorterCb
+  return p->tokenizer_v2.xTokenize(
+      p->pTokenizer, (void*)&sCtx, flags, pText, nText, pLoc, nLoc, fts5PorterCb
  );
 }

@@ -1290,40 +1291,46 @@ static int fts5TriCreate(
  Fts5Tokenizer **ppOut
 ){
  int rc = SQLITE_OK;
-  TrigramTokenizer *pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
+  TrigramTokenizer *pNew = 0;
  UNUSED_PARAM(pUnused);
-  if( pNew==0 ){
-    rc = SQLITE_NOMEM;
+  if( nArg%2 ){
+    rc = SQLITE_ERROR;
  }else{
    int i;
-    pNew->bFold = 1;
-    pNew->iFoldParam = 0;
-    for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
-      const char *zArg = azArg[i+1];
-      if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
-        if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
-          rc = SQLITE_ERROR;
+    pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
+    if( pNew==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      pNew->bFold = 1;
+      pNew->iFoldParam = 0;
+  
+      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
+        const char *zArg = azArg[i+1];
+        if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
+          if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
+            rc = SQLITE_ERROR;
+          }else{
+            pNew->bFold = (zArg[0]=='0');
+          }
+        }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
+          if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
+            rc = SQLITE_ERROR;
+          }else{
+            pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
+          }
        }else{
-          pNew->bFold = (zArg[0]=='0');
-        }
-      }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
-        if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
          rc = SQLITE_ERROR;
-        }else{
-          pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
        }
-      }else{
+      }
+  
+      if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
        rc = SQLITE_ERROR;
      }
-    }
-
-    if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
-      rc = SQLITE_ERROR;
-    }
-
-    if( rc!=SQLITE_OK ){
-      fts5TriDelete((Fts5Tokenizer*)pNew);
-      pNew = 0;
+  
+      if( rc!=SQLITE_OK ){
+        fts5TriDelete((Fts5Tokenizer*)pNew);
+        pNew = 0;
+      }
    }
  }
  *ppOut = (Fts5Tokenizer*)pNew;
@@ -1428,6 +1435,16 @@ int sqlite3Fts5TokenizerPattern(
  return FTS5_PATTERN_NONE;
 }

+/*
+** Return true if the tokenizer described by p->azArg[] is the trigram
+** tokenizer. This tokenizer needs to be loaded before xBestIndex is
+** called for the first time in order to correctly handle LIKE/GLOB.
+*/
+int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig *p){
+  return (p->nArg>=1 && 0==sqlite3_stricmp(p->azArg[0], "trigram"));
+}
+
+
 /*
 ** Register all built-in tokenizers with FTS5.
 */
@@ -1438,7 +1455,6 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){
  } aBuiltin[] = {
    { "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}},
    { "ascii",     {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }},
-    { "porter",    {fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize }},
    { "trigram",   {fts5TriCreate, fts5TriDelete, fts5TriTokenize}},
  };
  
@@ -1453,6 +1469,19 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){
        0
    );
  }
-
+  if( rc==SQLITE_OK ){
+    fts5_tokenizer_v2 sPorter = {
+      2,
+      fts5PorterCreate,
+      fts5PorterDelete,
+      fts5PorterTokenize
+    };
+    rc = pApi->xCreateTokenizer_v2(pApi,
+        "porter",
+        (void*)pApi,
+        &sPorter,
+        0
+    );
+  }
  return rc;
 }
@@ -364,6 +364,9 @@ int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){
            default: return 1;          }
          break;

+
+    default:
+      return 1;
  }
  return 0;
 }
@@ -64,6 +64,7 @@ struct Fts5VocabCursor {

  int nLeTerm;                    /* Size of zLeTerm in bytes */
  char *zLeTerm;                  /* (term <= $zLeTerm) paramater, or NULL */
+  int colUsed;                    /* Copy of sqlite3_index_info.colUsed */

  /* These are used by 'col' tables only */
  int iCol;
@@ -90,9 +91,11 @@ struct Fts5VocabCursor {
 /*
 ** Bits for the mask used as the idxNum value by xBestIndex/xFilter.
 */
-#define FTS5_VOCAB_TERM_EQ 0x01
-#define FTS5_VOCAB_TERM_GE 0x02
-#define FTS5_VOCAB_TERM_LE 0x04
+#define FTS5_VOCAB_TERM_EQ 0x0100
+#define FTS5_VOCAB_TERM_GE 0x0200
+#define FTS5_VOCAB_TERM_LE 0x0400
+
+#define FTS5_VOCAB_COLUSED_MASK 0xFF


 /*
@@ -269,11 +272,13 @@ static int fts5VocabBestIndexMethod(
  int iTermEq = -1;
  int iTermGe = -1;
  int iTermLe = -1;
-  int idxNum = 0;
+  int idxNum = (int)pInfo->colUsed;
  int nArg = 0;

  UNUSED_PARAM(pUnused);

+  assert( (pInfo->colUsed & FTS5_VOCAB_COLUSED_MASK)==pInfo->colUsed );
+
  for(i=0; i<pInfo->nConstraint; i++){
    struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
    if( p->usable==0 ) continue;
@@ -365,7 +370,7 @@ static int fts5VocabOpenMethod(
      if( rc==SQLITE_OK ){
        pVTab->zErrMsg = sqlite3_mprintf(
            "no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl
-            );
+        );
        rc = SQLITE_ERROR;
      }
    }else{
@@ -525,9 +530,19 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){

        switch( pTab->eType ){
          case FTS5_VOCAB_ROW:
-            if( eDetail==FTS5_DETAIL_FULL ){
-              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
-                pCsr->aCnt[0]++;
+            /* Do not bother counting the number of instances if the "cnt"
+            ** column is not being read (according to colUsed).  */
+            if( eDetail==FTS5_DETAIL_FULL && (pCsr->colUsed & 0x04) ){
+              while( iPos<nPos ){
+                u32 ii;
+                fts5FastGetVarint32(pPos, iPos, ii);
+                if( ii==1 ){ 
+                  /* New column in the position list */
+                  fts5FastGetVarint32(pPos, iPos, ii);
+                }else{
+                  /* An instance - increment pCsr->aCnt[] */
+                  pCsr->aCnt[0]++;
+                }
              }
            }
            pCsr->aDoc[0]++;
@@ -625,6 +640,7 @@ static int fts5VocabFilterMethod(
  if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++];
  if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++];
  if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++];
+  pCsr->colUsed = (idxNum & FTS5_VOCAB_COLUSED_MASK);

  if( pEq ){
    zTerm = (const char *)sqlite3_value_text(pEq);
@@ -51,6 +51,10 @@ proc fts5_test_poslist2 {cmd} {
  sort_poslist $res
 }

+proc fts5_test_insttoken {cmd iInst iToken} {
+  $cmd xInstToken $iInst $iToken
+}
+
 proc fts5_test_collist {cmd} {
  set res [list]

@@ -78,6 +82,9 @@ proc fts5_test_columnsize {cmd} {
 proc fts5_columntext {cmd iCol} {
  $cmd xColumnText $iCol
 }
+proc fts5_columnlocale {cmd iCol} {
+  $cmd xColumnLocale $iCol
+}

 proc fts5_test_columntext {cmd} {
  set res [list]
@@ -87,6 +94,14 @@ proc fts5_test_columntext {cmd} {
  set res
 }

+proc fts5_test_columnlocale {cmd} {
+  set res [list]
+  for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
+    lappend res [$cmd xColumnLocale $i]
+  }
+  set res
+}
+
 proc fts5_test_columntotalsize {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
@@ -114,6 +129,10 @@ proc fts5_test_rowcount {cmd} {
  $cmd xRowCount
 }

+proc fts5_test_rowid {cmd} {
+  $cmd xRowid
+}
+
 proc test_queryphrase_cb {cnt cmd} {
  upvar $cnt L 
  for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
@@ -161,17 +180,21 @@ proc fts5_aux_test_functions {db} {
  foreach f {
    fts5_test_columnsize
    fts5_test_columntext
+    fts5_test_columnlocale
    fts5_test_columntotalsize
    fts5_test_poslist
    fts5_test_poslist2
    fts5_test_collist
+    fts5_test_insttoken
    fts5_test_tokenize
    fts5_test_rowcount
+    fts5_test_rowid
    fts5_test_all

    fts5_test_queryphrase
    fts5_test_phrasecount
    fts5_columntext
+    fts5_columnlocale
    fts5_queryphrase
    fts5_collist
  } {
@@ -440,7 +440,7 @@ do_execsql_test 16.1 {
 proc funk {} {
  db eval { UPDATE n1_config SET v=50 WHERE k='version' }
  set fd [db incrblob main n1_data block 10]
-  fconfigure $fd -encoding binary -translation binary
+  fconfigure $fd -translation binary
 #  puts -nonewline $fd "\x44\x45"
  close $fd
 }
@@ -453,7 +453,7 @@ db func funk funk
 # statement no longer fails.
 #
 do_catchsql_test 16.2 {
-  SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
+  SELECT funk(), format('%g',bm25(n1)), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
 } {0 {{} -1e-06 {}}}
 # {1 {SQL logic error}}

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ab

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ac

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -18,7 +18,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ad

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ae

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -18,7 +18,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5af

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ag

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ah

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -163,6 +163,17 @@ do_execsql_test 1.8.2 {
  SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid < 'text';
 } {10000}

+do_execsql_test 1.8.3 {
+  SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid<5000 AND rowid < 'text';
+} {4999}
+do_execsql_test 1.8.4 {
+  SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid>5000 AND rowid > 'text';
+} {0}
+
+do_catchsql_test 1.9 {
+  SELECT * FROM t1('*xy');
+} {1 {unknown special query: xy}}
+
 } ;# foreach_detail_mode

 #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}
@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ai

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -19,7 +19,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5aj

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ak

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5al

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -293,6 +293,16 @@ do_catchsql_test 4.4.4 {
  SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL
 } {1 {parse error in rank function: }}

+# Check that the second and subsequent rank= constraints are ignored.
+#
+do_catchsql_test 4.3.3 {
+  SELECT *, rank FROM t3
+  WHERE t3 MATCH 'a' AND 
+  rank MATCH 'nosuch()' AND
+  rank MATCH 'rowidmod(3)'  
+  ORDER BY rank ASC
+} {1 {unable to use function MATCH in the requested context}}
+
 } ;# foreach_detail_mode


@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5alter

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5auto

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5aux

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -377,4 +377,28 @@ do_catchsql_test 12.3.3 {
  SELECT fts5_collist(t1, 1) FROM t1('one AND two');
 } {0 1}

+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 13.1 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=ascii);
+  INSERT INTO t1 VALUES('a b c'), ('d e f');
+  PRAGMA integrity_check;
+} {ok}
+
+do_catchsql_test 13.2 {
+  SELECT highlight(t1, 0, '[', ']') FROM t1
+} {0 {{a b c} {d e f}}}
+
+do_execsql_test 13.3 {
+  PRAGMA writable_schema = 1;
+  UPDATE sqlite_schema SET sql = 'CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=blah)'
+  WHERE name = 't1';
+}
+
+db close
+sqlite3 db test.db
+do_catchsql_test 13.4 {
+  SELECT highlight(t1, 0, '[', ']') FROM t1
+} {1 {SQL logic error}}
+
 finish_test
@@ -0,0 +1,71 @@
+# 2024 June 24
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focusing on the auxiliary function APIs.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5aux
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
+  INSERT INTO x1 VALUES('a b', 'c d');
+  INSERT INTO x1 VALUES('d e', 'a b');
+  INSERT INTO x1 VALUES('a b', 'e f');
+  INSERT INTO x1 VALUES('d e', 'c d');
+}
+
+fts5_aux_test_functions db
+do_execsql_test 1.1 {
+  SELECT fts5_test_all(x1) FROM x1 WHERE rowid=2
+} [list [list {*}{
+  columnsize {2 2}
+  columntext {{d e} {a b}}
+  columntotalsize {8 8}
+  poslist {}
+  tokenize {{d e} {a b}}
+  rowcount 4
+}]]
+
+do_execsql_test 1.2 {
+  SELECT fts5_test_columntext(x1) FROM x1
+} {
+  {{a b} {c d}}
+  {{d e} {a b}}
+  {{a b} {e f}}
+  {{d e} {c d}}
+}
+
+do_execsql_test 1.3 {
+  SELECT fts5_test_rowid(x1) FROM x1
+} {
+  1 2 3 4
+}
+do_execsql_test 1.4 {
+  SELECT fts5_test_phrasecount(x1) FROM x1
+} {
+  0 0 0 0
+}
+do_catchsql_test 1.5 {
+  SELECT fts5_queryphrase(x1, 0) FROM x1
+} {1 SQLITE_RANGE} 
+do_execsql_test 1.6 {
+  SELECT fts5_test_rowcount(x1) FROM x1
+} {4 4 4 4}
+
+
+finish_test
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5auxdata

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5bigpl

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -0,0 +1,166 @@
+# 2024 July 30
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file verifies that:
+#
+#   * blob values may be written to locale=0 tables.
+#
+#   * blob values - other than fts5_locale() values - may not be written 
+#     to locale=0 tables. This is an SQLITE_MISMATCH error
+#
+#   * blob values may be returned by queries on the external-content table
+#     of a locale=0 table.
+#
+#   * blob values not may be returned by queries on the external-content 
+#     table of a locale=1 table, apart from fts5_locale() blobs. This is an
+#     SQLITE_MISMATCH error.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5blob
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+# Test that blobs may be stored in normal locale=0 tables.
+#
+foreach {tn enc} {
+  1 utf8  
+  2 utf16 
+} {
+  reset_db
+  fts5_aux_test_functions db
+
+  execsql "PRAGMA encoding = $enc"
+
+  execsql "
+    CREATE VIRTUAL TABLE t1 USING fts5(x, y);
+  "
+  do_execsql_test 1.$tn.0 {
+    CREATE VIRTUAL TABLE tt USING fts5vocab('t1', 'instance');
+    INSERT INTO t1(rowid, x, y) VALUES(1, 555, X'0000000041424320444546');
+    INSERT INTO t1(rowid, x, y) VALUES(2, 666, X'41424300444546');
+    INSERT INTO t1(rowid, x, y) VALUES(3, 777, 'xyz');
+  }
+
+  do_execsql_test 1.$tn.1 {
+    SELECT rowid, quote(x), quote(y) FROM t1
+  } {
+    1 555 X'0000000041424320444546'
+    2 666 X'41424300444546'
+    3 777 'xyz'
+  }
+
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t1 WHERE rowid=2;
+    DELETE FROM t1 WHERE rowid=1;
+  }
+
+  do_execsql_test 1.$tn.3 {
+    PRAGMA integrity_check;
+  } {ok}
+}
+
+#--------------------------------------------------------------------------
+# Test that a blob may be stored and retrieved in an unindexed column of
+# a regular table with locale=1.
+#
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y UNINDEXED, locale=1);
+  INSERT INTO t1(rowid, x, y) VALUES(12, 'twelve', X'0000000041424320444546');
+}
+
+do_execsql_test 2.1 {
+  select rowid, x, quote(y) FROM t1
+} {
+  12 twelve X'0000000041424320444546'
+}
+
+#--------------------------------------------------------------------------
+# Test that blobs may not be written to any type of table with locale=1
+# set. Except, they may be written to UNINDEXED columns.
+#
+reset_db
+do_execsql_test 3.0 {
+  CREATE TABLE t1(a, b);
+
+  CREATE VIRTUAL TABLE x1 USING fts5(a, b, locale=1);
+  CREATE VIRTUAL TABLE x2 USING fts5(a, b, locale=1, content=t2);
+  CREATE VIRTUAL TABLE x3 USING fts5(a, b, locale=1, content=);
+}
+
+do_catchsql_test 3.1 {
+  INSERT INTO x1(rowid, a, b) VALUES(113, 'hello world', X'123456');
+} {0 {}}
+do_catchsql_test 3.2 {
+  INSERT INTO x2(rowid, a, b) VALUES(113, 'hello world', X'123456');
+} {0 {}}
+do_catchsql_test 3.3 {
+  INSERT INTO x3(rowid, a, b) VALUES(113, 'hello world', X'123456');
+} {0 {}}
+
+
+#--------------------------------------------------------------------------
+# Test that fts5_locale() values may not be written to any type of table
+# without locale=1 set. Even to an UNINDEXED column.
+#
+reset_db
+do_execsql_test 3.0 {
+  CREATE TABLE t1(a, b);
+
+  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
+  CREATE VIRTUAL TABLE x2 USING fts5(a, b, content=t2);
+  CREATE VIRTUAL TABLE x3 USING fts5(a, b, content=);
+
+  CREATE VIRTUAL TABLE x4 USING fts5(a, b, c UNINDEXED);
+}
+
+do_catchsql_test 3.1 {
+  INSERT INTO x1(rowid, a, b) 
+    VALUES(113, 'hello world', fts5_locale('en_AU', 'abc'));
+} {1 {fts5_locale() requires locale=1}}
+do_catchsql_test 3.2 {
+  INSERT INTO x2(rowid, a, b) 
+    VALUES(113, 'hello world', fts5_locale('en_AU', 'abc'));
+} {1 {fts5_locale() requires locale=1}}
+do_catchsql_test 3.3 {
+  INSERT INTO x3(rowid, a, b) 
+    VALUES(113, 'hello world', fts5_locale('en_AU', 'abc'));
+} {1 {fts5_locale() requires locale=1}}
+do_catchsql_test 3.4 {
+  INSERT INTO x4(rowid, a, b, c) 
+    VALUES(113, 'hello world', 'yesno', fts5_locale('en_AU', 'abc'));
+} {1 {fts5_locale() requires locale=1}}
+
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(x);
+}
+
+foreach {tn sql} {
+  1 { INSERT INTO x1(rowid, x) VALUES(4.5, 'abcd') }
+  2 { INSERT INTO x1(rowid, x) VALUES('xyz', 'abcd') }
+  3 { INSERT INTO x1(rowid, x) VALUES(X'001122', 'abcd') }
+} {
+  do_catchsql_test 4.1.$tn $sql {1 {datatype mismatch}}
+}
+
+
+finish_test
+
+
@@ -55,5 +55,22 @@ do_execsql_test 1.5 {
  SELECT * FROM t4t
 } {สนามกีฬา 1 1}

+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0 "
+  CREATE VIRTUAL TABLE x1 USING fts5(c, 
+      tokenize=\"unicode61 categories ' \t'\");
+"
+
+do_catchsql_test 2.1 "
+  CREATE VIRTUAL TABLE x2 USING fts5(c, 
+      tokenize=\"unicode61 categories 'N*\t\tMYZ'\");
+" {1 {error in tokenizer constructor}}
+
+do_catchsql_test 2.2 "
+  CREATE VIRTUAL TABLE x2 USING fts5(c, 
+      tokenize=\"unicode61 categories 'N*\t\tXYZ'\");
+" {1 {error in tokenizer constructor}}
+

 finish_test
@@ -79,7 +79,7 @@ foreach_detail_mode $::testprefix {

  do_catchsql_test 4.1 {
    SELECT * FROM t1 WHERE rowid MATCH 'a'
-  } {1 {unable to use function MATCH in the requested context}}
+  } {1 {no query solution}}
 }

 #-------------------------------------------------------------------------
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5columnsize

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5config

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5content

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -328,4 +328,41 @@ do_execsql_test 8.3.2 { INSERT INTO t1(t1) VALUES('rebuild') }
 do_execsql_test 8.3.3 { SELECT * FROM t1 WHERE rowid=1 } {one}
 do_execsql_test 8.3.4 { SELECT rowid FROM t1('two') } {2}

+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 9.1 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
+  INSERT INTO t1 VALUES(1, 'one two three');
+  INSERT INTO t1 VALUES(2, 'one two three');
+
+  CREATE VIRTUAL TABLE ft USING fts5(b, content=t1, content_rowid=a);
+  INSERT INTO ft(ft) VALUES('rebuild');
+}
+
+do_execsql_test 9.2 {
+  SELECT rowid, b FROM ft('two');
+} {
+  1 {one two three}
+  2 {one two three}
+}
+
+do_execsql_test 9.3 {
+  DELETE FROM t1 WHERE a=2;
+} 
+
+do_catchsql_test 9.4 {
+  SELECT rowid FROM ft('two');
+} {0 {1 2}}
+
+do_catchsql_test 9.5 {
+  SELECT * FROM ft('two');
+} {1 {fts5: missing row 2 from content table 'main'.'t1'}}
+
+fts5_aux_test_functions db
+
+do_catchsql_test 9.6 {
+  SELECT rowid, fts5_columntext(ft, 0) FROM ft('two');
+} {1 SQLITE_CORRUPT_VTAB}
+
 finish_test
+
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5contentless

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -267,4 +267,24 @@ do_execsql_test 8.2 {
  SELECT rowid FROM ft('four');
 } {}

+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 9.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=0);
+  INSERT INTO ft VALUES('hello world');
+  INSERT INTO ft VALUES('one two three');
+}
+
+do_catchsql_test 9.1 {
+  INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, 'hello world');
+} {0 {}}
+
+do_catchsql_test 9.2 {
+  CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', contentless_delete=2);
+} {1 {malformed contentless_delete=... directive}}
+
+do_catchsql_test 9.3 {
+  CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', contentless_delete=11);
+} {1 {malformed contentless_delete=... directive}}
+
 finish_test
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5contentless2

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5contentless3

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5contentless4

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,11 +15,12 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5contentless5

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
 }
+unset -nocomplain res

 do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, content='', contentless_delete=1);
@@ -34,8 +35,8 @@ do_execsql_test 1.01 {
 }

 # explain_i "UPDATE t1 SET a='a' WHERE t1.rowid=1"    
-breakpoint
-explain_i "UPDATE t1 SET a='a' FROM t2 WHERE t1.rowid=1 AND b IS NULL"     
+#breakpoint
+#explain_i "UPDATE t1 SET a='a' FROM t2 WHERE t1.rowid=1 AND b IS NULL"     

 #breakpoint
 #explain_i "UPDATE t1 SET a='a' WHERE b IS NULL AND rowid=?"
@@ -55,4 +56,56 @@ foreach {tn up err} {
  do_catchsql_test 1.$tn $up $res($err)
 }

+#-------------------------------------------------------------------------
+reset_db
+
+proc random {n} { expr {abs(int(rand()*$n))} }
+proc select_one {list} {
+  set n [llength $list]
+  lindex $list [random $n]
+}
+proc vocab {} {
+  list abc def ghi jkl mno pqr stu vwx yza
+}
+proc term {} {
+  select_one [vocab]
+}
+proc document {} {
+  set nTerm [expr [random 3] + 7]
+  set doc ""
+  for {set ii 0} {$ii < $nTerm} {incr ii} {
+    lappend doc [term]
+  }
+  set doc
+}
+db func document document
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(a, contentless_delete=1, content='');
+  INSERT INTO ft(ft, rank) VALUES('pgsz', 64);
+}
+
+do_test 2.1 {
+  for {set ii 1} {$ii < 12} {incr ii} {
+    db transaction {
+      for {set jj 0} {$jj < 10} {incr jj} {
+        set doc [document]
+        execsql { INSERT INTO ft VALUES($doc); }
+      }
+    }
+  }
+} {}
+
+do_test 2.2 {
+  foreach r [db eval {SELECT rowid FROM ft}] {
+    execsql { DELETE FROM ft WHERE rowid=$r }
+  }
+} {}
+
+set doc [document]
+do_execsql_test 2.3 {
+  INSERT INTO ft VALUES($doc)
+}
+
+
 finish_test
@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -99,6 +99,27 @@ sqlite3_db_config db DEFENSIVE 0
 do_catchsql_test 3.1 {
  DELETE FROM t3_content WHERE rowid = 3;
  SELECT * FROM t3 WHERE t3 MATCH 'o';
+} {1 {fts5: missing row 3 from content table 'main'.'t3_content'}}
+
+#--------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(x);
+  INSERT INTO t2 VALUES('one two three');
+  INSERT INTO t2 VALUES('four five six');
+  INSERT INTO t2 VALUES('seven eight nine');
+  INSERT INTO t2 VALUES('ten eleven twelve');
+}
+do_execsql_test 4.1 {
+  SELECT hex(block) FROM t2_data WHERE id=1;
+} {040C}
+do_execsql_test 4.2 {
+  UPDATE t2_data SET block = X'0402' WHERE id=1
+}
+breakpoint
+do_catchsql_test 4.3 {
+  DELETE FROM t2 WHERE rowid=3
 } {1 {database disk image is malformed}}

 finish_test
@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt2

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -100,6 +100,7 @@ set lrowid [db one {SELECT max(rowid) FROM t1_data WHERE (rowid & $mask)=0}]
 set nbyte [db one {SELECT length(block) FROM t1_data WHERE rowid=$lrowid}]
 set all [db eval {SELECT rowid FROM t1}]
 sqlite3_db_config db DEFENSIVE 0
+unset -nocomplain res
 for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} {
  do_execsql_test 2.$i.1 {
    BEGIN;
@@ -152,7 +153,7 @@ foreach {tn hdr} {
      execsql BEGIN

      set fd [db incrblob main x3_data block $rowid]
-      fconfigure $fd -encoding binary -translation binary
+      fconfigure $fd -translation binary
      set existing [read $fd [string length $hdr]]
      seek $fd 0
      puts -nonewline $fd $hdr
@@ -238,7 +239,7 @@ foreach {tn hdr} {
      execsql BEGIN

      set fd [db incrblob main x5_data block $rowid]
-      fconfigure $fd -encoding binary -translation binary
+      fconfigure $fd -translation binary
      puts -nonewline $fd $hdr
      close $fd

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt3

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -680,11 +680,11 @@ do_test 12.0 {

 do_catchsql_test 11.1 {
  SELECT * FROM t1 WHERE t1 MATCH 'abandon';
-} {1 {vtable constructor failed: t1}}
+} {1 {database disk image is malformed}}

 do_catchsql_test 11.2 {
  INSERT INTO t1(t1, rank) VALUES('merge', 500);
-} {1 {vtable constructor failed: t1}}
+} {1 {database disk image is malformed}}

 #-------------------------------------------------------------------------
 #
@@ -1040,7 +1040,7 @@ do_test 16.0 {

 do_catchsql_test 16.1 {
 INSERT INTO t1(t1) VALUES('integrity-check');
-} {1 {vtable constructor failed: t1}}
+} {1 {database disk image is malformed}}

 #--------------------------------------------------------------------------
 reset_db
@@ -1126,7 +1126,7 @@ do_test 17.0 {

 do_catchsql_test 17.1 {
  SELECT * FROM t1 WHERE t1 MATCH 'abandon';
-} {1 {vtable constructor failed: t1}}
+} {1 {database disk image is malformed}}

 #--------------------------------------------------------------------------
 reset_db
@@ -1630,7 +1630,7 @@ do_test 20.0 {

 do_catchsql_test 20.1 {
  SELECT * FROM t1 WHERE t1 MATCH 'abandon';
-} {1 {vtable constructor failed: t1}}
+} {1 {database disk image is malformed}}

 #-------------------------------------------------------------------------
 reset_db
@@ -2100,7 +2100,7 @@ do_test 22.0 {

 do_catchsql_test 22.1 {
  INSERT INTO t1(t1) VALUES('optimize');
-} {1 {vtable constructor failed: t1}}
+} {1 {database disk image is malformed}}

 #--------------------------------------------------------------------------
 reset_db
@@ -3700,7 +3700,7 @@ do_catchsql_test 32.1 {
         highlight(t1, 2, '[', ']') 
  FROM t1('g + h') 
  WHERE rank MATCH 'bm25(1.0, 1.0)' ORDER BY rank;
-} {1 {vtable constructor failed: t1}}
+} {1 {database disk image is malformed}}

 do_catchsql_test 32.2 {
  SELECT * FROM t3;
@@ -4267,7 +4267,7 @@ do_test 35.0 {

 do_catchsql_test 35.1 {
  SELECT * FROM t1 WHERE t1 MATCH 'e*';
-} {1 {database disk image is malformed}}
+} {1 {fts5: missing row 14 from content table 'main'.'t1_content'}}

 #-------------------------------------------------------------------------
 reset_db
@@ -8958,7 +8958,6 @@ do_catchsql_test 61.2 {
  SELECT * FROM t3 ORDER BY rowid;
 } {/*malformed database schema*/}

-breakpoint
 #-------------------------------------------------------------------------
 do_test 62.0 {
  sqlite3 db {}
@@ -10768,6 +10767,7 @@ do_catchsql_test 73.1 {
 reset_db
 do_test 74.0 {
  sqlite3 db {}
+  sqlite3_fts5_register_matchinfo db
  db deserialize [decode_hexdb {
 | size 106496 pagesize 4096 filename x.db
 | page 1 offset 0
@@ -14587,14 +14587,19 @@ do_test 74.0 {
 | end x.db
 }]} {}

-do_catchsql_test 74.1 {
-  SELECT rowid, quote(matchinfo(t1,'p�xyb<s')) FROM t1 WHERE t1 MATCH 'e*';
+do_catchsql_test 74.0.5 {
+  SELECT matchinfo(1,2);
 } {1 {unable to use function matchinfo in the requested context}}

+do_catchsql_test 74.1 {
+  SELECT rowid, quote(matchinfo(t1,'pxyb<s')) FROM t1 WHERE t1 MATCH 'e*';
+} {1 {unrecognized matchinfo flag: <}}
+
 #-------------------------------------------------------------------------
 reset_db
 do_test 75.0 {
  sqlite3 db {}
+  sqlite3_fts5_register_matchinfo db
  db deserialize [decode_hexdb {
 | size 32768 pagesize 4096 filename crash-033d665d5caa8d.db
 | page 1 offset 0
@@ -14791,7 +14796,7 @@ do_test 75.0 {

 do_catchsql_test 75.1 {
  SELECT rowid, quote(matchinfo(t1,'pcxybs')) FROM t1 WHERE t1 MATCH 'e*';
-} {1 {unable to use function matchinfo in the requested context}}
+} {1 {database disk image is malformed}}

 #-------------------------------------------------------------------------
 reset_db
@@ -15519,6 +15524,160 @@ do_catchsql_test 80.1 {
 SELECT snippet(rowid, -1, '.', '..', '[', '(]'),snippet(rowid, -1, '.', '.', '', '(]'), highlight(t1, 29, 1 , '') FROM t1('g+ h') WHERE rank MATCH 'bm25(1.0, 10)' ORDER BY NOT (SELECT 1 FROM t1('g+ æ') WHERE rank MATCH 'bm25(1.0, 10)' ORDER BY rank);
 } {1 {database disk image is malformed}}

+#-------------------------------------------------------------------------
+reset_db
+do_test 81.0 {
+  sqlite3 db {}
+  db deserialize [decode_hexdb {
+.open --hexdb
+| size 40960 pagesize 4096 filename crash-44e8035a976422.db
+| page 1 offset 0
+|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
+|     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a   .....@  ........
+|     32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04   ................
+|     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
+|     96: 00 00 00 00 0d 00 00 00 0d 0b 6e 00 0f a3 0f 4c   ..........n....L
+|    112: 0e e1 0e 81 0e 24 0d cc 0d 72 0d 1b 0c b0 0c 50   .....$...r.....P
+|    128: 0b f8 0b b3 0b 6e 00 00 00 00 00 00 00 00 00 00   .....n..........
+|   2912: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 43 0d   ..............C.
+|   2928: 06 17 11 11 08 75 74 61 62 6c 65 74 34 74 34 43   .....utablet4t4C
+|   2944: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41   REATE VIRTUAL TA
+|   2960: 42 4c 45 20 74 34 20 55 53 49 4e 47 20 66 74 73   BLE t4 USING fts
+|   2976: 35 76 6f 63 61 62 28 27 74 32 27 2c 20 27 72 6f   5vocab('t2', 'ro
+|   2992: 77 27 29 43 0c 06 17 11 11 08 75 74 61 62 6c 65   w')C......utable
+|   3008: 74 33 74 33 43 52 45 41 54 45 20 56 49 52 54 55   t3t3CREATE VIRTU
+|   3024: 41 4c 20 54 41 42 4c 45 20 74 33 20 55 53 49 4e   AL TABLE t3 USIN
+|   3040: 47 20 66 74 73 35 76 6f 63 61 62 28 27 74 31 27   G fts5vocab('t1'
+|   3056: 2c 20 27 72 6f 77 27 29 56 0b 06 17 1f 1f 01 7d   , 'row')V.......
+|   3072: 74 61 62 6c 65 74 32 5f 63 6f 6e 66 69 67 74 32   tablet2_configt2
+|   3088: 5f 63 6f 6e 66 69 67 0a 43 52 45 41 54 45 20 54   _config.CREATE T
+|   3104: 41 42 4c 45 20 27 74 32 5f 63 6f 6e 66 69 67 27   ABLE 't2_config'
+|   3120: 28 6b 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20   (k PRIMARY KEY, 
+|   3136: 76 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44   v) WITHOUT ROWID
+|   3152: 5e 0a 07 17 21 21 01 81 07 74 61 62 6c 65 74 32   ^...!!...tablet2
+|   3168: 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 6f 6e 74 65   _contentt2_conte
+|   3184: 6e 74 09 43 52 45 41 54 45 20 54 41 42 4c 45 20   nt.CREATE TABLE 
+|   3200: 27 74 32 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20   't2_content'(id 
+|   3216: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20   INTEGER PRIMARY 
+|   3232: 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 29   KEY, c0, c1, c2)
+|   3248: 69 09 07 17 19 19 01 81 2d 74 61 62 6c 65 74 32   i.......-tablet2
+|   3264: 5f 69 64 78 74 32 5f 69 64 78 08 43 52 45 41 54   _idxt2_idx.CREAT
+|   3280: 45 20 54 41 42 4c 45 20 27 74 32 5f 69 64 78 27   E TABLE 't2_idx'
+|   3296: 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67   (segid, term, pg
+|   3312: 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28   no, PRIMARY KEY(
+|   3328: 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49   segid, term)) WI
+|   3344: 54 48 4f 55 54 20 52 4f 57 49 44 55 08 07 17 1b   THOUT ROWIDU....
+|   3360: 1b 01 81 01 74 61 62 6c 65 74 32 5f 64 61 74 61   ....tablet2_data
+|   3376: 74 32 5f 64 61 74 61 07 43 52 45 41 54 45 20 54   t2_data.CREATE T
+|   3392: 41 42 4c 45 20 27 74 32 5f 64 61 74 61 27 28 69   ABLE 't2_data'(i
+|   3408: 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52   d INTEGER PRIMAR
+|   3424: 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c 4f   Y KEY, block BLO
+|   3440: 42 29 58 07 07 17 11 11 08 81 1d 74 61 62 6c 65   B)X........table
+|   3456: 74 32 74 32 43 52 45 41 54 45 20 56 49 52 54 55   t2t2CREATE VIRTU
+|   3472: 41 4c 20 54 41 42 4c 45 20 74 32 20 55 53 49 4e   AL TABLE t2 USIN
+|   3488: 47 20 66 74 73 35 28 27 61 27 2c 5b 62 5d 2c 22   G fts5('a',[b],.
+|   3504: 63 22 2c 64 65 74 61 69 6c 3d 6e 6f 6e 65 2c 63   c.,detail=none,c
+|   3520: 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 56 06 06 17   olumnsize=0)V...
+|   3536: 1f 1f 01 7d 74 61 62 6c 65 74 31 5f 63 6f 6e 66   ....tablet1_conf
+|   3552: 69 67 74 31 5f 63 6f 6e 66 69 67 06 43 52 45 41   igt1_config.CREA
+|   3568: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f 6e   TE TABLE 't1_con
+|   3584: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b   fig'(k PRIMARY K
+|   3600: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52   EY, v) WITHOUT R
+|   3616: 4f 57 49 44 5b 05 07 17 21 21 01 81 01 74 61 62   OWID[...!!...tab
+|   3632: 6c 65 74 31 5f 64 6f 63 73 69 7a 65 74 31 5f 64   let1_docsizet1_d
+|   3648: 6f 63 73 69 7a 65 05 43 52 45 41 54 45 20 54 41   ocsize.CREATE TA
+|   3664: 42 4c 45 20 27 74 31 5f 64 6f 63 73 69 7a 65 27   BLE 't1_docsize'
+|   3680: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d   (id INTEGER PRIM
+|   3696: 41 52 59 20 4b 45 59 2c 20 73 7a 20 42 4c 4f 42   ARY KEY, sz BLOB
+|   3712: 29 5e 04 07 17 21 21 01 81 07 74 61 62 6c 65 74   )^...!!...tablet
+|   3728: 31 5f 63 6f 6e 74 65 6e 74 74 31 5f 63 6f 6e 74   1_contentt1_cont
+|   3744: 65 6e 74 04 43 52 45 41 54 45 20 54 41 42 4c 45   ent.CREATE TABLE
+|   3760: 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 69 64    't1_content'(id
+|   3776: 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59    INTEGER PRIMARY
+|   3792: 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32    KEY, c0, c1, c2
+|   3808: 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 74   )i.......-tablet
+|   3824: 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 41   1_idxt1_idx.CREA
+|   3840: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 78   TE TABLE 't1_idx
+|   3856: 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70   '(segid, term, p
+|   3872: 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59   gno, PRIMARY KEY
+|   3888: 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57   (segid, term)) W
+|   3904: 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 17   ITHOUT ROWIDU...
+|   3920: 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 74   .....tablet1_dat
+|   3936: 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 20   at1_data.CREATE 
+|   3952: 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 28   TABLE 't1_data'(
+|   3968: 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41   id INTEGER PRIMA
+|   3984: 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c   RY KEY, block BL
+|   4000: 4f 42 29 5b 01 07 17 11 11 08 81 23 74 61 62 6c   OB)[.......#tabl
+|   4016: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54   et1t1CREATE VIRT
+|   4032: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49   UAL TABLE t1 USI
+|   4048: 4e 47 20 66 74 73 35 28 61 2c 62 20 75 6e 69 6e   NG fts5(a,b unin
+|   4064: 64 65 78 65 64 2c 63 2c 74 6f 6b 65 6e 69 7a 65   dexed,c,tokenize
+|   4080: 3d 22 70 6f 72 74 65 72 20 61 73 63 69 69 22 29   =.porter ascii.)
+| page 2 offset 4096
+|      0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c   ..h............|
+|     16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00   .*..............
+|   3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00   .......0........
+|   3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80   ..........7.....
+|   3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01   ...t.....0a.....
+|   3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01   b.....c.....g...
+|   3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01   ....h.......i...
+|   3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00   ............*...
+|   3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80   ............%...
+|   3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02   .....P.....0g...
+|   3984: 01 02 02 01 01 68 02 08 03 8d 02 03 01 01 6a 42   .....h........jB
+|   4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 80 01 03   ........7.......
+|   4016: 00 74 00 00 00 2e 02 30 61 01 12 02 01 01 62 01   .t.....0a.....b.
+|   4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02   ....c.....g.....
+|   4048: 01 01 68 01 05 01 02 03 01 01 69 01 06 01 02 04   ..h.......i.....
+|   4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00   ................
+|   4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01   ...$............
+| page 3 offset 8192
+|      0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00   ................
+|   4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c   ................
+|   4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02   ................
+| page 4 offset 12288
+|      0: 0d 00 00 00 03 0f be 00 0f ea 00 00 00 00 00 00   ................
+|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03   ................
+|   4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67   .....a b cg h ig
+|   4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69    h i.......g h i
+|   4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17   a b cg h i......
+|   4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69   .a b cd e fg h i
+| page 5 offset 16384
+|      0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00   ................
+|   4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03   ................
+|   4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03   ................
+| page 6 offset 20480
+|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
+| page 7 offset 24576
+|      0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00   ................
+|   3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84   ..............A.
+|   4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61   ............4.0a
+|   4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01   .....b.....c....
+|   4032: e6 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01   .d...e...f...g..
+|   4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06   ...h.....i......
+|   4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f   ................
+|   4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01   ...$............
+| page 8 offset 28672
+|      0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02   ................
+| page 9 offset 32768
+|      0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00   ................
+|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03   ................
+|   4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67   .....a b cg h ig
+|   4048: 20 68 21 69 14 02 05 00 17 17 17 67 20 68 20 69    h!i.......g h i
+|   4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17   a b cg h i......
+|   4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69   .a b cd e fg h i
+| page 10 offset 36864
+|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
+| end crash-44e8035a976422.db
+}]} {}
+
+do_catchsql_test 81.2 {
+  UPDATE t1 SET b=zeroblob(299);
+} {1 {database disk image is malformed}}
+

 sqlite3_fts5_may_be_corrupt 0
 finish_test
@@ -14,7 +14,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt4

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt5

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt6

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -13,7 +13,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt7

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -0,0 +1,94 @@
+# 2024 Aug 28
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5corrupt8
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+}
+
+do_execsql_test 1.1 {
+  UPDATE t1_data SET block='hello world' WHERE id=10
+}
+
+db close
+sqlite3 db test.db
+
+do_catchsql_test 1.2 {
+  SELECT * FROM t1
+} {1 {database disk image is malformed}}
+do_catchsql_test 1.3 {
+  DROP TABLE t1
+} {0 {}}
+do_execsql_test 1.4 {
+  SELECT * FROM sqlite_schema
+}
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+}
+do_execsql_test 2.1 {
+  UPDATE t1_config SET v=555 WHERE k='version'
+}
+db close
+sqlite3 db test.db
+do_catchsql_test 2.2 {
+  SELECT * FROM t1
+} {1 {invalid fts5 file format (found 555, expected 4 or 5) - run 'rebuild'}}
+do_catchsql_test 2.3 {
+  DROP TABLE t1
+} {1 {invalid fts5 file format (found 555, expected 4 or 5) - run 'rebuild'}}
+do_test 2.4 {
+  sqlite3_fts5_drop_corrupt_table db main t1
+} {}
+do_execsql_test 2.5 {
+  SELECT * FROM sqlite_schema
+}
+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+}
+do_execsql_test 3.1 {
+  DELETE FROM t1_config;
+}
+db close
+sqlite3 db test.db
+do_catchsql_test 3.2 {
+  SELECT * FROM t1
+} {1 {invalid fts5 file format (found 0, expected 4 or 5) - run 'rebuild'}}
+do_catchsql_test 3.3 {
+  DROP TABLE t1
+} {1 {invalid fts5 file format (found 0, expected 4 or 5) - run 'rebuild'}}
+
+
+do_test 3.4 {
+  sqlite3_db_config db DEFENSIVE 1
+} {1}
+do_test 3.5 {
+  sqlite3_fts5_drop_corrupt_table db main t1
+} {}
+do_test 3.6 {
+  sqlite3_db_config db DEFENSIVE -1
+} {1}
+do_execsql_test 3.7 {
+  SELECT * FROM sqlite_schema
+}
+
+finish_test
+
@@ -113,5 +113,58 @@ do_execsql_test 3.4 {
  INSERT INTO tx(tx) VALUES('integrity-check');
 }

+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE ft1 USING fts5(a, b UNINDEXED, 
+      content='', contentless_unindexed=1
+  );
+  CREATE VIRTUAL TABLE ft2 USING fts5(a, b UNINDEXED, 
+      content='', contentless_unindexed=1, contentless_delete=1
+  );
+
+  INSERT INTO ft1(rowid, a, b) VALUES(1, 'one', 'i');
+  INSERT INTO ft1(rowid, a, b) VALUES(2, 'two', 'ii');
+  INSERT INTO ft1(rowid, a, b) VALUES(3, 'three', 'iii');
+  INSERT INTO ft2(rowid, a, b) VALUES(1, 'one', 'i');
+  INSERT INTO ft2(rowid, a, b) VALUES(2, 'two', 'ii');
+  INSERT INTO ft2(rowid, a, b) VALUES(3, 'three', 'iii');
+}
+
+do_catchsql_test 4.1 {
+  DELETE FROM ft1 WHERE rowid=2
+} {1 {cannot DELETE from contentless fts5 table: ft1}}
+do_catchsql_test 4.2 {
+  DELETE FROM ft2 WHERE rowid=2
+} {0 {}}
+
+do_catchsql_test 4.3 {
+  INSERT INTO ft1(ft1, rowid, a) VALUES('delete', 2, 'two');
+} {0 {}}
+do_catchsql_test 4.2 {
+  INSERT INTO ft2(ft2, rowid, a) VALUES('delete', 2, 'two');
+} {1 {'delete' may not be used with a contentless_delete=1 table}}
+
+do_execsql_test 4.3 {
+  SELECT rowid, * FROM ft1;
+} {
+  1 {} i
+  3 {} iii
+}
+do_execsql_test 4.4 {
+  SELECT rowid, * FROM ft2;
+} {
+  1 {} i
+  3 {} iii
+}
+
+do_execsql_test 4.5 {
+  SELECT * FROM ft1_content
+} {1 i 3 iii}
+
+do_execsql_test 4.6 {
+  SELECT * FROM ft2_content
+} {1 i 3 iii}
+
 finish_test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5dlidx

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5doclist

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ea

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -13,7 +13,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5eb

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -86,13 +86,13 @@ do_execsql_test 3.0 {
  INSERT INTO e1 VALUES ('just a few words with a / inside');
 }
 do_execsql_test 3.1 {
-  SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"just"' ORDER BY rank;
+  SELECT rowid, format('%g',bm25(e1)) FROM e1 WHERE e1 MATCH '"just"' ORDER BY rank;
 } {1 -1e-06}
 do_execsql_test 3.2 {
  SELECT rowid FROM e1 WHERE e1 MATCH '"/" OR "just"'
 } 1
 do_execsql_test 3.3 {
-  SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank;
+  SELECT rowid, format('%g',bm25(e1)) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank;
 } {1 -1e-06}

 do_execsql_test 3.4 "
@@ -0,0 +1,52 @@
+# 2024 August 8
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this script is testing the FTS5 module.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5expr
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(a);
+  INSERT INTO x1(rowid, a) VALUES (113, 'fts5 expr test');
+}
+
+do_execsql_test 1.1 {
+  SELECT rowid FROM x1('expr');
+} {113}
+
+for {set ii 0} {$ii < 300} {incr ii} {
+  set expr "expr "
+  append expr [string repeat "NOT abcd " $ii]
+
+  if {$ii<257} {
+    set res {0 113}
+  } else {
+    set res {1 {fts5 expression tree is too large (maximum depth 256)}}
+  }
+  do_catchsql_test 1.1.$ii {
+    SELECT rowid FROM x1($expr)
+  } $res
+}
+
+do_execsql_test 1.2 {
+  SELECT rowid FROM x1 WHERE a MATCH '"..."'
+} {}
+
+finish_test
+
@@ -90,7 +90,7 @@ set ::res [db eval {SELECT rowid, x1 FROM x1 WHERE x1 MATCH '*reads'}]
 do_faultsim_test 4 -faults oom-* -body {
  db eval {SELECT rowid, x, x1 FROM x1 WHERE x1 MATCH '*reads'}
 } -test {
-  faultsim_test_result {0 {0 {} 3}}
+  faultsim_test_result {0 {0 {} 2}}
 }

 #-------------------------------------------------------------------------
@@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl]
 source $testdir/malloc_common.tcl
 set testprefix fts5fault6

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl]
 source $testdir/malloc_common.tcl
 set testprefix fts5faultG

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl]
 source $testdir/malloc_common.tcl
 set testprefix fts5faultG

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -0,0 +1,294 @@
+# 2010 June 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5faultI
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+set ::testprefix fts5faultI
+
+do_execsql_test 1.0 {
+  PRAGMA encoding = utf16;
+  CREATE VIRTUAL TABLE t1 USING fts5(x, locale=1);
+  INSERT INTO t1 VALUES('origintext unicode61 ascii porter trigram');
+}
+
+faultsim_save_and_close
+faultsim_restore_and_reopen
+
+do_faultsim_test 1 -faults oom* -prep { 
+} -body {
+  execsql {
+    SELECT rowid FROM t1(fts5_locale('en_US', 'origintext'));
+  }
+} -test {
+  faultsim_test_result {0 1}
+}
+
+do_faultsim_test 2 -faults oom* -prep { 
+  faultsim_restore_and_reopen
+  execsql {
+    SELECT * FROM t1('ascii');
+  }
+} -body {
+  execsql {
+    UPDATE t1 SET rowid=rowid+1;
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+fts5_aux_test_functions db
+do_faultsim_test 3 -faults oom* -prep { 
+} -body {
+  execsql {
+    SELECT fts5_columnlocale(t1, 0) FROM t1('unicode*');
+  }
+} -test {
+  faultsim_test_result {0 {{}}} {1 SQLITE_NOMEM}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE w1 USING fts5(a);
+}
+faultsim_save_and_close
+
+do_faultsim_test 4 -faults oom* -prep { 
+  faultsim_restore_and_reopen
+  execsql {
+    BEGIN;
+      INSERT INTO w1 VALUES('token token token');
+  }
+} -body {
+  execsql {
+    INSERT INTO w1(w1, rank) VALUES('rank', 'bm25()');
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+do_faultsim_test 5 -faults oom* -prep { 
+  faultsim_restore_and_reopen
+  execsql { 
+    BEGIN;
+      INSERT INTO w1 VALUES('one');
+      SAVEPOINT one;
+        INSERT INTO w1 VALUES('two');
+      ROLLBACK TO one;
+  }
+
+} -body {
+  execsql {
+    INSERT INTO w1 VALUES('string');
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE w1 USING fts5(a);
+  INSERT INTO w1 VALUES('one two three');
+}
+fts5_aux_test_functions db
+
+do_faultsim_test 5 -faults oom* -prep {
+} -body {
+  execsql {
+    SELECT fts5_test_insttoken(w1, 0, 0) FROM w1('two');
+  }
+} -test {
+  faultsim_test_result {0 two} {1 SQLITE_NOMEM}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE w1 USING fts5(a);
+  INSERT INTO w1 VALUES('one two three');
+}
+fts5_aux_test_functions db
+faultsim_save_and_close
+
+do_faultsim_test 6 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  db eval {
+    BEGIN;
+      INSERT INTO w1 VALUES('four five six');
+      SAVEPOINT abc;
+        INSERT INTO w1 VALUES('seven eight nine');
+        SAVEPOINT def;
+          INSERT INTO w1 VALUES('ten eleven twelve');
+  }
+} -body {
+  execsql {
+    RELEASE abc;
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 7.0 {
+  CREATE VIRTUAL TABLE w1 USING fts5(a);
+  INSERT INTO w1 VALUES('one two three');
+  INSERT INTO w1 VALUES('three two one');
+  DELETE FROM w1_content WHERE rowid=1;
+}
+
+faultsim_save_and_close
+
+do_faultsim_test 7 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  db eval { SELECT * FROM w1 }
+} -body {
+  execsql {
+    PRAGMA integrity_check;
+  }
+} -test {
+}
+
+#-------------------------------------------------------------------------
+reset_db
+fts5_tclnum_register db
+fts5_aux_test_functions db
+
+do_execsql_test 8.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize = "tclnum query", detail=columns
+  );
+  INSERT INTO ft VALUES('one two three i ii iii');
+  INSERT INTO ft VALUES('four five six iv v vi');
+  INSERT INTO ft VALUES('eight nine ten viii ix x');
+} {}
+
+do_faultsim_test 8.1 -faults oom* -prep {
+} -body {
+  execsql {
+    SELECT fts5_test_collist (ft) FROM ft('one two');
+  }
+} -test {
+  faultsim_test_result {0 {{0.0 1.0}}} {1 {SQL logic error}} {1 SQLITE_NOMEM}
+}
+
+do_faultsim_test 8.2 -faults oom* -prep {
+} -body {
+  execsql {
+    SELECT rowid FROM ft('one two') ORDER BY rank;
+  }
+} -test {
+  faultsim_test_result {0 1} {1 {SQL logic error}} {1 SQLITE_NOMEM}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 9.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x);
+  INSERT INTO ft VALUES('one two three i ii iii');
+  INSERT INTO ft VALUES('four five six iv v vi');
+  INSERT INTO ft VALUES('eight nine ten viii ix x');
+} {}
+
+faultsim_save_and_close
+
+do_faultsim_test 9.1 -faults oom* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql {
+    UPDATE ft SET rowid=4 WHERE rowid=1
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+do_faultsim_test 9.2 -faults oom* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql {
+    SELECT rowid FROM ft WHERE x MATCH 'one AND two AND three'
+  }
+} -test {
+  faultsim_test_result {0 1}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 10.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, locale=1);
+  INSERT INTO ft VALUES(fts5_locale('hello', 'one two three i ii iii'));
+  INSERT INTO ft VALUES('four five six iv v vi');
+  INSERT INTO ft VALUES('eight nine ten viii ix x');
+} {}
+
+do_execsql_test 10.1 {
+  SELECT fts5_get_locale(ft, 0) FROM ft WHERE x MATCH 'one AND two AND three'
+} {hello}
+
+faultsim_save_and_close
+do_faultsim_test 10.1 -faults oom* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql {
+    SELECT fts5_get_locale(ft, 0) FROM ft WHERE x MATCH 'one AND two AND three'
+  }
+} -test {
+  faultsim_test_result {0 hello}
+}
+
+breakpoint
+faultsim_save_and_close
+do_faultsim_test 10.2 -faults oom-t* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql {
+    INSERT INTO ft VALUES(zeroblob(10000));
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 11.0 {
+  CREATE VIRTUAL TABLE f1 USING fts5(content);
+  CREATE TABLE g1(id, content);
+  INSERT INTO g1 VALUES(30000, 'a b c');
+  INSERT INTO g1 VALUES(40000, 'd e f');
+}
+
+faultsim_save_and_close
+
+do_faultsim_test 11 -faults oom* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql {
+    INSERT INTO f1(rowid, content) SELECT id, content FROM g1;
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+finish_test
+
@@ -22,6 +22,7 @@ do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
 }

+unset -nocomplain res
 foreach {tn expr ok} {
  1 {^abc}           1
  2 {^abc + def}     1
@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5full

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5hash

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5integrity

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
@@ -153,6 +153,7 @@ do_execsql_test 5.3 {
  INSERT INTO gg(gg) VALUES('integrity-check');
 }

+unset -nocomplain res
 do_test 5.4.1 {
  set ok 0
  for {set i 0} {$i < 10000} {incr i} {
@@ -380,5 +381,32 @@ do_execsql_test 12.3 {
 } {ok}


+#-------------------------------------------------------------------
+reset_db
+do_execsql_test 13.1 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=ascii);
+  INSERT INTO t1 VALUES('a b c'), ('d e f');
+  PRAGMA integrity_check;
+} {ok}
+
+db close
+sqlite3 db test.db
+do_catchsql_test 13.2 {
+  PRAGMA integrity_check;
+} {0 ok}
+
+do_execsql_test 13.3 {
+  PRAGMA writable_schema = 1;
+  UPDATE sqlite_schema SET sql = 'CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=blah)'
+  WHERE name = 't1';
+}
+
+db close
+sqlite3 db test.db
+breakpoint
+do_catchsql_test 13.4 {
+  PRAGMA integrity_check;
+} {1 {SQL logic error}}
+

 finish_test
@@ -0,0 +1,56 @@
+# 2024 September 3
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file contains tests focused on the integrity-check procedure.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5integrity2
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(a, detail='none');
+  BEGIN;
+    INSERT INTO t2(rowid, a) VALUES(-1, 'hello world');
+    INSERT INTO t2(rowid, a) VALUES(9223372036854775807, 'hello world');
+  COMMIT;
+}
+
+do_execsql_test 2.1 {
+  SELECT rowid FROM t2('hello AND world');
+} {-1 9223372036854775807}
+
+#-------------------------------------------------------------------------
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, detail='none');
+  CREATE TABLE r1(r);
+  
+  WITH c(x) AS (VALUES(1) UNION SELECT x<<1 FROM c)
+    INSERT INTO r1(r) SELECT -1-x FROM c;
+
+  INSERT INTO t1(rowid, a) SELECT r, 'abc' FROM r1;
+}
+
+do_execsql_test 2.1 {
+  PRAGMA integrity_check;
+} {ok}
+
+do_execsql_test 2.2 {
+  SELECT rowid FROM t1('abc') ORDER BY +rowid;
+} [db eval {SELECT r FROM r1 ORDER BY r}]
+
+
+finish_test
@@ -33,6 +33,7 @@ proc progress_handler {args} {
  return 0
 }

+unset -nocomplain res
 foreach {tn sql} {
  1 { INSERT INTO t1(rowid, a) VALUES(0, 'z z z z') }
  2 { COMMIT }
@@ -64,4 +65,3 @@ foreach {tn sql} {
 }

 finish_test
-
@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5lastrowid

-# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
 ifcapable !fts5 {
  finish_test
  return
--- a/Show More
+++ b/Show More
@@ -1 +1 @@
 .46.0
 .47.0