Partial hand-merge of shell.c.in (not buildable)

FossilOrigin-Name: 62e90c9ba80c257da417ab57549f142f9628c10bd924ab3334a2c518976b3c6d
Merge console I/O changes for Windows CLI.
2023-12-03 19:32:37 +00:00 · 2023-11-21 18:26:06 +00:00 · 2023-11-21 15:55:31 +00:00 · 2023-11-21 12:02:04 +00:00 · 2023-11-20 15:54:00 +00:00 · 2023-11-20 13:06:59 +00:00
432 changed files with 48021 additions and 20375 deletions
@@ -52,11 +52,24 @@ TCC += @TCL_INCLUDE_SPEC@
 # The library that programs using TCL must link against.
 #
 LIBTCL = @TCL_LIB_SPEC@
+# The preprocessing options that are needed for images using the TCL library.
+INCTCL = @TCL_INCLUDE_SPEC@
+
+# TCCX is the C Compile and options for use in building shell extensions
+# that will run on the target platform. It relies on no in-tree includes,
+# using only pre-glommed source/header files in the current directory.
+TCCX = ${CC} ${CFLAGS} -fPIC -shared -I.
+TCCX += @EXTENSION_DEBUG@
+
+# TCXX is the C++ compile for building a shell extension from a C++ source.
+TCXX = ${CXX} ${CFLAGS} -fPIC -shared -I.
+TCXX += @EXTENSION_DEBUG@

 # Compiler options needed for programs that use the readline() library.
 #
 READLINE_FLAGS = -DHAVE_READLINE=@TARGET_HAVE_READLINE@ @TARGET_READLINE_INC@
 READLINE_FLAGS += -DHAVE_EDITLINE=@TARGET_HAVE_EDITLINE@
+READLINE_FLAGS += -DHAVE_LINENOISE=@TARGET_HAVE_LINENOISE@

 # The library that programs using readline() must link against.
 #
@@ -161,6 +174,9 @@ LTCOMPILE = $(LIBTOOL) --mode=compile --tag=CC $(TCC) $(LTCOMPILE_EXTRAS)
 LTLINK = $(LIBTOOL) --mode=link $(TCC) $(LTCOMPILE_EXTRAS) @LDFLAGS@ $(LTLINK_EXTRAS)
 LTINSTALL = $(LIBTOOL) --mode=install $(INSTALL)

+# Shell and extension source builder tool
+MKSHELL_TOOL = $(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl
+
 # You should not have to change anything below this line
 ###############################################################################

@@ -570,6 +586,11 @@ TESTPROGS = \
  dbhash$(TEXE) \
  sqltclsh$(TEXE)

+TESTPROGS_SHX = \
+  testfixture$(TEXE) \
+  sqlite3x$(TEXE) \
+  shell_extensions
+
 # Databases containing fuzzer test cases
 #
 FUZZDATA = \
@@ -646,6 +667,9 @@ SQLITE3_SHELL_TARGET_  = sqlite3$(TEXE)
 SQLITE3_SHELL_TARGET_1 =
 SQLITE3_SHELL_TARGET   = $(SQLITE3_SHELL_TARGET_@HAVE_WASI_SDK@)

+#
+SHELL_OPT_NOEXT = $(SHELL_OPT) -DSHELL_OMIT_EXTENSIONS=7
+
 # This is the default Makefile target.  The objects listed here
 # are what get build when you type just "make" with no arguments.
 #
@@ -670,6 +694,11 @@ libtclsqlite3.la:	tclsqlite.lo libsqlite3.la
 		-avoid-version

 sqlite3$(TEXE):	shell.c sqlite3.c
+	$(LTLINK) $(READLINE_FLAGS) $(SHELL_OPT_NOEXT) -o $@ \
+		shell.c sqlite3.c \
+		$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"
+
+sqlite3x$(TEXE): shell.c sqlite3.c
 	$(LTLINK) $(READLINE_FLAGS) $(SHELL_OPT) -o $@ \
 		shell.c sqlite3.c \
 		$(LIBREADLINE) $(TLIBS) -rpath "$(libdir)"
@@ -706,6 +735,24 @@ fuzzcheck$(TEXE):	$(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP)
 fuzzcheck-asan$(TEXE):	$(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP)
 	$(LTLINK) -o $@ -fsanitize=address $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)

+fuzzcheck-ubsan$(TEXE):	$(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP)
+	$(LTLINK) -o $@ -fsanitize=undefined $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)
+
+# Usage:    FUZZDB=filename make run-fuzzcheck
+#
+# Where filename is a fuzzcheck database, this target builds and runs
+# fuzzcheck, fuzzcheck-asan, and fuzzcheck-ubsan on that database.
+#
+# FUZZDB can be a glob pattern of two or more databases. Example:
+#
+#     FUZZDB=test/fuzzdata*.db make run-fuzzcheck
+#
+run-fuzzcheck:	fuzzcheck$(TEXE) fuzzcheck-asan$(TEXE) fuzzcheck-ubsan$(TEXE)
+	@if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi
+	./fuzzcheck$(TEXE) --spinner $(FUZZDB)
+	./fuzzcheck-asan$(TEXE) --spinner $(FUZZDB)
+	./fuzzcheck-ubsan$(TEXE) --spinner $(FUZZDB)
+
 ossshell$(TEXE):	$(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \
             $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS)
@@ -765,13 +812,22 @@ mptest:	mptester$(TEXE)
 	$(MPTEST2) --journalmode DELETE


+has_tclsh84:
+	sh $(TOP)/tool/cktclsh.sh 8.4 $(TCLSH_CMD)
+	touch has_tclsh84
+
+has_tclsh85:
+	sh $(TOP)/tool/cktclsh.sh 8.5 $(TCLSH_CMD)
+	touch has_tclsh85
+
+
 # This target creates a directory named "tsrc" and fills it with
 # copies of all of the C source code and header files needed to
 # build on the target system.  Some of the C source code and header
 # files are automatically generated.  This target takes care of
 # all that automatic generation.
 #
-.target_source:	$(SRC) $(TOP)/tool/vdbe-compress.tcl fts5.c
+.target_source:	$(SRC) $(TOP)/tool/vdbe-compress.tcl has_tclsh84 fts5.c
 	rm -rf tsrc
 	mkdir tsrc
 	cp -f $(SRC) tsrc
@@ -781,15 +837,15 @@ mptest:	mptester$(TEXE)
 	cp fts5.c fts5.h tsrc
 	touch .target_source

-sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl src-verify
+sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl src-verify has_tclsh84
 	$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl $(AMALGAMATION_LINE_MACROS)
 	cp tsrc/sqlite3ext.h .
 	cp $(TOP)/ext/session/sqlite3session.h .

-sqlite3r.h: sqlite3.h
+sqlite3r.h: sqlite3.h has_tclsh84
 	$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) --enable-recover >sqlite3r.h

-sqlite3r.c: sqlite3.c sqlite3r.h
+sqlite3r.c: sqlite3.c sqlite3r.h has_tclsh84
 	cp $(TOP)/ext/recover/sqlite3recover.c tsrc/
 	cp $(TOP)/ext/recover/sqlite3recover.h tsrc/
 	cp $(TOP)/ext/recover/dbdata.c tsrc/
@@ -804,7 +860,7 @@ tclsqlite3.c:	sqlite3.c
 	echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c
 	cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c

-sqlite3-all.c:	sqlite3.c $(TOP)/tool/split-sqlite3c.tcl
+sqlite3-all.c:	sqlite3.c $(TOP)/tool/split-sqlite3c.tcl has_tclsh84
 	$(TCLSH_CMD) $(TOP)/tool/split-sqlite3c.tcl

 # Rule to build the amalgamation
@@ -1092,10 +1148,10 @@ tclsqlite3$(TEXE):	tclsqlite-shell.lo libsqlite3.la

 # Rules to build opcodes.c and opcodes.h
 #
-opcodes.c:	opcodes.h $(TOP)/tool/mkopcodec.tcl
+opcodes.c:	opcodes.h $(TOP)/tool/mkopcodec.tcl has_tclsh84
 	$(TCLSH_CMD) $(TOP)/tool/mkopcodec.tcl opcodes.h >opcodes.c

-opcodes.h:	parse.h $(TOP)/src/vdbe.c $(TOP)/tool/mkopcodeh.tcl
+opcodes.h:	parse.h $(TOP)/src/vdbe.c $(TOP)/tool/mkopcodeh.tcl has_tclsh84
 	cat parse.h $(TOP)/src/vdbe.c | $(TCLSH_CMD) $(TOP)/tool/mkopcodeh.tcl >opcodes.h

 # Rules to build parse.c and parse.h - the outputs of lemon.
@@ -1106,10 +1162,10 @@ parse.c:	$(TOP)/src/parse.y lemon$(BEXE)
 	cp $(TOP)/src/parse.y .
 	./lemon$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) -S parse.y

-sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid$(BEXE) $(TOP)/VERSION
+sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid$(BEXE) $(TOP)/VERSION has_tclsh84
 	$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h

-sqlite3rc.h:	$(TOP)/src/sqlite3.rc $(TOP)/VERSION
+sqlite3rc.h:	$(TOP)/src/sqlite3.rc $(TOP)/VERSION has_tclsh84
 	echo '#ifndef SQLITE_RESOURCE_VERSION' >$@
 	echo -n '#define SQLITE_RESOURCE_VERSION ' >>$@
 	cat $(TOP)/VERSION | $(TCLSH_CMD) $(TOP)/tool/replace.tcl exact . , >>$@
@@ -1119,37 +1175,40 @@ keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
 	$(BCC) -o mkkeywordhash$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c
 	./mkkeywordhash$(BEXE) >keywordhash.h

+$(TOP)/ext/misc/basexx.c: $(TOP)/ext/misc/base64.c $(TOP)/ext/misc/base85.c
+
 # Source files that go into making shell.c
 SHELL_SRC = \
-	$(TOP)/src/shell.c.in \
-        $(TOP)/ext/misc/appendvfs.c \
-	$(TOP)/ext/misc/completion.c \
-        $(TOP)/ext/misc/decimal.c \
-        $(TOP)/ext/misc/basexx.c \
-        $(TOP)/ext/misc/base64.c \
-        $(TOP)/ext/misc/base85.c \
-	$(TOP)/ext/misc/fileio.c \
-        $(TOP)/ext/misc/ieee754.c \
-        $(TOP)/ext/misc/regexp.c \
-        $(TOP)/ext/misc/series.c \
-	$(TOP)/ext/misc/shathree.c \
-	$(TOP)/ext/misc/sqlar.c \
-        $(TOP)/ext/misc/uint.c \
-	$(TOP)/ext/expert/sqlite3expert.c \
-	$(TOP)/ext/expert/sqlite3expert.h \
-	$(TOP)/ext/misc/zipfile.c \
-	$(TOP)/ext/misc/memtrace.c \
-	$(TOP)/ext/misc/pcachetrace.c \
-	$(TOP)/ext/recover/dbdata.c \
-	$(TOP)/ext/recover/sqlite3recover.c \
-	$(TOP)/ext/recover/sqlite3recover.h \
-        $(TOP)/src/test_windirent.c
-
-shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
-	$(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl >shell.c
-
-
+  $(TOP)/src/shell.c.in \
+  $(TOP)/ext/consio/console_io.c \
+  $(TOP)/ext/consio/console_io.h \
+  $(TOP)/ext/misc/appendvfs.c \
+  $(TOP)/ext/misc/completion.c \
+  $(TOP)/ext/expert/sqlite3expert.c \
+  $(TOP)/ext/expert/sqlite3expert.h \
+  $(TOP)/ext/misc/appendvfs.c \
+  $(TOP)/ext/misc/basexx.c \
+  $(TOP)/ext/misc/completion.c \
+  $(TOP)/ext/misc/decimal.c \
+  $(TOP)/ext/misc/fileio.c \
+  $(TOP)/ext/misc/ieee754.c \
+  $(TOP)/ext/misc/regexp.c \
+  $(TOP)/ext/misc/series.c \
+  $(TOP)/ext/misc/shathree.c \
+  $(TOP)/ext/misc/sqlar.c \
+  $(TOP)/ext/misc/uint.c \
+  $(TOP)/ext/misc/memtrace.c \
+  $(TOP)/ext/misc/pcachetrace.c \
+  $(TOP)/src/shext_linkage.h \
+  $(TOP)/src/obj_interfaces.h \
+  $(TOP)/src/resmanage.h \
+  $(TOP)/src/resmanage.c \
+  $(TOP)/ext/recover/dbdata.c \
+  $(TOP)/ext/misc/zipfile.c \
+  $(TOP)/src/test_windirent.c

+shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl has_tclsh84
+	$(MKSHELL_TOOL) >shell.c

 # Rules to build the extension objects.
 #
@@ -1233,7 +1292,7 @@ fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon$(BEXE)

 fts5parse.h: fts5parse.c

-fts5.c: $(FTS5_SRC)
+fts5.c: $(FTS5_SRC) has_tclsh84
 	$(TCLSH_CMD) $(TOP)/ext/fts5/tool/mkfts5c.tcl
 	cp $(TOP)/ext/fts5/fts5.h .

@@ -1243,6 +1302,54 @@ fts5.lo:	fts5.c $(HDR) $(EXTHDR)
 sqlite3rbu.lo:	$(TOP)/ext/rbu/sqlite3rbu.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)/ext/rbu/sqlite3rbu.c

+# Rule to build shx_link.h (for use by shell extensions)
+
+SHX_LINK_SRC = \
+ $(TOP)/src/shext_linkage.h \
+ $(TOP)/src/obj_interfaces.h \
+ sqlite3ext.h \
+ sqlite3.h
+
+shx_link.h:	$(SHX_LINK_SRC)
+	$(MKSHELL_TOOL) -short-head -header-gen \
+                $(TOP)/src/shext_linkage.h > $@
+
+# Rules to build shell extensions used for testing extensible shell
+
+shell_extensions:	tcl_shell_extension \
+  test_shellext_cpp$(SHLIB_SUFFIX)  test_shellext_c$(SHLIB_SUFFIX)
+
+test_shellext_cpp$(SHLIB_SUFFIX):	$(TOP)/src/test_shellext_cpp.cpp \
+  $(TOP)/shx_link.h
+	$(TCXX) $(TOP)/src/test_shellext_cpp.cpp -o $@
+
+test_shellext_c$(SHLIB_SUFFIX):	$(TOP)/src/test_shellext_c.c \
+  $(TOP)/shx_link.h
+	$(TCCX) $(TOP)/src/test_shellext_c.c -o $@
+
+# Rules to build the TCL shell extension (tclshext.{so,dll,dylib})
+
+tcl_shell_extension:	tclshext$(SHLIB_SUFFIX)
+
+# This is a work-around for building Tk variant. Ultimately, the configure
+# script will find the Tk library with the same version as the Tcl library.
+# This has the same effect with the lib naming now used by the Tcl project,
+# but, if it will fail, does so when make rather than configure is run.
+TCL_LIBS = $(LIBTCL)
+ifdef WITH_TK
+ ifneq ($(WITH_TK),0)
+TCLEXT_OPTS += -DSHELL_ENABLE_TK
+LIBTK := $(patsubst -ltcl%,-ltk%,$(LIBTCL))
+TCL_LIBS += $(LIBTK)
+ endif
+endif
+
+tclshext.c:	$(TOP)/ext/misc/tclshext.c.in $(TOP)/src/tclsqlite.c
+	$(MKSHELL_TOOL) $(TOP)/ext/misc/tclshext.c.in > $@
+
+tclshext$(SHLIB_SUFFIX): tclshext.c shx_link.h
+	$(TCCX) $(INCTCL) $(TCLEXT_OPTS) tclshext.c \
+ -o $@ $(TCL_LIBS) $(TCLEXT_LDOPTS)

 # Rules to build the 'testfixture' application.
 #
@@ -1267,7 +1374,7 @@ TESTFIXTURE_SRC1 = sqlite3.c
 TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c
 TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION))

-testfixture$(TEXE):	$(TESTFIXTURE_SRC)
+testfixture$(TEXE):	has_tclsh85 $(TESTFIXTURE_SRC)
 	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \
 		-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)

@@ -1291,11 +1398,17 @@ fulltestonly:	$(TESTPROGS) fuzztest
 	./testfixture$(TEXE) $(TOP)/test/full.test

 # Fuzz testing
-fuzztest:	fuzzcheck$(TEXE) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessionfuzz-data1.db
+#
+# WARNING: When the "fuzztest" target is run by the testrunner.tcl script,
+# it does not actually run this code. Instead, it schedules equivalent 
+# commands. Therefore, if this target is updated, then code in
+# testrunner_data.tcl (search for "trd_fuzztest_data") must also be updated.
+#
+fuzztest:	fuzzcheck$(TEXE) $(FUZZDATA) sessionfuzz$(TEXE)
 	./fuzzcheck$(TEXE) $(FUZZDATA)
 	./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db

-valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessionfuzz-data1.db
+valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA) sessionfuzz$(TEXE)
 	valgrind ./fuzzcheck$(TEXE) --cell-size-check --limit-mem 10M $(FUZZDATA)
 	valgrind ./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db

@@ -1312,11 +1425,23 @@ testrunner:	testfixture$(TEXE)

 # Runs both fuzztest and testrunner, consecutively.
 #
-devtest:	testfixture$(TEXE) fuzztest testrunner
+devtest:	srctree-check testfixture$(TEXE) fuzztest testrunner
+
+mdevtest: srctree-check has_tclsh85
+	$(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest
+
+sdevtest: has_tclsh85
+	$(TCLSH_CMD) $(TOP)/test/testrunner.tcl sdevtest
+
+# Validate that various generated files in the source tree
+# are up-to-date.
+#
+srctree-check:	$(TOP)/tool/srctree-check.tcl
+	$(TCLSH_CMD) $(TOP)/tool/srctree-check.tcl

 # Testing for a release
 #
-releasetest: testfixture$(TEXE)
+releasetest: srctree-check testfixture$(TEXE)
 	./testfixture$(TEXE) $(TOP)/test/testrunner.tcl release

 # Minimal testing that runs in less than 3 minutes
@@ -1327,7 +1452,7 @@ quicktest:	./testfixture$(TEXE)
 # This is the common case.  Run many tests that do not take too long,
 # including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
 #
-test:	fuzztest sourcetest $(TESTPROGS) tcltest
+test:	srctree-check fuzztest sourcetest $(TESTPROGS) tcltest

 # Run a test using valgrind.  This can take a really long time
 # because valgrind is so much slower than a native machine.
@@ -1342,16 +1467,23 @@ valgrindtest:	$(TESTPROGS) valgrindfuzz
 smoketest:	$(TESTPROGS) fuzzcheck$(TEXE)
 	./testfixture$(TEXE) $(TOP)/test/main.test $(TESTOPTS)

-shelltest: $(TESTPROGS)
-	./testfixture$(TEXT) $(TOP)/test/permutations.test shell
+TESTRUNS_SHX = \
+ $(TOP)/test/shell_x/shell9.test \
+ $(TOP)/test/shell_x/shell10.test

-sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in
+shellxtest: $(TESTPROGS_SHX)
+	./testfixture$(TEXE) $(TESTRUNS_SHX)
+
+shelltest: $(TESTPROGS)
+	./testfixture$(TEXE) $(TOP)/test/permutations.test shell
+
+sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in has_tclsh85
 	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c

 sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
 	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)

-sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in
+sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in has_tclsh85
 	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c

 sqltclsh$(TEXE): sqltclsh.c
@@ -1370,7 +1502,7 @@ CHECKER_DEPS =\
  $(TOP)/ext/misc/btreeinfo.c \
  $(TOP)/ext/repair/sqlite3_checker.c.in

-sqlite3_checker.c:	$(CHECKER_DEPS)
+sqlite3_checker.c:	$(CHECKER_DEPS) has_tclsh85
 	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@

 sqlite3_checker$(TEXE):	sqlite3_checker.c
@@ -1456,6 +1588,11 @@ amalgamation-tarball: sqlite3.c sqlite3rc.h
 snapshot-tarball: sqlite3.c sqlite3rc.h
 	TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --snapshot

+# Build a ZIP archive containing various command-line tools.
+#
+tool-zip:	testfixture sqlite3 sqldiff sqlite3_analyzer $(TOP)/tool/mktoolzip.tcl
+	./testfixture $(TOP)/tool/mktoolzip.tcl
+
 # The next two rules are used to support the "threadtest" target. Building
 # threadtest runs a few thread-safety tests that are implemented in C. This
 # target is invoked by the releasetest.tcl script.
@@ -1518,6 +1655,7 @@ clean:
 	rm -f sqlite3.c
 	rm -f sqlite3rc.h
 	rm -f shell.c sqlite3ext.h
+	rm -f shx_link.h tclshext.c tclshext$(SHLIB_SUFFIX)
 	rm -f sqlite3_analyzer$(TEXE) sqlite3_analyzer.c
 	rm -f sqlite-*-output.vsix
 	rm -f mptester mptester.exe
@@ -1531,6 +1669,7 @@ clean:
 	rm -f threadtest5
 	rm -f src-verify
 	rm -f custom.rws
+	rm -f has_tclsh84 has_tclsh85

 distclean:	clean
 	rm -f sqlite_cfg.h config.log config.status libtool Makefile sqlite3.pc \
@@ -1575,7 +1714,7 @@ fiddle: sqlite3.c shell.c
 	@echo 'Updating custom dictionary from tool/custom.txt'
 	aspell --lang=en create master ./custom.rws < $<

-misspell: ./custom.rws
+misspell: ./custom.rws has_tclsh84
 	$(TCLSH_CMD) ./tool/spellsift.tcl ./src/*.c ./src/*.h ./src/*.in

 #
@@ -52,11 +52,11 @@ MINIMAL_AMALGAMATION = 0
 USE_STDCALL = 0
 !ENDIF

-# Set this non-0 to use structured exception handling (SEH) for WAL mode
-# in the core library.
+# Use the USE_SEH=0 option on the nmake command line to omit structured
+# exception handling (SEH) support.  SEH is on by default.
 #
 !IFNDEF USE_SEH
-USE_SEH = 0
+USE_SEH = 1
 !ENDIF

 # Set this non-0 to have the shell executable link against the core dynamic
@@ -225,6 +225,12 @@ WIN32HEAP = 0
 OSTRACE = 0
 !ENDIF

+# enable address sanitizer using ASAN=1 on the command-line.
+#
+!IFNDEF ASAN
+ASAN = 0
+!ENDIF
+
 # Set this to one of the following values to enable various debugging
 # features.  Each level includes the debugging options from the previous
 # levels.  Currently, the recognized values for DEBUG are:
@@ -318,6 +324,7 @@ SQLITE3EXE = winsqlite3shell.exe
 SQLITE3EXE = sqlite3.exe
 !ENDIF
 !ENDIF
+SQLITE3XEXE = sqlite3x.exe

 # This is the argument used to set the program database (PDB) file for the
 # SQLite shell executable (EXE).
@@ -368,6 +375,7 @@ SQLITE_TCL_DEP =
 !IFNDEF OPT_FEATURE_FLAGS
 !IF $(MINIMAL_AMALGAMATION)==0
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS5=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
@@ -397,10 +405,11 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RBU=1
 !ENDIF

 # Should structured exception handling (SEH) be enabled for WAL mode in
-# the core library?
+# the core library?  It is on by default.  Only omit it if the
+# USE_SEH=0 option is provided on the nmake command-line.
 #
-!IF $(USE_SEH)!=0
-OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_USE_SEH=1
+!IF $(USE_SEH)==0
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_OMIT_SEH=1
 !ENDIF

 # These are the "extended" SQLite compilation options used when compiling for
@@ -891,6 +900,13 @@ RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
 !ENDIF
 !ENDIF

+
+# Address sanitizer if ASAN=1
+#
+!IF $(ASAN)>0
+TCC = $(TCC) /fsanitize=address
+!ENDIF
+
 # <<mark>>
 # The locations of the Tcl header and library files.  Also, the library that
 # non-stubs enabled programs using Tcl must link against.  These variables
@@ -922,6 +938,14 @@ TCLLIBDIR = $(TCLDIR)\lib
 LIBTCL = tcl$(TCLVERSION)$(TCLSUFFIX).lib
 !ENDIF

+# For Tcl shell extension only:
+!IFDEF WITH_TK
+!IFNDEF LIBTK
+LIBTK = tk$(TCLVERSION)$(TCLSUFFIX).lib
+!ENDIF
+!ENDIF
+
+
 !IFNDEF LIBTCLSTUB
 LIBTCLSTUB = tclstub$(TCLVERSION)$(TCLSUFFIX).lib
 !ENDIF
@@ -1579,8 +1603,7 @@ TESTEXT = \
  $(TOP)\ext\rtree\test_rtreedoc.c \
  $(TOP)\ext\recover\sqlite3recover.c \
  $(TOP)\ext\recover\test_recover.c \
-  $(TOP)\ext\recover\dbdata.c \
-  fts5.c
+  $(TOP)\ext\recover\dbdata.c 

 # If use of zlib is enabled, add the "zipfile.c" source file.
 #
@@ -1596,7 +1619,8 @@ TESTSRC2 = \
  $(SRC01) \
  $(SRC07) \
  $(SRC10) \
-  $(TOP)\ext\async\sqlite3async.c
+  $(TOP)\ext\async\sqlite3async.c \
+  fts5.c

 # Header files used by all library source files.
 #
@@ -1654,6 +1678,11 @@ TESTPROGS = \
  dbhash.exe \
  sqltclsh.exe

+TESTPROGS_SHX = \
+  testfixture.exe \
+  $(SQLITE3XEXE) \
+  shell_extensions
+
 # Databases containing fuzzer test cases
 #
 FUZZDATA = \
@@ -1675,8 +1704,13 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_DQS=0
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1
 !ENDIF

+# 
+SHELL_NOEXT_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSHELL_OMIT_EXTENSIONS=7
+
 # <<mark>>
 # Extra compiler options for various test tools.
 #
@@ -1802,12 +1836,16 @@ sqlite3.def:	libsqlite3.lib
 # <</block2>>

 $(SQLITE3EXE):	shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
+	$(LTLINK) $(SHELL_NOEXT_COMPILE_OPTS) $(READLINE_FLAGS) shell.c $(SHELL_CORE_SRC) \
+		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
+
+$(SQLITE3XEXE):	shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLITE3H)
 	$(LTLINK) $(SHELL_COMPILE_OPTS) $(READLINE_FLAGS) shell.c $(SHELL_CORE_SRC) \
 		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)

 # <<mark>>
-sqldiff.exe:	$(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H)
-	$(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+sqldiff.exe:	$(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS)
+	$(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS)

 dbhash.exe:	$(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
@@ -2243,26 +2281,30 @@ keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe

 # Source files that go into making shell.c
 SHELL_SRC = \
-	$(TOP)\src\shell.c.in \
-	$(TOP)\ext\misc\appendvfs.c \
-	$(TOP)\ext\misc\completion.c \
-	$(TOP)\ext\misc\base64.c \
-	$(TOP)\ext\misc\base85.c \
-	$(TOP)\ext\misc\decimal.c \
-	$(TOP)\ext\misc\fileio.c \
-	$(TOP)\ext\misc\ieee754.c \
-	$(TOP)\ext\misc\regexp.c \
-	$(TOP)\ext\misc\series.c \
-	$(TOP)\ext\misc\shathree.c \
-	$(TOP)\ext\misc\uint.c \
-	$(TOP)\ext\expert\sqlite3expert.c \
-	$(TOP)\ext\expert\sqlite3expert.h \
-	$(TOP)\ext\misc\memtrace.c \
-	$(TOP)\ext\misc\pcachetrace.c \
-	$(TOP)\ext\recover\dbdata.c \
-	$(TOP)\ext\recover\sqlite3recover.c \
-	$(TOP)\ext\recover\sqlite3recover.h \
-	$(TOP)\src\test_windirent.c
+  $(TOP)\src\shell.c.in \
+  $(TOP)\ext\consio\console_io.c \
+  $(TOP)\ext\consio\console_io.h \
+  $(TOP)\ext\misc\appendvfs.c \
+  $(TOP)\ext\misc\completion.c \
+  $(TOP)\ext\misc\base64.c \
+  $(TOP)\ext\misc\base85.c \
+  $(TOP)\ext\misc\decimal.c \
+  $(TOP)\ext\misc\fileio.c \
+  $(TOP)\ext\misc\ieee754.c \
+  $(TOP)\ext\misc\regexp.c \
+  $(TOP)\ext\misc\series.c \
+  $(TOP)\ext\misc\shathree.c \
+  $(TOP)\ext\misc\uint.c \
+  $(TOP)\ext\expert\sqlite3expert.c \
+  $(TOP)\ext\expert\sqlite3expert.h \
+  $(TOP)\ext\misc\memtrace.c \
+  $(TOP)\ext\misc\pcachetrace.c \
+  $(TOP)\ext\recover\dbdata.c \
+  $(TOP)\ext\recover\sqlite3recover.c \
+  $(TOP)\ext\recover\sqlite3recover.h \
+  $(TOP)\src\resmanage.h \
+  $(TOP)\src\resmanage.c \
+  $(TOP)\src\test_windirent.c

 # If use of zlib is enabled, add the "zipfile.c" source file.
 #
@@ -2397,6 +2439,45 @@ fts5.dll:	fts5_ext.lo
 sqlite3rbu.lo:	$(TOP)\ext\rbu\sqlite3rbu.c $(HDR) $(EXTHDR)
 	$(LTCOMPILE) -DSQLITE_CORE -c $(TOP)\ext\rbu\sqlite3rbu.c

+# Rule to build shx_link.h (for use by shell extensions)
+
+SHX_LINK_SRC = \
+ $(TOP)/src/shext_linkage.h \
+ $(TOP)/src/obj_interfaces.h \
+ sqlite3ext.h \
+ sqlite3.h
+
+shx_link.h:	$(SHX_LINK_SRC)
+	$(TCLSH_CMD) $(TOP)\tool\mkshellc.tcl -short-head -header-gen \
+     $(TOP)/src/shext_linkage.h > $@
+
+# Rules to build shell extensions used for testing extensible shell
+
+shell_extensions:	tcl_shell_extension \
+  test_shellext_cpp.dll test_shellext_c.dll
+
+test_shellext_cpp.dll:	$(TOP)/src/test_shellext_cpp.cpp $(TOP)/shx_link.h
+	$(TCC) $(TOP)/src/test_shellext_cpp.cpp -LD -Fe$@
+
+test_shellext_c.dll:	$(TOP)/src/test_shellext_c.c $(TOP)/shx_link.h
+	$(TCC) $(TOP)/src/test_shellext_c.c -LD -Fe$@
+
+tcl_shell_extension:	tclshext.dll
+
+!IFDEF WITH_TK
+TCLEXT_OPTS=-DSHELL_ENABLE_TK
+TCLEXT_LIBS = $(TCLLIBS) $(LIBTK)
+!ELSE
+TCLEXT_OPTS=
+TCLEXT_LIBS = $(TCLLIBS)
+!ENDIF
+tclshext.c:	$(TOP)/ext/misc/tclshext.c.in $(TOP)/src/tclsqlite.c
+	$(TCLSH_CMD) $(TOP)\tool\mkshellc.tcl $(TOP)/ext/misc/tclshext.c.in > $@
+
+tclshext.dll: tclshext.c shx_link.h
+	$(TCC) -I$(TCLINCDIR) $(TCLEXT_OPTS) tclshext.c -Fe$@ -link -DLL \
+  -LIBPATH:$(TCLLIBDIR) $(TCLEXT_LIBS)
+
 # Rules to build the 'testfixture' application.
 #
 # If using the amalgamation, use sqlite3.c directly to build the test
@@ -2453,6 +2534,9 @@ extensiontest:	testfixture.exe testloadext.dll
 	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS)

+tool-zip:	testfixture.exe sqlite3.exe sqldiff.exe sqlite3_analyzer.exe $(TOP)\tool\mktoolzip.tcl
+	.\testfixture.exe $(TOP)\tool\mktoolzip.tcl
+
 coretestprogs:	$(TESTPROGS)

 testprogs:	coretestprogs srcck1.exe fuzzcheck.exe sessionfuzz.exe
@@ -2505,6 +2589,9 @@ testrunner:	testfixture.exe
 #
 devtest:	testfixture.exe fuzztest testrunner

+mdevtest:
+	$(TCLSH_CMD) $(TOP)\test\testrunner.tcl mdevtest
+
 # Testing for a release
 #
 releasetest: testfixture.exe fuzztest
@@ -2515,6 +2602,13 @@ smoketest:	$(TESTPROGS)
 	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\main.test $(TESTOPTS)

+TESTRUNS_SHX = \
+ $(TOP)/test/shell_x/shell9.test \
+ $(TOP)/test/shell_x/shell10.test
+
+shellxtest: $(TESTPROGS_SHX)
+	.\testfixture.exe $(TESTRUNS_SHX)
+
 shelltest: $(TESTPROGS)
 	.\testfixture.exe $(TOP)\test\permutations.test shell

@@ -41,11 +41,10 @@ email to drh at sqlite dot org.
 The SQLite source code is in the public domain.  See
 <https://sqlite.org/copyright.html> for details. 

-Because SQLite is in the public domain,
-we cannot accept pull requests, because
-if we did accept a pull request, the changes in that pull request would
-carry a copyright and the SQLite source code would no longer be fully in
-the public domain.
+Because SQLite is in the public domain, we do not normally accept pull
+requests, because if we did take a pull request, the changes in that
+pull request might carry a copyright and the SQLite source code would
+then no longer be fully in the public domain.

 ## Obtaining The SQLite Source Code

@@ -104,9 +103,9 @@ For example:
        mkdir bld                ;#  Build will occur in a sibling directory
        cd bld                   ;#  Change to the build directory
        ../sqlite/configure      ;#  Run the configure script
-        make                     ;#  Run the makefile.
+        make                     ;#  Builds the "sqlite3" command-line tool
        make sqlite3.c           ;#  Build the "amalgamation" source file
-        make test                ;#  Run some tests (requires Tcl)
+        make devtest             ;#  Run some tests (requires Tcl)

 See the makefile for additional targets.

@@ -116,32 +115,33 @@ script does not work out for you, there is a generic makefile named
 can copy and edit to suit your needs.  Comments on the generic makefile
 show what changes are needed.

-## Using MSVC for Windows systems
+## Compiling for Windows Using MSVC

 On Windows, all applicable build products can be compiled with MSVC.
-First open the command prompt window associated with the desired compiler
-version (e.g. "Developer Command Prompt for VS2013").  Next, use NMAKE
-with the provided "Makefile.msc" to build one of the supported targets.
+You will also need a working installation of TCL.
+See the [compile-for-windows.md](doc/compile-for-windows.md) document for
+additional information about how to install MSVC and TCL and configure your
+build environment.

-For example, from the parent directory of the source subtree named "sqlite":
+If you want to run tests, you need to let SQLite know the location of your
+TCL library, using a command like this:

-        mkdir bld
-        cd bld
-        nmake /f ..\sqlite\Makefile.msc TOP=..\sqlite
-        nmake /f ..\sqlite\Makefile.msc sqlite3.c TOP=..\sqlite
-        nmake /f ..\sqlite\Makefile.msc sqlite3.dll TOP=..\sqlite
-        nmake /f ..\sqlite\Makefile.msc sqlite3.exe TOP=..\sqlite
-        nmake /f ..\sqlite\Makefile.msc test TOP=..\sqlite
+        set TCLDIR=c:\Tcl

-There are several build options that can be set via the NMAKE command
-line.  For example, to build for WinRT, simply add "FOR_WINRT=1" argument
-to the "sqlite3.dll" command line above.  When debugging into the SQLite
-code, adding the "DEBUG=1" argument to one of the above command lines is
-recommended.
+SQLite uses "tclsh.exe" as part of the build process, and so that utility
+program will need to be somewhere on your %PATH%.  The finished SQLite library
+does not contain any TCL code, but it does use TCL to help with the build process
+and to run tests.

-SQLite does not require [Tcl](http://www.tcl.tk/) to run, but a Tcl installation
-is required by the makefiles (including those for MSVC).  SQLite contains
-a lot of generated code and Tcl is used to do much of that code generation.
+Build using Makefile.msc.  Example:
+
+        nmake /f Makefile.msc
+        nmake /f Makefile.msc sqlite3.c
+        nmake /f Makefile.msc devtest
+        nmake /f Makefile.msc releasetest
+ 
+There are many other makefile targets.  See comments in Makefile.msc for
+details.

 ## Source Code Tour

@@ -1 +1 @@
-3.43.0
+3.45.0
@@ -9,7 +9,7 @@ sqlite3_SOURCES = shell.c sqlite3.h
 EXTRA_sqlite3_SOURCES = sqlite3.c
 sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@
 sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@
-sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB $(SHELL_CFLAGS)
+sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB $(SHELL_CFLAGS)

 include_HEADERS = sqlite3.h sqlite3ext.h

@@ -52,11 +52,11 @@ MINIMAL_AMALGAMATION = 0
 USE_STDCALL = 0
 !ENDIF

-# Set this non-0 to use structured exception handling (SEH) for WAL mode
-# in the core library.
+# Use the USE_SEH=0 option on the nmake command line to omit structured
+# exception handling (SEH) support.  SEH is on by default.
 #
 !IFNDEF USE_SEH
-USE_SEH = 0
+USE_SEH = 1
 !ENDIF

 # Set this non-0 to have the shell executable link against the core dynamic
@@ -187,6 +187,12 @@ WIN32HEAP = 0
 OSTRACE = 0
 !ENDIF

+# enable address sanitizer using ASAN=1 on the command-line.
+#
+!IFNDEF ASAN
+ASAN = 0
+!ENDIF
+
 # Set this to one of the following values to enable various debugging
 # features.  Each level includes the debugging options from the previous
 # levels.  Currently, the recognized values for DEBUG are:
@@ -290,6 +296,7 @@ SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
 !IFNDEF OPT_FEATURE_FLAGS
 !IF $(MINIMAL_AMALGAMATION)==0
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS5=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
@@ -319,10 +326,11 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RBU=1
 !ENDIF

 # Should structured exception handling (SEH) be enabled for WAL mode in
-# the core library?
+# the core library?  It is on by default.  Only omit it if the
+# USE_SEH=0 option is provided on the nmake command-line.
 #
-!IF $(USE_SEH)!=0
-OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_USE_SEH=1
+!IF $(USE_SEH)==0
+OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_OMIT_SEH=1
 !ENDIF

 # These are the "extended" SQLite compilation options used when compiling for
@@ -732,6 +740,13 @@ RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
 !ENDIF


+# Address sanitizer if ASAN=1
+#
+!IF $(ASAN)>0
+TCC = $(TCC) /fsanitize=address
+!ENDIF
+
+
 # Compiler options needed for programs that use the readline() library.
 #
 !IFNDEF READLINE_FLAGS
@@ -973,6 +988,8 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_DQS=0
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1
+SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1
 !ENDIF


@@ -19,7 +19,7 @@ dnl	to configure the system for the local environment.
 # so that we create the export library with the dll.
 #-----------------------------------------------------------------------

-AC_INIT([sqlite],[3.42.0])
+AC_INIT([sqlite],[3.45.0])

 #--------------------------------------------------------------------
 # Call TEA_INIT as the first TEA_ macro to set up initial vars.
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.43.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.45.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.43.0'
-PACKAGE_STRING='sqlite 3.43.0'
+PACKAGE_VERSION='3.45.0'
+PACKAGE_STRING='sqlite 3.45.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''

@@ -776,6 +776,8 @@ OPT_FEATURE_FLAGS
 HAVE_ZLIB
 USE_AMALGAMATION
 TARGET_DEBUG
+EXTENSION_DEBUG
+TARGET_HAVE_LINENOISE
 TARGET_HAVE_EDITLINE
 TARGET_HAVE_READLINE
 TARGET_READLINE_INC
@@ -903,6 +905,7 @@ enable_editline
 enable_readline
 with_readline_lib
 with_readline_inc
+with_linenoise
 enable_debug
 enable_amalgamation
 enable_load_extension
@@ -1470,7 +1473,7 @@ if test "$ac_init_help" = "long"; then
  # Omit some internal or obsolete options to make the list less imposing.
  # This message is too long to be a string in the A/UX 3.1 sh.
  cat <<_ACEOF
-\`configure' configures sqlite 3.43.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.45.0 to adapt to many kinds of systems.

 Usage: $0 [OPTION]... [VAR=VALUE]...

@@ -1535,7 +1538,7 @@ fi

 if test -n "$ac_init_help"; then
  case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.43.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.45.0:";;
   esac
  cat <<\_ACEOF

@@ -1587,6 +1590,7 @@ Optional Packages:
                          (tclConfig.sh)
  --with-readline-lib     specify readline library
  --with-readline-inc     specify readline include paths
+  --with-linenoise=DIR    source directory for linenoise library

 Some influential environment variables:
  CC          C compiler command
@@ -1665,7 +1669,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
  cat <<\_ACEOF
-sqlite configure 3.43.0
+sqlite configure 3.45.0
 generated by GNU Autoconf 2.69

 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2084,7 +2088,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.

-It was created by sqlite $as_me 3.43.0, which was
+It was created by sqlite $as_me 3.45.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  $ $0 $@
@@ -3942,13 +3946,13 @@ if ${lt_cv_nm_interface+:} false; then :
 else
  lt_cv_nm_interface="BSD nm"
  echo "int some_variable = 0;" > conftest.$ac_ext
-  (eval echo "\"\$as_me:3945: $ac_compile\"" >&5)
+  (eval echo "\"\$as_me:3948: $ac_compile\"" >&5)
  (eval "$ac_compile" 2>conftest.err)
  cat conftest.err >&5
-  (eval echo "\"\$as_me:3948: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
+  (eval echo "\"\$as_me:3951: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
  (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
  cat conftest.err >&5
-  (eval echo "\"\$as_me:3951: output\"" >&5)
+  (eval echo "\"\$as_me:3954: output\"" >&5)
  cat conftest.out >&5
  if $GREP 'External.*some_variable' conftest.out > /dev/null; then
    lt_cv_nm_interface="MS dumpbin"
@@ -5154,7 +5158,7 @@ ia64-*-hpux*)
  ;;
 *-*-irix6*)
  # Find out which ABI we are using.
-  echo '#line 5157 "configure"' > conftest.$ac_ext
+  echo '#line 5160 "configure"' > conftest.$ac_ext
  if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
  (eval $ac_compile) 2>&5
  ac_status=$?
@@ -6679,11 +6683,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:6682: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:6685: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>conftest.err)
   ac_status=$?
   cat conftest.err >&5
-   echo "$as_me:6686: \$? = $ac_status" >&5
+   echo "$as_me:6689: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s "$ac_outfile"; then
     # The compiler can only warn and ignore the option if not recognized
     # So say no if there are warnings other than the usual output.
@@ -7018,11 +7022,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7021: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7024: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>conftest.err)
   ac_status=$?
   cat conftest.err >&5
-   echo "$as_me:7025: \$? = $ac_status" >&5
+   echo "$as_me:7028: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s "$ac_outfile"; then
     # The compiler can only warn and ignore the option if not recognized
     # So say no if there are warnings other than the usual output.
@@ -7123,11 +7127,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7126: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7129: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>out/conftest.err)
   ac_status=$?
   cat out/conftest.err >&5
-   echo "$as_me:7130: \$? = $ac_status" >&5
+   echo "$as_me:7133: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s out/conftest2.$ac_objext
   then
     # The compiler can only warn and ignore the option if not recognized
@@ -7178,11 +7182,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7181: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7184: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>out/conftest.err)
   ac_status=$?
   cat out/conftest.err >&5
-   echo "$as_me:7185: \$? = $ac_status" >&5
+   echo "$as_me:7188: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s out/conftest2.$ac_objext
   then
     # The compiler can only warn and ignore the option if not recognized
@@ -9558,7 +9562,7 @@ else
  lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  lt_status=$lt_dlunknown
  cat > conftest.$ac_ext <<_LT_EOF
-#line 9561 "configure"
+#line 9564 "configure"
 #include "confdefs.h"

 #if HAVE_DLFCN_H
@@ -9654,7 +9658,7 @@ else
  lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  lt_status=$lt_dlunknown
  cat > conftest.$ac_ext <<_LT_EOF
-#line 9657 "configure"
+#line 9660 "configure"
 #include "confdefs.h"

 #if HAVE_DLFCN_H
@@ -11245,6 +11249,27 @@ fi
 	fi
 fi

+# Check whether --with-linenoise was given.
+if test "${with_linenoise+set}" = set; then :
+  withval=$with_linenoise; with_linenoise=$withval
+else
+  with_linenoise="no"
+fi
+
+if test "x$with_linenoise" != "xno"; then
+   TARGET_HAVE_READLINE=0
+   TARGET_HAVE_EDITLINE=0
+   TARGET_HAVE_LINENOISE=1
+   TARGET_READLINE_INC="-I${with_linenoise}"
+   TARGET_READLINE_LIBS="${with_linenoise}/linenoise.c"
+   echo "using linenoise source code at ${with_linenoise}"
+else
+   TARGET_HAVE_LINENOISE=0
+   echo "not using linenoise"
+fi
+
+
+



@@ -11321,11 +11346,13 @@ fi
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking build type" >&5
 $as_echo_n "checking build type... " >&6; }
 if test "${enable_debug}" = "yes" ; then
-  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
+  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0 -Wall"
+  EXTENSION_DEBUG="-DSQLITE_DEBUG=1"
  { $as_echo "$as_me:${as_lineno-$LINENO}: result: debug" >&5
 $as_echo "debug" >&6; }
 else
  TARGET_DEBUG="-DNDEBUG"
+  EXTENSION_DEBUG="-DNDEBUG"
  { $as_echo "$as_me:${as_lineno-$LINENO}: result: release" >&5
 $as_echo "release" >&6; }
 fi
@@ -12457,7 +12484,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.43.0, which was
+This file was extended by sqlite $as_me 3.45.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was

  CONFIG_FILES    = $CONFIG_FILES
@@ -12523,7 +12550,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.43.0
+sqlite config.status 3.45.0
 configured by $0, generated by GNU Autoconf 2.69,
  with options \\"\$ac_cs_config\\"

@@ -74,7 +74,7 @@
 # you don't need (for example BLT) by erasing or commenting out
 # the corresponding code.
 #
-AC_INIT([sqlite],[m4_esyscmd(cat VERSION | tr -d '\n')])
+AC_INIT([sqlite],m4_esyscmd(cat VERSION | tr -d '\n'))

 dnl Make sure the local VERSION file matches this configure script
 sqlite_version_sanity_check=`cat $srcdir/VERSION | tr -d '\n'`
@@ -598,11 +598,28 @@ if test x"$with_readline" != xno; then
 		TARGET_HAVE_READLINE=1
 	fi
 fi
+AC_ARG_WITH([linenoise],
+            [AS_HELP_STRING([--with-linenoise=DIR],[source directory for linenoise library])],
+            [with_linenoise=$withval],
+            [with_linenoise="no"])
+if test "x$with_linenoise" != "xno"; then
+   TARGET_HAVE_READLINE=0
+   TARGET_HAVE_EDITLINE=0
+   TARGET_HAVE_LINENOISE=1
+   TARGET_READLINE_INC="-I${with_linenoise}"
+   TARGET_READLINE_LIBS="${with_linenoise}/linenoise.c"
+   echo "using linenoise source code at ${with_linenoise}"
+else
+   TARGET_HAVE_LINENOISE=0
+   echo "not using linenoise"
+fi

 AC_SUBST(TARGET_READLINE_LIBS)
 AC_SUBST(TARGET_READLINE_INC)
 AC_SUBST(TARGET_HAVE_READLINE)
 AC_SUBST(TARGET_HAVE_EDITLINE)
+AC_SUBST(TARGET_HAVE_LINENOISE)
+

 ##########
 # Figure out what C libraries are required to compile programs
@@ -615,13 +632,16 @@ AC_SEARCH_LIBS(fdatasync, [rt])
 AC_ARG_ENABLE(debug, AS_HELP_STRING([--enable-debug],[enable debugging & verbose explain]))
 AC_MSG_CHECKING([build type])
 if test "${enable_debug}" = "yes" ; then
-  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
+  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0 -Wall"
+  EXTENSION_DEBUG="-DSQLITE_DEBUG=1"
  AC_MSG_RESULT([debug])
 else
  TARGET_DEBUG="-DNDEBUG"
+  EXTENSION_DEBUG="-DNDEBUG"
  AC_MSG_RESULT([release])
 fi
 AC_SUBST(TARGET_DEBUG)
+AC_SUBST(EXTENSION_DEBUG)

 #########
 # See whether we should use the amalgamation to build
@@ -0,0 +1,148 @@
+# Notes On Compiling SQLite On Windows 11
+
+Here are step-by-step instructions on how to build SQLite from
+canonical source on a new Windows 11 PC, as of 2023-11-01:
+
+  1.  Install Microsoft Visual Studio. The free "community edition" 
+      will work fine.  Do a standard install for C++ development.
+      SQLite only needs the
+      "cl" compiler and the "nmake" build tool.
+
+  2.  Under the "Start" menu, find "All Apps" then go to "Visual Studio 20XX"
+      and find "x64 Native Tools Command Prompt for VS 20XX".  Pin that
+      application to your task bar, as you will use it a lot.  Bring up
+      an instance of this command prompt and do all of the subsequent steps
+      in that "x64 Native Tools" command prompt.  (Or use "x86" if you want
+      a 32-bit build.)  The subsequent steps will not work in a vanilla
+      DOS prompt.  Nor will they work in PowerShell.
+
+  3.  Install TCL development libraries.  This note assumes that you will
+      install the TCL development libraries in the "`c:\Tcl`" directory.
+      Make adjustments
+      if you want TCL installed somewhere else.  SQLite needs both the
+      "tclsh.exe" command-line tool as part of the build process, and
+      the "tcl86.lib" library in order to run tests.  You will need
+      TCL version 8.6 or later.
+      <ol type="a">
+      <li>Get the TCL source archive, perhaps from
+      [https://www.tcl.tk/software/tcltk/download.html](https://www.tcl.tk/software/tcltk/download.html).
+      <li>Untar or unzip the source archive.  CD into the "win/" subfolder
+          of the source tree.
+      <li>Run: `nmake /f makefile.vc release`
+      <li>Run: `nmake /f makefile.vc INSTALLDIR=c:\Tcl install`
+      <li>CD to `c:\Tcl\lib`.  In that subfolder make a copy of the
+          "`tcl86t.lib`" file to the alternative name "`tcl86.lib`"
+          (omitting the second 't').  Leave the copy in the same directory
+          as the original.
+      <li>CD to `c:\Tcl\bin`.  Make a copy of the "`tclsh86t.exe`"
+          file into "`tclsh.exe`" (without the "86t") in the same directory.
+      <li>Add `c:\Tcl\bin` to your %PATH%.  To do this, go to Settings
+          and search for "path".  Select "edit environment variables for
+          your account" and modify your default PATH accordingly.
+          You will need to close and reopen your command prompts after
+          making this change.
+      </ol>
+
+  4.  Download the SQLite source tree and unpack it. CD into the
+      toplevel directory of the source tree.
+
+  5.  Set the TCLDIR environment variable to point to your TCL installation.
+      Like this:
+      <ul>
+      <li> `set TCLDIR=c:\Tcl`
+      </ul>
+
+  6.  Run the "`Makefile.msc`" makefile with an appropriate target.
+      Examples:
+      <ul>
+      <li>  `nmake /f makefile.msc`
+      <li>  `nmake /f makefile.msc sqlite3.c`
+      <li>  `nmake /f makefile.msc devtest`
+      <li>  `nmake /f makefile.msc releasetest`
+      </ul>
+
+## 32-bit Builds
+
+Doing a 32-bit build is just like doing a 64-bit build with the
+following minor changes:
+
+  1.  Use the "x86 Native Tools Command Prompt" instead of
+      "x64 Native Tools Command Prompt".  "**x86**" instead of "**x64**".
+
+  2.  Use a different installation directory for TCL.
+      The recommended directory is `c:\tcl32`.  Thus you end up
+      with two TCL builds:
+      <ul>
+      <li> `c:\tcl` &larr;  64-bit (the default)
+      <li> `c:\tcl32` &larr;  32-bit
+      </ul>
+
+  3.  Ensure that `c:\tcl32\bin` comes before `c:\tcl\bin` on
+      your PATH environment variable.  You can achieve this using
+      a command like:
+      <ul>
+      <li>  `set PATH=c:\tcl32\bin;%PATH%`
+      </ul>
+
+## Building a DLL
+
+The command the developers use for building the deliverable DLL on the 
+[download page](https://sqlite.org/download.html) is as follows:
+
+> ~~~~
+nmake /f Makefile.msc sqlite3.dll USE_NATIVE_LIBPATHS=1 "OPTS=-DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_FTS5=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_JSON1=1 -DSQLITE_ENABLE_GEOPOLY=1 -DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 -DSQLITE_ENABLE_SERIALIZE=1 -DSQLITE_ENABLE_MATH_FUNCTIONS=1"
+~~~~
+
+That command generates both the sqlite3.dll and sqlite3.def files.  The same
+command works for both 32-bit and 64-bit builds.
+
+## Statically Linking The TCL Library
+
+Some utility programs associated with SQLite need to be linked
+with TCL in order to function.  The [sqlite3_analyzer.exe program](https://sqlite.org/sqlanalyze.html)
+is an example.  You can build as described above, and then
+enter:
+
+> ~~~~
+nmake /f Makefile.msc sqlite3_analyzer.exe
+~~~~
+
+And you will end up with a working executable.  However, that executable
+will depend on having the "tcl86.dll" library somewhere on your %PATH%.
+Use the following steps to build an executable that has the TCL library
+statically linked so that it does not depend on separate DLL:
+
+  1.  Use the appropriate "Command Prompt" window - either x86 or
+      x64, depending on whether you want a 32-bit or 64-bit executable.
+
+  2.  Untar the TCL source tarball into a fresh directory.  CD into
+      the "win/" subfolder.
+
+  3.  Run: `nmake /f makefile.vc OPTS=nothreads,static shell`
+
+
+  4.  CD into the "Release*" subfolder that is created (note the
+      wildcard - the full name of the directory might vary).  There
+      you will find the "tcl86s.lib" file.  Copy this file into the
+      same directory that you put the "tcl86.lib" on your initial
+      installation.  (In this document, that directory is
+      "`C:\Tcl32\lib`" for 32-bit builds and
+      "`C:\Tcl\lib`" for 64-bit builds.)
+
+  5.  CD into your SQLite source code directory and build the desired
+      utility program, but add the following extra arguments to the
+      nmake command line:
+      <blockquote><pre>
+      CCOPTS="-DSTATIC_BUILD" LIBTCL="tcl86s.lib netapi32.lib user32.lib"
+      </pre></blockquote>
+      <p>So, for example, to build a statically linked version of
+      sqlite3_analyzer.exe, you might type:
+      <blockquote><pre>
+      nmake /f Makefile.msc CCOPTS="-DSTATIC_BUILD" LIBTCL="tcl86s.lib netapi32.lib user32.lib" sqlite3_analyzer.exe
+      </pre></blockquote>
+
+  6.  After your executable is built, you can verify that it does not
+      depend on the TCL DLL by running:
+      <blockquote><pre>
+      dumpbin /dependents sqlite3_analyzer.exe
+      </pre></blockquote>
@@ -0,0 +1,73 @@
+# Introduction to the SQLite Extensible Shell #
+
+This article introduces a set of enhancements to the SQLite "CLI" shell
+which make it extensible as a shell and provide other useful features.
+
+What "extensible" means as of June 2023 is that additional dot-commands
+can be implemented by a dynamically loaded extension and integrated
+into the shell's command repertoire and help for same. It also means
+that scripting support can be provided by an extension and integrated
+into the shell's input collection and dispatching facility.
+
+In the future, "extensible" will also mean that additional forms of query
+result output formatting (or other disposition), and additional methods
+of data collection into tables (or other disposition), can be provided
+by an extension and utilized via the shell's .mode and .import commands.
+(This integration with .mode and .import commands is in-work.)
+
+Scripting support for the shell is presently implemented by a
+[Tcl Shell Extension](https://sqlite.org/src/file?name=doc/tcl_extension_intro.md&ci=cli_extension).
+
+Some new features are available via these new or altered dot-commands:
+
+**.eval** will interpret its arguments as shell input. This is mainly useful
+from within a script to run fixed or formulated shell commands.
+
+**.parameter** has several enhancements: The new "edit" subcommand can be
+used to create or modify SQL binding parameters, either by name or from
+being referenced in the most recently run SQL. The new "save" and "load"
+subcommands will store or retrieve all or named parameters into/from a file.
+
+**.shxload** is used to load shell extensions at runtime.
+
+**.shxopts** either shows or alters extended shell features. Presently,
+this allows shell input parsing to be enhanced as described below or to be
+reverted to the traditional rule where dot-commands always occupy one line
+and neither dot-commands nor #-prefixed comments may have leading whitespace.
+
+**.tables** has new options to show only tables, views, or system tables.
+
+**.vars** is used to create or modify "shell" variables. These are
+key/value pairs which are associated with the shell session rather than
+a specific database such as .databases would list. These variables may
+be the subject of "edit", "set", "save" and "load" subcommands with
+effect similar to what they do with the .parameter dot-command.
+
+**.x** is a general purpose, "run these" command. By default, its arguments
+are treated as shell variable names, values of which are interpreted as
+shell input. Optionally, its arguments are interpreted similarly to the
+.eval and .read subcommand. They differ in that interpretation of the
+arguments stops upon any error.
+
+Other new shell features are:
+
+By default, when invoked as "sqlite3x", or after ".shxopts +parsing" is run,
+the shell effects enhanced command parsing whereby quoted dot-command arguments
+may span multiple lines or dot-command input lines may be spliced together
+with trailing backslash, and all shell input may have leading whitespace.
+
+When not invoked as "sqlite3x", or after ".shxopts -parsing" is run, any
+dot-command arguments and the dot-command itself end at the first newline.
+Also, input lines with leading whitespace will not be accepted as dot-commands
+or #-prefixed comments.
+This might be needed to run legacy shell scripts having some dot-command(s)
+with a final argument that has an initial quote but no closing quote or
+which happen to end with backslash.
+
+The shell's handling of certain fatal conditions has been changed to make
+it more suitable being embedded into other applications. It no longer calls
+exit() for OOM conditions or -safe mode violations. This feature has not
+yet been fully tested; work remains to be sure that the shell can be called,
+then return to its caller, repeatedly without leaking resources or leaving
+the console in an odd state. But the infrastructure is in place to make
+that all work with only minor (or possibly no) revisions.
@@ -1229,8 +1229,8 @@ first syntax error, of course, if there are no instances of the

 <p>Lemon was originally written by Richard Hipp sometime in the late
 1980s on a Sun4 Workstation using K&amp;R C.  
-There was a companion LL(1) parser generator program named "Lime", the
-source code to which as been lost.</p>
+There was a companion LL(1) parser generator program named "Lime".
+The Lime source code has been lost.</p>

 <p>The lemon.c source file was originally many separate files that were
 compiled together to generate the "lemon" executable.  Sometime in the
@@ -0,0 +1,608 @@
+<!doctype html>
+<html>
+  <head>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta http-equiv="content-type" content="text/html; charset=UTF-8">
+    <title>CLI Shell Extensibility</title>
+  </head>
+  <body>
+<style>
+   body{counter-reset: section}
+   h2{counter-reset: h2counter; text-indent: 10px}
+   h3{counter-reset: h3counter; text-indent: 25px}
+   h4{counter-reset: h4counter; text-indent: 40px}
+
+   h2:before{
+     counter-increment: section;
+     content: counter(section) " ";
+   }
+   h3:before{
+     counter-increment: h2counter;
+     content: counter(section) "." counter(h2counter) " ";
+   }
+   h4:before{
+     counter-increment: h3counter;
+     content: counter(section) "." counter(h2counter) "." counter(h3counter) " ";
+   }
+   h5:before{
+     counter-increment: h4counter;
+     content: counter(section) "." counter(h2counter) "." counter(h3counter) "." counter(h4counter) " ";
+   }
+   </style>
+<img class="logo" src="https://sqlite.org/images/sqlite370_banner.gif"
+     alt="SQLite" border="0"/>
+
+    <h1>CLI Shell Extensibility</h1>
+    <hr/>
+    <h2>Introduction</h2>
+    <p>The command line shell for SQLite can be customized
+      to modify or add certain kinds of features, without altering its source.
+      This document details this extensibility
+      and how such extension may be accomplished by shell users.
+    <p>Shell extensibility serves to reduce the tension between
+      keeping the shell simple with broadly useful features
+      and allowing the shell to become an ever-growing tool
+      meeting diverse needs for those not ready to satisfy them
+      with a custom program using the SQLite libary.
+      Extensions contributed by the SQLite developers or others
+      will become (or are) available for use in situations
+      that may not justify permanently adding the same features
+      to the core shell published by the SQLite project in binary form.
+      <h2>Extensible Features</h2>
+    <p>Only certain categories of features may be
+      added or modified via extension, namely:
+
+      <h3>New or Revised Meta-Commands</h3>
+    <p>The "dot" commands implemented by the shell,
+      along with help text for them, may be augmented or overridden.
+      A meta-command effected with an extension may be used
+      in the same ways as one available in the core, non-extended shell.
+
+      <h3>New or Revised Import Modes (work-in-progress)</h3>
+    <p>The ways in which data may be imported to a DB table
+      may be augmented or overriden. Once that is done, the .import
+      meta-command will either have a new option to specify a new import
+      handler or an existing .import option can be overridden for this.
+
+      <h3>New or Revised Query Result Handling (work-in-progress)</h3>
+    <p>The display formatting or other disposition of query results
+      may be augmented or overriden. Once that is done, the .mode
+      meta-command will either have a new option to specify a new result
+      handler or an existing .mode option can be overridden for this.
+
+      <h3>Generalized Import Modes and Query Result Handling</h3>
+    <p>New handlers added via extension are not restricted to
+      importing data from a file or result display formatting.
+      They may be considered more generally to be either
+      a data source or a data sink, producing or accepting
+      data row sets. The origin or destination of
+      the data is up to the handler, as may be affected
+      by arguments to the .import or most recent .mode command.
+
+      <h3>Scriptability</h3>
+    <p>An shell extension may provide for integration of a scripting
+      subsystem with the shell. Such a subsystem may participate in
+      classification of input, extending the recognized classes from
+      { meta-command SQL-statement(s) comment } to add { script }, and
+      then handle input in the "script" input class. The shell exposes
+      some its internal data and code to script implementations so that
+      they may act in a manner providing adaptation or enhancement of
+      shell features rather than entirely independent new features.
+
+      <h2>Extension Methods, Dynamic or Static</h2>
+    <p>Shell extension may be effected at different times
+      in different ways according to convenience and need.
+
+      <h3>Runtime Extension</h3>
+    <p>Extension at runtime (when the shell is running)
+      is effected via the .shxload command.
+      In this way, a dynamically loaded library (DLL) is loaded with
+      provision made for its sqlite3_X_init() function to obtain
+      the shell extension API entry points and thereby
+      register with the shell core any
+      new meta-commands, or import or query result handlers
+      or scripting support that it implements.
+
+      <h3>Build-Time Extension</h3>
+    <p>Extension when the shell is built is effected by specifying
+      certain option values to either the "make" invocation or
+      to a utility, (tool/mkshellc.tcl), which assembles and transforms
+      sources to produce the shell's unitary source file (shell.c).
+
+      <h3>Extension Source Code</h3>
+    <p>With certain coding conventions and methods followed,
+      the same source code can be used either
+      to produce most of a runtime shell extension DLL or to
+      be incorporated into shell.c as a built-in shell extension.
+      (See the extension code samples for details. (Not fully implemented.))
+
+      <h3>Build-Time Diminuation</h3>
+    <p>Just as new meta-commands can be readily incorporated into
+      the shell when it is built, many of the core meta-commands
+      can be readily omitted from the shell build. This is done
+      with a variation of the option values that may be given
+      to "make" or to tool/mkshellc.tcl as the shell is built.
+      (See tool/mkshellc.tcl --help output for details.)
+      Such omission of meta-commands might be done when building
+      a customized shell which need not have the various meta-commands
+      which exist for the purpose of testing the SQLite library.
+
+      <h2>Interface</h2>
+    <p>The following details relate to src/shext_linkage.h, a header
+      in which declarations appear for objects and functions
+      that facilitate runtime interaction between the shell core
+      and shell extensions written in C/C++.
+      Comments in that header tersely summarize these explanations:
+
+      <h3>struct ShellInState and ShellExState</h3>
+    <p>The Shell{In,Ex}State objects, are maintained by the shell and known
+      as shell state. The shell state consists of a public portion, which
+      is available and stable for use in shell/extension interactions,
+      and a private portion which may not be stable. Shell extension code
+      used only for build-time extension might use the private part,
+      (to which it has access because such code is compiled within the same
+      translation unit as the core shell code),
+      but such usage generally precludes (or makes hazardous)
+      use of runtime loadable extensions built from such code.
+
+      <h3>ExtensionId typedef and eid Member</h3>
+    <p>An object of this type serves to uniquely identify an extension
+      loaded at runtime so that it may be unloaded later. It must be
+      passed back to the shell (in the ShellExtensionLink eid member)
+      by the sqlite3_X_init() function if the extension DLL is ever
+      to be unloaded during that shell session.
+
+      <h3>ShellExState typedef</h3>
+    <p>An object of this type is passed between the shell core and its
+      import or query result handlers to: (1) convey or keep parameters and
+      data related to formatting or parsing data rows in an external form;
+      (2) keep state associated with the progression of an import or result
+      handling operation from initiation to completion; (3) to facilitate
+      access to exposed shell state generally useful to such handlers or
+      meta-commands; or (4) to provide for abnormal shell exits.
+    <p>The shell core .mode and .import implementations also use the same
+      instance of this type to affect result output and import operations.
+      That instance resides in a ShellStateX object kept by the shell so
+      that extension meta-commands can access it, possibly to change it.
+      Meta-commands or handlers which alter this instance for their own
+      purposes (rather than for intended effect) should take care to
+      restore its prior value as the operation completes.
+
+      <h3>extensionDestruct member</h3>
+    <p>The function addressed by this member will be called prior to
+      exit or the extension being unloaded (if the pointer is non-zero.)
+      This is an out parameter from the sqlite3_X_init() function.
+      It may perform any cleanup or deallocations necessitated by
+      successful initialization generally (and will never be called
+      after failed initialization.)
+
+      <h3>Notes Regarding Object Interfaces for C vs C++ Writers</h3>
+    <p>The objects registered with the shell core to provided extension
+      functionality may be implemented in C or C++ (or anything else
+      presenting the same ABI.) In the below descriptions of their
+      interfaces, it should be understood that: C++ implementations
+      need not explicitly deal with anything like a Whatsit_Vtable
+      struct and will refer to the object pointer, passed implicitly,
+      as "this"; and C implementations will need to populate a static
+      Whatsit_Vtable and refer to the initial object pointer as "pThis".
+    <p>All shell extension interfaces have a method, destruct(), which
+      is (or may be) called by the shell core prior to deactivating any
+      registered meta-command, output result or import handler.
+      This call will be made in addition to any automatic (or implicit)
+      takedown that may occur due to atexit() or C++ destructor calls,
+      so destruct()'s responsibility should be limited to reversing
+      the per-registered-object effects of sqlite3_X_init().
+    <p>A registered object is deactivated when either: the extension
+      is immanently going to be unloaded; the registered object is
+      being overridden by some like-named object (such that it can no
+      longer be reached by the core shell); or the shell is about to
+      return or exit.
+
+      <h3>DotCommand typedef</h3>
+    <p>These objects represent an extension meta-command, including a
+      dispatch table for the public interface and any accompanying
+      data (which is opaque to the core shell.) Such objects are created
+      by extensions and passed to the core shell only by reference.
+      They are made known to the shell core via registerDotCommand() calls.
+
+      <h3>DotCommand_Vtable typedef</h3>
+    <p>These objects represent the dispatch table of a DotCommand object.
+    <p>All methods are given the same leading (or lone) argument:<br>
+      (1) the address of the registered DotCommand object.
+
+      <h4>destruct method</h4>
+    <p>This method is called prior to unloading a runtime extension
+      for any registered DotCommand object, provided its dispatch
+      table entry is non-zero.
+      It should free resources allocated during the sqlite3_X_init() call
+      associated with creation or preparation of the object.
+
+      <h4>name method</h4>
+    <p>This method returns the name of the meta-command (sans leading '.'.)
+      The returned pointer must remain valid throughout the lifetime of
+      the registered DotCommand object.
+
+      <h4>help method</h4>
+    <p>This method returns help text for the meta-command. This text
+      should be formatted and aligned with the built-in meta-command
+      help text so that it can be displayed seamlessly.
+    <p>There is one additional argument:<br>
+      (2) a const char * directing what help text to return, with
+      0 indicating primary, single-line help, or
+      &"" indicating more detailed help beyond the primary level, or
+      &"(other)" reserved for expansion.
+    <p>The return for the first 2 forms is either a C string or
+      null pointer. The C string will not be freed by the core shell, and
+      must remain valid during the lifetime of the DotCommand object.
+
+      <h4>argsCheck method</h4>
+    <p>This method returns a DotCmdRC value indicating the validity
+      (or invalidity) of arguments for the meta-command, using the
+      coding scheme defined for the DotCmdRC type.
+    <p>There are three additional arguments:<br>
+      (2) a char** to possibly receive an error message, which the
+      caller must eventually pass to sqlite3_free().<br>
+      (3) a count of all provided arguments; and<br>
+      (4) a char* array of given arguments, led by meta-command's name.
+
+      <h4>execute method</h4>
+    <p>This method performs whatever work the meta-command is supposed
+      to do when invoked. It has 4 additional arguments:<br>
+      (2) present ShellExState, passed by reference (an in/out parameter);<br>
+      (3) an error message pointer, passed by reference, set upon error but
+      otherwise not modified, to be freed by the shell core;<br>
+      (4) the number of invocation arguments; and<br>
+      (5) an array of C strings constituting the invocation arguments;<br>
+    <p>The return is a DotCmdRC value, indicating success, error, or
+      other disposition as documented for the DotCmdRC type.
+
+      <h3>ExportHandler typedef</h3>
+    <p>These objects represent an extension query result handler, including
+      a dispatch table for the public interface and any accompanying
+      data which is opaque to the core shell. Such objects are created
+      by extensions and passed to the core shell only by reference.
+      They are made known to the shell core via registerExporter() calls.
+
+      <h3>ExportHandler_Vtable typedef</h3>
+    <p>These objects represent the dispatch table of an ExportHandler object.
+      All methods in the dispatch table are given at
+      least this leading argument:<br>
+      (1) The ExportHandler address registered via registerExporter();<br>
+
+      <h4>destruct method</h4>
+    <p>This method is called prior to the shell's return or exit or
+      unloading the runtime extension for a registered ExportHandler object,
+      provided its dispatch table entry is non-zero.
+      It should free resources allocated during the sqlite3_X_init() call
+      associated with creation or preparation of the object.
+
+      <h4>name method</h4>
+    <p>This method returns the name of the ExportHandler, which users
+      specify to the .mode command (as the mode's name) to designate use
+      of the registered ExportHandler for subsequent query results.
+      The returned pointer must remain valid throughout the lifetime of
+      the registered ExportHandler object.
+
+      <h4>help method</h4>
+    <p>This method returns help text for the ExportHandler.
+    <p>There is one additional argument:<br>
+      (2) a const char * directing what help text to return, with
+      0 indicating primary, single-line help, or
+      &"" indicating more detailed help beyond the primary level, or
+      &"(other)" reserved for expansion.
+    <p>The primary help is included in the .mode command's own
+      detailed help text, so it should be aligned accordingly.
+      The detailed help is shown by the .mode command's --help option.
+    <p>The return is either a C string or null pointer. The C string
+      will not be freed by the core shell, and must remain valid
+      during the lifetime of the ExportHandler object.
+
+      <h4>Common arguments</h4>
+    <p>The following methods are given these 2 additional arguments:<br>
+      (2) A ShellExState object passed by reference; and<br>
+      (3) An error message pointer, passed by reference, to receive errors.
+
+      <h4>openResultsOutStream method</h4>
+    <p>This method is called when a query output is setup,
+      (via the .mode command with the handler's name given as a --flag.)
+    <p>It is given 3 additional arguments:<br>
+      (4) the number of arguments in the said .mode command;<br>
+      (5) an array of C strings with the said argument values; and<br>
+      (6) the name given as a --flag which caused the handler to be used.
+    <p>When the extension handler is activated via a .mode command,
+      parsing that command and behaving accordingly is the responsibility
+      of this method alone. (The command is parsed and acted upon by the
+      default .mode implementation only when no ExportHandler is used.)
+    <p>Once this method is called and succeeds, it is guaranteed that the
+      closeResultsOutStream method will be called.
+    <p>This method should return SQLITE_OK only upon success.
+      Any other return will abort remaining calls in the handling sequence.
+
+      <h4>prependResultsOut method</h4>
+    <p>This method is called when a query succeeds for which this handler
+      will be given the results. This is purely preparatory; zero or
+      more result rows may follow. It is up to this method to determine
+      if any results can even be had (by considering the return from
+      sqlite3_column_count()) and acting accordingly.
+    <p>It is given 1 additional argument:<br>
+      (4) the query prepared statement (pointer), not yet stepped.
+    <p>This method should return SQLITE_OK only upon success.
+      Any other return will abort the next 2 calls in the handling sequence.
+      (TBD: Such aborts without error should be supported for DDL and DML.)
+
+      <h4>rowResultsOut method</h4>
+    <p>This method is called when a query's prepared statement is stepped,
+      for each result row available.
+    <p>It is given 1 additional argument:<br>
+      (4) the query prepared statement (pointer), stepped with
+      a result row available. It is permitted for this method to
+      perform the remaining stepping, as indicated by the return.
+    <p>This method should return SQLITE_OK only upon success
+      without having completed stepping. If this method completes stepping,
+      (by making its own calls to sqlite3_step() until it returns SQLITE_DONE),
+      it must return SQLITE_DONE. Any other return will avoid the next call
+      in the handling sequence.
+
+      <h4>appendResultsOut method</h4>
+    <p>This method is called when result set stepping has been completed.
+    <p>It is given  additional argument:<br>
+      (4) the query prepared statement (pointer), completely stepped.
+    <p>This method should return SQLITE_OK upon success, or may return
+      something else to indicate error with no effect upon succeeding calls.
+
+      <h4>closeResultsOutStream method</h4>
+    <p>This method is called when a new .mode command is invoked specifying
+      some output mode not using this ExportHandler or when the extension
+      is about to be unloaded with this ExportHandler selected for output.
+      It should free resources allocated or held as a result of the
+      previous openResultsOutStream call.
+
+      <h3>ImportHandler typedef</h3>
+    <p>These objects represent an extension data import handler, including
+      a dispatch table for the public interface and any accompanying
+      data which is opaque to the core shell. Such objects are created
+      by extensions and passed to the core shell only by reference.
+      They are made known to the shell core via registerImporter() calls.
+
+      <h3>ImportHandler_Vtable typedef</h3>
+    <p>These objects represent the dispatch table of an ImportHandler object.
+      All methods in the dispatch table are given this 1 leading argument:<br>
+      (1) The ImportHandler address registered via registerImporter().
+
+      <h4>destruct method</h4>
+    <p>This method is called prior to the shell's return or exit or
+      unloading the runtime extension for a registered ImportHandler object,
+      provided its dispatch table entry is non-zero.
+      It should free resources allocated during the sqlite3_X_init() call
+      associated with creation or preparation of the object.
+
+      <h4>name method</h4>
+    <p>This method returns the name of the importer, which is to
+      be passed with leading '--' (or '-') to the .import command
+      to specify use of the registered importer for that invocation.
+      The returned pointer must remain valid throughout the lifetime of
+      the registered ImportHandler object.
+
+      <h4>help method</h4>
+    <p>This method returns help text for the ImportHandler.
+    <p>There is one additional argument:<br>
+      (2) a const char * directing what help text to return, with
+      0 indicating primary, single-line help, or
+      &"" indicating more detailed help beyond the primary level, or
+      &"(other)" reserved for expansion.
+    <p>The primary help is included in the .import command's own
+      detailed help text, so it should be aligned accordingly.
+      The detailed help is shown by the .import command's --help option.
+    <p>The return is either a C string or null pointer. The C string
+      will not be freed by the core shell, and must remain valid
+      during the lifetime of the ImportHandler object.
+
+      <h4>Common arguments</h4>
+    <p>The following methods are given these 2 additional arguments:<br>
+      (2) A ShellExState object passed by reference; and<br>
+      (3) An error message pointer, passed by reference, to receive errors.
+
+      <h4>openDataInStream method</h4>
+    <p>This method is called when a .import command is invoked with
+      the '--' (or '-') prefixed name of the ImportHandler as an argument.
+    <p>These 3 additional arguments are passed:<br>
+      (4) the number of arguments to said .import command;<br>
+      (5) an array of C strings with the said argument values; and<br>
+      (6) the name of the ImportHandler (as its name method would return.)
+    <p>When the extension handler is activated via a .import command,
+      responsibility for parsing that command and behaving accordingly is
+      shared between the shell core .import implementation and this method.
+      The shell core normally takes the last argument as the name of a table
+      (to be created if necessary) which will receive the imported data.
+      (But see return values for exceptions to this treatment.)
+      The shell core also detects any --flag argument selecting a registered
+      ImportHandler (giving effect to the last one) and calls this and
+      following methods to perform the input part of the import operation.
+      This method may interpret any or all of the arguments as needed
+      and (supposedly) documented by the associated help(...) method returns.
+    <p>Once this method is called and succeeds, it is guaranteed that the
+      closeDataInStream method will be called.
+    <p>This method should return SQLITE_OK upon success where disposition
+      of the imported data (into a table named by the last argument) is
+      to be handled normally, by the shell core.
+      An alternative success return is SQLITE_DONE, indicating that
+      disposition of the imported data has been done by the ImportHandler.
+      In that case, the next 3 methods (*DataInput(...)) will not be called
+      and this method should have completed the whole import operation.
+      After a success return, closeDataInStream is guaranteed to be called.
+      Other returns will abort all remaining calls in the handling sequence.
+
+      <h4>prepareDataInput method</h4>
+    <p>This method prepares for the particular import operation commenced
+      with a .import invocation. It may take into account the shape of
+      the input data (and possibly its type) as discovered during the call.
+    <p>This 1 additional argument is passed:<br>
+      (4) a to-be-prepared statement (pointer), passed by reference.
+      This is an out parameter conveying a prepared statement created
+      by this method specifically for the single import operation.
+    <p>The prepared statement should, in some manner wholly determined
+      by the extension handler, incorporate compiled SQL (possibly with
+      as-yet unbound parameters) which will (or may) produce a result set.
+      (This may be no more than a query such as "SELECT @1 as one, ...",
+      or could be a SELECT from a temporary table used for buffering.)
+    <p>The return should be SQLITE_OK upon success, in which case the
+      following 2 methods will also be called. Any other return will
+      abort the {prepare,row,finish}DataInput() call sequence. In that
+      case, no prepared statement should be returned either.
+
+      <h4>rowDataInput method</h4>
+    <p>This method is called to collect imported data, making it available
+      through the prepared statement passed as the last argument with as
+      many steps as are needed to get SQLITE_DONE from sqlite3_step().
+      It will be called repeatedly until its return indicates no more data.
+    <p>It is given 1 additional argument:<br>
+      (4) the prepared statement (pointer) returned by prepareDataInput().<br>
+      This prepared statement can have values bound to it or be reset as
+      necessary to return some or more data. However, it must remain the
+      same sqlite3_statement instance returned by prepareDataInput().
+    <p>The return should be SQLITE_DONE when no more data is available,
+      or SQLITE_ROW to indicate that more data is or might be available.
+      Any other return (including SQLITE_OK) indicates a condition which
+      will abort the data collection phase of the import operation. Any
+      of those 3 returns is treated as success by the shell core.
+
+      <h4>finishDataInput method</h4>
+    <p>This method is called to complete the data input phase of the
+      import operation commenced by prepareDataInput().
+    <p>It is given 1 additional argument:<br>
+      (4) the prepared statement (pointer) returned by prepareDataInput().<br>
+      This prepared statement should be finalized by this method. Other
+      cleanup or import wrap-up related to the transfer may also be performed.
+
+      <h4>closeDataInStream method</h4>
+    <p>This method is called as a .import command which specified this
+      ImportHandler completes.
+      It should free resources allocated or held as a result of the
+      previous openDataInStream call.
+
+      <h3>ScriptSupport typedef</h3>
+    <p>An object of this type is passed by an extension to registerScripting()
+      to enable the shell core to gain scripting functionality provided by
+      the extension.
+    <p>All methods in the dispatch table have this 1 leading argument:<br>
+      (1) The object address registered via registerScripting().
+    <p>The object's v-table contains these methods:
+
+      <h4>destruct method</h4>
+    <p>This method is called prior to the shell's return or exit or unloading
+      the runtime extension, provided its dispatch table entry is non-zero.
+      It should free resources allocated during the sqlite3_X_init() call
+      associated with creation or preparation of the object.
+
+      <h4>name</h4>
+    <p>This method returns the name of the ScriptSupport object.
+      There is no presently planned scheme for using this to switch among
+      scripting providers or even to identify one that is loaded.
+
+      <h4>help</h4>
+    <p>This method returns help text for the ExportHandler.
+      There is no presently implemented use for this method's return.
+    <p>There is one additional argument:<br>
+      (2) a const char * directing what help text to return.
+
+      <h4>configure</h4>
+    <p>This method allows load-time configuration of the extension.
+      It returns a DotCmdRC code indicating status of the extension
+      load and subsequent configuration.
+      It has these 4 additional arguments:<br>
+      (2) ShellExState reference, for general use by the extension;<br>
+      (3) A char ** for an error message associated with configuration;<br>
+      (4) Count of arguments passed in next parameter; and<br>
+      (5) A char * array of arguments passed in the .shxload command tail.
+
+      <h4>isScriptLeader</h4>
+    <p>This method returns TRUE if the provided line text should be
+      considered part of an input line group to be handled by the extension
+      (rather than being treated as a dot-command or SQL), otherwise FALSE.
+      There is one additional argument:<br>
+      (2) char * zLineLead, the initial line of a shell input group.
+
+      <h4>scriptIsComplete</h4>
+    <p>This method returns TRUE if the provided line group text should be
+      considered ready to execute, otherwise FALSE.
+      There are two additional arguments:<br>
+      (2) char * zScript, the possibly ready-to-exectue text; and<br>
+      (3) char ** pzWhyNot, for possibly saying why it is not ready.
+
+      <h4>resetCompletionScan</h4>
+    <p>This method is called by the shell, with no additional arguments,
+      to (possibly) restore the extension's input line scanning state.
+      It has no return.
+
+      <h4>runScript</h4>
+    <p>This method executes the provided script text, returning a
+      DotCmdRC value to indicate success, error, or other dispositions.
+      It has three additional arguments:<br>
+      (2) char * zScript, the to-be-executed text;<br>
+      (3) ShellExState *, (same struct as is passed to configure); and<br>
+      (4) A char ** for an error message associated with the execution.
+
+      <h3>ShellExtensionLink typedef</h3>
+    <p>An object of this type is passed (somewhat indirectly) to the
+      sqlite3_X_init(...) function which is called when a runtime
+      extension is loaded via a .load command with a --shell flag.
+      It is used to establish linkage between the loaded extension
+      and a shell core API exposed specifically for extensibility.
+    <p>At present, the extension API is limited to registration of
+      meta-commands, query result handlers, import handlers, and
+      scripting support implemented by extensions. This API may be
+      extended in future versions of the core shell, in a backwards-
+      compatible manner.
+      (See the pExtra sentinel, whose offset may increase.)
+
+      <h4>Establishing Shell/Extension Linkage</h4>
+    <p>An extension's sqlite3_X_init() function is called soon after
+      the extension is dynamically loaded. It may obtain a pointer
+      to the ShellExtensionLink object by the following means. (Or,
+      without reference to that object, it can do nothing useful.) 
+    <p>During extension build, a macro provided by shext_linkage.h,
+      SHDB_TO_SHEXTLINK(link_function_name), is used to define a
+      function named per its argument. This function accepts a
+      sqlite3 *, as passed into sqlite3_X_init(db, ...), and reads
+      that DB to retrieve the address of a ShellExtensionLink object,
+      which it returns. (Or, it returns NULL if somebody has loaded
+      the extension erroneously using the .load meta-command.)
+    <p>The sqlite3_X_init() function may call this macro-written
+      function to obtain a reference to a ShellExtensionLink object
+      which will remain valid while the extension is loaded.
+
+      <h2>Development Aspects of Extensibility and Its Evolution</h2>
+
+    <p>There is critical tension between competing uses of the ShellState
+      object which is kept and used by the core shell for state which
+      must persist between meta-command invocations. For unhindered
+      implementation and feature expansion flexibility, the structure
+      of the ShellState data should be unconstrained across versions
+      of the shell. However, unless extension meta-commands and data
+      transferers are to act entirely independently of built-in meta-commands,
+      (excepting interaction through the above-described extension methods),
+      some portion of the ShellState data needs to be shared in a stable
+      manner between extensions and the core shell code.
+    <p>With respect to interface stability concerns, it is nearly immaterial
+      whether the sharing occurs through exposed data structures
+      or an extension API devised to convey similar data. (Only data layout
+      and possible change notification are at stake in that choice.)
+    <p>For simplicity, and because it can work in a way consistent with how
+      built-in shell features are implemented now, the chosen sharing method
+      is to simply directly expose a subset of the ShellState data. That
+      subset will be extremely limited to minimize hinderance of what has
+      previously been unfettered change to that data structure/meaning.
+      It will initially be: the present output stream, as affected by
+      .output and .once commands; the currently open user DB (if any);
+      the dedicated shell DB; and the data related to formatting
+      and transfer of data in external forms. This data resides in the
+      ShellStateX object as a ShellExState struct.
+    <p>As shell extensibility evolves, additional data items may need to
+      move into the publicly exposed portion of the ShellStateX object,
+      either directly (via exposed data members) or by means of additional
+      extension APIs defined in the ShellExtensionLink object (which has
+      been defined to accommodate growth of its function pointer list in
+      a backwards-compatible manner.)
+
+  </body>
+</html>
@@ -0,0 +1,431 @@
+# Introduction to the TCL Extension for the SQLite Extensible Shell #
+
+This article introduces an extension,
+written to be hosted by the SQLite Extensible Shell,
+which adds [Tcl](https://www.tcl.tk/about/index.html) features to it.
+
+## Motivation ##
+
+An understanding of the motivation for this feature may prove useful
+for putting what follows into context.
+Over its 2+ decade existence, the sqlite3 CLI shell has evolved,
+from its original utility as a test vehicle for the SQLite library,
+to a tool used by many people who want to use SQLite databases
+without writing a specialized application. While the shell exposes
+many of the library features, its original design was best suited
+for performing short, fixed sequences of DB operations.
+
+Effecting more complex, algorithmically determined operations
+can be difficult,
+usually requiring use of a general purpose scripting tool
+to drive the sqlite3 shell.
+This can work well where the task can be decomposed into
+"Prepare a sequence of elementary DB operations,"
+followed by
+"then feed the sequence into sqlite3 as its command/SQL input."
+However, where the task requires multiple such steps,
+with sqlite3 either being run multiple times or
+being made to engage in two-way communication with a driving process,
+getting the combination to work as intended can be tedious and tricky.
+
+Those difficulties are enough to make one wish for a better way
+to drive the tool, and to muse: If only there was a "Tool Command Language".
+Fortunately, there is; it is known as "Tcl".
+The sqlite3 shell is a tool, in need of being flexibly commanded,
+and Tcl is more than adequate for that purpose, as will be shown.
+
+Not to be overlooked is that SQLite itself originated as a
+library for Tcl, and that its integration with Tcl makes it both
+powerful and easy to use in Tcl programs.
+In fact, Tcl with SQLite can be a very good substitute for
+ad hoc combinations of general purpose scripting tools with the SQLite shell,
+and has been
+[since year 2000](https://www.sqlite.org/src/timeline?c=2000-05-29+14:26:00).
+
+Beyond the utility of [Tcl with SQLite](https://www.tcl.tk/about/uses.html),
+SQLite (the shell) with Tcl has virtues of its own.
+These include instrumentation,
+query plan display, ready-to-use data import and export features,
+among others familiar to sqlite3 shell users.
+
+## Basics, Terminology and Getting Started ##
+
+A variant of the SQLite shell can be built, called "sqlite3x" here,
+whose long name is "The SQLite Extensible Shell".
+It may be built on Unix-like systems by running "make sqlite3x".
+On Windows systems, using MSVC, with suitable environment setup,
+it may be built by running "nmake -f Makefile.msc sqlite3x.exe".
+
+The Tcl extension for sqlite3x, called "tclshext" here,
+may be built from the top directory of a SQLite project checkout.
+(See [Obtaining Code ...](https://sqlite.org/getthecode.html#obtaining_code_directly_from_the_version_control_system).)
+
+Build the extension for Unix-like systems,
+after configure is run with a "--enable-tcl" option
+(and a "--with-tcl=..." option if necessary),
+by invoking make with "tcl_shell_extension" as a target.
+On Windows systems, using MSVC, with nmake variables TCLDIR and TCLSUFFIX set
+appropriately, it may be built by running
+"nmake -f Makefile.msc tcl_shell_extension".
+
+It may be necessary to first install the Tcl development package or library
+in order for configure to find the Tcl interpreter
+and specify how to locate it as a library to be linked with the extension.
+
+To manually get a Tcl-extended-shell started,
+(assuming the above-mentioned images were built and have
+been placed where the shell and OS loader can find them [^ loader ]),
+either of these inputs is needed:<br>
+```
+  From a running shell:
+    sqlite3x
+    .shxload tclshext
+ or
+  At shell invocation:
+    sqlite3x -cmd ".shxload tclshext"
+```
+[^loader]: A directory path may need to be prepended to the extension's
+  name for the OS loader to find it unless it is in one of the locations
+  designated to the loader as a candidate for dynamic libraries. How such
+  designation is made is beyond the scope of this introduction.
+
+Provided this results in another prompt without any error message(s),
+the Tcl-extended shell is ready to go.
+For brevity, the shell in this state will be called "sqlite3xt".
+
+## Yet Another Prompt -- Now What? ##
+
+When sqlite3xt is ready for input, it acts very much like the sqlite3 shell.
+(^ The sqlite3x shell passes a test suite which is substantially
+identical to that used for the traditional shell, differing only
+in the content and format of error messages.)
+When given sensible inputs (as discussed below in "Parsing"),
+that are recognized by and valid for the sqlite3 shell,
+the same outputs and effects will occur as would with sqlite3.
+This condition,
+where SQL and dot commands recognized by sqlite3 may be input
+and acted upon as sqlite3 would,
+is referred to below as the "shell execution environment".
+For sqlite3xt, that is just the beginning.
+
+## Execution Environments and Input Parsing Changes ##
+
+The effect of loading tclshext can be briefly summarized as
+the introduction of an alternative "execution environment",
+together with a way of entering and leaving it,
+explication of which follows this on the primary "execution environment":
+
+### Shell execution environment ###
+
+The "shell execution environment", in effect upon sqlite3xt startup,
+has these characteristics pertinent here:
+
+* The leading non-whitespace token of an input line group
+is interpreted as a dot command if it begins with a single '.';
+
+* if that token is nothing but a '.', the line is a no-op;
+
+* or, if it begins with anything but a '.', it is collected as SQL,
+until terminated with ';', '/' or 'go',
+then submitted to the SQL execution (prepare/step) engine.
+
+* or, if it begins with '#', it is ignored (as a comment).
+
+* Whitespace-delimited arguments after the leading token
+are parsed according to the section on "Dot-command arguments"
+[here](https://sqlite.org/cli.html#rules_for_dot_commands_sql_and_more),
+with exceptions as noted next.
+
+The main exception to legacy argument parsing is that open quote
+constructs are not auto-closed by the end of an input line.
+(This legacy behavior may be reinstated if desired by: entering
+".shxopts -parsing" at the prompt; renaming the sqlite3x image
+to sqlite3 before executing it; or invoking it with the option
+pair '-shxopt' '0'.)
+Instead, arguments are collected, potentially across multiple lines,
+until an input line ends outside of any open quote construct.
+(Input which does not depend on the legacy, auto-close-on-newline
+behavior is what the term "sensible inputs" means as used above.)
+For example, this would not be sensible input to sqlite3:<br>
+```
+   .print 'Hello.
+   This was entered as multi-line input.'
+```
+as it would result in an error, (malformed SQL), while this input:<br>
+```
+   .print &quot;I'm not fond of closing my quotations.
+```
+is acceptable to sqlite3, but deemed "not sensible" here.
+When either is input to sqlite3x, a continuation prompt will be issued
+(in interactive mode) and the rest of the multi-line input
+will be taken as part of the .print dot-command.
+
+Of course, this is (mostly) review to those familiar with the sqlite3 shell.
+
+### Tcl execution environment ###
+
+The "Tcl execution environment" differs in several important ways
+from the familiar execution environment describe above.
+How to get into this alternative execution environment is described later.
+
+In this alternative execution environment, these critical differences exist:
+
+* An expanded set of command words is available and readily expanded further.
+
+* The available command words generally do not begin with '.'.
+(^In the Tcl execution environment, enter
+"info&nbsp;commands" to see the available command words.)
+
+* When in interactive mode, commands whose
+initial token is not defined as a Tcl command,
+but which can be found as an executable
+in a directory named in the PATH environment variable,
+will be executed in a sub-process
+(unless blocked by Tcl global variable auto_noexec having been set.)
+
+* The command word and arguments are collected,
+parsed and expanded according to the usual
+[rules for the Tcl language](https://www.tcl.tk/about/language.html).
+In particular, input line groups are collected until deemed "complete"
+by the Tcl parser. This means no open brace, quote or bracket constructs.
+
+* New [command words can be readily defined]
+(https://www.tcl.tk/man/tcl8.6/TclCmd/proc.html),
+and [variables can be set](https://www.tcl.tk/man/tcl8.6/TclCmd/set.html),
+either of which may affect argument expansion
+per the usual Tcl rules.
+
+This environment will be familiar to those who use Tcl.
+There are a few differences however. These are:<br>
+
+* A single '.' on an input line which is not inside of an incomplete
+Tcl line group is treated as a (momentary) end-of-input by the REPL.
+(^ "REPL" is an acronym for "Read, Evaluate, Print Loop".)
+
+* The shell's dot commands, while not visible via \[info commands\],
+are discoverable by the "unknown" command and will be executed
+if their names (with the '.' prefix) would be found and resolved
+unambiguously in the shell execution environment.
+Commands whose names begin with '.' which are not found as unique in the
+shell execution environment produce an "invalid command name" error.
+Except for that treatment, the unknown command in effect
+acts like the standard Tcl version.
+(That standard version remains available as _original_unkown,
+to which handling of non-dot-prefixed commands is delegated.)
+
+* A few non-standard-Tcl commands are available. In particular:
+
+ + Commands udb and shdb act nearly like commands creatable
+by the "sqlite3" Tcl package to represent an open database. They
+differ in that they do not accept the "close" subcommand, which
+reflects the fact that they exist to allow the shell's current
+user database and the shell's own database to be accessed just
+like ones created in Tcl via "sqlite3&nbsp;someDbName&nbsp;itsFilename".
+
+ + Commands sqlite_shell_REPL, get_tcl_group and now_interactive
+permit input to be collected in the same manner (and from the same
+sources) as the shell's REPL does. Here, "collected in the same manner"
+does not include the execution environment switching or SQL execution
+that the shell execution environment implements.
+
+ + Command register_adhoc_command permits a newly (or oldly) defined
+Tcl command, likely with a leading '.' in its name, to be associated
+by name with some help text in a table kept by the shell
+for augmentation of its .help facility.
+In this way, the .help command can emit help
+text, in summary or long forms, for Tcl commands that might be
+executable from the shell execution environment.
+(More on this below.)
+
+ + Finally, the command ".." (sans quotes) exists for reasons made
+evident below. With no arguments,
+it does nothing, quietly and successfully, with the empty result.
+
+* The present implementation does not run the event loop processing
+that the standard Tcl REPL uses to support certain functionality
+(such as sockets, asynchronous I/O and the Tk GUI subsystem.)
+A future enhancement is to support use of event loop and Tk.
+
+## Switching Execution Environments ##
+
+The simplest execution environment switching
+is effected with lone dots and dot pairs.
+Some examples should make this clear. For now, focus on "." or ".."
+entered on a line by itself.
+(The # comments are not required. The leading prompts reflect
+what would be seen in interactive use; they are not to be typed either.)<br>
+```
+   sqlite> # Now in shell execution environment, but it is time to do Tcl stuff.
+   sqlite> ..
+   tcl% # Now in Tcl execution environment, ready to roll a trivial dot command.
+   tcl% proc .do {what} {
+      >  .eval $what
+      > }
+   tcl% register_adhoc_command {.do   Does whatever one thing it is told to do
+      >    from the shell execution environment and little else}
+   tcl% # Time to return to shell execution environment.
+   tcl% .
+   sqlite> # See how ad hoc command creation worked.
+   sqlite> .help do
+   .do   Does whatever one thing it is told to do
+      from the shell execution environment and little else
+   sqlite> .do .conn
+   ACTIVE 0: :memory:
+   sqlite> .do ".. puts Putting"
+   Putting
+   sqlite> # Be sure about being in shell execution environment.
+   sqlite> .
+   sqlite> # Oh, the prompt would have sufficed. Now, for some Tcl ...
+   sqlite> ..
+   tcl% # Let's see what that .. command does.
+   tcl% ..
+   tcl% # Apparently, it either gets to the Tcl environment or stays there.
+```
+
+The use of lone . and .. to switch environments is easiest to understand.
+However, as explained in the next section,
+use of .. not alone on an input line exploits other Tcl functionality.
+
+Another effect of loading tclshext is that a new dot command, .tcl ,
+becomes part of the shell's repertoire.
+When entered without any argument(s)
+from the shell execution environment,
+the .tcl command has the same effect as .. by itself.
+(^When run from the Tcl execution environment with no arguments,
+.tcl acts as a no-op rather than entering a recursive REPL.)
+With arguments, it will read file content into the Tcl interpreter
+just as Tcl's source command would.
+This may be useful for getting the Tcl execution environment
+customized via sqlite3x invocation options such as
+"'-cmd'&nbsp;'.tcl&nbsp;my_sqlite3xt_init.tcl'"
+
+A side benefit of the .tcl command's existence is that
+it appears in .help output, which can remind users how
+to use the extension's main feature.
+
+## Momentary Use of Tcl Execution Environment ##
+
+When .. is not the sole non-whitespace content of an input line group
+which has been submitted from the shell execution environment,
+that causes argument collection and expansion to be performed
+according to Tcl rules by the Tcl interpreter,
+without entering and staying in the Tcl execution environment.
+These two variations exist (where "..." stands for the provided argument set):
+```
+   sqlite> ..dotcmd ...
+ or
+   sqlite> .. tclcmd ...
+```
+The former, where no space separates .. from dotcmd, causes the
+dot command known as .dotcmd to be executed (if it can be found.)
+The latter, where space separates .. from tclcmd, causes the
+Tcl command known as tclcmd to be executed (if it can be found.)
+In either case, when the execution terminates (or fails with a
+"command does not yet exist" or "invalid command name" error),
+the shell execution environment remains in effect afterward.
+
+This temporary use of the Tcl interpreter serves two purposes.
+One is to exploit the more powerful capabilities of Tcl for argument processing.
+Within the text of arguments as provided,
+variables can be accessed, computations can be done,
+and Tcl commands can be invoked to yield results (or produce side effects),
+all of which can affect what the expanded arguments finally contain.
+The other (miniscule) effect is to avoid the need for extra
+input lines to switch to the Tcl execution environment and back
+when "one-shot" use of Tcl is all that is needed.
+In other words:<br>
+```
+   ..dotcmd whatever ...
+ acts as a shorter form of
+   ..
+   .dotcmd whatever ...
+   .
+```
+
+Whenever '..' leads an input line group
+submitted in the shell execution environment,
+then most of that input is given to the Tcl interpreter for
+processing on a one-shot basis.
+
+## Cross-Execution Environment Interactions ##
+
+Study of the above examples might give rise to these questions:<br>
+&nbsp;&nbsp;"How can '..dotcmd ...' work in the Tcl execution environment?"<br>
+and<br>
+&nbsp;&nbsp;"Why does '.do anything' work in the shell execution environment?"<br>
+There are two features at work behind those examples working:
+
+In the Tcl execution environment, (as touched upon above),
+when a prospective Tcl command is not found to be defined
+in the namespace(s) searched by the interpreter, it is
+passed to the Tcl command named "unknown".
+In the tclshext-augmented Tcl environment,
+that procedure is implemented by C code which treats
+a purported command with a leading '.' specially,
+by attempting to find it in the shell's repertoire of dot commands.
+If found (unambiguously), it is then executed
+and the result (such as it is) returned to the caller.
+(The sqlite3 shell dot-commands return the empty result on success.)
+So, assuming there is a dot command invokable as .dotcmd,
+(which there could be if another extension provided it),
+it can be found and executed from the Tcl execution environment
+with arguments as collected and expanded by the Tcl interpreter.
+
+In the shell execution environment, if any extensions have been
+loaded when a prospective dot command is not found in the
+existing repertoire of dot commands kept by the shell,
+the shell's dispatcher gives the (undocumented) .unknown
+dot command a chance to execute it. This has no chance
+of succeeding unless some extension has overridden the
+built-in .unknown implementation. However, tclshext does
+override it, and the replacement attempts to find any
+dot command name it is given in the Tcl root namespace.
+If it can be found, (such as will be true for the above
+example invoking .do from the shell execution environment),
+then that command is executed as a Tcl command
+with whatever arguments were passed to .unknown .
+
+Because of how this works, there is not often any reason or need to
+leave the Tcl execution environment. Dot commands and Tcl commands
+can be freely intermixed and executed in that environment. The main
+reason one might revert to the shell execution environment would
+be to evaluate SQL statements in the usual, sqlite3 way, (by just
+typing them followed by ";".) However, SQL can be evaluated without
+leaving the Tcl execution environment by use of the recently added
+dot command, .eval, which evaluates its arguments in the shell
+execution environment. For example:<br>
+```
+  tcl% .mode box
+  tcl% .eval {
+     >   .print "Let's have a one,"
+     >   select 1 as one;
+     >  } {
+     >   .print "and a two."
+     >   select 2 as two;
+     >  }
+  Let's have a one,
+  ┌─────┐
+  │ one │
+  ├─────┤
+  │ 1   │
+  └─────┘
+  and a two.
+  ┌─────┐
+  │ two │
+  ├─────┤
+  │ 2   │
+  └─────┘
+  tcl%
+```
+
+It should be noted, for those new to Tcl, that brace-quoting in Tcl
+means "Whatever is between these matching braces is the (single) value."
+It can be used for nearly any content, excepting explicit mismatched braces.
+
+## Summary, More to Come ##
+
+The Tcl extension should prove useful to those who like the features
+of the sqlite3 shell but find using it programmatically to be challenging.
+
+A future article will cover using Tk, a graphical user interface
+toolkit originally created for use with Tcl, with sqlite3xt.
@@ -0,0 +1,284 @@
+
+
+# The testrunner.tcl Script
+
+# 1. Overview
+
+testrunner.tcl is a Tcl script used to run multiple SQLite tests using 
+multiple jobs. It supports the following types of tests:
+
+  *  Tcl test scripts.
+
+  *  Tests run with [make] commands. Specifically, at time of writing, 
+     [make fuzztest], [make mptest], [make sourcetest] and [make threadtest].
+
+testrunner.tcl pipes the output of all tests and builds run into log file
+**testrunner.log**, created in the cwd directory. Searching this file for
+"failed" is a good way to find the output of a failed test.
+
+testrunner.tcl also populates SQLite database **testrunner.db**. This database
+contains details of all tests run, running and to be run. A useful query
+might be:
+
+```
+  SELECT * FROM script WHERE state='failed'
+```
+
+Running the command:
+
+```
+  ./testfixture $(TESTDIR)/testrunner.tcl status
+```
+
+in the directory containing the testrunner.db database runs various queries
+to produce a succinct report on the state of a running testrunner.tcl script.
+Running:
+
+```
+  watch ./testfixture $(TESTDIR)/testrunner.tcl status
+```
+
+in another terminal is a good way to keep an eye on a long running test.
+
+Sometimes testrunner.tcl uses the [testfixture] binary that it is run with
+to run tests (see "Binary Tests" below). Sometimes it builds testfixture and
+other binaries in specific configurations to test (see "Source Tests").
+
+# 2. Binary Tests
+
+The commands described in this section all run various combinations of the Tcl
+test scripts using the [testfixture] binary used to run the testrunner.tcl
+script (i.e. they do not invoke the compiler to build new binaries, or the
+[make] command to run tests that are not Tcl scripts). The procedure to run
+these tests is therefore:
+
+  1. Build the "testfixture" (or "testfixture.exe" for windows) binary using
+     whatever method seems convenient.
+
+  2. Test the binary built in step 1 by running testrunner.tcl with it, 
+     perhaps with various options.
+
+The following sub-sections describe the various options that can be
+passed to testrunner.tcl to test binary testfixture builds.
+
+## 2.1. Organization of Tcl Tests
+
+Tcl tests are stored in files that match the pattern *\*.test*. They are
+found in both the $TOP/test/ directory, and in the various sub-directories
+of the $TOP/ext/ directory of the source tree. Not all *\*.test* files
+contain Tcl tests - a handful are Tcl scripts designed to invoke other
+*\*.test* files.
+
+The **veryquick** set of tests is a subset of all Tcl test scripts in the
+source tree. In includes most tests, but excludes some that are very slow.
+Almost all fault-injection tests (those that test the response of the library
+to OOM or IO errors) are excluded. It is defined in source file 
+*test/permutations.test*.
+
+The **full** set of tests includes all Tcl test scripts in the source tree.
+To run a "full" test is to run all Tcl test scripts that can be found in the
+source tree.
+
+File *permutations.test* defines various test "permutations". A permutation
+consists of:
+
+  *  A subset of Tcl test scripts, and 
+
+  *  Runtime configuration to apply before running each test script 
+     (e.g. enabling auto-vacuum, or disable lookaside).
+
+Running **all** tests is to run all tests in the full test set, plus a dozen
+or so permutations. The specific permutations that are run as part of "all"
+are defined in file *testrunner_data.tcl*.
+
+## 2.2. Commands to Run Tests
+
+To run the "veryquick" test set, use either of the following:
+
+```
+  ./testfixture $TESTDIR/testrunner.tcl
+  ./testfixture $TESTDIR/testrunner.tcl veryquick
+```
+
+To run the "full" test suite:
+
+```
+  ./testfixture $TESTDIR/testrunner.tcl full
+```
+
+To run the subset of the "full" test suite for which the test file name matches
+a specified pattern (e.g. all tests that start with "fts5"), either of:
+
+```
+  ./testfixture $TESTDIR/testrunner.tcl fts5%
+  ./testfixture $TESTDIR/testrunner.tcl 'fts5*'
+```
+
+To run "all" tests (full + permutations):
+
+```
+  ./testfixture $TESTDIR/testrunner.tcl all
+```
+
+<a name=binary_test_failures></a>
+## 2.3. Investigating Binary Test Failures
+
+If a test fails, testrunner.tcl reports name of the Tcl test script and, if
+applicable, the name of the permutation, to stdout. This information can also
+be retrieved from either *testrunner.log* or *testrunner.db*.
+
+If there is no permutation, the individual test script may be run with:
+
+```
+  ./testfixture $PATH_TO_SCRIPT
+```
+
+Or, if the failure occured as part of a permutation:
+
+```
+  ./testfixture $TESTDIR/testrunner.tcl $PERMUTATION $PATH_TO_SCRIPT
+```
+
+TODO: An example instead of "$PERMUTATION" and $PATH\_TO\_SCRIPT?
+
+# 3. Source Code Tests
+
+The commands described in this section invoke the C compiler to build 
+binaries from the source tree, then use those binaries to run Tcl and
+other tests. The advantages of this are that:
+
+  *  it is possible to test multiple build configurations with a single
+     command, and 
+
+  *  it ensures that tests are always run using binaries created with the
+     same set of compiler options.
+
+The testrunner.tcl commands described in this section may be run using
+either a *testfixture* (or testfixture.exe) build, or with any other Tcl
+shell that supports SQLite 3.31.1 or newer via "package require sqlite3".
+
+TODO: ./configure + Makefile.msc build systems.
+
+## Commands to Run SQLite Tests
+
+The **mdevtest** command is equivalent to running the veryquick tests and
+the [make fuzztest] target once for each of two --enable-all builds - one 
+with debugging enabled and one without:
+
+```
+  tclsh $TESTDIR/testrunner.tcl mdevtest
+```
+
+In other words, it is equivalent to running:
+
+```
+  $TOP/configure --enable-all --enable-debug
+  make fuzztest
+  make testfixture
+  ./testfixture $TOP/test/testrunner.tcl veryquick
+
+  # Then, after removing files created by the tests above:
+  $TOP/configure --enable-all OPTS="-O0"
+  make fuzztest
+  make testfixture
+  ./testfixture $TOP/test/testrunner.tcl veryquick
+```
+
+The **sdevtest** command is identical to the mdevtest command, except that the
+second of the two builds is a sanitizer build. Specifically, this means that
+OPTS="-fsanitize=address,undefined" is specified instead of OPTS="-O0":
+
+```
+  tclsh $TESTDIR/testrunner.tcl sdevtest
+```
+
+The **release** command runs lots of tests under lots of builds. It runs
+different combinations of builds and tests depending on whether it is run
+on Linux, Windows or OSX. Refer to *testrunner\_data.tcl* for the details
+of the specific tests run.
+
+```
+  tclsh $TESTDIR/testrunner.tcl release
+```
+
+## Running ZipVFS Tests
+
+testrunner.tcl can build a zipvfs-enabled testfixture and use it to run
+tests from the Zipvfs project with the following command:
+
+```
+  tclsh $TESTDIR/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS
+```
+
+This can be combined with any of "mdevtest", "sdevtest" or "release" to
+test both SQLite and Zipvfs with a single command:
+
+```
+  tclsh $TESTDIR/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS mdevtest
+```
+
+## Investigating Source Code Test Failures
+
+Investigating a test failure that occurs during source code testing is a
+two step process:
+
+  1. Recreating the build configuration in which the test failed, and
+
+  2. Re-running the actual test.
+
+To recreate a build configuration, use the testrunner.tcl **script** command
+to create a build script. A build script is a bash script on Linux or OSX, or
+a dos \*.bat file on windows. For example:
+
+```
+  # Create a script that recreates build configuration "Device-One" on 
+  # Linux or OSX:
+  tclsh $TESTDIR/testrunner.tcl script Device-One > make.sh 
+
+  # Create a script that recreates build configuration "Have-Not" on Windows:
+  tclsh $TESTDIR/testrunner.tcl script Have-Not > make.bat 
+```
+
+The generated bash or \*.bat file script accepts a single argument - a makefile
+target to build. This may be used either to run a [make] command test directly,
+or else to build a testfixture (or testfixture.exe) binary with which to
+run a Tcl test script, as <a href=#binary_test_failures>described above</a>.
+
+
+
+# 4. Controlling CPU Core Utilization
+
+When running either binary or source code tests, testrunner.tcl reports the
+number of jobs it intends to use to stdout. e.g.
+
+```
+  $ ./testfixture $TESTDIR/testrunner.tcl
+  splitting work across 16 jobs
+  ... more output ...
+```
+
+By default, testfixture.tcl attempts to set the number of jobs to the number 
+of real cores on the machine. This can be overridden using the "--jobs" (or -j)
+switch:
+
+```
+  $ ./testfixture $TESTDIR/testrunner.tcl --jobs 8
+  splitting work across 8 jobs
+  ... more output ...
+```
+
+The number of jobs may also be changed while an instance of testrunner.tcl is
+running by exucuting the following command from the directory containing the
+testrunner.log and testrunner.db files:
+
+```
+  $ ./testfixture $TESTDIR/testrunner.tcl njob $NEW_NUMBER_OF_JOBS
+```
+
+
+
+
+
+
+
+
@@ -0,0 +1,676 @@
+/*
+** 2023 November 4
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+********************************************************************************
+** This file implements various interfaces used for console and stream I/O
+** by the SQLite project command-line tools, as explained in console_io.h .
+** Functions prefixed by "SQLITE_INTERNAL_LINKAGE" behave as described there.
+*/
+
+#ifndef SQLITE_CDECL
+# define SQLITE_CDECL
+#endif
+
+#ifndef SHELL_NO_SYSINC
+# include <stdarg.h>
+# include <string.h>
+# include <stdlib.h>
+# include <limits.h>
+# include <assert.h>
+# include "console_io.h"
+# include "sqlite3.h"
+#endif
+
+#ifndef SQLITE_CIO_NO_TRANSLATE
+# if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT
+#  ifndef SHELL_NO_SYSINC
+#   include <io.h>
+#   include <fcntl.h>
+#   undef WIN32_LEAN_AND_MEAN
+#   define WIN32_LEAN_AND_MEAN
+#   include <windows.h>
+#  endif
+#  define CIO_WIN_WC_XLATE 1 /* Use WCHAR Windows APIs for console I/O */
+# else
+#  ifndef SHELL_NO_SYSINC
+#   include <unistd.h>
+#  endif
+#  define CIO_WIN_WC_XLATE 0 /* Use plain C library stream I/O at console */
+# endif
+#else
+# define CIO_WIN_WC_XLATE 0 /* Not exposing translation routines at all */
+#endif
+
+#if CIO_WIN_WC_XLATE
+/* Character used to represent a known-incomplete UTF-8 char group (�) */
+static WCHAR cBadGroup = 0xfffd;
+#endif
+
+#if CIO_WIN_WC_XLATE
+static HANDLE handleOfFile(FILE *pf){
+  int fileDesc = _fileno(pf);
+  union { intptr_t osfh; HANDLE fh; } fid = {
+    (fileDesc>=0)? _get_osfhandle(fileDesc) : (intptr_t)INVALID_HANDLE_VALUE
+  };
+  return fid.fh;
+}
+#endif
+
+#ifndef SQLITE_CIO_NO_TRANSLATE
+typedef struct PerStreamTags {
+# if CIO_WIN_WC_XLATE
+  HANDLE hx;
+  DWORD consMode;
+  char acIncomplete[4];
+# else
+  short reachesConsole;
+# endif
+  FILE *pf;
+} PerStreamTags;
+
+/* Define NULL-like value for things which can validly be 0. */
+# define SHELL_INVALID_FILE_PTR ((FILE *)~0)
+# if CIO_WIN_WC_XLATE
+#  define SHELL_INVALID_CONS_MODE 0xFFFF0000
+# endif
+
+# if CIO_WIN_WC_XLATE
+#  define PST_INITIALIZER { INVALID_HANDLE_VALUE, SHELL_INVALID_CONS_MODE, \
+      {0,0,0,0}, SHELL_INVALID_FILE_PTR }
+# else
+#  define PST_INITIALIZER { 0, SHELL_INVALID_FILE_PTR }
+# endif
+
+/* Quickly say whether a known output is going to the console. */
+# if CIO_WIN_WC_XLATE
+static short pstReachesConsole(PerStreamTags *ppst){
+  return (ppst->hx != INVALID_HANDLE_VALUE);
+}
+# else
+#  define pstReachesConsole(ppst) 0
+# endif
+
+# if CIO_WIN_WC_XLATE
+static void restoreConsoleArb(PerStreamTags *ppst){
+  if( pstReachesConsole(ppst) ) SetConsoleMode(ppst->hx, ppst->consMode);
+}
+# else
+#  define restoreConsoleArb(ppst)
+# endif
+
+/* Say whether FILE* appears to be a console, collect associated info. */
+static short streamOfConsole(FILE *pf, /* out */ PerStreamTags *ppst){
+# if CIO_WIN_WC_XLATE
+  short rv = 0;
+  DWORD dwCM = SHELL_INVALID_CONS_MODE;
+  HANDLE fh = handleOfFile(pf);
+  ppst->pf = pf;
+  if( INVALID_HANDLE_VALUE != fh ){
+    rv = (GetFileType(fh) == FILE_TYPE_CHAR && GetConsoleMode(fh,&dwCM));
+  }
+  ppst->hx = (rv)? fh : INVALID_HANDLE_VALUE;
+  ppst->consMode = dwCM;
+  return rv;
+# else
+  ppst->pf = pf;
+  ppst->reachesConsole = ( (short)isatty(fileno(pf)) );
+  return ppst->reachesConsole;
+# endif
+}
+
+# if CIO_WIN_WC_XLATE
+/* Define console modes for use with the Windows Console API. */
+#  define SHELL_CONI_MODE \
+  (ENABLE_ECHO_INPUT | ENABLE_INSERT_MODE | ENABLE_LINE_INPUT | 0x80 \
+  | ENABLE_QUICK_EDIT_MODE | ENABLE_EXTENDED_FLAGS | ENABLE_PROCESSED_INPUT)
+#  define SHELL_CONO_MODE (ENABLE_PROCESSED_OUTPUT | ENABLE_WRAP_AT_EOL_OUTPUT \
+  | ENABLE_VIRTUAL_TERMINAL_PROCESSING)
+# endif
+
+typedef struct ConsoleInfo {
+  PerStreamTags pstSetup[3];
+  PerStreamTags pstDesignated[3];
+  StreamsAreConsole sacSetup;
+} ConsoleInfo;
+
+static short isValidStreamInfo(PerStreamTags *ppst){
+  return (ppst->pf != SHELL_INVALID_FILE_PTR);
+}
+
+static ConsoleInfo consoleInfo = {
+  { /* pstSetup */ PST_INITIALIZER, PST_INITIALIZER, PST_INITIALIZER },
+  { /* pstDesignated[] */ PST_INITIALIZER, PST_INITIALIZER, PST_INITIALIZER },
+  SAC_NoConsole /* sacSetup */
+};
+
+SQLITE_INTERNAL_LINKAGE FILE* invalidFileStream = (FILE *)~0;
+
+# if CIO_WIN_WC_XLATE
+static void maybeSetupAsConsole(PerStreamTags *ppst, short odir){
+  if( pstReachesConsole(ppst) ){
+    DWORD cm = odir? SHELL_CONO_MODE : SHELL_CONI_MODE;
+    SetConsoleMode(ppst->hx, cm);
+  }
+}
+# else
+#  define maybeSetupAsConsole(ppst,odir)
+# endif
+
+SQLITE_INTERNAL_LINKAGE void consoleRenewSetup(void){
+# if CIO_WIN_WC_XLATE
+  int ix = 0;
+  while( ix < 6 ){
+    PerStreamTags *ppst = (ix<3)?
+      &consoleInfo.pstSetup[ix] : &consoleInfo.pstDesignated[ix-3];
+    maybeSetupAsConsole(ppst, (ix % 3)>0);
+    ++ix;
+  }
+# endif
+}
+
+SQLITE_INTERNAL_LINKAGE StreamsAreConsole
+consoleClassifySetup( FILE *pfIn, FILE *pfOut, FILE *pfErr ){
+  StreamsAreConsole rv = SAC_NoConsole;
+  FILE* apf[3] = { pfIn, pfOut, pfErr };
+  int ix;
+  for( ix = 2; ix >= 0; --ix ){
+    PerStreamTags *ppst = &consoleInfo.pstSetup[ix];
+    if( streamOfConsole(apf[ix], ppst) ){
+      rv |= (SAC_InConsole<<ix);
+    }
+    consoleInfo.pstDesignated[ix] = *ppst;
+    if( ix > 0 ) fflush(apf[ix]);
+  }
+  consoleInfo.sacSetup = rv;
+  consoleRenewSetup();
+  return rv;
+}
+
+SQLITE_INTERNAL_LINKAGE void SQLITE_CDECL consoleRestore( void ){
+# if CIO_WIN_WC_XLATE
+  static ConsoleInfo *pci = &consoleInfo;
+  if( pci->sacSetup ){
+    int ix;
+    for( ix=0; ix<3; ++ix ){
+      if( pci->sacSetup & (SAC_InConsole<<ix) ){
+        PerStreamTags *ppst = &pci->pstSetup[ix];
+        SetConsoleMode(ppst->hx, ppst->consMode);
+      }
+    }
+  }
+# endif
+}
+#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
+
+#ifdef SQLITE_CIO_INPUT_REDIR
+/* Say whether given FILE* is among those known, via either
+** consoleClassifySetup() or set{Output,Error}Stream, as
+** readable, and return an associated PerStreamTags pointer
+** if so. Otherwise, return 0.
+*/
+static PerStreamTags * isKnownReadable(FILE *pf){
+  static PerStreamTags *apst[] = {
+    &consoleInfo.pstDesignated[0], &consoleInfo.pstSetup[0], 0
+  };
+  int ix = 0;
+  do {
+    if( apst[ix]->pf == pf ) break;
+  } while( apst[++ix] != 0 );
+  return apst[ix];
+}
+#endif
+
+#ifndef SQLITE_CIO_NO_TRANSLATE
+/* Say whether given FILE* is among those known, via either
+** consoleClassifySetup() or set{Output,Error}Stream, as
+** writable, and return an associated PerStreamTags pointer
+** if so. Otherwise, return 0.
+*/
+static PerStreamTags * isKnownWritable(FILE *pf){
+  static PerStreamTags *apst[] = {
+    &consoleInfo.pstDesignated[1], &consoleInfo.pstDesignated[2],
+    &consoleInfo.pstSetup[1], &consoleInfo.pstSetup[2], 0
+  };
+  int ix = 0;
+  do {
+    if( apst[ix]->pf == pf ) break;
+  } while( apst[++ix] != 0 );
+  return apst[ix];
+}
+
+static FILE *designateEmitStream(FILE *pf, unsigned chix){
+  FILE *rv = consoleInfo.pstDesignated[chix].pf;
+  if( pf == invalidFileStream ) return rv;
+  else{
+    /* Setting a possibly new output stream. */
+    PerStreamTags *ppst = isKnownWritable(pf);
+    if( ppst != 0 ){
+      PerStreamTags pst = *ppst;
+      consoleInfo.pstDesignated[chix] = pst;
+    }else streamOfConsole(pf, &consoleInfo.pstDesignated[chix]);
+  }
+  return rv;
+}
+
+SQLITE_INTERNAL_LINKAGE FILE *setOutputStream(FILE *pf){
+  return designateEmitStream(pf, 1);
+}
+# ifdef CONSIO_SET_ERROR_STREAM
+SQLITE_INTERNAL_LINKAGE FILE *setErrorStream(FILE *pf){
+  return designateEmitStream(pf, 2);
+}
+# endif
+#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
+
+#ifndef SQLITE_CIO_NO_SETMODE
+# if CIO_WIN_WC_XLATE
+static void setModeFlushQ(FILE *pf, short bFlush, int mode){
+  if( bFlush ) fflush(pf);
+  _setmode(_fileno(pf), mode);
+}
+# else
+#  define setModeFlushQ(f, b, m) if(b) fflush(f)
+# endif
+
+SQLITE_INTERNAL_LINKAGE void setBinaryMode(FILE *pf, short bFlush){
+  setModeFlushQ(pf, bFlush, _O_BINARY);
+}
+SQLITE_INTERNAL_LINKAGE void setTextMode(FILE *pf, short bFlush){
+  setModeFlushQ(pf, bFlush, _O_TEXT);
+}
+# undef setModeFlushQ
+
+#else /* defined(SQLITE_CIO_NO_SETMODE) */
+# define setBinaryMode(f, bFlush) do{ if((bFlush)) fflush(f); }while(0)
+# define setTextMode(f, bFlush) do{ if((bFlush)) fflush(f); }while(0)
+#endif /* defined(SQLITE_CIO_NO_SETMODE) */
+
+#ifndef SQLITE_CIO_NO_TRANSLATE
+# if CIO_WIN_WC_XLATE
+/* Write buffer cBuf as output to stream known to reach console,
+** limited to ncTake char's. Return ncTake on success, else 0. */
+static int conZstrEmit(PerStreamTags *ppst, const char *z, int ncTake){
+  int rv = 0;
+  if( z!=NULL ){
+    int nwc = MultiByteToWideChar(CP_UTF8,0, z,ncTake, 0,0);
+    if( nwc > 0 ){
+      WCHAR *zw = sqlite3_malloc64(nwc*sizeof(WCHAR));
+      if( zw!=NULL ){
+        nwc = MultiByteToWideChar(CP_UTF8,0, z,ncTake, zw,nwc);
+        if( nwc > 0 ){
+          /* Translation from UTF-8 to UTF-16, then WCHARs out. */
+          if( WriteConsoleW(ppst->hx, zw,nwc, 0, NULL) ){
+            rv = ncTake;
+          }
+        }
+        sqlite3_free(zw);
+      }
+    }
+  }
+  return rv;
+}
+
+/* For {f,o,e}PrintfUtf8() when stream is known to reach console. */
+static int conioVmPrintf(PerStreamTags *ppst, const char *zFormat, va_list ap){
+  char *z = sqlite3_vmprintf(zFormat, ap);
+  if( z ){
+    int rv = conZstrEmit(ppst, z, (int)strlen(z));
+    sqlite3_free(z);
+    return rv;
+  }else return 0;
+}
+# endif /* CIO_WIN_WC_XLATE */
+
+# ifdef CONSIO_GET_EMIT_STREAM
+static PerStreamTags * getDesignatedEmitStream(FILE *pf, unsigned chix,
+                                               PerStreamTags *ppst){
+  PerStreamTags *rv = isKnownWritable(pf);
+  short isValid = (rv!=0)? isValidStreamInfo(rv) : 0;
+  if( rv != 0 && isValid ) return rv;
+  streamOfConsole(pf, ppst);
+  return ppst;
+}
+# endif
+
+/* Get stream info, either for designated output or error stream when
+** chix equals 1 or 2, or for an arbitrary stream when chix == 0.
+** In either case, ppst references a caller-owned PerStreamTags
+** struct which may be filled in if none of the known writable
+** streams is being held by consoleInfo. The ppf parameter is an
+** output when chix!=0 and an input when chix==0.
+ */
+static PerStreamTags *
+getEmitStreamInfo(unsigned chix, PerStreamTags *ppst,
+                  /* in/out */ FILE **ppf){
+  PerStreamTags *ppstTry;
+  FILE *pfEmit;
+  if( chix > 0 ){
+    ppstTry = &consoleInfo.pstDesignated[chix];
+    if( !isValidStreamInfo(ppstTry) ){
+      ppstTry = &consoleInfo.pstSetup[chix];
+      pfEmit = ppst->pf;
+    }else pfEmit = ppstTry->pf;
+    if( !isValidStreamInfo(ppst) ){
+      pfEmit = (chix > 1)? stderr : stdout;
+      ppstTry = ppst;
+      streamOfConsole(pfEmit, ppstTry);
+    }
+    *ppf = pfEmit;
+  }else{
+    ppstTry = isKnownWritable(*ppf);
+    if( ppstTry != 0 ) return ppstTry;
+    streamOfConsole(*ppf, ppst);
+    return ppst;
+  }
+  return ppstTry;
+}
+
+SQLITE_INTERNAL_LINKAGE int oPrintfUtf8(const char *zFormat, ...){
+  va_list ap;
+  int rv;
+  FILE *pfOut;
+  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
+# if CIO_WIN_WC_XLATE
+  PerStreamTags *ppst = getEmitStreamInfo(1, &pst, &pfOut);
+# else
+  getEmitStreamInfo(1, &pst, &pfOut);
+# endif
+  assert(zFormat!=0);
+  va_start(ap, zFormat);
+# if CIO_WIN_WC_XLATE
+  if( pstReachesConsole(ppst) ){
+    rv = conioVmPrintf(ppst, zFormat, ap);
+  }else{
+# endif
+    rv = vfprintf(pfOut, zFormat, ap);
+# if CIO_WIN_WC_XLATE
+  }
+# endif
+  va_end(ap);
+  return rv;
+}
+
+SQLITE_INTERNAL_LINKAGE int ePrintfUtf8(const char *zFormat, ...){
+  va_list ap;
+  int rv;
+  FILE *pfErr;
+  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
+# if CIO_WIN_WC_XLATE
+  PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr);
+# else
+  getEmitStreamInfo(2, &pst, &pfErr);
+# endif
+  assert(zFormat!=0);
+  va_start(ap, zFormat);
+# if CIO_WIN_WC_XLATE
+  if( pstReachesConsole(ppst) ){
+    rv = conioVmPrintf(ppst, zFormat, ap);
+  }else{
+# endif
+    rv = vfprintf(pfErr, zFormat, ap);
+# if CIO_WIN_WC_XLATE
+  }
+# endif
+  va_end(ap);
+  return rv;
+}
+
+SQLITE_INTERNAL_LINKAGE int fPrintfUtf8(FILE *pfO, const char *zFormat, ...){
+  va_list ap;
+  int rv;
+  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
+# if CIO_WIN_WC_XLATE
+  PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO);
+# else
+  getEmitStreamInfo(0, &pst, &pfO);
+# endif
+  assert(zFormat!=0);
+  va_start(ap, zFormat);
+# if CIO_WIN_WC_XLATE
+  if( pstReachesConsole(ppst) ){
+    maybeSetupAsConsole(ppst, 1);
+    rv = conioVmPrintf(ppst, zFormat, ap);
+    if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst);
+  }else{
+# endif
+    rv = vfprintf(pfO, zFormat, ap);
+# if CIO_WIN_WC_XLATE
+  }
+# endif
+  va_end(ap);
+  return rv;
+}
+
+SQLITE_INTERNAL_LINKAGE int fPutsUtf8(const char *z, FILE *pfO){
+  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
+# if CIO_WIN_WC_XLATE
+  PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO);
+# else
+  getEmitStreamInfo(0, &pst, &pfO);
+# endif
+  assert(z!=0);
+# if CIO_WIN_WC_XLATE
+  if( pstReachesConsole(ppst) ){
+    int rv;
+    maybeSetupAsConsole(ppst, 1);
+    rv = conZstrEmit(ppst, z, (int)strlen(z));
+    if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst);
+    return rv;
+  }else {
+# endif
+    return (fputs(z, pfO)<0)? 0 : (int)strlen(z);
+# if CIO_WIN_WC_XLATE
+  }
+# endif
+}
+
+SQLITE_INTERNAL_LINKAGE int ePutsUtf8(const char *z){
+  FILE *pfErr;
+  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
+# if CIO_WIN_WC_XLATE
+  PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr);
+# else
+  getEmitStreamInfo(2, &pst, &pfErr);
+# endif
+  assert(z!=0);
+# if CIO_WIN_WC_XLATE
+  if( pstReachesConsole(ppst) ) return conZstrEmit(ppst, z, (int)strlen(z));
+  else {
+# endif
+    return (fputs(z, pfErr)<0)? 0 : (int)strlen(z);
+# if CIO_WIN_WC_XLATE
+  }
+# endif
+}
+
+SQLITE_INTERNAL_LINKAGE int oPutsUtf8(const char *z){
+  FILE *pfOut;
+  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
+# if CIO_WIN_WC_XLATE
+  PerStreamTags *ppst = getEmitStreamInfo(1, &pst, &pfOut);
+# else
+  getEmitStreamInfo(1, &pst, &pfOut);
+# endif
+  assert(z!=0);
+# if CIO_WIN_WC_XLATE
+  if( pstReachesConsole(ppst) ) return conZstrEmit(ppst, z, (int)strlen(z));
+  else {
+# endif
+    return (fputs(z, pfOut)<0)? 0 : (int)strlen(z);
+# if CIO_WIN_WC_XLATE
+  }
+# endif
+}
+
+#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
+
+#if !(defined(SQLITE_CIO_NO_UTF8SCAN) && defined(SQLITE_CIO_NO_TRANSLATE))
+/* Skip over as much z[] input char sequence as is valid UTF-8,
+** limited per nAccept char's or whole characters and containing
+** no char cn such that ((1<<cn) & ccm)!=0. On return, the
+** sequence z:return (inclusive:exclusive) is validated UTF-8.
+** Limit: nAccept>=0 => char count, nAccept<0 => character
+ */
+SQLITE_INTERNAL_LINKAGE const char*
+zSkipValidUtf8(const char *z, int nAccept, long ccm){
+  int ng = (nAccept<0)? -nAccept : 0;
+  const char *pcLimit = (nAccept>=0)? z+nAccept : 0;
+  assert(z!=0);
+  while( (pcLimit)? (z<pcLimit) : (ng-- != 0) ){
+    char c = *z;
+    if( (c & 0x80) == 0 ){
+      if( ccm != 0L && c < 0x20 && ((1L<<c) & ccm) != 0 ) return z;
+      ++z; /* ASCII */
+    }else if( (c & 0xC0) != 0xC0 ) return z; /* not a lead byte */
+    else{
+      const char *zt = z+1; /* Got lead byte, look at trail bytes.*/
+      do{
+        if( pcLimit && zt >= pcLimit ) return z;
+        else{
+          char ct = *zt++;
+          if( ct==0 || (zt-z)>4 || (ct & 0xC0)!=0x80 ){
+            /* Trailing bytes are too few, too many, or invalid. */
+            return z;
+          }
+        }
+      } while( ((c <<= 1) & 0x40) == 0x40 ); /* Eat lead byte's count. */
+      z = zt;
+    }
+  }
+  return z;
+}
+#endif /*!(defined(SQLITE_CIO_NO_UTF8SCAN)&&defined(SQLITE_CIO_NO_TRANSLATE))*/
+
+#ifndef SQLITE_CIO_NO_TRANSLATE
+SQLITE_INTERNAL_LINKAGE int
+fPutbUtf8(FILE *pfO, const char *cBuf, int nAccept){
+  assert(pfO!=0);
+# if CIO_WIN_WC_XLATE
+  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
+  PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO);
+  if( pstReachesConsole(ppst) ){
+    int rv;
+    maybeSetupAsConsole(ppst, 1);
+    rv = conZstrEmit(ppst, cBuf, nAccept);
+    if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst);
+    return rv;
+  }else {
+# endif
+    return (int)fwrite(cBuf, 1, nAccept, pfO);
+# if CIO_WIN_WC_XLATE
+  }
+# endif
+}
+
+SQLITE_INTERNAL_LINKAGE int
+oPutbUtf8(const char *cBuf, int nAccept){
+  FILE *pfOut;
+  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
+# if CIO_WIN_WC_XLATE
+  PerStreamTags *ppst = getEmitStreamInfo(1, &pst, &pfOut);
+# else
+  getEmitStreamInfo(1, &pst, &pfOut);
+# endif
+# if CIO_WIN_WC_XLATE
+  if( pstReachesConsole(ppst) ){
+    return conZstrEmit(ppst, cBuf, nAccept);
+  }else {
+# endif
+    return (int)fwrite(cBuf, 1, nAccept, pfOut);
+# if CIO_WIN_WC_XLATE
+  }
+# endif
+}
+
+# ifdef CONSIO_EPUTB
+SQLITE_INTERNAL_LINKAGE int
+ePutbUtf8(const char *cBuf, int nAccept){
+  FILE *pfErr;
+  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
+  PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr);
+#  if CIO_WIN_WC_XLATE
+  if( pstReachesConsole(ppst) ){
+    return conZstrEmit(ppst, cBuf, nAccept);
+  }else {
+#  endif
+    return (int)fwrite(cBuf, 1, nAccept, pfErr);
+#  if CIO_WIN_WC_XLATE
+  }
+#  endif
+}
+# endif /* defined(CONSIO_EPUTB) */
+
+SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){
+  if( pfIn==0 ) pfIn = stdin;
+# if CIO_WIN_WC_XLATE
+  if( pfIn == consoleInfo.pstSetup[0].pf
+      && (consoleInfo.sacSetup & SAC_InConsole)!=0 ){
+#  if CIO_WIN_WC_XLATE==1
+#   define SHELL_GULP 150 /* Count of WCHARS to be gulped at a time */
+    WCHAR wcBuf[SHELL_GULP+1];
+    int lend = 0, noc = 0;
+    if( ncMax > 0 ) cBuf[0] = 0;
+    while( noc < ncMax-8-1 && !lend ){
+      /* There is room for at least 2 more characters and a 0-terminator. */
+      int na = (ncMax > SHELL_GULP*4+1 + noc)? SHELL_GULP : (ncMax-1 - noc)/4;
+#   undef SHELL_GULP
+      DWORD nbr = 0;
+      BOOL bRC = ReadConsoleW(consoleInfo.pstSetup[0].hx, wcBuf, na, &nbr, 0);
+      if( bRC && nbr>0 && (wcBuf[nbr-1]&0xF800)==0xD800 ){
+        /* Last WHAR read is first of a UTF-16 surrogate pair. Grab its mate. */
+        DWORD nbrx;
+        bRC &= ReadConsoleW(consoleInfo.pstSetup[0].hx, wcBuf+nbr, 1, &nbrx, 0);
+        if( bRC ) nbr += nbrx;
+      }
+      if( !bRC || (noc==0 && nbr==0) ) return 0;
+      if( nbr > 0 ){
+        int nmb = WideCharToMultiByte(CP_UTF8, 0, wcBuf,nbr,0,0,0,0);
+        if( nmb != 0 && noc+nmb <= ncMax ){
+          int iseg = noc;
+          nmb = WideCharToMultiByte(CP_UTF8, 0, wcBuf,nbr,cBuf+noc,nmb,0,0);
+          noc += nmb;
+          /* Fixup line-ends as coded by Windows for CR (or "Enter".)
+          ** This is done without regard for any setMode{Text,Binary}()
+          ** call that might have been done on the interactive input.
+          */
+          if( noc > 0 ){
+            if( cBuf[noc-1]=='\n' ){
+              lend = 1;
+              if( noc > 1 && cBuf[noc-2]=='\r' ) cBuf[--noc-1] = '\n';
+            }
+          }
+          /* Check for ^Z (anywhere in line) too, to act as EOF. */
+          while( iseg < noc ){
+            if( cBuf[iseg]=='\x1a' ){
+              noc = iseg; /* Chop ^Z and anything following. */
+              lend = 1; /* Counts as end of line too. */
+              break;
+            }
+            ++iseg;
+          }
+        }else break; /* Drop apparent garbage in. (Could assert.) */
+      }else break;
+    }
+    /* If got nothing, (after ^Z chop), must be at end-of-file. */
+    if( noc > 0 ){
+      cBuf[noc] = 0;
+      return cBuf;
+    }else return 0;
+#  endif
+  }else{
+# endif
+    return fgets(cBuf, ncMax, pfIn);
+# if CIO_WIN_WC_XLATE
+  }
+# endif
+}
+#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
+
+#undef SHELL_INVALID_FILE_PTR
@@ -0,0 +1,278 @@
+/*
+** 2023 November 1
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+********************************************************************************
+** This file exposes various interfaces used for console and other I/O
+** by the SQLite project command-line tools. These interfaces are used
+** at either source conglomeration time, compilation time, or run time.
+** This source provides for either inclusion into conglomerated,
+** "single-source" forms or separate compilation then linking.
+**
+** Platform dependencies are "hidden" here by various stratagems so
+** that, provided certain conditions are met, the programs using this
+** source or object code compiled from it need no explicit conditional
+** compilation in their source for their console and stream I/O.
+**
+** The symbols and functionality exposed here are not a public API.
+** This code may change in tandem with other project code as needed.
+**
+** When this .h file and its companion .c are directly incorporated into
+** a source conglomeration (such as shell.c), the preprocessor symbol
+** CIO_WIN_WC_XLATE is defined as 0 or 1, reflecting whether console I/O
+** translation for Windows is effected for the build.
+*/
+
+#ifndef SQLITE_INTERNAL_LINKAGE
+# define SQLITE_INTERNAL_LINKAGE extern /* external to translation unit */
+# include <stdio.h>
+#else
+# define SHELL_NO_SYSINC /* Better yet, modify mkshellc.tcl for this. */
+#endif
+
+#ifndef SQLITE3_H
+# include "sqlite3.h"
+#endif
+
+#ifndef SQLITE_CIO_NO_CLASSIFY
+
+/* Define enum for use with following function. */
+typedef enum StreamsAreConsole {
+  SAC_NoConsole = 0,
+  SAC_InConsole = 1, SAC_OutConsole = 2, SAC_ErrConsole = 4,
+  SAC_AnyConsole = 0x7
+} StreamsAreConsole;
+
+/*
+** Classify the three standard I/O streams according to whether
+** they are connected to a console attached to the process.
+**
+** Returns the bit-wise OR of SAC_{In,Out,Err}Console values,
+** or SAC_NoConsole if none of the streams reaches a console.
+**
+** This function should be called before any I/O is done with
+** the given streams. As a side-effect, the given inputs are
+** recorded so that later I/O operations on them may be done
+** differently than the C library FILE* I/O would be done,
+** iff the stream is used for the I/O functions that follow,
+** and to support the ones that use an implicit stream.
+**
+** On some platforms, stream or console mode alteration (aka
+** "Setup") may be made which is undone by consoleRestore().
+*/
+SQLITE_INTERNAL_LINKAGE StreamsAreConsole
+consoleClassifySetup( FILE *pfIn, FILE *pfOut, FILE *pfErr );
+/* A usual call for convenience: */
+#define SQLITE_STD_CONSOLE_INIT() consoleClassifySetup(stdin,stdout,stderr)
+
+/*
+** After an initial call to consoleClassifySetup(...), renew
+** the same setup it effected. (A call not after is an error.)
+** This will restore state altered by consoleRestore();
+**
+** Applications which run an inferior (child) process which
+** inherits the same I/O streams may call this function after
+** such a process exits to guard against console mode changes.
+*/
+SQLITE_INTERNAL_LINKAGE void consoleRenewSetup(void);
+
+/*
+** Undo any side-effects left by consoleClassifySetup(...).
+**
+** This should be called after consoleClassifySetup() and
+** before the process terminates normally. It is suitable
+** for use with the atexit() C library procedure. After
+** this call, no console I/O should be done until one of
+** console{Classify or Renew}Setup(...) is called again.
+**
+** Applications which run an inferior (child) process that
+** inherits the same I/O streams might call this procedure
+** before so that said process will have a console setup
+** however users have configured it or come to expect.
+*/
+SQLITE_INTERNAL_LINKAGE void SQLITE_CDECL consoleRestore( void );
+
+#else /* defined(SQLITE_CIO_NO_CLASSIFY) */
+# define consoleClassifySetup(i,o,e)
+# define consoleRenewSetup()
+# define consoleRestore()
+#endif /* defined(SQLITE_CIO_NO_CLASSIFY) */
+
+#ifndef SQLITE_CIO_NO_REDIRECT
+/*
+** Set stream to be used for the functions below which write
+** to "the designated X stream", where X is Output or Error.
+** Returns the previous value.
+**
+** Alternatively, pass the special value, invalidFileStream,
+** to get the designated stream value without setting it.
+**
+** Before the designated streams are set, they default to
+** those passed to consoleClassifySetup(...), and before
+** that is called they default to stdout and stderr.
+**
+** It is error to close a stream so designated, then, without
+** designating another, use the corresponding {o,e}Emit(...).
+*/
+SQLITE_INTERNAL_LINKAGE FILE *invalidFileStream;
+SQLITE_INTERNAL_LINKAGE FILE *setOutputStream(FILE *pf);
+# ifdef CONSIO_SET_ERROR_STREAM
+SQLITE_INTERNAL_LINKAGE FILE *setErrorStream(FILE *pf);
+# endif
+#else
+# define setOutputStream(pf)
+# define setErrorStream(pf)
+#endif /* !defined(SQLITE_CIO_NO_REDIRECT) */
+
+#ifndef SQLITE_CIO_NO_TRANSLATE
+/*
+** Emit output like fprintf(). If the output is going to the
+** console and translation from UTF-8 is necessary, perform
+** the needed translation. Otherwise, write formatted output
+** to the provided stream almost as-is, possibly with newline
+** translation as specified by set{Binary,Text}Mode().
+*/
+SQLITE_INTERNAL_LINKAGE int fPrintfUtf8(FILE *pfO, const char *zFormat, ...);
+/* Like fPrintfUtf8 except stream is always the designated output. */
+SQLITE_INTERNAL_LINKAGE int oPrintfUtf8(const char *zFormat, ...);
+/* Like fPrintfUtf8 except stream is always the designated error. */
+SQLITE_INTERNAL_LINKAGE int ePrintfUtf8(const char *zFormat, ...);
+
+/*
+** Emit output like fputs(). If the output is going to the
+** console and translation from UTF-8 is necessary, perform
+** the needed translation. Otherwise, write given text to the
+** provided stream almost as-is, possibly with newline
+** translation as specified by set{Binary,Text}Mode().
+*/
+SQLITE_INTERNAL_LINKAGE int fPutsUtf8(const char *z, FILE *pfO);
+/* Like fPutsUtf8 except stream is always the designated output. */
+SQLITE_INTERNAL_LINKAGE int oPutsUtf8(const char *z);
+/* Like fPutsUtf8 except stream is always the designated error. */
+SQLITE_INTERNAL_LINKAGE int ePutsUtf8(const char *z);
+
+/*
+** Emit output like fPutsUtf8(), except that the length of the
+** accepted char or character sequence is limited by nAccept.
+**
+** Returns the number of accepted char values.
+*/
+SQLITE_INTERNAL_LINKAGE int
+fPutbUtf8(FILE *pfOut, const char *cBuf, int nAccept);
+/* Like fPutbUtf8 except stream is always the designated output. */
+SQLITE_INTERNAL_LINKAGE int
+oPutbUtf8(const char *cBuf, int nAccept);
+/* Like fPutbUtf8 except stream is always the designated error. */
+#ifdef CONSIO_EPUTB
+SQLITE_INTERNAL_LINKAGE int
+ePutbUtf8(const char *cBuf, int nAccept);
+#endif
+
+/*
+** Collect input like fgets(...) with special provisions for input
+** from the console on platforms that require same. Defers to the
+** C library fgets() when input is not from the console. Newline
+** translation may be done as set by set{Binary,Text}Mode(). As a
+** convenience, pfIn==NULL is treated as stdin.
+*/
+SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn);
+/* Like fGetsUtf8 except stream is always the designated input. */
+/* SQLITE_INTERNAL_LINKAGE char* iGetsUtf8(char *cBuf, int ncMax); */
+
+#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
+
+#ifndef SQLITE_CIO_NO_SETMODE
+/*
+** Set given stream for binary mode, where newline translation is
+** not done, or for text mode where, for some platforms, newlines
+** are translated to the platform's conventional char sequence.
+** If bFlush true, flush the stream.
+**
+** An additional side-effect is that if the stream is one passed
+** to consoleClassifySetup() as an output, it is flushed first.
+**
+** Note that binary/text mode has no effect on console I/O
+** translation. On all platforms, newline to the console starts
+** a new line and CR,LF chars from the console become a newline.
+*/
+SQLITE_INTERNAL_LINKAGE void setBinaryMode(FILE *, short bFlush);
+SQLITE_INTERNAL_LINKAGE void setTextMode(FILE *, short bFlush);
+#endif
+
+#ifdef SQLITE_CIO_PROMPTED_IN
+typedef struct Prompts {
+  int numPrompts;
+  const char **azPrompts;
+} Prompts;
+
+/*
+** Macros for use of a line editor.
+**
+** The following macros define operations involving use of a
+** line-editing library or simple console interaction.
+** A "T" argument is a text (char *) buffer or filename.
+** A "N" argument is an integer.
+**
+** SHELL_ADD_HISTORY(T) // Record text as line(s) of history.
+** SHELL_READ_HISTORY(T) // Read history from file named by T.
+** SHELL_WRITE_HISTORY(T) // Write history to file named by T.
+** SHELL_STIFLE_HISTORY(N) // Limit history to N entries.
+**
+** A console program which does interactive console input is
+** expected to call:
+** SHELL_READ_HISTORY(T) before collecting such input;
+** SHELL_ADD_HISTORY(T) as record-worthy input is taken;
+** SHELL_STIFLE_HISTORY(N) after console input ceases; then
+** SHELL_WRITE_HISTORY(T) before the program exits.
+*/
+
+/*
+** Retrieve a single line of input text from an input stream.
+**
+** If pfIn is the input stream passed to consoleClassifySetup(),
+** and azPrompt is not NULL, then a prompt is issued before the
+** line is collected, as selected by the isContinuation flag.
+** Array azPrompt[{0,1}] holds the {main,continuation} prompt.
+**
+** If zBufPrior is not NULL then it is a buffer from a prior
+** call to this routine that can be reused, or will be freed.
+**
+** The result is stored in space obtained from malloc() and
+** must either be freed by the caller or else passed back to
+** this function as zBufPrior for reuse.
+**
+** This function may call upon services of a line-editing
+** library to interactively collect line edited input.
+*/
+SQLITE_INTERNAL_LINKAGE char *
+shellGetLine(FILE *pfIn, char *zBufPrior, int nLen,
+             short isContinuation, Prompts azPrompt);
+#endif /* defined(SQLITE_CIO_PROMPTED_IN) */
+/*
+** TBD: Define an interface for application(s) to generate
+** completion candidates for use by the line-editor.
+**
+** This may be premature; the CLI is the only application
+** that does this. Yet, getting line-editing melded into
+** console I/O is desirable because a line-editing library
+** may have to establish console operating mode, possibly
+** in a way that interferes with the above functionality.
+*/
+
+#if !(defined(SQLITE_CIO_NO_UTF8SCAN)&&defined(SQLITE_CIO_NO_TRANSLATE))
+/* Skip over as much z[] input char sequence as is valid UTF-8,
+** limited per nAccept char's or whole characters and containing
+** no char cn such that ((1<<cn) & ccm)!=0. On return, the
+** sequence z:return (inclusive:exclusive) is validated UTF-8.
+** Limit: nAccept>=0 => char count, nAccept<0 => character
+ */
+SQLITE_INTERNAL_LINKAGE const char*
+zSkipValidUtf8(const char *z, int nAccept, long ccm);
+
+#endif
@@ -464,4 +464,23 @@ do_execsql_test 5.3 {
  t2 t2_idx_0001295b {100 20 5}
 }

+if 0 {
+do_test expert1-6.0 {
+  catchcmd :memory: {
+.expert
+select base64('');
+.expert
+select name from pragma_collation_list order by name collate uint;
+}
+} {0 {(no new indexes)
+
+SCAN CONSTANT ROW
+
+(no new indexes)
+
+SCAN pragma_collation_list VIRTUAL TABLE INDEX 0:
+USE TEMP B-TREE FOR ORDER BY
+}}
+}
+
 finish_test
@@ -32,7 +32,7 @@
 #endif /* !defined(SQLITE_AMALGAMATION) */


-#ifndef SQLITE_OMIT_VIRTUALTABLE 
+#ifndef SQLITE_OMIT_VIRTUALTABLE

 typedef sqlite3_int64 i64;
 typedef sqlite3_uint64 u64;
@@ -662,6 +662,7 @@ static int idxRegisterVtab(sqlite3expert *p){
    0,                            /* xRelease */
    0,                            /* xRollbackTo */
    0,                            /* xShadowName */
+    0,                            /* xIntegrity */
  };

  return sqlite3_create_module(p->dbv, "expert", &expertModule, (void*)p);
@@ -1818,6 +1819,88 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
  return rc;
 }

+/*
+** Define and possibly pretend to use a useless collation sequence.
+** This pretense allows expert to accept SQL using custom collations.
+*/
+int dummyCompare(void *up1, int up2, const void *up3, int up4, const void *up5){
+  (void)up1;
+  (void)up2;
+  (void)up3;
+  (void)up4;
+  (void)up5;
+  assert(0); /* VDBE should never be run. */
+  return 0;
+}
+/* And a callback to register above upon actual need */
+void useDummyCS(void *up1, sqlite3 *db, int etr, const char *zName){
+  (void)up1;
+  sqlite3_create_collation_v2(db, zName, etr, 0, dummyCompare, 0);
+}
+
+#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) \
+  && !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS)
+/*
+** dummy functions for no-op implementation of UDFs during expert's work
+*/
+void dummyUDF(sqlite3_context *up1, int up2, sqlite3_value **up3){
+  (void)up1;
+  (void)up2;
+  (void)up3;
+  assert(0); /* VDBE should never be run. */
+}
+void dummyUDFvalue(sqlite3_context *up1){
+  (void)up1;
+  assert(0); /* VDBE should never be run. */
+}
+
+/*
+** Register UDFs from user database with another.
+*/
+int registerUDFs(sqlite3 *dbSrc, sqlite3 *dbDst){
+  sqlite3_stmt *pStmt;
+  int rc = sqlite3_prepare_v2(dbSrc,
+            "SELECT name,type,enc,narg,flags "
+            "FROM pragma_function_list() "
+            "WHERE builtin==0", -1, &pStmt, 0);
+  if( rc==SQLITE_OK ){
+    while( SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
+      int nargs = sqlite3_column_int(pStmt,3);
+      int flags = sqlite3_column_int(pStmt,4);
+      const char *name = (char*)sqlite3_column_text(pStmt,0);
+      const char *type = (char*)sqlite3_column_text(pStmt,1);
+      const char *enc = (char*)sqlite3_column_text(pStmt,2);
+      if( name==0 || type==0 || enc==0 ){
+        /* no-op.  Only happens on OOM */
+      }else{
+        int ienc = SQLITE_UTF8;
+        int rcf = SQLITE_ERROR;
+        if( strcmp(enc,"utf16le")==0 ) ienc = SQLITE_UTF16LE;
+        else if( strcmp(enc,"utf16be")==0 ) ienc = SQLITE_UTF16BE;
+        ienc |= (flags & (SQLITE_DETERMINISTIC|SQLITE_DIRECTONLY));
+        if( strcmp(type,"w")==0 ){
+          rcf = sqlite3_create_window_function(dbDst,name,nargs,ienc,0,
+                                               dummyUDF,dummyUDFvalue,0,0,0);
+        }else if( strcmp(type,"a")==0 ){
+          rcf = sqlite3_create_function(dbDst,name,nargs,ienc,0,
+                                        0,dummyUDF,dummyUDFvalue);
+        }else if( strcmp(type,"s")==0 ){
+          rcf = sqlite3_create_function(dbDst,name,nargs,ienc,0,
+                                        dummyUDF,0,0);
+        }
+        if( rcf!=SQLITE_OK ){
+          rc = rcf;
+          break;
+        }
+      }
+    }
+    sqlite3_finalize(pStmt);
+    if( rc==SQLITE_DONE ) rc = SQLITE_OK;
+  }
+  return rc;
+}
+#endif
+
 /*
 ** Allocate a new sqlite3expert object.
 */
@@ -1844,7 +1927,21 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
      sqlite3_db_config(pNew->dbm, SQLITE_DBCONFIG_TRIGGER_EQP, 1, (int*)0);
    }
  }
-  
+
+  /* Allow custom collations to be dealt with through prepare. */
+  if( rc==SQLITE_OK ) rc = sqlite3_collation_needed(pNew->dbm,0,useDummyCS);
+  if( rc==SQLITE_OK ) rc = sqlite3_collation_needed(pNew->dbv,0,useDummyCS);
+
+#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) \
+  && !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS)
+  /* Register UDFs from database [db] with [dbm] and [dbv]. */
+  if( rc==SQLITE_OK ){
+    rc = registerUDFs(pNew->db, pNew->dbm);
+  }
+  if( rc==SQLITE_OK ){
+    rc = registerUDFs(pNew->db, pNew->dbv);
+  }
+#endif

  /* Copy the entire schema of database [db] into [dbm]. */
  if( rc==SQLITE_OK ){
@@ -1920,6 +2017,10 @@ int sqlite3_expert_sql(

  while( rc==SQLITE_OK && zStmt && zStmt[0] ){
    sqlite3_stmt *pStmt = 0;
+    /* Ensure that the provided statement compiles against user's DB. */
+    rc = idxPrepareStmt(p->db, &pStmt, pzErr, zStmt);
+    if( rc!=SQLITE_OK ) break;
+    sqlite3_finalize(pStmt);
    rc = sqlite3_prepare_v2(p->dbv, zStmt, -1, &pStmt, &zStmt);
    if( rc==SQLITE_OK ){
      if( pStmt ){
@@ -640,6 +640,7 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){

    zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid");
    sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1);
+    sqlite3_vtab_config(p->db, SQLITE_VTAB_INNOCUOUS);

    /* Create a list of user columns for the virtual table */
    zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]);
@@ -3889,6 +3890,8 @@ static int fts3RenameMethod(
    rc = sqlite3Fts3PendingTermsFlush(p);
  }

+  p->bIgnoreSavepoint = 1;
+
  if( p->zContentTbl==0 ){
    fts3DbExec(&rc, db,
      "ALTER TABLE %Q.'%q_content'  RENAME TO '%q_content';",
@@ -3916,6 +3919,8 @@ static int fts3RenameMethod(
    "ALTER TABLE %Q.'%q_segdir'   RENAME TO '%q_segdir';",
    p->zDb, p->zName, zName
  );
+
+  p->bIgnoreSavepoint = 0;
  return rc;
 }

@@ -3926,12 +3931,28 @@ static int fts3RenameMethod(
 */
 static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
  int rc = SQLITE_OK;
-  UNUSED_PARAMETER(iSavepoint);
-  assert( ((Fts3Table *)pVtab)->inTransaction );
-  assert( ((Fts3Table *)pVtab)->mxSavepoint <= iSavepoint );
-  TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
-  if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){
-    rc = fts3SyncMethod(pVtab);
+  Fts3Table *pTab = (Fts3Table*)pVtab;
+  assert( pTab->inTransaction );
+  assert( pTab->mxSavepoint<=iSavepoint );
+  TESTONLY( pTab->mxSavepoint = iSavepoint );
+
+  if( pTab->bIgnoreSavepoint==0 ){
+    if( fts3HashCount(&pTab->aIndex[0].hPending)>0 ){
+      char *zSql = sqlite3_mprintf("INSERT INTO %Q.%Q(%Q) VALUES('flush')",
+          pTab->zDb, pTab->zName, pTab->zName
+          );
+      if( zSql ){
+        pTab->bIgnoreSavepoint = 1;
+        rc = sqlite3_exec(pTab->db, zSql, 0, 0, 0);
+        pTab->bIgnoreSavepoint = 0;
+        sqlite3_free(zSql);
+      }else{
+        rc = SQLITE_NOMEM;
+      }
+    }
+    if( rc==SQLITE_OK ){
+      pTab->iSavepoint = iSavepoint+1;
+    }
  }
  return rc;
 }
@@ -3942,12 +3963,11 @@ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
 ** This is a no-op.
 */
 static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
-  TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
-  UNUSED_PARAMETER(iSavepoint);
-  UNUSED_PARAMETER(pVtab);
-  assert( p->inTransaction );
-  assert( p->mxSavepoint >= iSavepoint );
-  TESTONLY( p->mxSavepoint = iSavepoint-1 );
+  Fts3Table *pTab = (Fts3Table*)pVtab;
+  assert( pTab->inTransaction );
+  assert( pTab->mxSavepoint >= iSavepoint );
+  TESTONLY( pTab->mxSavepoint = iSavepoint-1 );
+  pTab->iSavepoint = iSavepoint;
  return SQLITE_OK;
 }

@@ -3957,11 +3977,13 @@ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
 ** Discard the contents of the pending terms table.
 */
 static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
-  Fts3Table *p = (Fts3Table*)pVtab;
+  Fts3Table *pTab = (Fts3Table*)pVtab;
  UNUSED_PARAMETER(iSavepoint);
-  assert( p->inTransaction );
-  TESTONLY( p->mxSavepoint = iSavepoint );
-  sqlite3Fts3PendingTermsClear(p);
+  assert( pTab->inTransaction );
+  TESTONLY( pTab->mxSavepoint = iSavepoint );
+  if( (iSavepoint+1)<=pTab->iSavepoint ){
+    sqlite3Fts3PendingTermsClear(pTab);
+  }
  return SQLITE_OK;
 }

@@ -3980,8 +4002,49 @@ static int fts3ShadowName(const char *zName){
  return 0;
 }

+/*
+** Implementation of the xIntegrity() method on the FTS3/FTS4 virtual
+** table.
+*/
+static int fts3Integrity(
+  sqlite3_vtab *pVtab,      /* The virtual table to be checked */
+  const char *zSchema,      /* Name of schema in which pVtab lives */
+  const char *zTabname,     /* Name of the pVTab table */
+  int isQuick,              /* True if this is a quick_check */
+  char **pzErr              /* Write error message here */
+){
+  Fts3Table *p = (Fts3Table*)pVtab;
+  char *zSql;
+  int rc;
+  char *zErr = 0;
+
+  assert( pzErr!=0 );
+  assert( *pzErr==0 );
+  UNUSED_PARAMETER(isQuick);
+  zSql = sqlite3_mprintf(
+            "INSERT INTO \"%w\".\"%w\"(\"%w\") VALUES('integrity-check');",
+            zSchema, zTabname, zTabname);
+  if( zSql==0 ){
+    return SQLITE_NOMEM;
+  }
+  rc = sqlite3_exec(p->db, zSql, 0, 0, &zErr);
+  sqlite3_free(zSql);
+  if( (rc&0xff)==SQLITE_CORRUPT ){
+    *pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s",
+                p->bFts4 ? 4 : 3, zSchema, zTabname);
+  }else if( rc!=SQLITE_OK ){
+    *pzErr = sqlite3_mprintf("unable to validate the inverted index for"
+                             " FTS%d table %s.%s: %s",
+                p->bFts4 ? 4 : 3, zSchema, zTabname, zErr);
+  }
+  sqlite3_free(zErr);
+  return SQLITE_OK;
+}
+
+
+
 static const sqlite3_module fts3Module = {
-  /* iVersion      */ 3,
+  /* iVersion      */ 4,
  /* xCreate       */ fts3CreateMethod,
  /* xConnect      */ fts3ConnectMethod,
  /* xBestIndex    */ fts3BestIndexMethod,
@@ -4005,6 +4068,7 @@ static const sqlite3_module fts3Module = {
  /* xRelease      */ fts3ReleaseMethod,
  /* xRollbackTo   */ fts3RollbackToMethod,
  /* xShadowName   */ fts3ShadowName,
+  /* xIntegrity    */ fts3Integrity,
 };

 /*
@@ -265,6 +265,7 @@ struct Fts3Table {
  int nPgsz;                      /* Page size for host database */
  char *zSegmentsTbl;             /* Name of %_segments table */
  sqlite3_blob *pSegments;        /* Blob handle open on %_segments table */
+  int iSavepoint;

  /* 
  ** The following array of hash tables is used to buffer pending index 
@@ -545,7 +545,8 @@ int sqlite3Fts3InitAux(sqlite3 *db){
     0,                           /* xSavepoint    */
     0,                           /* xRelease      */
     0,                           /* xRollbackTo   */
-     0                            /* xShadowName   */
+     0,                           /* xShadowName   */
+     0                            /* xIntegrity    */
  };
  int rc;                         /* Return code */

@@ -362,7 +362,8 @@ int sqlite3Fts3InitTerm(sqlite3 *db){
     0,                           /* xSavepoint    */
     0,                           /* xRelease      */
     0,                           /* xRollbackTo   */
-     0                            /* xShadowName   */
+     0,                           /* xShadowName   */
+     0                            /* xIntegrity    */
  };
  int rc;                         /* Return code */

@@ -445,7 +445,8 @@ int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash, void(*xDestroy)(void*)){
     0,                           /* xSavepoint    */
     0,                           /* xRelease      */
     0,                           /* xRollbackTo   */
-     0                            /* xShadowName   */
+     0,                           /* xShadowName   */
+     0                            /* xIntegrity    */
  };
  int rc;                         /* Return code */

@@ -3325,7 +3325,6 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
    rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING);
    if( rc==SQLITE_DONE ) rc = SQLITE_OK;
  }
-  sqlite3Fts3PendingTermsClear(p);

  /* Determine the auto-incr-merge setting if unknown.  If enabled,
  ** estimate the number of leaf blocks of content to be written
@@ -3347,6 +3346,10 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
      rc = sqlite3_reset(pStmt);
    }
  }
+
+  if( rc==SQLITE_OK ){
+    sqlite3Fts3PendingTermsClear(p);
+  }
  return rc;
 }

@@ -3978,6 +3981,8 @@ static int fts3AppendToNode(

  blobGrowBuffer(pPrev, nTerm, &rc);
  if( rc!=SQLITE_OK ) return rc;
+  assert( pPrev!=0 );
+  assert( pPrev->a!=0 );

  nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm);
  nSuffix = nTerm - nPrefix;
@@ -4034,9 +4039,13 @@ static int fts3IncrmergeAppend(
  nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist;

  /* If the current block is not empty, and if adding this term/doclist
-  ** to the current block would make it larger than Fts3Table.nNodeSize
-  ** bytes, write this block out to the database. */
-  if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){
+  ** to the current block would make it larger than Fts3Table.nNodeSize bytes,
+  ** and if there is still room for another leaf page, write this block out to
+  ** the database. */
+  if( pLeaf->block.n>0 
+   && (pLeaf->block.n + nSpace)>p->nNodeSize 
+   && pLeaf->iBlock < (pWriter->iStart + pWriter->nLeafEst)
+  ){
    rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n);
    pWriter->nWork++;

@@ -4368,7 +4377,7 @@ static int fts3IncrmergeLoad(
              rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock,0);
              blobGrowBuffer(&pNode->block, 
                  MAX(nBlock, p->nNodeSize)+FTS3_NODE_PADDING, &rc
-                  );
+              );
              if( rc==SQLITE_OK ){
                memcpy(pNode->block.a, aBlock, nBlock);
                pNode->block.n = nBlock;
@@ -5218,7 +5227,7 @@ static u64 fts3ChecksumIndex(
  int rc;
  u64 cksum = 0;

-  assert( *pRc==SQLITE_OK );
+  if( *pRc ) return 0;

  memset(&filter, 0, sizeof(filter));
  memset(&csr, 0, sizeof(csr));
@@ -5433,8 +5442,11 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
    rc = fts3DoIncrmerge(p, &zVal[6]);
  }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){
    rc = fts3DoAutoincrmerge(p, &zVal[10]);
+  }else if( nVal==5 && 0==sqlite3_strnicmp(zVal, "flush", 5) ){
+    rc = sqlite3Fts3PendingTermsFlush(p);
+  }
 #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
-  }else{
+  else{
    int v;
    if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
      v = atoi(&zVal[9]);
@@ -5452,8 +5464,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
      if( v>=4 && v<=FTS3_MERGE_COUNT && (v&1)==0 ) p->nMergeCount = v;
      rc = SQLITE_OK;
    }
-#endif
  }
+#endif
  return rc;
 }

@@ -492,8 +492,8 @@ struct Fts5ExtensionApi {
 **   as separate queries of the FTS index are required for each synonym.
 **
 **   When using methods (2) or (3), it is important that the tokenizer only
-**   provide synonyms when tokenizing document text (method (2)) or query
-**   text (method (3)), not both. Doing so will not cause any errors, but is
+**   provide synonyms when tokenizing document text (method (3)) or query
+**   text (method (2)), not both. Doing so will not cause any errors, but is
 **   inefficient.
 */
 typedef struct Fts5Tokenizer Fts5Tokenizer;
@@ -110,15 +110,19 @@ static int fts5CInstIterInit(
 */
 typedef struct HighlightContext HighlightContext;
 struct HighlightContext {
-  CInstIter iter;                 /* Coalesced Instance Iterator */
-  int iPos;                       /* Current token offset in zIn[] */
+  /* Constant parameters to fts5HighlightCb() */
  int iRangeStart;                /* First token to include */
  int iRangeEnd;                  /* If non-zero, last token to include */
  const char *zOpen;              /* Opening highlight */
  const char *zClose;             /* Closing highlight */
  const char *zIn;                /* Input text */
  int nIn;                        /* Size of input text in bytes */
-  int iOff;                       /* Current offset within zIn[] */
+
+  /* Variables modified by fts5HighlightCb() */
+  CInstIter iter;                 /* Coalesced Instance Iterator */
+  int iPos;                       /* Current token offset in zIn[] */
+  int iOff;                       /* Have copied up to this offset in zIn[] */
+  int bOpen;                      /* True if highlight is open */
  char *zOut;                     /* Output value */
 };

@@ -151,8 +155,8 @@ static int fts5HighlightCb(
  int tflags,                     /* Mask of FTS5_TOKEN_* flags */
  const char *pToken,             /* Buffer containing token */
  int nToken,                     /* Size of token in bytes */
-  int iStartOff,                  /* Start offset of token */
-  int iEndOff                     /* End offset of token */
+  int iStartOff,                  /* Start byte offset of token */
+  int iEndOff                     /* End byte offset of token */
 ){
  HighlightContext *p = (HighlightContext*)pContext;
  int rc = SQLITE_OK;
@@ -168,30 +172,55 @@ static int fts5HighlightCb(
    if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff;
  }

-  if( iPos==p->iter.iStart ){
+  /* If the parenthesis is open, and this token is not part of the current
+  ** phrase, and the starting byte offset of this token is past the point
+  ** that has currently been copied into the output buffer, close the
+  ** parenthesis. */
+  if( p->bOpen 
+   && (iPos<=p->iter.iStart || p->iter.iStart<0)
+   && iStartOff>p->iOff 
+  ){
+    fts5HighlightAppend(&rc, p, p->zClose, -1);
+    p->bOpen = 0;
+  }
+
+  /* If this is the start of a new phrase, and the highlight is not open:
+  **
+  **   * copy text from the input up to the start of the phrase, and
+  **   * open the highlight.
+  */
+  if( iPos==p->iter.iStart && p->bOpen==0 ){
    fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff);
    fts5HighlightAppend(&rc, p, p->zOpen, -1);
    p->iOff = iStartOff;
+    p->bOpen = 1;
  }

  if( iPos==p->iter.iEnd ){
-    if( p->iRangeEnd>=0 && p->iter.iStart<p->iRangeStart ){
+    if( p->bOpen==0 ){
+      assert( p->iRangeEnd>=0 );
      fts5HighlightAppend(&rc, p, p->zOpen, -1);
+      p->bOpen = 1;
    }
    fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
-    fts5HighlightAppend(&rc, p, p->zClose, -1);
    p->iOff = iEndOff;
+
    if( rc==SQLITE_OK ){
      rc = fts5CInstIterNext(&p->iter);
    }
  }

-  if( p->iRangeEnd>=0 && iPos==p->iRangeEnd ){
+  if( iPos==p->iRangeEnd ){
+    if( p->bOpen ){
+      if( p->iter.iStart>=0 && iPos>=p->iter.iStart ){
+        fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
+        p->iOff = iEndOff;
+      }
+      fts5HighlightAppend(&rc, p, p->zClose, -1);
+      p->bOpen = 0;
+    }
    fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
    p->iOff = iEndOff;
-    if( iPos>=p->iter.iStart && iPos<p->iter.iEnd ){
-      fts5HighlightAppend(&rc, p, p->zClose, -1);
-    }
  }

  return rc;
@@ -232,6 +261,9 @@ static void fts5HighlightFunction(
    if( rc==SQLITE_OK ){
      rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
    }
+    if( ctx.bOpen ){
+      fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1);
+    }
    fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);

    if( rc==SQLITE_OK ){
@@ -510,6 +542,9 @@ static void fts5SnippetFunction(
    if( rc==SQLITE_OK ){
      rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
    }
+    if( ctx.bOpen ){
+      fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1);
+    }
    if( ctx.iRangeEnd>=(nColSize-1) ){
      fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
    }else{
@@ -2583,6 +2583,8 @@ static char *fts5ExprPrintTcl(
      if( zRet==0 ) return 0;
    }

+  }else if( pExpr->eType==0 ){
+    zRet = sqlite3_mprintf("{}");
  }else{
    char const *zOp = 0;
    int i;
@@ -432,10 +432,8 @@ static Fts5HashEntry *fts5HashEntryMerge(
 }

 /*
-** Extract all tokens from hash table iHash and link them into a list
-** in sorted order. The hash table is cleared before returning. It is
-** the responsibility of the caller to free the elements of the returned
-** list.
+** Link all tokens from hash table iHash into a list in sorted order. The
+** tokens are not removed from the hash table.
 */
 static int fts5HashEntrySort(
  Fts5Hash *pHash, 
@@ -2719,6 +2719,14 @@ static void fts5SegIterHashInit(
        pLeaf->p = (u8*)pList;
      }
    }
+
+    /* The call to sqlite3Fts5HashScanInit() causes the hash table to
+    ** fill the size field of all existing position lists. This means they
+    ** can no longer be appended to. Since the only scenario in which they
+    ** can be appended to is if the previous operation on this table was
+    ** a DELETE, by clearing the Fts5Index.bDelete flag we can avoid this
+    ** possibility altogether.  */
+    p->bDelete = 0;
  }else{
    p->rc = sqlite3Fts5HashQuery(p->pHash, sizeof(Fts5Data), 
        (const char*)pTerm, nTerm, (void**)&pLeaf, &nList
@@ -2904,7 +2912,6 @@ static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){
      assert_nc( i2!=0 );
      pRes->bTermEq = 1;
      if( p1->iRowid==p2->iRowid ){
-        p1->bDel = p2->bDel;
        return i2;
      }
      res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : +1;
@@ -3272,7 +3279,7 @@ static Fts5Iter *fts5MultiIterAlloc(
  int nSeg
 ){
  Fts5Iter *pNew;
-  int nSlot;                      /* Power of two >= nSeg */
+  i64 nSlot;                      /* Power of two >= nSeg */

  for(nSlot=2; nSlot<nSeg; nSlot=nSlot*2);
  pNew = fts5IdxMalloc(p, 
@@ -4397,7 +4404,7 @@ static void fts5WriteAppendPoslistData(
  const u8 *a = aData;
  int n = nData;
  
-  assert( p->pConfig->pgsz>0 );
+  assert( p->pConfig->pgsz>0 || p->rc!=SQLITE_OK );
  while( p->rc==SQLITE_OK 
     && (pPage->buf.n + pPage->pgidx.n + n)>=p->pConfig->pgsz 
  ){
@@ -5048,7 +5055,6 @@ static void fts5DoSecureDelete(
  int iPgIdx = pSeg->pLeaf->szLeaf;

  u64 iDelta = 0;
-  u64 iNextDelta = 0;
  int iNextOff = 0;
  int iOff = 0;
  int nIdx = 0;
@@ -5056,8 +5062,6 @@ static void fts5DoSecureDelete(
  int bLastInDoclist = 0;
  int iIdx = 0;
  int iStart = 0;
-  int iKeyOff = 0;
-  int iPrevKeyOff = 0;
  int iDelKeyOff = 0;       /* Offset of deleted key, if any */

  nIdx = nPg-iPgIdx;
@@ -5082,10 +5086,21 @@ static void fts5DoSecureDelete(
  ** This block sets the following variables:
  **
  **   iStart:
+  **     The offset of the first byte of the rowid or delta-rowid
+  **     value for the doclist entry being removed.
+  **
  **   iDelta:
+  **     The value of the rowid or delta-rowid value for the doclist
+  **     entry being removed.
+  **
+  **   iNextOff:
+  **     The offset of the next entry following the position list
+  **     for the one being removed. If the position list for this
+  **     entry overflows onto the next leaf page, this value will be
+  **     greater than pLeaf->szLeaf.
  */
  {
-    int iSOP;
+    int iSOP;                     /* Start-Of-Position-list */
    if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){
      iStart = pSeg->iTermLeafOffset;
    }else{
@@ -5121,47 +5136,75 @@ static void fts5DoSecureDelete(
  }

  iOff = iStart;
-  if( iNextOff>=iPgIdx ){
-    int pgno = pSeg->iLeafPgno+1;
-    fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist);
-    iNextOff = iPgIdx;
-  }else{
-    /* Set bLastInDoclist to true if the entry being removed is the last
-    ** in its doclist.  */
-    for(iIdx=0, iKeyOff=0; iIdx<nIdx; /* no-op */){
-      u32 iVal = 0;
-      iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
-      iKeyOff += iVal;
-      if( iKeyOff==iNextOff ){
-        bLastInDoclist = 1;
+
+  /* Set variable bLastInDoclist to true if this entry happens to be
+  ** the last rowid in the doclist for its term.  */
+  if( pSeg->bDel==0 ){
+    if( iNextOff>=iPgIdx ){
+      int pgno = pSeg->iLeafPgno+1;
+      fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist);
+      iNextOff = iPgIdx;
+    }else{
+      /* Loop through the page-footer. If iNextOff (offset of the
+      ** entry following the one we are removing) is equal to the 
+      ** offset of a key on this page, then the entry is the last 
+      ** in its doclist.  */
+      int iKeyOff = 0;
+      for(iIdx=0; iIdx<nIdx; /* no-op */){
+        u32 iVal = 0;
+        iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
+        iKeyOff += iVal;
+        if( iKeyOff==iNextOff ){
+          bLastInDoclist = 1;
+        }
      }
    }
+
+    /* If this is (a) the first rowid on a page and (b) is not followed by
+    ** another position list on the same page, set the "first-rowid" field
+    ** of the header to 0.  */
+    if( fts5GetU16(&aPg[0])==iStart && (bLastInDoclist || iNextOff==iPgIdx) ){
+      fts5PutU16(&aPg[0], 0);
+    }
  }

-  if( fts5GetU16(&aPg[0])==iStart && (bLastInDoclist||iNextOff==iPgIdx) ){
-    fts5PutU16(&aPg[0], 0);
-  }
-
-  if( bLastInDoclist==0 ){
+  if( pSeg->bDel ){
+    iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta);
+    aPg[iOff++] = 0x01;
+  }else if( bLastInDoclist==0 ){
    if( iNextOff!=iPgIdx ){
+      u64 iNextDelta = 0;
      iNextOff += fts5GetVarint(&aPg[iNextOff], &iNextDelta);
      iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta + iNextDelta);
    }
  }else if( 
-      iStart==pSeg->iTermLeafOffset && pSeg->iLeafPgno==pSeg->iTermLeafPgno 
+      pSeg->iLeafPgno==pSeg->iTermLeafPgno 
+   && iStart==pSeg->iTermLeafOffset 
  ){
    /* The entry being removed was the only position list in its
    ** doclist. Therefore the term needs to be removed as well. */
    int iKey = 0;
-    for(iIdx=0, iKeyOff=0; iIdx<nIdx; iKey++){
+    int iKeyOff = 0;
+
+    /* Set iKeyOff to the offset of the term that will be removed - the
+    ** last offset in the footer that is not greater than iStart. */
+    for(iIdx=0; iIdx<nIdx; iKey++){
      u32 iVal = 0;
      iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
      if( (iKeyOff+iVal)>(u32)iStart ) break;
      iKeyOff += iVal;
    }
+    assert_nc( iKey>=1 );

+    /* Set iDelKeyOff to the value of the footer entry to remove from 
+    ** the page. */
    iDelKeyOff = iOff = iKeyOff;
+
    if( iNextOff!=iPgIdx ){
+      /* This is the only position-list associated with the term, and there
+      ** is another term following it on this page. So the subsequent term
+      ** needs to be moved to replace the term associated with the entry
+      ** being removed. */
      int nPrefix = 0;
      int nSuffix = 0;
      int nPrefix2 = 0;
@@ -5198,80 +5241,88 @@ static void fts5DoSecureDelete(
      }
    }
  }else if( iStart==4 ){
-      int iPgno;
+    int iPgno;

-      assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno );
-      /* The entry being removed may be the only position list in
-      ** its doclist. */
-      for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){
-        Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno));
-        int bEmpty = (pPg && pPg->nn==4);
-        fts5DataRelease(pPg);
-        if( bEmpty==0 ) break;
-      }
+    assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno );
+    /* The entry being removed may be the only position list in
+    ** its doclist. */
+    for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){
+      Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno));
+      int bEmpty = (pPg && pPg->nn==4);
+      fts5DataRelease(pPg);
+      if( bEmpty==0 ) break;
+    }

-      if( iPgno==pSeg->iTermLeafPgno ){
-        i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno);
-        Fts5Data *pTerm = fts5DataRead(p, iId);
-        if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){
-          u8 *aTermIdx = &pTerm->p[pTerm->szLeaf];
-          int nTermIdx = pTerm->nn - pTerm->szLeaf;
-          int iTermIdx = 0;
-          int iTermOff = 0;
+    if( iPgno==pSeg->iTermLeafPgno ){
+      i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno);
+      Fts5Data *pTerm = fts5DataRead(p, iId);
+      if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){
+        u8 *aTermIdx = &pTerm->p[pTerm->szLeaf];
+        int nTermIdx = pTerm->nn - pTerm->szLeaf;
+        int iTermIdx = 0;
+        int iTermOff = 0;

-          while( 1 ){
-            u32 iVal = 0;
-            int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal);
-            iTermOff += iVal;
-            if( (iTermIdx+nByte)>=nTermIdx ) break;
-            iTermIdx += nByte;
-          }
-          nTermIdx = iTermIdx;
-
-          memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx);
-          fts5PutU16(&pTerm->p[2], iTermOff);
-
-          fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx);
-          if( nTermIdx==0 ){
-            fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno);
-          }
+        while( 1 ){
+          u32 iVal = 0;
+          int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal);
+          iTermOff += iVal;
+          if( (iTermIdx+nByte)>=nTermIdx ) break;
+          iTermIdx += nByte;
        }
-        fts5DataRelease(pTerm);
+        nTermIdx = iTermIdx;
+
+        memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx);
+        fts5PutU16(&pTerm->p[2], iTermOff);
+
+        fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx);
+        if( nTermIdx==0 ){
+          fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno);
+        }
+      }
+      fts5DataRelease(pTerm);
+    }
+  }
+
+  /* Assuming no error has occurred, this block does final edits to the
+  ** leaf page before writing it back to disk. Input variables are:
+  **
+  **   nPg: Total initial size of leaf page.
+  **   iPgIdx: Initial offset of page footer.
+  **
+  **   iOff: Offset to move data to
+  **   iNextOff: Offset to move data from
+  */
+  if( p->rc==SQLITE_OK ){
+    const int nMove = nPg - iNextOff;     /* Number of bytes to move */
+    int nShift = iNextOff - iOff;         /* Distance to move them */
+
+    int iPrevKeyOut = 0;
+    int iKeyIn = 0;
+
+    memmove(&aPg[iOff], &aPg[iNextOff], nMove);
+    iPgIdx -= nShift;
+    nPg = iPgIdx;
+    fts5PutU16(&aPg[2], iPgIdx);
+
+    for(iIdx=0; iIdx<nIdx; /* no-op */){
+      u32 iVal = 0;
+      iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
+      iKeyIn += iVal;
+      if( iKeyIn!=iDelKeyOff ){
+        int iKeyOut = (iKeyIn - (iKeyIn>iOff ? nShift : 0));
+        nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOut - iPrevKeyOut);
+        iPrevKeyOut = iKeyOut;
      }
    }

-    if( p->rc==SQLITE_OK ){
-      const int nMove = nPg - iNextOff;
-      int nShift = 0;
-
-      memmove(&aPg[iOff], &aPg[iNextOff], nMove);
-      iPgIdx -= (iNextOff - iOff);
-      nPg = iPgIdx;
-      fts5PutU16(&aPg[2], iPgIdx);
-
-      nShift = iNextOff - iOff;
-      for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdx<nIdx; /* no-op */){
-        u32 iVal = 0;
-        iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
-        iKeyOff += iVal;
-        if( iKeyOff!=iDelKeyOff ){
-          if( iKeyOff>iOff ){
-            iKeyOff -= nShift;
-            nShift = 0;
-          }
-          nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff);
-          iPrevKeyOff = iKeyOff;
-        }
-      }
-
-      if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){
-        fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno);
-      }
-
-      assert_nc( nPg>4 || fts5GetU16(aPg)==0 );
-      fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg,nPg);
+    if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){
+      fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno);
    }
-    sqlite3_free(aIdx);
+
+    assert_nc( nPg>4 || fts5GetU16(aPg)==0 );
+    fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg, nPg);
+  }
+  sqlite3_free(aIdx);
 }

 /*
@@ -5421,7 +5472,8 @@ static void fts5FlushOneHash(Fts5Index *p){
              writer.bFirstRowidInPage = 0;
              fts5WriteDlidxAppend(p, &writer, iRowid);
            }else{
-              pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid-iPrev);
+              u64 iRowidDelta = (u64)iRowid - (u64)iPrev;
+              pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowidDelta);
            }
            if( p->rc!=SQLITE_OK ) break;
            assert( pBuf->n<=pBuf->nSpace );
@@ -5440,10 +5492,16 @@ static void fts5FlushOneHash(Fts5Index *p){
                fts5WriteFlushLeaf(p, &writer);
              }
            }else{
-              int bDummy;
-              int nPos;
-              int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy);
-              nCopy += nPos;
+              int bDel = 0;
+              int nPos = 0;
+              int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDel);
+              if( bDel && bSecureDelete ){
+                fts5BufferAppendVarint(&p->rc, pBuf, nPos*2);
+                iOff += nCopy;
+                nCopy = nPos;
+              }else{
+                nCopy += nPos;
+              }
              if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){
                /* The entire poslist will fit on the current leaf. So copy
                ** it in one go. */
@@ -5481,7 +5539,6 @@ static void fts5FlushOneHash(Fts5Index *p){
        assert( pBuf->n<=pBuf->nSpace );
        if( p->rc==SQLITE_OK ) sqlite3Fts5HashScanNext(pHash);
      }
-      sqlite3Fts5HashClear(pHash);
      fts5WriteFinish(p, &writer, &pgnoLast);
  
      assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 );
@@ -5514,7 +5571,6 @@ static void fts5FlushOneHash(Fts5Index *p){
  fts5IndexCrisismerge(p, &pStruct);
  fts5StructureWrite(p, pStruct);
  fts5StructureRelease(pStruct);
-  p->nContentlessDelete = 0;
 }

 /*
@@ -5525,8 +5581,12 @@ static void fts5IndexFlush(Fts5Index *p){
  if( p->nPendingData || p->nContentlessDelete ){
    assert( p->pHash );
    fts5FlushOneHash(p);
-    p->nPendingData = 0;
-    p->nPendingRow = 0;
+    if( p->rc==SQLITE_OK ){
+      sqlite3Fts5HashClear(p->pHash);
+      p->nPendingData = 0;
+      p->nPendingRow = 0;
+      p->nContentlessDelete = 0;
+    }
  }
 }

@@ -5604,8 +5664,9 @@ int sqlite3Fts5IndexOptimize(Fts5Index *p){

  assert( p->rc==SQLITE_OK );
  fts5IndexFlush(p);
-  assert( p->nContentlessDelete==0 );
+  assert( p->rc!=SQLITE_OK || p->nContentlessDelete==0 );
  pStruct = fts5StructureRead(p);
+  assert( p->rc!=SQLITE_OK || pStruct!=0 );
  fts5StructureInvalidate(p);

  if( pStruct ){
@@ -6152,7 +6213,7 @@ int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){
  /* Flush the hash table to disk if required */
  if( iRowid<p->iWriteRowid 
   || (iRowid==p->iWriteRowid && p->bDelete==0)
-   || (p->nPendingData > p->pConfig->nHashSize) 
+   || (p->nPendingData > p->pConfig->nHashSize)
  ){
    fts5IndexFlush(p);
  }
@@ -7815,7 +7876,7 @@ static void fts5DecodeFunction(
    fts5DecodeRowidList(&rc, &s, &a[4], iTermOff-4);

    iOff = iTermOff;
-    while( iOff<szLeaf ){
+    while( iOff<szLeaf && rc==SQLITE_OK ){
      int nAppend;

      /* Read the term data for the next term*/
@@ -7835,8 +7896,11 @@ static void fts5DecodeFunction(
      }else{
        iTermOff = szLeaf;
      }
-
-      fts5DecodeRowidList(&rc, &s, &a[iOff], iTermOff-iOff);
+      if( iTermOff>szLeaf ){
+        rc = FTS5_CORRUPT;
+      }else{
+        fts5DecodeRowidList(&rc, &s, &a[iOff], iTermOff-iOff);
+      }
      iOff = iTermOff;
      if( iOff<szLeaf ){
        iOff += fts5GetVarint32(&a[iOff], nKeep);
@@ -8265,7 +8329,8 @@ int sqlite3Fts5IndexInit(sqlite3 *db){
      0,                           /* xSavepoint    */
      0,                           /* xRelease      */
      0,                           /* xRollbackTo   */
-      0                            /* xShadowName   */
+      0,                           /* xShadowName   */
+      0                            /* xIntegrity    */
    };
    rc = sqlite3_create_module(db, "fts5_structure", &fts5structure_module, 0);
  }
@@ -117,6 +117,8 @@ struct Fts5FullTable {
  Fts5Storage *pStorage;          /* Document store */
  Fts5Global *pGlobal;            /* Global (connection wide) data */
  Fts5Cursor *pSortCsr;           /* Sort data from this cursor */
+  int iSavepoint;                 /* Successful xSavepoint()+1 */
+  int bInSavepoint;
 #ifdef SQLITE_DEBUG
  struct Fts5TransactionState ts;
 #endif
@@ -405,6 +407,13 @@ static int fts5InitVtab(
    pConfig->pzErrmsg = 0;
  }

+  if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
+    rc = sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, (int)1);
+  }
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS);
+  }
+
  if( rc!=SQLITE_OK ){
    fts5FreeVtab(pTab);
    pTab = 0;
@@ -1329,6 +1338,9 @@ static int fts5FilterMethod(
    pCsr->iFirstRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64);
  }

+  rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
+  if( rc!=SQLITE_OK ) goto filter_out;
+
  if( pTab->pSortCsr ){
    /* If pSortCsr is non-NULL, then this call is being made as part of 
    ** processing for a "... MATCH <expr> ORDER BY rank" query (ePlan is
@@ -1351,6 +1363,7 @@ static int fts5FilterMethod(
    pCsr->pExpr = pTab->pSortCsr->pExpr;
    rc = fts5CursorFirst(pTab, pCsr, bDesc);
  }else if( pCsr->pExpr ){
+    assert( rc==SQLITE_OK );
    rc = fts5CursorParseRank(pConfig, pCsr, pRank);
    if( rc==SQLITE_OK ){
      if( bOrderByRank ){
@@ -1522,6 +1535,7 @@ static int fts5SpecialInsert(
  Fts5Config *pConfig = pTab->p.pConfig;
  int rc = SQLITE_OK;
  int bError = 0;
+  int bLoadConfig = 0;

  if( 0==sqlite3_stricmp("delete-all", zCmd) ){
    if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
@@ -1533,6 +1547,7 @@ static int fts5SpecialInsert(
    }else{
      rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage);
    }
+    bLoadConfig = 1;
  }else if( 0==sqlite3_stricmp("rebuild", zCmd) ){
    if( pConfig->eContent==FTS5_CONTENT_NONE ){
      fts5SetVtabError(pTab, 
@@ -1542,6 +1557,7 @@ static int fts5SpecialInsert(
    }else{
      rc = sqlite3Fts5StorageRebuild(pTab->pStorage);
    }
+    bLoadConfig = 1;
  }else if( 0==sqlite3_stricmp("optimize", zCmd) ){
    rc = sqlite3Fts5StorageOptimize(pTab->pStorage);
  }else if( 0==sqlite3_stricmp("merge", zCmd) ){
@@ -1554,6 +1570,8 @@ static int fts5SpecialInsert(
  }else if( 0==sqlite3_stricmp("prefix-index", zCmd) ){
    pConfig->bPrefixIndex = sqlite3_value_int(pVal);
 #endif
+  }else if( 0==sqlite3_stricmp("flush", zCmd) ){
+    rc = sqlite3Fts5FlushToDisk(&pTab->p);
  }else{
    rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
    if( rc==SQLITE_OK ){
@@ -1567,6 +1585,12 @@ static int fts5SpecialInsert(
      }
    }
  }
+
+  if( rc==SQLITE_OK && bLoadConfig ){
+    pTab->p.pConfig->iCookie--;
+    rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
+  }
+
  return rc;
 }

@@ -1685,7 +1709,7 @@ static int fts5UpdateMethod(
    assert( nArg!=1 || eType0==SQLITE_INTEGER );

    /* Filter out attempts to run UPDATE or DELETE on contentless tables.
-    ** This is not suported. Except - DELETE is supported if the CREATE
+    ** This is not suported. Except - they are both supported if the CREATE
    ** VIRTUAL TABLE statement contained "contentless_delete=1". */
    if( eType0==SQLITE_INTEGER 
     && pConfig->eContent==FTS5_CONTENT_NONE 
@@ -1714,7 +1738,8 @@ static int fts5UpdateMethod(
      }

      else if( eType0!=SQLITE_INTEGER ){     
-        /* If this is a REPLACE, first remove the current entry (if any) */
+        /* An INSERT statement. If the conflict-mode is REPLACE, first remove
+        ** the current entry (if any). */
        if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){
          i64 iNew = sqlite3_value_int64(apVal[1]);  /* Rowid to delete */
          rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0);
@@ -2588,8 +2613,12 @@ static int fts5RenameMethod(
  sqlite3_vtab *pVtab,            /* Virtual table handle */
  const char *zName               /* New name of table */
 ){
+  int rc;
  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
-  return sqlite3Fts5StorageRename(pTab->pStorage, zName);
+  pTab->bInSavepoint = 1;
+  rc = sqlite3Fts5StorageRename(pTab->pStorage, zName);
+  pTab->bInSavepoint = 0;
+  return rc;
 }

 int sqlite3Fts5FlushToDisk(Fts5Table *pTab){
@@ -2603,9 +2632,29 @@ int sqlite3Fts5FlushToDisk(Fts5Table *pTab){
 ** Flush the contents of the pending-terms table to disk.
 */
 static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
-  UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
-  fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_SAVEPOINT, iSavepoint);
-  return sqlite3Fts5FlushToDisk((Fts5Table*)pVtab);
+  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
+  int rc = SQLITE_OK;
+  char *zSql = 0;
+  fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint);
+
+  if( pTab->bInSavepoint==0 ){
+    zSql = sqlite3_mprintf("INSERT INTO %Q.%Q(%Q) VALUES('flush')",
+        pTab->p.pConfig->zDb, pTab->p.pConfig->zName, pTab->p.pConfig->zName
+    );
+    if( zSql ){
+      pTab->bInSavepoint = 1;
+      rc = sqlite3_exec(pTab->p.pConfig->db, zSql, 0, 0, 0);
+      pTab->bInSavepoint = 0;
+      sqlite3_free(zSql);
+    }else{
+      rc = SQLITE_NOMEM;
+    }
+    if( rc==SQLITE_OK ){
+      pTab->iSavepoint = iSavepoint+1;
+    }
+  }
+
+  return rc;
 }

 /*
@@ -2614,9 +2663,16 @@ static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
 ** This is a no-op.
 */
 static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
-  UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
-  fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_RELEASE, iSavepoint);
-  return sqlite3Fts5FlushToDisk((Fts5Table*)pVtab);
+  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
+  int rc = SQLITE_OK;
+  fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint);
+  if( (iSavepoint+1)<pTab->iSavepoint ){
+    rc = sqlite3Fts5FlushToDisk(&pTab->p);
+    if( rc==SQLITE_OK ){
+      pTab->iSavepoint = iSavepoint;
+    }
+  }
+  return rc;
 }

 /*
@@ -2626,11 +2682,14 @@ static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
 */
 static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
-  UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
+  int rc = SQLITE_OK;
  fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint);
  fts5TripCursors(pTab);
  pTab->p.pConfig->pgsz = 0;
-  return sqlite3Fts5StorageRollback(pTab->pStorage);
+  if( (iSavepoint+1)<=pTab->iSavepoint ){
+    rc = sqlite3Fts5StorageRollback(pTab->pStorage);
+  }
+  return rc;
 }

 /*
@@ -2850,9 +2909,46 @@ static int fts5ShadowName(const char *zName){
  return 0;
 }

+/*
+** Run an integrity check on the FTS5 data structures.  Return a string
+** if anything is found amiss.  Return a NULL pointer if everything is
+** OK.
+*/
+static int fts5Integrity(
+  sqlite3_vtab *pVtab,    /* the FTS5 virtual table to check */
+  const char *zSchema,    /* Name of schema in which this table lives */
+  const char *zTabname,   /* Name of the table itself */
+  int isQuick,            /* True if this is a quick-check */
+  char **pzErr            /* Write error message here */
+){
+  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
+  Fts5Config *pConfig = pTab->p.pConfig;
+  char *zSql;
+  char *zErr = 0;
+  int rc;
+  assert( pzErr!=0 && *pzErr==0 );
+  UNUSED_PARAM(isQuick);
+  zSql = sqlite3_mprintf(
+            "INSERT INTO \"%w\".\"%w\"(\"%w\") VALUES('integrity-check');",
+            zSchema, zTabname, pConfig->zName);
+  if( zSql==0 ) return SQLITE_NOMEM;
+  rc = sqlite3_exec(pConfig->db, zSql, 0, 0, &zErr);
+  sqlite3_free(zSql);
+  if( (rc&0xff)==SQLITE_CORRUPT ){
+    *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s",
+                zSchema, zTabname);
+  }else if( rc!=SQLITE_OK ){
+    *pzErr = sqlite3_mprintf("unable to validate the inverted index for"
+                             " FTS5 table %s.%s: %s",
+                zSchema, zTabname, zErr);
+  }
+  sqlite3_free(zErr);
+  return SQLITE_OK;
+}
+
 static int fts5Init(sqlite3 *db){
  static const sqlite3_module fts5Mod = {
-    /* iVersion      */ 3,
+    /* iVersion      */ 4,
    /* xCreate       */ fts5CreateMethod,
    /* xConnect      */ fts5ConnectMethod,
    /* xBestIndex    */ fts5BestIndexMethod,
@@ -2875,7 +2971,8 @@ static int fts5Init(sqlite3 *db){
    /* xSavepoint    */ fts5SavepointMethod,
    /* xRelease      */ fts5ReleaseMethod,
    /* xRollbackTo   */ fts5RollbackToMethod,
-    /* xShadowName   */ fts5ShadowName
+    /* xShadowName   */ fts5ShadowName,
+    /* xIntegrity    */ fts5Integrity
  };

  int rc;
@@ -1184,7 +1184,9 @@ int sqlite3Fts5StorageSync(Fts5Storage *p){
  i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db);
  if( p->bTotalsValid ){
    rc = fts5StorageSaveTotals(p);
-    p->bTotalsValid = 0;
+    if( rc==SQLITE_OK ){
+      p->bTotalsValid = 0;
+    }
  }
  if( rc==SQLITE_OK ){
    rc = sqlite3Fts5IndexSync(p->pIndex);
@@ -472,7 +472,8 @@ int sqlite3Fts5TestRegisterTok(sqlite3 *db, fts5_api *pApi){
     0,                           /* xSavepoint    */
     0,                           /* xRelease      */
     0,                           /* xRollbackTo   */
-     0                            /* xShadowName   */
+     0,                           /* xShadowName   */
+     0                            /* xIntegrity    */
  };
  int rc;                         /* Return code */

@@ -229,6 +229,12 @@ static const unsigned char sqlite3Utf8Trans1[] = {

 #endif /* ifndef SQLITE_AMALGAMATION */

+#define FTS5_SKIP_UTF8(zIn) {                               \
+  if( ((unsigned char)(*(zIn++)))>=0xc0 ){                              \
+    while( (((unsigned char)*zIn) & 0xc0)==0x80 ){ zIn++; }             \
+  }                                                    \
+}
+
 typedef struct Unicode61Tokenizer Unicode61Tokenizer;
 struct Unicode61Tokenizer {
  unsigned char aTokenChar[128];  /* ASCII range token characters */
@@ -1264,6 +1270,7 @@ static int fts5PorterTokenize(
 typedef struct TrigramTokenizer TrigramTokenizer;
 struct TrigramTokenizer {
  int bFold;                      /* True to fold to lower-case */
+  int iFoldParam;                 /* Parameter to pass to Fts5UnicodeFold() */
 };

 /*
@@ -1290,6 +1297,7 @@ static int fts5TriCreate(
  }else{
    int i;
    pNew->bFold = 1;
+    pNew->iFoldParam = 0;
    for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
      const char *zArg = azArg[i+1];
      if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
@@ -1298,10 +1306,21 @@ static int fts5TriCreate(
        }else{
          pNew->bFold = (zArg[0]=='0');
        }
+      }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
+        if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
+          rc = SQLITE_ERROR;
+        }else{
+          pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
+        }
      }else{
        rc = SQLITE_ERROR;
      }
    }
+
+    if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
+      rc = SQLITE_ERROR;
+    }
+
    if( rc!=SQLITE_OK ){
      fts5TriDelete((Fts5Tokenizer*)pNew);
      pNew = 0;
@@ -1324,40 +1343,62 @@ static int fts5TriTokenize(
  TrigramTokenizer *p = (TrigramTokenizer*)pTok;
  int rc = SQLITE_OK;
  char aBuf[32];
+  char *zOut = aBuf;
+  int ii;
  const unsigned char *zIn = (const unsigned char*)pText;
  const unsigned char *zEof = &zIn[nText];
  u32 iCode;
+  int aStart[3];                  /* Input offset of each character in aBuf[] */

  UNUSED_PARAM(unusedFlags);
-  while( 1 ){
-    char *zOut = aBuf;
-    int iStart = zIn - (const unsigned char*)pText;
-    const unsigned char *zNext; 

-    READ_UTF8(zIn, zEof, iCode);
-    if( iCode==0 ) break;
-    zNext = zIn;
-    if( zIn<zEof ){
-      if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
-      WRITE_UTF8(zOut, iCode);
+  /* Populate aBuf[] with the characters for the first trigram. */
+  for(ii=0; ii<3; ii++){
+    do {
+      aStart[ii] = zIn - (const unsigned char*)pText;
+      READ_UTF8(zIn, zEof, iCode);
+      if( iCode==0 ) return SQLITE_OK;
+      if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam);
+    }while( iCode==0 );
+    WRITE_UTF8(zOut, iCode);
+  }
+
+  /* At the start of each iteration of this loop:
+  **
+  **  aBuf:      Contains 3 characters. The 3 characters of the next trigram.
+  **  zOut:      Points to the byte following the last character in aBuf.
+  **  aStart[3]: Contains the byte offset in the input text corresponding
+  **             to the start of each of the three characters in the buffer.
+  */
+  assert( zIn<=zEof );
+  while( 1 ){
+    int iNext;                    /* Start of character following current tri */
+    const char *z1;
+
+    /* Read characters from the input up until the first non-diacritic */
+    do {
+      iNext = zIn - (const unsigned char*)pText;
      READ_UTF8(zIn, zEof, iCode);
      if( iCode==0 ) break;
-    }else{
-      break;
-    }
-    if( zIn<zEof ){
-      if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
-      WRITE_UTF8(zOut, iCode);
-      READ_UTF8(zIn, zEof, iCode);
-      if( iCode==0 ) break;
-      if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
-      WRITE_UTF8(zOut, iCode);
-    }else{
-      break;
-    }
-    rc = xToken(pCtx, 0, aBuf, zOut-aBuf, iStart, iStart + zOut-aBuf);
-    if( rc!=SQLITE_OK ) break;
-    zIn = zNext;
+      if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam);
+    }while( iCode==0 );
+
+    /* Pass the current trigram back to fts5 */
+    rc = xToken(pCtx, 0, aBuf, zOut-aBuf, aStart[0], iNext);
+    if( iCode==0 || rc!=SQLITE_OK ) break;
+
+    /* Remove the first character from buffer aBuf[]. Append the character
+    ** with codepoint iCode.  */
+    z1 = aBuf;
+    FTS5_SKIP_UTF8(z1);
+    memmove(aBuf, z1, zOut - z1);
+    zOut -= (z1 - aBuf);
+    WRITE_UTF8(zOut, iCode);
+
+    /* Update the aStart[] array */
+    aStart[0] = aStart[1];
+    aStart[1] = aStart[2];
+    aStart[2] = iNext;
  }

  return rc;
@@ -1380,7 +1421,9 @@ int sqlite3Fts5TokenizerPattern(
 ){
  if( xCreate==fts5TriCreate ){
    TrigramTokenizer *p = (TrigramTokenizer*)pTok;
-    return p->bFold ? FTS5_PATTERN_LIKE : FTS5_PATTERN_GLOB;
+    if( p->iFoldParam==0 ){
+      return p->bFold ? FTS5_PATTERN_LIKE : FTS5_PATTERN_GLOB;
+    }
  }
  return FTS5_PATTERN_NONE;
 }
@@ -783,7 +783,8 @@ int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){
    /* xSavepoint    */ 0,
    /* xRelease      */ 0,
    /* xRollbackTo   */ 0,
-    /* xShadowName   */ 0
+    /* xShadowName   */ 0,
+    /* xIntegrity    */ 0
  };
  void *p = (void*)pGlobal;

@@ -65,7 +65,9 @@ foreach w {a b c d e f} {

 do_execsql_test 2.4 {
  INSERT INTO t1(t1) VALUES('integrity-check');
-}
+  PRAGMA integrity_check;
+  PRAGMA integrity_check(t1);
+} {ok ok}


 #-------------------------------------------------------------------------
@@ -88,6 +90,7 @@ foreach {i x y} {
 } {
  do_execsql_test 3.$i.1 { INSERT INTO t1 VALUES($x, $y) }
  do_execsql_test 3.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
+  do_execsql_test 3.$i.3 { PRAGMA integrity_check(t1) } ok
  if {[set_test_counter errors]} break
 }

@@ -135,7 +138,7 @@ foreach {i x y} {
   10 {ddd abcde dddd dd c} {dddd c c d abcde}
 } {
  do_execsql_test 5.$i.1 { INSERT INTO t1 VALUES($x, $y) }
-  do_execsql_test 5.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
+  do_execsql_test 5.$i.2 { PRAGMA integrity_check(t1) } ok
  if {[set_test_counter errors]} break
 }

@@ -307,5 +307,31 @@ do_catchsql_test 10.1.4 {
  SELECT group_concat(firstcol(t1), '.') FROM t1 GROUP BY rowid
 } {1 {unable to use function firstcol in the requested context}}

-finish_test
+#-------------------------------------------------------------------------
+# Test that xInstCount() works from within an xPhraseQuery() callback.
+#
+reset_db

+proc xCallback {cmd} {
+  incr ::hitcount [$cmd xInstCount]
+  return SQLITE_OK
+}
+proc fts5_hitcount {cmd} {
+  set ::hitcount 0
+  $cmd xQueryPhrase 0 xCallback
+  return $::hitcount
+}
+sqlite3_fts5_create_function db fts5_hitcount fts5_hitcount
+
+do_execsql_test 11.1 {
+  CREATE VIRTUAL TABLE x1 USING fts5(z);
+  INSERT INTO x1 VALUES('one two three');
+  INSERT INTO x1 VALUES('one two one three one');
+  INSERT INTO x1 VALUES('one two three');
+}
+
+do_execsql_test 11.2 {
+  SELECT fts5_hitcount(x1) FROM x1('one') LIMIT 1;
+} {5}
+
+finish_test
@@ -65,4 +65,44 @@ do_execsql_test 2.1 {
  INSERT INTO fts_idx(fts_idx) VALUES('integrity-check');
 }

+#-------------------------------------------------------------------------
+# Tests for OR IGNORE conflict handling.
+#
+reset_db
+foreach_detail_mode $::testprefix {
+
+  do_execsql_test 3.0 {
+    CREATE VIRTUAL TABLE t1 USING fts5(xyz, detail=%DETAIL%);
+
+    BEGIN;
+    INSERT INTO t1(rowid, xyz) VALUES(13, 'thirteen documents');
+    INSERT INTO t1(rowid, xyz) VALUES(14, 'fourteen documents');
+    INSERT INTO t1(rowid, xyz) VALUES(15, 'fifteen documents');
+    COMMIT;
+  }
+
+  set db_cksum [cksum]
+    foreach {tn sql} {
+    1 {
+      INSERT OR IGNORE INTO t1(rowid, xyz) VALUES(14, 'new text');
+    }
+    2 {
+      UPDATE OR IGNORE t1 SET rowid=13 WHERE rowid=15;
+    }
+    3 {
+      INSERT OR IGNORE INTO t1(rowid, xyz) 
+      SELECT 13, 'some text'
+      UNION ALL
+      SELECT 14, 'some text'
+      UNION ALL
+      SELECT 15, 'some text'
+    }
+  } {
+    do_execsql_test 3.1.$tn.1 $sql
+    do_test 3.1.$tn.2 { cksum } $db_cksum
+  }
+
+}
+
+
 finish_test
@@ -294,4 +294,3 @@ do_catchsql_test 7.2.5 {
 } {1 {recursively defined fts5 content table}}

 finish_test
-
@@ -268,4 +268,3 @@ do_execsql_test 8.2 {
 } {}

 finish_test
-
@@ -205,4 +205,3 @@ foreach {tn step} {


 finish_test
-
@@ -193,4 +193,3 @@ do_execsql_test 3.7 {


 finish_test
-
@@ -245,4 +245,3 @@ do_execsql_test 4.3 {
 }

 finish_test
-
@@ -56,4 +56,3 @@ foreach {tn up err} {
 }

 finish_test
-
@@ -48,6 +48,10 @@ do_test 1.3 {
  }
  catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
 } {1 {database disk image is malformed}}
+do_execsql_test 1.3b {
+  PRAGMA integrity_check(t1);
+} {{malformed inverted index for FTS5 table main.t1}}
+

 do_test 1.4 {
  db_restore_and_reopen
@@ -167,6 +167,9 @@ foreach {tn hdr} {
      do_test 3.$tn.$tn2.2 {
        catchsql { INSERT INTO x3(x3) VALUES('integrity-check') }
      } {1 {database disk image is malformed}}
+      do_execsql_test 3.$tn.$tn2.3 {
+        PRAGMA integrity_check(x3);
+      } {{malformed inverted index for FTS5 table main.x3}}
    }

    execsql ROLLBACK
@@ -95,6 +95,9 @@ do_execsql_test 3.3 {
  SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank;
 } {1 -1e-06}

+do_execsql_test 3.4 "
+  SELECT fts5_expr_tcl('e AND \" \"');
+" {{AND [nearset -- {e}] [{}]}}


 finish_test
@@ -0,0 +1,76 @@
+# 2010 June 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5faultG
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+set ::testprefix fts5faultG
+
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a);
+  INSERT INTO t1 VALUES('test renaming the table');
+  INSERT INTO t1 VALUES(' after it has been written');
+  INSERT INTO t1 VALUES(' actually other stuff instead');
+}
+faultsim_save_and_close
+do_faultsim_test 1 -faults oom* -prep { 
+  faultsim_restore_and_reopen
+  execsql {
+    BEGIN;
+      DELETE FROM t1 WHERE rowid=2;
+  }
+} -body {
+  execsql {
+    DELETE FROM t1;
+  }
+} -test {
+  catchsql { COMMIT }
+  faultsim_integrity_check
+  faultsim_test_result {0 {}}
+}
+
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, content=, contentless_delete=1);
+  BEGIN;
+    INSERT INTO t1 VALUES('here''s some text');
+    INSERT INTO t1 VALUES('useful stuff, text');
+    INSERT INTO t1 VALUES('what would we do without text!'); 
+  COMMIT;
+}
+faultsim_save_and_close
+do_faultsim_test 2 -faults oom* -prep { 
+  faultsim_restore_and_reopen
+  execsql {
+    BEGIN;
+      DELETE FROM t1 WHERE rowid=2;
+  }
+} -body {
+  execsql {
+    INSERT INTO t1(t1) VALUES('optimize');
+  }
+} -test {
+  faultsim_integrity_check
+  faultsim_test_result {0 {}}
+}
+
+
+
+finish_test
@@ -77,6 +77,9 @@ do_catchsql_test 4.2 {
    UPDATE aa_docsize SET sz = X'44' WHERE rowid = 3;
    INSERT INTO aa(aa) VALUES('integrity-check'); 
 } {1 {database disk image is malformed}}
+do_execsql_test 4.2.1 {
+  PRAGMA integrity_check(aa);
+} {{malformed inverted index for FTS5 table main.aa}}

 do_catchsql_test 4.3 { 
  ROLLBACK;
@@ -317,4 +320,39 @@ do_catchsql_test 10.5.3 {
  INSERT INTO vt0(vt0) VALUES('integrity-check');
 } {0 {}}

+reset_db
+proc slang {in} {return [string map {th d e eh} $in]}
+db function slang -deterministic -innocuous slang
+do_execsql_test 11.0 {
+  CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c TEXT AS (slang(b)));
+  INSERT INTO t1(b) VALUES('the quick fox jumps over the lazy brown dog');
+  SELECT c FROM t1;
+} {{deh quick fox jumps ovehr deh lazy brown dog}}
+
+do_execsql_test 11.1 {
+  CREATE VIRTUAL TABLE t2 USING fts5(content="t1", c);
+  INSERT INTO t2(t2) VALUES('rebuild');
+  SELECT rowid FROM t2 WHERE t2 MATCH 'deh';
+} {1}
+
+do_execsql_test 11.2 {
+  PRAGMA integrity_check(t2);
+} {ok}
+db close
+sqlite3 db test.db
+
+# FIX ME?
+#
+# FTS5 integrity-check does not care if the content table is unreadable or
+# does not exist.  It only looks for internal inconsistencies in the
+# inverted index.
+#
+do_execsql_test 11.3 {
+  PRAGMA integrity_check(t2);
+} {ok}
+do_execsql_test 11.4 {
+  DROP TABLE t1;
+  PRAGMA integrity_check(t2);
+} {ok}
+
 finish_test
@@ -44,12 +44,12 @@ do_catchsql_test 1.2.2 {

 do_catchsql_test 1.3.1 { 
  SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads'); 
-} {1 {no such cursor: 1}}
+} {1 {no such cursor: 2}}

 do_catchsql_test 1.3.2 {
  SELECT a FROM t1
    WHERE rank = (SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads'));
-} {1 {no such cursor: 1}}
+} {1 {no such cursor: 2}}

 db close
 sqlite3 db test.db
@@ -424,10 +424,12 @@ do_execsql_test -db db2 15.3 {
  SAVEPOINT one;
 } {}
 do_execsql_test 15.4 END
-do_test 15.4 {
+do_test 15.5 {
  list [catch { db2 eval COMMIT } msg] $msg
 } {0 {}}

+db2 close
+
 #-------------------------------------------------------------------------
 reset_db
 forcedelete test.db2
@@ -469,6 +471,8 @@ do_execsql_test -db db2 16.6 {
  SELECT * FROM x1
 } {abc def}

+db2 close
+
 #-------------------------------------------------------------------------
 reset_db
 do_execsql_test 17.1 {
@@ -9,7 +9,7 @@
 #
 #***********************************************************************
 #
-# TESTRUNNER: slow
+# TESTRUNNER: superslow
 #

 source [file join [file dirname [info script]] fts5_common.tcl]
@@ -42,23 +42,4 @@ do_execsql_test 1.2 {
  SELECT count(*) FROM t1('mno')
 } $nLoop

-do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE t2 USING fts5(x);
-  INSERT INTO t2(t2, rank) VALUES('pgsz', 32);
-}
-
-do_test 2.1 {
-  for {set ii 0} {$ii < $nLoop} {incr ii} {
-    execsql {
-      INSERT INTO t2 VALUES('abc def ghi');
-      INSERT INTO t2 VALUES('jkl mno pqr');
-      INSERT INTO t2(t2, rank) VALUES('merge', -1);
-    }
-  }
-} {}
-
-do_execsql_test 2.2 {
-  SELECT count(*) FROM t2('mno')
-} $nLoop
-
 finish_test
@@ -0,0 +1,45 @@
+# 2023 Aug 27
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# TESTRUNNER: superslow
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5optimize2
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+set nLoop 2500
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(x);
+  INSERT INTO t2(t2, rank) VALUES('pgsz', 32);
+}
+
+do_test 1.1 {
+  for {set ii 0} {$ii < $nLoop} {incr ii} {
+    execsql {
+      INSERT INTO t2 VALUES('abc def ghi');
+      INSERT INTO t2 VALUES('jkl mno pqr');
+      INSERT INTO t2(t2, rank) VALUES('merge', -1);
+    }
+  }
+} {}
+
+do_execsql_test 1.2 {
+  SELECT count(*) FROM t2('mno')
+} $nLoop
+
+finish_test
@@ -52,6 +52,36 @@ do_execsql_test 2.1 {
  SELECT * FROM t2('to*');
 } {top to tommy}

+#-------------------------------------------------------------------------
+
+foreach {tn newrowid} {
+  1   122
+  2   123
+  3   124
+} {
+  reset_db
+  do_execsql_test 3.$tn.0 {
+    CREATE VIRTUAL TABLE t12 USING fts5(x);
+    INSERT INTO t12(rowid, x) VALUES(123, 'wwww');
+  }
+  do_execsql_test 3.$tn.1 {
+    BEGIN;
+    DELETE FROM t12 WHERE rowid=123;
+    SELECT * FROM t12('wwww*');
+    INSERT INTO t12(rowid, x) VALUES($newrowid, 'wwww');
+    SELECT * FROM t12('wwww*');
+    END;
+  } {wwww}
+  do_execsql_test 3.$tn.2 {
+    INSERT INTO t12(t12) VALUES('integrity-check');
+  }
+  do_execsql_test 3.$tn.3 {
+    SELECT rowid FROM t12('wwww*');
+  } $newrowid
+}
+
+finish_test
+


 finish_test
@@ -180,4 +180,28 @@ do_execsql_test 6.1 {
  {table table table} {the table names.} {rank on an fts5 table}
 }

+
+#-------------------------------------------------------------------------
+# forum post: https://sqlite.org/forum/forumpost/a2dd636330
+#
+reset_db
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t USING fts5 (a, b);  
+  INSERT INTO t (a, b) VALUES ('data1', 'sentence1'), ('data2', 'sentence2'); 
+  INSERT INTO t(t, rank) VALUES ('rank', 'bm25(10.0,1.0)');
+}
+
+sqlite3 db2 test.db 
+do_execsql_test -db db2 1.1 {
+  SELECT *, rank<0.0 FROM t('data*') ORDER BY RANK;
+} {data1 sentence1 1 data2 sentence2 1}
+
+do_execsql_test 1.2 {
+  INSERT INTO t(t, rank) VALUES ('rank', 'bm25(10.0,1.0)');
+}
+do_execsql_test -db db2 1.3 {
+  SELECT *, rank<0.0 FROM t('data*') ORDER BY RANK;
+} {data1 sentence1 1 data2 sentence2 1}
+db2 close
+
 finish_test
@@ -71,7 +71,7 @@ ifcapable fts3 {

  do_catchsql_test 3.2 {
    DROP TABLE vt1;
-  } {1 {SQL logic error}}
+  } {0 {}}

  do_execsql_test 3.3 {
    SAVEPOINT x;
@@ -273,6 +273,76 @@ do_execsql_test 5.3 {
 do_execsql_test 5.4 { SELECT rowid FROM t1('abc'); } 2
 do_execsql_test 5.5 { SELECT rowid FROM t1('aa'); } 2

+#-------------------------------------------------------------------------
+# Tests for the bug fixed by https://sqlite.org/src/info/4b60a1c3
+#
+reset_db
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE fts USING fts5(content);
+  INSERT INTO fts(fts, rank) VALUES ('secure-delete', 1);
+  INSERT INTO fts(rowid, content) VALUES
+    (3407, 'profile profile profile profile profile profile profile profile pull pulling pulling really');
+  DELETE FROM fts WHERE rowid IS 3407;
+  INSERT INTO fts(fts) VALUES ('integrity-check');
+}
+
+foreach {tn detail} {
+  1 full
+  2 column
+  3 none
+} {
+  do_execsql_test 6.1.$detail "
+    DROP TABLE IF EXISTS t1;
+    CREATE VIRTUAL TABLE t1 USING fts5(x, detail=$detail);
+  "
+
+  do_execsql_test 6.2.$detail {
+    INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+  }
+
+  for {set ii 1} {$ii < 100} {incr ii} {
+    do_execsql_test 6.3.$detail.$ii.1 {
+      BEGIN;
+        INSERT INTO t1(rowid, x) VALUES(10, 'word1');
+        WITH s(i) AS (
+          SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<CAST($ii AS integer)
+        )
+        INSERT INTO t1(x) SELECT 'word3' FROM s;
+      COMMIT;
+      INSERT INTO t1(t1) VALUES('optimize');
+    }
+
+    do_execsql_test 6.3.$detail.$ii.2 {
+      DELETE FROM t1 WHERE rowid=10;
+      INSERT INTO t1(t1) VALUES ('integrity-check');
+    }
+
+    do_execsql_test 6.3.$detail.$ii.3 {
+      DELETE FROM t1;
+    }
+
+    do_execsql_test 6.3.$detail.$ii.4 {
+      BEGIN;
+        INSERT INTO t1(rowid, x) VALUES(10, 'tokenA');
+        WITH s(i) AS (
+          SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<CAST($ii AS integer)
+        )
+        INSERT INTO t1(x) SELECT group_concat('tokenB ') FROM s;
+      COMMIT;
+      INSERT INTO t1(t1) VALUES('optimize');
+    }
+
+    do_execsql_test 6.3.$detail.$ii.5 {
+      DELETE FROM t1 WHERE rowid=10;
+      INSERT INTO t1(t1) VALUES ('integrity-check');
+    }
+
+    do_execsql_test 6.3.$detail.$ii.6 {
+      DELETE FROM t1;
+    }
+  }
+}
+

 finish_test

@@ -18,7 +18,7 @@ db progress 1 progress_handler
 set ::PHC 0
 proc progress_handler {args} {
  incr ::PHC
-  if {($::PHC % 100000)==0}  breakpoint
+  # if {($::PHC % 100000)==0}  breakpoint
  return 0
 }

@@ -50,6 +50,92 @@ do_test 1.3 {
  expr $phc(1)*5 < $phc(2)
 } {1}

+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  INSERT INTO t1(t1, rank) VALUES('secure-delete', $sd) 
+}
+
+do_execsql_test 2.1 {
+  BEGIN;
+    INSERT INTO t1(rowid, x) VALUES(-100000, 'abc def ghi');
+    INSERT INTO t1(rowid, x) VALUES(-99999, 'abc def ghi');
+    INSERT INTO t1(rowid, x) VALUES(9223372036854775800, 'abc def ghi');
+  COMMIT;
+}
+
+do_execsql_test 2.2 {
+  SELECT rowid FROM t1('def')
+} {-100000 -99999 9223372036854775800}
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  INSERT INTO t1(t1, rank) VALUES('secure-delete', $sd) 
+}
+
+do_execsql_test 3.1 {
+  BEGIN;
+    INSERT INTO t1(rowid, x) 
+      VALUES(51869, 'when whenever where weress what turn'), 
+            (51871, 'to were');
+  COMMIT;
+}
+
+do_execsql_test 3.2 {
+  DELETE FROM t1 WHERE rowid=51871;
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x);
+  INSERT INTO t1(rowid, x) VALUES(10, 'one two');
+}
+do_execsql_test 4.1 {
+  UPDATE t1 SET x = 'one three' WHERE rowid=10;
+  INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+}
+do_execsql_test 4.2 {
+  DELETE FROM t1 WHERE rowid=10;
+}
+do_execsql_test 4.3 {
+  INSERT INTO t1(t1) VALUES('integrity-check');
+}
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(content);
+
+  INSERT INTO t1(t1,rank) VALUES('secure-delete',1);
+  INSERT INTO t1 VALUES('active'),('boomer'),('atom'),('atomic'),
+         ('alpha channel backup abandon test aback boomer atom alpha active');
+  DELETE FROM t1 WHERE t1 MATCH 'abandon';
+} 
+
+do_execsql_test 5.1 {
+  INSERT INTO t1(t1) VALUES('rebuild');
+}
+
+do_execsql_test 5.2 {
+  DELETE FROM t1 WHERE rowid NOTNULL<5;
+}
+
+db close
+sqlite3 db test.db
+
+do_execsql_test 5.3 {
+  PRAGMA integrity_check;
+} {ok}
+

 finish_test

@@ -0,0 +1,116 @@
+# 2023 Feb 17
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# TESTRUNNER: slow
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5 { finish_test ; return }
+set ::testprefix fts5secure7
+
+
+set NVOCAB 500
+set NDOC [expr 1000]
+
+set NREP 100
+set nDeletePerRep [expr 5]
+
+set VOCAB [list]
+
+proc select_one {list} {
+  set n [llength $list]
+  lindex $list [expr {abs(int(rand()*$n))}]
+}
+
+proc init_vocab {} {
+  set L [split "abcdefghijklmnopqrstuvwxyz" {}]
+  set nL [llength $L]
+  for {set i 0} {$i < $::NVOCAB} {incr i} {
+    set n [expr {6 + int(rand()*8)}]
+    set word ""
+    for {set j 0} {$j < $n} {incr j} {
+      append word [select_one $L]
+    }
+    lappend ::VOCAB $word
+  }
+}
+
+proc get_word {} {
+  select_one $::VOCAB
+}
+
+proc get_document {nWord} {
+  set ret [list]
+  for {set i 0} {$i < $nWord} {incr i} {
+    lappend ret [get_word]
+  }
+  return $ret
+}
+
+init_vocab
+
+db func document [list get_document 12]
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(body);
+  INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+}
+do_execsql_test 1.1 {
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$NDOC
+  )
+  INSERT INTO t1 SELECT document() FROM s;
+}
+
+for {set iRep 0} {$iRep < $NREP} {incr iRep} {
+  set lRowid [db eval {SELECT rowid FROM t1}]
+  for {set iDel 0} {$iDel < $nDeletePerRep} {incr iDel} {
+    set idx [select_one $lRowid]
+    db eval {
+      DELETE FROM t1 WHERE rowid=$idx
+    }
+  }
+  db eval {
+    WITH s(i) AS (
+      SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$nDeletePerRep
+    )
+    INSERT INTO t1 SELECT document() FROM s;
+  }
+  do_execsql_test 1.2.$iRep {
+    INSERT INTO t1(t1) VALUES('integrity-check');
+  }
+}
+
+reset_db
+db func document [list get_document 12]
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(body);
+  INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
+  INSERT INTO t1(t1, rank) VALUES('pgsz', 128);
+}
+do_execsql_test 2.1 {
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<$NDOC
+  )
+  INSERT INTO t1 SELECT document() FROM s;
+}
+for {set ii 0} {$ii < $NDOC} {incr ii} {
+  set lRowid [db eval {SELECT rowid FROM t1}]
+  set idx [select_one $lRowid]
+  db eval { DELETE FROM t1 WHERE rowid=$idx }
+  do_execsql_test 2.2.$ii {
+    INSERT INTO t1(t1) VALUES('integrity-check');
+  }
+}
+
+finish_test
+
+
@@ -122,6 +122,9 @@ foreach {tn expr} {
  4.1 "NEAR(one two, 2)"
  4.2 "NEAR(one two three, 2)"
  4.3 "NEAR(eight nine, 1) OR NEAR(six seven, 1)"
+
+  5.1 "one + two"
+  5.2 "1 + two"
 } {
  if {[fts5_expr_ok $expr ss]==0} {
    do_test 1.$tok.$tn.OMITTED { list } [list]
@@ -0,0 +1,89 @@
+# 2023 Nov 03
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focusing on the built-in fts5 tokenizers. 
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5tokenizer2
+
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+sqlite3_fts5_create_tokenizer db tst get_tst_tokenizer
+proc get_tst_tokenizer {args} {
+  return "tst_tokenizer"
+}
+proc tst_tokenizer {flags txt} {
+  set token ""
+  set lTok [list]
+
+  foreach c [split $txt {}] {
+    if {$token==""} {
+      append token $c
+    } else {
+      set t1 [string is upper $token]
+      set t2 [string is upper $c]
+
+      if {$t1!=$t2} {
+        lappend lTok $token
+        set token ""
+      }
+      append token $c
+    }
+  }
+  if {$token!=""} { lappend lTok $token }
+
+  set iOff 0
+  foreach t $lTok {
+    set n [string length $t]
+    sqlite3_fts5_token $t $iOff [expr $iOff+$n]
+    incr iOff $n
+  }
+}
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(t, tokenize=tst);
+}
+
+do_execsql_test 1.1 {
+  INSERT INTO t1 VALUES('AAdontBBmess');
+}
+
+do_execsql_test 1.2 {
+  SELECT snippet(t1, 0, '>', '<', '...', 4) FROM t1('BB');
+} {AAdont>BB<mess}
+
+do_execsql_test 1.3 {
+  SELECT highlight(t1, 0, '>', '<') FROM t1('BB');
+} {AAdont>BB<mess}
+
+do_execsql_test 1.4 {
+  SELECT highlight(t1, 0, '>', '<') FROM t1('AA');
+} {>AA<dontBBmess}
+
+do_execsql_test 1.5 {
+  SELECT highlight(t1, 0, '>', '<') FROM t1('dont');
+} {AA>dont<BBmess}
+
+do_execsql_test 1.6 {
+  SELECT highlight(t1, 0, '>', '<') FROM t1('mess');
+} {AAdontBB>mess<}
+
+do_execsql_test 1.7 {
+  SELECT highlight(t1, 0, '>', '<') FROM t1('BB mess');
+} {AAdont>BBmess<}
+
+
+finish_test
@@ -215,4 +215,42 @@ do_execsql_test 7.2 {
  SELECT rowid FROM f WHERE filename GLOB '*ир*';
 } {20}

+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 8.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(y, tokenize=trigram);
+  INSERT INTO t1 VALUES('abcdefghijklm');
+}
+
+foreach {tn match res} {
+  1 "abc ghi"         "(abc)def(ghi)jklm"
+  2 "def ghi"         "abc(defghi)jklm"
+  3 "efg ghi"         "abcd(efghi)jklm"
+  4 "efghi"           "abcd(efghi)jklm"
+  5 "abcd jklm"       "(abcd)efghi(jklm)"
+  6 "ijkl jklm"       "abcdefgh(ijklm)"
+  7 "ijk ijkl hijk"   "abcdefg(hijkl)m"
+
+} {
+  do_execsql_test 8.1.$tn {
+    SELECT highlight(t1, 0, '(', ')') FROM t1($match)
+  } $res
+}
+
+do_execsql_test 8.2 {
+  CREATE VIRTUAL TABLE ft2 USING fts5(a, tokenize="trigram");
+  INSERT INTO ft2 VALUES('abc x cde');
+  INSERT INTO ft2 VALUES('abc cde');
+  INSERT INTO ft2 VALUES('abcde');
+}
+
+do_execsql_test 8.3 {
+  SELECT highlight(ft2, 0, '[', ']') FROM ft2 WHERE ft2 MATCH 'abc AND cde';
+} {
+  {[abc] x [cde]}
+  {[abc] [cde]}
+  {[abcde]}
+}
+
 finish_test
@@ -0,0 +1,109 @@
+# 2023 October 24
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# Tests for the fts5 "trigram" tokenizer.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+ifcapable !fts5 { finish_test ; return }
+set ::testprefix fts5trigram2
+
+do_execsql_test 1.0 "
+  CREATE VIRTUAL TABLE t1 USING fts5(y, tokenize='trigram remove_diacritics 1');
+  INSERT INTO t1 VALUES('abc\u0303defghijklm');
+  INSERT INTO t1 VALUES('a\u0303b\u0303c\u0303defghijklm');
+"
+
+do_execsql_test 1.1 {
+  SELECT highlight(t1, 0, '(', ')') FROM t1('abc');
+} [list \
+  "(abc\u0303)defghijklm"                          \
+  "(a\u0303b\u0303c\u0303)defghijklm"              \
+]
+
+do_execsql_test 1.2 {
+  SELECT highlight(t1, 0, '(', ')') FROM t1('bcde');
+} [list \
+  "a(bc\u0303de)fghijklm"                          \
+  "a\u0303(b\u0303c\u0303de)fghijklm"              \
+]
+
+do_execsql_test 1.3 {
+  SELECT highlight(t1, 0, '(', ')') FROM t1('cdef');
+} [list \
+  "ab(c\u0303def)ghijklm"                          \
+  "a\u0303b\u0303(c\u0303def)ghijklm"              \
+]
+
+do_execsql_test 1.4 {
+  SELECT highlight(t1, 0, '(', ')') FROM t1('def');
+} [list \
+  "abc\u0303(def)ghijklm"                          \
+  "a\u0303b\u0303c\u0303(def)ghijklm"              \
+]
+
+
+#-------------------------------------------------------------------------
+do_catchsql_test 2.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(
+      z, tokenize='trigram case_sensitive 1 remove_diacritics 1'
+  );
+} {1 {error in tokenizer constructor}}
+
+do_execsql_test 2.1 {
+  CREATE VIRTUAL TABLE t2 USING fts5(
+      z, tokenize='trigram case_sensitive 0 remove_diacritics 1'
+  );
+}
+do_execsql_test 2.2 "
+  INSERT INTO t2 VALUES('\u00E3bcdef');
+  INSERT INTO t2 VALUES('b\u00E3cdef');
+  INSERT INTO t2 VALUES('bc\u00E3def');
+  INSERT INTO t2 VALUES('bcd\u00E3ef');
+"
+
+do_execsql_test 2.3 {
+  SELECT highlight(t2, 0, '(', ')') FROM t2('abc');
+} "(\u00E3bc)def"
+do_execsql_test 2.4 {
+  SELECT highlight(t2, 0, '(', ')') FROM t2('bac');
+} "(b\u00E3c)def"
+do_execsql_test 2.5 {
+  SELECT highlight(t2, 0, '(', ')') FROM t2('bca');
+} "(bc\u00E3)def"
+do_execsql_test 2.6 "
+  SELECT highlight(t2, 0, '(', ')') FROM t2('\u00E3bc');
+" "(\u00E3bc)def"
+
+#-------------------------------------------------------------------------
+do_execsql_test 3.0 {
+  CREATE VIRTUAL TABLE t3 USING fts5(
+      z, tokenize='trigram remove_diacritics 1'
+  );
+} {}
+do_execsql_test 3.1 "
+  INSERT INTO t3 VALUES ('\u0303abc\u0303');
+"
+do_execsql_test 3.2 {
+  SELECT highlight(t3, 0, '(', ')') FROM t3('abc');
+} "\u0303(abc\u0303)"
+
+#-------------------------------------------------------------------------
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t4 USING fts5(z, tokenize=trigram);
+} {}
+
+breakpoint
+do_execsql_test 4.1 {
+  INSERT INTO t4 VALUES('ABCD');
+} {}
+
+finish_test
@@ -6,9 +6,10 @@ JAVA_HOME ?= $(HOME)/jdk/current
 # e.g. /usr/lib/jvm/default-javajava-19-openjdk-amd64
 JDK_HOME ?= $(JAVA_HOME)
 # ^^^ JDK_HOME is not as widely used as JAVA_HOME
-bin.javac := $(JDK_HOME)/bin/javac
-bin.java  := $(JDK_HOME)/bin/java
-bin.jar   := $(JDK_HOME)/bin/jar
+bin.jar     := $(JDK_HOME)/bin/jar
+bin.java    := $(JDK_HOME)/bin/java
+bin.javac   := $(JDK_HOME)/bin/javac
+bin.javadoc := $(JDK_HOME)/bin/javadoc
 ifeq (,$(wildcard $(JDK_HOME)))
 $(error set JDK_HOME to the top-most dir of your JDK installation.)
 endif
@@ -17,18 +18,29 @@ $(MAKEFILE):

 package.jar := sqlite3-jni.jar

-dir.top := ../..
-dir.tool := ../../tool
-dir.jni := $(patsubst %/,%,$(dir $(MAKEFILE)))
-
+dir.top     := ../..
+dir.tool    := ../../tool
+dir.jni     := $(patsubst %/,%,$(dir $(MAKEFILE)))
 dir.src     := $(dir.jni)/src
 dir.src.c   := $(dir.src)/c
 dir.bld     := $(dir.jni)/bld
 dir.bld.c   := $(dir.bld)
 dir.src.jni := $(dir.src)/org/sqlite/jni
-dir.src.jni.tester := $(dir.src.jni)/tester
+dir.src.capi := $(dir.src.jni)/capi
+dir.src.fts5 := $(dir.src.jni)/fts5
+dir.tests   := $(dir.src)/tests
+mkdir       ?= mkdir -p
 $(dir.bld.c):
-	mkdir -p $@
+	$(mkdir) $@
+
+javac.flags ?= -Xlint:unchecked -Xlint:deprecation
+java.flags ?=
+javac.flags += -encoding utf8
+# -------------^^^^^^^^^^^^^^ required for Windows builds
+jnicheck ?= 1
+ifeq (1,$(jnicheck))
+  java.flags += -Xcheck:jni
+endif

 classpath := $(dir.src)
 CLEAN_FILES := $(package.jar)
@@ -36,17 +48,28 @@ DISTCLEAN_FILES := $(dir.jni)/*~ $(dir.src.c)/*~ $(dir.src.jni)/*~

 sqlite3-jni.h := $(dir.src.c)/sqlite3-jni.h
 .NOTPARALLEL: $(sqlite3-jni.h)
-SQLite3Jni.java := src/org/sqlite/jni/SQLite3Jni.java
-SQLTester.java := src/org/sqlite/jni/tester/SQLTester.java
-SQLite3Jni.class := $(SQLite3Jni.java:.java=.class)
+CApi.java := $(dir.src.capi)/CApi.java
+SQLTester.java := $(dir.src.capi)/SQLTester.java
+CApi.class := $(CApi.java:.java=.class)
 SQLTester.class := $(SQLTester.java:.java=.class)

 ########################################################################
 # The future of FTS5 customization in this API is as yet unclear.
-# It would be a real doozy to bind to JNI.
+# The pieces are all in place, and are all thin proxies so not much
+# complexity, but some semantic changes were required in porting
+# which are largely untested.
+#
+# Reminder: this flag influences the contents of $(sqlite3-jni.h),
+# which is checked in. Please do not check in changes to that file in
+# which the fts5 APIs have been stripped unless that feature is
+# intended to be stripped for good.
 enable.fts5 ?= 1
-# If enable.tester is 0, the org/sqlite/jni/tester/* bits are elided.
-enable.tester ?= 1
+
+ifeq (,$(wildcard $(dir.tests)/*))
+  enable.tester := 0
+else
+  enable.tester := 1
+endif

 # bin.version-info = binary to output various sqlite3 version info
 # building the distribution zip file.
@@ -57,68 +80,99 @@ $(bin.version-info): $(dir.tool)/version-info.c $(sqlite3.h) $(dir.top)/Makefile

 # Be explicit about which Java files to compile so that we can work on
 # in-progress files without requiring them to be in a compilable statae.
-JAVA_FILES.main := $(patsubst %,$(dir.src.jni)/%,\
-  BusyHandler.java \
-  Collation.java \
-  CollationNeeded.java \
-  CommitHook.java \
+JAVA_FILES.main := $(patsubst %,$(dir.src.jni)/annotation/%,\
+  Experimental.java \
+  NotNull.java \
+  Nullable.java \
+) $(patsubst %,$(dir.src.capi)/%,\
+  AbstractCollationCallback.java \
+  AggregateFunction.java \
+  AuthorizerCallback.java \
+  AutoExtensionCallback.java \
+  BusyHandlerCallback.java \
+  CollationCallback.java \
+  CollationNeededCallback.java \
+  CommitHookCallback.java \
+  ConfigLogCallback.java \
+  ConfigSqlLogCallback.java \
  NativePointerHolder.java \
  OutputPointer.java \
-  ProgressHandler.java \
+  PrepareMultiCallback.java \
+  PreupdateHookCallback.java \
+  ProgressHandlerCallback.java \
  ResultCode.java \
-  RollbackHook.java \
+  RollbackHookCallback.java \
+  ScalarFunction.java \
  SQLFunction.java \
-  sqlite3_context.java \
+  CallbackProxy.java \
+  CApi.java \
+  TableColumnMetadata.java \
+  TraceV2Callback.java \
+  UpdateHookCallback.java \
+  ValueHolder.java \
+  WindowFunction.java \
+  XDestroyCallback.java \
  sqlite3.java \
-  SQLite3Jni.java \
+  sqlite3_blob.java \
+  sqlite3_context.java \
  sqlite3_stmt.java \
  sqlite3_value.java \
-  Tester1.java \
-  Tracer.java \
-  UpdateHook.java \
+) $(patsubst %,$(dir.src.jni)/wrapper1/%,\
+  AggregateFunction.java \
+  ScalarFunction.java \
+  SqlFunction.java \
+  Sqlite.java \
+  SqliteException.java \
  ValueHolder.java \
+  WindowFunction.java \
+)
+
+JAVA_FILES.unittest := $(patsubst %,$(dir.src.jni)/%,\
+  capi/Tester1.java \
+  wrapper1/Tester2.java \
 )
 ifeq (1,$(enable.fts5))
-  JAVA_FILES.main += $(patsubst %,$(dir.src.jni)/%,\
+  JAVA_FILES.unittest += $(patsubst %,$(dir.src.fts5)/%,\
+    TesterFts5.java \
+  )
+  JAVA_FILES.main += $(patsubst %,$(dir.src.fts5)/%,\
    fts5_api.java \
    fts5_extension_function.java \
    fts5_tokenizer.java \
    Fts5.java \
    Fts5Context.java \
    Fts5ExtensionApi.java \
-    Fts5Function.java \
    Fts5PhraseIter.java \
    Fts5Tokenizer.java \
-    TesterFts5.java \
+    XTokenizeCallback.java \
  )
 endif
-JAVA_FILES.tester := $(dir.src.jni.tester)/SQLTester.java
+JAVA_FILES.tester := $(SQLTester.java)
+JAVA_FILES.package.info := \
+  $(dir.src.jni)/package-info.java \
+  $(dir.src.jni)/annotation/package-info.java

 CLASS_FILES.main := $(JAVA_FILES.main:.java=.class)
+CLASS_FILES.unittest := $(JAVA_FILES.unittest:.java=.class)
 CLASS_FILES.tester := $(JAVA_FILES.tester:.java=.class)

-JAVA_FILES += $(JAVA_FILES.main)
+JAVA_FILES += $(JAVA_FILES.main) $(JAVA_FILES.unittest)
 ifeq (1,$(enable.tester))
  JAVA_FILES += $(JAVA_FILES.tester)
 endif

 CLASS_FILES :=
-define DOTCLASS_DEPS
-$(1).class: $(1).java $(MAKEFILE)
+define CLASSFILE_DEPS
 all: $(1).class
+$(1).class: $(1).java
 CLASS_FILES += $(1).class
 endef
-$(foreach B,$(basename $(JAVA_FILES)),$(eval $(call DOTCLASS_DEPS,$(B))))
-$(CLASS_FILES.tester): $(CLASS_FILES.main)
-javac.flags ?= -Xlint:unchecked -Xlint:deprecation
-java.flags ?=
-jnicheck ?= 1
-ifeq (1,$(jnicheck))
-  java.flags += -Xcheck:jni
-endif
-$(SQLite3Jni.class): $(JAVA_FILES)
+$(foreach B,$(basename \
+  $(JAVA_FILES.main) $(JAVA_FILES.unittest) $(JAVA_FILES.tester)),\
+  $(eval $(call CLASSFILE_DEPS,$(B))))
+$(CLASS_FILES): $(MAKEFILE)
 	$(bin.javac) $(javac.flags) -h $(dir.bld.c) -cp $(classpath) $(JAVA_FILES)
-all: $(SQLite3Jni.class)
+
 #.PHONY: classfiles

 ########################################################################
@@ -152,27 +206,41 @@ $(sqlite3.h):
 	$(MAKE) -C $(dir.top) sqlite3.c
 $(sqlite3.c): $(sqlite3.h)

+opt.threadsafe ?= 1
+opt.fatal-oom ?= 1
+opt.debug ?= 1
+opt.metrics ?= 1
 SQLITE_OPT = \
-  -DSQLITE_ENABLE_RTREE \
+  -DSQLITE_THREADSAFE=$(opt.threadsafe) \
+  -DSQLITE_TEMP_STORE=2 \
+  -DSQLITE_USE_URI=1 \
+  -DSQLITE_OMIT_LOAD_EXTENSION \
+  -DSQLITE_OMIT_DEPRECATED \
+  -DSQLITE_OMIT_SHARED_CACHE \
+  -DSQLITE_C=$(sqlite3.c) \
+  -DSQLITE_JNI_FATAL_OOM=$(opt.fatal-oom) \
+  -DSQLITE_JNI_ENABLE_METRICS=$(opt.metrics)
+
+opt.extras ?= 1
+ifeq (1,$(opt.extras))
+SQLITE_OPT += -DSQLITE_ENABLE_RTREE \
  -DSQLITE_ENABLE_EXPLAIN_COMMENTS \
  -DSQLITE_ENABLE_STMTVTAB \
  -DSQLITE_ENABLE_DBPAGE_VTAB \
  -DSQLITE_ENABLE_DBSTAT_VTAB \
  -DSQLITE_ENABLE_BYTECODE_VTAB \
  -DSQLITE_ENABLE_OFFSET_SQL_FUNC \
-  -DSQLITE_OMIT_LOAD_EXTENSION \
-  -DSQLITE_OMIT_DEPRECATED \
-  -DSQLITE_OMIT_SHARED_CACHE \
-  -DSQLITE_THREADSAFE=0 \
-  -DSQLITE_TEMP_STORE=2 \
-  -DSQLITE_USE_URI=1 \
-  -DSQLITE_C=$(sqlite3.c) \
-  -DSQLITE_DEBUG
-# -DSQLITE_DEBUG is just to work around a -Wall warning
-# for a var which gets set in all builds but only read
-# via assert().
+  -DSQLITE_ENABLE_PREUPDATE_HOOK \
+  -DSQLITE_ENABLE_NORMALIZE \
+  -DSQLITE_ENABLE_SQLLOG \
+  -DSQLITE_ENABLE_COLUMN_METADATA
+endif

-SQLITE_OPT += -g -DDEBUG -UNDEBUG
+ifeq (1,$(opt.debug))
+  SQLITE_OPT += -DSQLITE_DEBUG -g -DDEBUG -UNDEBUG
+else
+  SQLITE_OPT += -Os
+endif

 ifeq (1,$(enable.fts5))
  SQLITE_OPT += -DSQLITE_ENABLE_FTS5
@@ -181,25 +249,30 @@ endif
 sqlite3-jni.c := $(dir.src.c)/sqlite3-jni.c
 sqlite3-jni.o := $(dir.bld.c)/sqlite3-jni.o
 sqlite3-jni.h   := $(dir.src.c)/sqlite3-jni.h
-sqlite3-jni.dll := $(dir.bld.c)/libsqlite3-jni.so
+package.dll := $(dir.bld.c)/libsqlite3-jni.so
 # All javac-generated .h files must be listed in $(sqlite3-jni.h.in):
 sqlite3-jni.h.in :=
+# $(java.with.jni) lists all Java files which contain JNI decls:
+java.with.jni :=
 define ADD_JNI_H
-sqlite3-jni.h.in += $$(dir.bld.c)/org_sqlite_jni_$(1).h
-$$(dir.bld.c)/org_sqlite_jni_$(1).h: $$(dir.src.jni)/$(1).java
+sqlite3-jni.h.in += $$(dir.bld.c)/org_sqlite_jni$(3)_$(2).h
+java.with.jni += $(1)/$(2).java
+$$(dir.bld.c)/org_sqlite_jni$(3)_$(2).h: $(1)/$(2).java
 endef
-$(eval $(call ADD_JNI_H,SQLite3Jni))
+# Invoke ADD_JNI_H once for each Java file which includes JNI
+# declarations:
+$(eval $(call ADD_JNI_H,$(dir.src.capi),CApi,_capi))
+$(eval $(call ADD_JNI_H,$(dir.src.capi),SQLTester,_capi))
 ifeq (1,$(enable.fts5))
-  $(eval $(call ADD_JNI_H,Fts5ExtensionApi))
-  $(eval $(call ADD_JNI_H,fts5_api))
-  $(eval $(call ADD_JNI_H,fts5_tokenizer))
+ $(eval $(call ADD_JNI_H,$(dir.src.fts5),Fts5ExtensionApi,_fts5))
+ $(eval $(call ADD_JNI_H,$(dir.src.fts5),fts5_api,_fts5))
+ $(eval $(call ADD_JNI_H,$(dir.src.fts5),fts5_tokenizer,_fts5))
 endif
-ifeq (1,$(enable.tester))
-  sqlite3-jni.h.in += $(dir.bld.c)/org_sqlite_jni_tester_SQLTester.h
-  $(dir.bld.c)/org_sqlite_jni_tester_SQLTester.h: $(dir.src.jni.tester)/SQLTester.java
-endif
-#sqlite3-jni.dll.cfiles := $(dir.src.c)
-sqlite3-jni.dll.cflags = \
+$(sqlite3-jni.h.in): $(dir.bld.c)
+
+#package.dll.cfiles :=
+package.dll.cflags = \
+  -std=c99 \
  -fPIC \
  -I. \
  -I$(dir $(sqlite3.h)) \
@@ -207,54 +280,82 @@ sqlite3-jni.dll.cflags = \
  -I$(JDK_HOME)/include \
  $(patsubst %,-I%,$(patsubst %.h,,$(wildcard $(JDK_HOME)/include/*))) \
  -Wall
-# Using (-Wall -Wextra) triggers an untennable number of
-# gcc warnings from sqlite3.c for mundane things like
-# unused parameters.
-#
 # The gross $(patsubst...) above is to include the platform-specific
 # subdir which lives under $(JDK_HOME)/include and is a required
 # include path for client-level code.
+#
+# Using (-Wall -Wextra) triggers an untennable number of
+# gcc warnings from sqlite3.c for mundane things like
+# unused parameters.
 ########################################################################
 ifeq (1,$(enable.tester))
-  sqlite3-jni.dll.cflags += -DS3JNI_ENABLE_SQLTester
+  package.dll.cflags += -DSQLITE_JNI_ENABLE_SQLTester
 endif
-$(sqlite3-jni.h): $(sqlite3-jni.h.in) $(MAKEFILE)
-	cat $(sqlite3-jni.h.in) > $@
-$(sqlite3-jni.dll): $(sqlite3-jni.h) $(sqlite3.c) $(sqlite3.h)
-$(sqlite3-jni.dll): $(dir.bld.c) $(sqlite3-jni.c) $(SQLite3Jni.java) $(MAKEFILE)
-	$(CC) $(sqlite3-jni.dll.cflags) $(SQLITE_OPT) \
-			$(sqlite3-jni.c) -shared -o $@
-all: $(sqlite3-jni.dll)

-.PHONY: test
-test.flags ?= -v
-test: $(SQLite3Jni.class) $(sqlite3-jni.dll)
-	$(bin.java) -ea -Djava.library.path=$(dir.bld.c) \
-		$(java.flags) -cp $(classpath) \
-		org.sqlite.jni.Tester1 $(if $(test.flags),-- $(test.flags),)
+$(sqlite3-jni.h): $(sqlite3-jni.h.in) $(MAKEFILE)
+	@cat $(sqlite3-jni.h.in) > $@.tmp
+	@if cmp $@ $@.tmp >/dev/null; then \
+		rm -f $@.tmp; \
+		echo "$@ not modified"; \
+	else \
+		mv $@.tmp $@; \
+		echo "Updated $@"; \
+	fi
+	@if [ x1 != x$(enable.fts5) ]; then \
+		echo "*** REMINDER:"; \
+		echo "*** enable.fts5=0, so please do not check in changes to $@."; \
+	fi
+
+$(package.dll): $(sqlite3-jni.h) $(sqlite3.c) $(sqlite3.h)
+$(package.dll): $(sqlite3-jni.c) $(MAKEFILE)
+	$(CC) $(package.dll.cflags) $(SQLITE_OPT) \
+			$(sqlite3-jni.c) -shared -o $@
+all: $(package.dll)
+
+.PHONY: test test-one
+Tester1.flags ?=
+Tester2.flags ?=
+test.flags.jvm = -ea -Djava.library.path=$(dir.bld.c) \
+                  $(java.flags) -cp $(classpath)
+test.deps := $(CLASS_FILES) $(package.dll)
+test-one: $(test.deps)
+	$(bin.java) $(test.flags.jvm) org.sqlite.jni.capi.Tester1 $(Tester1.flags)
+	$(bin.java) $(test.flags.jvm) org.sqlite.jni.wrapper1.Tester2 $(Tester2.flags)
+test-sqllog: $(test.deps)
+	@echo "Testing with -sqllog..."
+	$(bin.java) $(test.flags.jvm) org.sqlite.jni.capi.Tester1 $(Tester1.flags) -sqllog
+test-mt: $(test.deps)
+	@echo "Testing in multi-threaded mode:";
+	$(bin.java) $(test.flags.jvm) org.sqlite.jni.capi.Tester1 \
+    -t 7 -r 50 -shuffle $(Tester1.flags)
+	$(bin.java) $(test.flags.jvm) org.sqlite.jni.wrapper1.Tester2 \
+    -t 7 -r 50 -shuffle $(Tester2.flags)
+
+test: test-one test-mt
+tests: test test-sqllog

 tester.scripts := $(sort $(wildcard $(dir.src)/tests/*.test))
 tester.flags ?= # --verbose
 .PHONY: tester tester-local tester-ext
 ifeq (1,$(enable.tester))
-tester-local: $(CLASS_FILES.tester) $(sqlite3-jni.dll)
+tester-local: $(CLASS_FILES.tester) $(package.dll)
 	$(bin.java) -ea -Djava.library.path=$(dir.bld.c) \
 		$(java.flags) -cp $(classpath) \
-		org.sqlite.jni.tester.SQLTester $(tester.flags) $(tester.scripts)
+		org.sqlite.jni.capi.SQLTester $(tester.flags) $(tester.scripts)
 tester: tester-local
 else
 tester:
-	@echo "SQLTester support is disabled. Build with enable.tester=1 to enable it."
+	@echo "SQLTester support is disabled."
 endif

-tester.extdir.default := src/tests/ext
+tester.extdir.default := $(dir.tests)/ext
 tester.extdir ?= $(tester.extdir.default)
 tester.extern-scripts := $(wildcard $(tester.extdir)/*.test)
 ifneq (,$(tester.extern-scripts))
 tester-ext:
 	$(bin.java) -ea -Djava.library.path=$(dir.bld.c) \
 		$(java.flags) -cp $(classpath) \
-		org.sqlite.jni.tester.SQLTester $(tester.flags) $(tester.extern-scripts)
+		org.sqlite.jni.capi.SQLTester $(tester.flags) $(tester.extern-scripts)
 else
 tester-ext:
 	@echo "******************************************************"; \
@@ -267,25 +368,79 @@ endif

 tester-ext: tester-local
 tester: tester-ext
-tests: test tester
+tests: tester
+########################################################################
+# Build each SQLITE_THREADMODE variant and run all tests against them.
+multitest: clean
+define MULTIOPT
+multitest: multitest-$(1)
+multitest-$(1):
+	$$(MAKE) opt.debug=$$(opt.debug) $(patsubst %,opt.%,$(2)) \
+		tests clean enable.fts5=1
+endef
+
+$(eval $(call MULTIOPT,01,threadsafe=0 oom=1))
+$(eval $(call MULTIOPT,00,threadsafe=0 oom=0))
+$(eval $(call MULTIOPT,11,threadsafe=1 oom=1))
+$(eval $(call MULTIOPT,10,threadsafe=1 oom=0))
+$(eval $(call MULTIOPT,21,threadsafe=2 oom=1))
+$(eval $(call MULTIOPT,20,threadsafe=2 oom=0))
+
+
+########################################################################
+# jar bundle...
 package.jar.in := $(abspath $(dir.src)/jar.in)
 CLEAN_FILES += $(package.jar.in)
-$(package.jar.in): $(MAKEFILE) $(CLASS_FILES.main)
-	cd $(dir.src); ls -1 org/sqlite/jni/*.java org/sqlite/jni/*.class > $@
-	@ls -la $@
-	@echo "To use this jar you will need the -Djava.library.path=DIR/WITH/libsqlite3-jni.so flag."
-	@echo "e.g. java -jar $@ -Djava.library.path=bld"
+JAVA_FILES.jar := $(JAVA_FILES.main) $(JAVA_FILES.unittest) $(JAVA_FILES.package.info)
+CLASS_FILES.jar := $(filter-out %/package-info.class,$(JAVA_FILES.jar:.java=.class))
+$(package.jar.in): $(package.dll) $(MAKEFILE)
+	ls -1 \
+		$(dir.src.jni)/*/*.java $(dir.src.jni)/*/*.class \
+	| sed -e 's,^$(dir.src)/,,' | sort > $@

-$(package.jar): $(CLASS_FILES) $(MAKEFILE) $(package.jar.in)
-	rm -f $(dir.src)/c/*~ $(dir.src.jni)/*~
-	cd $(dir.src); $(bin.jar) -cfe ../$@ org.sqlite.jni.Tester1 @$(package.jar.in)
+$(package.jar): $(CLASS_FILES.jar) $(MAKEFILE) $(package.jar.in)
+	@rm -f $(dir.src)/c/*~ $(dir.src.jni)/*~
+	cd $(dir.src); $(bin.jar) -cfe ../$@ org.sqlite.jni.capi.Tester1 @$(package.jar.in)
+	@ls -la $@
+	@echo "To use this jar you will need the -Djava.library.path=DIR/CONTAINING/libsqlite3-jni.so flag."
+	@echo "e.g. java -Djava.library.path=bld -jar $@"

 jar: $(package.jar)
+run-jar: $(package.jar) $(package.dll)
+	$(bin.java) -Djava.library.path=$(dir.bld) -jar $(package.jar) $(run-jar.flags)

+########################################################################
+# javadoc...
+dir.doc   := $(dir.jni)/javadoc
+doc.index := $(dir.doc)/index.html
+javadoc.exclude := -exclude org.sqlite.jni.fts5
+# ^^^^ 2023-09-13: elide the fts5 parts from the public docs for
+# the time being, as it's not clear where the Java bindings for
+# those bits are going.
+# javadoc.exclude += -exclude org.sqlite.jni.capi
+# ^^^^ exclude the capi API only for certain builds (TBD)
+$(doc.index): $(JAVA_FILES.main) $(MAKEFILE)
+	@if [ -d $(dir.doc) ]; then rm -fr $(dir.doc)/*; fi
+	$(bin.javadoc) -cp $(classpath) -d $(dir.doc) -quiet \
+		-subpackages org.sqlite.jni $(javadoc.exclude)
+	@echo "javadoc output is in $@"
+
+.PHONY: doc javadoc docserve
+.FORCE: doc
+doc: $(doc.index)
+javadoc: $(doc.index)
+# Force rebild of docs
+redoc:
+	@rm -f $(doc.index)
+	@$(MAKE) doc
+docserve: $(doc.index)
+	cd $(dir.doc) && althttpd -max-age 1 -page index.html
+########################################################################
+# Clean up...
 CLEAN_FILES += $(dir.bld.c)/* \
  $(dir.src.jni)/*.class \
-  $(dir.src.jni.tester)/*.class \
-  $(sqlite3-jni.dll) \
+  $(dir.src.jni)/*/*.class \
+  $(package.dll) \
  hs_err_pid*.log

 .PHONY: clean distclean
@@ -293,7 +448,7 @@ clean:
 	-rm -f $(CLEAN_FILES)
 distclean: clean
 	-rm -f $(DISTCLEAN_FILES)
-	-rm -fr $(dir.bld.c)
+	-rm -fr $(dir.bld.c) $(dir.doc)

 ########################################################################
 # disttribution bundle rules...
@@ -317,6 +472,10 @@ dist: \
    $(bin.version-info) $(sqlite3.canonical.c) \
    $(package.jar) $(MAKEFILE)
 	@echo "Making end-user deliverables..."
+	@echo "****************************************************************************"; \
+	echo  "*** WARNING: be sure to build this with JDK8 (javac 1.8) for compatibility."; \
+	echo  "*** reasons!"; $$($(bin.javac) -version); \
+	echo  "****************************************************************************"
 	@rm -fr $(dist-dir.top)
 	@mkdir -p $(dist-dir.src)
 	@cp -p $(dist.top.extras) $(dist-dir.top)/.
@@ -15,7 +15,9 @@ Technical support is available in the forum:

 > **FOREWARNING:** this subproject is very much in development and
  subject to any number of changes. Please do not rely on any
-  information about its API until this disclaimer is removed.
+  information about its API until this disclaimer is removed.  The JNI
+  bindings released with version 3.43 are a "tech preview." Once
+  finalized, strong backward compatibility guarantees will apply.

 Project goals/requirements:

@@ -40,12 +42,43 @@ Non-goals:
 - Creation of high-level OO wrapper APIs. Clients are free to create
  them off of the C-style API.

+- Virtual tables are unlikely to be supported due to the amount of
+  glue code needed to fit them into Java.

-Significant TODOs
-========================================================================
+- Support for mixed-mode operation, where client code accesses SQLite
+  both via the Java-side API and the C API via their own native
+  code. Such cases would be a minefield of potential mis-interactions
+  between this project's JNI bindings and mixed-mode client code.

- Lots of APIs left to bind. Most "day-to-day" functionality is already
-  in place and is believed to work well.
+
+Hello World
+-----------------------------------------------------------------------
+
+```java
+import org.sqlite.jni.*;
+import static org.sqlite.jni.CApi.*;
+
+...
+
+final sqlite3 db = sqlite3_open(":memory:");
+try {
+  final int rc = sqlite3_errcode(db);
+  if( 0 != rc ){
+    if( null != db ){
+      System.out.print("Error opening db: "+sqlite3_errmsg(db));
+    }else{
+      System.out.print("Error opening db: rc="+rc);
+    }
+    ... handle error ...
+  }
+  // ... else use the db ...
+}finally{
+  // ALWAYS close databases using sqlite3_close() or sqlite3_close_v2()
+  // when done with them. All of their active statement handles must
+  // first have been passed to sqlite3_finalize().
+  sqlite3_close_v2(db);
+}
+```


 Building
@@ -59,55 +92,99 @@ The canonical builds assumes a Linux-like environment and requires:

 Put simply:

-```
+```console
 $ export JAVA_HOME=/path/to/jdk/root
 $ make
 $ make test
 $ make clean
 ```

+The jar distribution can be created with `make jar`, but note that it
+does not contain the binary DLL file. A different DLL is needed for
+each target platform.
+
+
 <a id='1to1ish'></a>
 One-to-One(-ish) Mapping to C
 ========================================================================

 This JNI binding aims to provide as close to a 1-to-1 experience with
-the C API as cross-language semantics allow. Exceptions are
+the C API as cross-language semantics allow. Interface changes are
 necessarily made where cross-language semantics do not allow a 1-to-1,
 and judiciously made where a 1-to-1 mapping would be unduly cumbersome
-to use in Java.
+to use in Java. In all cases, this binding makes every effort to
+provide semantics compatible with the C API documentation even if the
+interface to those semantics is slightly different.  Any cases which
+deviate from those semantics (either removing or adding semantics) are
+clearly documented.

-Golden Rule: _Never_ Throw from Callbacks
+Where it makes sense to do so for usability, Java-side overloads are
+provided which accept or return data in alternative forms or provide
+sensible default argument values. In all such cases they are thin
+proxies around the corresponding C APIs and do not introduce new
+semantics.
+
+In a few cases, Java-specific capabilities have been added in
+new APIs, all of which have "_java" somewhere in their names.
+Examples include:
+
+- `sqlite3_result_java_object()`
+- `sqlite3_column_java_object()`
+- `sqlite3_value_java_object()`
+
+which, as one might surmise, collectively enable the passing of
+arbitrary Java objects from user-defined SQL functions through to the
+caller.
+
+
+Golden Rule: Garbage Collection Cannot Free SQLite Resources
 ------------------------------------------------------------------------

-JNI bindings which accept client-defined functions _must never throw
-exceptions_ unless _very explicitly documented_ as being
-throw-safe. Exceptions are generally reserved for higher-level
-bindings which are constructed to specifically deal with them and
-ensure that they do not leak C-level resources. Some of the JNI
-bindings are provided as Java functions which expect this rule to
-always hold.
+It is important that all databases and prepared statement handles get
+cleaned up by client code. A database cannot be closed if it has open
+statement handles. `sqlite3_close()` fails if the db cannot be closed
+whereas `sqlite3_close_v2()` recognizes that case and marks the db as
+a "zombie," pending finalization when the library detects that all
+pending statements have been closed. Be aware that Java garbage
+collection _cannot_ close a database or finalize a prepared statement.
+Those things require explicit API calls.

-UTF-8(-ish)
+Classes for which it is sensible support Java's `AutoCloseable`
+interface so can be used with try-with-resources constructs.
+
+
+Golden Rule #2: _Never_ Throw from Callbacks (Unless...)
 ------------------------------------------------------------------------

-SQLite internally uses UTF-8 encoding, whereas Java natively uses
-UTF-16.  Java JNI has routines for converting to and from UTF-8, _but_
-Java uses what its docs call "[modified UTF-8][modutf8]." Care must be
-taken when converting Java strings to UTF-8 to ensure that the proper
-conversion is performed. In short,
-`String.getBytes(StandardCharsets.UTF_8)` performs the proper
-conversion in Java, and there is no JNI C API for that conversion
-(JNI's `NewStringUTF()` returns MUTF-8).
+All routines in this API, barring explicitly documented exceptions,
+retain C-like semantics. For example, they are not permitted to throw
+or propagate exceptions and must return error information (if any) via
+result codes or `null`. The only cases where the C-style APIs may
+throw is through client-side misuse, e.g. passing in a null where it
+may cause a `NullPointerException`. The APIs clearly mark function
+parameters which should not be null, but does not generally actively
+defend itself against such misuse. Some C-style APIs explicitly accept
+`null` as a no-op for usability's sake, and some of the JNI APIs
+deliberately return an error code, instead of segfaulting, when passed
+a `null`.

-Known consequences and limitations of this discrepancy include:
+Client-defined callbacks _must never throw exceptions_ unless _very
+explitly documented_ as being throw-safe. Exceptions are generally
+reserved for higher-level bindings which are constructed to
+specifically deal with them and ensure that they do not leak C-level
+resources. In some cases, callback handlers are permitted to throw, in
+which cases they get translated to C-level result codes and/or
+messages. If a callback which is not permitted to throw throws, its
+exception may trigger debug output but will otherwise be suppressed.

- Names of databases, tables, and collations must not contain
-  characters which differ in MUTF-8 and UTF-8, or certain APIs will
-  mis-translate them on their way between languages. APIs which
-  transfer other client-side data to Java take extra care to
-  convert the data at the cost of performance.
-
-[modutf8]: https://docs.oracle.com/javase/8/docs/api/java/io/DataInput.html#modified-utf-8
+The reason some callbacks are permitted to throw and others not is
+because all such callbacks act as proxies for C function callback
+interfaces and some of those interfaces have no error-reporting
+mechanism. Those which are capable of propagating errors back through
+the library convert exceptions from callbacks into corresponding
+C-level error information. Those which cannot propagate errors
+necessarily suppress any exceptions in order to maintain the C-style
+semantics of the APIs.


 Unwieldy Constructs are Re-mapped
@@ -125,7 +202,7 @@ A prime example of where interface changes for Java are necessary for
 usability is [registration of a custom
 collation](https://sqlite.org/c3ref/create_collation.html):

-```
+```c
 // C:
 int sqlite3_create_collation(sqlite3 * db, const char * name, int eTextRep,
                             void *pUserData,
@@ -144,7 +221,7 @@ passed that object as their first argument. That data is passed around
 bind that part as-is to Java, the result would be awkward to use (^Yes,
 we tried this.):

-```
+```java
 // Java:
 int sqlite3_create_collation(sqlite3 db, String name, int eTextRep,
                             Object pUserData, xCompareType xCompare);
@@ -159,20 +236,20 @@ for callbacks and (B) having their internal state provided separately,
 which is ill-fitting in Java. For the sake of usability, C APIs which
 follow that pattern use a slightly different Java interface:

-```
+```java
 int sqlite3_create_collation(sqlite3 db, String name, int eTextRep,
-                             Collation collation);
+                             SomeCallbackType collation);
 ```

-Where the `Collation` class has an abstract `xCompare()` method and
+Where the `Collation` class has an abstract `call()` method and
 no-op `xDestroy()` method which can be overridden if needed, leading to
 a much more Java-esque usage:

-```
-int rc = sqlite3_create_collation(db, "mycollation", SQLITE_UTF8, new Collation(){
+```java
+int rc = sqlite3_create_collation(db, "mycollation", SQLITE_UTF8, new SomeCallbackType(){

  // Required comparison function:
-  @Override public int xCompare(byte[] lhs, byte[] rhs){ ... }
+  @Override public int call(byte[] lhs, byte[] rhs){ ... }

  // Optional finalizer function:
  @Override public void xDestroy(){ ... }
@@ -187,8 +264,8 @@ int rc = sqlite3_create_collation(db, "mycollation", SQLITE_UTF8, new Collation(

 Noting that:

- It is still possible to bind in call-scope-local state via closures,
-  if desired.
+- It is possible to bind in call-scope-local state via closures, if
+  desired, as opposed to packing it into the Collation object.

 - No capabilities of the C API are lost or unduly obscured via the
  above API reshaping, so power users need not make any compromises.
@@ -199,6 +276,7 @@ Noting that:
  overriding the `xDestroy()` method effectively gives it v2
  semantics.

+
 ### User-defined SQL Functions (a.k.a. UDFs)

 The [`sqlite3_create_function()`](https://sqlite.org/c3ref/create_function.html)
@@ -206,28 +284,33 @@ family of APIs make heavy use of function pointers to provide
 client-defined callbacks, necessitating interface changes in the JNI
 binding. The Java API has only one core function-registration function:

-```
+```java
 int sqlite3_create_function(sqlite3 db, String funcName, int nArgs,
                            int encoding, SQLFunction func);
 ```

-> Design question: does the encoding argument serve any purpose in JS?
+> Design question: does the encoding argument serve any purpose in
+  Java? That's as-yet undetermined. If not, it will be removed.

 `SQLFunction` is not used directly, but is instead instantiated via
 one of its three subclasses:

- `SQLFunction.Scalar` implements simple scalar functions using but a
+- `ScalarFunction` implements simple scalar functions using but a
  single callback.
- `SQLFunction.Aggregate` implements aggregate functions using two
+- `AggregateFunction` implements aggregate functions using two
  callbacks.
- `SQLFunction.Window` implements window functions using four
+- `WindowFunction` implements window functions using four
  callbacks.

-Search [`Tester1.java`](/file/ext/jni/src/org/sqlite/jni/Tester1.java) for
+Search [`Tester1.java`](/file/ext/jni/src/org/sqlite/jni/capi/Tester1.java) for
 `SQLFunction` for how it's used.

 Reminder: see the disclaimer at the top of this document regarding the
 in-flux nature of this API.

-[jsrc]: /file/
-[www]: https://sqlite.org
+### And so on...
+
+Various APIs which accept callbacks, e.g. `sqlite3_trace_v2()` and
+`sqlite3_update_hook()`, use interfaces similar to those shown above.
+Despite the changes in signature, the JNI layer makes every effort to
+provide the same semantics as the C API documentation suggests.
@@ -6,7 +6,9 @@
 # proper top-level JDK directory and, depending on the platform, add a
 # platform-specific -I directory. It should build as-is with any
 # 2020s-era version of gcc or clang. It requires JDK version 8 or
-# higher.
+# higher and that JAVA_HOME points to the top-most installation
+# directory of that JDK. On Ubuntu-style systems the JDK is typically
+# installed under /usr/lib/jvm/java-VERSION-PLATFORM.

 default: all

@@ -31,14 +33,11 @@ SQLITE_OPT = \
  -DSQLITE_OMIT_LOAD_EXTENSION \
  -DSQLITE_OMIT_DEPRECATED \
  -DSQLITE_OMIT_SHARED_CACHE \
-  -DSQLITE_THREADSAFE=0 \
+  -DSQLITE_THREADSAFE=1 \
  -DSQLITE_TEMP_STORE=2 \
  -DSQLITE_USE_URI=1 \
  -DSQLITE_ENABLE_FTS5 \
  -DSQLITE_DEBUG
-# -DSQLITE_DEBUG is just to work around a -Wall warning
-# for a var which gets set in all builds but only read
-# via assert().

 sqlite3-jni.dll = libsqlite3-jni.so
 $(sqlite3-jni.dll):
@@ -50,8 +49,10 @@ $(sqlite3-jni.dll):
 		src/sqlite3-jni.c -shared -o $@
 	@echo "Now try running it with: make test"

+test.flags = -Djava.library.path=. sqlite3-jni-*.jar
 test: $(sqlite3-jni.dll)
-	java -jar -Djava.library.path=. sqlite3-jni-*.jar
+	java -jar $(test.flags)
+	java -jar $(test.flags) -t 7 -r 10 -shuffle

 clean:
 	-rm -f $(sqlite3-jni.dll)
@@ -1,32 +0,0 @@
-/*
-** 2023-08-05
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This file is part of the JNI bindings for the sqlite3 C API.
-*/
-package org.sqlite.jni;
-
-/**
-   A callback for use with sqlite3_set_authorizer().
-*/
-public interface Authorizer {
-  /**
-     Must functions as described for the sqlite3_set_authorizer()
-     callback, with one caveat: the string values passed here were
-     initially (at the C level) encoded in standard UTF-8. If they
-     contained any constructs which are not compatible with MUTF-8,
-     these strings will not have the expected values. For further
-     details, see the documentation for the SQLite3Jni class.
-
-     Must not throw.
-  */
-  int xAuth(int opId, @Nullable String s1, @Nullable String s2,
-            @Nullable String s3, @Nullable String s4);
-}
@@ -1,31 +0,0 @@
-/*
-** 2023-08-05
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This file is part of the JNI bindings for the sqlite3 C API.
-*/
-package org.sqlite.jni;
-
-/**
-   A callback for use with sqlite3_auto_extension().
-*/
-public interface AutoExtension {
-  /**
-     Must function as described for the sqlite3_auto_extension(),
-     with the caveat that the signature is more limited.
-
-     As an exception (as it were) to the callbacks-must-not-throw
-     rule, AutoExtensions may do so and the exception's error message
-     will be set as the db's error string.
-
-     Results are undefined if db is closed by an auto-extension.
-  */
-  int xEntryPoint(sqlite3 db);
-}
@@ -1,45 +0,0 @@
-/*
-** 2023-07-22
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This file is part of the JNI bindings for the sqlite3 C API.
-*/
-package org.sqlite.jni;
-
-/**
-   Callback proxy for use with sqlite3_busy_handler().
-*/
-public abstract class BusyHandler {
-  /**
-     Must function as documented for the sqlite3_busy_handler()
-     callback argument, minus the (void*) argument the C-level
-     function requires.
-
-     Any exceptions thrown by this callback are suppressed in order to
-     retain the C-style API semantics of the JNI bindings.
-  */
-  public abstract int xCallback(int n);
-
-  /**
-     Optionally override to perform any cleanup when this busy
-     handler is destroyed. It is destroyed when:
-
-     - The associated db is passed to sqlite3_close() or
-       sqlite3_close_v2().
-
-     - sqlite3_busy_handler() is called to replace the handler,
-       whether it's passed a null handler or any other instance of
-       this class.
-
-     - sqlite3_busy_timeout() is called, which implicitly installs
-       a busy handler.
-  */
-  public void xDestroy(){}
-}
@@ -1,27 +0,0 @@
-/*
-** 2023-08-04
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This file is part of the JNI bindings for the sqlite3 C API.
-*/
-package org.sqlite.jni;
-
-/**
-   Fts5Function is used in conjunction with the
-   sqlite3_create_fts_function() JNI-bound API to give that native code
-   access to the callback functions needed in order to implement
-   FTS5 auxiliary functions in Java.
-*/
-public abstract class Fts5Function {
-
-  public abstract void xFunction(Fts5ExtensionApi pApi, Fts5Context pFts,
-                                 sqlite3_context pCtx, sqlite3_value argv[]);
-  public void xDestroy() {}
-}
@@ -1,165 +0,0 @@
-/*
-** 2023-07-21
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This file is part of the JNI bindings for the sqlite3 C API.
-*/
-package org.sqlite.jni;
-
-/**
-   Helper classes for handling JNI output pointers.
-
-   We do not use a generic OutputPointer<T> because working with those
-   from the native JNI code is unduly quirky due to a lack of
-   autoboxing at that level.
-
-   The usage is similar for all of thes types:
-
-   ```
-   OutputPointer.sqlite3 out = new OutputPointer.sqlite3();
-   assert( null==out.get() );
-   int rc = sqlite3_open(":memory:", out);
-   if( 0!=rc ) ... error;
-   assert( null!=out.get() );
-   sqlite3 db = out.take();
-   assert( null==out.get() );
-   ```
-
-   With the minor exception that the primitive types permit direct
-   access to the object's value via the `value` property, whereas the
-   JNI-level opaque types do not permit client-level code to set that
-   property.
-*/
-public final class OutputPointer {
-
-  /**
-     Output pointer for use with routines, such as sqlite3_open(),
-     which return a database handle via an output pointer. These
-     pointers can only be set by the JNI layer, not by client-level
-     code.
-  */
-  public static final class sqlite3 {
-    private org.sqlite.jni.sqlite3 value;
-    //! Initializes with a null value.
-    public sqlite3(){value = null;}
-    //! Sets the current value to null.
-    public void clear(){value = null;}
-    //! Returns the current value.
-    public final org.sqlite.jni.sqlite3 get(){return value;}
-    //! Equivalent to calling get() then clear().
-    public final org.sqlite.jni.sqlite3 take(){
-      final org.sqlite.jni.sqlite3 v = value;
-      value = null;
-      return v;
-    }
-  }
-
-  /**
-     Output pointer for use with routines, such as sqlite3_prepare(),
-     which return a statement handle via an output pointer. These
-     pointers can only be set by the JNI layer, not by client-level
-     code.
-  */
-  public static final class sqlite3_stmt {
-    private org.sqlite.jni.sqlite3_stmt value;
-    //! Initializes with a null value.
-    public sqlite3_stmt(){value = null;}
-    //! Sets the current value to null.
-    public void clear(){value = null;}
-    //! Returns the current value.
-    public final org.sqlite.jni.sqlite3_stmt get(){return value;}
-    //! Equivalent to calling get() then clear().
-    public final org.sqlite.jni.sqlite3_stmt take(){
-      final org.sqlite.jni.sqlite3_stmt v = value;
-      value = null;
-      return v;
-    }
-  }
-
-  /**
-     Output pointer for use with native routines which return integers via
-     output pointers.
-  */
-  public static final class Int32 {
-    /**
-       This is public for ease of use. Accessors are provided for
-       consistency with the higher-level types.
-    */
-    public int value;
-    //! Initializes with the value 0.
-    public Int32(){this(0);}
-    //! Initializes with the value v.
-    public Int32(int v){value = v;}
-    //! Returns the current value.
-    public final int get(){return value;}
-    //! Sets the current value to v.
-    public final void set(int v){value = v;}
-  }
-
-  /**
-     Output pointer for use with native routines which return 64-bit integers
-     via output pointers.
-  */
-  public static final class Int64 {
-    /**
-       This is public for ease of use. Accessors are provided for
-       consistency with the higher-level types.
-    */
-    public long value;
-    //! Initializes with the value 0.
-    public Int64(){this(0);}
-    //! Initializes with the value v.
-    public Int64(long v){value = v;}
-    //! Returns the current value.
-    public final long get(){return value;}
-    //! Sets the current value.
-    public final void set(long v){value = v;}
-  }
-
-  /**
-     Output pointer for use with native routines which return strings via
-     output pointers.
-  */
-  public static final class String {
-    /**
-       This is public for ease of use. Accessors are provided for
-       consistency with the higher-level types.
-    */
-    public java.lang.String value;
-    //! Initializes with a null value.
-    public String(){this(null);}
-    //! Initializes with the value v.
-    public String(java.lang.String v){value = v;}
-    //! Returns the current value.
-    public final java.lang.String get(){return value;}
-    //! Sets the current value.
-    public final void set(java.lang.String v){value = v;}
-  }
-
-  /**
-     Output pointer for use with native routines which return byte
-     arrays via output pointers.
-  */
-  public static final class ByteArray {
-    /**
-       This is public for ease of use. Accessors are provided for
-       consistency with the higher-level types.
-    */
-    public byte[] value;
-    //! Initializes with the value null.
-    public ByteArray(){this(null);}
-    //! Initializes with the value v.
-    public ByteArray(byte[] v){value = v;}
-    //! Returns the current value.
-    public final byte[] get(){return value;}
-    //! Sets the current value.
-    public final void set(byte[] v){value = v;}
-  }
-}
@@ -1,155 +0,0 @@
-/*
-** 2023-07-21
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This file is part of the JNI bindings for the sqlite3 C API.
-*/
-package org.sqlite.jni;
-
-/**
-   This enum contains all of the core and "extended" result codes used
-   by the sqlite3 library. It is provided not for use with the C-style
-   API (with which it won't work) but for higher-level code which may
-   find it useful to map SQLite result codes to human-readable names.
-*/
-public enum ResultCode {
-  SQLITE_OK(SQLite3Jni.SQLITE_OK),
-  SQLITE_ERROR(SQLite3Jni.SQLITE_ERROR),
-  SQLITE_INTERNAL(SQLite3Jni.SQLITE_INTERNAL),
-  SQLITE_PERM(SQLite3Jni.SQLITE_PERM),
-  SQLITE_ABORT(SQLite3Jni.SQLITE_ABORT),
-  SQLITE_BUSY(SQLite3Jni.SQLITE_BUSY),
-  SQLITE_LOCKED(SQLite3Jni.SQLITE_LOCKED),
-  SQLITE_NOMEM(SQLite3Jni.SQLITE_NOMEM),
-  SQLITE_READONLY(SQLite3Jni.SQLITE_READONLY),
-  SQLITE_INTERRUPT(SQLite3Jni.SQLITE_INTERRUPT),
-  SQLITE_IOERR(SQLite3Jni.SQLITE_IOERR),
-  SQLITE_CORRUPT(SQLite3Jni.SQLITE_CORRUPT),
-  SQLITE_NOTFOUND(SQLite3Jni.SQLITE_NOTFOUND),
-  SQLITE_FULL(SQLite3Jni.SQLITE_FULL),
-  SQLITE_CANTOPEN(SQLite3Jni.SQLITE_CANTOPEN),
-  SQLITE_PROTOCOL(SQLite3Jni.SQLITE_PROTOCOL),
-  SQLITE_EMPTY(SQLite3Jni.SQLITE_EMPTY),
-  SQLITE_SCHEMA(SQLite3Jni.SQLITE_SCHEMA),
-  SQLITE_TOOBIG(SQLite3Jni.SQLITE_TOOBIG),
-  SQLITE_CONSTRAINT(SQLite3Jni.SQLITE_CONSTRAINT),
-  SQLITE_MISMATCH(SQLite3Jni.SQLITE_MISMATCH),
-  SQLITE_MISUSE(SQLite3Jni.SQLITE_MISUSE),
-  SQLITE_NOLFS(SQLite3Jni.SQLITE_NOLFS),
-  SQLITE_AUTH(SQLite3Jni.SQLITE_AUTH),
-  SQLITE_FORMAT(SQLite3Jni.SQLITE_FORMAT),
-  SQLITE_RANGE(SQLite3Jni.SQLITE_RANGE),
-  SQLITE_NOTADB(SQLite3Jni.SQLITE_NOTADB),
-  SQLITE_NOTICE(SQLite3Jni.SQLITE_NOTICE),
-  SQLITE_WARNING(SQLite3Jni.SQLITE_WARNING),
-  SQLITE_ROW(SQLite3Jni.SQLITE_ROW),
-  SQLITE_DONE(SQLite3Jni.SQLITE_DONE),
-  SQLITE_ERROR_MISSING_COLLSEQ(SQLite3Jni.SQLITE_ERROR_MISSING_COLLSEQ),
-  SQLITE_ERROR_RETRY(SQLite3Jni.SQLITE_ERROR_RETRY),
-  SQLITE_ERROR_SNAPSHOT(SQLite3Jni.SQLITE_ERROR_SNAPSHOT),
-  SQLITE_IOERR_READ(SQLite3Jni.SQLITE_IOERR_READ),
-  SQLITE_IOERR_SHORT_READ(SQLite3Jni.SQLITE_IOERR_SHORT_READ),
-  SQLITE_IOERR_WRITE(SQLite3Jni.SQLITE_IOERR_WRITE),
-  SQLITE_IOERR_FSYNC(SQLite3Jni.SQLITE_IOERR_FSYNC),
-  SQLITE_IOERR_DIR_FSYNC(SQLite3Jni.SQLITE_IOERR_DIR_FSYNC),
-  SQLITE_IOERR_TRUNCATE(SQLite3Jni.SQLITE_IOERR_TRUNCATE),
-  SQLITE_IOERR_FSTAT(SQLite3Jni.SQLITE_IOERR_FSTAT),
-  SQLITE_IOERR_UNLOCK(SQLite3Jni.SQLITE_IOERR_UNLOCK),
-  SQLITE_IOERR_RDLOCK(SQLite3Jni.SQLITE_IOERR_RDLOCK),
-  SQLITE_IOERR_DELETE(SQLite3Jni.SQLITE_IOERR_DELETE),
-  SQLITE_IOERR_BLOCKED(SQLite3Jni.SQLITE_IOERR_BLOCKED),
-  SQLITE_IOERR_NOMEM(SQLite3Jni.SQLITE_IOERR_NOMEM),
-  SQLITE_IOERR_ACCESS(SQLite3Jni.SQLITE_IOERR_ACCESS),
-  SQLITE_IOERR_CHECKRESERVEDLOCK(SQLite3Jni.SQLITE_IOERR_CHECKRESERVEDLOCK),
-  SQLITE_IOERR_LOCK(SQLite3Jni.SQLITE_IOERR_LOCK),
-  SQLITE_IOERR_CLOSE(SQLite3Jni.SQLITE_IOERR_CLOSE),
-  SQLITE_IOERR_DIR_CLOSE(SQLite3Jni.SQLITE_IOERR_DIR_CLOSE),
-  SQLITE_IOERR_SHMOPEN(SQLite3Jni.SQLITE_IOERR_SHMOPEN),
-  SQLITE_IOERR_SHMSIZE(SQLite3Jni.SQLITE_IOERR_SHMSIZE),
-  SQLITE_IOERR_SHMLOCK(SQLite3Jni.SQLITE_IOERR_SHMLOCK),
-  SQLITE_IOERR_SHMMAP(SQLite3Jni.SQLITE_IOERR_SHMMAP),
-  SQLITE_IOERR_SEEK(SQLite3Jni.SQLITE_IOERR_SEEK),
-  SQLITE_IOERR_DELETE_NOENT(SQLite3Jni.SQLITE_IOERR_DELETE_NOENT),
-  SQLITE_IOERR_MMAP(SQLite3Jni.SQLITE_IOERR_MMAP),
-  SQLITE_IOERR_GETTEMPPATH(SQLite3Jni.SQLITE_IOERR_GETTEMPPATH),
-  SQLITE_IOERR_CONVPATH(SQLite3Jni.SQLITE_IOERR_CONVPATH),
-  SQLITE_IOERR_VNODE(SQLite3Jni.SQLITE_IOERR_VNODE),
-  SQLITE_IOERR_AUTH(SQLite3Jni.SQLITE_IOERR_AUTH),
-  SQLITE_IOERR_BEGIN_ATOMIC(SQLite3Jni.SQLITE_IOERR_BEGIN_ATOMIC),
-  SQLITE_IOERR_COMMIT_ATOMIC(SQLite3Jni.SQLITE_IOERR_COMMIT_ATOMIC),
-  SQLITE_IOERR_ROLLBACK_ATOMIC(SQLite3Jni.SQLITE_IOERR_ROLLBACK_ATOMIC),
-  SQLITE_IOERR_DATA(SQLite3Jni.SQLITE_IOERR_DATA),
-  SQLITE_IOERR_CORRUPTFS(SQLite3Jni.SQLITE_IOERR_CORRUPTFS),
-  SQLITE_LOCKED_SHAREDCACHE(SQLite3Jni.SQLITE_LOCKED_SHAREDCACHE),
-  SQLITE_LOCKED_VTAB(SQLite3Jni.SQLITE_LOCKED_VTAB),
-  SQLITE_BUSY_RECOVERY(SQLite3Jni.SQLITE_BUSY_RECOVERY),
-  SQLITE_BUSY_SNAPSHOT(SQLite3Jni.SQLITE_BUSY_SNAPSHOT),
-  SQLITE_BUSY_TIMEOUT(SQLite3Jni.SQLITE_BUSY_TIMEOUT),
-  SQLITE_CANTOPEN_NOTEMPDIR(SQLite3Jni.SQLITE_CANTOPEN_NOTEMPDIR),
-  SQLITE_CANTOPEN_ISDIR(SQLite3Jni.SQLITE_CANTOPEN_ISDIR),
-  SQLITE_CANTOPEN_FULLPATH(SQLite3Jni.SQLITE_CANTOPEN_FULLPATH),
-  SQLITE_CANTOPEN_CONVPATH(SQLite3Jni.SQLITE_CANTOPEN_CONVPATH),
-  SQLITE_CANTOPEN_SYMLINK(SQLite3Jni.SQLITE_CANTOPEN_SYMLINK),
-  SQLITE_CORRUPT_VTAB(SQLite3Jni.SQLITE_CORRUPT_VTAB),
-  SQLITE_CORRUPT_SEQUENCE(SQLite3Jni.SQLITE_CORRUPT_SEQUENCE),
-  SQLITE_CORRUPT_INDEX(SQLite3Jni.SQLITE_CORRUPT_INDEX),
-  SQLITE_READONLY_RECOVERY(SQLite3Jni.SQLITE_READONLY_RECOVERY),
-  SQLITE_READONLY_CANTLOCK(SQLite3Jni.SQLITE_READONLY_CANTLOCK),
-  SQLITE_READONLY_ROLLBACK(SQLite3Jni.SQLITE_READONLY_ROLLBACK),
-  SQLITE_READONLY_DBMOVED(SQLite3Jni.SQLITE_READONLY_DBMOVED),
-  SQLITE_READONLY_CANTINIT(SQLite3Jni.SQLITE_READONLY_CANTINIT),
-  SQLITE_READONLY_DIRECTORY(SQLite3Jni.SQLITE_READONLY_DIRECTORY),
-  SQLITE_ABORT_ROLLBACK(SQLite3Jni.SQLITE_ABORT_ROLLBACK),
-  SQLITE_CONSTRAINT_CHECK(SQLite3Jni.SQLITE_CONSTRAINT_CHECK),
-  SQLITE_CONSTRAINT_COMMITHOOK(SQLite3Jni.SQLITE_CONSTRAINT_COMMITHOOK),
-  SQLITE_CONSTRAINT_FOREIGNKEY(SQLite3Jni.SQLITE_CONSTRAINT_FOREIGNKEY),
-  SQLITE_CONSTRAINT_FUNCTION(SQLite3Jni.SQLITE_CONSTRAINT_FUNCTION),
-  SQLITE_CONSTRAINT_NOTNULL(SQLite3Jni.SQLITE_CONSTRAINT_NOTNULL),
-  SQLITE_CONSTRAINT_PRIMARYKEY(SQLite3Jni.SQLITE_CONSTRAINT_PRIMARYKEY),
-  SQLITE_CONSTRAINT_TRIGGER(SQLite3Jni.SQLITE_CONSTRAINT_TRIGGER),
-  SQLITE_CONSTRAINT_UNIQUE(SQLite3Jni.SQLITE_CONSTRAINT_UNIQUE),
-  SQLITE_CONSTRAINT_VTAB(SQLite3Jni.SQLITE_CONSTRAINT_VTAB),
-  SQLITE_CONSTRAINT_ROWID(SQLite3Jni.SQLITE_CONSTRAINT_ROWID),
-  SQLITE_CONSTRAINT_PINNED(SQLite3Jni.SQLITE_CONSTRAINT_PINNED),
-  SQLITE_CONSTRAINT_DATATYPE(SQLite3Jni.SQLITE_CONSTRAINT_DATATYPE),
-  SQLITE_NOTICE_RECOVER_WAL(SQLite3Jni.SQLITE_NOTICE_RECOVER_WAL),
-  SQLITE_NOTICE_RECOVER_ROLLBACK(SQLite3Jni.SQLITE_NOTICE_RECOVER_ROLLBACK),
-  SQLITE_WARNING_AUTOINDEX(SQLite3Jni.SQLITE_WARNING_AUTOINDEX),
-  SQLITE_AUTH_USER(SQLite3Jni.SQLITE_AUTH_USER),
-  SQLITE_OK_LOAD_PERMANENTLY(SQLite3Jni.SQLITE_OK_LOAD_PERMANENTLY);
-
-  public final int value;
-
-  ResultCode(int rc){
-    value = rc;
-    ResultCodeMap.set(rc, this);
-  }
-
-  /**
-     Returns the entry from this enum for the given result code, or
-     null if no match is found.
-  */
-  public static ResultCode getEntryForInt(int rc){
-    return ResultCodeMap.get(rc);
-  }
-
-  /**
-     Internal level of indirection required because we cannot initialize
-     static enum members in an enum before the enum constructor is
-     invoked.
-  */
-  private static final class ResultCodeMap {
-    private static final java.util.Map<Integer,ResultCode> i2e
-      = new java.util.HashMap<>();
-    private static void set(int rc, ResultCode e){ i2e.put(rc, e); }
-    private static ResultCode get(int rc){ return i2e.get(rc); }
-  }
-
-}
@@ -1,172 +0,0 @@
-/*
-** 2023-07-22
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This file is part of the JNI bindings for the sqlite3 C API.
-*/
-package org.sqlite.jni;
-
-/**
-   SQLFunction is used in conjunction with the
-   sqlite3_create_function() JNI-bound API to give that native code
-   access to the callback functions needed in order to implement SQL
-   functions in Java.
-
-   This class is not used by itself, but is a marker base class. The
-   three UDF types are modelled by the inner classes Scalar,
-   Aggregate<T>, and Window<T>. Most simply, clients may create
-   anonymous classes from those to implement UDFs. Clients are free to
-   create their own classes for use with UDFs, so long as they conform
-   to the public interfaces defined by those three classes. The JNI
-   layer only actively relies on the SQLFunction base class.
-*/
-public abstract class SQLFunction {
-
-  /**
-     PerContextState assists aggregate and window functions in
-     managinga their accumulator state across calls to the UDF's
-     callbacks.
-
-     If a given aggregate or window function is called multiple times
-     in a single SQL statement, e.g. SELECT MYFUNC(A), MYFUNC(B)...,
-     then the clients need some way of knowing which call is which so
-     that they can map their state between their various UDF callbacks
-     and reset it via xFinal(). This class takes care of such
-     mappings.
-
-     This class works by mapping
-     sqlite3_context.getAggregateContext() to a single piece of
-     state, of a client-defined type (the T part of this class), which
-     persists across a "matching set" of the UDF's callbacks.
-
-     This class is a helper providing commonly-needed functionality -
-     it is not required for use with aggregate or window functions.
-     Client UDFs are free to perform such mappings using custom
-     approaches. The provided Aggregate<T> and Window<T> classes
-     use this.
-  */
-  public static final class PerContextState<T> {
-    private final java.util.Map<Long,ValueHolder<T>> map
-      = new java.util.HashMap<>();
-
-    /**
-       Should be called from a UDF's xStep(), xValue(), and xInverse()
-       methods, passing it that method's first argument and an initial
-       value for the persistent state. If there is currently no
-       mapping for cx.getAggregateContext() within the map, one is
-       created using the given initial value, else the existing one is
-       used and the 2nd argument is ignored.  It returns a
-       ValueHolder<T> which can be used to modify that state directly
-       without requiring that the client update the underlying map's
-       entry.
-
-       T must be of a type which can be legally stored as a value in
-       java.util.HashMap<KeyType,T>.
-    */
-    public ValueHolder<T> getAggregateState(sqlite3_context cx, T initialValue){
-      ValueHolder<T> rc = map.get(cx.getAggregateContext());
-      if(null == rc){
-        map.put(cx.getAggregateContext(), rc = new ValueHolder<>(initialValue));
-      }
-      return rc;
-    }
-
-    /**
-       Should be called from a UDF's xFinal() method and passed that
-       method's first argument. This function removes the value
-       associated with cx.getAggregateContext() from the map and
-       returns it, returning null if no other UDF method has been
-       called to set up such a mapping. The latter condition will be
-       the case if a UDF is used in a statement which has no result
-       rows.
-    */
-    public T takeAggregateState(sqlite3_context cx){
-      final ValueHolder<T> h = map.remove(cx.getAggregateContext());
-      return null==h ? null : h.value;
-    }
-  }
-
-  //! Subclass for creating scalar functions.
-  public static abstract class Scalar extends SQLFunction {
-
-    //! As for the xFunc() argument of the C API's sqlite3_create_function()
-    public abstract void xFunc(sqlite3_context cx, sqlite3_value[] args);
-
-    /**
-       Optionally override to be notified when the UDF is finalized by
-       SQLite.
-    */
-    public void xDestroy() {}
-  }
-
-  /**
-     SQLFunction Subclass for creating aggregate functions.  Its T is
-     the data type of its "accumulator" state, an instance of which is
-     intended to be be managed using the getAggregateState() and
-     takeAggregateState() methods.
-  */
-  public static abstract class Aggregate<T> extends SQLFunction {
-
-    //! As for the xStep() argument of the C API's sqlite3_create_function()
-    public abstract void xStep(sqlite3_context cx, sqlite3_value[] args);
-
-    //! As for the xFinal() argument of the C API's sqlite3_create_function()
-    public abstract void xFinal(sqlite3_context cx);
-
-    /**
-       Optionally override to be notified when the UDF is finalized by
-       SQLite.
-    */
-    public void xDestroy() {}
-
-    //! Per-invocation state for the UDF.
-    private final PerContextState<T> map = new PerContextState<>();
-
-    /**
-       To be called from the implementation's xStep() method, as well
-       as the xValue() and xInverse() methods of the Window<T>
-       subclass, to fetch the current per-call UDF state. On the
-       first call to this method for any given sqlite3_context
-       argument, the context is set to the given initial value. On all other
-       calls, the 2nd argument is ignored.
-
-       @see PerContextState<T>#takeAggregateState()
-    */
-    protected final ValueHolder<T> getAggregateState(sqlite3_context cx, T initialValue){
-      return map.getAggregateState(cx, initialValue);
-    }
-
-    /**
-       To be called from the implementation's xFinal() method to fetch
-       the final state of the UDF and remove its mapping.
-
-       @see PerContextState<T>#takeAggregateState()
-    */
-    protected final T takeAggregateState(sqlite3_context cx){
-      return map.takeAggregateState(cx);
-    }
-  }
-
-  /**
-     An SQLFunction subclass for creating window functions.  Note that
-     Window<T> inherits from Aggregate<T> and each instance is
-     required to implement the inherited abstract methods from that
-     class. See Aggregate<T> for information on managing the UDF's
-     invocation-specific state.
-  */
-  public static abstract class Window<T> extends Aggregate<T> {
-
-    //! As for the xInverse() argument of the C API's sqlite3_create_window_function()
-    public abstract void xInverse(sqlite3_context cx, sqlite3_value[] args);
-
-    //! As for the xValue() argument of the C API's sqlite3_create_window_function()
-    public abstract void xValue(sqlite3_context cx);
-  }
-}
@@ -1,87 +0,0 @@
-/*
-** 2023-08-04
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This file contains a set of tests for the sqlite3 JNI bindings.
-*/
-package org.sqlite.jni;
-import static org.sqlite.jni.SQLite3Jni.*;
-import static org.sqlite.jni.Tester1.*;
-
-public class TesterFts5 {
-
-  private static void test1(){
-    Fts5ExtensionApi fea = Fts5ExtensionApi.getInstance();
-    affirm( null != fea );
-    affirm( fea.getNativePointer() != 0 );
-    affirm( fea == Fts5ExtensionApi.getInstance() )/*singleton*/;
-
-    sqlite3 db = createNewDb();
-    fts5_api fApi = fts5_api.getInstanceForDb(db);
-    affirm( fApi != null );
-    affirm( fApi == fts5_api.getInstanceForDb(db) /* singleton per db */ );
-
-    execSql(db, new String[] {
-        "CREATE VIRTUAL TABLE ft USING fts5(a, b);",
-        "INSERT INTO ft(rowid, a, b) VALUES(1, 'X Y', 'Y Z');",
-        "INSERT INTO ft(rowid, a, b) VALUES(2, 'A Z', 'Y Y');"
-      });
-
-    final String pUserData = "This is pUserData";
-    ValueHolder<Boolean> xDestroyCalled = new ValueHolder<>(false);
-    ValueHolder<Integer> xFuncCount = new ValueHolder<>(0);
-    final fts5_extension_function func = new fts5_extension_function(){
-        public void xFunction(Fts5ExtensionApi ext, Fts5Context fCx,
-                              sqlite3_context pCx, sqlite3_value argv[]){
-          int nCols = ext.xColumnCount(fCx);
-          affirm( 2 == nCols );
-          affirm( nCols == argv.length );
-          affirm( ext.xUserData(fCx) == pUserData );
-          if(true){
-            OutputPointer.String op = new OutputPointer.String();
-            for(int i = 0; i < nCols; ++i ){
-              int rc = ext.xColumnText(fCx, i, op);
-              affirm( 0 == rc );
-              final String val = op.value;
-              affirm( val.equals(sqlite3_value_text(argv[i])) );
-              //outln("xFunction col "+i+": "+val);
-            }
-          }
-          ++xFuncCount.value;
-        }
-        public void xDestroy(){
-          xDestroyCalled.value = true;
-        }
-      };
-
-    int rc = fApi.xCreateFunction("myaux", pUserData, func);
-    affirm( 0==rc );
-
-    affirm( 0==xFuncCount.value );
-    execSql(db, "select myaux(ft,a,b) from ft;");
-    affirm( 2==xFuncCount.value );
-    affirm( !xDestroyCalled.value );
-    sqlite3_close_v2(db);
-    affirm( xDestroyCalled.value );
-  }
-
-  public TesterFts5(){
-    int oldAffirmCount = Tester1.affirmCount;
-    Tester1.affirmCount = 0;
-    final long timeStart = System.nanoTime();
-    test1();
-    final long timeEnd = System.nanoTime();
-    outln("FTS5 Tests done. Metrics:");
-    outln("\tAssertions checked: "+Tester1.affirmCount);
-    outln("\tTotal time = "
-          +((timeEnd - timeStart)/1000000.0)+"ms");
-    Tester1.affirmCount = oldAffirmCount;
-  }
-}
@@ -1,62 +0,0 @@
-/*
-** 2023-07-22
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-*************************************************************************
-** This file is part of the JNI bindings for the sqlite3 C API.
-*/
-package org.sqlite.jni;
-
-/**
-   Callback proxy for use with sqlite3_trace_v2().
-*/
-public interface Tracer {
-  /**
-     Achtung: this interface is subject to change because the current
-     approach to mapping the passed-in natives back to Java is
-     uncomfortably quirky.
-
-     Called by sqlite3 for various tracing operations, as per
-     sqlite3_trace_v2(). Note that this interface elides the 2nd
-     argument to the native trace callback, as that role is better
-     filled by instance-local state.
-
-     The 2nd argument to this function, if non-0, will be a native
-     pointer to either an sqlite3 or sqlite3_stmt object, depending on
-     the first argument (see below). Client code can pass it to the
-     sqlite3 resp. sqlite3_stmt constructor to create a wrapping
-     object, if necessary. This API does not do so by default because
-     tracing can be called frequently, creating such a wrapper for
-     each call is comparatively expensive, and the objects are
-     probably only seldom useful.
-
-     The final argument to this function is the "X" argument
-     documented for sqlite3_trace() and sqlite3_trace_v2(). Its type
-     depends on value of the first argument:
-
-     - SQLITE_TRACE_STMT: pNative is a sqlite3_stmt. pX is a string
-       containing the prepared SQL, with one caveat: JNI only provides
-       us with the ability to convert that string to MUTF-8, as
-       opposed to standard UTF-8, and is cannot be ruled out that that
-       difference may be significant for certain inputs. The
-       alternative would be that we first convert it to UTF-16 before
-       passing it on, but there's no readily-available way to do that
-       without calling back into the db to peform the conversion
-       (which would lead to further tracing).
-
-     - SQLITE_TRACE_PROFILE: pNative is a sqlite3_stmt. pX is a Long
-       holding an approximate number of nanoseconds the statement took
-       to run.
-
-     - SQLITE_TRACE_ROW: pNative is a sqlite3_stmt. pX is null.
-
-     - SQLITE_TRACE_CLOSE: pNative is a sqlite3. pX is null.
-  */
-  int xCallback(int traceFlag, Object pNative, Object pX);
-}
@@ -0,0 +1,30 @@
+/*
+** 2023-09-27
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file houses the Experimental annotation for the sqlite3 C API.
+*/
+package org.sqlite.jni.annotation;
+import java.lang.annotation.*;
+
+/**
+   This annotation is for flagging methods, constructors, and types
+   which are expressly experimental and subject to any amount of
+   change or outright removal. Client code should not rely on such
+   features.
+*/
+@Documented
+@Retention(RetentionPolicy.SOURCE)
+@Target({
+    ElementType.METHOD,
+    ElementType.CONSTRUCTOR,
+    ElementType.TYPE
+})
+public @interface Experimental{}
@@ -0,0 +1,71 @@
+/*
+** 2023-09-27
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file houses the NotNull annotation for the sqlite3 C API.
+*/
+package org.sqlite.jni.annotation;
+import java.lang.annotation.*;
+
+/**
+   This annotation is for flagging parameters which may not legally be
+   null or point to closed/finalized C-side resources.
+
+   <p>In the case of Java types which map directly to C struct types
+   (e.g. {@link org.sqlite.jni.capi.sqlite3}, {@link
+   org.sqlite.jni.capi.sqlite3_stmt}, and {@link
+   org.sqlite.jni.capi.sqlite3_context}), a closed/finalized resource
+   is also considered to be null for purposes this annotation because
+   the C-side effect of passing such a handle is the same as if null
+   is passed.</p>
+
+   <p>When used in the context of Java interfaces which are called
+   from the C APIs, this annotation communicates that the C API will
+   never pass a null value to the callback for that parameter.</p>
+
+   <p>Passing a null, for this annotation's definition of null, for
+   any parameter marked with this annoation specifically invokes
+   undefined behavior (see below).</p>
+
+   <p>Passing 0 (i.e. C NULL) or a negative value for any long-type
+   parameter marked with this annoation specifically invokes undefined
+   behavior (see below). Such values are treated as C pointers in the
+   JNI layer.</p>
+
+   <p><b>Undefined behaviour:</b> the JNI build uses the {@code
+   SQLITE_ENABLE_API_ARMOR} build flag, meaning that the C code
+   invoked with invalid NULL pointers and the like will not invoke
+   undefined behavior in the conventional C sense, but may, for
+   example, return result codes which are not documented for the
+   affected APIs or may otherwise behave unpredictably. In no known
+   cases will such arguments result in C-level code dereferencing a
+   NULL pointer or accessing out-of-bounds (or otherwise invalid)
+   memory. In other words, they may cause unexpected behavior but
+   should never cause an outright crash or security issue.</p>
+
+   <p>Note that the C-style API does not throw any exceptions on its
+   own because it has a no-throw policy in order to retain its C-style
+   semantics, but it may trigger NullPointerExceptions (or similar) if
+   passed a null for a parameter flagged with this annotation.</p>
+
+   <p>This annotation is informational only. No policy is in place to
+   programmatically ensure that NotNull is conformed to in client
+   code.</p>
+
+   <p>This annotation is solely for the use by the classes in the
+   org.sqlite.jni package and subpackages, but is made public so that
+   javadoc will link to it from the annotated functions. It is not
+   part of the public API and client-level code must not rely on
+   it.</p>
+*/
+@Documented
+@Retention(RetentionPolicy.SOURCE)
+@Target(ElementType.PARAMETER)
+public @interface NotNull{}
@@ -0,0 +1,33 @@
+/*
+** 2023-09-27
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file houses the Nullable annotation for the sqlite3 C API.
+*/
+package org.sqlite.jni.annotation;
+import java.lang.annotation.*;
+
+/**
+   This annotation is for flagging parameters which may legally be
+   null, noting that they may behave differently if passed null but
+   are prepared to expect null as a value. When used in the context of
+   callback methods which are called into from the C APIs, this
+   annotation communicates that the C API may pass a null value to the
+   callback.
+
+   <p>This annotation is solely for the use by the classes in this
+   package but is made public so that javadoc will link to it from the
+   annotated functions. It is not part of the public API and
+   client-level code must not rely on it.
+*/
+@Documented
+@Retention(RetentionPolicy.SOURCE)
+@Target(ElementType.PARAMETER)
+public @interface Nullable{}
@@ -0,0 +1,17 @@
+/*
+** 2023-09-27
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+*/
+/**
+   This package houses annotations specific to the JNI bindings of the
+   SQLite3 C API.
+*/
+package org.sqlite.jni.annotation;
@@ -0,0 +1,34 @@
+/*
+** 2023-08-25
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+import org.sqlite.jni.annotation.NotNull;
+
+/**
+   An implementation of {@link CollationCallback} which provides a
+   no-op xDestroy() method.
+*/
+public abstract class AbstractCollationCallback
+  implements CollationCallback, XDestroyCallback {
+  /**
+     Must compare the given byte arrays and return the result using
+     {@code memcmp()} semantics.
+  */
+  public abstract int call(@NotNull byte[] lhs, @NotNull byte[] rhs);
+
+  /**
+     Optionally override to be notified when the UDF is finalized by
+     SQLite. This implementation does nothing.
+  */
+  public void xDestroy(){}
+}
@@ -0,0 +1,138 @@
+/*
+** 2023-08-25
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+
+
+/**
+   A SQLFunction implementation for aggregate functions.  Its T is the
+   data type of its "accumulator" state, an instance of which is
+   intended to be be managed using the getAggregateState() and
+   takeAggregateState() methods.
+*/
+public abstract class AggregateFunction<T> implements SQLFunction {
+
+  /**
+     As for the xStep() argument of the C API's
+     sqlite3_create_function().  If this function throws, the
+     exception is not propagated and a warning might be emitted to a
+     debugging channel.
+  */
+  public abstract void xStep(sqlite3_context cx, sqlite3_value[] args);
+
+  /**
+     As for the xFinal() argument of the C API's sqlite3_create_function().
+     If this function throws, it is translated into an sqlite3_result_error().
+  */
+  public abstract void xFinal(sqlite3_context cx);
+
+  /**
+     Optionally override to be notified when the UDF is finalized by
+     SQLite.
+  */
+  public void xDestroy() {}
+
+  /**
+     PerContextState assists aggregate and window functions in
+     managing their accumulator state across calls to the UDF's
+     callbacks.
+
+     <p>T must be of a type which can be legally stored as a value in
+     java.util.HashMap<KeyType,T>.
+
+     <p>If a given aggregate or window function is called multiple times
+     in a single SQL statement, e.g. SELECT MYFUNC(A), MYFUNC(B)...,
+     then the clients need some way of knowing which call is which so
+     that they can map their state between their various UDF callbacks
+     and reset it via xFinal(). This class takes care of such
+     mappings.
+
+     <p>This class works by mapping
+     sqlite3_context.getAggregateContext() to a single piece of
+     state, of a client-defined type (the T part of this class), which
+     persists across a "matching set" of the UDF's callbacks.
+
+     <p>This class is a helper providing commonly-needed functionality
+     - it is not required for use with aggregate or window functions.
+     Client UDFs are free to perform such mappings using custom
+     approaches. The provided {@link AggregateFunction} and {@link
+     WindowFunction} classes use this.
+  */
+  public static final class PerContextState<T> {
+    private final java.util.Map<Long,ValueHolder<T>> map
+      = new java.util.HashMap<>();
+
+    /**
+       Should be called from a UDF's xStep(), xValue(), and xInverse()
+       methods, passing it that method's first argument and an initial
+       value for the persistent state. If there is currently no
+       mapping for the given context within the map, one is created
+       using the given initial value, else the existing one is used
+       and the 2nd argument is ignored.  It returns a ValueHolder<T>
+       which can be used to modify that state directly without
+       requiring that the client update the underlying map's entry.
+
+       <p>The caller is obligated to eventually call
+       takeAggregateState() to clear the mapping.
+    */
+    public ValueHolder<T> getAggregateState(sqlite3_context cx, T initialValue){
+      final Long key = cx.getAggregateContext(true);
+      ValueHolder<T> rc = null==key ? null : map.get(key);
+      if( null==rc ){
+        map.put(key, rc = new ValueHolder<>(initialValue));
+      }
+      return rc;
+    }
+
+    /**
+       Should be called from a UDF's xFinal() method and passed that
+       method's first argument. This function removes the value
+       associated with cx.getAggregateContext() from the map and
+       returns it, returning null if no other UDF method has been
+       called to set up such a mapping. The latter condition will be
+       the case if a UDF is used in a statement which has no result
+       rows.
+    */
+    public T takeAggregateState(sqlite3_context cx){
+      final ValueHolder<T> h = map.remove(cx.getAggregateContext(false));
+      return null==h ? null : h.value;
+    }
+  }
+
+  /** Per-invocation state for the UDF. */
+  private final PerContextState<T> map = new PerContextState<>();
+
+  /**
+     To be called from the implementation's xStep() method, as well
+     as the xValue() and xInverse() methods of the {@link WindowFunction}
+     subclass, to fetch the current per-call UDF state. On the
+     first call to this method for any given sqlite3_context
+     argument, the context is set to the given initial value. On all other
+     calls, the 2nd argument is ignored.
+
+     @see SQLFunction.PerContextState#getAggregateState
+  */
+  protected final ValueHolder<T> getAggregateState(sqlite3_context cx, T initialValue){
+    return map.getAggregateState(cx, initialValue);
+  }
+
+  /**
+     To be called from the implementation's xFinal() method to fetch
+     the final state of the UDF and remove its mapping.
+
+     see SQLFunction.PerContextState#takeAggregateState
+  */
+  protected final T takeAggregateState(sqlite3_context cx){
+    return map.takeAggregateState(cx);
+  }
+}
@@ -0,0 +1,29 @@
+/*
+** 2023-08-25
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+import org.sqlite.jni.annotation.*;
+
+/**
+   Callback for use with {@link CApi#sqlite3_set_authorizer}.
+*/
+public interface AuthorizerCallback extends CallbackProxy {
+  /**
+     Must function as described for the C-level
+     sqlite3_set_authorizer() callback. If it throws, the error is
+     converted to a db-level error and the exception is suppressed.
+  */
+  int call(int opId, @Nullable String s1, @Nullable String s2,
+           @Nullable String s3, @Nullable String s4);
+
+}
@@ -0,0 +1,40 @@
+/*
+** 2023-08-25
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+
+/**
+   Callback for use with the {@link CApi#sqlite3_auto_extension}
+   family of APIs.
+*/
+public interface AutoExtensionCallback extends CallbackProxy {
+  /**
+     Must function as described for a C-level
+     sqlite3_auto_extension() callback.
+
+     <p>This callback may throw and the exception's error message will
+     be set as the db's error string.
+
+     <p>Tips for implementations:
+
+     <p>- Opening a database from an auto-extension handler will lead to
+     an endless recursion of the auto-handler triggering itself
+     indirectly for each newly-opened database.
+
+     <p>- If this routine is stateful, it may be useful to make the
+     overridden method synchronized.
+
+     <p>- Results are undefined if the given db is closed by an auto-extension.
+  */
+  int call(sqlite3 db);
+}
@@ -0,0 +1,26 @@
+/*
+** 2023-08-25
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+
+/**
+   Callback for use with {@link CApi#sqlite3_busy_handler}.
+*/
+public interface BusyHandlerCallback extends CallbackProxy {
+  /**
+     Must function as documented for the C-level
+     sqlite3_busy_handler() callback argument, minus the (void*)
+     argument the C-level function requires.
+  */
+  int call(int n);
+}
@@ -0,0 +1,45 @@
+/*
+** 2023-08-25
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+/**
+   This marker interface exists soley for use as a documentation and
+   class-grouping tool. It should be applied to interfaces or
+   classes which have a call() method implementing some specific
+   callback interface on behalf of the C library.
+
+   <p>Unless very explicitely documented otherwise, callbacks must
+   never throw. Any which do throw but should not might trigger debug
+   output regarding the error, but the exception will not be
+   propagated.  For callback interfaces which support returning error
+   info to the core, the JNI binding will convert any exceptions to
+   C-level error information. For callback interfaces which do not
+   support returning error information, all exceptions will
+   necessarily be suppressed in order to retain the C-style no-throw
+   semantics and avoid invoking undefined behavior in the C layer.
+
+   <p>Callbacks of this style follow a common naming convention:
+
+   <p>1) They use the UpperCamelCase form of the C function they're
+   proxying for, minus the {@code sqlite3_} prefix, plus a {@code
+   Callback} suffix. e.g. {@code sqlite3_busy_handler()}'s callback is
+   named {@code BusyHandlerCallback}. Exceptions are made where that
+   would potentially be ambiguous, e.g. {@link ConfigSqllogCallback}
+   instead of {@code ConfigCallback} because the {@code
+   sqlite3_config()} interface may need to support more callback types
+   in the future.
+
+   <p>2) They all have a {@code call()} method but its signature is
+   callback-specific.
+*/
+public interface CallbackProxy {}
@@ -0,0 +1,35 @@
+/*
+** 2023-08-25
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+import org.sqlite.jni.annotation.NotNull;
+
+/**
+   Callback for use with {@link CApi#sqlite3_create_collation}.
+
+   @see AbstractCollationCallback
+*/
+public interface CollationCallback
+  extends CallbackProxy, XDestroyCallback {
+  /**
+     Must compare the given byte arrays and return the result using
+     {@code memcmp()} semantics.
+  */
+  int call(@NotNull byte[] lhs, @NotNull byte[] rhs);
+
+  /**
+     Called by SQLite when the collation is destroyed. If a collation
+     requires custom cleanup, override this method.
+  */
+  void xDestroy();
+}
@@ -1,5 +1,5 @@
 /*
-** 2023-07-30
+** 2023-08-25
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
@@ -11,18 +11,19 @@
 *************************************************************************
 ** This file is part of the JNI bindings for the sqlite3 C API.
 */
-package org.sqlite.jni;
+package org.sqlite.jni.capi;

 /**
-   Callback proxy for use with sqlite3_collation_needed().
+   Callback for use with {@link CApi#sqlite3_collation_needed}.
 */
-public interface CollationNeeded {
+public interface CollationNeededCallback extends CallbackProxy {
  /**
     Has the same semantics as the C-level sqlite3_create_collation()
     callback.

-     If it throws, the exception message is passed on to the db and
-     the exception is suppressed.
+     <p>Because the C API has no mechanism for reporting errors
+     from this callbacks, any exceptions thrown by this callback
+     are suppressed.
  */
-  int xCollationNeeded(sqlite3 db, int eTextRep, String collationName);
+  void call(sqlite3 db, int eTextRep, String collationName);
 }
@@ -1,5 +1,5 @@
 /*
-** 2023-07-22
+** 2023-08-25
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
@@ -11,15 +11,16 @@
 *************************************************************************
 ** This file is part of the JNI bindings for the sqlite3 C API.
 */
-package org.sqlite.jni;
+package org.sqlite.jni.capi;

 /**
-   Callback proxy for use with sqlite3_update_hook().
+   Callback for use with {@link CApi#sqlite3_commit_hook}.
 */
-public interface UpdateHook {
+public interface CommitHookCallback extends CallbackProxy {
  /**
-     Works as documented for the sqlite3_update_hook() callback.
-     Must not throw.
+     Works as documented for the C-level sqlite3_commit_hook()
+     callback. If it throws, the exception is translated into
+     a db-level error.
  */
-  void xUpdateHook(int opId, String dbName, String tableName, long rowId);
+  int call();
 }
@@ -1,5 +1,5 @@
 /*
-** 2023-07-22
+** 2023-08-23
 **
 ** The author disclaims copyright to this source code.  In place of
 ** a legal notice, here is a blessing:
@@ -11,15 +11,15 @@
 *************************************************************************
 ** This file is part of the JNI bindings for the sqlite3 C API.
 */
-package org.sqlite.jni;
+package org.sqlite.jni.capi;

 /**
-   Callback proxy for use with sqlite3_rollback_hook().
+   A callback for use with sqlite3_config().
 */
-public interface RollbackHook {
+public interface ConfigLogCallback {
  /**
-     Works as documented for the sqlite3_rollback_hook() callback.
-     Must not throw.
+     Must function as described for a C-level callback for
+     {@link CApi#sqlite3_config(ConfigLogCallback)}, with the slight signature change.
  */
-  void xRollbackHook();
+  void call(int errCode, String msg);
 }
@@ -0,0 +1,25 @@
+/*
+** 2023-08-23
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the JNI bindings for the sqlite3 C API.
+*/
+package org.sqlite.jni.capi;
+
+/**
+   A callback for use with sqlite3_config().
+*/
+public interface ConfigSqlLogCallback {
+  /**
+     Must function as described for a C-level callback for
+     {@link CApi#sqlite3_config(ConfigSqlLogCallback)}, with the slight signature change.
+  */
+  void call(sqlite3 db, String msg, int msgType );
+}
@@ -11,7 +11,7 @@
 *************************************************************************
 ** This file is part of the JNI bindings for the sqlite3 C API.
 */
-package org.sqlite.jni;
+package org.sqlite.jni.capi;

 /**
   A helper for passing pointers between JNI C code and Java, in
@@ -23,11 +23,24 @@ package org.sqlite.jni;
   NativePointerHolder is not inadvertently passed to an incompatible
   function signature.

-   These objects do not _own_ the pointer they refer to.  They are
+   These objects do not own the pointer they refer to.  They are
   intended simply to communicate that pointer between C and Java.
 */
 public class NativePointerHolder<ContextType> {
  //! Only set from JNI, where access permissions don't matter.
-  private long nativePointer = 0;
+  private volatile long nativePointer = 0;
+  /**
+     For use ONLY by package-level APIs which act as proxies for
+     close/finalize operations. Such ops must call this to zero out
+     the pointer so that this object is not carrying a stale
+     pointer. This function returns the prior value of the pointer and
+     sets it to 0.
+  */
+  final long clearNativePointer() {
+    final long rv = nativePointer;
+    nativePointer= 0;
+    return rv;
+  }
+
  public final long getNativePointer(){ return nativePointer; }
 }
--- a/Show More
+++ b/Show More
@@ -1 +1 @@
 .43.0
 .45.0