Make the "unref" operation on database pages a method which is part of the

PgHdr object.  This is a potential performance optimization.

FossilOrigin-Name: 37c5c8aa0ed93a39169f26297c4c145224e7f612260aed010c2e6add5a99d03d

Makefile.in

@@ -57,7 +57,6 @@ LIBTCL = @TCL_LIB_SPEC@
 #
 READLINE_FLAGS = -DHAVE_READLINE=@TARGET_HAVE_READLINE@ @TARGET_READLINE_INC@
 READLINE_FLAGS += -DHAVE_EDITLINE=@TARGET_HAVE_EDITLINE@
-READLINE_FLAGS += -DHAVE_LINENOISE=@TARGET_HAVE_LINENOISE@
 
 # The library that programs using readline() must link against.
 #
@@ -121,6 +120,10 @@ TCLSH_CMD = @TCLSH_CMD@
 #
 TCLLIBDIR = @TCLLIBDIR@
 
+# The suffix used on shared libraries.  Ex:  ".dll", ".so", ".dylib"
+#
+SHLIB_SUFFIX = @TCL_SHLIB_SUFFIX@
+
 # If gcov support was enabled by the configure script, add the appropriate
 # flags here.  It's not always as easy as just having the user add the right
 # CFLAGS / LDFLAGS, because libtool wants to use CFLAGS when linking, which
@@ -414,8 +417,6 @@ TESTSRC = \
   $(TOP)/ext/recover/sqlite3recover.c \
   $(TOP)/ext/recover/dbdata.c \
   $(TOP)/ext/recover/test_recover.c \
-  $(TOP)/ext/intck/test_intck.c  \
-  $(TOP)/ext/intck/sqlite3intck.c \
   $(TOP)/ext/rbu/test_rbu.c
 
 # Statically linked extensions
@@ -445,12 +446,10 @@ TESTSRC += \
   $(TOP)/ext/misc/percentile.c \
   $(TOP)/ext/misc/prefixes.c \
   $(TOP)/ext/misc/qpvtab.c \
-  $(TOP)/ext/misc/randomjson.c \
   $(TOP)/ext/misc/regexp.c \
   $(TOP)/ext/misc/remember.c \
   $(TOP)/ext/misc/series.c \
   $(TOP)/ext/misc/spellfix.c \
-  $(TOP)/ext/misc/stmtrand.c \
   $(TOP)/ext/misc/totype.c \
   $(TOP)/ext/misc/unionvtab.c \
   $(TOP)/ext/misc/wholenumber.c \
@@ -600,48 +599,26 @@ SHELL_OPT += -DSQLITE_ENABLE_DBPAGE_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB
 SHELL_OPT += -DSQLITE_ENABLE_OFFSET_SQL_FUNC
-SHELL_OPT += -DSQLITE_STRICT_SUBTYPE=1
-FUZZERSHELL_OPT =
+FUZZERSHELL_OPT = 
 FUZZCHECK_OPT += -I$(TOP)/test
 FUZZCHECK_OPT += -I$(TOP)/ext/recover
-FUZZCHECK_OPT += \
-  -DSQLITE_OSS_FUZZ \
-  -DSQLITE_ENABLE_BYTECODE_VTAB \
-  -DSQLITE_ENABLE_DBPAGE_VTAB \
-  -DSQLITE_ENABLE_DBSTAT_VTAB \
-  -DSQLITE_ENABLE_BYTECODE_VTAB \
-  -DSQLITE_ENABLE_DESERIALIZE \
-  -DSQLITE_ENABLE_EXPLAIN_COMMENTS \
-  -DSQLITE_ENABLE_FTS3_PARENTHESIS \
-  -DSQLITE_ENABLE_FTS4 \
-  -DSQLITE_ENABLE_FTS5 \
-  -DSQLITE_ENABLE_GEOPOLY \
-  -DSQLITE_ENABLE_MATH_FUNCTIONS \
-  -DSQLITE_ENABLE_MEMSYS5 \
-  -DSQLITE_ENABLE_NORMALIZE \
-  -DSQLITE_ENABLE_OFFSET_SQL_FUNC \
-  -DSQLITE_ENABLE_PREUPDATE_HOOK \
-  -DSQLITE_ENABLE_RTREE \
-  -DSQLITE_ENABLE_SESSION \
-  -DSQLITE_ENABLE_STMTVTAB \
-  -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION \
-  -DSQLITE_ENABLE_STAT4 \
-  -DSQLITE_ENABLE_STMT_SCANSTATUS \
-  -DSQLITE_MAX_MEMORY=50000000 \
-  -DSQLITE_MAX_MMAP_SIZE=0 \
-  -DSQLITE_OMIT_LOAD_EXTENSION \
-  -DSQLITE_PRINTF_PRECISION_LIMIT=1000 \
-  -DSQLITE_PRIVATE="" \
-  -DSQLITE_STRICT_SUBTYPE=1 \
-  -DSQLITE_STATIC_RANDOMJSON
-
+FUZZCHECK_OPT += -DSQLITE_OMIT_LOAD_EXTENSION
+FUZZCHECK_OPT += -DSQLITE_ENABLE_MEMSYS5 -DSQLITE_OSS_FUZZ
+FUZZCHECK_OPT += -DSQLITE_MAX_MEMORY=50000000
+FUZZCHECK_OPT += -DSQLITE_PRINTF_PRECISION_LIMIT=1000
+FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS4
+FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS3_PARENTHESIS
+FUZZCHECK_OPT += -DSQLITE_ENABLE_FTS5
+FUZZCHECK_OPT += -DSQLITE_ENABLE_RTREE
+FUZZCHECK_OPT += -DSQLITE_ENABLE_GEOPOLY
+FUZZCHECK_OPT += -DSQLITE_ENABLE_DBSTAT_VTAB
+FUZZCHECK_OPT += -DSQLITE_ENABLE_BYTECODE_VTAB
 FUZZCHECK_SRC += $(TOP)/test/fuzzcheck.c
 FUZZCHECK_SRC += $(TOP)/test/ossfuzz.c
 FUZZCHECK_SRC += $(TOP)/test/fuzzinvariants.c
 FUZZCHECK_SRC += $(TOP)/ext/recover/dbdata.c
 FUZZCHECK_SRC += $(TOP)/ext/recover/sqlite3recover.c
 FUZZCHECK_SRC += $(TOP)/test/vt02.c
-FUZZCHECK_SRC += $(TOP)/ext/misc/randomjson.c
 DBFUZZ_OPT =
 ST_OPT = -DSQLITE_OS_KV_OPTIONAL
 
@@ -695,12 +672,6 @@ srcck1$(BEXE):	$(TOP)/tool/srcck1.c
 sourcetest:	srcck1$(BEXE) sqlite3.c
 	./srcck1 sqlite3.c
 
-src-verify:	$(TOP)/tool/src-verify.c
-	$(BCC) -o src-verify$(BEXE) $(TOP)/tool/src-verify.c
-
-verify-source:	./src-verify
-	./src-verify $(TOP)
-
 fuzzershell$(TEXE):	$(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(FUZZERSHELL_OPT) \
 	  $(TOP)/tool/fuzzershell.c sqlite3.c $(TLIBS)
@@ -708,27 +679,6 @@ fuzzershell$(TEXE):	$(TOP)/tool/fuzzershell.c sqlite3.c sqlite3.h
 fuzzcheck$(TEXE):	$(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP)
 	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)
 
-fuzzcheck-asan$(TEXE):	$(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP)
-	$(LTLINK) -o $@ -fsanitize=address $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)
-
-fuzzcheck-ubsan$(TEXE):	$(FUZZCHECK_SRC) sqlite3.c sqlite3.h $(FUZZCHECK_DEP)
-	$(LTLINK) -o $@ -fsanitize=undefined $(FUZZCHECK_OPT) $(FUZZCHECK_SRC) sqlite3.c $(TLIBS)
-
-# Usage:    FUZZDB=filename make run-fuzzcheck
-#
-# Where filename is a fuzzcheck database, this target builds and runs
-# fuzzcheck, fuzzcheck-asan, and fuzzcheck-ubsan on that database.
-#
-# FUZZDB can be a glob pattern of two or more databases. Example:
-#
-#     FUZZDB=test/fuzzdata*.db make run-fuzzcheck
-#
-run-fuzzcheck:	fuzzcheck$(TEXE) fuzzcheck-asan$(TEXE) fuzzcheck-ubsan$(TEXE)
-	@if test "$(FUZZDB)" = ""; then echo 'ERROR: No FUZZDB specified. Rerun with FUZZDB=filename'; exit 1; fi
-	./fuzzcheck$(TEXE) --spinner $(FUZZDB)
-	./fuzzcheck-asan$(TEXE) --spinner $(FUZZDB)
-	./fuzzcheck-ubsan$(TEXE) --spinner $(FUZZDB)
-
 ossshell$(TEXE):	$(TOP)/test/ossfuzz.c $(TOP)/test/ossshell.c sqlite3.c sqlite3.h
 	$(LTLINK) -o $@ $(FUZZCHECK_OPT) $(TOP)/test/ossshell.c \
              $(TOP)/test/ossfuzz.c sqlite3.c $(TLIBS)
@@ -788,26 +738,13 @@ mptest:	mptester$(TEXE)
 	$(MPTEST2) --journalmode DELETE
 
 
-has_tclsh84:
-	sh $(TOP)/tool/cktclsh.sh 8.4 $(TCLSH_CMD)
-	touch has_tclsh84
-
-has_tclsh85:
-	sh $(TOP)/tool/cktclsh.sh 8.5 $(TCLSH_CMD)
-	touch has_tclsh85
-
-has_tclconfig:
-	@ if test x"$(HAVE_TCL)" != "x1"; then	echo 'ERROR: Requires access to "tclConfig.sh" which "configure" was not able to locate'; exit 1; fi
-	touch has_tclconfig
-
-
 # This target creates a directory named "tsrc" and fills it with
 # copies of all of the C source code and header files needed to
 # build on the target system.  Some of the C source code and header
 # files are automatically generated.  This target takes care of
 # all that automatic generation.
 #
-.target_source:	$(SRC) $(TOP)/tool/vdbe-compress.tcl has_tclsh84 fts5.c
+.target_source:	$(SRC) $(TOP)/tool/vdbe-compress.tcl fts5.c
 	rm -rf tsrc
 	mkdir tsrc
 	cp -f $(SRC) tsrc
@@ -817,19 +754,19 @@ has_tclconfig:
 	cp fts5.c fts5.h tsrc
 	touch .target_source
 
-sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl src-verify has_tclsh84
-	$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl $(AMALGAMATION_LINE_MACROS) $(EXTRA_SRC)
+sqlite3.c:	.target_source $(TOP)/tool/mksqlite3c.tcl
+	$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl $(AMALGAMATION_LINE_MACROS)
 	cp tsrc/sqlite3ext.h .
 	cp $(TOP)/ext/session/sqlite3session.h .
 
-sqlite3r.h: sqlite3.h has_tclsh84
+sqlite3r.h: sqlite3.h
 	$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) --enable-recover >sqlite3r.h
 
-sqlite3r.c: sqlite3.c sqlite3r.h has_tclsh84
+sqlite3r.c: sqlite3.c sqlite3r.h
 	cp $(TOP)/ext/recover/sqlite3recover.c tsrc/
 	cp $(TOP)/ext/recover/sqlite3recover.h tsrc/
 	cp $(TOP)/ext/recover/dbdata.c tsrc/
-	$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl --enable-recover $(AMALGAMATION_LINE_MACROS) $(EXTRA_SRC)
+	$(TCLSH_CMD) $(TOP)/tool/mksqlite3c.tcl --enable-recover $(AMALGAMATION_LINE_MACROS)
 
 sqlite3ext.h:	.target_source
 	cp tsrc/sqlite3ext.h .
@@ -840,7 +777,7 @@ tclsqlite3.c:	sqlite3.c
 	echo '#endif /* USE_SYSTEM_SQLITE */' >>tclsqlite3.c
 	cat $(TOP)/src/tclsqlite.c >>tclsqlite3.c
 
-sqlite3-all.c:	sqlite3.c $(TOP)/tool/split-sqlite3c.tcl has_tclsh84
+sqlite3-all.c:	sqlite3.c $(TOP)/tool/split-sqlite3c.tcl
 	$(TCLSH_CMD) $(TOP)/tool/split-sqlite3c.tcl
 
 # Rule to build the amalgamation
@@ -1122,16 +1059,16 @@ tclsqlite-shell.lo:	$(TOP)/src/tclsqlite.c $(HDR)
 tclsqlite-stubs.lo:	$(TOP)/src/tclsqlite.c $(HDR)
 	$(LTCOMPILE) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c
 
-tclsqlite3$(TEXE):	has_tclconfig tclsqlite-shell.lo libsqlite3.la
+tclsqlite3$(TEXE):	tclsqlite-shell.lo libsqlite3.la
 	$(LTLINK) -o $@ tclsqlite-shell.lo \
 		 libsqlite3.la $(LIBTCL)
 
 # Rules to build opcodes.c and opcodes.h
 #
-opcodes.c:	opcodes.h $(TOP)/tool/mkopcodec.tcl has_tclsh84
+opcodes.c:	opcodes.h $(TOP)/tool/mkopcodec.tcl
 	$(TCLSH_CMD) $(TOP)/tool/mkopcodec.tcl opcodes.h >opcodes.c
 
-opcodes.h:	parse.h $(TOP)/src/vdbe.c $(TOP)/tool/mkopcodeh.tcl has_tclsh84
+opcodes.h:	parse.h $(TOP)/src/vdbe.c $(TOP)/tool/mkopcodeh.tcl
 	cat parse.h $(TOP)/src/vdbe.c | $(TCLSH_CMD) $(TOP)/tool/mkopcodeh.tcl >opcodes.h
 
 # Rules to build parse.c and parse.h - the outputs of lemon.
@@ -1142,10 +1079,10 @@ parse.c:	$(TOP)/src/parse.y lemon$(BEXE)
 	cp $(TOP)/src/parse.y .
 	./lemon$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) -S parse.y
 
-sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid$(BEXE) $(TOP)/VERSION has_tclsh84
+sqlite3.h:	$(TOP)/src/sqlite.h.in $(TOP)/manifest mksourceid$(BEXE) $(TOP)/VERSION
 	$(TCLSH_CMD) $(TOP)/tool/mksqlite3h.tcl $(TOP) >sqlite3.h
 
-sqlite3rc.h:	$(TOP)/src/sqlite3.rc $(TOP)/VERSION has_tclsh84
+sqlite3rc.h:	$(TOP)/src/sqlite3.rc $(TOP)/VERSION
 	echo '#ifndef SQLITE_RESOURCE_VERSION' >$@
 	echo -n '#define SQLITE_RESOURCE_VERSION ' >>$@
 	cat $(TOP)/VERSION | $(TCLSH_CMD) $(TOP)/tool/replace.tcl exact . , >>$@
@@ -1155,37 +1092,32 @@ keywordhash.h:	$(TOP)/tool/mkkeywordhash.c
 	$(BCC) -o mkkeywordhash$(BEXE) $(OPT_FEATURE_FLAGS) $(OPTS) $(TOP)/tool/mkkeywordhash.c
 	./mkkeywordhash$(BEXE) >keywordhash.h
 
-# Source and header files that shell.c depends on
-SHELL_DEP = \
-    $(TOP)/src/shell.c.in \
-    $(TOP)/ext/consio/console_io.c \
-    $(TOP)/ext/consio/console_io.h \
-    $(TOP)/ext/expert/sqlite3expert.c \
-    $(TOP)/ext/expert/sqlite3expert.h \
-    $(TOP)/ext/intck/sqlite3intck.c \
-    $(TOP)/ext/intck/sqlite3intck.h \
-    $(TOP)/ext/misc/appendvfs.c \
-    $(TOP)/ext/misc/base64.c \
-    $(TOP)/ext/misc/base85.c \
-    $(TOP)/ext/misc/completion.c \
-    $(TOP)/ext/misc/decimal.c \
-    $(TOP)/ext/misc/fileio.c \
-    $(TOP)/ext/misc/ieee754.c \
-    $(TOP)/ext/misc/memtrace.c \
-    $(TOP)/ext/misc/pcachetrace.c \
-    $(TOP)/ext/misc/regexp.c \
-    $(TOP)/ext/misc/series.c \
-    $(TOP)/ext/misc/shathree.c \
-    $(TOP)/ext/misc/sqlar.c \
-    $(TOP)/ext/misc/uint.c \
-    $(TOP)/ext/misc/zipfile.c \
-    $(TOP)/ext/recover/dbdata.c \
-    $(TOP)/ext/recover/sqlite3recover.c \
-    $(TOP)/ext/recover/sqlite3recover.h \
-    $(TOP)/src/test_windirent.c \
-    $(TOP)/src/test_windirent.h
+# Source files that go into making shell.c
+SHELL_SRC = \
+	$(TOP)/src/shell.c.in \
+        $(TOP)/ext/misc/appendvfs.c \
+	$(TOP)/ext/misc/completion.c \
+        $(TOP)/ext/misc/decimal.c \
+        $(TOP)/ext/misc/basexx.c \
+        $(TOP)/ext/misc/base64.c \
+        $(TOP)/ext/misc/base85.c \
+	$(TOP)/ext/misc/fileio.c \
+        $(TOP)/ext/misc/ieee754.c \
+        $(TOP)/ext/misc/regexp.c \
+        $(TOP)/ext/misc/series.c \
+	$(TOP)/ext/misc/shathree.c \
+	$(TOP)/ext/misc/sqlar.c \
+        $(TOP)/ext/misc/uint.c \
+	$(TOP)/ext/expert/sqlite3expert.c \
+	$(TOP)/ext/expert/sqlite3expert.h \
+	$(TOP)/ext/misc/zipfile.c \
+	$(TOP)/ext/misc/memtrace.c \
+	$(TOP)/ext/recover/dbdata.c \
+	$(TOP)/ext/recover/sqlite3recover.c \
+	$(TOP)/ext/recover/sqlite3recover.h \
+        $(TOP)/src/test_windirent.c
 
-shell.c:	$(SHELL_DEP) $(TOP)/tool/mkshellc.tcl has_tclsh84
+shell.c:	$(SHELL_SRC) $(TOP)/tool/mkshellc.tcl
 	$(TCLSH_CMD) $(TOP)/tool/mkshellc.tcl >shell.c
 
 
@@ -1273,7 +1205,7 @@ fts5parse.c:	$(TOP)/ext/fts5/fts5parse.y lemon$(BEXE)
 
 fts5parse.h: fts5parse.c
 
-fts5.c: $(FTS5_SRC) has_tclsh84
+fts5.c: $(FTS5_SRC)
 	$(TCLSH_CMD) $(TOP)/ext/fts5/tool/mkfts5c.tcl
 	cp $(TOP)/ext/fts5/fts5.h .
 
@@ -1301,21 +1233,19 @@ TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_STMTVTAB
 TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_DBPAGE_VTAB
 TESTFIXTURE_FLAGS += -DSQLITE_ENABLE_BYTECODE_VTAB
 TESTFIXTURE_FLAGS += -DSQLITE_CKSUMVFS_STATIC
-TESTFIXTURE_FLAGS += -DSQLITE_STATIC_RANDOMJSON
-TESTFIXTURE_FLAGS += -DSQLITE_STRICT_SUBTYPE=1
 
 TESTFIXTURE_SRC0 = $(TESTSRC2) libsqlite3.la
 TESTFIXTURE_SRC1 = sqlite3.c
 TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c
 TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION))
 
-testfixture$(TEXE):	has_tclconfig has_tclsh85 $(TESTFIXTURE_SRC)
+testfixture$(TEXE):	$(TESTFIXTURE_SRC)
 	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \
 		-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)
 
-coretestprogs:	testfixture$(BEXE) sqlite3$(BEXE)
+coretestprogs:	$(TESTPROGS)
 
-testprogs:	$(TESTPROGS) srcck1$(BEXE) fuzzcheck$(TEXE) sessionfuzz$(TEXE)
+testprogs:	coretestprogs srcck1$(BEXE) fuzzcheck$(TEXE) sessionfuzz$(TEXE)
 
 # A very detailed test running most or all test cases
 fulltest:	alltest fuzztest
@@ -1333,17 +1263,11 @@ fulltestonly:	$(TESTPROGS) fuzztest
 	./testfixture$(TEXE) $(TOP)/test/full.test
 
 # Fuzz testing
-#
-# WARNING: When the "fuzztest" target is run by the testrunner.tcl script,
-# it does not actually run this code. Instead, it schedules equivalent 
-# commands. Therefore, if this target is updated, then code in
-# testrunner_data.tcl (search for "trd_fuzztest_data") must also be updated.
-#
-fuzztest:	fuzzcheck$(TEXE) $(FUZZDATA) sessionfuzz$(TEXE)
+fuzztest:	fuzzcheck$(TEXE) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessionfuzz-data1.db
 	./fuzzcheck$(TEXE) $(FUZZDATA)
 	./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db
 
-valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA) sessionfuzz$(TEXE)
+valgrindfuzz:	fuzzcheck$(TEXT) $(FUZZDATA) sessionfuzz$(TEXE) $(TOP)/test/sessionfuzz-data1.db
 	valgrind ./fuzzcheck$(TEXE) --cell-size-check --limit-mem 10M $(FUZZDATA)
 	valgrind ./sessionfuzz$(TEXE) run $(TOP)/test/sessionfuzz-data1.db
 
@@ -1360,23 +1284,11 @@ testrunner:	testfixture$(TEXE)
 
 # Runs both fuzztest and testrunner, consecutively.
 #
-devtest:	srctree-check testfixture$(TEXE) fuzztest testrunner
-
-mdevtest: srctree-check has_tclsh85
-	$(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest
-
-sdevtest: has_tclsh85
-	$(TCLSH_CMD) $(TOP)/test/testrunner.tcl sdevtest
-
-# Validate that various generated files in the source tree
-# are up-to-date.
-#
-srctree-check:	$(TOP)/tool/srctree-check.tcl
-	$(TCLSH_CMD) $(TOP)/tool/srctree-check.tcl
+devtest:	testfixture$(TEXE) fuzztest testrunner
 
 # Testing for a release
 #
-releasetest: srctree-check testfixture$(TEXE)
+releasetest: testfixture$(TEXE)
 	./testfixture$(TEXE) $(TOP)/test/testrunner.tcl release
 
 # Minimal testing that runs in less than 3 minutes
@@ -1387,7 +1299,7 @@ quicktest:	./testfixture$(TEXE)
 # This is the common case.  Run many tests that do not take too long,
 # including fuzzcheck, sqlite3_analyzer, and sqldiff tests.
 #
-test:	srctree-check fuzztest sourcetest $(TESTPROGS) tcltest
+test:	fuzztest sourcetest $(TESTPROGS) tcltest
 
 # Run a test using valgrind.  This can take a really long time
 # because valgrind is so much slower than a native machine.
@@ -1405,16 +1317,16 @@ smoketest:	$(TESTPROGS) fuzzcheck$(TEXE)
 shelltest: $(TESTPROGS)
 	./testfixture$(TEXT) $(TOP)/test/permutations.test shell
 
-sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in has_tclsh85
+sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in
 	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c
 
-sqlite3_analyzer$(TEXE): has_tclconfig sqlite3_analyzer.c
+sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
 	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
 
-sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in has_tclsh85
+sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in
 	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c
 
-sqltclsh$(TEXE): has_tclconfig sqltclsh.c
+sqltclsh$(TEXE): sqltclsh.c
 	$(LTLINK) sqltclsh.c -o $@ $(LIBTCL) $(TLIBS)
 
 sqlite3_expert$(TEXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
@@ -1430,10 +1342,10 @@ CHECKER_DEPS =\
   $(TOP)/ext/misc/btreeinfo.c \
   $(TOP)/ext/repair/sqlite3_checker.c.in
 
-sqlite3_checker.c:	$(CHECKER_DEPS) has_tclsh85
+sqlite3_checker.c:	$(CHECKER_DEPS)
 	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@
 
-sqlite3_checker$(TEXE):	has_tclconfig sqlite3_checker.c
+sqlite3_checker$(TEXE):	sqlite3_checker.c
 	$(LTLINK) sqlite3_checker.c -o $@ $(LIBTCL) $(TLIBS)
 
 dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
@@ -1516,11 +1428,6 @@ amalgamation-tarball: sqlite3.c sqlite3rc.h
 snapshot-tarball: sqlite3.c sqlite3rc.h
 	TOP=$(TOP) sh $(TOP)/tool/mkautoconfamal.sh --snapshot
 
-# Build a ZIP archive containing various command-line tools.
-#
-tool-zip:	testfixture sqlite3 sqldiff sqlite3_analyzer $(TOP)/tool/mktoolzip.tcl
-	./testfixture $(TOP)/tool/mktoolzip.tcl
-
 # The next two rules are used to support the "threadtest" target. Building
 # threadtest runs a few thread-safety tests that are implemented in C. This
 # target is invoked by the releasetest.tcl script.
@@ -1558,37 +1465,12 @@ install:	sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_instal
 
 pkgIndex.tcl:
 	echo 'package ifneeded sqlite3 $(RELEASE) [list load [file join $$dir libtclsqlite3[info sharedlibextension]] sqlite3]' > $@
-
 tcl_install:	lib_install libtclsqlite3.la pkgIndex.tcl
 	$(INSTALL) -d $(DESTDIR)$(TCLLIBDIR)
 	$(LTINSTALL) libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)
 	rm -f $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.a
 	$(INSTALL) -m 0644 pkgIndex.tcl $(DESTDIR)$(TCLLIBDIR)
 
-# Build the SQLite TCL extension in a way that make it compatible
-# with whatever version of TCL is running as $TCLSH_CMD, possibly defined
-# by --with-tclsh=
-#
-tclextension:	tclsqlite3.c
-	$(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --build-only --cc $(CC) $(CFLAGS) $(OPT_FEATURE_FLAGS) $(OPTS)
-
-# Install the SQLite TCL extension in a way that is appropriate for $TCLSH_CMD
-# to find it.
-#
-tclextension-install:	tclsqlite3.c
-	$(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --cc $(CC) $(CFLAGS) $(OPT_FEATURE_FLAGS) $(OPTS)
-
-# Install the SQLite TCL extension that is used by $TCLSH_CMD
-#
-tclextension-uninstall: 
-	$(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --uninstall
-
-# List all installed the SQLite TCL extension that is are accessible
-# by $TCLSH_CMD, included prior versions.
-#
-tclextension-list: 
-	$(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --info
-
 clean:
 	rm -f *.lo *.la *.o sqlite3$(TEXE) libsqlite3.la
 	rm -f sqlite3.h opcodes.*
@@ -1603,7 +1485,6 @@ clean:
 	rm -f LogEst$(TEXE) fts3view$(TEXE) rollback-test$(TEXE) showdb$(TEXE)
 	rm -f showjournal$(TEXE) showstat4$(TEXE) showwal$(TEXE) speedtest1$(TEXE)
 	rm -f wordcount$(TEXE) changeset$(TEXE)
-	rm -f version-info$(TEXT)
 	rm -f sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def
 	rm -f sqlite3.c
 	rm -f sqlite3rc.h
@@ -1619,9 +1500,6 @@ clean:
 	rm -f dbhash dbhash.exe
 	rm -f fts5.* fts5parse.*
 	rm -f threadtest5
-	rm -f src-verify
-	rm -f custom.rws
-	rm -f has_tclsh84 has_tclsh85
 
 distclean:	clean
 	rm -f sqlite_cfg.h config.log config.status libtool Makefile sqlite3.pc \
@@ -1650,28 +1528,3 @@ sqlite3.dll: $(REAL_LIBOBJ) sqlite3.def
 #
 fiddle: sqlite3.c shell.c
 	make -C ext/wasm fiddle emcc_opt=-Os
-
-#
-# Spell-checking for source comments
-# The sources checked are either C sources or C source templates.
-# Their comments are extracted and processed through aspell using
-# a custom dictionary that contains scads of odd identifiers that
-# find their way into the comments.
-#
-# Currently, this target is setup to be "made" in-tree only.
-# The output is ephemeral. Redirect it to guide spelling fixups,
-# either to correct spelling or add words to tool/custom.txt.
-#
-./custom.rws: ./tool/custom.txt
-	@echo 'Updating custom dictionary from tool/custom.txt'
-	aspell --lang=en create master ./custom.rws < $<
-
-misspell: ./custom.rws has_tclsh84
-	$(TCLSH_CMD) ./tool/spellsift.tcl ./src/*.c ./src/*.h ./src/*.in
-
-#
-# tool/version-info: a utility for emitting sqlite3 version info
-# in various forms.
-#
-version-info$(TEXE):	$(TOP)/tool/version-info.c Makefile sqlite3.h
-	$(LTLINK) $(ST_OPT) -o $@ $(TOP)/tool/version-info.c

Makefile.linux-gcc

@@ -22,6 +22,12 @@ TOP = ../sqlite
 BCC = gcc -g -O0
 #BCC = /opt/ancic/bin/c89 -0
 
+#### If the target operating system supports the "usleep()" system
+#    call, then define the HAVE_USLEEP macro for all C modules.
+#
+#USLEEP = 
+USLEEP = -DHAVE_USLEEP=1
+
 #### If you want the SQLite library to be safe for use within a 
 #    multi-threaded program, then define the following macro
 #    appropriately:

Makefile.msc

@@ -18,13 +18,6 @@ USE_AMALGAMATION = 1
 !ENDIF
 # <</mark>>
 
-# Optionally set EXTRA_SRC to a list of C files to append to
-# the generated sqlite3.c.
-#
-!IFNDEF EXTRA_SRC
-EXTRA_SRC =
-!ENDIF
-
 # Set this non-0 to enable full warnings (-W4, etc) when compiling.
 #
 !IFNDEF USE_FULLWARN
@@ -59,13 +52,6 @@ MINIMAL_AMALGAMATION = 0
 USE_STDCALL = 0
 !ENDIF
 
-# Use the USE_SEH=0 option on the nmake command line to omit structured
-# exception handling (SEH) support.  SEH is on by default.
-#
-!IFNDEF USE_SEH
-USE_SEH = 1
-!ENDIF
-
 # Set this non-0 to have the shell executable link against the core dynamic
 # link library.
 #
@@ -232,12 +218,6 @@ WIN32HEAP = 0
 OSTRACE = 0
 !ENDIF
 
-# enable address sanitizer using ASAN=1 on the command-line.
-#
-!IFNDEF ASAN
-ASAN = 0
-!ENDIF
-
 # Set this to one of the following values to enable various debugging
 # features.  Each level includes the debugging options from the previous
 # levels.  Currently, the recognized values for DEBUG are:
@@ -381,7 +361,6 @@ SQLITE_TCL_DEP =
 !IFNDEF OPT_FEATURE_FLAGS
 !IF $(MINIMAL_AMALGAMATION)==0
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
-OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS5=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
@@ -410,14 +389,6 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_MATH_FUNCTIONS
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RBU=1
 !ENDIF
 
-# Should structured exception handling (SEH) be enabled for WAL mode in
-# the core library?  It is on by default.  Only omit it if the
-# USE_SEH=0 option is provided on the nmake command-line.
-#
-!IF $(USE_SEH)==0
-OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_OMIT_SEH=1
-!ENDIF
-
 # These are the "extended" SQLite compilation options used when compiling for
 # the Windows 10 platform.
 #
@@ -906,13 +877,6 @@ RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
 !ENDIF
 !ENDIF
 
-
-# Address sanitizer if ASAN=1
-#
-!IF $(ASAN)>0
-TCC = $(TCC) /fsanitize=address
-!ENDIF
-
 # <<mark>>
 # The locations of the Tcl header and library files.  Also, the library that
 # non-stubs enabled programs using Tcl must link against.  These variables
@@ -1562,7 +1526,7 @@ TESTSRC = \
   $(TOP)\ext\fts3\fts3_term.c \
   $(TOP)\ext\fts3\fts3_test.c \
   $(TOP)\ext\rbu\test_rbu.c \
-  $(TOP)\ext\session\test_session.c
+  $(TOP)\ext\session\test_session.c 
 
 # Statically linked extensions.
 #
@@ -1591,21 +1555,18 @@ TESTEXT = \
   $(TOP)\ext\misc\percentile.c \
   $(TOP)\ext\misc\prefixes.c \
   $(TOP)\ext\misc\qpvtab.c \
-  $(TOP)\ext\misc\randomjson.c \
   $(TOP)\ext\misc\regexp.c \
   $(TOP)\ext\misc\remember.c \
   $(TOP)\ext\misc\series.c \
   $(TOP)\ext\misc\spellfix.c \
-  $(TOP)\ext\misc\stmtrand.c \
   $(TOP)\ext\misc\totype.c \
   $(TOP)\ext\misc\unionvtab.c \
   $(TOP)\ext\misc\wholenumber.c \
   $(TOP)\ext\rtree\test_rtreedoc.c \
   $(TOP)\ext\recover\sqlite3recover.c \
   $(TOP)\ext\recover\test_recover.c \
-  $(TOP)\ext\intck\test_intck.c  \
-  $(TOP)\ext\intck\sqlite3intck.c \
-  $(TOP)\ext\recover\dbdata.c 
+  $(TOP)\ext\recover\dbdata.c \
+  fts5.c
 
 # If use of zlib is enabled, add the "zipfile.c" source file.
 #
@@ -1621,8 +1582,7 @@ TESTSRC2 = \
   $(SRC01) \
   $(SRC07) \
   $(SRC10) \
-  $(TOP)\ext\async\sqlite3async.c \
-  fts5.c
+  $(TOP)\ext\async\sqlite3async.c
 
 # Header files used by all library source files.
 #
@@ -1701,9 +1661,6 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_DQS=0
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_STRICT_SUBTYPE=1
 !ENDIF
 
 # <<mark>>
@@ -1714,35 +1671,6 @@ FUZZERSHELL_COMPILE_OPTS =
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -I$(TOP)\test -I$(TOP)\ext\recover
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MEMSYS5
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OSS_FUZZ
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBPAGE_VTAB
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DBSTAT_VTAB
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_BYTECODE_VTAB
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_DESERIALIZE
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS3_PARENTHESIS
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS4
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_FTS5
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_GEOPOLY
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MATH_FUNCTIONS
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_MEMSYS5
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_NORMALIZE
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_PREUPDATE_HOOK
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_RTREE
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_SESSION
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_STMTVTAB
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_STAT4
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MEMORY=50000000
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MMAP_SIZE=0
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRIVATE=""
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_STRICT_SUBTYPE=1
-FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_STATIC_RANDOMJSON
-
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_MAX_MEMORY=50000000
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_PRINTF_PRECISION_LIMIT=1000
 FUZZCHECK_OPTS = $(FUZZCHECK_OPTS) -DSQLITE_OMIT_LOAD_EXTENSION
@@ -1758,7 +1686,6 @@ FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\fuzzinvariants.c
 FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\test\vt02.c
 FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\dbdata.c
 FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\recover\sqlite3recover.c
-FUZZCHECK_SRC = $(FUZZCHECK_SRC) $(TOP)\ext\misc\randomjson.c
 
 OSSSHELL_SRC = $(TOP)\test\ossshell.c $(TOP)\test\ossfuzz.c
 DBFUZZ_COMPILE_OPTS = -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION
@@ -1820,19 +1747,6 @@ pkgIndex.tcl:	$(TOP)\VERSION
 
 $(SQLITE3TCLDLL):	libtclsqlite3.lib $(LIBRESOBJS) tclsqlite3.def pkgIndex.tcl
 	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /DEF:tclsqlite3.def /OUT:$@ libtclsqlite3.lib $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
-
-tclextension:	$(SQLITE3TCLDLL)
-
-tclextension-install:	$(SQLITE3TCLDLL)
-	$(TCLSH_CMD) $(TOP)\tool\buildtclext.tcl --install-only
-
-tclextension-uninstall:
-	$(TCLSH_CMD) $(TOP)\tool\buildtclext.tcl --uninstall
-
-tclextension-list:
-	$(TCLSH_CMD) $(TOP)\tool\buildtclext.tcl --info
-
-
 # <</mark>>
 
 $(SQLITE3DLL):	$(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)
@@ -1851,8 +1765,8 @@ $(SQLITE3EXE):	shell.c $(SHELL_CORE_DEP) $(LIBRESOBJS) $(SHELL_CORE_SRC) $(SQLIT
 		/link $(SQLITE3EXEPDB) $(LDFLAGS) $(LTLINKOPTS) $(SHELL_LINK_OPTS) $(LTLIBPATHS) $(LIBRESOBJS) $(LIBREADLINE) $(LTLIBS) $(TLIBS)
 
 # <<mark>>
-sqldiff.exe:	$(TOP)\tool\sqldiff.c $(TOP)\ext\consio\console_io.h $(TOP)\ext\consio\console_io.c $(SQLITE3C) $(SQLITE3H) $(LIBRESOBJS)
-	$(LTLINK) $(NO_WARN) -I$(TOP)\ext\consio $(TOP)\tool\sqldiff.c $(TOP)\ext\consio\console_io.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS) $(LIBRESOBJS)
+sqldiff.exe:	$(TOP)\tool\sqldiff.c $(SQLITE3C) $(SQLITE3H)
+	$(LTLINK) $(NO_WARN) $(TOP)\tool\sqldiff.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
 dbhash.exe:	$(TOP)\tool\dbhash.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(TOP)\tool\dbhash.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
@@ -1866,12 +1780,6 @@ srcck1.exe:	$(TOP)\tool\srcck1.c
 sourcetest:	srcck1.exe $(SQLITE3C)
 	srcck1.exe $(SQLITE3C)
 
-src-verify.exe:	$(TOP)\tool\src-verify.c
-	$(LTLINK) $(NO_WARN) $(TOP)\tool\src-verify.c
-
-verify-source:	src-verify.exe
-	src-verify.exe $(TOP)
-
 fuzzershell.exe:	$(TOP)\tool\fuzzershell.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(FUZZERSHELL_COMPILE_OPTS) $(TOP)\tool\fuzzershell.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
@@ -1879,10 +1787,7 @@ dbfuzz.exe:	$(TOP)\test\dbfuzz.c $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(DBFUZZ_COMPILE_OPTS) $(TOP)\test\dbfuzz.c $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
 fuzzcheck.exe:	$(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H)
-	$(LTLINK) /F 8388608 $(NO_WARN) $(FUZZCHECK_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
-
-fuzzcheck-asan.exe:	$(FUZZCHECK_SRC) $(SQLITE3C) $(SQLITE3H)
-	$(LTLINK) $(NO_WARN) /fsanitize=address $(FUZZCHECK_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
+	$(LTLINK) $(NO_WARN) $(FUZZCHECK_OPTS) $(FUZZCHECK_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
 
 ossshell.exe:	$(OSSSHELL_SRC) $(SQLITE3C) $(SQLITE3H)
 	$(LTLINK) $(NO_WARN) $(FUZZCHECK_OPTS) $(OSSSHELL_SRC) $(SQLITE3C) /link $(LDFLAGS) $(LTLINKOPTS)
@@ -1935,8 +1840,8 @@ mptest:	mptester.exe
 	move vdbe.new tsrc\vdbe.c
 	echo > .target_source
 
-sqlite3.c:	.target_source sqlite3ext.h sqlite3session.h $(MKSQLITE3C_TOOL) src-verify.exe
-	$(TCLSH_CMD) $(MKSQLITE3C_TOOL) $(MKSQLITE3C_ARGS) $(EXTRA_SRC)
+sqlite3.c:	.target_source sqlite3ext.h sqlite3session.h $(MKSQLITE3C_TOOL)
+	$(TCLSH_CMD) $(MKSQLITE3C_TOOL) $(MKSQLITE3C_ARGS)
 
 sqlite3-all.c:	sqlite3.c $(TOP)\tool\split-sqlite3c.tcl
 	$(TCLSH_CMD) $(TOP)\tool\split-sqlite3c.tcl
@@ -2286,44 +2191,36 @@ mkkeywordhash.exe:	$(TOP)\tool\mkkeywordhash.c
 keywordhash.h:	$(TOP)\tool\mkkeywordhash.c mkkeywordhash.exe
 	.\mkkeywordhash.exe > keywordhash.h
 
-# Source and header files that shell.c depends on
-SHELL_DEP = \
-    $(TOP)\src\shell.c.in \
-    $(TOP)\ext\consio\console_io.c \
-    $(TOP)\ext\consio\console_io.h \
-    $(TOP)\ext\expert\sqlite3expert.c \
-    $(TOP)\ext\expert\sqlite3expert.h \
-    $(TOP)\ext\intck\sqlite3intck.c \
-    $(TOP)\ext\intck\sqlite3intck.h \
-    $(TOP)\ext\misc\appendvfs.c \
-    $(TOP)\ext\misc\base64.c \
-    $(TOP)\ext\misc\base85.c \
-    $(TOP)\ext\misc\completion.c \
-    $(TOP)\ext\misc\decimal.c \
-    $(TOP)\ext\misc\fileio.c \
-    $(TOP)\ext\misc\ieee754.c \
-    $(TOP)\ext\misc\memtrace.c \
-    $(TOP)\ext\misc\pcachetrace.c \
-    $(TOP)\ext\misc\regexp.c \
-    $(TOP)\ext\misc\series.c \
-    $(TOP)\ext\misc\shathree.c \
-    $(TOP)\ext\misc\sqlar.c \
-    $(TOP)\ext\misc\uint.c \
-    $(TOP)\ext\misc\zipfile.c \
-    $(TOP)\ext\recover\dbdata.c \
-    $(TOP)\ext\recover\sqlite3recover.c \
-    $(TOP)\ext\recover\sqlite3recover.h \
-    $(TOP)\src\test_windirent.c \
-    $(TOP)\src\test_windirent.h
+# Source files that go into making shell.c
+SHELL_SRC = \
+	$(TOP)\src\shell.c.in \
+	$(TOP)\ext\misc\appendvfs.c \
+	$(TOP)\ext\misc\completion.c \
+        $(TOP)\ext\misc\base64.c \
+        $(TOP)\ext\misc\base85.c \
+	$(TOP)\ext\misc\decimal.c \
+	$(TOP)\ext\misc\fileio.c \
+	$(TOP)\ext\misc\ieee754.c \
+	$(TOP)\ext\misc\regexp.c \
+	$(TOP)\ext\misc\series.c \
+	$(TOP)\ext\misc\shathree.c \
+	$(TOP)\ext\misc\uint.c \
+	$(TOP)\ext\expert\sqlite3expert.c \
+	$(TOP)\ext\expert\sqlite3expert.h \
+	$(TOP)\ext\misc\memtrace.c \
+	$(TOP)/ext/recover/dbdata.c \
+	$(TOP)/ext/recover/sqlite3recover.c \
+	$(TOP)/ext/recover/sqlite3recover.h \
+	$(TOP)\src\test_windirent.c
 
 # If use of zlib is enabled, add the "zipfile.c" source file.
 #
 !IF $(USE_ZLIB)!=0
-SHELL_DEP = $(SHELL_DEP) $(TOP)\ext\misc\sqlar.c
-SHELL_DEP = $(SHELL_DEP) $(TOP)\ext\misc\zipfile.c
+SHELL_SRC = $(SHELL_SRC) $(TOP)\ext\misc\sqlar.c
+SHELL_SRC = $(SHELL_SRC) $(TOP)\ext\misc\zipfile.c
 !ENDIF
 
-shell.c:	$(SHELL_DEP) $(TOP)\tool\mkshellc.tcl
+shell.c:	$(SHELL_SRC) $(TOP)\tool\mkshellc.tcl
 	$(TCLSH_CMD) $(TOP)\tool\mkshellc.tcl > shell.c
 
 zlib:
@@ -2466,8 +2363,6 @@ TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_DBPAGE_VTAB=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_ENABLE_BYTECODE_VTAB=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_CKSUMVFS_STATIC=1
 TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) $(TEST_CCONV_OPTS)
-TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_STATIC_RANDOMJSON
-TESTFIXTURE_FLAGS = $(TESTFIXTURE_FLAGS) -DSQLITE_STRICT_SUBTYPE=1
 
 TESTFIXTURE_SRC0 = $(TESTEXT) $(TESTSRC2)
 TESTFIXTURE_SRC1 = $(TESTEXT) $(SQLITE3C)
@@ -2507,12 +2402,9 @@ extensiontest:	testfixture.exe testloadext.dll
 	@set PATH=$(LIBTCLPATH);$(PATH)
 	.\testfixture.exe $(TOP)\test\loadext.test $(TESTOPTS)
 
-tool-zip:	testfixture.exe sqlite3.exe sqldiff.exe sqlite3_analyzer.exe $(TOP)\tool\mktoolzip.tcl
-	.\testfixture.exe $(TOP)\tool\mktoolzip.tcl
+coretestprogs:	$(TESTPROGS)
 
-coretestprogs:	testfixture.exe sqlite3.exe
-
-testprogs:	$(TESTPROGS) srcck1.exe fuzzcheck.exe sessionfuzz.exe
+testprogs:	coretestprogs srcck1.exe fuzzcheck.exe sessionfuzz.exe
 
 fulltest:	alltest fuzztest
 
@@ -2562,13 +2454,10 @@ testrunner:	testfixture.exe
 #
 devtest:	testfixture.exe fuzztest testrunner
 
-mdevtest:
-	$(TCLSH_CMD) $(TOP)\test\testrunner.tcl mdevtest
-
 # Testing for a release
 #
-releasetest: testfixture.exe
-	testfixture.exe $(TOP)\test\testrunner.tcl release
+releasetest: testfixture.exe fuzztest
+	testfixture.exe $(TOP)/test/testrunner.tcl release
 
 
 smoketest:	$(TESTPROGS)
@@ -2578,7 +2467,7 @@ smoketest:	$(TESTPROGS)
 shelltest: $(TESTPROGS)
 	.\testfixture.exe $(TOP)\test\permutations.test shell
 
-sqlite3_analyzer.c:	$(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(TOP)\ext\consio\console_io.h $(TOP)\ext\consio\console_io.c $(SQLITE_TCL_DEP)
+sqlite3_analyzer.c:	$(SQLITE3C) $(SQLITE3H) $(TOP)\src\tclsqlite.c $(TOP)\tool\spaceanal.tcl $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in $(SQLITE_TCL_DEP)
 	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\tool\sqlite3_analyzer.c.in > $@
 
 sqlite3_analyzer.exe:	sqlite3_analyzer.c $(LIBRESOBJS)
@@ -2596,14 +2485,14 @@ sqlite3_expert.exe: $(SQLITE3C) $(TOP)\ext\expert\sqlite3expert.h $(TOP)\ext\exp
 	$(LTLINK) $(NO_WARN)	$(TOP)\ext\expert\sqlite3expert.c $(TOP)\ext\expert\expert.c $(SQLITE3C) $(TLIBS)
 
 CHECKER_DEPS =\
-  $(TOP)\tool\mkccode.tcl \
+  $(TOP)/tool/mkccode.tcl \
   sqlite3.c \
-  $(TOP)\src\tclsqlite.c \
-  $(TOP)\ext\repair\sqlite3_checker.tcl \
-  $(TOP)\ext\repair\checkindex.c \
-  $(TOP)\ext\repair\checkfreelist.c \
-  $(TOP)\ext\misc\btreeinfo.c \
-  $(TOP)\ext\repair\sqlite3_checker.c.in
+  $(TOP)/src/tclsqlite.c \
+  $(TOP)/ext/repair/sqlite3_checker.tcl \
+  $(TOP)/ext/repair/checkindex.c \
+  $(TOP)/ext/repair/checkfreelist.c \
+  $(TOP)/ext/misc/btreeinfo.c \
+  $(TOP)/ext/repair/sqlite3_checker.c.in
 
 sqlite3_checker.c:	$(CHECKER_DEPS)
 	$(TCLSH_CMD) $(TOP)\tool\mkccode.tcl $(TOP)\ext\repair\sqlite3_checker.c.in > $@
@@ -2742,5 +2631,4 @@ clean:
 	del /Q showshm.exe sqlite3_checker.* sqlite3_expert.exe 2>NUL
 	del /Q fts5.* fts5parse.* 2>NUL
 	del /Q lsm.h lsm1.c 2>NUL
-	del /q src-verify.exe 2>NUL
 # <</mark>>

README.md

@@ -7,7 +7,7 @@ and most of the documentation are managed separately.
 
 ## Version Control
 
-SQLite sources are managed using
+SQLite sources are managed using the
 [Fossil](https://www.fossil-scm.org/), a distributed version control system
 that was specifically designed and written to support SQLite development.
 The [Fossil repository](https://sqlite.org/src/timeline) contains the urtext.
@@ -25,28 +25,7 @@ If you pulled your SQLite source code from a secondary source and want to
 verify its integrity, there are hints on how to do that in the
 [Verifying Code Authenticity](#vauth) section below.
 
-## Contacting The SQLite Developers
-
-The preferred way to ask questions or make comments about SQLite or to
-report bugs against SQLite is to visit the 
-[SQLite Forum](https://sqlite.org/forum) at <https://sqlite.org/forum/>.
-Anonymous postings are permitted.
-
-If you think you have found a bug that has security implications and
-you do not want to report it on the public forum, you can send a private
-email to drh at sqlite dot org.
-
-## Public Domain
-
-The SQLite source code is in the public domain.  See
-<https://sqlite.org/copyright.html> for details. 
-
-Because SQLite is in the public domain, we do not normally accept pull
-requests, because if we did take a pull request, the changes in that
-pull request might carry a copyright and the SQLite source code would
-then no longer be fully in the public domain.
-
-## Obtaining The SQLite Source Code
+## Obtaining The Code
 
 If you do not want to use Fossil, you can download tarballs or ZIP
 archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows:
@@ -103,9 +82,9 @@ For example:
         mkdir bld                ;#  Build will occur in a sibling directory
         cd bld                   ;#  Change to the build directory
         ../sqlite/configure      ;#  Run the configure script
-        make                     ;#  Builds the "sqlite3" command-line tool
+        make                     ;#  Run the makefile.
         make sqlite3.c           ;#  Build the "amalgamation" source file
-        make devtest             ;#  Run some tests (requires Tcl)
+        make test                ;#  Run some tests (requires Tcl)
 
 See the makefile for additional targets.
 
@@ -115,33 +94,32 @@ script does not work out for you, there is a generic makefile named
 can copy and edit to suit your needs.  Comments on the generic makefile
 show what changes are needed.
 
-## Compiling for Windows Using MSVC
+## Using MSVC for Windows systems
 
 On Windows, all applicable build products can be compiled with MSVC.
-You will also need a working installation of TCL.
-See the [compile-for-windows.md](doc/compile-for-windows.md) document for
-additional information about how to install MSVC and TCL and configure your
-build environment.
+First open the command prompt window associated with the desired compiler
+version (e.g. "Developer Command Prompt for VS2013").  Next, use NMAKE
+with the provided "Makefile.msc" to build one of the supported targets.
 
-If you want to run tests, you need to let SQLite know the location of your
-TCL library, using a command like this:
+For example, from the parent directory of the source subtree named "sqlite":
 
-        set TCLDIR=c:\Tcl
+        mkdir bld
+        cd bld
+        nmake /f ..\sqlite\Makefile.msc TOP=..\sqlite
+        nmake /f ..\sqlite\Makefile.msc sqlite3.c TOP=..\sqlite
+        nmake /f ..\sqlite\Makefile.msc sqlite3.dll TOP=..\sqlite
+        nmake /f ..\sqlite\Makefile.msc sqlite3.exe TOP=..\sqlite
+        nmake /f ..\sqlite\Makefile.msc test TOP=..\sqlite
 
-SQLite uses "tclsh.exe" as part of the build process, and so that utility
-program will need to be somewhere on your %PATH%.  The finished SQLite library
-does not contain any TCL code, but it does use TCL to help with the build process
-and to run tests.
+There are several build options that can be set via the NMAKE command
+line.  For example, to build for WinRT, simply add "FOR_WINRT=1" argument
+to the "sqlite3.dll" command line above.  When debugging into the SQLite
+code, adding the "DEBUG=1" argument to one of the above command lines is
+recommended.
 
-Build using Makefile.msc.  Example:
-
-        nmake /f Makefile.msc
-        nmake /f Makefile.msc sqlite3.c
-        nmake /f Makefile.msc devtest
-        nmake /f Makefile.msc releasetest
- 
-There are many other makefile targets.  See comments in Makefile.msc for
-details.
+SQLite does not require [Tcl](http://www.tcl.tk/) to run, but a Tcl installation
+is required by the makefiles (including those for MSVC).  SQLite contains
+a lot of generated code and Tcl is used to do much of that code generation.
 
 ## Source Code Tour
 
@@ -159,7 +137,7 @@ extension and only later escaped to the wild as an independent library.)
 
 Test scripts and programs are found in the **test/** subdirectory.
 Additional test code is found in other source repositories.
-See [How SQLite Is Tested](https://www.sqlite.org/testing.html) for
+See [How SQLite Is Tested](http://www.sqlite.org/testing.html) for
 additional information.
 
 The **ext/** subdirectory contains code for extensions.  The
@@ -183,7 +161,7 @@ manually-edited files and automatically-generated files.
 
 The SQLite interface is defined by the **sqlite3.h** header file, which is
 generated from src/sqlite.h.in, ./manifest.uuid, and ./VERSION.  The
-[Tcl script](https://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion.
+[Tcl script](http://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion.
 The manifest.uuid file contains the SHA3 hash of the particular check-in
 and is used to generate the SQLITE\_SOURCE\_ID macro.  The VERSION file
 contains the current SQLite version number.  The sqlite3.h header is really
@@ -250,14 +228,14 @@ individual source file exceeds 32K lines in length.
 ## How It All Fits Together
 
 SQLite is modular in design.
-See the [architectural description](https://www.sqlite.org/arch.html)
+See the [architectural description](http://www.sqlite.org/arch.html)
 for details. Other documents that are useful in
 (helping to understand how SQLite works include the
-[file format](https://www.sqlite.org/fileformat2.html) description,
-the [virtual machine](https://www.sqlite.org/opcode.html) that runs
+[file format](http://www.sqlite.org/fileformat2.html) description,
+the [virtual machine](http://www.sqlite.org/opcode.html) that runs
 prepared statements, the description of
-[how transactions work](https://www.sqlite.org/atomiccommit.html), and
-the [overview of the query planner](https://www.sqlite.org/optoverview.html).
+[how transactions work](http://www.sqlite.org/atomiccommit.html), and
+the [overview of the query planner](http://www.sqlite.org/optoverview.html).
 
 Years of effort have gone into optimizing SQLite, both
 for small size and high performance.  And optimizations tend to result in
@@ -312,13 +290,16 @@ Key files:
      is not part of the core SQLite library.  But as most of the tests in this
      repository are written in Tcl, the Tcl language bindings are important.
 
-  *  **test\*.c** - Files in the src/ folder that begin with "test" go into
+  *  **test*.c** - Files in the src/ folder that begin with "test" go into
      building the "testfixture.exe" program.  The testfixture.exe program is
      an enhanced Tcl shell.  The testfixture.exe program runs scripts in the
      test/ folder to validate the core SQLite code.  The testfixture program
      (and some other test programs too) is built and run when you type
      "make test".
 
+  *  **ext/misc/json1.c** - This file implements the various JSON functions
+     that are built into SQLite.
+
 There are many other source files.  Each has a succinct header comment that
 describes its purpose and role within the larger system.
 
@@ -326,8 +307,8 @@ describes its purpose and role within the larger system.
 ## Verifying Code Authenticity
 
 The `manifest` file at the root directory of the source tree
-contains either a SHA3-256 hash or a SHA1 hash
-for every source file in the repository.
+contains either a SHA3-256 hash (for newer files) or a SHA1 hash (for
+older files) for every source file in the repository.
 The name of the version of the entire source tree is just the
 SHA3-256 hash of the `manifest` file itself, possibly with the
 last line of that file omitted if the last line begins with
@@ -335,25 +316,14 @@ last line of that file omitted if the last line begins with
 The `manifest.uuid` file should contain the SHA3-256 hash of the
 `manifest` file. If all of the above hash comparisons are correct, then
 you can be confident that your source tree is authentic and unadulterated.
-Details on the format for the `manifest` files are available
-[on the Fossil website](https://fossil-scm.org/fossil/doc/trunk/www/fileformat.wiki#manifest).
 
-The process of checking source code authenticity is automated by the 
-makefile:
-
->   make verify-source
-
-Or on windows:
-
->   nmake /f Makefile.msc verify-source
-
-Using the makefile to verify source integrity is good for detecting
-accidental changes to the source tree, but malicious changes could be
-hidden by also modifying the makefiles.
+The format of the `manifest` file should be mostly self-explanatory, but
+if you want details, they are available
+[here](https://fossil-scm.org/fossil/doc/trunk/www/fileformat.wiki#manifest).
 
 ## Contacts
 
-The main SQLite website is [https://sqlite.org/](https://sqlite.org/)
+The main SQLite website is [http://www.sqlite.org/](http://www.sqlite.org/)
 with geographically distributed backups at
-[https://www2.sqlite.org/](https://www2.sqlite.org) and
-[https://www3.sqlite.org/](https://www3.sqlite.org).
+[http://www2.sqlite.org/](http://www2.sqlite.org) and
+[http://www3.sqlite.org/](http://www3.sqlite.org).

VERSION

@@ -1 +1 @@
-3.47.0
+3.42.0

autoconf/Makefile.am

@@ -9,7 +9,7 @@ sqlite3_SOURCES = shell.c sqlite3.h
 EXTRA_sqlite3_SOURCES = sqlite3.c
 sqlite3_LDADD = @EXTRA_SHELL_OBJ@ @READLINE_LIBS@
 sqlite3_DEPENDENCIES = @EXTRA_SHELL_OBJ@
-sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_DQS=0 -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB $(SHELL_CFLAGS)
+sqlite3_CFLAGS = $(AM_CFLAGS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS -DSQLITE_ENABLE_DBPAGE_VTAB -DSQLITE_ENABLE_STMTVTAB -DSQLITE_ENABLE_DBSTAT_VTAB $(SHELL_CFLAGS)
 
 include_HEADERS = sqlite3.h sqlite3ext.h
 

autoconf/Makefile.msc

@@ -18,13 +18,6 @@
 TOP = .
 
 
-# Optionally set EXTRA_SRC to a list of C files to append to
-# the generated sqlite3.c.
-#
-!IFNDEF EXTRA_SRC
-EXTRA_SRC =
-!ENDIF
-
 # Set this non-0 to enable full warnings (-W4, etc) when compiling.
 #
 !IFNDEF USE_FULLWARN
@@ -59,13 +52,6 @@ MINIMAL_AMALGAMATION = 0
 USE_STDCALL = 0
 !ENDIF
 
-# Use the USE_SEH=0 option on the nmake command line to omit structured
-# exception handling (SEH) support.  SEH is on by default.
-#
-!IFNDEF USE_SEH
-USE_SEH = 1
-!ENDIF
-
 # Set this non-0 to have the shell executable link against the core dynamic
 # link library.
 #
@@ -194,12 +180,6 @@ WIN32HEAP = 0
 OSTRACE = 0
 !ENDIF
 
-# enable address sanitizer using ASAN=1 on the command-line.
-#
-!IFNDEF ASAN
-ASAN = 0
-!ENDIF
-
 # Set this to one of the following values to enable various debugging
 # features.  Each level includes the debugging options from the previous
 # levels.  Currently, the recognized values for DEBUG are:
@@ -303,7 +283,6 @@ SQLITE3EXEPDB = /pdb:sqlite3sh.pdb
 !IFNDEF OPT_FEATURE_FLAGS
 !IF $(MINIMAL_AMALGAMATION)==0
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS3=1
-OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_FTS5=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RTREE=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_GEOPOLY=1
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_STMTVTAB=1
@@ -332,14 +311,6 @@ OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_MATH_FUNCTIONS
 OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_ENABLE_RBU=1
 !ENDIF
 
-# Should structured exception handling (SEH) be enabled for WAL mode in
-# the core library?  It is on by default.  Only omit it if the
-# USE_SEH=0 option is provided on the nmake command-line.
-#
-!IF $(USE_SEH)==0
-OPT_FEATURE_FLAGS = $(OPT_FEATURE_FLAGS) -DSQLITE_OMIT_SEH=1
-!ENDIF
-
 # These are the "extended" SQLite compilation options used when compiling for
 # the Windows 10 platform.
 #
@@ -747,13 +718,6 @@ RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1
 !ENDIF
 
 
-# Address sanitizer if ASAN=1
-#
-!IF $(ASAN)>0
-TCC = $(TCC) /fsanitize=address
-!ENDIF
-
-
 # Compiler options needed for programs that use the readline() library.
 #
 !IFNDEF READLINE_FLAGS
@@ -995,9 +959,6 @@ SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_DQS=0
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_FTS4=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_EXPLAIN_COMMENTS=1
 SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_OFFSET_SQL_FUNC=1
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_UNKNOWN_SQL_FUNCTION=1
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_ENABLE_STMT_SCANSTATUS=1
-SHELL_COMPILE_OPTS = $(SHELL_COMPILE_OPTS) -DSQLITE_STRICT_SUBTYPE=1
 !ENDIF
 
 

autoconf/tea/configure.ac

@@ -19,7 +19,7 @@ dnl	to configure the system for the local environment.
 # so that we create the export library with the dll.
 #-----------------------------------------------------------------------
 
-AC_INIT([sqlite],[3.47.0])
+AC_INIT([sqlite],[3.41.0])
 
 #--------------------------------------------------------------------
 # Call TEA_INIT as the first TEA_ macro to set up initial vars.

autoconf/tea/tclconfig/tcl.m4

@@ -4064,4 +4064,4 @@ AC_DEFUN([TEA_ZIPFS_SUPPORT], [
 
 # Local Variables:
 # mode: autoconf
-# End:
+# End:

autoconf/tea/win/rules.vc

@@ -708,3 +708,4 @@ TK_INCLUDES     = -I"$(_TKDIR)\generic" -I"$(_TKDIR)\win" -I"$(_TKDIR)\xlib"
 !message *** Link options '$(LINKERFLAGS)'
 
 !endif
+

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.47.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.42.0.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.47.0'
-PACKAGE_STRING='sqlite 3.47.0'
+PACKAGE_VERSION='3.42.0'
+PACKAGE_STRING='sqlite 3.42.0'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -776,11 +776,22 @@ OPT_FEATURE_FLAGS
 HAVE_ZLIB
 USE_AMALGAMATION
 TARGET_DEBUG
-TARGET_HAVE_LINENOISE
 TARGET_HAVE_EDITLINE
 TARGET_HAVE_READLINE
 TARGET_READLINE_INC
 TARGET_READLINE_LIBS
+HAVE_TCL
+TCL_SHLIB_SUFFIX
+TCL_STUB_LIB_SPEC
+TCL_STUB_LIB_FLAG
+TCL_STUB_LIB_FILE
+TCL_LIB_SPEC
+TCL_LIB_FLAG
+TCL_LIB_FILE
+TCL_INCLUDE_SPEC
+TCL_SRC_DIR
+TCL_BIN_DIR
+TCL_VERSION
 TARGET_EXEEXT
 SQLITE_OS_WIN
 SQLITE_OS_UNIX
@@ -794,10 +805,6 @@ RELEASE
 VERSION
 program_prefix
 TCLLIBDIR
-HAVE_TCL
-TCL_STUB_LIB_SPEC
-TCL_LIB_SPEC
-TCL_INCLUDE_SPEC
 TCLSH_CMD
 INSTALL_DATA
 INSTALL_SCRIPT
@@ -886,18 +893,16 @@ enable_fast_install
 with_gnu_ld
 enable_libtool_lock
 enable_largefile
-with_tclsh
-with_tcl
-enable_tcl
 with_wasi_sdk
 enable_threadsafe
 enable_releasemode
 enable_tempstore
+enable_tcl
+with_tcl
 enable_editline
 enable_readline
 with_readline_lib
 with_readline_inc
-with_linenoise
 enable_debug
 enable_amalgamation
 enable_load_extension
@@ -1465,7 +1470,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures sqlite 3.47.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.42.0 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1530,7 +1535,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.47.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.42.0:";;
    esac
   cat <<\_ACEOF
 
@@ -1544,12 +1549,11 @@ Optional Features:
                           optimize for fast installation [default=yes]
   --disable-libtool-lock  avoid locking (might break parallel builds)
   --disable-largefile     omit support for large files
-  --disable-tcl           omit building accessory programs that require
-                          TCL-dev
   --disable-threadsafe    Disable mutexing
   --enable-releasemode    Support libtool link to release mode
   --enable-tempstore      Use an in-ram database for temporary tables
                           (never,no,yes,always)
+  --disable-tcl           do not build TCL extension
   --enable-editline       enable BSD editline support
   --disable-readline      disable readline support
   --enable-debug          enable debugging & verbose explain
@@ -1577,13 +1581,12 @@ Optional Packages:
   --with-pic              try to use only PIC/non-PIC objects [default=use
                           both]
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
-  --with-tclsh=PATHNAME   full pathname of a tclsh to use
-  --with-tcl=DIR          directory containing (tclConfig.sh)
   --with-wasi-sdk=DIR     directory containing the WASI SDK. Triggers
                           cross-compile to WASM.
+  --with-tcl=DIR          directory containing tcl configuration
+                          (tclConfig.sh)
   --with-readline-lib     specify readline library
   --with-readline-inc     specify readline include paths
-  --with-linenoise=DIR    source directory for linenoise library
 
 Some influential environment variables:
   CC          C compiler command
@@ -1662,7 +1665,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-sqlite configure 3.47.0
+sqlite configure 3.42.0
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2081,7 +2084,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by sqlite $as_me 3.47.0, which was
+It was created by sqlite $as_me 3.42.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -3939,13 +3942,13 @@ if ${lt_cv_nm_interface+:} false; then :
 else
   lt_cv_nm_interface="BSD nm"
   echo "int some_variable = 0;" > conftest.$ac_ext
-  (eval echo "\"\$as_me:3942: $ac_compile\"" >&5)
+  (eval echo "\"\$as_me:3945: $ac_compile\"" >&5)
   (eval "$ac_compile" 2>conftest.err)
   cat conftest.err >&5
-  (eval echo "\"\$as_me:3945: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
+  (eval echo "\"\$as_me:3948: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
   (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
   cat conftest.err >&5
-  (eval echo "\"\$as_me:3948: output\"" >&5)
+  (eval echo "\"\$as_me:3951: output\"" >&5)
   cat conftest.out >&5
   if $GREP 'External.*some_variable' conftest.out > /dev/null; then
     lt_cv_nm_interface="MS dumpbin"
@@ -5151,7 +5154,7 @@ ia64-*-hpux*)
   ;;
 *-*-irix6*)
   # Find out which ABI we are using.
-  echo '#line 5154 "configure"' > conftest.$ac_ext
+  echo '#line 5157 "configure"' > conftest.$ac_ext
   if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
   (eval $ac_compile) 2>&5
   ac_status=$?
@@ -6676,11 +6679,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:6679: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:6682: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>conftest.err)
    ac_status=$?
    cat conftest.err >&5
-   echo "$as_me:6683: \$? = $ac_status" >&5
+   echo "$as_me:6686: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s "$ac_outfile"; then
      # The compiler can only warn and ignore the option if not recognized
      # So say no if there are warnings other than the usual output.
@@ -7015,11 +7018,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7018: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7021: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>conftest.err)
    ac_status=$?
    cat conftest.err >&5
-   echo "$as_me:7022: \$? = $ac_status" >&5
+   echo "$as_me:7025: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s "$ac_outfile"; then
      # The compiler can only warn and ignore the option if not recognized
      # So say no if there are warnings other than the usual output.
@@ -7120,11 +7123,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7123: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7126: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>out/conftest.err)
    ac_status=$?
    cat out/conftest.err >&5
-   echo "$as_me:7127: \$? = $ac_status" >&5
+   echo "$as_me:7130: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s out/conftest2.$ac_objext
    then
      # The compiler can only warn and ignore the option if not recognized
@@ -7175,11 +7178,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7178: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7181: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>out/conftest.err)
    ac_status=$?
    cat out/conftest.err >&5
-   echo "$as_me:7182: \$? = $ac_status" >&5
+   echo "$as_me:7185: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s out/conftest2.$ac_objext
    then
      # The compiler can only warn and ignore the option if not recognized
@@ -9555,7 +9558,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 9558 "configure"
+#line 9561 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -9651,7 +9654,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 9654 "configure"
+#line 9657 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -10308,31 +10311,11 @@ done
 USE_AMALGAMATION=1
 
 #########
-# Figure out all the name of a working tclsh and parameters needed to compile against Tcl.
-# The --with-tcl= and/or --with-tclsh= configuration arguments might be useful for this.
+# See whether we can run specific tclsh versions known to work well;
+# if not, then we fall back to plain tclsh.
+# TODO: try other versions before falling back?
 #
-
-# Check whether --with-tclsh was given.
-if test "${with_tclsh+set}" = set; then :
-  withval=$with_tclsh;
-fi
-
-
-# Check whether --with-tcl was given.
-if test "${with_tcl+set}" = set; then :
-  withval=$with_tcl;
-fi
-
-# Check whether --enable-tcl was given.
-if test "${enable_tcl+set}" = set; then :
-  enableval=$enable_tcl; use_tcl=$enableval
-else
-  use_tcl=yes
-fi
-
-original_use_tcl=${use_tcl}
-if test x"${with_tclsh}" == x -a x"${with_tcl}" == x; then
-  for ac_prog in tclsh8.6 tclsh tclsh9.0
+for ac_prog in tclsh8.7 tclsh8.6 tclsh8.5 tclsh
 do
   # Extract the first word of "$ac_prog", so it can be a program name with args.
 set dummy $ac_prog; ac_word=$2
@@ -10375,99 +10358,10 @@ fi
 done
 test -n "$TCLSH_CMD" || TCLSH_CMD="none"
 
-  with_tclsh=${TCLSH_CMD}
-fi
-if test x"${with_tclsh}" != x -a x"${with_tclsh}" != xnone; then
-  TCLSH_CMD=${with_tclsh}
-  { $as_echo "$as_me:${as_lineno-$LINENO}: result: using tclsh at \"$TCLSH_CMD\"" >&5
-$as_echo "using tclsh at \"$TCLSH_CMD\"" >&6; }
-  if test x"${use_tcl}" = "xyes"; then
-    with_tcl=`${with_tclsh} <${srcdir}/tool/find_tclconfig.tcl`
-    if test x"${with_tcl}" != x; then
-      { $as_echo "$as_me:${as_lineno-$LINENO}: result: $TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}" >&5
-$as_echo "$TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}" >&6; }
-    else
-      { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $TCLSH_CMD is unable to recommend a tclConfig.sh" >&5
-$as_echo "$as_me: WARNING: $TCLSH_CMD is unable to recommend a tclConfig.sh" >&2;}
-      use_tcl=no
-    fi
-  fi
-fi
-if test x"${use_tcl}" = "xyes"; then
-  if test x"${with_tcl}" != x; then
-    if test -r ${with_tcl}/tclConfig.sh; then
-      tclconfig="${with_tcl}/tclConfig.sh"
-    else
-      for i in tcl8.6 tcl9.0 lib; do
-        if test -r ${with_tcl}/$i/tclConfig.sh; then
-          tclconfig=${with_tcl}/$i/tclConfig.sh
-          break
-        fi
-      done
-    fi
-    if test ! -r "${tclconfig}"; then
-      as_fn_error $? "no tclConfig.sh file found under ${with_tcl}" "$LINENO" 5
-    fi
-  else
-    # If we have not yet found a tclConfig.sh file, look in $libdir whic is
-    # set automatically by autoconf or by the --prefix command-line option.
-    # See https://sqlite.org/forum/forumpost/e04e693439a22457
-    libdir=${prefix}/lib
-    if test -r ${libdir}/tclConfig.sh; then
-      tclconfig=${libdir}/tclConfig.sh
-    else
-      for i in tcl8.6 tcl9.0 lib; do
-        if test -r ${libdir}/$i/tclConfig.sh; then
-          tclconfig=${libdir}/$i/tclConfig.sh
-          break
-        fi
-      done
-    fi
-    if test ! -r "${tclconfig}"; then
-      as_fn_error $? "cannot find a usable tclConfig.sh file.
-        Use --with-tcl=DIR to specify a directory where tclConfig.sh can be found.
-        SQLite does not use TCL internally, but TCL is required to build SQLite
-        from canonical sources and TCL is required for testing." "$LINENO" 5
-    fi
-  fi
-  { $as_echo "$as_me:${as_lineno-$LINENO}: result: loading TCL configuration from ${tclconfig}" >&5
-$as_echo "loading TCL configuration from ${tclconfig}" >&6; }
-  . ${tclconfig}
-
-
-
-  # There are lots of other configuration variables that are provided by the
-  # tclConfig.sh file and that could be included here.  But as of right now,
-  # TCL_LIB_SPEC is the only what that the Makefile uses.
-  HAVE_TCL=1
-elif test x"${original_use_tcl}" = "xno"; then
-  { $as_echo "$as_me:${as_lineno-$LINENO}: result: unable to run tests because of --disable-tcl" >&5
-$as_echo "unable to run tests because of --disable-tcl" >&6; }
-  HAVE_TCL=0
-else
-  { $as_echo "$as_me:${as_lineno-$LINENO}: result: unable to run tests because no tclConfig.sh file could be located" >&5
-$as_echo "unable to run tests because no tclConfig.sh file could be located" >&6; }
-  HAVE_TCL=0
-fi
-
-if test x"$TCLSH_CMD" == x; then
-  TCLSH_CMD=${TCL_EXEC_PREFIX}/bin/tclsh${TCL_VERSION}
-  if test ! -x ${TCLSH_CMD}; then
-    TCLSH_CMD_2=${TCL_EXEC_PREFIX}/bin/tclsh
-    if test ! -x ${TCLSH_CMD_2}; then
-      { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: cannot find a usable tclsh at either ${TCLSH_CMD} or ${TCLSH_CMD_2}" >&5
-$as_echo "$as_me: WARNING: cannot find a usable tclsh at either ${TCLSH_CMD} or ${TCLSH_CMD_2}" >&2;}
-      TCLSH_CMD=none
-    else
-      TCLSH_CMD=${TCLSH_CMD_2}
-    fi
-  fi
-fi
 if test "$TCLSH_CMD" = "none"; then
   # If we can't find a local tclsh, then building the amalgamation will fail.
   # We act as though --disable-amalgamation has been used.
-  { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Warning: can't find tclsh - defaulting to non-amalgamation build." >&5
-$as_echo "$as_me: WARNING: Warning: can't find tclsh - defaulting to non-amalgamation build." >&2;}
+  echo "Warning: can't find tclsh - defaulting to non-amalgamation build."
   USE_AMALGAMATION=0
   TCLSH_CMD="tclsh"
 fi
@@ -10475,6 +10369,7 @@ fi
 
 
 if test "x${TCLLIBDIR+set}" != "xset" ; then
+  TCLLIBDIR='$(libdir)'
   for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` ; do
     if test -d $i ; then
       TCLLIBDIR=$i
@@ -10871,6 +10766,213 @@ fi
 
 
 
+##########
+# Figure out all the parameters needed to compile against Tcl.
+#
+# This code is derived from the SC_PATH_TCLCONFIG and SC_LOAD_TCLCONFIG
+# macros in the in the tcl.m4 file of the standard TCL distribution.
+# Those macros could not be used directly since we have to make some
+# minor changes to accomodate systems that do not have TCL installed.
+#
+# Check whether --enable-tcl was given.
+if test "${enable_tcl+set}" = set; then :
+  enableval=$enable_tcl; use_tcl=$enableval
+else
+  use_tcl=yes
+fi
+
+if test "${use_tcl}" = "yes" ; then
+
+# Check whether --with-tcl was given.
+if test "${with_tcl+set}" = set; then :
+  withval=$with_tcl; with_tclconfig=${withval}
+fi
+
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for Tcl configuration" >&5
+$as_echo_n "checking for Tcl configuration... " >&6; }
+  if ${ac_cv_c_tclconfig+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+
+    # First check to see if --with-tcl was specified.
+    if test x"${with_tclconfig}" != x ; then
+      if test -f "${with_tclconfig}/tclConfig.sh" ; then
+        ac_cv_c_tclconfig=`(cd ${with_tclconfig}; pwd)`
+      else
+        as_fn_error $? "${with_tclconfig} directory doesn't contain tclConfig.sh" "$LINENO" 5
+      fi
+    fi
+
+    # Start autosearch by asking tclsh
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      if test x"$cross_compiling" = xno; then
+        for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}`
+        do
+          if test -f "$i/tclConfig.sh" ; then
+            ac_cv_c_tclconfig="$i"
+            break
+          fi
+        done
+      fi
+    fi
+
+    # On ubuntu 14.10, $auto_path on tclsh is not quite correct.
+    # So try again after applying corrections.
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      if test x"$cross_compiling" = xno; then
+        for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD} | sed 's,/tcltk/tcl,/tcl,g'`
+        do
+          if test -f "$i/tclConfig.sh" ; then
+            ac_cv_c_tclconfig="$i"
+            break
+          fi
+        done
+      fi
+    fi
+
+    # Recent versions of Xcode on Macs hid the tclConfig.sh file
+    # in a strange place.
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      if test x"$cross_compiling" = xno; then
+        for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib
+        do
+          if test -f "$i/tclConfig.sh" ; then
+            ac_cv_c_tclconfig="$i"
+            break
+          fi
+        done
+      fi
+    fi
+
+    # then check for a private Tcl installation
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      for i in \
+            ../tcl \
+            `ls -dr ../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \
+            `ls -dr ../tcl[8-9].[0-9] 2>/dev/null` \
+            `ls -dr ../tcl[8-9].[0-9]* 2>/dev/null` \
+            ../../tcl \
+            `ls -dr ../../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \
+            `ls -dr ../../tcl[8-9].[0-9] 2>/dev/null` \
+            `ls -dr ../../tcl[8-9].[0-9]* 2>/dev/null` \
+            ../../../tcl \
+            `ls -dr ../../../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \
+            `ls -dr ../../../tcl[8-9].[0-9] 2>/dev/null` \
+            `ls -dr ../../../tcl[8-9].[0-9]* 2>/dev/null`
+      do
+        if test -f "$i/unix/tclConfig.sh" ; then
+          ac_cv_c_tclconfig=`(cd $i/unix; pwd)`
+          break
+        fi
+      done
+    fi
+
+    # check in a few common install locations
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      for i in \
+            `ls -d ${libdir} 2>/dev/null` \
+            `ls -d /usr/local/lib 2>/dev/null` \
+            `ls -d /usr/contrib/lib 2>/dev/null` \
+            `ls -d /usr/lib 2>/dev/null`
+      do
+        if test -f "$i/tclConfig.sh" ; then
+           ac_cv_c_tclconfig=`(cd $i; pwd)`
+           break
+        fi
+      done
+    fi
+
+    # check in a few other private locations
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      for i in \
+         ${srcdir}/../tcl \
+         `ls -dr ${srcdir}/../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \
+         `ls -dr ${srcdir}/../tcl[8-9].[0-9] 2>/dev/null` \
+         `ls -dr ${srcdir}/../tcl[8-9].[0-9]* 2>/dev/null`
+      do
+        if test -f "$i/unix/tclConfig.sh" ; then
+          ac_cv_c_tclconfig=`(cd $i/unix; pwd)`
+          break
+        fi
+      done
+    fi
+
+fi
+
+
+  if test x"${ac_cv_c_tclconfig}" = x ; then
+    use_tcl=no
+    { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Can't find Tcl configuration definitions" >&5
+$as_echo "$as_me: WARNING: Can't find Tcl configuration definitions" >&2;}
+    { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: *** Without Tcl the regression tests cannot be executed ***" >&5
+$as_echo "$as_me: WARNING: *** Without Tcl the regression tests cannot be executed ***" >&2;}
+    { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: *** Consider using --with-tcl=... to define location of Tcl ***" >&5
+$as_echo "$as_me: WARNING: *** Consider using --with-tcl=... to define location of Tcl ***" >&2;}
+  else
+    TCL_BIN_DIR=${ac_cv_c_tclconfig}
+    { $as_echo "$as_me:${as_lineno-$LINENO}: result: found $TCL_BIN_DIR/tclConfig.sh" >&5
+$as_echo "found $TCL_BIN_DIR/tclConfig.sh" >&6; }
+
+    { $as_echo "$as_me:${as_lineno-$LINENO}: checking for existence of $TCL_BIN_DIR/tclConfig.sh" >&5
+$as_echo_n "checking for existence of $TCL_BIN_DIR/tclConfig.sh... " >&6; }
+    if test -f "$TCL_BIN_DIR/tclConfig.sh" ; then
+      { $as_echo "$as_me:${as_lineno-$LINENO}: result: loading" >&5
+$as_echo "loading" >&6; }
+      . $TCL_BIN_DIR/tclConfig.sh
+    else
+      { $as_echo "$as_me:${as_lineno-$LINENO}: result: file not found" >&5
+$as_echo "file not found" >&6; }
+    fi
+
+    #
+    # If the TCL_BIN_DIR is the build directory (not the install directory),
+    # then set the common variable name to the value of the build variables.
+    # For example, the variable TCL_LIB_SPEC will be set to the value
+    # of TCL_BUILD_LIB_SPEC. An extension should make use of TCL_LIB_SPEC
+    # instead of TCL_BUILD_LIB_SPEC since it will work with both an
+    # installed and uninstalled version of Tcl.
+    #
+
+    if test -f $TCL_BIN_DIR/Makefile ; then
+      TCL_LIB_SPEC=${TCL_BUILD_LIB_SPEC}
+      TCL_STUB_LIB_SPEC=${TCL_BUILD_STUB_LIB_SPEC}
+      TCL_STUB_LIB_PATH=${TCL_BUILD_STUB_LIB_PATH}
+    fi
+
+    #
+    # eval is required to do the TCL_DBGX substitution
+    #
+
+    eval "TCL_LIB_FILE=\"${TCL_LIB_FILE}\""
+    eval "TCL_LIB_FLAG=\"${TCL_LIB_FLAG}\""
+    eval "TCL_LIB_SPEC=\"${TCL_LIB_SPEC}\""
+
+    eval "TCL_STUB_LIB_FILE=\"${TCL_STUB_LIB_FILE}\""
+    eval "TCL_STUB_LIB_FLAG=\"${TCL_STUB_LIB_FLAG}\""
+    eval "TCL_STUB_LIB_SPEC=\"${TCL_STUB_LIB_SPEC}\""
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  fi
+fi
+if test "${use_tcl}" = "no" ; then
+  HAVE_TCL=""
+else
+  HAVE_TCL=1
+fi
+
+
 ##########
 # Figure out what C libraries are required to compile programs
 # that use "readline()" library.
@@ -11143,27 +11245,6 @@ fi
 	fi
 fi
 
-# Check whether --with-linenoise was given.
-if test "${with_linenoise+set}" = set; then :
-  withval=$with_linenoise; with_linenoise=$withval
-else
-  with_linenoise="no"
-fi
-
-if test "x$with_linenoise" != "xno"; then
-   TARGET_HAVE_READLINE=0
-   TARGET_HAVE_EDITLINE=0
-   TARGET_HAVE_LINENOISE=1
-   TARGET_READLINE_INC="-I${with_linenoise}"
-   TARGET_READLINE_LIBS="${with_linenoise}/linenoise.c"
-   echo "using linenoise source code at ${with_linenoise}"
-else
-   TARGET_HAVE_LINENOISE=0
-   echo "not using linenoise"
-fi
-
-
-
 
 
 
@@ -11240,7 +11321,7 @@ fi
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking build type" >&5
 $as_echo_n "checking build type... " >&6; }
 if test "${enable_debug}" = "yes" ; then
-  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0 -Wall"
+  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
   { $as_echo "$as_me:${as_lineno-$LINENO}: result: debug" >&5
 $as_echo "debug" >&6; }
 else
@@ -12376,7 +12457,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.47.0, which was
+This file was extended by sqlite $as_me 3.42.0, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -12442,7 +12523,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.47.0
+sqlite config.status 3.42.0
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -74,7 +74,7 @@
 # you don't need (for example BLT) by erasing or commenting out
 # the corresponding code.
 #
-AC_INIT([sqlite],m4_esyscmd(cat VERSION | tr -d '\n'))
+AC_INIT([sqlite],[m4_esyscmd(cat VERSION | tr -d '\n')])
 
 dnl Make sure the local VERSION file matches this configure script
 sqlite_version_sanity_check=`cat $srcdir/VERSION | tr -d '\n'`
@@ -116,101 +116,15 @@ AC_CHECK_FUNCS([fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_s
 USE_AMALGAMATION=1
 
 #########
-# Figure out all the name of a working tclsh and parameters needed to compile against Tcl.
-# The --with-tcl= and/or --with-tclsh= configuration arguments might be useful for this.
-#
-AC_ARG_WITH(tclsh, AS_HELP_STRING([--with-tclsh=PATHNAME],[full pathname of a tclsh to use]))
-AC_ARG_WITH(tcl, AS_HELP_STRING([--with-tcl=DIR],[directory containing (tclConfig.sh)]))
-AC_ARG_ENABLE(tcl, AS_HELP_STRING([--disable-tcl],[omit building accessory programs that require TCL-dev]),
-      [use_tcl=$enableval],[use_tcl=yes])
-original_use_tcl=${use_tcl}
-if test x"${with_tclsh}" == x -a x"${with_tcl}" == x; then
-  AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.6 tclsh tclsh9.0],none)
-  with_tclsh=${TCLSH_CMD}
-fi
-if test x"${with_tclsh}" != x -a x"${with_tclsh}" != xnone; then
-  TCLSH_CMD=${with_tclsh}
-  AC_MSG_RESULT([using tclsh at "$TCLSH_CMD"])
-  if test x"${use_tcl}" = "xyes"; then
-    with_tcl=`${with_tclsh} <${srcdir}/tool/find_tclconfig.tcl`
-    if test x"${with_tcl}" != x; then
-      AC_MSG_RESULT([$TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}])
-    else
-      AC_MSG_WARN([$TCLSH_CMD is unable to recommend a tclConfig.sh])
-      use_tcl=no
-    fi
-  fi
-fi
-if test x"${use_tcl}" = "xyes"; then
-  if test x"${with_tcl}" != x; then
-    if test -r ${with_tcl}/tclConfig.sh; then
-      tclconfig="${with_tcl}/tclConfig.sh"
-    else
-      for i in tcl8.6 tcl9.0 lib; do
-        if test -r ${with_tcl}/$i/tclConfig.sh; then
-          tclconfig=${with_tcl}/$i/tclConfig.sh
-          break
-        fi
-      done
-    fi
-    if test ! -r "${tclconfig}"; then
-      AC_MSG_ERROR([no tclConfig.sh file found under ${with_tcl}])
-    fi
-  else
-    # If we have not yet found a tclConfig.sh file, look in $libdir whic is
-    # set automatically by autoconf or by the --prefix command-line option.
-    # See https://sqlite.org/forum/forumpost/e04e693439a22457
-    libdir=${prefix}/lib
-    if test -r ${libdir}/tclConfig.sh; then
-      tclconfig=${libdir}/tclConfig.sh
-    else
-      for i in tcl8.6 tcl9.0 lib; do
-        if test -r ${libdir}/$i/tclConfig.sh; then
-          tclconfig=${libdir}/$i/tclConfig.sh
-          break
-        fi
-      done
-    fi
-    if test ! -r "${tclconfig}"; then
-      AC_MSG_ERROR([cannot find a usable tclConfig.sh file. 
-        Use --with-tcl=DIR to specify a directory where tclConfig.sh can be found.
-        SQLite does not use TCL internally, but TCL is required to build SQLite
-        from canonical sources and TCL is required for testing.])
-    fi
-  fi
-  AC_MSG_RESULT([loading TCL configuration from ${tclconfig}])
-  . ${tclconfig}
-  AC_SUBST(TCL_INCLUDE_SPEC)
-  AC_SUBST(TCL_LIB_SPEC)
-  AC_SUBST(TCL_STUB_LIB_SPEC)
-  # There are lots of other configuration variables that are provided by the
-  # tclConfig.sh file and that could be included here.  But as of right now,
-  # TCL_LIB_SPEC is the only what that the Makefile uses.
-  HAVE_TCL=1
-elif test x"${original_use_tcl}" = "xno"; then
-  AC_MSG_RESULT([unable to run tests because of --disable-tcl])
-  HAVE_TCL=0
-else
-  AC_MSG_RESULT([unable to run tests because no tclConfig.sh file could be located])
-  HAVE_TCL=0
-fi
-AC_SUBST(HAVE_TCL)
-if test x"$TCLSH_CMD" == x; then
-  TCLSH_CMD=${TCL_EXEC_PREFIX}/bin/tclsh${TCL_VERSION}
-  if test ! -x ${TCLSH_CMD}; then
-    TCLSH_CMD_2=${TCL_EXEC_PREFIX}/bin/tclsh
-    if test ! -x ${TCLSH_CMD_2}; then
-      AC_MSG_WARN([cannot find a usable tclsh at either ${TCLSH_CMD} or ${TCLSH_CMD_2}])
-      TCLSH_CMD=none
-    else
-      TCLSH_CMD=${TCLSH_CMD_2}
-    fi
-  fi
-fi
+# See whether we can run specific tclsh versions known to work well;
+# if not, then we fall back to plain tclsh.
+# TODO: try other versions before falling back?
+# 
+AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.7 tclsh8.6 tclsh8.5 tclsh], none)
 if test "$TCLSH_CMD" = "none"; then
   # If we can't find a local tclsh, then building the amalgamation will fail.
   # We act as though --disable-amalgamation has been used.
-  AC_MSG_WARN([Warning: can't find tclsh - defaulting to non-amalgamation build.])
+  echo "Warning: can't find tclsh - defaulting to non-amalgamation build."
   USE_AMALGAMATION=0
   TCLSH_CMD="tclsh"
 fi
@@ -218,6 +132,7 @@ AC_SUBST(TCLSH_CMD)
 
 AC_ARG_VAR([TCLLIBDIR], [Where to install tcl plugin])
 if test "x${TCLLIBDIR+set}" != "xset" ; then
+  TCLLIBDIR='$(libdir)'
   for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` ; do
     if test -d $i ; then
       TCLLIBDIR=$i
@@ -424,6 +339,190 @@ AC_SUBST(SQLITE_OS_UNIX)
 AC_SUBST(SQLITE_OS_WIN)
 AC_SUBST(TARGET_EXEEXT)
 
+##########
+# Figure out all the parameters needed to compile against Tcl.
+#
+# This code is derived from the SC_PATH_TCLCONFIG and SC_LOAD_TCLCONFIG
+# macros in the in the tcl.m4 file of the standard TCL distribution.
+# Those macros could not be used directly since we have to make some
+# minor changes to accomodate systems that do not have TCL installed.
+#
+AC_ARG_ENABLE(tcl, AS_HELP_STRING([--disable-tcl],[do not build TCL extension]),
+      [use_tcl=$enableval],[use_tcl=yes])
+if test "${use_tcl}" = "yes" ; then
+  AC_ARG_WITH(tcl, AS_HELP_STRING([--with-tcl=DIR],[directory containing tcl configuration (tclConfig.sh)]), with_tclconfig=${withval})
+  AC_MSG_CHECKING([for Tcl configuration])
+  AC_CACHE_VAL(ac_cv_c_tclconfig,[
+    # First check to see if --with-tcl was specified.
+    if test x"${with_tclconfig}" != x ; then
+      if test -f "${with_tclconfig}/tclConfig.sh" ; then
+        ac_cv_c_tclconfig=`(cd ${with_tclconfig}; pwd)`
+      else
+        AC_MSG_ERROR([${with_tclconfig} directory doesn't contain tclConfig.sh])
+      fi
+    fi
+
+    # Start autosearch by asking tclsh
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      if test x"$cross_compiling" = xno; then
+        for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}`
+        do
+          if test -f "$i/tclConfig.sh" ; then
+            ac_cv_c_tclconfig="$i"
+            break
+          fi
+        done
+      fi
+    fi
+
+    # On ubuntu 14.10, $auto_path on tclsh is not quite correct.
+    # So try again after applying corrections.
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      if test x"$cross_compiling" = xno; then
+        for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD} | sed 's,/tcltk/tcl,/tcl,g'`
+        do
+          if test -f "$i/tclConfig.sh" ; then
+            ac_cv_c_tclconfig="$i"
+            break
+          fi
+        done
+      fi
+    fi
+
+    # Recent versions of Xcode on Macs hid the tclConfig.sh file
+    # in a strange place.
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      if test x"$cross_compiling" = xno; then
+        for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib
+        do
+          if test -f "$i/tclConfig.sh" ; then
+            ac_cv_c_tclconfig="$i"
+            break
+          fi
+        done
+      fi
+    fi
+
+    # then check for a private Tcl installation
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      for i in \
+            ../tcl \
+            `ls -dr ../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \
+            `ls -dr ../tcl[[8-9]].[[0-9]] 2>/dev/null` \
+            `ls -dr ../tcl[[8-9]].[[0-9]]* 2>/dev/null` \
+            ../../tcl \
+            `ls -dr ../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \
+            `ls -dr ../../tcl[[8-9]].[[0-9]] 2>/dev/null` \
+            `ls -dr ../../tcl[[8-9]].[[0-9]]* 2>/dev/null` \
+            ../../../tcl \
+            `ls -dr ../../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \
+            `ls -dr ../../../tcl[[8-9]].[[0-9]] 2>/dev/null` \
+            `ls -dr ../../../tcl[[8-9]].[[0-9]]* 2>/dev/null`
+      do
+        if test -f "$i/unix/tclConfig.sh" ; then
+          ac_cv_c_tclconfig=`(cd $i/unix; pwd)`
+          break
+        fi
+      done
+    fi
+
+    # check in a few common install locations
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      for i in \
+            `ls -d ${libdir} 2>/dev/null` \
+            `ls -d /usr/local/lib 2>/dev/null` \
+            `ls -d /usr/contrib/lib 2>/dev/null` \
+            `ls -d /usr/lib 2>/dev/null`
+      do
+        if test -f "$i/tclConfig.sh" ; then
+           ac_cv_c_tclconfig=`(cd $i; pwd)`
+           break
+        fi
+      done
+    fi
+
+    # check in a few other private locations
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      for i in \
+         ${srcdir}/../tcl \
+         `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \
+         `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]] 2>/dev/null` \
+         `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]]* 2>/dev/null`
+      do
+        if test -f "$i/unix/tclConfig.sh" ; then
+          ac_cv_c_tclconfig=`(cd $i/unix; pwd)`
+          break
+        fi
+      done
+    fi
+  ])
+
+  if test x"${ac_cv_c_tclconfig}" = x ; then
+    use_tcl=no
+    AC_MSG_WARN(Can't find Tcl configuration definitions)
+    AC_MSG_WARN(*** Without Tcl the regression tests cannot be executed ***)
+    AC_MSG_WARN(*** Consider using --with-tcl=... to define location of Tcl ***)
+  else
+    TCL_BIN_DIR=${ac_cv_c_tclconfig}
+    AC_MSG_RESULT(found $TCL_BIN_DIR/tclConfig.sh)
+
+    AC_MSG_CHECKING([for existence of $TCL_BIN_DIR/tclConfig.sh])
+    if test -f "$TCL_BIN_DIR/tclConfig.sh" ; then
+      AC_MSG_RESULT([loading])
+      . $TCL_BIN_DIR/tclConfig.sh
+    else
+      AC_MSG_RESULT([file not found])
+    fi
+    
+    #
+    # If the TCL_BIN_DIR is the build directory (not the install directory),
+    # then set the common variable name to the value of the build variables.
+    # For example, the variable TCL_LIB_SPEC will be set to the value
+    # of TCL_BUILD_LIB_SPEC. An extension should make use of TCL_LIB_SPEC
+    # instead of TCL_BUILD_LIB_SPEC since it will work with both an
+    # installed and uninstalled version of Tcl.
+    #
+    
+    if test -f $TCL_BIN_DIR/Makefile ; then
+      TCL_LIB_SPEC=${TCL_BUILD_LIB_SPEC}
+      TCL_STUB_LIB_SPEC=${TCL_BUILD_STUB_LIB_SPEC}
+      TCL_STUB_LIB_PATH=${TCL_BUILD_STUB_LIB_PATH}
+    fi
+    
+    #
+    # eval is required to do the TCL_DBGX substitution
+    #
+    
+    eval "TCL_LIB_FILE=\"${TCL_LIB_FILE}\""
+    eval "TCL_LIB_FLAG=\"${TCL_LIB_FLAG}\""
+    eval "TCL_LIB_SPEC=\"${TCL_LIB_SPEC}\""
+    
+    eval "TCL_STUB_LIB_FILE=\"${TCL_STUB_LIB_FILE}\""
+    eval "TCL_STUB_LIB_FLAG=\"${TCL_STUB_LIB_FLAG}\""
+    eval "TCL_STUB_LIB_SPEC=\"${TCL_STUB_LIB_SPEC}\""
+    
+    AC_SUBST(TCL_VERSION)
+    AC_SUBST(TCL_BIN_DIR)
+    AC_SUBST(TCL_SRC_DIR)
+    AC_SUBST(TCL_INCLUDE_SPEC)
+    
+    AC_SUBST(TCL_LIB_FILE)
+    AC_SUBST(TCL_LIB_FLAG)
+    AC_SUBST(TCL_LIB_SPEC)
+    
+    AC_SUBST(TCL_STUB_LIB_FILE)
+    AC_SUBST(TCL_STUB_LIB_FLAG)
+    AC_SUBST(TCL_STUB_LIB_SPEC)
+    AC_SUBST(TCL_SHLIB_SUFFIX)
+  fi
+fi
+if test "${use_tcl}" = "no" ; then
+  HAVE_TCL=""
+else
+  HAVE_TCL=1
+fi
+AC_SUBST(HAVE_TCL)
+
 ##########
 # Figure out what C libraries are required to compile programs
 # that use "readline()" library.
@@ -499,28 +598,11 @@ if test x"$with_readline" != xno; then
 		TARGET_HAVE_READLINE=1
 	fi
 fi
-AC_ARG_WITH([linenoise],
-            [AS_HELP_STRING([--with-linenoise=DIR],[source directory for linenoise library])],
-            [with_linenoise=$withval],
-            [with_linenoise="no"])
-if test "x$with_linenoise" != "xno"; then
-   TARGET_HAVE_READLINE=0
-   TARGET_HAVE_EDITLINE=0
-   TARGET_HAVE_LINENOISE=1
-   TARGET_READLINE_INC="-I${with_linenoise}"
-   TARGET_READLINE_LIBS="${with_linenoise}/linenoise.c"
-   echo "using linenoise source code at ${with_linenoise}"
-else
-   TARGET_HAVE_LINENOISE=0
-   echo "not using linenoise"
-fi
 
 AC_SUBST(TARGET_READLINE_LIBS)
 AC_SUBST(TARGET_READLINE_INC)
 AC_SUBST(TARGET_HAVE_READLINE)
 AC_SUBST(TARGET_HAVE_EDITLINE)
-AC_SUBST(TARGET_HAVE_LINENOISE)
-
 
 ##########
 # Figure out what C libraries are required to compile programs
@@ -533,7 +615,7 @@ AC_SEARCH_LIBS(fdatasync, [rt])
 AC_ARG_ENABLE(debug, AS_HELP_STRING([--enable-debug],[enable debugging & verbose explain]))
 AC_MSG_CHECKING([build type])
 if test "${enable_debug}" = "yes" ; then
-  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0 -Wall"
+  TARGET_DEBUG="-DSQLITE_DEBUG=1 -DSQLITE_ENABLE_SELECTTRACE -DSQLITE_ENABLE_WHERETRACE -O0"
   AC_MSG_RESULT([debug])
 else
   TARGET_DEBUG="-DNDEBUG"

doc/compile-for-windows.md

@@ -1,156 +0,0 @@
-# Notes On Compiling SQLite On Windows 11
-
-Here are step-by-step instructions on how to build SQLite from
-canonical source on a new Windows 11 PC, as of 2023-11-01:
-
-  1.  Install Microsoft Visual Studio. The free "community edition" 
-      will work fine.  Do a standard install for C++ development.
-      SQLite only needs the
-      "cl" compiler and the "nmake" build tool.
-
-  2.  Under the "Start" menu, find "All Apps" then go to "Visual Studio 20XX"
-      and find "x64 Native Tools Command Prompt for VS 20XX".  Pin that
-      application to your task bar, as you will use it a lot.  Bring up
-      an instance of this command prompt and do all of the subsequent steps
-      in that "x64 Native Tools" command prompt.  (Or use "x86" if you want
-      a 32-bit build.)  The subsequent steps will not work in a vanilla
-      DOS prompt.  Nor will they work in PowerShell.
-
-  3.  Install TCL development libraries.  This note assumes that you will
-      install the TCL development libraries in the "`c:\Tcl`" directory.
-      Make adjustments
-      if you want TCL installed somewhere else.  SQLite needs both the
-      "tclsh.exe" command-line tool as part of the build process, and
-      the "tcl86.lib" library in order to run tests.  You will need
-      TCL version 8.6 or later.
-      <ol type="a">
-      <li>Get the TCL source archive, perhaps from
-      [https://www.tcl.tk/software/tcltk/download.html](https://www.tcl.tk/software/tcltk/download.html).
-      <li>Untar or unzip the source archive.  CD into the "win/" subfolder
-          of the source tree.
-      <li>Run: `nmake /f makefile.vc release`
-      <li>Run: `nmake /f makefile.vc INSTALLDIR=c:\Tcl install`
-      <li>CD to `c:\Tcl\lib`.  In that subfolder make a copy of the
-          "`tcl86t.lib`" file to the alternative name "`tcl86.lib`"
-          (omitting the second 't').  Leave the copy in the same directory
-          as the original.
-      <li>CD to `c:\Tcl\bin`.  Make a copy of the "`tclsh86t.exe`"
-          file into "`tclsh.exe`" (without the "86t") in the same directory.
-      <li>Add `c:\Tcl\bin` to your %PATH%.  To do this, go to Settings
-          and search for "path".  Select "edit environment variables for
-          your account" and modify your default PATH accordingly.
-          You will need to close and reopen your command prompts after
-          making this change.
-      </ol>
-
-  4.  Download the SQLite source tree and unpack it. CD into the
-      toplevel directory of the source tree.
-
-  5.  Set the TCLDIR environment variable to point to your TCL installation.
-      Like this:
-      <ul>
-      <li> `set TCLDIR=c:\Tcl`
-      </ul>
-
-  6.  Run the "`Makefile.msc`" makefile with an appropriate target.
-      Examples:
-      <ul>
-      <li>  `nmake /f makefile.msc`
-      <li>  `nmake /f makefile.msc sqlite3.c`
-      <li>  `nmake /f makefile.msc devtest`
-      <li>  `nmake /f makefile.msc releasetest`
-      <li>  `nmake /f makefile.msc sqlite3.exe`
-      </ul>
-
-  7.  For a debugging build of the CLI, where the ".treetrace" and ".wheretrace"
-      commands work, add the DEBUG=3 argument to nmake.  Like this:
-      <ul>
-      <li> `nmake /f makefile.msc DEBUG=3 clean sqlite3.exe`
-      </ul>
-   
-
-## 32-bit Builds
-
-Doing a 32-bit build is just like doing a 64-bit build with the
-following minor changes:
-
-  1.  Use the "x86 Native Tools Command Prompt" instead of
-      "x64 Native Tools Command Prompt".  "**x86**" instead of "**x64**".
-
-  2.  Use a different installation directory for TCL.
-      The recommended directory is `c:\tcl32`.  Thus you end up
-      with two TCL builds:
-      <ul>
-      <li> `c:\tcl` &larr;  64-bit (the default)
-      <li> `c:\tcl32` &larr;  32-bit
-      </ul>
-
-  3.  Ensure that `c:\tcl32\bin` comes before `c:\tcl\bin` on
-      your PATH environment variable.  You can achieve this using
-      a command like:
-      <ul>
-      <li>  `set PATH=c:\tcl32\bin;%PATH%`
-      </ul>
-
-## Building a DLL
-
-The command the developers use for building the deliverable DLL on the 
-[download page](https://sqlite.org/download.html) is as follows:
-
-> ~~~~
-nmake /f Makefile.msc sqlite3.dll USE_NATIVE_LIBPATHS=1 "OPTS=-DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_FTS5=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_JSON1=1 -DSQLITE_ENABLE_GEOPOLY=1 -DSQLITE_ENABLE_SESSION=1 -DSQLITE_ENABLE_PREUPDATE_HOOK=1 -DSQLITE_ENABLE_SERIALIZE=1 -DSQLITE_ENABLE_MATH_FUNCTIONS=1"
-~~~~
-
-That command generates both the sqlite3.dll and sqlite3.def files.  The same
-command works for both 32-bit and 64-bit builds.
-
-## Statically Linking The TCL Library
-
-Some utility programs associated with SQLite need to be linked
-with TCL in order to function.  The [sqlite3_analyzer.exe program](https://sqlite.org/sqlanalyze.html)
-is an example.  You can build as described above, and then
-enter:
-
-> ~~~~
-nmake /f Makefile.msc sqlite3_analyzer.exe
-~~~~
-
-And you will end up with a working executable.  However, that executable
-will depend on having the "tcl86.dll" library somewhere on your %PATH%.
-Use the following steps to build an executable that has the TCL library
-statically linked so that it does not depend on separate DLL:
-
-  1.  Use the appropriate "Command Prompt" window - either x86 or
-      x64, depending on whether you want a 32-bit or 64-bit executable.
-
-  2.  Untar the TCL source tarball into a fresh directory.  CD into
-      the "win/" subfolder.
-
-  3.  Run: `nmake /f makefile.vc OPTS=nothreads,static shell`
-
-
-  4.  CD into the "Release*" subfolder that is created (note the
-      wildcard - the full name of the directory might vary).  There
-      you will find the "tcl86s.lib" file.  Copy this file into the
-      same directory that you put the "tcl86.lib" on your initial
-      installation.  (In this document, that directory is
-      "`C:\Tcl32\lib`" for 32-bit builds and
-      "`C:\Tcl\lib`" for 64-bit builds.)
-
-  5.  CD into your SQLite source code directory and build the desired
-      utility program, but add the following extra arguments to the
-      nmake command line:
-      <blockquote><pre>
-      CCOPTS="-DSTATIC_BUILD" LIBTCL="tcl86s.lib netapi32.lib user32.lib"
-      </pre></blockquote>
-      <p>So, for example, to build a statically linked version of
-      sqlite3_analyzer.exe, you might type:
-      <blockquote><pre>
-      nmake /f Makefile.msc CCOPTS="-DSTATIC_BUILD" LIBTCL="tcl86s.lib netapi32.lib user32.lib" sqlite3_analyzer.exe
-      </pre></blockquote>
-
-  6.  After your executable is built, you can verify that it does not
-      depend on the TCL DLL by running:
-      <blockquote><pre>
-      dumpbin /dependents sqlite3_analyzer.exe
-      </pre></blockquote>

doc/jsonb.md

@@ -1,290 +0,0 @@
-# The JSONB Format
-
-This document describes SQLite's JSONB binary encoding of
-JSON.
-
-## 1.0 What Is JSONB?
-
-Beginning with version 3.45.0 (circa 2024-01-01), SQLite supports an
-alternative binary encoding of JSON which we call "JSONB".  JSONB is
-a binary format that stored as a BLOB.
-
-The advantage of JSONB over ordinary text RFC 8259 JSON is that JSONB
-is both slightly smaller (by between 5% and 10% in most cases) and
-can be processed in less than half the number of CPU cycles.  The built-in
-[JSON SQL functions] of SQLite can accept either ordinary text JSON
-or the binary JSONB encoding for any of their JSON inputs.
-
-The "JSONB" name is inspired by [PostgreSQL](https://postgresql.org), but the
-on-disk format for SQLite's JSONB is not the same as PostgreSQL's.
-The two formats have the same name, but they have wildly different internal
-representations and are not in any way binary compatible.
-
-The central idea behind this JSONB specification is that each element
-begins with a header that includes the size and type of that element.
-The header takes the place of punctuation such as double-quotes,
-curly-brackes, square-brackets, commas, and colons.  Since the size
-and type of each element is contained in its header, the element can
-be read faster since it is no longer necessary to carefully scan forward
-looking for the closing delimiter.  The payload of JSONB is the same
-as for corresponding text JSON.  The same payload bytes occur in the
-same order.  The only real difference between JSONB and ordinary text
-JSON is that JSONB includes a binary header on
-each element and omits delimiter and separator punctuation.
-
-### 1.1 Internal Use Only
-
-The details of the JSONB are not intended to be visible to application
-developers.  Application developers should look at JSONB as an opaque BLOB
-used internally by SQLite.  Nevertheless, we want the format to be backwards
-compatible across all future versions of SQLite.  To that end, the format
-is documented by this file in the source tree.  But this file should be
-used only by SQLite core developers, not by developers of applications
-that only use SQLite.
-
-## 2.0 The Purpose Of This Document
-
-JSONB is not intended as an external format to be used by
-applications.  JSONB is designed for internal use by SQLite only.
-Programmers do not need to understand the JSONB format in order to
-use it effectively.
-Applications should access JSONB only through the [JSON SQL functions],
-not by looking at individual bytes of the BLOB.
-
-However, JSONB is intended to be portable and backwards compatible
-for all future versions of SQLite.  In other words, you should not have
-to export and reimport your SQLite database files when you upgrade to
-a newer SQLite version.  For that reason, the JSONB format needs to
-be well-defined.
-
-This document is therefore similar in purpose to the
-[SQLite database file format] document that describes the on-disk
-format of an SQLite database file.  Applications are not expected
-to directly read and write the bits and bytes of SQLite database files.
-The SQLite database file format is carefully documented so that it
-can be stable and enduring.  In the same way, the JSONB representation
-of JSON is documented here so that it too can be stable and enduring,
-not so that applications can read or writes individual bytes.
-
-## 3.0 Encoding
-
-JSONB is a direct translation of the underlying text JSON. The difference
-is that JSONB uses a binary encoding that is faster to parse compared to
-the detailed syntax of text JSON.
-
-Each JSON element is encoded as a header and a payload.  The header
-determines type of element (string, numeric, boolean, null, object, or
-array) and the size of the payload.  The header can be between 1 and
-9 bytes in size.  The payload can be any size from zero bytes up to the
-maximum allowed BLOB size.
-
-### 3.1 Payload Size
-
-The upper four bits of the first byte of the header determine size of the
-header and possibly also the size of the payload.
-If the upper four bits have a value between 0 and 11, then the header is
-exactly one byte in size and the payload size is determined by those
-upper four bits.  If the upper four bits have a value between 12 and 15,
-that means that the total header size is 2, 3, 5, or 9 bytes and the
-payload size is unsigned big-endian integer that is contained in the
-subsequent bytes.  The size integer is the one byte that following the
-initial header byte if the upper four bits
-are 12, two bytes if the upper bits are 13, four bytes if the upper bits
-are 14, and eight bytes if the upper bits are 15.  The current design
-of SQLite does not support BLOB values larger than 2GiB, so the eight-byte
-variant of the payload size integer will never be used by the current code.
-The eight-byte payload size integer is included in the specification
-to allow for future expansion.
-
-The header for an element does *not* need to be in its simplest
-form.  For example, consider the JSON numeric value "`1`".
-That element can be encode in five different ways:
-
-  *  `0x13 0x31`
-  *  `0xc3 0x01 0x31`
-  *  `0xd3 0x00 0x01 0x31`
-  *  `0xe3 0x00 0x00 0x00 0x01 0x31`
-  *  `0xf3 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x01 0x31`
-
-The shortest encoding is preferred, of course, and usually happens with
-primitive elements such as numbers.  However the total size of an array
-or object might not be known exactly when the header of the element is
-first generated.  It is convenient to reserve space for the largest
-possible header and then go back and fill in the correct payload size
-at the end.  This technique can result in array or object headers that
-are larger than absolutely necessary.
-
-### 3.2 Element Type
-
-The least-significant four bits of the first byte of the header (the first
-byte masked against 0x0f) determine element type.  The following codes are
-used:
-
-<ol>
-<li type="0"><p><b>NULL</b> &rarr;
-The element is a JSON "null".  The payload size for a true JSON NULL must
-must be zero.  Future versions of SQLite might extend the JSONB format
-with elements that have a zero element type but a non-zero size.  In that
-way, legacy versions of SQLite will interpret the element as a NULL 
-for backwards compatibility while newer versions will interpret the
-element in some other way.
-
-<li value="1"><p><b>TRUE</b> &rarr;
-The element is a JSON "true".  The payload size must be zero for a actual
-"true" value.  Elements with type 1 and a non-zero payload size are
-reserved for future expansion.  Legacy implementations that see an element
-type of 1 with a non-zero payload size should continue to interpret that
-element as "true" for compatibility.
-
-<li value="2"><p><b>FALSE</b> &rarr;
-The element is a JSON "false".  The payload size must be zero for a actual
-"false" value.  Elements with type 2 and a non-zero payload size are
-reserved for future expansion.  Legacy implementations that see an element
-type of 2 with a non-zero payload size should continue to interpret that
-element as "false" for compatibility.
-
-<li value="3"><p><b>INT</b> &rarr;
-The element is a JSON integer value in the canonical
-RFC 8259 format, without extensions.  The payload is the ASCII
-text representation of that numeric value.
-
-<li value="4"><p><b>INT5</b> &rarr;
-The element is a JSON integer value that is not in the
-canonical format.   The payload is the ASCII
-text representation of that numeric value.  Because the payload is in a
-non-standard format, it will need to be translated when the JSONB is
-converted into RFC 8259 text JSON.
-
-<li value="5"><p><b>FLOAT</b> &rarr;
-The element is a JSON floating-point value in the canonical
-RFC 8259 format, without extensions.  The payload is the ASCII
-text representation of that numeric value.
-
-<li value="6"><p><b>FLOAT5</b> &rarr;
-The element is a JSON floating-point value that is not in the
-canonical format.   The payload is the ASCII
-text representation of that numeric value.  Because the payload is in a
-non-standard format, it will need to be translated when the JSONB is
-converted into RFC 8259 text JSON.
-
-<li value="7"><p><b>TEXT</b> &rarr;
-The element is a JSON string value that does not contain
-any escapes nor any characters that need to be escaped for either SQL or
-JSON.  The payload is the UTF8 text representation of the string value.
-The payload does <i>not</i> include string delimiters.
-
-<li value="8"><p><b>TEXTJ</b> &rarr;
-The element is a JSON string value that contains
-RFC 8259 character escapes (such as "<tt>\n</tt>" or "<tt>\u0020</tt>").
-Those escapes will need to be translated into actual UTF8 if this element
-is [json_extract|extracted] into SQL.
-The payload is the UTF8 text representation of the escaped string value.
-The payload does <i>not</i> include string delimiters.
-
-<li value="9"><p><b>TEXT5</b> &rarr;
-The element is a JSON string value that contains
-character escapes, including some character escapes that part of JSON5
-and which are not found in the canonical RFC 8259 spec.
-Those escapes will need to be translated into standard JSON prior to
-rendering the JSON as text, or into their actual UTF8 characters if this
-element is [json_extract|extracted] into SQL.
-The payload is the UTF8 text representation of the escaped string value.
-The payload does <i>not</i> include string delimiters.
-
-<li value="10"><p><b>TEXTRAW</b> &rarr;
-The element is a JSON string value that contains
-UTF8 characters that need to be escaped if this string is rendered into
-standard JSON text.
-The payload does <i>not</i> include string delimiters.
-
-<li value="11"><p><b>ARRAY</b> &rarr;
-The element is a JSON array.  The payload contains
-JSONB elements that comprise values contained within the array.
-
-<li value="12"><p><b>OBJECT</b> &rarr;
-The element is a JSON object.  The payload contains
-pairs of JSONB elements that comprise entries for the JSON object.
-The first element in each pair must be a string (types 7 through 10).
-The second element of each pair may be any types, including nested
-arrays or objects.
-
-<li value="13"><p><b>RESERVED-13</b> &rarr;
-Reserved for future expansion.  Legacy implements that encounter this
-element type should raise an error.
-
-<li value="14"><p><b>RESERVED-14</b> &rarr;
-Reserved for future expansion.  Legacy implements that encounter this
-element type should raise an error.
-
-<li value="15"><p><b>RESERVED-15</b> &rarr;
-Reserved for future expansion.  Legacy implements that encounter this
-element type should raise an error.
-</ol>
-
-Element types outside the range of 0 to 12 are reserved for future
-expansion.  The current implement raises an error if see an element type
-other than those listed above.  However, future versions of SQLite might
-use of the three remaining element types to implement indexing or similar
-optimizations, to speed up lookup against large JSON arrays and/or objects.
-
-### 3.3 Design Rationale For Element Types
-
-A key goal of JSONB is that it should be quick to translate
-to and from text JSON and/or be constructed from SQL values.
-When converting from text into JSONB, we do not want the
-converter subroutine to burn CPU cycles converting elements
-values into some standard format which might never be used.
-Format conversion is "lazy" - it is deferred until actually
-needed.  This has implications for the JSONB format design:
-
-  1.   Numeric values are stored as text, not a numbers.  The values are
-       a direct copy of the text JSON values from which they are derived.
-
-  2.   There are multiple element types depending on the details of value
-       formats.  For example, INT is used for pure RFC-8259 integer
-       literals and INT5 exists for JSON5 extensions such as hexadecimal
-       notation.  FLOAT is used for pure RFC-8259 floating point literals
-       and FLOAT5 is used for JSON5 extensions.  There are four different
-       representations of strings, depending on where the string came from
-       and how special characters within the string are escaped.
-
-A second goal of JSONB is that it should be capable of serving as the
-"parse tree" for JSON when a JSON value is being processed by the
-various [JSON SQL functions] built into SQLite.  Before JSONB was
-developed, operations such [json_replace()] and [json_patch()]
-and similar worked in three stages:
-
-
-  1.  Translate the text JSON into a internal format that is
-      easier to scan and edit.
-  2.  Perform the requested operation on the JSON.
-  3.  Translate the internal format back into text.
-
-JSONB seeks to serve as the internal format directly - bypassing
-the first and third stages of that process.  Since most of the CPU
-cycles are spent on the first and third stages, that suggests that
-JSONB processing will be much faster than text JSON processing.
-
-So when processing JSONB, only the second stage of the three-stage
-process is required.  But when processing text JSON, it is still necessary
-to do stages one and three.  If JSONB is to be used as the internal
-binary representation, this is yet another reason to store numeric
-values as text.  Storing numbers as text minimizes the amount of
-conversion work needed for stages one and three.  This is also why
-there are four different representations of text in JSONB.  Different
-text representations are used for text coming from different sources
-(RFC-8259 JSON, JSON5, or SQL string values) and conversions only
-happen if and when they are actually needed.
-
-### 3.4 Valid JSONB BLOBs
-
-A valid JSONB BLOB consists of a single JSON element.  The element must
-exactly fill the BLOB.  This one element is often a JSON object or array
-and those usually contain additional elements as its payload, but the
-element can be a primite value such a string, number, boolean, or null.
-
-When the built-in JSON functions are attempting to determine if a BLOB
-argument is a JSONB or just a random BLOB, they look at the header of
-the outer element to see that it is well-formed and that the element
-completely fills the BLOB.  If these conditions are met, then the BLOB
-is accepted as a JSONB value.

doc/lemon.html

@@ -322,7 +322,7 @@ These are the steps:
 <ul>
 <li> Declare a local variable of type "yyParser"
 <li> Initialize the variable using ParseInit()
-<li> Pass a pointer to the variable in calls to Parse()
+<li> Pass a pointer to the variable in calls ot Parse()
 <li> Deallocate substructure in the parse variable using ParseFinalize().
 </ul>
 
@@ -683,7 +683,6 @@ other than that, the order of directives in Lemon is arbitrary.</p>
 <li><tt><a href='#pifdef'>%endif</a></tt>
 <li><tt><a href='#extraarg'>%extra_argument</a></tt>
 <li><tt><a href='#pfallback'>%fallback</a></tt>
-<li><tt><a href='#reallc'>%free</a></tt>
 <li><tt><a href='#pifdef'>%if</a></tt>
 <li><tt><a href='#pifdef'>%ifdef</a></tt>
 <li><tt><a href='#pifdef'>%ifndef</a></tt>
@@ -694,7 +693,6 @@ other than that, the order of directives in Lemon is arbitrary.</p>
 <li><tt><a href='#parse_accept'>%parse_accept</a></tt>
 <li><tt><a href='#parse_failure'>%parse_failure</a></tt>
 <li><tt><a href='#pright'>%right</a></tt>
-<li><tt><a href='#reallc'>%realloc</a></tt>
 <li><tt><a href='#stack_overflow'>%stack_overflow</a></tt>
 <li><tt><a href='#stack_size'>%stack_size</a></tt>
 <li><tt><a href='#start_symbol'>%start_symbol</a></tt>
@@ -1202,21 +1200,6 @@ match any input token.</p>
 the wildcard token and some other token, the other token is always used.
 The wildcard token is only matched if there are no alternatives.</p>
 
-<a id='reallc'></a>
-<h4>4.4.26 The <tt>%realloc</tt> and <tt>%free</tt> directives</h4>
-
-<p>The <tt>%realloc</tt> and <tt>%free</tt> directives defines function
-that allocate and free heap memory.  The signatures of these functions
-should be the same as the realloc() and free() functions from the standard
-C library.
-
-<p>If both of these functions are defined
-then these functions are used to allocate and free
-memory for supplemental parser stack space, if the initial
-parse stack space is exceeded.  The initial parser stack size
-is specified by either <tt>%stack_size</tt> or the
--DYYSTACKDEPTH compile-time flag.
-
 <a id='errors'></a>
 <h2>5.0 Error Processing</h2>
 
@@ -1241,14 +1224,13 @@ to begin parsing a new file.  This is what will happen at the very
 first syntax error, of course, if there are no instances of the
 "error" non-terminal in your grammar.</p>
 
-
 <a id='history'></a>
 <h2>6.0 History of Lemon</h2>
 
 <p>Lemon was originally written by Richard Hipp sometime in the late
 1980s on a Sun4 Workstation using K&amp;R C.  
-There was a companion LL(1) parser generator program named "Lime".
-The Lime source code has been lost.</p>
+There was a companion LL(1) parser generator program named "Lime", the
+source code to which as been lost.</p>
 
 <p>The lemon.c source file was originally many separate files that were
 compiled together to generate the "lemon" executable.  Sometime in the

doc/testrunner.md

@@ -1,356 +0,0 @@
-
-
-# The testrunner.tcl Script
-
-<ul type=none>
-  <li> 1. <a href=#overview>Overview</a>
-  <li> 2. <a href=#binary_tests>Binary Tests</a>
-<ul type=none>
-  <li> 2.1. <a href=#organization_tests>Organization of Tcl Tests</a>
-  <li> 2.2. <a href=#run_tests>Commands to Run Tests</a>
-  <li> 2.3. <a href=#binary_test_failures>Investigating Binary Test Failures</a>
-</ul>
-  <li> 3. <a href=#source_code_tests>Source Tests</a>
-<ul type=none>
-  <li> 3.1. <a href=#commands_to_run_tests>Commands to Run SQLite Tests</a>
-  <li> 3.2. <a href=#zipvfs_tests>Running ZipVFS Tests</a>
-  <li> 3.3. <a href=#source_code_test_failures>Investigating Source Code Test Failures</a>
-</ul>
-  <li> 4. <a href=#testrunner_options>Extra testrunner.tcl Options</a>
-  <li> 5. <a href=#cpu_cores>Controlling CPU Core Utilization</a>
-</ul>
-
-<a name=overview></a>
-# 1. Overview
-
-testrunner.tcl is a Tcl script used to run multiple SQLite tests using 
-multiple jobs. It supports the following types of tests:
-
-  *  Tcl test scripts.
-
-  *  Tests run with `make` commands.  Examples:
-      -  `make mdevtest`
-      -  `make releasetest`
-      -  `make sdevtest`
-      -  `make testrunner`
-
-testrunner.tcl pipes the output of all tests and builds run into log file
-**testrunner.log**, created in the current working directory. Search this
-file to find details of errors.  Suggested search commands:
-
-   *  `grep "^!" testrunner.log`
-   *  `grep failed testrunner.log`
-
-testrunner.tcl also populates SQLite database **testrunner.db**. This database
-contains details of all tests run, running and to be run. A useful query
-might be:
-
-```
-  SELECT * FROM script WHERE state='failed'
-```
-
-Running the command:
-
-```
-  ./testfixture $(TESTDIR)/testrunner.tcl status
-```
-
-in the directory containing the testrunner.db database runs various queries
-to produce a succinct report on the state of a running testrunner.tcl script.
-Running:
-
-```
-  watch ./testfixture $(TESTDIR)/testrunner.tcl status
-```
-
-in another terminal is a good way to keep an eye on a long running test.
-
-Sometimes testrunner.tcl uses the `testfixture` binary that it is run with
-to run tests (see "Binary Tests" below). Sometimes it builds testfixture and
-other binaries in specific configurations to test (see "Source Tests").
-
-<a name=binary_tests></a>
-# 2. Binary Tests
-
-The commands described in this section all run various combinations of the Tcl
-test scripts using the `testfixture` binary used to run the testrunner.tcl
-script (i.e. they do not invoke the compiler to build new binaries, or the
-`make` command to run tests that are not Tcl scripts). The procedure to run
-these tests is therefore:
-
-  1. Build the "testfixture" (or "testfixture.exe" for windows) binary using
-     whatever method seems convenient.
-
-  2. Test the binary built in step 1 by running testrunner.tcl with it, 
-     perhaps with various options.
-
-The following sub-sections describe the various options that can be
-passed to testrunner.tcl to test binary testfixture builds.
-
-<a name=organization_tests></a>
-## 2.1. Organization of Tcl Tests
-
-Tcl tests are stored in files that match the pattern *\*.test*. They are
-found in both the $TOP/test/ directory, and in the various sub-directories
-of the $TOP/ext/ directory of the source tree. Not all *\*.test* files
-contain Tcl tests - a handful are Tcl scripts designed to invoke other
-*\*.test* files.
-
-The **veryquick** set of tests is a subset of all Tcl test scripts in the
-source tree. In includes most tests, but excludes some that are very slow.
-Almost all fault-injection tests (those that test the response of the library
-to OOM or IO errors) are excluded. It is defined in source file 
-*test/permutations.test*.
-
-The **full** set of tests includes all Tcl test scripts in the source tree.
-To run a "full" test is to run all Tcl test scripts that can be found in the
-source tree.
-
-File *permutations.test* defines various test "permutations". A permutation
-consists of:
-
-  *  A subset of Tcl test scripts, and 
-
-  *  Runtime configuration to apply before running each test script 
-     (e.g. enabling auto-vacuum, or disable lookaside).
-
-Running **all** tests is to run all tests in the full test set, plus a dozen
-or so permutations. The specific permutations that are run as part of "all"
-are defined in file *testrunner_data.tcl*.
-
-<a name=run_tests></a>
-## 2.2. Commands to Run Tests
-
-To run the "veryquick" test set, use either of the following:
-
-```
-  ./testfixture $TESTDIR/testrunner.tcl
-  ./testfixture $TESTDIR/testrunner.tcl veryquick
-```
-
-To run the "full" test suite:
-
-```
-  ./testfixture $TESTDIR/testrunner.tcl full
-```
-
-To run the subset of the "full" test suite for which the test file name matches
-a specified pattern (e.g. all tests that start with "fts5"), either of:
-
-```
-  ./testfixture $TESTDIR/testrunner.tcl fts5%
-  ./testfixture $TESTDIR/testrunner.tcl 'fts5*'
-```
-
-Strictly speaking, for a test to be run the pattern must match the script
-filename, not including the directory, using the rules of Tcl's 
-\[string match\] command. Except that before the matching is done, any "%"
-characters specified as part of the pattern are transformed to "\*".
-
-
-To run "all" tests (full + permutations):
-
-```
-  ./testfixture $TESTDIR/testrunner.tcl all
-```
-
-<a name=binary_test_failures></a>
-## 2.3. Investigating Binary Test Failures
-
-If a test fails, testrunner.tcl reports name of the Tcl test script and, if
-applicable, the name of the permutation, to stdout. This information can also
-be retrieved from either *testrunner.log* or *testrunner.db*.
-
-If there is no permutation, the individual test script may be run with:
-
-```
-  ./testfixture $PATH_TO_SCRIPT
-```
-
-Or, if the failure occured as part of a permutation:
-
-```
-  ./testfixture $TESTDIR/testrunner.tcl $PERMUTATION $PATH_TO_SCRIPT
-```
-
-TODO: An example instead of "$PERMUTATION" and $PATH\_TO\_SCRIPT?
-
-<a name=source_code_tests></a>
-# 3. Source Code Tests
-
-The commands described in this section invoke the C compiler to build 
-binaries from the source tree, then use those binaries to run Tcl and
-other tests. The advantages of this are that:
-
-  *  it is possible to test multiple build configurations with a single
-     command, and 
-
-  *  it ensures that tests are always run using binaries created with the
-     same set of compiler options.
-
-The testrunner.tcl commands described in this section may be run using
-either a *testfixture* (or testfixture.exe) build, or with any other Tcl
-shell that supports SQLite 3.31.1 or newer via "package require sqlite3".
-
-TODO: ./configure + Makefile.msc build systems.
-
-<a name=commands_to_run_tests></a>
-## 3.1. Commands to Run SQLite Tests
-
-The **mdevtest** command is equivalent to running the veryquick tests and
-the `make fuzztest` target once for each of two --enable-all builds - one 
-with debugging enabled and one without:
-
-```
-  tclsh $TESTDIR/testrunner.tcl mdevtest
-```
-
-In other words, it is equivalent to running:
-
-```
-  $TOP/configure --enable-all --enable-debug
-  make fuzztest
-  make testfixture
-  ./testfixture $TOP/test/testrunner.tcl veryquick
-
-  # Then, after removing files created by the tests above:
-  $TOP/configure --enable-all OPTS="-O0"
-  make fuzztest
-  make testfixture
-  ./testfixture $TOP/test/testrunner.tcl veryquick
-```
-
-The **sdevtest** command is identical to the mdevtest command, except that the
-second of the two builds is a sanitizer build. Specifically, this means that
-OPTS="-fsanitize=address,undefined" is specified instead of OPTS="-O0":
-
-```
-  tclsh $TESTDIR/testrunner.tcl sdevtest
-```
-
-The **release** command runs lots of tests under lots of builds. It runs
-different combinations of builds and tests depending on whether it is run
-on Linux, Windows or OSX. Refer to *testrunner\_data.tcl* for the details
-of the specific tests run.
-
-```
-  tclsh $TESTDIR/testrunner.tcl release
-```
-
-As with <a href=#source code tests>source code tests</a>, one or more patterns
-may be appended to any of the above commands (mdevtest, sdevtest or release).
-In that case only Tcl tests (no fuzz or other tests) that match the specified
-pattern are run. For example, to run the just the Tcl rtree tests in all 
-builds and configurations supported by "release":
-
-```
-  tclsh $TESTDIR/testrunner.tcl release rtree%
-```
-
-<a name=zipvfs_tests></a>
-## 3.2. Running ZipVFS Tests
-
-testrunner.tcl can build a zipvfs-enabled testfixture and use it to run
-tests from the Zipvfs project with the following command:
-
-```
-  tclsh $TESTDIR/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS
-```
-
-This can be combined with any of "mdevtest", "sdevtest" or "release" to
-test both SQLite and Zipvfs with a single command:
-
-```
-  tclsh $TESTDIR/testrunner.tcl --zipvfs $PATH_TO_ZIPVFS mdevtest
-```
-
-<a name=source_code_test_failures></a>
-## 3.3. Investigating Source Code Test Failures
-
-Investigating a test failure that occurs during source code testing is a
-two step process:
-
-  1. Recreating the build configuration in which the test failed, and
-
-  2. Re-running the actual test.
-
-To recreate a build configuration, use the testrunner.tcl **script** command
-to create a build script. A build script is a bash script on Linux or OSX, or
-a dos \*.bat file on windows. For example:
-
-```
-  # Create a script that recreates build configuration "Device-One" on 
-  # Linux or OSX:
-  tclsh $TESTDIR/testrunner.tcl script Device-One > make.sh 
-
-  # Create a script that recreates build configuration "Have-Not" on Windows:
-  tclsh $TESTDIR/testrunner.tcl script Have-Not > make.bat 
-```
-
-The generated bash or \*.bat file script accepts a single argument - a makefile
-target to build. This may be used either to run a `make` command test directly,
-or else to build a testfixture (or testfixture.exe) binary with which to
-run a Tcl test script, as <a href=#binary_test_failures>described above</a>.
-
-<a name=testrunner_options></a>
-# 4. Extra testrunner.tcl Options
-
-The testrunner.tcl script options in this section may be used with both source
-code and binary tests.
-
-The **--buildonly** option instructs testrunner.tcl just to build the binaries
-required by a test, not to run any actual tests. For example:
-
-```
-  # Build binaries required by release test.
-  tclsh $TESTDIR/testrunner.tcl --buildonly release"
-```
-
-The **--dryrun** option prevents testrunner.tcl from building any binaries
-or running any tests. Instead, it just writes the shell commands that it
-would normally execute into the testrunner.log file. Example:
-
-```
-  # Log the shell commmands that make up the mdevtest test.
-  tclsh $TESTDIR/testrunner.tcl --dryrun mdevtest"
-```
-
-The **--explain** option is similar to --dryrun in that it prevents testrunner.tcl
-from building any binaries or running any tests.  The difference is that --explain
-prints on standard output a human-readable summary of all the builds and tests that
-would have been run.
-
-```
-  # Show what builds and tests would have been run
-  tclsh $TESTDIR/testrunner.tcl --explain mdevtest
-```
-
-<a name=cpu_cores></a>
-# 5. Controlling CPU Core Utilization
-
-When running either binary or source code tests, testrunner.tcl reports the
-number of jobs it intends to use to stdout. e.g.
-
-```
-  $ ./testfixture $TESTDIR/testrunner.tcl
-  splitting work across 16 jobs
-  ... more output ...
-```
-
-By default, testfixture.tcl attempts to set the number of jobs to the number 
-of real cores on the machine. This can be overridden using the "--jobs" (or -j)
-switch:
-
-```
-  $ ./testfixture $TESTDIR/testrunner.tcl --jobs 8
-  splitting work across 8 jobs
-  ... more output ...
-```
-
-The number of jobs may also be changed while an instance of testrunner.tcl is
-running by exucuting the following command from the directory containing the
-testrunner.log and testrunner.db files:
-
-```
-  $ ./testfixture $TESTDIR/testrunner.tcl njob $NEW_NUMBER_OF_JOBS
-```

ext/consio/console_io.c

@@ -1,686 +0,0 @@
-/*
-** 2023 November 4
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-********************************************************************************
-** This file implements various interfaces used for console and stream I/O
-** by the SQLite project command-line tools, as explained in console_io.h .
-** Functions prefixed by "SQLITE_INTERNAL_LINKAGE" behave as described there.
-*/
-
-#ifndef SQLITE_CDECL
-# define SQLITE_CDECL
-#endif
-
-#ifndef SHELL_NO_SYSINC
-# include <stdarg.h>
-# include <string.h>
-# include <stdlib.h>
-# include <limits.h>
-# include <assert.h>
-# include "sqlite3.h"
-#endif
-#ifndef HAVE_CONSOLE_IO_H
-# include "console_io.h"
-#endif
-#if defined(_MSC_VER)
-# pragma warning(disable : 4204)
-#endif
-
-#ifndef SQLITE_CIO_NO_TRANSLATE
-# if (defined(_WIN32) || defined(WIN32)) && !SQLITE_OS_WINRT
-#  ifndef SHELL_NO_SYSINC
-#   include <io.h>
-#   include <fcntl.h>
-#   undef WIN32_LEAN_AND_MEAN
-#   define WIN32_LEAN_AND_MEAN
-#   include <windows.h>
-#  endif
-#  define CIO_WIN_WC_XLATE 1 /* Use WCHAR Windows APIs for console I/O */
-# else
-#  ifndef SHELL_NO_SYSINC
-#   include <unistd.h>
-#  endif
-#  define CIO_WIN_WC_XLATE 0 /* Use plain C library stream I/O at console */
-# endif
-#else
-# define CIO_WIN_WC_XLATE 0 /* Not exposing translation routines at all */
-#endif
-
-#if CIO_WIN_WC_XLATE
-static HANDLE handleOfFile(FILE *pf){
-  int fileDesc = _fileno(pf);
-  union { intptr_t osfh; HANDLE fh; } fid = {
-    (fileDesc>=0)? _get_osfhandle(fileDesc) : (intptr_t)INVALID_HANDLE_VALUE
-  };
-  return fid.fh;
-}
-#endif
-
-#ifndef SQLITE_CIO_NO_TRANSLATE
-typedef struct PerStreamTags {
-# if CIO_WIN_WC_XLATE
-  HANDLE hx;
-  DWORD consMode;
-  char acIncomplete[4];
-# else
-  short reachesConsole;
-# endif
-  FILE *pf;
-} PerStreamTags;
-
-/* Define NULL-like value for things which can validly be 0. */
-# define SHELL_INVALID_FILE_PTR ((FILE *)~0)
-# if CIO_WIN_WC_XLATE
-#  define SHELL_INVALID_CONS_MODE 0xFFFF0000
-# endif
-
-# if CIO_WIN_WC_XLATE
-#  define PST_INITIALIZER { INVALID_HANDLE_VALUE, SHELL_INVALID_CONS_MODE, \
-      {0,0,0,0}, SHELL_INVALID_FILE_PTR }
-# else
-#  define PST_INITIALIZER { 0, SHELL_INVALID_FILE_PTR }
-# endif
-
-/* Quickly say whether a known output is going to the console. */
-# if CIO_WIN_WC_XLATE
-static short pstReachesConsole(PerStreamTags *ppst){
-  return (ppst->hx != INVALID_HANDLE_VALUE);
-}
-# else
-#  define pstReachesConsole(ppst) 0
-# endif
-
-# if CIO_WIN_WC_XLATE
-static void restoreConsoleArb(PerStreamTags *ppst){
-  if( pstReachesConsole(ppst) ) SetConsoleMode(ppst->hx, ppst->consMode);
-}
-# else
-#  define restoreConsoleArb(ppst)
-# endif
-
-/* Say whether FILE* appears to be a console, collect associated info. */
-static short streamOfConsole(FILE *pf, /* out */ PerStreamTags *ppst){
-# if CIO_WIN_WC_XLATE
-  short rv = 0;
-  DWORD dwCM = SHELL_INVALID_CONS_MODE;
-  HANDLE fh = handleOfFile(pf);
-  ppst->pf = pf;
-  if( INVALID_HANDLE_VALUE != fh ){
-    rv = (GetFileType(fh) == FILE_TYPE_CHAR && GetConsoleMode(fh,&dwCM));
-  }
-  ppst->hx = (rv)? fh : INVALID_HANDLE_VALUE;
-  ppst->consMode = dwCM;
-  return rv;
-# else
-  ppst->pf = pf;
-  ppst->reachesConsole = ( (short)isatty(fileno(pf)) );
-  return ppst->reachesConsole;
-# endif
-}
-
-# ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
-#  define ENABLE_VIRTUAL_TERMINAL_PROCESSING  (0x4)
-# endif
-
-# if CIO_WIN_WC_XLATE
-/* Define console modes for use with the Windows Console API. */
-#  define SHELL_CONI_MODE \
-  (ENABLE_ECHO_INPUT | ENABLE_INSERT_MODE | ENABLE_LINE_INPUT | 0x80 \
-  | ENABLE_QUICK_EDIT_MODE | ENABLE_EXTENDED_FLAGS | ENABLE_PROCESSED_INPUT)
-#  define SHELL_CONO_MODE (ENABLE_PROCESSED_OUTPUT | ENABLE_WRAP_AT_EOL_OUTPUT \
-  | ENABLE_VIRTUAL_TERMINAL_PROCESSING)
-# endif
-
-typedef struct ConsoleInfo {
-  PerStreamTags pstSetup[3];
-  PerStreamTags pstDesignated[3];
-  StreamsAreConsole sacSetup;
-} ConsoleInfo;
-
-static short isValidStreamInfo(PerStreamTags *ppst){
-  return (ppst->pf != SHELL_INVALID_FILE_PTR);
-}
-
-static ConsoleInfo consoleInfo = {
-  { /* pstSetup */ PST_INITIALIZER, PST_INITIALIZER, PST_INITIALIZER },
-  { /* pstDesignated[] */ PST_INITIALIZER, PST_INITIALIZER, PST_INITIALIZER },
-  SAC_NoConsole /* sacSetup */
-};
-
-SQLITE_INTERNAL_LINKAGE FILE* invalidFileStream = (FILE *)~0;
-
-# if CIO_WIN_WC_XLATE
-static void maybeSetupAsConsole(PerStreamTags *ppst, short odir){
-  if( pstReachesConsole(ppst) ){
-    DWORD cm = odir? SHELL_CONO_MODE : SHELL_CONI_MODE;
-    SetConsoleMode(ppst->hx, cm);
-  }
-}
-# else
-#  define maybeSetupAsConsole(ppst,odir)
-# endif
-
-SQLITE_INTERNAL_LINKAGE void consoleRenewSetup(void){
-# if CIO_WIN_WC_XLATE
-  int ix = 0;
-  while( ix < 6 ){
-    PerStreamTags *ppst = (ix<3)?
-      &consoleInfo.pstSetup[ix] : &consoleInfo.pstDesignated[ix-3];
-    maybeSetupAsConsole(ppst, (ix % 3)>0);
-    ++ix;
-  }
-# endif
-}
-
-SQLITE_INTERNAL_LINKAGE StreamsAreConsole
-consoleClassifySetup( FILE *pfIn, FILE *pfOut, FILE *pfErr ){
-  StreamsAreConsole rv = SAC_NoConsole;
-  FILE* apf[3] = { pfIn, pfOut, pfErr };
-  int ix;
-  for( ix = 2; ix >= 0; --ix ){
-    PerStreamTags *ppst = &consoleInfo.pstSetup[ix];
-    if( streamOfConsole(apf[ix], ppst) ){
-      rv |= (SAC_InConsole<<ix);
-    }
-    consoleInfo.pstDesignated[ix] = *ppst;
-    if( ix > 0 ) fflush(apf[ix]);
-  }
-  consoleInfo.sacSetup = rv;
-  consoleRenewSetup();
-  return rv;
-}
-
-SQLITE_INTERNAL_LINKAGE void SQLITE_CDECL consoleRestore( void ){
-# if CIO_WIN_WC_XLATE
-  static ConsoleInfo *pci = &consoleInfo;
-  if( pci->sacSetup ){
-    int ix;
-    for( ix=0; ix<3; ++ix ){
-      if( pci->sacSetup & (SAC_InConsole<<ix) ){
-        PerStreamTags *ppst = &pci->pstSetup[ix];
-        SetConsoleMode(ppst->hx, ppst->consMode);
-      }
-    }
-  }
-# endif
-}
-#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
-
-#ifdef SQLITE_CIO_INPUT_REDIR
-/* Say whether given FILE* is among those known, via either
-** consoleClassifySetup() or set{Output,Error}Stream, as
-** readable, and return an associated PerStreamTags pointer
-** if so. Otherwise, return 0.
-*/
-static PerStreamTags * isKnownReadable(FILE *pf){
-  static PerStreamTags *apst[] = {
-    &consoleInfo.pstDesignated[0], &consoleInfo.pstSetup[0], 0
-  };
-  int ix = 0;
-  do {
-    if( apst[ix]->pf == pf ) break;
-  } while( apst[++ix] != 0 );
-  return apst[ix];
-}
-#endif
-
-#ifndef SQLITE_CIO_NO_TRANSLATE
-/* Say whether given FILE* is among those known, via either
-** consoleClassifySetup() or set{Output,Error}Stream, as
-** writable, and return an associated PerStreamTags pointer
-** if so. Otherwise, return 0.
-*/
-static PerStreamTags * isKnownWritable(FILE *pf){
-  static PerStreamTags *apst[] = {
-    &consoleInfo.pstDesignated[1], &consoleInfo.pstDesignated[2],
-    &consoleInfo.pstSetup[1], &consoleInfo.pstSetup[2], 0
-  };
-  int ix = 0;
-  do {
-    if( apst[ix]->pf == pf ) break;
-  } while( apst[++ix] != 0 );
-  return apst[ix];
-}
-
-static FILE *designateEmitStream(FILE *pf, unsigned chix){
-  FILE *rv = consoleInfo.pstDesignated[chix].pf;
-  if( pf == invalidFileStream ) return rv;
-  else{
-    /* Setting a possibly new output stream. */
-    PerStreamTags *ppst = isKnownWritable(pf);
-    if( ppst != 0 ){
-      PerStreamTags pst = *ppst;
-      consoleInfo.pstDesignated[chix] = pst;
-    }else streamOfConsole(pf, &consoleInfo.pstDesignated[chix]);
-  }
-  return rv;
-}
-
-SQLITE_INTERNAL_LINKAGE FILE *setOutputStream(FILE *pf){
-  return designateEmitStream(pf, 1);
-}
-# ifdef CONSIO_SET_ERROR_STREAM
-SQLITE_INTERNAL_LINKAGE FILE *setErrorStream(FILE *pf){
-  return designateEmitStream(pf, 2);
-}
-# endif
-#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
-
-#ifndef SQLITE_CIO_NO_SETMODE
-# if CIO_WIN_WC_XLATE
-static void setModeFlushQ(FILE *pf, short bFlush, int mode){
-  if( bFlush ) fflush(pf);
-  _setmode(_fileno(pf), mode);
-}
-# else
-#  define setModeFlushQ(f, b, m) if(b) fflush(f)
-# endif
-
-SQLITE_INTERNAL_LINKAGE void setBinaryMode(FILE *pf, short bFlush){
-  setModeFlushQ(pf, bFlush, _O_BINARY);
-}
-SQLITE_INTERNAL_LINKAGE void setTextMode(FILE *pf, short bFlush){
-  setModeFlushQ(pf, bFlush, _O_TEXT);
-}
-# undef setModeFlushQ
-
-#else /* defined(SQLITE_CIO_NO_SETMODE) */
-# define setBinaryMode(f, bFlush) do{ if((bFlush)) fflush(f); }while(0)
-# define setTextMode(f, bFlush) do{ if((bFlush)) fflush(f); }while(0)
-#endif /* defined(SQLITE_CIO_NO_SETMODE) */
-
-#ifndef SQLITE_CIO_NO_TRANSLATE
-# if CIO_WIN_WC_XLATE
-/* Write buffer cBuf as output to stream known to reach console,
-** limited to ncTake char's. Return ncTake on success, else 0. */
-static int conZstrEmit(PerStreamTags *ppst, const char *z, int ncTake){
-  int rv = 0;
-  if( z!=NULL ){
-    int nwc = MultiByteToWideChar(CP_UTF8,0, z,ncTake, 0,0);
-    if( nwc > 0 ){
-      WCHAR *zw = sqlite3_malloc64(nwc*sizeof(WCHAR));
-      if( zw!=NULL ){
-        nwc = MultiByteToWideChar(CP_UTF8,0, z,ncTake, zw,nwc);
-        if( nwc > 0 ){
-          /* Translation from UTF-8 to UTF-16, then WCHARs out. */
-          if( WriteConsoleW(ppst->hx, zw,nwc, 0, NULL) ){
-            rv = ncTake;
-          }
-        }
-        sqlite3_free(zw);
-      }
-    }
-  }
-  return rv;
-}
-
-/* For {f,o,e}PrintfUtf8() when stream is known to reach console. */
-static int conioVmPrintf(PerStreamTags *ppst, const char *zFormat, va_list ap){
-  char *z = sqlite3_vmprintf(zFormat, ap);
-  if( z ){
-    int rv = conZstrEmit(ppst, z, (int)strlen(z));
-    sqlite3_free(z);
-    return rv;
-  }else return 0;
-}
-# endif /* CIO_WIN_WC_XLATE */
-
-# ifdef CONSIO_GET_EMIT_STREAM
-static PerStreamTags * getDesignatedEmitStream(FILE *pf, unsigned chix,
-                                               PerStreamTags *ppst){
-  PerStreamTags *rv = isKnownWritable(pf);
-  short isValid = (rv!=0)? isValidStreamInfo(rv) : 0;
-  if( rv != 0 && isValid ) return rv;
-  streamOfConsole(pf, ppst);
-  return ppst;
-}
-# endif
-
-/* Get stream info, either for designated output or error stream when
-** chix equals 1 or 2, or for an arbitrary stream when chix == 0.
-** In either case, ppst references a caller-owned PerStreamTags
-** struct which may be filled in if none of the known writable
-** streams is being held by consoleInfo. The ppf parameter is a
-** byref output when chix!=0 and a byref input when chix==0.
- */
-static PerStreamTags *
-getEmitStreamInfo(unsigned chix, PerStreamTags *ppst,
-                  /* in/out */ FILE **ppf){
-  PerStreamTags *ppstTry;
-  FILE *pfEmit;
-  if( chix > 0 ){
-    ppstTry = &consoleInfo.pstDesignated[chix];
-    if( !isValidStreamInfo(ppstTry) ){
-      ppstTry = &consoleInfo.pstSetup[chix];
-      pfEmit = ppst->pf;
-    }else pfEmit = ppstTry->pf;
-    if( !isValidStreamInfo(ppstTry) ){
-      pfEmit = (chix > 1)? stderr : stdout;
-      ppstTry = ppst;
-      streamOfConsole(pfEmit, ppstTry);
-    }
-    *ppf = pfEmit;
-  }else{
-    ppstTry = isKnownWritable(*ppf);
-    if( ppstTry != 0 ) return ppstTry;
-    streamOfConsole(*ppf, ppst);
-    return ppst;
-  }
-  return ppstTry;
-}
-
-SQLITE_INTERNAL_LINKAGE int oPrintfUtf8(const char *zFormat, ...){
-  va_list ap;
-  int rv;
-  FILE *pfOut;
-  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
-# if CIO_WIN_WC_XLATE
-  PerStreamTags *ppst = getEmitStreamInfo(1, &pst, &pfOut);
-# else
-  getEmitStreamInfo(1, &pst, &pfOut);
-# endif
-  assert(zFormat!=0);
-  va_start(ap, zFormat);
-# if CIO_WIN_WC_XLATE
-  if( pstReachesConsole(ppst) ){
-    rv = conioVmPrintf(ppst, zFormat, ap);
-  }else{
-# endif
-    rv = vfprintf(pfOut, zFormat, ap);
-# if CIO_WIN_WC_XLATE
-  }
-# endif
-  va_end(ap);
-  return rv;
-}
-
-SQLITE_INTERNAL_LINKAGE int ePrintfUtf8(const char *zFormat, ...){
-  va_list ap;
-  int rv;
-  FILE *pfErr;
-  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
-# if CIO_WIN_WC_XLATE
-  PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr);
-# else
-  getEmitStreamInfo(2, &pst, &pfErr);
-# endif
-  assert(zFormat!=0);
-  va_start(ap, zFormat);
-# if CIO_WIN_WC_XLATE
-  if( pstReachesConsole(ppst) ){
-    rv = conioVmPrintf(ppst, zFormat, ap);
-  }else{
-# endif
-    rv = vfprintf(pfErr, zFormat, ap);
-# if CIO_WIN_WC_XLATE
-  }
-# endif
-  va_end(ap);
-  return rv;
-}
-
-SQLITE_INTERNAL_LINKAGE int fPrintfUtf8(FILE *pfO, const char *zFormat, ...){
-  va_list ap;
-  int rv;
-  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
-# if CIO_WIN_WC_XLATE
-  PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO);
-# else
-  getEmitStreamInfo(0, &pst, &pfO);
-# endif
-  assert(zFormat!=0);
-  va_start(ap, zFormat);
-# if CIO_WIN_WC_XLATE
-  if( pstReachesConsole(ppst) ){
-    maybeSetupAsConsole(ppst, 1);
-    rv = conioVmPrintf(ppst, zFormat, ap);
-    if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst);
-  }else{
-# endif
-    rv = vfprintf(pfO, zFormat, ap);
-# if CIO_WIN_WC_XLATE
-  }
-# endif
-  va_end(ap);
-  return rv;
-}
-
-SQLITE_INTERNAL_LINKAGE int fPutsUtf8(const char *z, FILE *pfO){
-  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
-# if CIO_WIN_WC_XLATE
-  PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO);
-# else
-  getEmitStreamInfo(0, &pst, &pfO);
-# endif
-  assert(z!=0);
-# if CIO_WIN_WC_XLATE
-  if( pstReachesConsole(ppst) ){
-    int rv;
-    maybeSetupAsConsole(ppst, 1);
-    rv = conZstrEmit(ppst, z, (int)strlen(z));
-    if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst);
-    return rv;
-  }else {
-# endif
-    return (fputs(z, pfO)<0)? 0 : (int)strlen(z);
-# if CIO_WIN_WC_XLATE
-  }
-# endif
-}
-
-SQLITE_INTERNAL_LINKAGE int ePutsUtf8(const char *z){
-  FILE *pfErr;
-  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
-# if CIO_WIN_WC_XLATE
-  PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr);
-# else
-  getEmitStreamInfo(2, &pst, &pfErr);
-# endif
-  assert(z!=0);
-# if CIO_WIN_WC_XLATE
-  if( pstReachesConsole(ppst) ) return conZstrEmit(ppst, z, (int)strlen(z));
-  else {
-# endif
-    return (fputs(z, pfErr)<0)? 0 : (int)strlen(z);
-# if CIO_WIN_WC_XLATE
-  }
-# endif
-}
-
-SQLITE_INTERNAL_LINKAGE int oPutsUtf8(const char *z){
-  FILE *pfOut;
-  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
-# if CIO_WIN_WC_XLATE
-  PerStreamTags *ppst = getEmitStreamInfo(1, &pst, &pfOut);
-# else
-  getEmitStreamInfo(1, &pst, &pfOut);
-# endif
-  assert(z!=0);
-# if CIO_WIN_WC_XLATE
-  if( pstReachesConsole(ppst) ) return conZstrEmit(ppst, z, (int)strlen(z));
-  else {
-# endif
-    return (fputs(z, pfOut)<0)? 0 : (int)strlen(z);
-# if CIO_WIN_WC_XLATE
-  }
-# endif
-}
-
-#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
-
-#if !(defined(SQLITE_CIO_NO_UTF8SCAN) && defined(SQLITE_CIO_NO_TRANSLATE))
-/* Skip over as much z[] input char sequence as is valid UTF-8,
-** limited per nAccept char's or whole characters and containing
-** no char cn such that ((1<<cn) & ccm)!=0. On return, the
-** sequence z:return (inclusive:exclusive) is validated UTF-8.
-** Limit: nAccept>=0 => char count, nAccept<0 => character
- */
-SQLITE_INTERNAL_LINKAGE const char*
-zSkipValidUtf8(const char *z, int nAccept, long ccm){
-  int ng = (nAccept<0)? -nAccept : 0;
-  const char *pcLimit = (nAccept>=0)? z+nAccept : 0;
-  assert(z!=0);
-  while( (pcLimit)? (z<pcLimit) : (ng-- != 0) ){
-    char c = *z;
-    if( (c & 0x80) == 0 ){
-      if( ccm != 0L && c < 0x20 && ((1L<<c) & ccm) != 0 ) return z;
-      ++z; /* ASCII */
-    }else if( (c & 0xC0) != 0xC0 ) return z; /* not a lead byte */
-    else{
-      const char *zt = z+1; /* Got lead byte, look at trail bytes.*/
-      do{
-        if( pcLimit && zt >= pcLimit ) return z;
-        else{
-          char ct = *zt++;
-          if( ct==0 || (zt-z)>4 || (ct & 0xC0)!=0x80 ){
-            /* Trailing bytes are too few, too many, or invalid. */
-            return z;
-          }
-        }
-      } while( ((c <<= 1) & 0x40) == 0x40 ); /* Eat lead byte's count. */
-      z = zt;
-    }
-  }
-  return z;
-}
-#endif /*!(defined(SQLITE_CIO_NO_UTF8SCAN)&&defined(SQLITE_CIO_NO_TRANSLATE))*/
-
-#ifndef SQLITE_CIO_NO_TRANSLATE
-# ifdef CONSIO_SPUTB
-SQLITE_INTERNAL_LINKAGE int
-fPutbUtf8(FILE *pfO, const char *cBuf, int nAccept){
-  assert(pfO!=0);
-#  if CIO_WIN_WC_XLATE
-  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
-  PerStreamTags *ppst = getEmitStreamInfo(0, &pst, &pfO);
-  if( pstReachesConsole(ppst) ){
-    int rv;
-    maybeSetupAsConsole(ppst, 1);
-    rv = conZstrEmit(ppst, cBuf, nAccept);
-    if( 0 == isKnownWritable(ppst->pf) ) restoreConsoleArb(ppst);
-    return rv;
-  }else {
-#  endif
-    return (int)fwrite(cBuf, 1, nAccept, pfO);
-#  if CIO_WIN_WC_XLATE
-  }
-#  endif
-}
-# endif
-
-SQLITE_INTERNAL_LINKAGE int
-oPutbUtf8(const char *cBuf, int nAccept){
-  FILE *pfOut;
-  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
-# if CIO_WIN_WC_XLATE
-  PerStreamTags *ppst = getEmitStreamInfo(1, &pst, &pfOut);
-# else
-  getEmitStreamInfo(1, &pst, &pfOut);
-# endif
-# if CIO_WIN_WC_XLATE
-  if( pstReachesConsole(ppst) ){
-    return conZstrEmit(ppst, cBuf, nAccept);
-  }else {
-# endif
-    return (int)fwrite(cBuf, 1, nAccept, pfOut);
-# if CIO_WIN_WC_XLATE
-  }
-# endif
-}
-
-# ifdef CONSIO_EPUTB
-SQLITE_INTERNAL_LINKAGE int
-ePutbUtf8(const char *cBuf, int nAccept){
-  FILE *pfErr;
-  PerStreamTags pst = PST_INITIALIZER; /* for unknown streams */
-  PerStreamTags *ppst = getEmitStreamInfo(2, &pst, &pfErr);
-#  if CIO_WIN_WC_XLATE
-  if( pstReachesConsole(ppst) ){
-    return conZstrEmit(ppst, cBuf, nAccept);
-  }else {
-#  endif
-    return (int)fwrite(cBuf, 1, nAccept, pfErr);
-#  if CIO_WIN_WC_XLATE
-  }
-#  endif
-}
-# endif /* defined(CONSIO_EPUTB) */
-
-SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn){
-  if( pfIn==0 ) pfIn = stdin;
-# if CIO_WIN_WC_XLATE
-  if( pfIn == consoleInfo.pstSetup[0].pf
-      && (consoleInfo.sacSetup & SAC_InConsole)!=0 ){
-#  if CIO_WIN_WC_XLATE==1
-#   define SHELL_GULP 150 /* Count of WCHARS to be gulped at a time */
-    WCHAR wcBuf[SHELL_GULP+1];
-    int lend = 0, noc = 0;
-    if( ncMax > 0 ) cBuf[0] = 0;
-    while( noc < ncMax-8-1 && !lend ){
-      /* There is room for at least 2 more characters and a 0-terminator. */
-      int na = (ncMax > SHELL_GULP*4+1 + noc)? SHELL_GULP : (ncMax-1 - noc)/4;
-#   undef SHELL_GULP
-      DWORD nbr = 0;
-      BOOL bRC = ReadConsoleW(consoleInfo.pstSetup[0].hx, wcBuf, na, &nbr, 0);
-      if( bRC && nbr>0 && (wcBuf[nbr-1]&0xF800)==0xD800 ){
-        /* Last WHAR read is first of a UTF-16 surrogate pair. Grab its mate. */
-        DWORD nbrx;
-        bRC &= ReadConsoleW(consoleInfo.pstSetup[0].hx, wcBuf+nbr, 1, &nbrx, 0);
-        if( bRC ) nbr += nbrx;
-      }
-      if( !bRC || (noc==0 && nbr==0) ) return 0;
-      if( nbr > 0 ){
-        int nmb = WideCharToMultiByte(CP_UTF8, 0, wcBuf,nbr,0,0,0,0);
-        if( nmb != 0 && noc+nmb <= ncMax ){
-          int iseg = noc;
-          nmb = WideCharToMultiByte(CP_UTF8, 0, wcBuf,nbr,cBuf+noc,nmb,0,0);
-          noc += nmb;
-          /* Fixup line-ends as coded by Windows for CR (or "Enter".)
-          ** This is done without regard for any setMode{Text,Binary}()
-          ** call that might have been done on the interactive input.
-          */
-          if( noc > 0 ){
-            if( cBuf[noc-1]=='\n' ){
-              lend = 1;
-              if( noc > 1 && cBuf[noc-2]=='\r' ) cBuf[--noc-1] = '\n';
-            }
-          }
-          /* Check for ^Z (anywhere in line) too, to act as EOF. */
-          while( iseg < noc ){
-            if( cBuf[iseg]=='\x1a' ){
-              noc = iseg; /* Chop ^Z and anything following. */
-              lend = 1; /* Counts as end of line too. */
-              break;
-            }
-            ++iseg;
-          }
-        }else break; /* Drop apparent garbage in. (Could assert.) */
-      }else break;
-    }
-    /* If got nothing, (after ^Z chop), must be at end-of-file. */
-    if( noc > 0 ){
-      cBuf[noc] = 0;
-      return cBuf;
-    }else return 0;
-#  endif
-  }else{
-# endif
-    return fgets(cBuf, ncMax, pfIn);
-# if CIO_WIN_WC_XLATE
-  }
-# endif
-}
-#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
-
-#if defined(_MSC_VER)
-# pragma warning(default : 4204)
-#endif
-
-#undef SHELL_INVALID_FILE_PTR

ext/consio/console_io.h

@@ -1,280 +0,0 @@
-/*
-** 2023 November 1
-**
-** The author disclaims copyright to this source code.  In place of
-** a legal notice, here is a blessing:
-**
-**    May you do good and not evil.
-**    May you find forgiveness for yourself and forgive others.
-**    May you share freely, never taking more than you give.
-**
-********************************************************************************
-** This file exposes various interfaces used for console and other I/O
-** by the SQLite project command-line tools. These interfaces are used
-** at either source conglomeration time, compilation time, or run time.
-** This source provides for either inclusion into conglomerated,
-** "single-source" forms or separate compilation then linking.
-**
-** Platform dependencies are "hidden" here by various stratagems so
-** that, provided certain conditions are met, the programs using this
-** source or object code compiled from it need no explicit conditional
-** compilation in their source for their console and stream I/O.
-**
-** The symbols and functionality exposed here are not a public API.
-** This code may change in tandem with other project code as needed.
-**
-** When this .h file and its companion .c are directly incorporated into
-** a source conglomeration (such as shell.c), the preprocessor symbol
-** CIO_WIN_WC_XLATE is defined as 0 or 1, reflecting whether console I/O
-** translation for Windows is effected for the build.
-*/
-#define HAVE_CONSOLE_IO_H 1
-#ifndef SQLITE_INTERNAL_LINKAGE
-# define SQLITE_INTERNAL_LINKAGE extern /* external to translation unit */
-# include <stdio.h>
-#else
-# define SHELL_NO_SYSINC /* Better yet, modify mkshellc.tcl for this. */
-#endif
-
-#ifndef SQLITE3_H
-# include "sqlite3.h"
-#endif
-
-#ifndef SQLITE_CIO_NO_CLASSIFY
-
-/* Define enum for use with following function. */
-typedef enum StreamsAreConsole {
-  SAC_NoConsole = 0,
-  SAC_InConsole = 1, SAC_OutConsole = 2, SAC_ErrConsole = 4,
-  SAC_AnyConsole = 0x7
-} StreamsAreConsole;
-
-/*
-** Classify the three standard I/O streams according to whether
-** they are connected to a console attached to the process.
-**
-** Returns the bit-wise OR of SAC_{In,Out,Err}Console values,
-** or SAC_NoConsole if none of the streams reaches a console.
-**
-** This function should be called before any I/O is done with
-** the given streams. As a side-effect, the given inputs are
-** recorded so that later I/O operations on them may be done
-** differently than the C library FILE* I/O would be done,
-** iff the stream is used for the I/O functions that follow,
-** and to support the ones that use an implicit stream.
-**
-** On some platforms, stream or console mode alteration (aka
-** "Setup") may be made which is undone by consoleRestore().
-*/
-SQLITE_INTERNAL_LINKAGE StreamsAreConsole
-consoleClassifySetup( FILE *pfIn, FILE *pfOut, FILE *pfErr );
-/* A usual call for convenience: */
-#define SQLITE_STD_CONSOLE_INIT() consoleClassifySetup(stdin,stdout,stderr)
-
-/*
-** After an initial call to consoleClassifySetup(...), renew
-** the same setup it effected. (A call not after is an error.)
-** This will restore state altered by consoleRestore();
-**
-** Applications which run an inferior (child) process which
-** inherits the same I/O streams may call this function after
-** such a process exits to guard against console mode changes.
-*/
-SQLITE_INTERNAL_LINKAGE void consoleRenewSetup(void);
-
-/*
-** Undo any side-effects left by consoleClassifySetup(...).
-**
-** This should be called after consoleClassifySetup() and
-** before the process terminates normally. It is suitable
-** for use with the atexit() C library procedure. After
-** this call, no console I/O should be done until one of
-** console{Classify or Renew}Setup(...) is called again.
-**
-** Applications which run an inferior (child) process that
-** inherits the same I/O streams might call this procedure
-** before so that said process will have a console setup
-** however users have configured it or come to expect.
-*/
-SQLITE_INTERNAL_LINKAGE void SQLITE_CDECL consoleRestore( void );
-
-#else /* defined(SQLITE_CIO_NO_CLASSIFY) */
-# define consoleClassifySetup(i,o,e)
-# define consoleRenewSetup()
-# define consoleRestore()
-#endif /* defined(SQLITE_CIO_NO_CLASSIFY) */
-
-#ifndef SQLITE_CIO_NO_REDIRECT
-/*
-** Set stream to be used for the functions below which write
-** to "the designated X stream", where X is Output or Error.
-** Returns the previous value.
-**
-** Alternatively, pass the special value, invalidFileStream,
-** to get the designated stream value without setting it.
-**
-** Before the designated streams are set, they default to
-** those passed to consoleClassifySetup(...), and before
-** that is called they default to stdout and stderr.
-**
-** It is error to close a stream so designated, then, without
-** designating another, use the corresponding {o,e}Emit(...).
-*/
-SQLITE_INTERNAL_LINKAGE FILE *invalidFileStream;
-SQLITE_INTERNAL_LINKAGE FILE *setOutputStream(FILE *pf);
-# ifdef CONSIO_SET_ERROR_STREAM
-SQLITE_INTERNAL_LINKAGE FILE *setErrorStream(FILE *pf);
-# endif
-#else
-# define setOutputStream(pf)
-# define setErrorStream(pf)
-#endif /* !defined(SQLITE_CIO_NO_REDIRECT) */
-
-#ifndef SQLITE_CIO_NO_TRANSLATE
-/*
-** Emit output like fprintf(). If the output is going to the
-** console and translation from UTF-8 is necessary, perform
-** the needed translation. Otherwise, write formatted output
-** to the provided stream almost as-is, possibly with newline
-** translation as specified by set{Binary,Text}Mode().
-*/
-SQLITE_INTERNAL_LINKAGE int fPrintfUtf8(FILE *pfO, const char *zFormat, ...);
-/* Like fPrintfUtf8 except stream is always the designated output. */
-SQLITE_INTERNAL_LINKAGE int oPrintfUtf8(const char *zFormat, ...);
-/* Like fPrintfUtf8 except stream is always the designated error. */
-SQLITE_INTERNAL_LINKAGE int ePrintfUtf8(const char *zFormat, ...);
-
-/*
-** Emit output like fputs(). If the output is going to the
-** console and translation from UTF-8 is necessary, perform
-** the needed translation. Otherwise, write given text to the
-** provided stream almost as-is, possibly with newline
-** translation as specified by set{Binary,Text}Mode().
-*/
-SQLITE_INTERNAL_LINKAGE int fPutsUtf8(const char *z, FILE *pfO);
-/* Like fPutsUtf8 except stream is always the designated output. */
-SQLITE_INTERNAL_LINKAGE int oPutsUtf8(const char *z);
-/* Like fPutsUtf8 except stream is always the designated error. */
-SQLITE_INTERNAL_LINKAGE int ePutsUtf8(const char *z);
-
-/*
-** Emit output like fPutsUtf8(), except that the length of the
-** accepted char or character sequence is limited by nAccept.
-**
-** Returns the number of accepted char values.
-*/
-#ifdef CONSIO_SPUTB
-SQLITE_INTERNAL_LINKAGE int
-fPutbUtf8(FILE *pfOut, const char *cBuf, int nAccept);
-/* Like fPutbUtf8 except stream is always the designated output. */
-#endif
-SQLITE_INTERNAL_LINKAGE int
-oPutbUtf8(const char *cBuf, int nAccept);
-/* Like fPutbUtf8 except stream is always the designated error. */
-#ifdef CONSIO_EPUTB
-SQLITE_INTERNAL_LINKAGE int
-ePutbUtf8(const char *cBuf, int nAccept);
-#endif
-
-/*
-** Collect input like fgets(...) with special provisions for input
-** from the console on platforms that require same. Defers to the
-** C library fgets() when input is not from the console. Newline
-** translation may be done as set by set{Binary,Text}Mode(). As a
-** convenience, pfIn==NULL is treated as stdin.
-*/
-SQLITE_INTERNAL_LINKAGE char* fGetsUtf8(char *cBuf, int ncMax, FILE *pfIn);
-/* Like fGetsUtf8 except stream is always the designated input. */
-/* SQLITE_INTERNAL_LINKAGE char* iGetsUtf8(char *cBuf, int ncMax); */
-
-#endif /* !defined(SQLITE_CIO_NO_TRANSLATE) */
-
-#ifndef SQLITE_CIO_NO_SETMODE
-/*
-** Set given stream for binary mode, where newline translation is
-** not done, or for text mode where, for some platforms, newlines
-** are translated to the platform's conventional char sequence.
-** If bFlush true, flush the stream.
-**
-** An additional side-effect is that if the stream is one passed
-** to consoleClassifySetup() as an output, it is flushed first.
-**
-** Note that binary/text mode has no effect on console I/O
-** translation. On all platforms, newline to the console starts
-** a new line and CR,LF chars from the console become a newline.
-*/
-SQLITE_INTERNAL_LINKAGE void setBinaryMode(FILE *, short bFlush);
-SQLITE_INTERNAL_LINKAGE void setTextMode(FILE *, short bFlush);
-#endif
-
-#ifdef SQLITE_CIO_PROMPTED_IN
-typedef struct Prompts {
-  int numPrompts;
-  const char **azPrompts;
-} Prompts;
-
-/*
-** Macros for use of a line editor.
-**
-** The following macros define operations involving use of a
-** line-editing library or simple console interaction.
-** A "T" argument is a text (char *) buffer or filename.
-** A "N" argument is an integer.
-**
-** SHELL_ADD_HISTORY(T) // Record text as line(s) of history.
-** SHELL_READ_HISTORY(T) // Read history from file named by T.
-** SHELL_WRITE_HISTORY(T) // Write history to file named by T.
-** SHELL_STIFLE_HISTORY(N) // Limit history to N entries.
-**
-** A console program which does interactive console input is
-** expected to call:
-** SHELL_READ_HISTORY(T) before collecting such input;
-** SHELL_ADD_HISTORY(T) as record-worthy input is taken;
-** SHELL_STIFLE_HISTORY(N) after console input ceases; then
-** SHELL_WRITE_HISTORY(T) before the program exits.
-*/
-
-/*
-** Retrieve a single line of input text from an input stream.
-**
-** If pfIn is the input stream passed to consoleClassifySetup(),
-** and azPrompt is not NULL, then a prompt is issued before the
-** line is collected, as selected by the isContinuation flag.
-** Array azPrompt[{0,1}] holds the {main,continuation} prompt.
-**
-** If zBufPrior is not NULL then it is a buffer from a prior
-** call to this routine that can be reused, or will be freed.
-**
-** The result is stored in space obtained from malloc() and
-** must either be freed by the caller or else passed back to
-** this function as zBufPrior for reuse.
-**
-** This function may call upon services of a line-editing
-** library to interactively collect line edited input.
-*/
-SQLITE_INTERNAL_LINKAGE char *
-shellGetLine(FILE *pfIn, char *zBufPrior, int nLen,
-             short isContinuation, Prompts azPrompt);
-#endif /* defined(SQLITE_CIO_PROMPTED_IN) */
-/*
-** TBD: Define an interface for application(s) to generate
-** completion candidates for use by the line-editor.
-**
-** This may be premature; the CLI is the only application
-** that does this. Yet, getting line-editing melded into
-** console I/O is desirable because a line-editing library
-** may have to establish console operating mode, possibly
-** in a way that interferes with the above functionality.
-*/
-
-#if !(defined(SQLITE_CIO_NO_UTF8SCAN)&&defined(SQLITE_CIO_NO_TRANSLATE))
-/* Skip over as much z[] input char sequence as is valid UTF-8,
-** limited per nAccept char's or whole characters and containing
-** no char cn such that ((1<<cn) & ccm)!=0. On return, the
-** sequence z:return (inclusive:exclusive) is validated UTF-8.
-** Limit: nAccept>=0 => char count, nAccept<0 => character
- */
-SQLITE_INTERNAL_LINKAGE const char*
-zSkipValidUtf8(const char *z, int nAccept, long ccm);
-
-#endif

ext/expert/expert1.test

@@ -8,7 +8,6 @@
 #    May you share freely, never taking more than you give.
 #
 #***********************************************************************
-# TESTRUNNER: shell
 #
 # The focus of this file is testing the CLI shell tool. Specifically,
 # the ".recommend" command.
@@ -465,30 +464,4 @@ do_execsql_test 5.3 {
   t2 t2_idx_0001295b {100 20 5}
 }
 
-do_catchsql_test 5.4 {
-  SELECT sqlite_expert_rem(123, 123);
-} {1 {no such function: sqlite_expert_rem}}
-do_catchsql_test 5.5 {
-  SELECT sqlite_expert_sample();
-} {1 {no such function: sqlite_expert_sample}}
-
-if 0 {
-do_test expert1-6.0 {
-  catchcmd :memory: {
-.expert
-select base64('');
-.expert
-select name from pragma_collation_list order by name collate uint;
-}
-} {0 {(no new indexes)
-
-SCAN CONSTANT ROW
-
-(no new indexes)
-
-SCAN pragma_collation_list VIRTUAL TABLE INDEX 0:
-USE TEMP B-TREE FOR ORDER BY
-}}
-}
-
 finish_test

ext/expert/sqlite3expert.c

@@ -32,7 +32,7 @@
 #endif /* !defined(SQLITE_AMALGAMATION) */
 
 
-#ifndef SQLITE_OMIT_VIRTUALTABLE
+#ifndef SQLITE_OMIT_VIRTUALTABLE 
 
 typedef sqlite3_int64 i64;
 typedef sqlite3_uint64 u64;
@@ -626,7 +626,7 @@ static int expertFilter(
   pCsr->pData = 0;
   if( rc==SQLITE_OK ){
     rc = idxPrintfPrepareStmt(pExpert->db, &pCsr->pData, &pVtab->base.zErrMsg,
-        "SELECT * FROM main.%Q WHERE sqlite_expert_sample()", pVtab->pTab->zName
+        "SELECT * FROM main.%Q WHERE sample()", pVtab->pTab->zName
     );
   }
 
@@ -662,7 +662,6 @@ static int idxRegisterVtab(sqlite3expert *p){
     0,                            /* xRelease */
     0,                            /* xRollbackTo */
     0,                            /* xShadowName */
-    0,                            /* xIntegrity */
   };
 
   return sqlite3_create_module(p->dbv, "expert", &expertModule, (void*)p);
@@ -1500,7 +1499,7 @@ struct IdxRemCtx {
 };
 
 /*
-** Implementation of scalar function sqlite_expert_rem().
+** Implementation of scalar function rem().
 */
 static void idxRemFunc(
   sqlite3_context *pCtx,
@@ -1513,7 +1512,7 @@ static void idxRemFunc(
   assert( argc==2 );
 
   iSlot = sqlite3_value_int(argv[0]);
-  assert( iSlot<p->nSlot );
+  assert( iSlot<=p->nSlot );
   pSlot = &p->aSlot[iSlot];
 
   switch( pSlot->eType ){
@@ -1624,8 +1623,7 @@ static int idxPopulateOneStat1(
     const char *zName = (const char*)sqlite3_column_text(pIndexXInfo, 0);
     const char *zColl = (const char*)sqlite3_column_text(pIndexXInfo, 1);
     zCols = idxAppendText(&rc, zCols, 
-        "%sx.%Q IS sqlite_expert_rem(%d, x.%Q) COLLATE %s", 
-        zComma, zName, nCol, zName, zColl
+        "%sx.%Q IS rem(%d, x.%Q) COLLATE %s", zComma, zName, nCol, zName, zColl
     );
     zOrder = idxAppendText(&rc, zOrder, "%s%d", zComma, ++nCol);
   }
@@ -1758,13 +1756,13 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
 
   if( rc==SQLITE_OK ){
     sqlite3 *dbrem = (p->iSample==100 ? p->db : p->dbv);
-    rc = sqlite3_create_function(dbrem, "sqlite_expert_rem", 
-        2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
+    rc = sqlite3_create_function(
+        dbrem, "rem", 2, SQLITE_UTF8, (void*)pCtx, idxRemFunc, 0, 0
     );
   }
   if( rc==SQLITE_OK ){
-    rc = sqlite3_create_function(p->db, "sqlite_expert_sample", 
-        0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0
+    rc = sqlite3_create_function(
+        p->db, "sample", 0, SQLITE_UTF8, (void*)&samplectx, idxSampleFunc, 0, 0
     );
   }
 
@@ -1816,95 +1814,10 @@ static int idxPopulateStat1(sqlite3expert *p, char **pzErr){
     rc = sqlite3_exec(p->dbm, "ANALYZE sqlite_schema", 0, 0, 0);
   }
 
-  sqlite3_create_function(p->db, "sqlite_expert_rem", 2, SQLITE_UTF8, 0,0,0,0);
-  sqlite3_create_function(p->db, "sqlite_expert_sample", 0,SQLITE_UTF8,0,0,0,0);
-
   sqlite3_exec(p->db, "DROP TABLE IF EXISTS temp."UNIQUE_TABLE_NAME,0,0,0);
   return rc;
 }
 
-/*
-** Define and possibly pretend to use a useless collation sequence.
-** This pretense allows expert to accept SQL using custom collations.
-*/
-int dummyCompare(void *up1, int up2, const void *up3, int up4, const void *up5){
-  (void)up1;
-  (void)up2;
-  (void)up3;
-  (void)up4;
-  (void)up5;
-  assert(0); /* VDBE should never be run. */
-  return 0;
-}
-/* And a callback to register above upon actual need */
-void useDummyCS(void *up1, sqlite3 *db, int etr, const char *zName){
-  (void)up1;
-  sqlite3_create_collation_v2(db, zName, etr, 0, dummyCompare, 0);
-}
-
-#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) \
-  && !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS)
-/*
-** dummy functions for no-op implementation of UDFs during expert's work
-*/
-void dummyUDF(sqlite3_context *up1, int up2, sqlite3_value **up3){
-  (void)up1;
-  (void)up2;
-  (void)up3;
-  assert(0); /* VDBE should never be run. */
-}
-void dummyUDFvalue(sqlite3_context *up1){
-  (void)up1;
-  assert(0); /* VDBE should never be run. */
-}
-
-/*
-** Register UDFs from user database with another.
-*/
-int registerUDFs(sqlite3 *dbSrc, sqlite3 *dbDst){
-  sqlite3_stmt *pStmt;
-  int rc = sqlite3_prepare_v2(dbSrc,
-            "SELECT name,type,enc,narg,flags "
-            "FROM pragma_function_list() "
-            "WHERE builtin==0", -1, &pStmt, 0);
-  if( rc==SQLITE_OK ){
-    while( SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){
-      int nargs = sqlite3_column_int(pStmt,3);
-      int flags = sqlite3_column_int(pStmt,4);
-      const char *name = (char*)sqlite3_column_text(pStmt,0);
-      const char *type = (char*)sqlite3_column_text(pStmt,1);
-      const char *enc = (char*)sqlite3_column_text(pStmt,2);
-      if( name==0 || type==0 || enc==0 ){
-        /* no-op.  Only happens on OOM */
-      }else{
-        int ienc = SQLITE_UTF8;
-        int rcf = SQLITE_ERROR;
-        if( strcmp(enc,"utf16le")==0 ) ienc = SQLITE_UTF16LE;
-        else if( strcmp(enc,"utf16be")==0 ) ienc = SQLITE_UTF16BE;
-        ienc |= (flags & (SQLITE_DETERMINISTIC|SQLITE_DIRECTONLY));
-        if( strcmp(type,"w")==0 ){
-          rcf = sqlite3_create_window_function(dbDst,name,nargs,ienc,0,
-                                               dummyUDF,dummyUDFvalue,0,0,0);
-        }else if( strcmp(type,"a")==0 ){
-          rcf = sqlite3_create_function(dbDst,name,nargs,ienc,0,
-                                        0,dummyUDF,dummyUDFvalue);
-        }else if( strcmp(type,"s")==0 ){
-          rcf = sqlite3_create_function(dbDst,name,nargs,ienc,0,
-                                        dummyUDF,0,0);
-        }
-        if( rcf!=SQLITE_OK ){
-          rc = rcf;
-          break;
-        }
-      }
-    }
-    sqlite3_finalize(pStmt);
-    if( rc==SQLITE_DONE ) rc = SQLITE_OK;
-  }
-  return rc;
-}
-#endif
-
 /*
 ** Allocate a new sqlite3expert object.
 */
@@ -1931,28 +1844,14 @@ sqlite3expert *sqlite3_expert_new(sqlite3 *db, char **pzErrmsg){
       sqlite3_db_config(pNew->dbm, SQLITE_DBCONFIG_TRIGGER_EQP, 1, (int*)0);
     }
   }
-
-  /* Allow custom collations to be dealt with through prepare. */
-  if( rc==SQLITE_OK ) rc = sqlite3_collation_needed(pNew->dbm,0,useDummyCS);
-  if( rc==SQLITE_OK ) rc = sqlite3_collation_needed(pNew->dbv,0,useDummyCS);
-
-#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) \
-  && !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS)
-  /* Register UDFs from database [db] with [dbm] and [dbv]. */
-  if( rc==SQLITE_OK ){
-    rc = registerUDFs(pNew->db, pNew->dbm);
-  }
-  if( rc==SQLITE_OK ){
-    rc = registerUDFs(pNew->db, pNew->dbv);
-  }
-#endif
+  
 
   /* Copy the entire schema of database [db] into [dbm]. */
   if( rc==SQLITE_OK ){
     sqlite3_stmt *pSql = 0;
     rc = idxPrintfPrepareStmt(pNew->db, &pSql, pzErrmsg, 
         "SELECT sql FROM sqlite_schema WHERE name NOT LIKE 'sqlite_%%'"
-        " AND sql NOT LIKE 'CREATE VIRTUAL %%' ORDER BY rowid"
+        " AND sql NOT LIKE 'CREATE VIRTUAL %%'"
     );
     while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSql) ){
       const char *zSql = (const char*)sqlite3_column_text(pSql, 0);
@@ -2021,10 +1920,6 @@ int sqlite3_expert_sql(
 
   while( rc==SQLITE_OK && zStmt && zStmt[0] ){
     sqlite3_stmt *pStmt = 0;
-    /* Ensure that the provided statement compiles against user's DB. */
-    rc = idxPrepareStmt(p->db, &pStmt, pzErr, zStmt);
-    if( rc!=SQLITE_OK ) break;
-    sqlite3_finalize(pStmt);
     rc = sqlite3_prepare_v2(p->dbv, zStmt, -1, &pStmt, &zStmt);
     if( rc==SQLITE_OK ){
       if( pStmt ){

ext/expert/test_expert.c

@@ -16,7 +16,15 @@
 #include "sqlite3expert.h"
 #include <assert.h>
 #include <string.h>
-#include "tclsqlite.h"
+
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 

ext/fts3/fts3.c

@@ -640,7 +640,6 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){
 
     zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid");
     sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1);
-    sqlite3_vtab_config(p->db, SQLITE_VTAB_INNOCUOUS);
 
     /* Create a list of user columns for the virtual table */
     zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]);
@@ -3890,8 +3889,6 @@ static int fts3RenameMethod(
     rc = sqlite3Fts3PendingTermsFlush(p);
   }
 
-  p->bIgnoreSavepoint = 1;
-
   if( p->zContentTbl==0 ){
     fts3DbExec(&rc, db,
       "ALTER TABLE %Q.'%q_content'  RENAME TO '%q_content';",
@@ -3919,8 +3916,6 @@ static int fts3RenameMethod(
     "ALTER TABLE %Q.'%q_segdir'   RENAME TO '%q_segdir';",
     p->zDb, p->zName, zName
   );
-
-  p->bIgnoreSavepoint = 0;
   return rc;
 }
 
@@ -3931,28 +3926,12 @@ static int fts3RenameMethod(
 */
 static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
   int rc = SQLITE_OK;
-  Fts3Table *pTab = (Fts3Table*)pVtab;
-  assert( pTab->inTransaction );
-  assert( pTab->mxSavepoint<=iSavepoint );
-  TESTONLY( pTab->mxSavepoint = iSavepoint );
-
-  if( pTab->bIgnoreSavepoint==0 ){
-    if( fts3HashCount(&pTab->aIndex[0].hPending)>0 ){
-      char *zSql = sqlite3_mprintf("INSERT INTO %Q.%Q(%Q) VALUES('flush')",
-          pTab->zDb, pTab->zName, pTab->zName
-          );
-      if( zSql ){
-        pTab->bIgnoreSavepoint = 1;
-        rc = sqlite3_exec(pTab->db, zSql, 0, 0, 0);
-        pTab->bIgnoreSavepoint = 0;
-        sqlite3_free(zSql);
-      }else{
-        rc = SQLITE_NOMEM;
-      }
-    }
-    if( rc==SQLITE_OK ){
-      pTab->iSavepoint = iSavepoint+1;
-    }
+  UNUSED_PARAMETER(iSavepoint);
+  assert( ((Fts3Table *)pVtab)->inTransaction );
+  assert( ((Fts3Table *)pVtab)->mxSavepoint <= iSavepoint );
+  TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint );
+  if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){
+    rc = fts3SyncMethod(pVtab);
   }
   return rc;
 }
@@ -3963,11 +3942,12 @@ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
 ** This is a no-op.
 */
 static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
-  Fts3Table *pTab = (Fts3Table*)pVtab;
-  assert( pTab->inTransaction );
-  assert( pTab->mxSavepoint >= iSavepoint );
-  TESTONLY( pTab->mxSavepoint = iSavepoint-1 );
-  pTab->iSavepoint = iSavepoint;
+  TESTONLY( Fts3Table *p = (Fts3Table*)pVtab );
+  UNUSED_PARAMETER(iSavepoint);
+  UNUSED_PARAMETER(pVtab);
+  assert( p->inTransaction );
+  assert( p->mxSavepoint >= iSavepoint );
+  TESTONLY( p->mxSavepoint = iSavepoint-1 );
   return SQLITE_OK;
 }
 
@@ -3977,13 +3957,11 @@ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
 ** Discard the contents of the pending terms table.
 */
 static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
-  Fts3Table *pTab = (Fts3Table*)pVtab;
+  Fts3Table *p = (Fts3Table*)pVtab;
   UNUSED_PARAMETER(iSavepoint);
-  assert( pTab->inTransaction );
-  TESTONLY( pTab->mxSavepoint = iSavepoint );
-  if( (iSavepoint+1)<=pTab->iSavepoint ){
-    sqlite3Fts3PendingTermsClear(pTab);
-  }
+  assert( p->inTransaction );
+  TESTONLY( p->mxSavepoint = iSavepoint );
+  sqlite3Fts3PendingTermsClear(p);
   return SQLITE_OK;
 }
 
@@ -4002,42 +3980,8 @@ static int fts3ShadowName(const char *zName){
   return 0;
 }
 
-/*
-** Implementation of the xIntegrity() method on the FTS3/FTS4 virtual
-** table.
-*/
-static int fts3IntegrityMethod(
-  sqlite3_vtab *pVtab,      /* The virtual table to be checked */
-  const char *zSchema,      /* Name of schema in which pVtab lives */
-  const char *zTabname,     /* Name of the pVTab table */
-  int isQuick,              /* True if this is a quick_check */
-  char **pzErr              /* Write error message here */
-){
-  Fts3Table *p = (Fts3Table*)pVtab;
-  int rc = SQLITE_OK;
-  int bOk = 0;
-
-  UNUSED_PARAMETER(isQuick);
-  rc = sqlite3Fts3IntegrityCheck(p, &bOk);
-  assert( rc!=SQLITE_CORRUPT_VTAB );
-  if( rc==SQLITE_ERROR || (rc&0xFF)==SQLITE_CORRUPT ){
-    *pzErr = sqlite3_mprintf("unable to validate the inverted index for"
-                             " FTS%d table %s.%s: %s",
-                p->bFts4 ? 4 : 3, zSchema, zTabname, sqlite3_errstr(rc));
-    if( *pzErr ) rc = SQLITE_OK;
-  }else if( rc==SQLITE_OK && bOk==0 ){
-    *pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s",
-                p->bFts4 ? 4 : 3, zSchema, zTabname);
-    if( *pzErr==0 ) rc = SQLITE_NOMEM;
-  }
-  sqlite3Fts3SegmentsClose(p);
-  return rc;
-}
-
-
-
 static const sqlite3_module fts3Module = {
-  /* iVersion      */ 4,
+  /* iVersion      */ 3,
   /* xCreate       */ fts3CreateMethod,
   /* xConnect      */ fts3ConnectMethod,
   /* xBestIndex    */ fts3BestIndexMethod,
@@ -4061,7 +4005,6 @@ static const sqlite3_module fts3Module = {
   /* xRelease      */ fts3ReleaseMethod,
   /* xRollbackTo   */ fts3RollbackToMethod,
   /* xShadowName   */ fts3ShadowName,
-  /* xIntegrity    */ fts3IntegrityMethod,
 };
 
 /*

ext/fts3/fts3Int.h

@@ -265,7 +265,6 @@ struct Fts3Table {
   int nPgsz;                      /* Page size for host database */
   char *zSegmentsTbl;             /* Name of %_segments table */
   sqlite3_blob *pSegments;        /* Blob handle open on %_segments table */
-  int iSavepoint;
 
   /* 
   ** The following array of hash tables is used to buffer pending index 
@@ -653,7 +652,5 @@ int sqlite3FtsUnicodeIsdiacritic(int);
 
 int sqlite3Fts3ExprIterate(Fts3Expr*, int (*x)(Fts3Expr*,int,void*), void*);
 
-int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk);
-
 #endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */
 #endif /* _FTSINT_H */

ext/fts3/fts3_aux.c

@@ -545,8 +545,7 @@ int sqlite3Fts3InitAux(sqlite3 *db){
      0,                           /* xSavepoint    */
      0,                           /* xRelease      */
      0,                           /* xRollbackTo   */
-     0,                           /* xShadowName   */
-     0                            /* xIntegrity    */
+     0                            /* xShadowName   */
   };
   int rc;                         /* Return code */
 

ext/fts3/fts3_snippet.c

@@ -398,7 +398,6 @@ static int fts3SnippetNextCandidate(SnippetIter *pIter){
       return 1;
     }
 
-    assert( pIter->nSnippet>=0 );
     pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1;
     for(i=0; i<pIter->nPhrase; i++){
       SnippetPhrase *pPhrase = &pIter->aPhrase[i];
@@ -447,7 +446,7 @@ static void fts3SnippetDetails(
         }
         mCover |= mPhrase;
 
-        for(j=0; j<pPhrase->nToken && j<pIter->nSnippet; j++){
+        for(j=0; j<pPhrase->nToken; j++){
           mHighlight |= (mPos>>j);
         }
 

ext/fts3/fts3_term.c

@@ -78,8 +78,6 @@ static int fts3termConnectMethod(
     iIndex = atoi(argv[4]);
     argc--;
   }
-  
-  *ppVtab = 0;
 
   /* The user should specify a single argument - the name of an fts3 table. */
   if( argc!=4 ){
@@ -97,17 +95,12 @@ static int fts3termConnectMethod(
   rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA);
   if( rc!=SQLITE_OK ) return rc;
 
-  nByte = sizeof(Fts3termTable);
-  p = (Fts3termTable *)sqlite3Fts3MallocZero(nByte);
+  nByte = sizeof(Fts3termTable) + sizeof(Fts3Table) + nDb + nFts3 + 2;
+  p = (Fts3termTable *)sqlite3_malloc64(nByte);
   if( !p ) return SQLITE_NOMEM;
+  memset(p, 0, (size_t)nByte);
 
-  p->pFts3Tab = (Fts3Table*)sqlite3Fts3MallocZero(
-      sizeof(Fts3Table) + nDb + nFts3 + 2
-  );
-  if( p->pFts3Tab==0 ){
-    sqlite3_free(p);
-    return SQLITE_NOMEM;
-  }
+  p->pFts3Tab = (Fts3Table *)&p[1];
   p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
   p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
   p->pFts3Tab->db = db;
@@ -137,7 +130,6 @@ static int fts3termDisconnectMethod(sqlite3_vtab *pVtab){
     sqlite3_finalize(pFts3->aStmt[i]);
   }
   sqlite3_free(pFts3->zSegmentsTbl);
-  sqlite3_free(pFts3);
   sqlite3_free(p);
   return SQLITE_OK;
 }
@@ -370,8 +362,7 @@ int sqlite3Fts3InitTerm(sqlite3 *db){
      0,                           /* xSavepoint    */
      0,                           /* xRelease      */
      0,                           /* xRollbackTo   */
-     0,                           /* xShadowName   */
-     0                            /* xIntegrity    */
+     0                            /* xShadowName   */
   };
   int rc;                         /* Return code */
 

ext/fts3/fts3_test.c

@@ -18,7 +18,14 @@
 ** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly.
 */
 
-#include "tclsqlite.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 #include <string.h>
 #include <assert.h>
 
@@ -160,8 +167,7 @@ static int SQLITE_TCLAPI fts3_near_match_cmd(
   Tcl_Obj *pPhrasecount = 0;
   
   Tcl_Obj **apExprToken;
-  Tcl_Size nExprToken;
-  Tcl_Size nn;
+  int nExprToken;
 
   UNUSED_PARAMETER(clientData);
 
@@ -195,25 +201,23 @@ static int SQLITE_TCLAPI fts3_near_match_cmd(
     }
   }
 
-  rc = Tcl_ListObjGetElements(interp, objv[1], &nn, &apDocToken);
-  doc.nToken = (int)nn;
+  rc = Tcl_ListObjGetElements(interp, objv[1], &doc.nToken, &apDocToken);
   if( rc!=TCL_OK ) goto near_match_out;
   doc.aToken = (NearToken *)ckalloc(doc.nToken*sizeof(NearToken));
   for(ii=0; ii<doc.nToken; ii++){
-    doc.aToken[ii].z = Tcl_GetStringFromObj(apDocToken[ii], &nn);
-    doc.aToken[ii].n = (int)nn;
+    doc.aToken[ii].z = Tcl_GetStringFromObj(apDocToken[ii], &doc.aToken[ii].n);
   }
 
   rc = Tcl_ListObjGetElements(interp, objv[2], &nExprToken, &apExprToken);
   if( rc!=TCL_OK ) goto near_match_out;
 
-  nPhrase = (int)(nExprToken + 1) / 2;
+  nPhrase = (nExprToken + 1) / 2;
   aPhrase = (NearPhrase *)ckalloc(nPhrase * sizeof(NearPhrase));
   memset(aPhrase, 0, nPhrase * sizeof(NearPhrase));
   for(ii=0; ii<nPhrase; ii++){
     Tcl_Obj *pPhrase = apExprToken[ii*2];
     Tcl_Obj **apToken;
-    Tcl_Size nToken;
+    int nToken;
     int jj;
 
     rc = Tcl_ListObjGetElements(interp, pPhrase, &nToken, &apToken);
@@ -223,12 +227,11 @@ static int SQLITE_TCLAPI fts3_near_match_cmd(
       rc = TCL_ERROR;
       goto near_match_out;
     }
-    for(jj=0; jj<(int)nToken; jj++){
+    for(jj=0; jj<nToken; jj++){
       NearToken *pT = &aPhrase[ii].aToken[jj];
-      pT->z = Tcl_GetStringFromObj(apToken[jj], &nn);
-      pT->n = (int)nn;
+      pT->z = Tcl_GetStringFromObj(apToken[jj], &pT->n);
     }
-    aPhrase[ii].nToken = (int)nToken;
+    aPhrase[ii].nToken = nToken;
   }
   for(ii=1; ii<nPhrase; ii++){
     Tcl_Obj *pNear = apExprToken[2*ii-1];

ext/fts3/fts3_tokenize_vtab.c

@@ -445,8 +445,7 @@ int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash, void(*xDestroy)(void*)){
      0,                           /* xSavepoint    */
      0,                           /* xRelease      */
      0,                           /* xRollbackTo   */
-     0,                           /* xShadowName   */
-     0                            /* xIntegrity    */
+     0                            /* xShadowName   */
   };
   int rc;                         /* Return code */
 

ext/fts3/fts3_tokenizer.c

@@ -226,7 +226,11 @@ int sqlite3Fts3InitTokenizer(
 
 #ifdef SQLITE_TEST
 
-#include "tclsqlite.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include <string.h>
 
 /*

ext/fts3/fts3_write.c

@@ -2667,18 +2667,16 @@ static int fts3MsrBufferData(
   char *pList,
   i64 nList
 ){
-  if( (nList+FTS3_NODE_PADDING)>pMsr->nBuffer ){
+  if( nList>pMsr->nBuffer ){
     char *pNew;
-    int nNew = nList*2 + FTS3_NODE_PADDING;
-    pNew = (char *)sqlite3_realloc64(pMsr->aBuffer, nNew);
+    pMsr->nBuffer = nList*2;
+    pNew = (char *)sqlite3_realloc64(pMsr->aBuffer, pMsr->nBuffer);
     if( !pNew ) return SQLITE_NOMEM;
     pMsr->aBuffer = pNew;
-    pMsr->nBuffer = nNew;
   }
 
   assert( nList>0 );
   memcpy(pMsr->aBuffer, pList, nList);
-  memset(&pMsr->aBuffer[nList], 0, FTS3_NODE_PADDING);
   return SQLITE_OK;
 }
 
@@ -3325,6 +3323,7 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
     rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING);
     if( rc==SQLITE_DONE ) rc = SQLITE_OK;
   }
+  sqlite3Fts3PendingTermsClear(p);
 
   /* Determine the auto-incr-merge setting if unknown.  If enabled,
   ** estimate the number of leaf blocks of content to be written
@@ -3346,10 +3345,6 @@ int sqlite3Fts3PendingTermsFlush(Fts3Table *p){
       rc = sqlite3_reset(pStmt);
     }
   }
-
-  if( rc==SQLITE_OK ){
-    sqlite3Fts3PendingTermsClear(p);
-  }
   return rc;
 }
 
@@ -3981,8 +3976,6 @@ static int fts3AppendToNode(
 
   blobGrowBuffer(pPrev, nTerm, &rc);
   if( rc!=SQLITE_OK ) return rc;
-  assert( pPrev!=0 );
-  assert( pPrev->a!=0 );
 
   nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm);
   nSuffix = nTerm - nPrefix;
@@ -4039,13 +4032,9 @@ static int fts3IncrmergeAppend(
   nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist;
 
   /* If the current block is not empty, and if adding this term/doclist
-  ** to the current block would make it larger than Fts3Table.nNodeSize bytes,
-  ** and if there is still room for another leaf page, write this block out to
-  ** the database. */
-  if( pLeaf->block.n>0 
-   && (pLeaf->block.n + nSpace)>p->nNodeSize 
-   && pLeaf->iBlock < (pWriter->iStart + pWriter->nLeafEst)
-  ){
+  ** to the current block would make it larger than Fts3Table.nNodeSize
+  ** bytes, write this block out to the database. */
+  if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){
     rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n);
     pWriter->nWork++;
 
@@ -4356,7 +4345,6 @@ static int fts3IncrmergeLoad(
 
       for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){
         NodeReader reader;
-        memset(&reader, 0, sizeof(reader));
         pNode = &pWriter->aNodeWriter[i];
 
         if( pNode->block.a){
@@ -4377,7 +4365,7 @@ static int fts3IncrmergeLoad(
               rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock,0);
               blobGrowBuffer(&pNode->block, 
                   MAX(nBlock, p->nNodeSize)+FTS3_NODE_PADDING, &rc
-              );
+                  );
               if( rc==SQLITE_OK ){
                 memcpy(pNode->block.a, aBlock, nBlock);
                 pNode->block.n = nBlock;
@@ -5227,7 +5215,7 @@ static u64 fts3ChecksumIndex(
   int rc;
   u64 cksum = 0;
 
-  if( *pRc ) return 0;
+  assert( *pRc==SQLITE_OK );
 
   memset(&filter, 0, sizeof(filter));
   memset(&csr, 0, sizeof(csr));
@@ -5294,7 +5282,7 @@ static u64 fts3ChecksumIndex(
 ** If an error occurs (e.g. an OOM or IO error), return an SQLite error 
 ** code. The final value of *pbOk is undefined in this case.
 */
-int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){
+static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){
   int rc = SQLITE_OK;             /* Return code */
   u64 cksum1 = 0;                 /* Checksum based on FTS index contents */
   u64 cksum2 = 0;                 /* Checksum based on %_content contents */
@@ -5372,12 +5360,7 @@ int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){
     sqlite3_finalize(pStmt);
   }
 
-  if( rc==SQLITE_CORRUPT_VTAB ){
-    rc = SQLITE_OK;
-    *pbOk = 0;
-  }else{
-    *pbOk = (rc==SQLITE_OK && cksum1==cksum2);
-  }
+  *pbOk = (cksum1==cksum2);
   return rc;
 }
 
@@ -5417,7 +5400,7 @@ static int fts3DoIntegrityCheck(
 ){
   int rc;
   int bOk = 0;
-  rc = sqlite3Fts3IntegrityCheck(p, &bOk);
+  rc = fts3IntegrityCheck(p, &bOk);
   if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB;
   return rc;
 }
@@ -5447,11 +5430,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
     rc = fts3DoIncrmerge(p, &zVal[6]);
   }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){
     rc = fts3DoAutoincrmerge(p, &zVal[10]);
-  }else if( nVal==5 && 0==sqlite3_strnicmp(zVal, "flush", 5) ){
-    rc = sqlite3Fts3PendingTermsFlush(p);
-  }
 #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST)
-  else{
+  }else{
     int v;
     if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){
       v = atoi(&zVal[9]);
@@ -5469,8 +5449,8 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){
       if( v>=4 && v<=FTS3_MERGE_COUNT && (v&1)==0 ) p->nMergeCount = v;
       rc = SQLITE_OK;
     }
-  }
 #endif
+  }
   return rc;
 }
 

ext/fts5/extract_api_docs.tcl

@@ -223,12 +223,10 @@ proc main {data} {
     Fts5ExtensionApi {
       set struct [get_fts5_struct $data "^struct Fts5ExtensionApi" "^.;"]
       set map [list]
-      set lKey [list]
       foreach {k v} [get_struct_members $data] {
         if {[string match x* $k]==0} continue
-        lappend lKey $k
+        lappend map $k "<a href=#$k>$k</a>"
       }
-      foreach k [lsort -decr $lKey] { lappend map $k "<a href=#$k>$k</a>" }
       output [string map $map $struct]
     }
 

ext/fts5/fts5.h

@@ -55,8 +55,8 @@ struct Fts5PhraseIter {
 ** EXTENSION API FUNCTIONS
 **
 ** xUserData(pFts):
-**   Return a copy of the pUserData pointer passed to the xCreateFunction()
-**   API when the extension function was registered.
+**   Return a copy of the context pointer the extension function was 
+**   registered with.
 **
 ** xColumnTotalSize(pFts, iCol, pnToken):
 **   If parameter iCol is less than zero, set output variable *pnToken
@@ -88,11 +88,8 @@ struct Fts5PhraseIter {
 **   created with the "columnsize=0" option.
 **
 ** xColumnText:
-**   If parameter iCol is less than zero, or greater than or equal to the
-**   number of columns in the table, SQLITE_RANGE is returned. 
-**
-**   Otherwise, this function attempts to retrieve the text of column iCol of
-**   the current document. If successful, (*pz) is set to point to a buffer
+**   This function attempts to retrieve the text of column iCol of the
+**   current document. If successful, (*pz) is set to point to a buffer
 **   containing the text in utf-8 encoding, (*pn) is set to the size in bytes
 **   (not characters) of the buffer and SQLITE_OK is returned. Otherwise,
 **   if an error occurs, an SQLite error code is returned and the final values
@@ -102,10 +99,8 @@ struct Fts5PhraseIter {
 **   Returns the number of phrases in the current query expression.
 **
 ** xPhraseSize:
-**   If parameter iCol is less than zero, or greater than or equal to the
-**   number of phrases in the current query, as returned by xPhraseCount, 
-**   0 is returned. Otherwise, this function returns the number of tokens in
-**   phrase iPhrase of the query. Phrases are numbered starting from zero.
+**   Returns the number of tokens in phrase iPhrase of the query. Phrases
+**   are numbered starting from zero.
 **
 ** xInstCount:
 **   Set *pnInst to the total number of occurrences of all phrases within
@@ -121,13 +116,12 @@ struct Fts5PhraseIter {
 **   Query for the details of phrase match iIdx within the current row.
 **   Phrase matches are numbered starting from zero, so the iIdx argument
 **   should be greater than or equal to zero and smaller than the value
-**   output by xInstCount(). If iIdx is less than zero or greater than
-**   or equal to the value returned by xInstCount(), SQLITE_RANGE is returned.
+**   output by xInstCount().
 **
-**   Otherwise, output parameter *piPhrase is set to the phrase number, *piCol
+**   Usually, output parameter *piPhrase is set to the phrase number, *piCol
 **   to the column in which it occurs and *piOff the token offset of the
-**   first token of the phrase. SQLITE_OK is returned if successful, or an
-**   error code (i.e. SQLITE_NOMEM) if an error occurs.
+**   first token of the phrase. Returns SQLITE_OK if successful, or an error
+**   code (i.e. SQLITE_NOMEM) if an error occurs.
 **
 **   This API can be quite slow if used with an FTS5 table created with the
 **   "detail=none" or "detail=column" option. 
@@ -153,10 +147,6 @@ struct Fts5PhraseIter {
 **   Invoking Api.xUserData() returns a copy of the pointer passed as 
 **   the third argument to pUserData.
 **
-**   If parameter iPhrase is less than zero, or greater than or equal to
-**   the number of phrases in the query, as returned by xPhraseCount(),
-**   this function returns SQLITE_RANGE.
-**
 **   If the callback function returns any value other than SQLITE_OK, the
 **   query is abandoned and the xQueryPhrase function returns immediately.
 **   If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK.
@@ -238,10 +228,6 @@ struct Fts5PhraseIter {
 **   (i.e. if it is a contentless table), then this API always iterates
 **   through an empty set (all calls to xPhraseFirst() set iCol to -1).
 **
-**   In all cases, matches are visited in (column ASC, offset ASC) order.
-**   i.e. all those in column 0, sorted by offset, followed by those in 
-**   column 1, etc.
-**
 ** xPhraseNext()
 **   See xPhraseFirst above.
 **
@@ -275,39 +261,6 @@ struct Fts5PhraseIter {
 **
 ** xPhraseNextColumn()
 **   See xPhraseFirstColumn above.
-**
-** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken)
-**   This is used to access token iToken of phrase iPhrase of the current
-**   query. Before returning, output parameter *ppToken is set to point
-**   to a buffer containing the requested token, and *pnToken to the
-**   size of this buffer in bytes.
-**
-**   If iPhrase or iToken are less than zero, or if iPhrase is greater than
-**   or equal to the number of phrases in the query as reported by 
-**   xPhraseCount(), or if iToken is equal to or greater than the number of
-**   tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken
-     are both zeroed.
-**
-**   The output text is not a copy of the query text that specified the
-**   token. It is the output of the tokenizer module. For tokendata=1
-**   tables, this includes any embedded 0x00 and trailing data.
-**
-** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken)
-**   This is used to access token iToken of phrase hit iIdx within the
-**   current row. If iIdx is less than zero or greater than or equal to the
-**   value returned by xInstCount(), SQLITE_RANGE is returned.  Otherwise,
-**   output variable (*ppToken) is set to point to a buffer containing the
-**   matching document token, and (*pnToken) to the size of that buffer in 
-**   bytes. This API is not available if the specified token matches a 
-**   prefix query term. In that case both output variables are always set 
-**   to 0.
-**
-**   The output text is not a copy of the document text that was tokenized.
-**   It is the output of the tokenizer module. For tokendata=1 tables, this 
-**   includes any embedded 0x00 and trailing data.
-**
-**   This API can be quite slow if used with an FTS5 table created with the
-**   "detail=none" or "detail=column" option.
 */
 struct Fts5ExtensionApi {
   int iVersion;                   /* Currently always set to 3 */
@@ -345,13 +298,6 @@ struct Fts5ExtensionApi {
 
   int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*);
   void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol);
-
-  /* Below this point are iVersion>=3 only */
-  int (*xQueryToken)(Fts5Context*, 
-      int iPhrase, int iToken, 
-      const char **ppToken, int *pnToken
-  );
-  int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*);
 };
 
 /* 
@@ -546,8 +492,8 @@ struct Fts5ExtensionApi {
 **   as separate queries of the FTS index are required for each synonym.
 **
 **   When using methods (2) or (3), it is important that the tokenizer only
-**   provide synonyms when tokenizing document text (method (3)) or query
-**   text (method (2)), not both. Doing so will not cause any errors, but is
+**   provide synonyms when tokenizing document text (method (2)) or query
+**   text (method (3)), not both. Doing so will not cause any errors, but is
 **   inefficient.
 */
 typedef struct Fts5Tokenizer Fts5Tokenizer;
@@ -595,7 +541,7 @@ struct fts5_api {
   int (*xCreateTokenizer)(
     fts5_api *pApi,
     const char *zName,
-    void *pUserData,
+    void *pContext,
     fts5_tokenizer *pTokenizer,
     void (*xDestroy)(void*)
   );
@@ -604,7 +550,7 @@ struct fts5_api {
   int (*xFindTokenizer)(
     fts5_api *pApi,
     const char *zName,
-    void **ppUserData,
+    void **ppContext,
     fts5_tokenizer *pTokenizer
   );
 
@@ -612,7 +558,7 @@ struct fts5_api {
   int (*xCreateFunction)(
     fts5_api *pApi,
     const char *zName,
-    void *pUserData,
+    void *pContext,
     fts5_extension_function xFunction,
     void (*xDestroy)(void*)
   );

ext/fts5/fts5Int.h

@@ -59,22 +59,6 @@ typedef sqlite3_uint64 u64;
 # define LARGEST_INT64  (0xffffffff|(((i64)0x7fffffff)<<32))
 # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
 
-/* The uptr type is an unsigned integer large enough to hold a pointer
-*/
-#if defined(HAVE_STDINT_H)
-  typedef uintptr_t uptr;
-#elif SQLITE_PTRSIZE==4
-  typedef u32 uptr;
-#else
-  typedef u64 uptr;
-#endif
-
-#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC
-# define EIGHT_BYTE_ALIGNMENT(X)   ((((uptr)(X) - (uptr)0)&3)==0)
-#else
-# define EIGHT_BYTE_ALIGNMENT(X)   ((((uptr)(X) - (uptr)0)&7)==0)
-#endif
-
 #endif
 
 /* Truncate very long tokens to this many bytes. Hard limit is 
@@ -158,15 +142,6 @@ struct Fts5Colset {
 */
 
 typedef struct Fts5Config Fts5Config;
-typedef struct Fts5TokenizerConfig Fts5TokenizerConfig;
-
-struct Fts5TokenizerConfig {
-  Fts5Tokenizer *pTok;
-  fts5_tokenizer *pTokApi;
-  const char **azArg;
-  int nArg;
-  int ePattern;                   /* FTS_PATTERN_XXX constant */
-};
 
 /*
 ** An instance of the following structure encodes all information that can
@@ -179,10 +154,6 @@ struct Fts5TokenizerConfig {
 **   attempt to merge together. A value of 1 sets the object to use the 
 **   compile time default. Zero disables auto-merge altogether.
 **
-** bContentlessDelete:
-**   True if the contentless_delete option was present in the CREATE 
-**   VIRTUAL TABLE statement.
-**
 ** zContent:
 **
 ** zContentRowid:
@@ -209,7 +180,6 @@ struct Fts5TokenizerConfig {
 */
 struct Fts5Config {
   sqlite3 *db;                    /* Database handle */
-  Fts5Global *pGlobal;            /* Global fts5 object for handle db */
   char *zDb;                      /* Database holding FTS index (e.g. "main") */
   char *zName;                    /* Name of FTS index */
   int nCol;                       /* Number of columns */
@@ -218,18 +188,17 @@ struct Fts5Config {
   int nPrefix;                    /* Number of prefix indexes */
   int *aPrefix;                   /* Sizes in bytes of nPrefix prefix indexes */
   int eContent;                   /* An FTS5_CONTENT value */
-  int bContentlessDelete;         /* "contentless_delete=" option (dflt==0) */
   char *zContent;                 /* content table */ 
   char *zContentRowid;            /* "content_rowid=" option value */ 
   int bColumnsize;                /* "columnsize=" option value (dflt==1) */
-  int bTokendata;                 /* "tokendata=" option value (dflt==0) */
   int eDetail;                    /* FTS5_DETAIL_XXX value */
   char *zContentExprlist;
-  Fts5TokenizerConfig t;
+  Fts5Tokenizer *pTok;
+  fts5_tokenizer *pTokApi;
   int bLock;                      /* True when table is preparing statement */
+  int ePattern;                   /* FTS_PATTERN_XXX constant */
 
   /* Values loaded from the %_config table */
-  int iVersion;                   /* fts5 file format 'version' */
   int iCookie;                    /* Incremented when %_config is modified */
   int pgsz;                       /* Approximate page size used in %_data */
   int nAutomerge;                 /* 'automerge' setting */
@@ -238,8 +207,6 @@ struct Fts5Config {
   int nHashSize;                  /* Bytes of memory for in-memory hash */
   char *zRank;                    /* Name of rank function */
   char *zRankArgs;                /* Arguments to rank function */
-  int bSecureDelete;              /* 'secure-delete' */
-  int nDeleteMerge;           /* 'deletemerge' */
 
   /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
   char **pzErrmsg;
@@ -249,11 +216,8 @@ struct Fts5Config {
 #endif
 };
 
-/* Current expected value of %_config table 'version' field. And
-** the expected version if the 'secure-delete' option has ever been
-** set on the table.  */
-#define FTS5_CURRENT_VERSION               4
-#define FTS5_CURRENT_VERSION_SECUREDELETE  5
+/* Current expected value of %_config table 'version' field */
+#define FTS5_CURRENT_VERSION  4
 
 #define FTS5_CONTENT_NORMAL   0
 #define FTS5_CONTENT_NONE     1
@@ -409,19 +373,16 @@ struct Fts5IndexIter {
 /*
 ** Values used as part of the flags argument passed to IndexQuery().
 */
-#define FTS5INDEX_QUERY_PREFIX      0x0001  /* Prefix query */
-#define FTS5INDEX_QUERY_DESC        0x0002  /* Docs in descending rowid order */
-#define FTS5INDEX_QUERY_TEST_NOIDX  0x0004  /* Do not use prefix index */
-#define FTS5INDEX_QUERY_SCAN        0x0008  /* Scan query (fts5vocab) */
+#define FTS5INDEX_QUERY_PREFIX     0x0001   /* Prefix query */
+#define FTS5INDEX_QUERY_DESC       0x0002   /* Docs in descending rowid order */
+#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004   /* Do not use prefix index */
+#define FTS5INDEX_QUERY_SCAN       0x0008   /* Scan query (fts5vocab) */
 
 /* The following are used internally by the fts5_index.c module. They are
 ** defined here only to make it easier to avoid clashes with the flags
 ** above. */
-#define FTS5INDEX_QUERY_SKIPEMPTY   0x0010
-#define FTS5INDEX_QUERY_NOOUTPUT    0x0020
-#define FTS5INDEX_QUERY_SKIPHASH    0x0040
-#define FTS5INDEX_QUERY_NOTOKENDATA 0x0080
-#define FTS5INDEX_QUERY_SCANONETERM 0x0100
+#define FTS5INDEX_QUERY_SKIPEMPTY  0x0010
+#define FTS5INDEX_QUERY_NOOUTPUT   0x0020
 
 /*
 ** Create/destroy an Fts5Index object.
@@ -490,10 +451,6 @@ void *sqlite3Fts5StructureRef(Fts5Index*);
 void sqlite3Fts5StructureRelease(void*);
 int sqlite3Fts5StructureTest(Fts5Index*, void*);
 
-/*
-** Used by xInstToken():
-*/
-int sqlite3Fts5IterToken(Fts5IndexIter*, i64, int, int, const char**, int*);
 
 /*
 ** Insert or remove data to or from the index. Each time a document is 
@@ -568,16 +525,6 @@ int sqlite3Fts5IndexReset(Fts5Index *p);
 
 int sqlite3Fts5IndexLoadConfig(Fts5Index *p);
 
-int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin);
-int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid);
-
-void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*);
-
-/* Used to populate hash tables for xInstToken in detail=none/column mode. */
-int sqlite3Fts5IndexIterWriteTokendata(
-    Fts5IndexIter*, const char*, int, i64 iRowid, int iCol, int iOff
-);
-
 /*
 ** End of interface to code in fts5_index.c.
 **************************************************************************/
@@ -590,7 +537,7 @@ int sqlite3Fts5GetVarintLen(u32 iVal);
 u8 sqlite3Fts5GetVarint(const unsigned char*, u64*);
 int sqlite3Fts5PutVarint(unsigned char *p, u64 v);
 
-#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&(b))
+#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b)
 #define fts5GetVarint    sqlite3Fts5GetVarint
 
 #define fts5FastGetVarint32(a, iOff, nVal) {      \
@@ -621,7 +568,13 @@ struct Fts5Table {
   Fts5Index *pIndex;              /* Full-text index */
 };
 
-int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig);
+int sqlite3Fts5GetTokenizer(
+  Fts5Global*, 
+  const char **azArg,
+  int nArg,
+  Fts5Config*,
+  char **pzErr
+);
 
 Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64);
 
@@ -656,11 +609,6 @@ int sqlite3Fts5HashWrite(
 */
 void sqlite3Fts5HashClear(Fts5Hash*);
 
-/*
-** Return true if the hash is empty, false otherwise.
-*/
-int sqlite3Fts5HashIsEmpty(Fts5Hash*);
-
 int sqlite3Fts5HashQuery(
   Fts5Hash*,                      /* Hash table to query */
   int nPre,
@@ -677,13 +625,11 @@ void sqlite3Fts5HashScanNext(Fts5Hash*);
 int sqlite3Fts5HashScanEof(Fts5Hash*);
 void sqlite3Fts5HashScanEntry(Fts5Hash *,
   const char **pzTerm,            /* OUT: term (nul-terminated) */
-  int *pnTerm,                    /* OUT: Size of term in bytes */
   const u8 **ppDoclist,           /* OUT: pointer to doclist */
   int *pnDoclist                  /* OUT: size of doclist in bytes */
 );
 
 
-
 /*
 ** End of interface to code in fts5_hash.c.
 **************************************************************************/
@@ -804,10 +750,6 @@ int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**);
 
 int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *);
 
-int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*);
-int sqlite3Fts5ExprInstToken(Fts5Expr*, i64, int, int, int, int, const char**, int*);
-void sqlite3Fts5ExprClearTokens(Fts5Expr*);
-
 /*******************************************
 ** The fts5_expr.c API above this point is used by the other hand-written
 ** C code in this module. The interfaces below this point are called by
@@ -884,7 +826,6 @@ int sqlite3Fts5TokenizerPattern(
     int (*xCreate)(void*, const char**, int, Fts5Tokenizer**),
     Fts5Tokenizer *pTok
 );
-int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig*);
 /*
 ** End of interface to code in fts5_tokenizer.c.
 **************************************************************************/

ext/fts5/fts5_aux.c

@@ -110,19 +110,15 @@ static int fts5CInstIterInit(
 */
 typedef struct HighlightContext HighlightContext;
 struct HighlightContext {
-  /* Constant parameters to fts5HighlightCb() */
+  CInstIter iter;                 /* Coalesced Instance Iterator */
+  int iPos;                       /* Current token offset in zIn[] */
   int iRangeStart;                /* First token to include */
   int iRangeEnd;                  /* If non-zero, last token to include */
   const char *zOpen;              /* Opening highlight */
   const char *zClose;             /* Closing highlight */
   const char *zIn;                /* Input text */
   int nIn;                        /* Size of input text in bytes */
-
-  /* Variables modified by fts5HighlightCb() */
-  CInstIter iter;                 /* Coalesced Instance Iterator */
-  int iPos;                       /* Current token offset in zIn[] */
-  int iOff;                       /* Have copied up to this offset in zIn[] */
-  int bOpen;                      /* True if highlight is open */
+  int iOff;                       /* Current offset within zIn[] */
   char *zOut;                     /* Output value */
 };
 
@@ -155,8 +151,8 @@ static int fts5HighlightCb(
   int tflags,                     /* Mask of FTS5_TOKEN_* flags */
   const char *pToken,             /* Buffer containing token */
   int nToken,                     /* Size of token in bytes */
-  int iStartOff,                  /* Start byte offset of token */
-  int iEndOff                     /* End byte offset of token */
+  int iStartOff,                  /* Start offset of token */
+  int iEndOff                     /* End offset of token */
 ){
   HighlightContext *p = (HighlightContext*)pContext;
   int rc = SQLITE_OK;
@@ -167,60 +163,35 @@ static int fts5HighlightCb(
   if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK;
   iPos = p->iPos++;
 
-  if( p->iRangeEnd>=0 ){
+  if( p->iRangeEnd>0 ){
     if( iPos<p->iRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK;
     if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff;
   }
 
-  /* If the parenthesis is open, and this token is not part of the current
-  ** phrase, and the starting byte offset of this token is past the point
-  ** that has currently been copied into the output buffer, close the
-  ** parenthesis. */
-  if( p->bOpen 
-   && (iPos<=p->iter.iStart || p->iter.iStart<0)
-   && iStartOff>p->iOff 
-  ){
-    fts5HighlightAppend(&rc, p, p->zClose, -1);
-    p->bOpen = 0;
-  }
-
-  /* If this is the start of a new phrase, and the highlight is not open:
-  **
-  **   * copy text from the input up to the start of the phrase, and
-  **   * open the highlight.
-  */
-  if( iPos==p->iter.iStart && p->bOpen==0 ){
+  if( iPos==p->iter.iStart ){
     fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff);
     fts5HighlightAppend(&rc, p, p->zOpen, -1);
     p->iOff = iStartOff;
-    p->bOpen = 1;
   }
 
   if( iPos==p->iter.iEnd ){
-    if( p->bOpen==0 ){
-      assert( p->iRangeEnd>=0 );
+    if( p->iRangeEnd && p->iter.iStart<p->iRangeStart ){
       fts5HighlightAppend(&rc, p, p->zOpen, -1);
-      p->bOpen = 1;
     }
     fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
+    fts5HighlightAppend(&rc, p, p->zClose, -1);
     p->iOff = iEndOff;
-
     if( rc==SQLITE_OK ){
       rc = fts5CInstIterNext(&p->iter);
     }
   }
 
-  if( iPos==p->iRangeEnd ){
-    if( p->bOpen ){
-      if( p->iter.iStart>=0 && iPos>=p->iter.iStart ){
-        fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
-        p->iOff = iEndOff;
-      }
-      fts5HighlightAppend(&rc, p, p->zClose, -1);
-      p->bOpen = 0;
-    }
+  if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){
     fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
     p->iOff = iEndOff;
+    if( iPos>=p->iter.iStart && iPos<p->iter.iEnd ){
+      fts5HighlightAppend(&rc, p, p->zClose, -1);
+    }
   }
 
   return rc;
@@ -250,12 +221,9 @@ static void fts5HighlightFunction(
   memset(&ctx, 0, sizeof(HighlightContext));
   ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
   ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
-  ctx.iRangeEnd = -1;
   rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn);
-  if( rc==SQLITE_RANGE ){
-    sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC);
-    rc = SQLITE_OK;
-  }else if( ctx.zIn ){
+
+  if( ctx.zIn ){
     if( rc==SQLITE_OK ){
       rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter);
     }
@@ -263,9 +231,6 @@ static void fts5HighlightFunction(
     if( rc==SQLITE_OK ){
       rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
     }
-    if( ctx.bOpen ){
-      fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1);
-    }
     fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
 
     if( rc==SQLITE_OK ){
@@ -441,7 +406,6 @@ static void fts5SnippetFunction(
   iCol = sqlite3_value_int(apVal[0]);
   ctx.zOpen = fts5ValueToText(apVal[1]);
   ctx.zClose = fts5ValueToText(apVal[2]);
-  ctx.iRangeEnd = -1;
   zEllips = fts5ValueToText(apVal[3]);
   nToken = sqlite3_value_int(apVal[4]);
 
@@ -544,9 +508,6 @@ static void fts5SnippetFunction(
     if( rc==SQLITE_OK ){
       rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
     }
-    if( ctx.bOpen ){
-      fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1);
-    }
     if( ctx.iRangeEnd>=(nColSize-1) ){
       fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff);
     }else{

ext/fts5/fts5_buffer.c

@@ -68,7 +68,6 @@ void sqlite3Fts5BufferAppendBlob(
 ){
   if( nData ){
     if( fts5BufferGrow(pRc, pBuf, nData) ) return;
-    assert( pBuf->p!=0 );
     memcpy(&pBuf->p[pBuf->n], pData, nData);
     pBuf->n += nData;
   }
@@ -170,7 +169,6 @@ int sqlite3Fts5PoslistNext64(
   i64 *piOff                      /* IN/OUT: Current offset */
 ){
   int i = *pi;
-  assert( a!=0 || i==0 );
   if( i>=n ){
     /* EOF */
     *piOff = -1;
@@ -178,7 +176,6 @@ int sqlite3Fts5PoslistNext64(
   }else{
     i64 iOff = *piOff;
     u32 iVal;
-    assert( a!=0 );
     fts5FastGetVarint32(a, i, iVal);
     if( iVal<=1 ){
       if( iVal==0 ){

ext/fts5/fts5_config.c

@@ -22,8 +22,6 @@
 #define FTS5_DEFAULT_CRISISMERGE   16
 #define FTS5_DEFAULT_HASHSIZE    (1024*1024)
 
-#define FTS5_DEFAULT_DELETE_AUTOMERGE 10      /* default 10% */
-
 /* Maximum allowed page size */
 #define FTS5_MAX_PAGE_SIZE (64*1024)
 
@@ -234,6 +232,7 @@ static int fts5ConfigSetEnum(
 ** eventually free any such error message using sqlite3_free().
 */
 static int fts5ConfigParseSpecial(
+  Fts5Global *pGlobal,
   Fts5Config *pConfig,            /* Configuration object to update */
   const char *zCmd,               /* Special command to parse */
   const char *zArg,               /* Argument to parse */
@@ -297,11 +296,12 @@ static int fts5ConfigParseSpecial(
   if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
     const char *p = (const char*)zArg;
     sqlite3_int64 nArg = strlen(zArg) + 1;
-    char **azArg = sqlite3Fts5MallocZero(&rc, (sizeof(char*) + 2) * nArg);
+    char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
+    char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2);
+    char *pSpace = pDel;
 
-    if( azArg ){
-      char *pSpace = (char*)&azArg[nArg];
-      if( pConfig->t.azArg ){
+    if( azArg && pSpace ){
+      if( pConfig->pTok ){
         *pzErr = sqlite3_mprintf("multiple tokenize=... directives");
         rc = SQLITE_ERROR;
       }else{
@@ -324,14 +324,16 @@ static int fts5ConfigParseSpecial(
           *pzErr = sqlite3_mprintf("parse error in tokenize directive");
           rc = SQLITE_ERROR;
         }else{
-          pConfig->t.azArg = (const char**)azArg;
-          pConfig->t.nArg = nArg;
-          azArg = 0;
+          rc = sqlite3Fts5GetTokenizer(pGlobal, 
+              (const char**)azArg, (int)nArg, pConfig,
+              pzErr
+          );
         }
       }
     }
-    sqlite3_free(azArg);
 
+    sqlite3_free(azArg);
+    sqlite3_free(pDel);
     return rc;
   }
 
@@ -350,16 +352,6 @@ static int fts5ConfigParseSpecial(
     return rc;
   }
 
-  if( sqlite3_strnicmp("contentless_delete", zCmd, nCmd)==0 ){
-    if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
-      *pzErr = sqlite3_mprintf("malformed contentless_delete=... directive");
-      rc = SQLITE_ERROR;
-    }else{
-      pConfig->bContentlessDelete = (zArg[0]=='1');
-    }
-    return rc;
-  }
-
   if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){
     if( pConfig->zContentRowid ){
       *pzErr = sqlite3_mprintf("multiple content_rowid=... directives");
@@ -394,20 +386,20 @@ static int fts5ConfigParseSpecial(
     return rc;
   }
 
-  if( sqlite3_strnicmp("tokendata", zCmd, nCmd)==0 ){
-    if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
-      *pzErr = sqlite3_mprintf("malformed tokendata=... directive");
-      rc = SQLITE_ERROR;
-    }else{
-      pConfig->bTokendata = (zArg[0]=='1');
-    }
-    return rc;
-  }
-
   *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd);
   return SQLITE_ERROR;
 }
 
+/*
+** Allocate an instance of the default tokenizer ("simple") at 
+** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
+** code if an error occurs.
+*/
+static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
+  assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
+  return sqlite3Fts5GetTokenizer(pGlobal, 0, 0, pConfig, 0);
+}
+
 /*
 ** Gobble up the first bareword or quoted word from the input buffer zIn.
 ** Return a pointer to the character immediately following the last in
@@ -540,7 +532,6 @@ int sqlite3Fts5ConfigParse(
   *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config));
   if( pRet==0 ) return SQLITE_NOMEM;
   memset(pRet, 0, sizeof(Fts5Config));
-  pRet->pGlobal = pGlobal;
   pRet->db = db;
   pRet->iCookie = -1;
 
@@ -559,7 +550,6 @@ int sqlite3Fts5ConfigParse(
     rc = SQLITE_ERROR;
   }
 
-  assert( (pRet->abUnindexed && pRet->azCol) || rc!=SQLITE_OK );
   for(i=3; rc==SQLITE_OK && i<nArg; i++){
     const char *zOrig = azArg[i];
     const char *z;
@@ -589,7 +579,7 @@ int sqlite3Fts5ConfigParse(
         rc = SQLITE_ERROR;
       }else{
         if( bOption ){
-          rc = fts5ConfigParseSpecial(pRet, 
+          rc = fts5ConfigParseSpecial(pGlobal, pRet, 
             ALWAYS(zOne)?zOne:"",
             zTwo?zTwo:"",
             pzErr
@@ -605,26 +595,11 @@ int sqlite3Fts5ConfigParse(
     sqlite3_free(zTwo);
   }
 
-  /* We only allow contentless_delete=1 if the table is indeed contentless. */
-  if( rc==SQLITE_OK 
-   && pRet->bContentlessDelete 
-   && pRet->eContent!=FTS5_CONTENT_NONE 
-  ){
-    *pzErr = sqlite3_mprintf(
-        "contentless_delete=1 requires a contentless table"
-    );
-    rc = SQLITE_ERROR;
-  }
-
-  /* We only allow contentless_delete=1 if columnsize=0 is not present. 
-  **
-  ** This restriction may be removed at some point. 
-  */
-  if( rc==SQLITE_OK && pRet->bContentlessDelete && pRet->bColumnsize==0 ){
-    *pzErr = sqlite3_mprintf(
-        "contentless_delete=1 is incompatible with columnsize=0"
-    );
-    rc = SQLITE_ERROR;
+  /* If a tokenizer= option was successfully parsed, the tokenizer has
+  ** already been allocated. Otherwise, allocate an instance of the default
+  ** tokenizer (unicode61) now.  */
+  if( rc==SQLITE_OK && pRet->pTok==0 ){
+    rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
   }
 
   /* If no zContent option was specified, fill in the default values. */
@@ -668,10 +643,9 @@ int sqlite3Fts5ConfigParse(
 void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
   if( pConfig ){
     int i;
-    if( pConfig->t.pTok ){
-      pConfig->t.pTokApi->xDelete(pConfig->t.pTok);
+    if( pConfig->pTok ){
+      pConfig->pTokApi->xDelete(pConfig->pTok);
     }
-    sqlite3_free((char*)pConfig->t.azArg);
     sqlite3_free(pConfig->zDb);
     sqlite3_free(pConfig->zName);
     for(i=0; i<pConfig->nCol; i++){
@@ -746,18 +720,10 @@ int sqlite3Fts5Tokenize(
   void *pCtx,                     /* Context passed to xToken() */
   int (*xToken)(void*, int, const char*, int, int, int)    /* Callback */
 ){
-  int rc = SQLITE_OK;
-  if( pText ){
-    if( pConfig->t.pTok==0 ){
-      rc = sqlite3Fts5LoadTokenizer(pConfig);
-    }
-    if( rc==SQLITE_OK ){
-      rc = pConfig->t.pTokApi->xTokenize(
-          pConfig->t.pTok, pCtx, flags, pText, nText, xToken
-      );
-    }
-  }
-  return rc;
+  if( pText==0 ) return SQLITE_OK;
+  return pConfig->pTokApi->xTokenize(
+      pConfig->pTok, pCtx, flags, pText, nText, xToken
+  );
 }
 
 /*
@@ -923,18 +889,6 @@ int sqlite3Fts5ConfigSetValue(
     }
   }
 
-  else if( 0==sqlite3_stricmp(zKey, "deletemerge") ){
-    int nVal = -1;
-    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
-      nVal = sqlite3_value_int(pVal);
-    }else{
-      *pbBadkey = 1;
-    }
-    if( nVal<0 ) nVal = FTS5_DEFAULT_DELETE_AUTOMERGE;
-    if( nVal>100 ) nVal = 0;
-    pConfig->nDeleteMerge = nVal;
-  }
-
   else if( 0==sqlite3_stricmp(zKey, "rank") ){
     const char *zIn = (const char*)sqlite3_value_text(pVal);
     char *zRank;
@@ -949,18 +903,6 @@ int sqlite3Fts5ConfigSetValue(
       rc = SQLITE_OK;
       *pbBadkey = 1;
     }
-  }
-
-  else if( 0==sqlite3_stricmp(zKey, "secure-delete") ){
-    int bVal = -1;
-    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
-      bVal = sqlite3_value_int(pVal);
-    }
-    if( bVal<0 ){
-      *pbBadkey = 1;
-    }else{
-      pConfig->bSecureDelete = (bVal ? 1 : 0);
-    }
   }else{
     *pbBadkey = 1;
   }
@@ -983,7 +925,6 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
   pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE;
   pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE;
   pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE;
-  pConfig->nDeleteMerge = FTS5_DEFAULT_DELETE_AUTOMERGE;
 
   zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName);
   if( zSql ){
@@ -1006,20 +947,15 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
     rc = sqlite3_finalize(p);
   }
   
-  if( rc==SQLITE_OK 
-   && iVersion!=FTS5_CURRENT_VERSION
-   && iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE
-  ){
+  if( rc==SQLITE_OK && iVersion!=FTS5_CURRENT_VERSION ){
     rc = SQLITE_ERROR;
     if( pConfig->pzErrmsg ){
       assert( 0==*pConfig->pzErrmsg );
-      *pConfig->pzErrmsg = sqlite3_mprintf("invalid fts5 file format "
-          "(found %d, expected %d or %d) - run 'rebuild'",
-          iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE
+      *pConfig->pzErrmsg = sqlite3_mprintf(
+          "invalid fts5 file format (found %d, expected %d) - run 'rebuild'",
+          iVersion, FTS5_CURRENT_VERSION
       );
     }
-  }else{
-    pConfig->iVersion = iVersion;
   }
 
   if( rc==SQLITE_OK ){

ext/fts5/fts5_expr.c

@@ -17,10 +17,6 @@
 #include "fts5Int.h"
 #include "fts5parse.h"
 
-#ifndef SQLITE_FTS5_MAX_EXPR_DEPTH
-# define SQLITE_FTS5_MAX_EXPR_DEPTH 256
-#endif
-
 /*
 ** All token types in the generated fts5parse.h file are greater than 0.
 */
@@ -54,28 +50,18 @@ struct Fts5Expr {
 
 /*
 ** eType:
-**   Expression node type. Usually one of:
+**   Expression node type. Always one of:
 **
 **       FTS5_AND                 (nChild, apChild valid)
 **       FTS5_OR                  (nChild, apChild valid)
 **       FTS5_NOT                 (nChild, apChild valid)
 **       FTS5_STRING              (pNear valid)
 **       FTS5_TERM                (pNear valid)
-**
-**   An expression node with eType==0 may also exist. It always matches zero
-**   rows. This is created when a phrase containing no tokens is parsed.
-**   e.g. "".
-**
-** iHeight:
-**   Distance from this node to furthest leaf. This is always 0 for nodes
-**   of type FTS5_STRING and FTS5_TERM. For all other nodes it is one 
-**   greater than the largest child value.
 */
 struct Fts5ExprNode {
   int eType;                      /* Node type */
   int bEof;                       /* True at EOF */
   int bNomatch;                   /* True if entry is not a match */
-  int iHeight;                    /* Distance to tree leaf nodes */
 
   /* Next method for this node. */
   int (*xNext)(Fts5Expr*, Fts5ExprNode*, int, i64);
@@ -104,9 +90,7 @@ struct Fts5ExprNode {
 struct Fts5ExprTerm {
   u8 bPrefix;                     /* True for a prefix term */
   u8 bFirst;                      /* True if token must be first in column */
-  char *pTerm;                    /* Term data */
-  int nQueryTerm;                 /* Effective size of term in bytes */
-  int nFullTerm;                  /* Size of term in bytes incl. tokendata */
+  char *zTerm;                    /* nul-terminated term */
   Fts5IndexIter *pIter;           /* Iterator for this term */
   Fts5ExprTerm *pSynonym;         /* Pointer to first in list of synonyms */
 };
@@ -147,31 +131,6 @@ struct Fts5Parse {
   int bPhraseToAnd;               /* Convert "a+b" to "a AND b" */
 };
 
-/*
-** Check that the Fts5ExprNode.iHeight variables are set correctly in
-** the expression tree passed as the only argument.
-*/
-#ifndef NDEBUG
-static void assert_expr_depth_ok(int rc, Fts5ExprNode *p){
-  if( rc==SQLITE_OK ){
-    if( p->eType==FTS5_TERM || p->eType==FTS5_STRING || p->eType==0 ){
-      assert( p->iHeight==0 );
-    }else{
-      int ii;
-      int iMaxChild = 0;
-      for(ii=0; ii<p->nChild; ii++){
-        Fts5ExprNode *pChild = p->apChild[ii];
-        iMaxChild = MAX(iMaxChild, pChild->iHeight);
-        assert_expr_depth_ok(SQLITE_OK, pChild);
-      }
-      assert( p->iHeight==iMaxChild+1 );
-    }
-  }
-}
-#else
-# define assert_expr_depth_ok(rc, p)
-#endif
-
 void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){
   va_list ap;
   va_start(ap, zFmt);
@@ -286,8 +245,6 @@ int sqlite3Fts5ExprNew(
   }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF );
   sqlite3Fts5ParserFree(pEngine, fts5ParseFree);
 
-  assert_expr_depth_ok(sParse.rc, sParse.pExpr);
-
   /* If the LHS of the MATCH expression was a user column, apply the
   ** implicit column-filter.  */
   if( iCol<pConfig->nCol && sParse.pExpr && sParse.rc==SQLITE_OK ){
@@ -328,11 +285,7 @@ int sqlite3Fts5ExprNew(
   }
 
   sqlite3_free(sParse.apPhrase);
-  if( 0==*pzErr ){
-    *pzErr = sParse.zErr;
-  }else{
-    sqlite3_free(sParse.zErr);
-  }
+  *pzErr = sParse.zErr;
   return sParse.rc;
 }
 
@@ -454,7 +407,7 @@ int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){
   Fts5Parse sParse;
   memset(&sParse, 0, sizeof(sParse));
 
-  if( *pp1 && p2 ){
+  if( *pp1 ){
     Fts5Expr *p1 = *pp1;
     int nPhrase = p1->nPhrase + p2->nPhrase;
 
@@ -479,7 +432,7 @@ int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){
     }
     sqlite3_free(p2->apExprPhrase);
     sqlite3_free(p2);
-  }else if( p2 ){
+  }else{
     *pp1 = p2;
   }
 
@@ -977,7 +930,7 @@ static int fts5ExprNearInitAll(
             p->pIter = 0;
           }
           rc = sqlite3Fts5IndexQuery(
-              pExpr->pIndex, p->pTerm, p->nQueryTerm,
+              pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm),
               (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) |
               (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0),
               pNear->pColset,
@@ -1614,7 +1567,7 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){
       Fts5ExprTerm *pSyn;
       Fts5ExprTerm *pNext;
       Fts5ExprTerm *pTerm = &pPhrase->aTerm[i];
-      sqlite3_free(pTerm->pTerm);
+      sqlite3_free(pTerm->zTerm);
       sqlite3Fts5IterClose(pTerm->pIter);
       for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){
         pNext = pSyn->pSynonym;
@@ -1712,7 +1665,6 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset(
 typedef struct TokenCtx TokenCtx;
 struct TokenCtx {
   Fts5ExprPhrase *pPhrase;
-  Fts5Config *pConfig;
   int rc;
 };
 
@@ -1746,12 +1698,8 @@ static int fts5ParseTokenize(
       rc = SQLITE_NOMEM;
     }else{
       memset(pSyn, 0, (size_t)nByte);
-      pSyn->pTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer);
-      pSyn->nFullTerm = pSyn->nQueryTerm = nToken;
-      if( pCtx->pConfig->bTokendata ){
-        pSyn->nQueryTerm = (int)strlen(pSyn->pTerm);
-      }
-      memcpy(pSyn->pTerm, pToken, nToken);
+      pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer);
+      memcpy(pSyn->zTerm, pToken, nToken);
       pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym;
       pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn;
     }
@@ -1776,11 +1724,7 @@ static int fts5ParseTokenize(
     if( rc==SQLITE_OK ){
       pTerm = &pPhrase->aTerm[pPhrase->nTerm++];
       memset(pTerm, 0, sizeof(Fts5ExprTerm));
-      pTerm->pTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
-      pTerm->nFullTerm = pTerm->nQueryTerm = nToken;
-      if( pCtx->pConfig->bTokendata && rc==SQLITE_OK ){ 
-        pTerm->nQueryTerm = (int)strlen(pTerm->pTerm);
-      }
+      pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken);
     }
   }
 
@@ -1847,7 +1791,6 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
 
   memset(&sCtx, 0, sizeof(TokenCtx));
   sCtx.pPhrase = pAppend;
-  sCtx.pConfig = pConfig;
 
   rc = fts5ParseStringFromToken(pToken, &z);
   if( rc==SQLITE_OK ){
@@ -1879,7 +1822,6 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
     }else if( sCtx.pPhrase->nTerm ){
       sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix;
     }
-    assert( pParse->apPhrase!=0 );
     pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
   }
 
@@ -1896,15 +1838,12 @@ int sqlite3Fts5ExprClonePhrase(
   Fts5Expr **ppNew
 ){
   int rc = SQLITE_OK;             /* Return code */
-  Fts5ExprPhrase *pOrig = 0;      /* The phrase extracted from pExpr */
+  Fts5ExprPhrase *pOrig;          /* The phrase extracted from pExpr */
   Fts5Expr *pNew = 0;             /* Expression to return via *ppNew */
-  TokenCtx sCtx = {0,0,0};        /* Context object for fts5ParseTokenize */
-  if( !pExpr || iPhrase<0 || iPhrase>=pExpr->nPhrase ){
-    rc = SQLITE_RANGE;
-  }else{
-    pOrig = pExpr->apExprPhrase[iPhrase];
-    pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr));
-  }
+  TokenCtx sCtx = {0,0};          /* Context object for fts5ParseTokenize */
+
+  pOrig = pExpr->apExprPhrase[iPhrase];
+  pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr));
   if( rc==SQLITE_OK ){
     pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, 
         sizeof(Fts5ExprPhrase*));
@@ -1917,7 +1856,7 @@ int sqlite3Fts5ExprClonePhrase(
     pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, 
         sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*));
   }
-  if( rc==SQLITE_OK && ALWAYS(pOrig!=0) ){
+  if( rc==SQLITE_OK ){
     Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset;
     if( pColsetOrig ){
       sqlite3_int64 nByte;
@@ -1931,27 +1870,26 @@ int sqlite3Fts5ExprClonePhrase(
     }
   }
 
-  if( rc==SQLITE_OK ){
-    if( pOrig->nTerm ){
-      int i;                          /* Used to iterate through phrase terms */
-      sCtx.pConfig = pExpr->pConfig;
-      for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
-        int tflags = 0;
-        Fts5ExprTerm *p;
-        for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){
-          rc = fts5ParseTokenize((void*)&sCtx,tflags,p->pTerm,p->nFullTerm,0,0);
-          tflags = FTS5_TOKEN_COLOCATED;
-        }
-        if( rc==SQLITE_OK ){
-          sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
-          sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst;
-        }
+  if( pOrig->nTerm ){
+    int i;                          /* Used to iterate through phrase terms */
+    for(i=0; rc==SQLITE_OK && i<pOrig->nTerm; i++){
+      int tflags = 0;
+      Fts5ExprTerm *p;
+      for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){
+        const char *zTerm = p->zTerm;
+        rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm),
+            0, 0);
+        tflags = FTS5_TOKEN_COLOCATED;
+      }
+      if( rc==SQLITE_OK ){
+        sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix;
+        sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst;
       }
-    }else{
-      /* This happens when parsing a token or quoted phrase that contains
-      ** no token characters at all. (e.g ... MATCH '""'). */
-      sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase));
     }
+  }else{
+    /* This happens when parsing a token or quoted phrase that contains
+    ** no token characters at all. (e.g ... MATCH '""'). */
+    sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase));
   }
 
   if( rc==SQLITE_OK && ALWAYS(sCtx.pPhrase) ){
@@ -2267,11 +2205,7 @@ static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
   }
 }
 
-/*
-** Add pSub as a child of p.
-*/
 static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){
-  int ii = p->nChild;
   if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){
     int nByte = sizeof(Fts5ExprNode*) * pSub->nChild;
     memcpy(&p->apChild[p->nChild], pSub->apChild, nByte);
@@ -2280,9 +2214,6 @@ static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){
   }else{
     p->apChild[p->nChild++] = pSub;
   }
-  for( ; ii<p->nChild; ii++){
-    p->iHeight = MAX(p->iHeight, p->apChild[ii]->iHeight + 1);
-  }
 }
 
 /*
@@ -2313,7 +2244,6 @@ static Fts5ExprNode *fts5ParsePhraseToAnd(
   if( pRet ){
     pRet->eType = FTS5_AND;
     pRet->nChild = nTerm;
-    pRet->iHeight = 1;
     fts5ExprAssignXNext(pRet);
     pParse->nPhrase--;
     for(ii=0; ii<nTerm; ii++){
@@ -2324,13 +2254,11 @@ static Fts5ExprNode *fts5ParsePhraseToAnd(
         if( parseGrowPhraseArray(pParse) ){
           fts5ExprPhraseFree(pPhrase);
         }else{
-          Fts5ExprTerm *p = &pNear->apPhrase[0]->aTerm[ii];
-          Fts5ExprTerm *pTo = &pPhrase->aTerm[0];
           pParse->apPhrase[pParse->nPhrase++] = pPhrase;
           pPhrase->nTerm = 1;
-          pTo->pTerm = sqlite3Fts5Strndup(&pParse->rc, p->pTerm, p->nFullTerm);
-          pTo->nQueryTerm = p->nQueryTerm;
-          pTo->nFullTerm = p->nFullTerm;
+          pPhrase->aTerm[0].zTerm = sqlite3Fts5Strndup(
+              &pParse->rc, pNear->apPhrase[0]->aTerm[ii].zTerm, -1
+          );
           pRet->apChild[ii] = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 
               0, 0, sqlite3Fts5ParseNearset(pParse, 0, pPhrase)
           );
@@ -2414,25 +2342,13 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
                   "fts5: %s queries are not supported (detail!=full)", 
                   pNear->nPhrase==1 ? "phrase": "NEAR"
               );
-              sqlite3Fts5ParseNodeFree(pRet);
+              sqlite3_free(pRet);
               pRet = 0;
-              pNear = 0;
-              assert( pLeft==0 && pRight==0 );
             }
           }
         }else{
-          assert( pNear==0 );
           fts5ExprAddChildren(pRet, pLeft);
           fts5ExprAddChildren(pRet, pRight);
-          pLeft = pRight = 0;
-          if( pRet->iHeight>SQLITE_FTS5_MAX_EXPR_DEPTH ){
-            sqlite3Fts5ParseError(pParse, 
-                "fts5 expression tree is too large (maximum depth %d)", 
-                SQLITE_FTS5_MAX_EXPR_DEPTH
-            );
-            sqlite3Fts5ParseNodeFree(pRet);
-            pRet = 0;
-          }
         }
       }
     }
@@ -2468,7 +2384,6 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
     assert( pRight->eType==FTS5_STRING 
         || pRight->eType==FTS5_TERM 
         || pRight->eType==FTS5_EOF 
-        || (pRight->eType==FTS5_AND && pParse->bPhraseToAnd) 
     );
 
     if( pLeft->eType==FTS5_AND ){
@@ -2482,8 +2397,6 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
         );
 
     if( pRight->eType==FTS5_EOF ){
-      assert( pParse->apPhrase!=0 );
-      assert( pParse->nPhrase>0 );
       assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] );
       sqlite3Fts5ParseNodeFree(pRight);
       pRet = pLeft;
@@ -2514,7 +2427,7 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
   return pRet;
 }
 
-#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
+#ifdef SQLITE_TEST
 static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
   sqlite3_int64 nByte = 0;
   Fts5ExprTerm *p;
@@ -2522,17 +2435,16 @@ static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){
 
   /* Determine the maximum amount of space required. */
   for(p=pTerm; p; p=p->pSynonym){
-    nByte += pTerm->nQueryTerm * 2 + 3 + 2;
+    nByte += (int)strlen(pTerm->zTerm) * 2 + 3 + 2;
   }
   zQuoted = sqlite3_malloc64(nByte);
 
   if( zQuoted ){
     int i = 0;
     for(p=pTerm; p; p=p->pSynonym){
-      char *zIn = p->pTerm;
-      char *zEnd = &zIn[p->nQueryTerm];
+      char *zIn = p->zTerm;
       zQuoted[i++] = '"';
-      while( zIn<zEnd ){
+      while( *zIn ){
         if( *zIn=='"' ) zQuoted[i++] = '"';
         zQuoted[i++] = *zIn++;
       }
@@ -2610,10 +2522,8 @@ static char *fts5ExprPrintTcl(
 
       zRet = fts5PrintfAppend(zRet, " {");
       for(iTerm=0; zRet && iTerm<pPhrase->nTerm; iTerm++){
-        Fts5ExprTerm *p = &pPhrase->aTerm[iTerm];
-        zRet = fts5PrintfAppend(zRet, "%s%.*s", iTerm==0?"":" ", 
-            p->nQueryTerm, p->pTerm
-        );
+        char *zTerm = pPhrase->aTerm[iTerm].zTerm;
+        zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm);
         if( pPhrase->aTerm[iTerm].bPrefix ){
           zRet = fts5PrintfAppend(zRet, "*");
         }
@@ -2623,8 +2533,6 @@ static char *fts5ExprPrintTcl(
       if( zRet==0 ) return 0;
     }
 
-  }else if( pExpr->eType==0 ){
-    zRet = sqlite3_mprintf("{}");
   }else{
     char const *zOp = 0;
     int i;
@@ -2886,14 +2794,14 @@ static void fts5ExprFold(
     sqlite3_result_int(pCtx, sqlite3Fts5UnicodeFold(iCode, bRemoveDiacritics));
   }
 }
-#endif /* if SQLITE_TEST || SQLITE_FTS5_DEBUG */
+#endif /* ifdef SQLITE_TEST */
 
 /*
 ** This is called during initialization to register the fts5_expr() scalar
 ** UDF with the SQLite handle passed as the only argument.
 */
 int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){
-#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG)
+#ifdef SQLITE_TEST
   struct Fts5ExprFunc {
     const char *z;
     void (*x)(sqlite3_context*,int,sqlite3_value**);
@@ -3014,17 +2922,6 @@ static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){
   return 0;
 }
 
-/*
-** pToken is a buffer nToken bytes in size that may or may not contain
-** an embedded 0x00 byte. If it does, return the number of bytes in
-** the buffer before the 0x00. If it does not, return nToken.
-*/
-static int fts5QueryTerm(const char *pToken, int nToken){
-  int ii;
-  for(ii=0; ii<nToken && pToken[ii]; ii++){}
-  return ii;
-}
-
 static int fts5ExprPopulatePoslistsCb(
   void *pCtx,                /* Copy of 2nd argument to xTokenize() */
   int tflags,                /* Mask of FTS5_TOKEN_* flags */
@@ -3036,33 +2933,22 @@ static int fts5ExprPopulatePoslistsCb(
   Fts5ExprCtx *p = (Fts5ExprCtx*)pCtx;
   Fts5Expr *pExpr = p->pExpr;
   int i;
-  int nQuery = nToken;
-  i64 iRowid = pExpr->pRoot->iRowid;
 
   UNUSED_PARAM2(iUnused1, iUnused2);
 
-  if( nQuery>FTS5_MAX_TOKEN_SIZE ) nQuery = FTS5_MAX_TOKEN_SIZE;
-  if( pExpr->pConfig->bTokendata ){
-    nQuery = fts5QueryTerm(pToken, nQuery);
-  }
+  if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE;
   if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++;
   for(i=0; i<pExpr->nPhrase; i++){
-    Fts5ExprTerm *pT;
+    Fts5ExprTerm *pTerm;
     if( p->aPopulator[i].bOk==0 ) continue;
-    for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){
-      if( (pT->nQueryTerm==nQuery || (pT->nQueryTerm<nQuery && pT->bPrefix))
-       && memcmp(pT->pTerm, pToken, pT->nQueryTerm)==0
+    for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){
+      int nTerm = (int)strlen(pTerm->zTerm);
+      if( (nTerm==nToken || (nTerm<nToken && pTerm->bPrefix))
+       && memcmp(pTerm->zTerm, pToken, nTerm)==0
       ){
         int rc = sqlite3Fts5PoslistWriterAppend(
             &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff
         );
-        if( rc==SQLITE_OK && pExpr->pConfig->bTokendata && !pT->bPrefix ){
-          int iCol = p->iOff>>32;
-          int iTokOff = p->iOff & 0x7FFFFFFF;
-          rc = sqlite3Fts5IndexIterWriteTokendata(
-              pT->pIter, pToken, nToken, iRowid, iCol, iTokOff
-          );
-        }
         if( rc ) return rc;
         break;
       }
@@ -3116,7 +3002,6 @@ static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){
   pNode->iRowid = iRowid;
   pNode->bEof = 0;
   switch( pNode->eType ){
-    case 0:
     case FTS5_TERM:
     case FTS5_STRING:
       return (pNode->pNear->apPhrase[0]->poslist.n>0);
@@ -3198,80 +3083,3 @@ int sqlite3Fts5ExprPhraseCollist(
 
   return rc;
 }
-
-/*
-** Does the work of the fts5_api.xQueryToken() API method.
-*/
-int sqlite3Fts5ExprQueryToken(
-  Fts5Expr *pExpr, 
-  int iPhrase, 
-  int iToken, 
-  const char **ppOut, 
-  int *pnOut
-){
-  Fts5ExprPhrase *pPhrase = 0;
-
-  if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
-    return SQLITE_RANGE;
-  }
-  pPhrase = pExpr->apExprPhrase[iPhrase];
-  if( iToken<0 || iToken>=pPhrase->nTerm ){
-    return SQLITE_RANGE;
-  }
-
-  *ppOut = pPhrase->aTerm[iToken].pTerm;
-  *pnOut = pPhrase->aTerm[iToken].nFullTerm;
-  return SQLITE_OK;
-}
-
-/*
-** Does the work of the fts5_api.xInstToken() API method.
-*/
-int sqlite3Fts5ExprInstToken(
-  Fts5Expr *pExpr, 
-  i64 iRowid,
-  int iPhrase, 
-  int iCol, 
-  int iOff, 
-  int iToken, 
-  const char **ppOut, 
-  int *pnOut
-){
-  Fts5ExprPhrase *pPhrase = 0;
-  Fts5ExprTerm *pTerm = 0;
-  int rc = SQLITE_OK;
-
-  if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
-    return SQLITE_RANGE;
-  }
-  pPhrase = pExpr->apExprPhrase[iPhrase];
-  if( iToken<0 || iToken>=pPhrase->nTerm ){
-    return SQLITE_RANGE;
-  }
-  pTerm = &pPhrase->aTerm[iToken];
-  if( pTerm->bPrefix==0 ){
-    if( pExpr->pConfig->bTokendata ){
-      rc = sqlite3Fts5IterToken(
-          pTerm->pIter, iRowid, iCol, iOff+iToken, ppOut, pnOut
-      );
-    }else{
-      *ppOut = pTerm->pTerm;
-      *pnOut = pTerm->nFullTerm;
-    }
-  }
-  return rc;
-}
-
-/*
-** Clear the token mappings for all Fts5IndexIter objects mannaged by 
-** the expression passed as the only argument.
-*/
-void sqlite3Fts5ExprClearTokens(Fts5Expr *pExpr){
-  int ii;
-  for(ii=0; ii<pExpr->nPhrase; ii++){
-    Fts5ExprTerm *pT;
-    for(pT=&pExpr->apExprPhrase[ii]->aTerm[0]; pT; pT=pT->pSynonym){
-      sqlite3Fts5IndexIterClearTokendata(pT->pIter);
-    }
-  }
-}

ext/fts5/fts5_hash.c

@@ -36,15 +36,10 @@ struct Fts5Hash {
 
 /*
 ** Each entry in the hash table is represented by an object of the 
-** following type. Each object, its key, and its current data are stored 
-** in a single memory allocation. The key immediately follows the object 
-** in memory. The position list data immediately follows the key data 
-** in memory.
-**
-** The key is Fts5HashEntry.nKey bytes in size. It consists of a single
-** byte identifying the index (either the main term index or a prefix-index),
-** followed by the term data. For example: "0token". There is no 
-** nul-terminator - in this case nKey=6.
+** following type. Each object, its key (a nul-terminated string) and 
+** its current data are stored in a single memory allocation. The 
+** key immediately follows the object in memory. The position list
+** data immediately follows the key data in memory.
 **
 ** The data that follows the key is in a similar, but not identical format
 ** to the doclist data stored in the database. It is:
@@ -179,7 +174,8 @@ static int fts5HashResize(Fts5Hash *pHash){
       unsigned int iHash;
       Fts5HashEntry *p = apOld[i];
       apOld[i] = p->pHashNext;
-      iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p), p->nKey);
+      iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p),
+                          (int)strlen(fts5EntryKey(p)));
       p->pHashNext = apNew[iHash];
       apNew[iHash] = p;
     }
@@ -263,7 +259,7 @@ int sqlite3Fts5HashWrite(
   for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
     char *zKey = fts5EntryKey(p);
     if( zKey[0]==bByte 
-     && p->nKey==nToken+1
+     && p->nKey==nToken
      && memcmp(&zKey[1], pToken, nToken)==0 
     ){
       break;
@@ -293,9 +289,9 @@ int sqlite3Fts5HashWrite(
     zKey[0] = bByte;
     memcpy(&zKey[1], pToken, nToken);
     assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) );
-    p->nKey = nToken+1;
+    p->nKey = nToken;
     zKey[nToken+1] = '\0';
-    p->nData = nToken+1 + sizeof(Fts5HashEntry);
+    p->nData = nToken+1 + 1 + sizeof(Fts5HashEntry);
     p->pHashNext = pHash->aSlot[iHash];
     pHash->aSlot[iHash] = p;
     pHash->nEntry++;
@@ -412,17 +408,12 @@ static Fts5HashEntry *fts5HashEntryMerge(
       *ppOut = p1;
       p1 = 0;
     }else{
+      int i = 0;
       char *zKey1 = fts5EntryKey(p1);
       char *zKey2 = fts5EntryKey(p2);
-      int nMin = MIN(p1->nKey, p2->nKey);
+      while( zKey1[i]==zKey2[i] ) i++;
 
-      int cmp = memcmp(zKey1, zKey2, nMin);
-      if( cmp==0 ){
-        cmp = p1->nKey - p2->nKey;
-      }
-      assert( cmp!=0 );
-
-      if( cmp>0 ){
+      if( ((u8)zKey1[i])>((u8)zKey2[i]) ){
         /* p2 is smaller */
         *ppOut = p2;
         ppOut = &p2->pScanNext;
@@ -441,8 +432,10 @@ static Fts5HashEntry *fts5HashEntryMerge(
 }
 
 /*
-** Link all tokens from hash table iHash into a list in sorted order. The
-** tokens are not removed from the hash table.
+** Extract all tokens from hash table iHash and link them into a list
+** in sorted order. The hash table is cleared before returning. It is
+** the responsibility of the caller to free the elements of the returned
+** list.
 */
 static int fts5HashEntrySort(
   Fts5Hash *pHash, 
@@ -464,7 +457,7 @@ static int fts5HashEntrySort(
     Fts5HashEntry *pIter;
     for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){
       if( pTerm==0 
-       || (pIter->nKey>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm))
+       || (pIter->nKey+1>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm))
       ){
         Fts5HashEntry *pEntry = pIter;
         pEntry->pScanNext = 0;
@@ -482,6 +475,7 @@ static int fts5HashEntrySort(
     pList = fts5HashEntryMerge(pList, ap[i]);
   }
 
+  pHash->nEntry = 0;
   sqlite3_free(ap);
   *ppSorted = pList;
   return SQLITE_OK;
@@ -503,11 +497,12 @@ int sqlite3Fts5HashQuery(
 
   for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){
     zKey = fts5EntryKey(p);
-    if( nTerm==p->nKey && memcmp(zKey, pTerm, nTerm)==0 ) break;
+    assert( p->nKey+1==(int)strlen(zKey) );
+    if( nTerm==p->nKey+1 && memcmp(zKey, pTerm, nTerm)==0 ) break;
   }
 
   if( p ){
-    int nHashPre = sizeof(Fts5HashEntry) + nTerm;
+    int nHashPre = sizeof(Fts5HashEntry) + nTerm + 1;
     int nList = p->nData - nHashPre;
     u8 *pRet = (u8*)(*ppOut = sqlite3_malloc64(nPre + nList + 10));
     if( pRet ){
@@ -534,28 +529,6 @@ int sqlite3Fts5HashScanInit(
   return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan);
 }
 
-#ifdef SQLITE_DEBUG
-static int fts5HashCount(Fts5Hash *pHash){
-  int nEntry = 0;
-  int ii;
-  for(ii=0; ii<pHash->nSlot; ii++){
-    Fts5HashEntry *p = 0;
-    for(p=pHash->aSlot[ii]; p; p=p->pHashNext){
-      nEntry++;
-    }
-  }
-  return nEntry;
-}
-#endif
-
-/*
-** Return true if the hash table is empty, false otherwise.
-*/
-int sqlite3Fts5HashIsEmpty(Fts5Hash *pHash){
-  assert( pHash->nEntry==fts5HashCount(pHash) );
-  return pHash->nEntry==0;
-}
-
 void sqlite3Fts5HashScanNext(Fts5Hash *p){
   assert( !sqlite3Fts5HashScanEof(p) );
   p->pScan = p->pScan->pScanNext;
@@ -568,22 +541,19 @@ int sqlite3Fts5HashScanEof(Fts5Hash *p){
 void sqlite3Fts5HashScanEntry(
   Fts5Hash *pHash,
   const char **pzTerm,            /* OUT: term (nul-terminated) */
-  int *pnTerm,                    /* OUT: Size of term in bytes */
   const u8 **ppDoclist,           /* OUT: pointer to doclist */
   int *pnDoclist                  /* OUT: size of doclist in bytes */
 ){
   Fts5HashEntry *p;
   if( (p = pHash->pScan) ){
     char *zKey = fts5EntryKey(p);
-    int nTerm = p->nKey;
+    int nTerm = (int)strlen(zKey);
     fts5HashAddPoslistSize(pHash, p, 0);
     *pzTerm = zKey;
-    *pnTerm = nTerm;
-    *ppDoclist = (const u8*)&zKey[nTerm];
-    *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm);
+    *ppDoclist = (const u8*)&zKey[nTerm+1];
+    *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm + 1);
   }else{
     *pzTerm = 0;
-    *pnTerm = 0;
     *ppDoclist = 0;
     *pnDoclist = 0;
   }

ext/fts5/fts5_main.c

@@ -117,8 +117,6 @@ struct Fts5FullTable {
   Fts5Storage *pStorage;          /* Document store */
   Fts5Global *pGlobal;            /* Global (connection wide) data */
   Fts5Cursor *pSortCsr;           /* Sort data from this cursor */
-  int iSavepoint;                 /* Successful xSavepoint()+1 */
-  
 #ifdef SQLITE_DEBUG
   struct Fts5TransactionState ts;
 #endif
@@ -377,12 +375,8 @@ static int fts5InitVtab(
     assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 );
   }
   if( rc==SQLITE_OK ){
-    pConfig->pzErrmsg = pzErr;
     pTab->p.pConfig = pConfig;
     pTab->pGlobal = pGlobal;
-    if( bCreate || sqlite3Fts5TokenizerPreload(&pConfig->t) ){
-      rc = sqlite3Fts5LoadTokenizer(pConfig);
-    }
   }
 
   /* Open the index sub-system */
@@ -404,18 +398,13 @@ static int fts5InitVtab(
 
   /* Load the initial configuration */
   if( rc==SQLITE_OK ){
+    assert( pConfig->pzErrmsg==0 );
+    pConfig->pzErrmsg = pzErr;
     rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
     sqlite3Fts5IndexRollback(pTab->p.pIndex);
+    pConfig->pzErrmsg = 0;
   }
 
-  if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){
-    rc = sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, (int)1);
-  }
-  if( rc==SQLITE_OK ){
-    rc = sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS);
-  }
-
-  if( pConfig ) pConfig->pzErrmsg = 0;
   if( rc!=SQLITE_OK ){
     fts5FreeVtab(pTab);
     pTab = 0;
@@ -483,10 +472,10 @@ static int fts5UsePatternMatch(
 ){
   assert( FTS5_PATTERN_GLOB==SQLITE_INDEX_CONSTRAINT_GLOB );
   assert( FTS5_PATTERN_LIKE==SQLITE_INDEX_CONSTRAINT_LIKE );
-  if( pConfig->t.ePattern==FTS5_PATTERN_GLOB && p->op==FTS5_PATTERN_GLOB ){
+  if( pConfig->ePattern==FTS5_PATTERN_GLOB && p->op==FTS5_PATTERN_GLOB ){
     return 1;
   }
-  if( pConfig->t.ePattern==FTS5_PATTERN_LIKE 
+  if( pConfig->ePattern==FTS5_PATTERN_LIKE 
    && (p->op==FTS5_PATTERN_LIKE || p->op==FTS5_PATTERN_GLOB)
   ){
     return 1;
@@ -533,11 +522,11 @@ static int fts5UsePatternMatch(
 ** This function ensures that there is at most one "r" or "=". And that if
 ** there exists an "=" then there is no "<" or ">".
 **
-** If an unusable MATCH operator is present in the WHERE clause, then
-** SQLITE_CONSTRAINT is returned.
-**
 ** Costs are assigned as follows:
 **
+**  a) If an unusable MATCH operator is present in the WHERE clause, the
+**     cost is unconditionally set to 1e50 (a really big number).
+**
 **  a) If a MATCH operator is present, the cost depends on the other
 **     constraints also present. As follows:
 **
@@ -569,7 +558,7 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
   int bSeenEq = 0;
   int bSeenGt = 0;
   int bSeenLt = 0;
-  int nSeenMatch = 0;
+  int bSeenMatch = 0;
   int bSeenRank = 0;
 
 
@@ -600,15 +589,18 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
       /* A MATCH operator or equivalent */
       if( p->usable==0 || iCol<0 ){
         /* As there exists an unusable MATCH constraint this is an 
-        ** unusable plan. Return SQLITE_CONSTRAINT. */
-        return SQLITE_CONSTRAINT;
+        ** unusable plan. Set a prohibitively high cost. */
+        pInfo->estimatedCost = 1e50;
+        assert( iIdxStr < pInfo->nConstraint*6 + 1 );
+        idxStr[iIdxStr] = 0;
+        return SQLITE_OK;
       }else{
         if( iCol==nCol+1 ){
           if( bSeenRank ) continue;
           idxStr[iIdxStr++] = 'r';
           bSeenRank = 1;
         }else if( iCol>=0 ){
-          nSeenMatch++;
+          bSeenMatch = 1;
           idxStr[iIdxStr++] = 'M';
           sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol);
           idxStr += strlen(&idxStr[iIdxStr]);
@@ -625,7 +617,6 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
         idxStr += strlen(&idxStr[iIdxStr]);
         pInfo->aConstraintUsage[i].argvIndex = ++iCons;
         assert( idxStr[iIdxStr]=='\0' );
-        nSeenMatch++;
       }else if( bSeenEq==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol<0 ){
         idxStr[iIdxStr++] = '=';
         bSeenEq = 1;
@@ -656,15 +647,12 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
   }
   idxStr[iIdxStr] = '\0';
 
-  /* Set idxFlags flags for the ORDER BY clause
-  **
-  ** Note that tokendata=1 tables cannot currently handle "ORDER BY rowid DESC".
-  */
+  /* Set idxFlags flags for the ORDER BY clause */
   if( pInfo->nOrderBy==1 ){
     int iSort = pInfo->aOrderBy[0].iColumn;
-    if( iSort==(pConfig->nCol+1) && nSeenMatch>0 ){
+    if( iSort==(pConfig->nCol+1) && bSeenMatch ){
       idxFlags |= FTS5_BI_ORDER_RANK;
-    }else if( iSort==-1 && (!pInfo->aOrderBy[0].desc || !pConfig->bTokendata) ){
+    }else if( iSort==-1 ){
       idxFlags |= FTS5_BI_ORDER_ROWID;
     }
     if( BitFlagTest(idxFlags, FTS5_BI_ORDER_RANK|FTS5_BI_ORDER_ROWID) ){
@@ -677,17 +665,14 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){
 
   /* Calculate the estimated cost based on the flags set in idxFlags. */
   if( bSeenEq ){
-    pInfo->estimatedCost = nSeenMatch ? 1000.0 : 10.0;
-    if( nSeenMatch==0 ) fts5SetUniqueFlag(pInfo);
+    pInfo->estimatedCost = bSeenMatch ? 100.0 : 10.0;
+    if( bSeenMatch==0 ) fts5SetUniqueFlag(pInfo);
   }else if( bSeenLt && bSeenGt ){
-    pInfo->estimatedCost = nSeenMatch ? 5000.0 : 250000.0;
+    pInfo->estimatedCost = bSeenMatch ? 500.0 : 250000.0;
   }else if( bSeenLt || bSeenGt ){
-    pInfo->estimatedCost = nSeenMatch ? 7500.0 : 750000.0;
+    pInfo->estimatedCost = bSeenMatch ? 750.0 : 750000.0;
   }else{
-    pInfo->estimatedCost = nSeenMatch ? 10000.0 : 1000000.0;
-  }
-  for(i=1; i<nSeenMatch; i++){
-    pInfo->estimatedCost *= 0.4;
+    pInfo->estimatedCost = bSeenMatch ? 1000.0 : 1000000.0;
   }
 
   pInfo->idxNum = idxFlags;
@@ -919,16 +904,6 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
   );
   assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) );
 
-  /* If this cursor uses FTS5_PLAN_MATCH and this is a tokendata=1 table,
-  ** clear any token mappings accumulated at the fts5_index.c level. In
-  ** other cases, specifically FTS5_PLAN_SOURCE and FTS5_PLAN_SORTED_MATCH,
-  ** we need to retain the mappings for the entire query.  */
-  if( pCsr->ePlan==FTS5_PLAN_MATCH 
-   && ((Fts5Table*)pCursor->pVtab)->pConfig->bTokendata 
-  ){
-    sqlite3Fts5ExprClearTokens(pCsr->pExpr);
-  }
-
   if( pCsr->ePlan<3 ){
     int bSkip = 0;
     if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc;
@@ -963,7 +938,6 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){
           }
         }else{
           rc = SQLITE_OK;
-          CsrFlagSet(pCsr, FTS5CSR_REQUIRE_DOCSIZE);
         }
         break;
       }
@@ -1217,18 +1191,6 @@ static i64 fts5GetRowidLimit(sqlite3_value *pVal, i64 iDefault){
   return iDefault;
 }
 
-/*
-** Set the error message on the virtual table passed as the first argument.
-*/
-static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){
-  va_list ap;                     /* ... printf arguments */
-  va_start(ap, zFormat);
-  sqlite3_free(p->p.base.zErrMsg);
-  p->p.base.zErrMsg = sqlite3_vmprintf(zFormat, ap);
-  va_end(ap);
-}
-
-
 /*
 ** This is the xFilter interface for the virtual table.  See
 ** the virtual table xFilter method documentation for additional
@@ -1367,9 +1329,6 @@ static int fts5FilterMethod(
     pCsr->iFirstRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64);
   }
 
-  rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
-  if( rc!=SQLITE_OK ) goto filter_out;
-
   if( pTab->pSortCsr ){
     /* If pSortCsr is non-NULL, then this call is being made as part of 
     ** processing for a "... MATCH <expr> ORDER BY rank" query (ePlan is
@@ -1392,7 +1351,6 @@ static int fts5FilterMethod(
     pCsr->pExpr = pTab->pSortCsr->pExpr;
     rc = fts5CursorFirst(pTab, pCsr, bDesc);
   }else if( pCsr->pExpr ){
-    assert( rc==SQLITE_OK );
     rc = fts5CursorParseRank(pConfig, pCsr, pRank);
     if( rc==SQLITE_OK ){
       if( bOrderByRank ){
@@ -1404,7 +1362,9 @@ static int fts5FilterMethod(
       }
     }
   }else if( pConfig->zContent==0 ){
-    fts5SetVtabError(pTab,"%s: table does not support scanning",pConfig->zName);
+    *pConfig->pzErrmsg = sqlite3_mprintf(
+        "%s: table does not support scanning", pConfig->zName
+    );
     rc = SQLITE_ERROR;
   }else{
     /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup
@@ -1447,13 +1407,9 @@ static i64 fts5CursorRowid(Fts5Cursor *pCsr){
   assert( pCsr->ePlan==FTS5_PLAN_MATCH 
        || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH 
        || pCsr->ePlan==FTS5_PLAN_SOURCE 
-       || pCsr->ePlan==FTS5_PLAN_SCAN 
-       || pCsr->ePlan==FTS5_PLAN_ROWID 
   );
   if( pCsr->pSorter ){
     return pCsr->pSorter->iRowid;
-  }else if( pCsr->ePlan>=FTS5_PLAN_SCAN ){
-    return sqlite3_column_int64(pCsr->pStmt, 0);
   }else{
     return sqlite3Fts5ExprRowid(pCsr->pExpr);
   }
@@ -1470,16 +1426,25 @@ static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
   int ePlan = pCsr->ePlan;
   
   assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 );
-  if( ePlan==FTS5_PLAN_SPECIAL ){
-    *pRowid = 0;
-  }else{
-    *pRowid = fts5CursorRowid(pCsr);
+  switch( ePlan ){
+    case FTS5_PLAN_SPECIAL:
+      *pRowid = 0;
+      break;
+
+    case FTS5_PLAN_SOURCE:
+    case FTS5_PLAN_MATCH:
+    case FTS5_PLAN_SORTED_MATCH:
+      *pRowid = fts5CursorRowid(pCsr);
+      break;
+
+    default:
+      *pRowid = sqlite3_column_int64(pCsr->pStmt, 0);
+      break;
   }
 
   return SQLITE_OK;
 }
 
-
 /*
 ** If the cursor requires seeking (bSeekRequired flag is set), seek it.
 ** Return SQLITE_OK if no error occurs, or an SQLite error code otherwise.
@@ -1516,13 +1481,8 @@ static int fts5SeekCursor(Fts5Cursor *pCsr, int bErrormsg){
       rc = sqlite3_reset(pCsr->pStmt);
       if( rc==SQLITE_OK ){
         rc = FTS5_CORRUPT;
-        fts5SetVtabError((Fts5FullTable*)pTab, 
-            "fts5: missing row %lld from content table %s",
-            fts5CursorRowid(pCsr), 
-            pTab->pConfig->zContent
-        );
       }else if( pTab->pConfig->pzErrmsg ){
-        fts5SetVtabError((Fts5FullTable*)pTab, 
+        *pTab->pConfig->pzErrmsg = sqlite3_mprintf(
             "%s", sqlite3_errmsg(pTab->pConfig->db)
         );
       }
@@ -1531,6 +1491,14 @@ static int fts5SeekCursor(Fts5Cursor *pCsr, int bErrormsg){
   return rc;
 }
 
+static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){
+  va_list ap;                     /* ... printf arguments */
+  va_start(ap, zFormat);
+  assert( p->p.base.zErrMsg==0 );
+  p->p.base.zErrMsg = sqlite3_vmprintf(zFormat, ap);
+  va_end(ap);
+}
+
 /*
 ** This function is called to handle an FTS INSERT command. In other words,
 ** an INSERT statement of the form:
@@ -1554,7 +1522,6 @@ static int fts5SpecialInsert(
   Fts5Config *pConfig = pTab->p.pConfig;
   int rc = SQLITE_OK;
   int bError = 0;
-  int bLoadConfig = 0;
 
   if( 0==sqlite3_stricmp("delete-all", zCmd) ){
     if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
@@ -1566,7 +1533,6 @@ static int fts5SpecialInsert(
     }else{
       rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage);
     }
-    bLoadConfig = 1;
   }else if( 0==sqlite3_stricmp("rebuild", zCmd) ){
     if( pConfig->eContent==FTS5_CONTENT_NONE ){
       fts5SetVtabError(pTab, 
@@ -1576,7 +1542,6 @@ static int fts5SpecialInsert(
     }else{
       rc = sqlite3Fts5StorageRebuild(pTab->pStorage);
     }
-    bLoadConfig = 1;
   }else if( 0==sqlite3_stricmp("optimize", zCmd) ){
     rc = sqlite3Fts5StorageOptimize(pTab->pStorage);
   }else if( 0==sqlite3_stricmp("merge", zCmd) ){
@@ -1589,13 +1554,8 @@ static int fts5SpecialInsert(
   }else if( 0==sqlite3_stricmp("prefix-index", zCmd) ){
     pConfig->bPrefixIndex = sqlite3_value_int(pVal);
 #endif
-  }else if( 0==sqlite3_stricmp("flush", zCmd) ){
-    rc = sqlite3Fts5FlushToDisk(&pTab->p);
   }else{
-    rc = sqlite3Fts5FlushToDisk(&pTab->p);
-    if( rc==SQLITE_OK ){
-      rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
-    }
+    rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
     if( rc==SQLITE_OK ){
       rc = sqlite3Fts5ConfigSetValue(pTab->p.pConfig, zCmd, pVal, &bError);
     }
@@ -1607,12 +1567,6 @@ static int fts5SpecialInsert(
       }
     }
   }
-
-  if( rc==SQLITE_OK && bLoadConfig ){
-    pTab->p.pConfig->iCookie--;
-    rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
-  }
-
   return rc;
 }
 
@@ -1669,7 +1623,6 @@ static int fts5UpdateMethod(
   Fts5Config *pConfig = pTab->p.pConfig;
   int eType0;                     /* value_type() of apVal[0] */
   int rc = SQLITE_OK;             /* Return code */
-  int bUpdateOrDelete = 0;
 
   /* A transaction must be open when this is called. */
   assert( pTab->ts.eState==1 || pTab->ts.eState==2 );
@@ -1680,11 +1633,6 @@ static int fts5UpdateMethod(
        || sqlite3_value_type(apVal[0])==SQLITE_NULL 
   );
   assert( pTab->p.pConfig->pzErrmsg==0 );
-  if( pConfig->pgsz==0 ){
-    rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
-    if( rc!=SQLITE_OK ) return rc;
-  }
-
   pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg;
 
   /* Put any active cursors into REQUIRE_SEEK state. */
@@ -1699,15 +1647,7 @@ static int fts5UpdateMethod(
     if( pConfig->eContent!=FTS5_CONTENT_NORMAL 
       && 0==sqlite3_stricmp("delete", z) 
     ){
-      if( pConfig->bContentlessDelete ){
-        fts5SetVtabError(pTab, 
-            "'delete' may not be used with a contentless_delete=1 table"
-        );
-        rc = SQLITE_ERROR;
-      }else{
-        rc = fts5SpecialDelete(pTab, apVal);
-        bUpdateOrDelete = 1;
-      }
+      rc = fts5SpecialDelete(pTab, apVal);
     }else{
       rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]);
     }
@@ -1724,7 +1664,7 @@ static int fts5UpdateMethod(
     ** Cases 3 and 4 may violate the rowid constraint.
     */
     int eConflict = SQLITE_ABORT;
-    if( pConfig->eContent==FTS5_CONTENT_NORMAL || pConfig->bContentlessDelete ){
+    if( pConfig->eContent==FTS5_CONTENT_NORMAL ){
       eConflict = sqlite3_vtab_on_conflict(pConfig->db);
     }
 
@@ -1732,12 +1672,8 @@ static int fts5UpdateMethod(
     assert( nArg!=1 || eType0==SQLITE_INTEGER );
 
     /* Filter out attempts to run UPDATE or DELETE on contentless tables.
-    ** This is not suported. Except - they are both supported if the CREATE
-    ** VIRTUAL TABLE statement contained "contentless_delete=1". */
-    if( eType0==SQLITE_INTEGER 
-     && pConfig->eContent==FTS5_CONTENT_NONE 
-     && pConfig->bContentlessDelete==0
-    ){
+    ** This is not suported.  */
+    if( eType0==SQLITE_INTEGER && fts5IsContentless(pTab) ){
       pTab->p.base.zErrMsg = sqlite3_mprintf(
           "cannot %s contentless fts5 table: %s", 
           (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName
@@ -1749,7 +1685,6 @@ static int fts5UpdateMethod(
     else if( nArg==1 ){
       i64 iDel = sqlite3_value_int64(apVal[0]);  /* Rowid to delete */
       rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0);
-      bUpdateOrDelete = 1;
     }
 
     /* INSERT or UPDATE */
@@ -1761,12 +1696,10 @@ static int fts5UpdateMethod(
       }
 
       else if( eType0!=SQLITE_INTEGER ){     
-        /* An INSERT statement. If the conflict-mode is REPLACE, first remove
-        ** the current entry (if any). */
+        /* If this is a REPLACE, first remove the current entry (if any) */
         if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){
           i64 iNew = sqlite3_value_int64(apVal[1]);  /* Rowid to delete */
           rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0);
-          bUpdateOrDelete = 1;
         }
         fts5StorageInsert(&rc, pTab, apVal, pRowid);
       }
@@ -1795,24 +1728,10 @@ static int fts5UpdateMethod(
           rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0);
           fts5StorageInsert(&rc, pTab, apVal, pRowid);
         }
-        bUpdateOrDelete = 1;
       }
     }
   }
 
-  if( rc==SQLITE_OK 
-   && bUpdateOrDelete 
-   && pConfig->bSecureDelete 
-   && pConfig->iVersion==FTS5_CURRENT_VERSION 
-  ){
-    rc = sqlite3Fts5StorageConfigValue(
-        pTab->pStorage, "version", 0, FTS5_CURRENT_VERSION_SECUREDELETE
-    );
-    if( rc==SQLITE_OK ){
-      pConfig->iVersion = FTS5_CURRENT_VERSION_SECUREDELETE;
-    }
-  }
-
   pTab->p.pConfig->pzErrmsg = 0;
   return rc;
 }
@@ -1825,7 +1744,8 @@ static int fts5SyncMethod(sqlite3_vtab *pVtab){
   Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
   fts5CheckTransactionState(pTab, FTS5_SYNC, 0);
   pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg;
-  rc = sqlite3Fts5FlushToDisk(&pTab->p);
+  fts5TripCursors(pTab);
+  rc = sqlite3Fts5StorageSync(pTab->pStorage);
   pTab->p.pConfig->pzErrmsg = 0;
   return rc;
 }
@@ -1921,10 +1841,7 @@ static int fts5ApiColumnText(
 ){
   int rc = SQLITE_OK;
   Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
-  Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab);
-  if( iCol<0 || iCol>=pTab->pConfig->nCol ){
-    rc = SQLITE_RANGE;
-  }else if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab)) 
+  if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab)) 
    || pCsr->ePlan==FTS5_PLAN_SPECIAL 
   ){
     *pz = 0;
@@ -1949,9 +1866,8 @@ static int fts5CsrPoslist(
   int rc = SQLITE_OK;
   int bLive = (pCsr->pSorter==0);
 
-  if( iPhrase<0 || iPhrase>=sqlite3Fts5ExprPhraseCount(pCsr->pExpr) ){
-    rc = SQLITE_RANGE;
-  }else if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){
+  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){
+
     if( pConfig->eDetail!=FTS5_DETAIL_FULL ){
       Fts5PoslistPopulator *aPopulator;
       int i;
@@ -1975,21 +1891,15 @@ static int fts5CsrPoslist(
     CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST);
   }
 
-  if( rc==SQLITE_OK ){
-    if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){
-      Fts5Sorter *pSorter = pCsr->pSorter;
-      int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
-      *pn = pSorter->aIdx[iPhrase] - i1;
-      *pa = &pSorter->aPoslist[i1];
-    }else{
-      *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa);
-    }
+  if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){
+    Fts5Sorter *pSorter = pCsr->pSorter;
+    int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]);
+    *pn = pSorter->aIdx[iPhrase] - i1;
+    *pa = &pSorter->aPoslist[i1];
   }else{
-    *pa = 0;
-    *pn = 0;
+    *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa);
   }
 
-
   return rc;
 }
 
@@ -2096,6 +2006,12 @@ static int fts5ApiInst(
   ){
     if( iIdx<0 || iIdx>=pCsr->nInstCount ){
       rc = SQLITE_RANGE;
+#if 0
+    }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){
+      *piPhrase = pCsr->aInst[iIdx*3];
+      *piCol = pCsr->aInst[iIdx*3 + 2];
+      *piOff = -1;
+#endif
     }else{
       *piPhrase = pCsr->aInst[iIdx*3];
       *piCol = pCsr->aInst[iIdx*3 + 1];
@@ -2235,10 +2151,11 @@ static void *fts5ApiGetAuxdata(Fts5Context *pCtx, int bClear){
 }
 
 static void fts5ApiPhraseNext(
-  Fts5Context *pCtx, 
+  Fts5Context *pUnused, 
   Fts5PhraseIter *pIter, 
   int *piCol, int *piOff
 ){
+  UNUSED_PARAM(pUnused);
   if( pIter->a>=pIter->b ){
     *piCol = -1;
     *piOff = -1;
@@ -2246,12 +2163,8 @@ static void fts5ApiPhraseNext(
     int iVal;
     pIter->a += fts5GetVarint32(pIter->a, iVal);
     if( iVal==1 ){
-      /* Avoid returning a (*piCol) value that is too large for the table,
-      ** even if the position-list is corrupt. The caller might not be
-      ** expecting it.  */
-      int nCol = ((Fts5Table*)(((Fts5Cursor*)pCtx)->base.pVtab))->pConfig->nCol;
       pIter->a += fts5GetVarint32(pIter->a, iVal);
-      *piCol = (iVal>=nCol ? nCol-1 : iVal);
+      *piCol = iVal;
       *piOff = 0;
       pIter->a += fts5GetVarint32(pIter->a, iVal);
     }
@@ -2353,56 +2266,13 @@ static int fts5ApiPhraseFirstColumn(
   return rc;
 }
 
-/*
-** xQueryToken() API implemenetation.
-*/
-static int fts5ApiQueryToken(
-  Fts5Context* pCtx, 
-  int iPhrase, 
-  int iToken, 
-  const char **ppOut, 
-  int *pnOut
-){
-  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
-  return sqlite3Fts5ExprQueryToken(pCsr->pExpr, iPhrase, iToken, ppOut, pnOut);
-}
-
-/*
-** xInstToken() API implemenetation.
-*/
-static int fts5ApiInstToken(
-  Fts5Context *pCtx,
-  int iIdx, 
-  int iToken,
-  const char **ppOut, int *pnOut
-){
-  Fts5Cursor *pCsr = (Fts5Cursor*)pCtx;
-  int rc = SQLITE_OK;
-  if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 
-   || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) 
-  ){
-    if( iIdx<0 || iIdx>=pCsr->nInstCount ){
-      rc = SQLITE_RANGE;
-    }else{
-      int iPhrase = pCsr->aInst[iIdx*3];
-      int iCol = pCsr->aInst[iIdx*3 + 1];
-      int iOff = pCsr->aInst[iIdx*3 + 2];
-      i64 iRowid = fts5CursorRowid(pCsr);
-      rc = sqlite3Fts5ExprInstToken(
-          pCsr->pExpr, iRowid, iPhrase, iCol, iOff, iToken, ppOut, pnOut
-      );
-    }
-  }
-  return rc;
-}
-
 
 static int fts5ApiQueryPhrase(Fts5Context*, int, void*, 
     int(*)(const Fts5ExtensionApi*, Fts5Context*, void*)
 );
 
 static const Fts5ExtensionApi sFts5Api = {
-  3,                            /* iVersion */
+  2,                            /* iVersion */
   fts5ApiUserData,
   fts5ApiColumnCount,
   fts5ApiRowCount,
@@ -2422,8 +2292,6 @@ static const Fts5ExtensionApi sFts5Api = {
   fts5ApiPhraseNext,
   fts5ApiPhraseFirstColumn,
   fts5ApiPhraseNextColumn,
-  fts5ApiQueryToken,
-  fts5ApiInstToken
 };
 
 /*
@@ -2507,10 +2375,7 @@ static void fts5ApiCallback(
     sqlite3_result_error(context, zErr, -1);
     sqlite3_free(zErr);
   }else{
-    sqlite3_vtab *pTab = pCsr->base.pVtab;
     fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]);
-    sqlite3_free(pTab->zErrMsg);
-    pTab->zErrMsg = 0;
   }
 }
 
@@ -2647,12 +2512,6 @@ static int fts5ColumnMethod(
       sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1));
     }
     pConfig->pzErrmsg = 0;
-  }else if( pConfig->bContentlessDelete && sqlite3_vtab_nochange(pCtx) ){
-    char *zErr = sqlite3_mprintf("cannot UPDATE a subset of "
-        "columns on fts5 contentless-delete table: %s", pConfig->zName
-    );
-    sqlite3_result_error(pCtx, zErr, -1);
-    sqlite3_free(zErr);
   }
   return rc;
 }
@@ -2691,10 +2550,8 @@ static int fts5RenameMethod(
   sqlite3_vtab *pVtab,            /* Virtual table handle */
   const char *zName               /* New name of table */
 ){
-  int rc;
   Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
-  rc = sqlite3Fts5StorageRename(pTab->pStorage, zName);
-  return rc;
+  return sqlite3Fts5StorageRename(pTab->pStorage, zName);
 }
 
 int sqlite3Fts5FlushToDisk(Fts5Table *pTab){
@@ -2708,15 +2565,9 @@ int sqlite3Fts5FlushToDisk(Fts5Table *pTab){
 ** Flush the contents of the pending-terms table to disk.
 */
 static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
-  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
-  int rc = SQLITE_OK;
-
-  fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint);
-  rc = sqlite3Fts5FlushToDisk((Fts5Table*)pVtab);
-  if( rc==SQLITE_OK ){
-    pTab->iSavepoint = iSavepoint+1;
-  }
-  return rc;
+  UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
+  fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_SAVEPOINT, iSavepoint);
+  return sqlite3Fts5FlushToDisk((Fts5Table*)pVtab);
 }
 
 /*
@@ -2725,16 +2576,9 @@ static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){
 ** This is a no-op.
 */
 static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
-  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
-  int rc = SQLITE_OK;
-  fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint);
-  if( (iSavepoint+1)<pTab->iSavepoint ){
-    rc = sqlite3Fts5FlushToDisk(&pTab->p);
-    if( rc==SQLITE_OK ){
-      pTab->iSavepoint = iSavepoint;
-    }
-  }
-  return rc;
+  UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
+  fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_RELEASE, iSavepoint);
+  return sqlite3Fts5FlushToDisk((Fts5Table*)pVtab);
 }
 
 /*
@@ -2744,14 +2588,10 @@ static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){
 */
 static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
   Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
-  int rc = SQLITE_OK;
+  UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
   fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint);
   fts5TripCursors(pTab);
-  if( (iSavepoint+1)<=pTab->iSavepoint ){
-    pTab->p.pConfig->pgsz = 0;
-    rc = sqlite3Fts5StorageRollback(pTab->pStorage);
-  }
-  return rc;
+  return sqlite3Fts5StorageRollback(pTab->pStorage);
 }
 
 /*
@@ -2873,7 +2713,7 @@ static int fts5FindTokenizer(
   return rc;
 }
 
-int fts5GetTokenizer(
+int sqlite3Fts5GetTokenizer(
   Fts5Global *pGlobal, 
   const char **azArg,
   int nArg,
@@ -2887,42 +2727,29 @@ int fts5GetTokenizer(
   if( pMod==0 ){
     assert( nArg>0 );
     rc = SQLITE_ERROR;
-    if( pzErr ) *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]);
+    *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]);
   }else{
     rc = pMod->x.xCreate(
-        pMod->pUserData, (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->t.pTok
+        pMod->pUserData, (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->pTok
     );
-    pConfig->t.pTokApi = &pMod->x;
+    pConfig->pTokApi = &pMod->x;
     if( rc!=SQLITE_OK ){
-      if( pzErr && rc!=SQLITE_NOMEM ){
-        *pzErr = sqlite3_mprintf("error in tokenizer constructor");
-      }
+      if( pzErr ) *pzErr = sqlite3_mprintf("error in tokenizer constructor");
     }else{
-      pConfig->t.ePattern = sqlite3Fts5TokenizerPattern(
-          pMod->x.xCreate, pConfig->t.pTok
+      pConfig->ePattern = sqlite3Fts5TokenizerPattern(
+          pMod->x.xCreate, pConfig->pTok
       );
     }
   }
 
   if( rc!=SQLITE_OK ){
-    pConfig->t.pTokApi = 0;
-    pConfig->t.pTok = 0;
+    pConfig->pTokApi = 0;
+    pConfig->pTok = 0;
   }
 
   return rc;
 }
 
-/*
-** Attempt to instantiate the tokenizer.
-*/
-int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig){
-  return fts5GetTokenizer(
-      pConfig->pGlobal, pConfig->t.azArg, pConfig->t.nArg, 
-      pConfig, pConfig->pzErrmsg
-  );
-}
-
-
 static void fts5ModuleDestroy(void *pCtx){
   Fts5TokenizerModule *pTok, *pNextTok;
   Fts5Auxiliary *pAux, *pNextAux;
@@ -2984,47 +2811,9 @@ static int fts5ShadowName(const char *zName){
   return 0;
 }
 
-/*
-** Run an integrity check on the FTS5 data structures.  Return a string
-** if anything is found amiss.  Return a NULL pointer if everything is
-** OK.
-*/
-static int fts5IntegrityMethod(
-  sqlite3_vtab *pVtab,    /* the FTS5 virtual table to check */
-  const char *zSchema,    /* Name of schema in which this table lives */
-  const char *zTabname,   /* Name of the table itself */
-  int isQuick,            /* True if this is a quick-check */
-  char **pzErr            /* Write error message here */
-){
-  Fts5FullTable *pTab = (Fts5FullTable*)pVtab;
-  int rc;
-
-  assert( pzErr!=0 && *pzErr==0 );
-  UNUSED_PARAM(isQuick);
-  assert( pTab->p.pConfig->pzErrmsg==0 );
-  pTab->p.pConfig->pzErrmsg = pzErr;
-  rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, 0);
-  if( *pzErr==0 && rc!=SQLITE_OK ){
-    if( (rc&0xff)==SQLITE_CORRUPT ){
-      *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s",
-          zSchema, zTabname);
-      rc = (*pzErr) ? SQLITE_OK : SQLITE_NOMEM;
-    }else{
-      *pzErr = sqlite3_mprintf("unable to validate the inverted index for"
-          " FTS5 table %s.%s: %s",
-          zSchema, zTabname, sqlite3_errstr(rc));
-    }
-  }
-
-  sqlite3Fts5IndexCloseReader(pTab->p.pIndex);
-  pTab->p.pConfig->pzErrmsg = 0;
-
-  return rc;
-}
-
 static int fts5Init(sqlite3 *db){
   static const sqlite3_module fts5Mod = {
-    /* iVersion      */ 4,
+    /* iVersion      */ 3,
     /* xCreate       */ fts5CreateMethod,
     /* xConnect      */ fts5ConnectMethod,
     /* xBestIndex    */ fts5BestIndexMethod,
@@ -3047,8 +2836,7 @@ static int fts5Init(sqlite3 *db){
     /* xSavepoint    */ fts5SavepointMethod,
     /* xRelease      */ fts5ReleaseMethod,
     /* xRollbackTo   */ fts5RollbackToMethod,
-    /* xShadowName   */ fts5ShadowName,
-    /* xIntegrity    */ fts5IntegrityMethod
+    /* xShadowName   */ fts5ShadowName
   };
 
   int rc;

ext/fts5/fts5_storage.c

@@ -77,10 +77,10 @@ static int fts5StorageGetStmt(
       "INSERT INTO %Q.'%q_content' VALUES(%s)",         /* INSERT_CONTENT  */
       "REPLACE INTO %Q.'%q_content' VALUES(%s)",        /* REPLACE_CONTENT */
       "DELETE FROM %Q.'%q_content' WHERE id=?",         /* DELETE_CONTENT  */
-      "REPLACE INTO %Q.'%q_docsize' VALUES(?,?%s)",     /* REPLACE_DOCSIZE  */
+      "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)",       /* REPLACE_DOCSIZE  */
       "DELETE FROM %Q.'%q_docsize' WHERE id=?",         /* DELETE_DOCSIZE  */
 
-      "SELECT sz%s FROM %Q.'%q_docsize' WHERE id=?",    /* LOOKUP_DOCSIZE  */
+      "SELECT sz FROM %Q.'%q_docsize' WHERE id=?",      /* LOOKUP_DOCSIZE  */
 
       "REPLACE INTO %Q.'%q_config' VALUES(?,?)",        /* REPLACE_CONFIG */
       "SELECT %s FROM %s AS T",                         /* SCAN */
@@ -128,19 +128,6 @@ static int fts5StorageGetStmt(
         break;
       }
 
-      case FTS5_STMT_REPLACE_DOCSIZE: 
-        zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName,
-          (pC->bContentlessDelete ? ",?" : "")
-        );
-        break;
-
-      case FTS5_STMT_LOOKUP_DOCSIZE: 
-        zSql = sqlite3_mprintf(azStmt[eStmt], 
-            (pC->bContentlessDelete ? ",origin" : ""),
-            pC->zDb, pC->zName
-        );
-        break;
-
       default:
         zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName);
         break;
@@ -330,11 +317,9 @@ int sqlite3Fts5StorageOpen(
     }
 
     if( rc==SQLITE_OK && pConfig->bColumnsize ){
-      const char *zCols = "id INTEGER PRIMARY KEY, sz BLOB";
-      if( pConfig->bContentlessDelete ){
-        zCols = "id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER";
-      }
-      rc = sqlite3Fts5CreateTable(pConfig, "docsize", zCols, 0, pzErr);
+      rc = sqlite3Fts5CreateTable(
+          pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr
+      );
     }
     if( rc==SQLITE_OK ){
       rc = sqlite3Fts5CreateTable(
@@ -411,7 +396,7 @@ static int fts5StorageDeleteFromIndex(
 ){
   Fts5Config *pConfig = p->pConfig;
   sqlite3_stmt *pSeek = 0;        /* SELECT to read row iDel from %_data */
-  int rc = SQLITE_OK;             /* Return code */
+  int rc;                         /* Return code */
   int rc2;                        /* sqlite3_reset() return code */
   int iCol;
   Fts5InsertCtx ctx;
@@ -427,6 +412,7 @@ static int fts5StorageDeleteFromIndex(
 
   ctx.pStorage = p;
   ctx.iCol = -1;
+  rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
   for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
     if( pConfig->abUnindexed[iCol-1]==0 ){
       const char *zText;
@@ -447,7 +433,7 @@ static int fts5StorageDeleteFromIndex(
           zText, nText, (void*)&ctx, fts5StorageInsertCallback
       );
       p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
-      if( p->aTotalSize[iCol-1]<0 && rc==SQLITE_OK ){
+      if( p->aTotalSize[iCol-1]<0 ){
         rc = FTS5_CORRUPT;
       }
     }
@@ -463,37 +449,6 @@ static int fts5StorageDeleteFromIndex(
   return rc;
 }
 
-/*
-** This function is called to process a DELETE on a contentless_delete=1
-** table. It adds the tombstone required to delete the entry with rowid 
-** iDel. If successful, SQLITE_OK is returned. Or, if an error occurs,
-** an SQLite error code.
-*/
-static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){
-  i64 iOrigin = 0;
-  sqlite3_stmt *pLookup = 0;
-  int rc = SQLITE_OK;
-
-  assert( p->pConfig->bContentlessDelete );
-  assert( p->pConfig->eContent==FTS5_CONTENT_NONE );
-
-  /* Look up the origin of the document in the %_docsize table. Store
-  ** this in stack variable iOrigin.  */
-  rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0);
-  if( rc==SQLITE_OK ){
-    sqlite3_bind_int64(pLookup, 1, iDel);
-    if( SQLITE_ROW==sqlite3_step(pLookup) ){
-      iOrigin = sqlite3_column_int64(pLookup, 1);
-    }
-    rc = sqlite3_reset(pLookup);
-  }
-
-  if( rc==SQLITE_OK && iOrigin!=0 ){
-    rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iOrigin, iDel);
-  }
-
-  return rc;
-}
 
 /*
 ** Insert a record into the %_docsize table. Specifically, do:
@@ -514,17 +469,10 @@ static int fts5StorageInsertDocsize(
     rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0);
     if( rc==SQLITE_OK ){
       sqlite3_bind_int64(pReplace, 1, iRowid);
-      if( p->pConfig->bContentlessDelete ){
-        i64 iOrigin = 0;
-        rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin);
-        sqlite3_bind_int64(pReplace, 3, iOrigin);
-      }
-      if( rc==SQLITE_OK ){
-        sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
-        sqlite3_step(pReplace);
-        rc = sqlite3_reset(pReplace);
-        sqlite3_bind_null(pReplace, 2);
-      }
+      sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
+      sqlite3_step(pReplace);
+      rc = sqlite3_reset(pReplace);
+      sqlite3_bind_null(pReplace, 2);
     }
   }
   return rc;
@@ -588,15 +536,7 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
 
   /* Delete the index records */
   if( rc==SQLITE_OK ){
-    rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel);
-  }
-
-  if( rc==SQLITE_OK ){
-    if( p->pConfig->bContentlessDelete ){
-      rc = fts5StorageContentlessDelete(p, iDel);
-    }else{
-      rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
-    }
+    rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
   }
 
   /* Delete the %_docsize record */
@@ -673,7 +613,7 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){
   }
 
   if( rc==SQLITE_OK ){
-    rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, pConfig->pzErrmsg);
+    rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0);
   }
 
   while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){
@@ -1184,9 +1124,7 @@ int sqlite3Fts5StorageSync(Fts5Storage *p){
   i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db);
   if( p->bTotalsValid ){
     rc = fts5StorageSaveTotals(p);
-    if( rc==SQLITE_OK ){
-      p->bTotalsValid = 0;
-    }
+    p->bTotalsValid = 0;
   }
   if( rc==SQLITE_OK ){
     rc = sqlite3Fts5IndexSync(p->pIndex);

ext/fts5/fts5_tcl.c

@@ -14,7 +14,14 @@
 
 
 #ifdef SQLITE_TEST
-#include "tclsqlite.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 
 #ifdef SQLITE_ENABLE_FTS5
 
@@ -237,9 +244,6 @@ static int SQLITE_TCLAPI xF5tApi(
     { "xGetAuxdataInt",    1, "CLEAR" },              /* 15 */
     { "xPhraseForeach",    4, "IPHRASE COLVAR OFFVAR SCRIPT" }, /* 16 */
     { "xPhraseColumnForeach", 3, "IPHRASE COLVAR SCRIPT" }, /* 17 */
-
-    { "xQueryToken",       2, "IPHRASE ITERM" },      /* 18 */
-    { "xInstToken",        2, "IDX ITERM" },          /* 19 */
     { 0, 0, 0}
   };
 
@@ -290,12 +294,12 @@ static int SQLITE_TCLAPI xF5tApi(
       break;
     }
     CASE(3, "xTokenize") {
-      Tcl_Size nText;
+      int nText;
       char *zText = Tcl_GetStringFromObj(objv[2], &nText);
       F5tFunction ctx;
       ctx.interp = interp;
       ctx.pScript = objv[3];
-      rc = p->pApi->xTokenize(p->pFts, zText, (int)nText, &ctx, xTokenizeCb);
+      rc = p->pApi->xTokenize(p->pFts, zText, nText, &ctx, xTokenizeCb);
       if( rc==SQLITE_OK ){
         Tcl_ResetResult(interp);
       }
@@ -496,38 +500,6 @@ static int SQLITE_TCLAPI xF5tApi(
       break;
     }
 
-    CASE(18, "xQueryToken") {
-      const char *pTerm = 0;
-      int nTerm = 0;
-      int iPhrase = 0;
-      int iTerm = 0;
-
-      if( Tcl_GetIntFromObj(interp, objv[2], &iPhrase) ) return TCL_ERROR;
-      if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR;
-      rc = p->pApi->xQueryToken(p->pFts, iPhrase, iTerm, &pTerm, &nTerm);
-      if( rc==SQLITE_OK ){
-        Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm));
-      }
-
-      break;
-    }
-
-    CASE(19, "xInstToken") {
-      const char *pTerm = 0;
-      int nTerm = 0;
-      int iIdx = 0;
-      int iTerm = 0;
-
-      if( Tcl_GetIntFromObj(interp, objv[2], &iIdx) ) return TCL_ERROR;
-      if( Tcl_GetIntFromObj(interp, objv[3], &iTerm) ) return TCL_ERROR;
-      rc = p->pApi->xInstToken(p->pFts, iIdx, iTerm, &pTerm, &nTerm);
-      if( rc==SQLITE_OK ){
-        Tcl_SetObjResult(interp, Tcl_NewStringObj(pTerm, nTerm));
-      }
-
-      break;
-    }
-
     default: 
       assert( 0 );
       break;
@@ -598,16 +570,15 @@ static void xF5tFunction(
     sqlite3_result_error(pCtx, Tcl_GetStringResult(p->interp), -1);
   }else{
     Tcl_Obj *pVar = Tcl_GetObjResult(p->interp);
+    int n;
     const char *zType = (pVar->typePtr ? pVar->typePtr->name : "");
     char c = zType[0];
     if( c=='b' && strcmp(zType,"bytearray")==0 && pVar->bytes==0 ){
       /* Only return a BLOB type if the Tcl variable is a bytearray and
       ** has no string representation. */
-      Tcl_Size nn;
-      unsigned char *data = Tcl_GetByteArrayFromObj(pVar, &nn);
-      sqlite3_result_blob(pCtx, data, (int)nn, SQLITE_TRANSIENT);
+      unsigned char *data = Tcl_GetByteArrayFromObj(pVar, &n);
+      sqlite3_result_blob(pCtx, data, n, SQLITE_TRANSIENT);
     }else if( c=='b' && strcmp(zType,"boolean")==0 ){
-      int n;
       Tcl_GetIntFromObj(0, pVar, &n);
       sqlite3_result_int(pCtx, n);
     }else if( c=='d' && strcmp(zType,"double")==0 ){
@@ -620,9 +591,8 @@ static void xF5tFunction(
       Tcl_GetWideIntFromObj(0, pVar, &v);
       sqlite3_result_int64(pCtx, v);
     }else{
-      Tcl_Size nn;
-      unsigned char *data = (unsigned char *)Tcl_GetStringFromObj(pVar, &nn);
-      sqlite3_result_text(pCtx, (char*)data, (int)nn, SQLITE_TRANSIENT);
+      unsigned char *data = (unsigned char *)Tcl_GetStringFromObj(pVar, &n);
+      sqlite3_result_text(pCtx, (char *)data, n, SQLITE_TRANSIENT);
     }
   }
 }
@@ -715,7 +685,7 @@ static int SQLITE_TCLAPI f5tTokenize(
   Tcl_Obj *CONST objv[]
 ){
   char *zText;
-  Tcl_Size nText;
+  int nText;
   sqlite3 *db = 0;
   fts5_api *pApi = 0;
   Fts5Tokenizer *pTok = 0;
@@ -724,7 +694,7 @@ static int SQLITE_TCLAPI f5tTokenize(
   void *pUserdata;
   int rc;
 
-  Tcl_Size nArg;
+  int nArg;
   const char **azArg;
   F5tTokenizeCtx ctx;
 
@@ -756,7 +726,7 @@ static int SQLITE_TCLAPI f5tTokenize(
     return TCL_ERROR;
   }
 
-  rc = tokenizer.xCreate(pUserdata, &azArg[1], (int)(nArg-1), &pTok);
+  rc = tokenizer.xCreate(pUserdata, &azArg[1], nArg-1, &pTok);
   if( rc!=SQLITE_OK ){
     Tcl_AppendResult(interp, "error in tokenizer.xCreate()", 0);
     return TCL_ERROR;
@@ -768,7 +738,7 @@ static int SQLITE_TCLAPI f5tTokenize(
   ctx.pRet = pRet;
   ctx.zInput = zText;
   rc = tokenizer.xTokenize(
-      pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, zText,(int)nText, xTokenizeCb2
+      pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, zText, nText, xTokenizeCb2
   );
   tokenizer.xDelete(pTok);
   if( rc!=SQLITE_OK ){
@@ -923,13 +893,13 @@ static int SQLITE_TCLAPI f5tTokenizerReturn(
   F5tTokenizerContext *p = (F5tTokenizerContext*)clientData;
   int iStart;
   int iEnd;
-  Tcl_Size nToken;
+  int nToken;
   int tflags = 0;
   char *zToken;
   int rc;
 
   if( objc==5 ){
-    Tcl_Size nArg;
+    int nArg;
     char *zArg = Tcl_GetStringFromObj(objv[1], &nArg);
     if( nArg<=10 && nArg>=2 && memcmp("-colocated", zArg, nArg)==0 ){
       tflags |= FTS5_TOKEN_COLOCATED;
@@ -954,7 +924,7 @@ static int SQLITE_TCLAPI f5tTokenizerReturn(
     return TCL_ERROR;
   }
 
-  rc = p->xToken(p->pCtx, tflags, zToken, (int)nToken, iStart, iEnd);
+  rc = p->xToken(p->pCtx, tflags, zToken, nToken, iStart, iEnd);
   Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
   return rc==SQLITE_OK ? TCL_OK : TCL_ERROR;
 
@@ -1078,7 +1048,7 @@ static int SQLITE_TCLAPI f5tTokenHash(
   Tcl_Obj *CONST objv[]
 ){
   char *z;
-  Tcl_Size n;
+  int n;
   unsigned int iVal;
   int nSlot;
 
@@ -1091,7 +1061,7 @@ static int SQLITE_TCLAPI f5tTokenHash(
   }
   z = Tcl_GetStringFromObj(objv[2], &n);
 
-  iVal = f5t_fts5HashKey(nSlot, z, (int)n);
+  iVal = f5t_fts5HashKey(nSlot, z, n);
   Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
   return TCL_OK;
 }
@@ -1147,176 +1117,6 @@ static int SQLITE_TCLAPI f5tRegisterTok(
   return TCL_OK;
 }
 
-typedef struct OriginTextCtx OriginTextCtx;
-struct OriginTextCtx {
-  sqlite3 *db;
-  fts5_api *pApi;
-};
-
-typedef struct OriginTextTokenizer OriginTextTokenizer;
-struct OriginTextTokenizer {
-  Fts5Tokenizer *pTok;            /* Underlying tokenizer object */
-  fts5_tokenizer tokapi;          /* API implementation for pTok */
-};
-
-/*
-** Delete the OriginTextCtx object indicated by the only argument.
-*/
-static void f5tOrigintextTokenizerDelete(void *pCtx){
-  OriginTextCtx *p = (OriginTextCtx*)pCtx;
-  ckfree((char*)p);
-}
-
-static int f5tOrigintextCreate(
-  void *pCtx, 
-  const char **azArg, 
-  int nArg, 
-  Fts5Tokenizer **ppOut
-){
-  OriginTextCtx *p = (OriginTextCtx*)pCtx;
-  OriginTextTokenizer *pTok = 0;
-  void *pTokCtx = 0;
-  int rc = SQLITE_OK;
-
-  pTok = (OriginTextTokenizer*)sqlite3_malloc(sizeof(OriginTextTokenizer));
-  if( pTok==0 ){
-    rc = SQLITE_NOMEM;
-  }else if( nArg<1 ){
-    rc = SQLITE_ERROR;
-  }else{
-    /* Locate the underlying tokenizer */
-    rc = p->pApi->xFindTokenizer(p->pApi, azArg[0], &pTokCtx, &pTok->tokapi);
-  }
-
-  /* Create the new tokenizer instance */
-  if( rc==SQLITE_OK ){
-    rc = pTok->tokapi.xCreate(pTokCtx, &azArg[1], nArg-1, &pTok->pTok);
-  }
-
-  if( rc!=SQLITE_OK ){
-    sqlite3_free(pTok);
-    pTok = 0;
-  }
-  *ppOut = (Fts5Tokenizer*)pTok;
-  return rc;
-}
-
-static void f5tOrigintextDelete(Fts5Tokenizer *pTokenizer){
-  OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer;
-  if( p->pTok ){
-    p->tokapi.xDelete(p->pTok);
-  }
-  sqlite3_free(p);
-}
-
-typedef struct OriginTextCb OriginTextCb;
-struct OriginTextCb {
-  void *pCtx;
-  const char *pText; 
-  int nText;
-  int (*xToken)(void *, int, const char *, int, int, int);
-
-  char *aBuf;                     /* Buffer to use */
-  int nBuf;                       /* Allocated size of aBuf[] */
-};
-
-static int xOriginToken(
-  void *pCtx,                     /* Copy of 2nd argument to xTokenize() */
-  int tflags,                     /* Mask of FTS5_TOKEN_* flags */
-  const char *pToken,             /* Pointer to buffer containing token */
-  int nToken,                     /* Size of token in bytes */
-  int iStart,                     /* Byte offset of token within input text */
-  int iEnd                        /* Byte offset of end of token within input */
-){
-  OriginTextCb *p = (OriginTextCb*)pCtx;
-  int ret = 0;
-
-  if( nToken==(iEnd-iStart) && 0==memcmp(pToken, &p->pText[iStart], nToken) ){
-    /* Token exactly matches document text. Pass it through as is. */
-    ret = p->xToken(p->pCtx, tflags, pToken, nToken, iStart, iEnd);
-  }else{
-    int nReq = nToken + 1 + (iEnd-iStart);
-    if( nReq>p->nBuf ){
-      sqlite3_free(p->aBuf);
-      p->aBuf = sqlite3_malloc(nReq*2);
-      if( p->aBuf==0 ) return SQLITE_NOMEM;
-      p->nBuf = nReq*2;
-    }
-
-    memcpy(p->aBuf, pToken, nToken);
-    p->aBuf[nToken] = '\0';
-    memcpy(&p->aBuf[nToken+1], &p->pText[iStart], iEnd-iStart);
-    ret = p->xToken(p->pCtx, tflags, p->aBuf, nReq, iStart, iEnd);
-  }
-
-  return ret;
-}
-
-
-static int f5tOrigintextTokenize(
-  Fts5Tokenizer *pTokenizer, 
-  void *pCtx,
-  int flags,                      /* Mask of FTS5_TOKENIZE_* flags */
-  const char *pText, int nText, 
-  int (*xToken)(void *, int, const char *, int, int, int)
-){
-  OriginTextTokenizer *p = (OriginTextTokenizer*)pTokenizer;
-  OriginTextCb cb;
-  int ret;
-
-  memset(&cb, 0, sizeof(cb));
-  cb.pCtx = pCtx;
-  cb.pText = pText;
-  cb.nText = nText;
-  cb.xToken = xToken;
-
-  ret = p->tokapi.xTokenize(p->pTok,(void*)&cb,flags,pText,nText,xOriginToken);
-  sqlite3_free(cb.aBuf);
-  return ret;
-}
-
-/*
-**      sqlite3_fts5_register_origintext DB
-**
-** Description...
-*/
-static int SQLITE_TCLAPI f5tRegisterOriginText(
-  void * clientData,
-  Tcl_Interp *interp,
-  int objc,
-  Tcl_Obj *CONST objv[]
-){
-  sqlite3 *db = 0;
-  fts5_api *pApi = 0;
-  int rc;
-  fts5_tokenizer tok = {0, 0, 0};
-  OriginTextCtx *pCtx = 0;
-
-  if( objc!=2 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "DB");
-    return TCL_ERROR;
-  }
-  if( f5tDbAndApi(interp, objv[1], &db, &pApi) ) return TCL_ERROR;
-
-  pCtx = (OriginTextCtx*)ckalloc(sizeof(OriginTextCtx));
-  pCtx->db = db;
-  pCtx->pApi = pApi;
-
-  tok.xCreate = f5tOrigintextCreate;
-  tok.xDelete = f5tOrigintextDelete;
-  tok.xTokenize = f5tOrigintextTokenize;
-  rc = pApi->xCreateTokenizer(
-      pApi, "origintext", (void*)pCtx, &tok, f5tOrigintextTokenizerDelete
-  );
-
-  Tcl_ResetResult(interp);
-  if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, "error: ", sqlite3_errmsg(db), 0);
-    return TCL_ERROR;
-  }
-  return TCL_OK;
-}
-
 /*
 ** Entry point.
 */
@@ -1333,8 +1133,7 @@ int Fts5tcl_Init(Tcl_Interp *interp){
     { "sqlite3_fts5_may_be_corrupt",     f5tMayBeCorrupt, 0 },
     { "sqlite3_fts5_token_hash",         f5tTokenHash, 0 },
     { "sqlite3_fts5_register_matchinfo", f5tRegisterMatchinfo, 0 },
-    { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 },
-    { "sqlite3_fts5_register_origintext",f5tRegisterOriginText, 0 }
+    { "sqlite3_fts5_register_fts5tokenize", f5tRegisterTok, 0 }
   };
   int i;
   F5tTokenizerContext *pContext;

ext/fts5/fts5_test_tok.c

@@ -472,8 +472,7 @@ int sqlite3Fts5TestRegisterTok(sqlite3 *db, fts5_api *pApi){
      0,                           /* xSavepoint    */
      0,                           /* xRelease      */
      0,                           /* xRollbackTo   */
-     0,                           /* xShadowName   */
-     0                            /* xIntegrity    */
+     0                            /* xShadowName   */
   };
   int rc;                         /* Return code */
 

ext/fts5/fts5_tokenize.c

@@ -79,7 +79,7 @@ static int fts5AsciiCreate(
       int i;
       memset(p, 0, sizeof(AsciiTokenizer));
       memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar));
-      for(i=0; rc==SQLITE_OK && i<nArg-1; i+=2){
+      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
         const char *zArg = azArg[i+1];
         if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
           fts5AsciiAddExceptions(p, zArg, 1);
@@ -90,7 +90,6 @@ static int fts5AsciiCreate(
           rc = SQLITE_ERROR;
         }
       }
-      if( rc==SQLITE_OK && i<nArg ) rc = SQLITE_ERROR;
       if( rc!=SQLITE_OK ){
         fts5AsciiDelete((Fts5Tokenizer*)p);
         p = 0;
@@ -230,12 +229,6 @@ static const unsigned char sqlite3Utf8Trans1[] = {
 
 #endif /* ifndef SQLITE_AMALGAMATION */
 
-#define FTS5_SKIP_UTF8(zIn) {                               \
-  if( ((unsigned char)(*(zIn++)))>=0xc0 ){                              \
-    while( (((unsigned char)*zIn) & 0xc0)==0x80 ){ zIn++; }             \
-  }                                                    \
-}
-
 typedef struct Unicode61Tokenizer Unicode61Tokenizer;
 struct Unicode61Tokenizer {
   unsigned char aTokenChar[128];  /* ASCII range token characters */
@@ -382,16 +375,17 @@ static int fts5UnicodeCreate(
       }
 
       /* Search for a "categories" argument */
-      for(i=0; rc==SQLITE_OK && i<nArg-1; i+=2){
+      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
         if( 0==sqlite3_stricmp(azArg[i], "categories") ){
           zCat = azArg[i+1];
         }
       }
+
       if( rc==SQLITE_OK ){
         rc = unicodeSetCategories(p, zCat);
       }
 
-      for(i=0; rc==SQLITE_OK && i<nArg-1; i+=2){
+      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
         const char *zArg = azArg[i+1];
         if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
           if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
@@ -416,7 +410,6 @@ static int fts5UnicodeCreate(
           rc = SQLITE_ERROR;
         }
       }
-      if( i<nArg && rc==SQLITE_OK ) rc = SQLITE_ERROR;
 
     }else{
       rc = SQLITE_NOMEM;
@@ -1271,7 +1264,6 @@ static int fts5PorterTokenize(
 typedef struct TrigramTokenizer TrigramTokenizer;
 struct TrigramTokenizer {
   int bFold;                      /* True to fold to lower-case */
-  int iFoldParam;                 /* Parameter to pass to Fts5UnicodeFold() */
 };
 
 /*
@@ -1298,8 +1290,7 @@ static int fts5TriCreate(
   }else{
     int i;
     pNew->bFold = 1;
-    pNew->iFoldParam = 0;
-    for(i=0; rc==SQLITE_OK && i<nArg-1; i+=2){
+    for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
       const char *zArg = azArg[i+1];
       if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
         if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
@@ -1307,22 +1298,10 @@ static int fts5TriCreate(
         }else{
           pNew->bFold = (zArg[0]=='0');
         }
-      }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
-        if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
-          rc = SQLITE_ERROR;
-        }else{
-          pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
-        }
       }else{
         rc = SQLITE_ERROR;
       }
     }
-    if( i<nArg && rc==SQLITE_OK ) rc = SQLITE_ERROR;
-
-    if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
-      rc = SQLITE_ERROR;
-    }
-
     if( rc!=SQLITE_OK ){
       fts5TriDelete((Fts5Tokenizer*)pNew);
       pNew = 0;
@@ -1345,62 +1324,40 @@ static int fts5TriTokenize(
   TrigramTokenizer *p = (TrigramTokenizer*)pTok;
   int rc = SQLITE_OK;
   char aBuf[32];
-  char *zOut = aBuf;
-  int ii;
   const unsigned char *zIn = (const unsigned char*)pText;
   const unsigned char *zEof = &zIn[nText];
   u32 iCode;
-  int aStart[3];                  /* Input offset of each character in aBuf[] */
 
   UNUSED_PARAM(unusedFlags);
-
-  /* Populate aBuf[] with the characters for the first trigram. */
-  for(ii=0; ii<3; ii++){
-    do {
-      aStart[ii] = zIn - (const unsigned char*)pText;
-      READ_UTF8(zIn, zEof, iCode);
-      if( iCode==0 ) return SQLITE_OK;
-      if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam);
-    }while( iCode==0 );
-    WRITE_UTF8(zOut, iCode);
-  }
-
-  /* At the start of each iteration of this loop:
-  **
-  **  aBuf:      Contains 3 characters. The 3 characters of the next trigram.
-  **  zOut:      Points to the byte following the last character in aBuf.
-  **  aStart[3]: Contains the byte offset in the input text corresponding
-  **             to the start of each of the three characters in the buffer.
-  */
-  assert( zIn<=zEof );
   while( 1 ){
-    int iNext;                    /* Start of character following current tri */
-    const char *z1;
+    char *zOut = aBuf;
+    int iStart = zIn - (const unsigned char*)pText;
+    const unsigned char *zNext; 
 
-    /* Read characters from the input up until the first non-diacritic */
-    do {
-      iNext = zIn - (const unsigned char*)pText;
+    READ_UTF8(zIn, zEof, iCode);
+    if( iCode==0 ) break;
+    zNext = zIn;
+    if( zIn<zEof ){
+      if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
+      WRITE_UTF8(zOut, iCode);
       READ_UTF8(zIn, zEof, iCode);
       if( iCode==0 ) break;
-      if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam);
-    }while( iCode==0 );
-
-    /* Pass the current trigram back to fts5 */
-    rc = xToken(pCtx, 0, aBuf, zOut-aBuf, aStart[0], iNext);
-    if( iCode==0 || rc!=SQLITE_OK ) break;
-
-    /* Remove the first character from buffer aBuf[]. Append the character
-    ** with codepoint iCode.  */
-    z1 = aBuf;
-    FTS5_SKIP_UTF8(z1);
-    memmove(aBuf, z1, zOut - z1);
-    zOut -= (z1 - aBuf);
-    WRITE_UTF8(zOut, iCode);
-
-    /* Update the aStart[] array */
-    aStart[0] = aStart[1];
-    aStart[1] = aStart[2];
-    aStart[2] = iNext;
+    }else{
+      break;
+    }
+    if( zIn<zEof ){
+      if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
+      WRITE_UTF8(zOut, iCode);
+      READ_UTF8(zIn, zEof, iCode);
+      if( iCode==0 ) break;
+      if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, 0);
+      WRITE_UTF8(zOut, iCode);
+    }else{
+      break;
+    }
+    rc = xToken(pCtx, 0, aBuf, zOut-aBuf, iStart, iStart + zOut-aBuf);
+    if( rc!=SQLITE_OK ) break;
+    zIn = zNext;
   }
 
   return rc;
@@ -1423,23 +1380,11 @@ int sqlite3Fts5TokenizerPattern(
 ){
   if( xCreate==fts5TriCreate ){
     TrigramTokenizer *p = (TrigramTokenizer*)pTok;
-    if( p->iFoldParam==0 ){
-      return p->bFold ? FTS5_PATTERN_LIKE : FTS5_PATTERN_GLOB;
-    }
+    return p->bFold ? FTS5_PATTERN_LIKE : FTS5_PATTERN_GLOB;
   }
   return FTS5_PATTERN_NONE;
 }
 
-/*
-** Return true if the tokenizer described by p->azArg[] is the trigram
-** tokenizer. This tokenizer needs to be loaded before xBestIndex is
-** called for the first time in order to correctly handle LIKE/GLOB.
-*/
-int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig *p){
-  return (p->nArg>=1 && 0==sqlite3_stricmp(p->azArg[0], "trigram"));
-}
-
-
 /*
 ** Register all built-in tokenizers with FTS5.
 */

ext/fts5/fts5_vocab.c

@@ -64,7 +64,6 @@ struct Fts5VocabCursor {
 
   int nLeTerm;                    /* Size of zLeTerm in bytes */
   char *zLeTerm;                  /* (term <= $zLeTerm) paramater, or NULL */
-  int colUsed;                    /* Copy of sqlite3_index_info.colUsed */
 
   /* These are used by 'col' tables only */
   int iCol;
@@ -91,11 +90,9 @@ struct Fts5VocabCursor {
 /*
 ** Bits for the mask used as the idxNum value by xBestIndex/xFilter.
 */
-#define FTS5_VOCAB_TERM_EQ 0x0100
-#define FTS5_VOCAB_TERM_GE 0x0200
-#define FTS5_VOCAB_TERM_LE 0x0400
-
-#define FTS5_VOCAB_COLUSED_MASK 0xFF
+#define FTS5_VOCAB_TERM_EQ 0x01
+#define FTS5_VOCAB_TERM_GE 0x02
+#define FTS5_VOCAB_TERM_LE 0x04
 
 
 /*
@@ -272,13 +269,11 @@ static int fts5VocabBestIndexMethod(
   int iTermEq = -1;
   int iTermGe = -1;
   int iTermLe = -1;
-  int idxNum = (int)pInfo->colUsed;
+  int idxNum = 0;
   int nArg = 0;
 
   UNUSED_PARAM(pUnused);
 
-  assert( (pInfo->colUsed & FTS5_VOCAB_COLUSED_MASK)==pInfo->colUsed );
-
   for(i=0; i<pInfo->nConstraint; i++){
     struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
     if( p->usable==0 ) continue;
@@ -370,7 +365,7 @@ static int fts5VocabOpenMethod(
       if( rc==SQLITE_OK ){
         pVTab->zErrMsg = sqlite3_mprintf(
             "no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl
-        );
+            );
         rc = SQLITE_ERROR;
       }
     }else{
@@ -530,19 +525,9 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
 
         switch( pTab->eType ){
           case FTS5_VOCAB_ROW:
-            /* Do not bother counting the number of instances if the "cnt"
-            ** column is not being read (according to colUsed).  */
-            if( eDetail==FTS5_DETAIL_FULL && (pCsr->colUsed & 0x04) ){
-              while( iPos<nPos ){
-                u32 ii;
-                fts5FastGetVarint32(pPos, iPos, ii);
-                if( ii==1 ){ 
-                  /* New column in the position list */
-                  fts5FastGetVarint32(pPos, iPos, ii);
-                }else{
-                  /* An instance - increment pCsr->aCnt[] */
-                  pCsr->aCnt[0]++;
-                }
+            if( eDetail==FTS5_DETAIL_FULL ){
+              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
+                pCsr->aCnt[0]++;
               }
             }
             pCsr->aDoc[0]++;
@@ -640,12 +625,11 @@ static int fts5VocabFilterMethod(
   if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++];
   if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++];
   if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++];
-  pCsr->colUsed = (idxNum & FTS5_VOCAB_COLUSED_MASK);
 
   if( pEq ){
     zTerm = (const char *)sqlite3_value_text(pEq);
     nTerm = sqlite3_value_bytes(pEq);
-    f = FTS5INDEX_QUERY_NOTOKENDATA;
+    f = 0;
   }else{
     if( pGe ){
       zTerm = (const char *)sqlite3_value_text(pGe);
@@ -799,8 +783,7 @@ int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){
     /* xSavepoint    */ 0,
     /* xRelease      */ 0,
     /* xRollbackTo   */ 0,
-    /* xShadowName   */ 0,
-    /* xIntegrity    */ 0
+    /* xShadowName   */ 0
   };
   void *p = (void*)pGlobal;
 

ext/fts5/test/fts5_common.tcl

@@ -61,12 +61,6 @@ proc fts5_test_collist {cmd} {
   set res
 }
 
-proc fts5_collist {cmd iPhrase} {
-  set res [list]
-  $cmd xPhraseColumnForeach $iPhrase c { lappend res $c }
-  set res
-}
-
 proc fts5_test_columnsize {cmd} {
   set res [list]
   for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
@@ -75,10 +69,6 @@ proc fts5_test_columnsize {cmd} {
   set res
 }
 
-proc fts5_columntext {cmd iCol} {
-  $cmd xColumnText $iCol
-}
-
 proc fts5_test_columntext {cmd} {
   set res [list]
   for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
@@ -114,10 +104,6 @@ proc fts5_test_rowcount {cmd} {
   $cmd xRowCount
 }
 
-proc fts5_test_rowid {cmd} {
-  $cmd xRowid
-}
-
 proc test_queryphrase_cb {cnt cmd} {
   upvar $cnt L 
   for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
@@ -139,13 +125,6 @@ proc fts5_test_queryphrase {cmd} {
   set res
 }
 
-proc fts5_queryphrase {cmd iPhrase} {
-  set cnt [list]
-  for {set j 0} {$j < [$cmd xColumnCount]} {incr j} { lappend cnt 0 }
-  $cmd xQueryPhrase $iPhrase [list test_queryphrase_cb cnt]
-  set cnt
-}
-
 proc fts5_test_phrasecount {cmd} {
   $cmd xPhraseCount
 }
@@ -171,14 +150,10 @@ proc fts5_aux_test_functions {db} {
     fts5_test_collist
     fts5_test_tokenize
     fts5_test_rowcount
-    fts5_test_rowid
     fts5_test_all
 
     fts5_test_queryphrase
     fts5_test_phrasecount
-    fts5_columntext
-    fts5_queryphrase
-    fts5_collist
   } {
     sqlite3_fts5_create_function $db $f $f
   }
@@ -463,20 +438,6 @@ proc detail_is_none {} { detail_check ; expr {$::detail == "none"} }
 proc detail_is_col {}  { detail_check ; expr {$::detail == "col" } }
 proc detail_is_full {} { detail_check ; expr {$::detail == "full"} }
 
-proc foreach_tokenizer_mode {prefix script} {
-  set saved $::testprefix
-  foreach {d mapping} {
-    ""              {}
-    "-origintext"   {, tokenize="origintext unicode61", tokendata=1}
-  } {
-    set s [string map [list %TOKENIZER% $mapping] $script]
-    set ::testprefix "$prefix$d"
-    reset_db
-    sqlite3_fts5_register_origintext db
-    uplevel $s
-  }
-  set ::testprefix $saved
-}
 
 #-------------------------------------------------------------------------
 # Convert a poslist of the type returned by fts5_test_poslist() to a 
@@ -633,10 +594,6 @@ proc nearset_rc {aCol args} {
   list
 }
 
-proc dump {tname} {
-  execsql_pp "SELECT * FROM ${tname}_idx"
-  execsql_pp "SELECT id, quote(block), fts5_decode(id,block) FROM ${tname}_data"
-}
 
 #-------------------------------------------------------------------------
 # Code for a simple Tcl tokenizer that supports synonyms at query time.

ext/fts5/test/fts5aa.test

@@ -22,7 +22,6 @@ ifcapable !fts5 {
 }
 
 foreach_detail_mode $::testprefix {
-foreach_tokenizer_mode $::testprefix {
 
 do_execsql_test 1.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
@@ -45,7 +44,7 @@ do_execsql_test 1.1 {
 #
 
 do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
 }
 do_execsql_test 2.1 {
   INSERT INTO t1 VALUES('a b c', 'd e f');
@@ -66,17 +65,14 @@ foreach w {a b c d e f} {
 
 do_execsql_test 2.4 {
   INSERT INTO t1(t1) VALUES('integrity-check');
-  PRAGMA integrity_check;
-  PRAGMA integrity_check(t1);
-} {ok ok}
+}
 
 
 #-------------------------------------------------------------------------
 #
 reset_db
-sqlite3_fts5_register_origintext db
 do_execsql_test 3.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
 }
 foreach {i x y} {
    1  {g f d b f} {h h e i a}
@@ -92,16 +88,14 @@ foreach {i x y} {
 } {
   do_execsql_test 3.$i.1 { INSERT INTO t1 VALUES($x, $y) }
   do_execsql_test 3.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
-  do_execsql_test 3.$i.3 { PRAGMA integrity_check(t1) } ok
   if {[set_test_counter errors]} break
 }
 
 #-------------------------------------------------------------------------
 #
 reset_db
-sqlite3_fts5_register_origintext db
 do_execsql_test 4.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
   INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
 }
 foreach {i x y} {
@@ -124,9 +118,8 @@ foreach {i x y} {
 #-------------------------------------------------------------------------
 #
 reset_db
-sqlite3_fts5_register_origintext db
 do_execsql_test 5.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
   INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
 }
 foreach {i x y} {
@@ -142,16 +135,15 @@ foreach {i x y} {
    10 {ddd abcde dddd dd c} {dddd c c d abcde}
 } {
   do_execsql_test 5.$i.1 { INSERT INTO t1 VALUES($x, $y) }
-  do_execsql_test 5.$i.2 { PRAGMA integrity_check(t1) } ok
+  do_execsql_test 5.$i.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
   if {[set_test_counter errors]} break
 }
 
 #-------------------------------------------------------------------------
 #
 reset_db
-sqlite3_fts5_register_origintext db
 do_execsql_test 6.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
   INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
 }
 
@@ -186,7 +178,6 @@ do_execsql_test 6.6 {
 #-------------------------------------------------------------------------
 #
 reset_db
-sqlite3_fts5_register_origintext db
 expr srand(0)
 do_execsql_test 7.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(x,y,z);
@@ -228,7 +219,6 @@ for {set i 1} {$i <= 10} {incr i} {
 #-------------------------------------------------------------------------
 #
 reset_db
-sqlite3_fts5_register_origintext db
 do_execsql_test 8.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(x, prefix="1,2,3");
   INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
@@ -243,7 +233,6 @@ do_execsql_test 8.1 {
 #-------------------------------------------------------------------------
 #
 reset_db
-sqlite3_fts5_register_origintext db
 
 expr srand(0)
 
@@ -288,9 +277,8 @@ for {set i 1} {$i <= 10} {incr i} {
 #-------------------------------------------------------------------------
 #
 reset_db
-sqlite3_fts5_register_origintext db
 do_execsql_test 10.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x,y, detail=%DETAIL%);
 }
 set d10 {
    1  {g f d b f} {h h e i a}
@@ -323,19 +311,19 @@ do_execsql_test 10.4.2 { INSERT INTO t1(t1) VALUES('integrity-check') }
 #-------------------------------------------------------------------------
 #
 do_catchsql_test 11.1 {
-  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rank, detail=%DETAIL%);
 } {1 {reserved fts5 column name: rank}}
 do_catchsql_test 11.2 {
-  CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE rank USING fts5(a, b, c, detail=%DETAIL%);
 } {1 {reserved fts5 table name: rank}}
 do_catchsql_test 11.3 {
-  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE t2 USING fts5(a, b, c, rowid, detail=%DETAIL%);
 } {1 {reserved fts5 column name: rowid}}
 
 #-------------------------------------------------------------------------
 #
 do_execsql_test 12.1 {
-  CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE t2 USING fts5(x,y, detail=%DETAIL%);
 } {}
 
 do_catchsql_test 12.2 {
@@ -350,9 +338,8 @@ do_test 12.3 {
 #-------------------------------------------------------------------------
 #
 reset_db
-sqlite3_fts5_register_origintext db
 do_execsql_test 13.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, detail=%DETAIL%);
   INSERT INTO t1(rowid, x) VALUES(1, 'o n e'), (2, 't w o');
 } {}
 
@@ -375,9 +362,8 @@ do_execsql_test 13.6 {
 #-------------------------------------------------------------------------
 #
 reset_db
-sqlite3_fts5_register_origintext db
 do_execsql_test 14.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y, detail=%DETAIL%);
   INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
   WITH d(x,y) AS (
     SELECT NULL, 'xyz xyz xyz xyz xyz xyz'
@@ -440,7 +426,7 @@ do_execsql_test 16.1 {
 proc funk {} {
   db eval { UPDATE n1_config SET v=50 WHERE k='version' }
   set fd [db incrblob main n1_data block 10]
-  fconfigure $fd -translation binary
+  fconfigure $fd -encoding binary -translation binary
 #  puts -nonewline $fd "\x44\x45"
   close $fd
 }
@@ -453,16 +439,15 @@ db func funk funk
 # statement no longer fails.
 #
 do_catchsql_test 16.2 {
-  SELECT funk(), format('%g',bm25(n1)), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
+  SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
 } {0 {{} -1e-06 {}}}
 # {1 {SQL logic error}}
 
 #-------------------------------------------------------------------------
 #
 reset_db
-sqlite3_fts5_register_origintext db
 do_execsql_test 17.1 {
-  CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE b2 USING fts5(x, detail=%DETAIL%);
   INSERT INTO b2 VALUES('a');
   INSERT INTO b2 VALUES('b');
   INSERT INTO b2 VALUES('c');
@@ -478,9 +463,8 @@ do_test 17.2 {
 
 if {[string match n* %DETAIL%]==0} {
   reset_db
-  sqlite3_fts5_register_origintext db
   do_execsql_test 17.3 {
-    CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL% %TOKENIZER%);
+    CREATE VIRTUAL TABLE c2 USING fts5(x, y, detail=%DETAIL%);
     INSERT INTO c2 VALUES('x x x', 'x x x');
     SELECT rowid FROM c2 WHERE c2 MATCH 'y:x';
   } {1}
@@ -489,9 +473,8 @@ if {[string match n* %DETAIL%]==0} {
 #-------------------------------------------------------------------------
 #
 reset_db
-sqlite3_fts5_register_origintext db
 do_execsql_test 17.1 {
-  CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE uio USING fts5(ttt, detail=%DETAIL%);
   INSERT INTO uio VALUES(NULL);
   INSERT INTO uio SELECT NULL FROM uio;
   INSERT INTO uio SELECT NULL FROM uio;
@@ -538,8 +521,8 @@ do_execsql_test 17.9 {
 #--------------------------------------------------------------------
 #
 do_execsql_test 18.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL% %TOKENIZER%);
-  CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE t1 USING fts5(a, b, detail=%DETAIL%);
+  CREATE VIRTUAL TABLE t2 USING fts5(c, d, detail=%DETAIL%);
   INSERT INTO t1 VALUES('abc*', NULL);
   INSERT INTO t2 VALUES(1, 'abcdefg');
 }
@@ -554,9 +537,8 @@ do_execsql_test 18.3 {
 # fts5 table in the temp schema.
 #
 reset_db
-sqlite3_fts5_register_origintext db
 do_execsql_test 19.0 {
-  CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE temp.t1 USING fts5(x, detail=%DETAIL%);
   INSERT INTO t1 VALUES('x y z');
   INSERT INTO t1 VALUES('w x 1');
   SELECT rowid FROM t1 WHERE t1 MATCH 'x';
@@ -566,9 +548,8 @@ do_execsql_test 19.0 {
 # Test that 6 and 7 byte varints can be read.
 #
 reset_db
-sqlite3_fts5_register_origintext db
 do_execsql_test 20.0 {
-  CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE temp.tmp USING fts5(x, detail=%DETAIL%);
 }
 set ::ids [list \
   0 [expr 1<<36] [expr 2<<36] [expr 1<<43] [expr 2<<43]
@@ -586,7 +567,7 @@ do_test 20.1 {
 #
 do_execsql_test 21.0 {
   CREATE TEMP TABLE t8(a, b);
-  CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE ft USING fts5(x, detail=%DETAIL%);
 }
 
 do_execsql_test 21.1 {
@@ -597,7 +578,7 @@ do_execsql_test 21.1 {
 }
 
 do_execsql_test 22.0 {
-  CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE t9 USING fts5(x, detail=%DETAIL%);
   INSERT INTO t9(rowid, x) VALUES(2, 'bbb');
   BEGIN;
     INSERT INTO t9(rowid, x) VALUES(1, 'aaa');
@@ -612,7 +593,7 @@ do_execsql_test 22.1 {
 
 #-------------------------------------------------------------------------
 do_execsql_test 23.0 {
-  CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE t10 USING fts5(x, detail=%DETAIL%);
   CREATE TABLE t11(x);
 }
 do_execsql_test 23.1 {
@@ -624,7 +605,7 @@ do_execsql_test 23.2 {
 
 #-------------------------------------------------------------------------
 do_execsql_test 24.0 {
-  CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE t12 USING fts5(x, detail=%DETAIL%);
   INSERT INTO t12 VALUES('aaaa');
 }
 do_execsql_test 24.1 {
@@ -634,9 +615,6 @@ do_execsql_test 24.1 {
     INSERT INTO t12 VALUES('aaaa');
   END;
 }
-execsql_pp {
-  SELECT rowid, hex(block) FROM t12_data
-}
 do_execsql_test 24.2 {
   INSERT INTO t12(t12) VALUES('integrity-check');
 }
@@ -646,7 +624,7 @@ do_execsql_test 24.3 {
 
 #-------------------------------------------------------------------------
 do_execsql_test 25.0 {
-  CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL% %TOKENIZER%);
+  CREATE VIRTUAL TABLE t13 USING fts5(x, detail=%DETAIL%);
 }
 do_execsql_test 25.1 {
   BEGIN;
@@ -657,7 +635,6 @@ SELECT * FROM t13('BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB
 }
 
 
-}
 }
 
 expand_all_sql db

ext/fts5/test/fts5ab.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ab
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -180,11 +180,7 @@ if {[detail_is_full]} {
   } {1 2}
 }
 
-do_execsql_test 4.5.1 {
-  SELECT rowid FROM s1 WHERE s1 MATCH 'a AND x'
-} {1 2}
-
-do_execsql_test 4.5.2 {
+do_execsql_test 4.5 {
   SELECT rowid FROM s1 WHERE s1 MATCH 'a x'
 } {1 2}
 

ext/fts5/test/fts5ac.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ac
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5ad.test

@@ -18,7 +18,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ad
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5ae.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ae
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5af.test

@@ -18,7 +18,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5af
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -193,34 +193,4 @@ do_execsql_test 5.6 {
 
 } ;# foreach_detail_mode 
 
-reset_db
-do_execsql_test 6.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(colA, colB);
-  INSERT INTO t1 VALUES('A B C', 'D E F');
-}
-
-do_execsql_test 6.1 {
-  SELECT colA, colB, snippet(t1,0,'[', ']','...',1) FROM t1 WHERE t1 MATCH 'B';
-} {{A B C} {D E F} ...[B]...}
-breakpoint
-do_execsql_test 6.2 {
-  SELECT colA, colB, snippet(t1, 1,'[',']','...',2) FROM t1 WHERE t1 MATCH 'B';
-} {{A B C} {D E F} {D E...}}
-do_execsql_test 6.3 {
-  SELECT colA, colB, snippet(t1, 1,'[',']','...',1) FROM t1 WHERE t1 MATCH 'B';
-} {{A B C} {D E F} {D...}}
-
-do_execsql_test 6.1 {
-  SELECT colA, colB, snippet(t1,0,'[', ']','...',1) FROM t1 WHERE t1 MATCH 'A';
-} {{A B C} {D E F} [A]...}
-breakpoint
-do_execsql_test 6.2 {
-  SELECT colA, colB, snippet(t1, 1,'[',']','...',2) FROM t1 WHERE t1 MATCH 'A';
-} {{A B C} {D E F} {D E...}}
-do_execsql_test 6.3 {
-  SELECT colA, colB, snippet(t1, 1,'[',']','...',1) FROM t1 WHERE t1 MATCH 'A';
-} {{A B C} {D E F} {D...}}
-
-
-
 finish_test

ext/fts5/test/fts5ag.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ag
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5ah.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ah
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5ai.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ai
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5aj.test

@@ -19,7 +19,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5aj
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5ak.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ak
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -154,30 +154,4 @@ do_execsql_test 3.2 {
 
 }
 
-# 2023-04-06 https://sqlite.org/forum/forumpost/cae4367d9b
-#
-# This is not a test of FTS5, but rather a test of the of what happens to
-# prepared statements that encounter SQLITE_SCHEMA while other prepared
-# statements are running.  The original problem POC used FTS5, and so
-# is seems reasonable to put the test here.
-#
-# The vdbeaux24.test module in TH3 also tests this same behavior but
-# without requiring FTS5 or an other extension.
-#
-reset_db
-db null NULL
-do_execsql_test 4.0 {
-  CREATE TABLE t5(a PRIMARY KEY);
-  INSERT INTO t5 VALUES(0);
-  CREATE VIRTUAL TABLE t6 USING fts5(0);
-  DELETE FROM t6;
-  CREATE TABLE t7(x);
-  WITH cte(a) AS (
-     SELECT a FROM t5
-      WHERE ((0,0) IN (SELECT 0, LAG(0) OVER (PARTITION BY 0) FROM t6), 0)
-               < (a,0)
-  )
-  SELECT max(a) FROM cte;
-} NULL
-
 finish_test

ext/fts5/test/fts5al.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5al
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5alter.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5alter
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5auto.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5auto
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5aux.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5aux
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -307,98 +307,5 @@ do_catchsql_test 10.1.4 {
   SELECT group_concat(firstcol(t1), '.') FROM t1 GROUP BY rowid
 } {1 {unable to use function firstcol in the requested context}}
 
-#-------------------------------------------------------------------------
-# Test that xInstCount() works from within an xPhraseQuery() callback.
-#
-reset_db
-
-proc xCallback {cmd} {
-  incr ::hitcount [$cmd xInstCount]
-  return SQLITE_OK
-}
-proc fts5_hitcount {cmd} {
-  set ::hitcount 0
-  $cmd xQueryPhrase 0 xCallback
-  return $::hitcount
-}
-sqlite3_fts5_create_function db fts5_hitcount fts5_hitcount
-
-do_execsql_test 11.1 {
-  CREATE VIRTUAL TABLE x1 USING fts5(z);
-  INSERT INTO x1 VALUES('one two three');
-  INSERT INTO x1 VALUES('one two one three one');
-  INSERT INTO x1 VALUES('one two three');
-}
-
-do_execsql_test 11.2 {
-  SELECT fts5_hitcount(x1) FROM x1('one') LIMIT 1;
-} {5}
-
-#-------------------------------------------------------------------------
-# Test that xColumnText returns SQLITE_RANGE when it should.
-#
-reset_db
-fts5_aux_test_functions db
-do_execsql_test 12.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c);
-  INSERT INTO t1 VALUES('one', 'two', 'three');
-  INSERT INTO t1 VALUES('one', 'one', 'one');
-  INSERT INTO t1 VALUES('two', 'two', 'two');
-  INSERT INTO t1 VALUES('three', 'three', 'three');
-}
-
-do_catchsql_test 12.1.1 {
-  SELECT fts5_columntext(t1, -1) FROM t1('two');
-} {1 SQLITE_RANGE}
-do_catchsql_test 12.1.2 {
-  SELECT fts5_columntext(t1, 3) FROM t1('two');
-} {1 SQLITE_RANGE}
-do_catchsql_test 12.1.2 {
-  SELECT fts5_columntext(t1, 1) FROM t1('one AND two');
-} {0 two}
-
-do_catchsql_test 12.2.1 {
-  SELECT fts5_queryphrase(t1, -1) FROM t1('one AND two');
-} {1 SQLITE_RANGE}
-do_catchsql_test 12.2.2 {
-  SELECT fts5_queryphrase(t1, 2) FROM t1('one AND two');
-} {1 SQLITE_RANGE}
-do_catchsql_test 12.2.3 {
-  SELECT fts5_queryphrase(t1, 1) FROM t1('one AND two');
-} {0 {{1 2 1}}}
-
-do_catchsql_test 12.3.1 {
-  SELECT fts5_collist(t1, -1) FROM t1('one AND two');
-} {1 SQLITE_RANGE}
-do_catchsql_test 12.3.2 {
-  SELECT fts5_collist(t1, 2) FROM t1('one AND two');
-} {1 SQLITE_RANGE}
-do_catchsql_test 12.3.3 {
-  SELECT fts5_collist(t1, 1) FROM t1('one AND two');
-} {0 1}
-
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 13.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=ascii);
-  INSERT INTO t1 VALUES('a b c'), ('d e f');
-  PRAGMA integrity_check;
-} {ok}
-
-do_catchsql_test 13.2 {
-  SELECT highlight(t1, 0, '[', ']') FROM t1
-} {0 {{a b c} {d e f}}}
-
-do_execsql_test 13.3 {
-  PRAGMA writable_schema = 1;
-  UPDATE sqlite_schema SET sql = 'CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=blah)'
-  WHERE name = 't1';
-}
-
-db close
-sqlite3 db test.db
-do_catchsql_test 13.4 {
-  SELECT highlight(t1, 0, '[', ']') FROM t1
-} {1 {SQL logic error}}
-
 finish_test
+

ext/fts5/test/fts5aux2.test

@@ -1,71 +0,0 @@
-# 2024 June 24
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-#
-# Tests focusing on the auxiliary function APIs.
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5aux
-
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
-  INSERT INTO x1 VALUES('a b', 'c d');
-  INSERT INTO x1 VALUES('d e', 'a b');
-  INSERT INTO x1 VALUES('a b', 'e f');
-  INSERT INTO x1 VALUES('d e', 'c d');
-}
-
-fts5_aux_test_functions db
-do_execsql_test 1.1 {
-  SELECT fts5_test_all(x1) FROM x1 WHERE rowid=2
-} [list [list {*}{
-  columnsize {2 2}
-  columntext {{d e} {a b}}
-  columntotalsize {8 8}
-  poslist {}
-  tokenize {{d e} {a b}}
-  rowcount 4
-}]]
-
-do_execsql_test 1.2 {
-  SELECT fts5_test_columntext(x1) FROM x1
-} {
-  {{a b} {c d}}
-  {{d e} {a b}}
-  {{a b} {e f}}
-  {{d e} {c d}}
-}
-
-do_execsql_test 1.3 {
-  SELECT fts5_test_rowid(x1) FROM x1
-} {
-  1 2 3 4
-}
-do_execsql_test 1.4 {
-  SELECT fts5_test_phrasecount(x1) FROM x1
-} {
-  0 0 0 0
-}
-do_catchsql_test 1.5 {
-  SELECT fts5_queryphrase(x1, 0) FROM x1
-} {1 SQLITE_RANGE} 
-do_execsql_test 1.6 {
-  SELECT fts5_test_rowcount(x1) FROM x1
-} {4 4 4 4}
-
-
-finish_test

ext/fts5/test/fts5auxdata.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5auxdata
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5bigid.test

@@ -1,62 +0,0 @@
-# 2023 May 28
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5bigid
-
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-
-set nRow 20000
-
-proc do_ascdesc_test {tn query} {
-  set ::lAsc  [db eval { SELECT rowid FROM x1($query) }] 
-  set ::lDesc [db eval { SELECT rowid FROM x1($query) ORDER BY rowid DESC }] 
-  do_test $tn.1 { lsort -integer $::lAsc } $::lAsc
-  do_test $tn.2 { lsort -integer -decr $::lDesc } $::lDesc
-  do_test $tn.3 { lsort -integer $::lDesc } $::lAsc
-}
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE x1 USING fts5(a);
-}
-
-do_test 1.1 {
-  for {set ii 0} {$ii < $nRow} {incr ii} {
-    db eval {
-      REPLACE INTO x1(rowid, a) VALUES(random(), 'movement at the station');
-    }
-  }
-} {}
-
-do_ascdesc_test 1.2 "the"
-
-do_execsql_test 1.3 {
-  DELETE FROM x1
-}
-
-do_test 1.4 {
-  for {set ii 0} {$ii < $nRow} {incr ii} {
-    db eval {
-      INSERT INTO x1(rowid, a) VALUES(
-          $ii + 0x6FFFFFFFFFFFFFFF, 'movement at the station'
-      );
-    }
-  }
-} {}
-
-do_ascdesc_test 1.5 "movement"
-
-finish_test

ext/fts5/test/fts5bigpl.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5bigpl
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5colset.test

@@ -79,7 +79,7 @@ foreach_detail_mode $::testprefix {
 
   do_catchsql_test 4.1 {
     SELECT * FROM t1 WHERE rowid MATCH 'a'
-  } {1 {no query solution}}
+  } {1 {unable to use function MATCH in the requested context}}
 }
 
 #-------------------------------------------------------------------------

ext/fts5/test/fts5columnsize.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5columnsize
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5config.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5config
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5conflict.test

@@ -65,44 +65,4 @@ do_execsql_test 2.1 {
   INSERT INTO fts_idx(fts_idx) VALUES('integrity-check');
 }
 
-#-------------------------------------------------------------------------
-# Tests for OR IGNORE conflict handling.
-#
-reset_db
-foreach_detail_mode $::testprefix {
-
-  do_execsql_test 3.0 {
-    CREATE VIRTUAL TABLE t1 USING fts5(xyz, detail=%DETAIL%);
-
-    BEGIN;
-    INSERT INTO t1(rowid, xyz) VALUES(13, 'thirteen documents');
-    INSERT INTO t1(rowid, xyz) VALUES(14, 'fourteen documents');
-    INSERT INTO t1(rowid, xyz) VALUES(15, 'fifteen documents');
-    COMMIT;
-  }
-
-  set db_cksum [cksum]
-    foreach {tn sql} {
-    1 {
-      INSERT OR IGNORE INTO t1(rowid, xyz) VALUES(14, 'new text');
-    }
-    2 {
-      UPDATE OR IGNORE t1 SET rowid=13 WHERE rowid=15;
-    }
-    3 {
-      INSERT OR IGNORE INTO t1(rowid, xyz) 
-      SELECT 13, 'some text'
-      UNION ALL
-      SELECT 14, 'some text'
-      UNION ALL
-      SELECT 15, 'some text'
-    }
-  } {
-    do_execsql_test 3.1.$tn.1 $sql
-    do_test 3.1.$tn.2 { cksum } $db_cksum
-  }
-
-}
-
-
 finish_test

ext/fts5/test/fts5content.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5content
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -293,76 +293,5 @@ do_catchsql_test 7.2.5 {
   SELECT * FROM t1('abc') ORDER BY rank;
 } {1 {recursively defined fts5 content table}}
 
-#---------------------------------------------------------------------------
-# Check that if the content table is a view, and that view contains an
-# error, a reasonable error message is returned if the user tries to
-# read from the view via the fts5 table.
-#
-reset_db
-do_execsql_test 8.1 {
-  CREATE VIEW a1 AS 
-    SELECT 1 AS r, text_value(1) AS t
-      UNION ALL
-    SELECT 2 AS r, text_value(2) AS t;
-
-  CREATE VIRTUAL TABLE t1 USING fts5(t, content='a1', content_rowid='r');
-}
-
-foreach {tn sql} {
-  1 "SELECT * FROM t1"
-  2 "INSERT INTO t1(t1) VALUES('rebuild')"
-  3 "SELECT * FROM t1 WHERE rowid=1"
-} {
-  do_catchsql_test 8.2.$tn $sql {1 {no such function: text_value}}
-}
-
-proc text_value {i} {
-  if {$i==1} { return "one" }
-  if {$i==2} { return "two" }
-  return "many"
-}
-db func text_value text_value
-
-do_execsql_test 8.3.1 { SELECT * FROM t1 } {one two}
-do_execsql_test 8.3.2 { INSERT INTO t1(t1) VALUES('rebuild') }
-do_execsql_test 8.3.3 { SELECT * FROM t1 WHERE rowid=1 } {one}
-do_execsql_test 8.3.4 { SELECT rowid FROM t1('two') } {2}
-
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 9.1 {
-  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
-  INSERT INTO t1 VALUES(1, 'one two three');
-  INSERT INTO t1 VALUES(2, 'one two three');
-
-  CREATE VIRTUAL TABLE ft USING fts5(b, content=t1, content_rowid=a);
-  INSERT INTO ft(ft) VALUES('rebuild');
-}
-
-do_execsql_test 9.2 {
-  SELECT rowid, b FROM ft('two');
-} {
-  1 {one two three}
-  2 {one two three}
-}
-
-do_execsql_test 9.3 {
-  DELETE FROM t1 WHERE a=2;
-} 
-
-do_catchsql_test 9.4 {
-  SELECT rowid FROM ft('two');
-} {0 {1 2}}
-
-do_catchsql_test 9.5 {
-  SELECT * FROM ft('two');
-} {1 {fts5: missing row 2 from content table 'main'.'t1'}}
-
-fts5_aux_test_functions db
-
-do_catchsql_test 9.6 {
-  SELECT rowid, fts5_columntext(ft, 0) FROM ft('two');
-} {1 SQLITE_CORRUPT_VTAB}
-
 finish_test
 

ext/fts5/test/fts5contentless.test

@@ -1,270 +0,0 @@
-# 2014 Dec 20
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-#
-# This file contains tests for the content= and content_rowid= options.
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5contentless
-
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-
-# Check that it is not possible to specify "contentless_delete=1" for 
-# anything other than a contentless table.
-#
-set res(0) {0 {}}
-set res(1) {1 {contentless_delete=1 requires a contentless table}}
-foreach {tn sql bError} {
-  1 "(a, b, contentless_delete=1)"              1
-  2 "(a, b, contentless_delete=1, content=abc)" 1
-  3 "(a, b, contentless_delete=1, content=)"    0
-  4 "(content=, contentless_delete=1, a)"       0
-  5 "(content='', contentless_delete=1, hello)" 0
-} {
-  execsql { BEGIN }
-  do_catchsql_test 1.$tn "CREATE VIRTUAL TABLE t1 USING fts5 $sql" $res($bError)
-  execsql { ROLLBACK }
-}
-
-# Check that it is not possible to specify "contentless_delete=1" 
-# along with columnsize=1. 
-#
-set res(0) {0 {}}
-set res(1) {1 {contentless_delete=1 is incompatible with columnsize=0}}
-foreach {tn sql bError} {
-  2 "(a, b, content='', contentless_delete=1, columnsize=0)" 1 
-} {
-  execsql { BEGIN }
-  do_catchsql_test 1.$tn "CREATE VIRTUAL TABLE t1 USING fts5 $sql" $res($bError)
-  execsql { ROLLBACK }
-}
-
-# Check that if contentless_delete=1 is specified, then the "origin" 
-# column is added to the %_docsize table.
-reset_db
-do_execsql_test 3.0 {
-  CREATE VIRTUAL TABLE x1 USING fts5(c, content='');
-  CREATE VIRTUAL TABLE x2 USING fts5(c, content='', contentless_delete=1);
-}
-do_execsql_test 3.1 {
-  SELECT sql FROM sqlite_schema WHERE name IN ('x1_docsize', 'x2_docsize');
-} {
-  {CREATE TABLE 'x1_docsize'(id INTEGER PRIMARY KEY, sz BLOB)} 
-  {CREATE TABLE 'x2_docsize'(id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER)}
-}
-
-do_execsql_test 3.2.1 {
-  SELECT hex(block) FROM x1_data WHERE id=10
-} {00000000000000}
-do_execsql_test 3.2.2 {
-  SELECT hex(block) FROM x2_data WHERE id=10
-} {00000000FF000001000000}
-
-do_execsql_test 3.3 {
-  INSERT INTO x2 VALUES('first text');
-  INSERT INTO x2 VALUES('second text');
-}
-do_execsql_test 3.4 {
-  SELECT id, origin FROM x2_docsize
-} {1 1 2 2}
-do_execsql_test 3.5 {
-  SELECT level, segment, loc1, loc2 FROM fts5_structure(
-    (SELECT block FROM x2_data WHERE id=10)
-  )
-} {
-  0 0   1 1
-  0 1   2 2
-}
-do_execsql_test 3.6 {
-  INSERT INTO x2(x2) VALUES('optimize');
-}
-do_execsql_test 3.7 {
-  SELECT level, segment, loc1, loc2 FROM fts5_structure(
-    (SELECT block FROM x2_data WHERE id=10)
-  )
-} {
-  1 0   1 2
-}
-
-do_execsql_test 3.8 {
-  DELETE FROM x2 WHERE rowid=2;
-}
-
-do_execsql_test 3.9 {
-  SELECT rowid FROM x2('text')
-} {1}
-
-#--------------------------------------------------------------------------
-reset_db
-proc document {n} {
-  set vocab [list A B C D E F G H I J K L M N O P Q R S T U V W X Y Z]
-  set ret [list]
-  for {set ii 0} {$ii < $n} {incr ii} {
-    lappend ret [lindex $vocab [expr int(rand()*[llength $vocab])]]
-  }
-  set ret
-}
-
-set nRow 1000
-
-do_execsql_test 4.0 {
-  CREATE TABLE t1(x);
-  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
-  INSERT INTO ft(ft, rank) VALUES('pgsz', 100);
-}
-do_test 4.1 {
-  for {set ii 0} {$ii < $nRow} {incr ii} {
-    set doc [document 6]
-    execsql {
-      INSERT INTO t1 VALUES($doc);
-      INSERT INTO ft VALUES($doc);
-    }
-  }
-} {}
-
-foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} {
-  set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}]
-  set L2 [execsql {SELECT rowid FROM ft($v)}]
-  do_test 4.2.$v { set L1 } $L2
-}
-
-do_test 4.3 {
-  for {set ii 1} {$ii < $nRow} {incr ii 2} { 
-    execsql {
-       DELETE FROM ft WHERE rowid=$ii;
-       DELETE FROM t1 WHERE rowid=$ii;
-    }
-  }
-} {}
-
-foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} {
-  set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}]
-  set L2 [execsql {SELECT rowid FROM ft($v)}]
-  do_test 4.4.$v { set L1 } $L2
-}
-
-do_execsql_test 4.5 {
-  INSERT INTO ft(ft) VALUES('optimize');
-} {}
-
-foreach v {A B C D E F G H I J K L M N O P Q R S T U V W X Y Z} {
-  set L1 [execsql {SELECT rowid FROM t1 WHERE x LIKE '%'||$v||'%'}]
-  set L2 [execsql {SELECT rowid FROM ft($v)}]
-  do_test 4.6.$v { set L1 } $L2
-}
-
-#execsql_pp { SELECT fts5_decode(id, block) FROM ft_data }
-
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 5.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
-  INSERT INTO ft(rowid, x) VALUES(1, 'one two three');
-  INSERT INTO ft(rowid, x) VALUES(2, 'one two four');
-  INSERT INTO ft(rowid, x) VALUES(3, 'one two five');
-  INSERT INTO ft(rowid, x) VALUES(4, 'one two seven');
-  INSERT INTO ft(rowid, x) VALUES(5, 'one two eight');
-}
-
-do_execsql_test 5.1 {
-  DELETE FROM ft WHERE rowid=2
-}
-
-do_execsql_test 5.2 {
-  SELECT rowid FROM ft
-} {1 3 4 5}
-
-do_catchsql_test 5.3 {
-  UPDATE ft SET x='four six' WHERE rowid=3
-} {0 {}}
-
-do_execsql_test 5.4 {
-  SELECT rowid FROM ft('one');
-} {1 4 5}
-
-do_execsql_test 5.5 {
-  REPLACE INTO ft(rowid, x) VALUES(3, 'four six');
-  SELECT rowid FROM ft('one');
-} {1 4 5}
-
-do_execsql_test 5.6 {
-  REPLACE INTO ft(rowid, x) VALUES(6, 'one two eleven');
-  SELECT rowid FROM ft('one');
-} {1 4 5 6}
-
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 6.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
-  INSERT INTO ft(rowid, x) VALUES(1, 'one two three');
-  INSERT INTO ft(rowid, x) VALUES(2, 'one two four');
-}
-
-do_test 6.1 {
-  db eval { SELECT rowid FROM ft('one two') } {
-    if {$rowid==1} {
-      db eval { INSERT INTO ft(rowid, x) VALUES(3, 'one two four') }
-    }
-  }
-} {}
-
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 7.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
-}
-
-set lRowid [list -450 0 1 2 42]
-
-do_test 7.1 {
-  execsql BEGIN
-  foreach r $lRowid {
-    execsql { INSERT INTO ft(rowid, x) VALUES($r, 'one one one'); }
-  }
-  execsql COMMIT
-} {}
-
-do_test 7.2 {
-  execsql BEGIN
-  foreach r $lRowid {
-    execsql { REPLACE INTO ft(rowid, x) VALUES($r, 'two two two'); }
-  }
-  execsql COMMIT
-} {}
-
-do_execsql_test 7.3 { SELECT rowid FROM ft('one'); } {}
-do_execsql_test 7.4 { SELECT rowid FROM ft('two'); } $lRowid
-
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 8.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
-  INSERT INTO ft VALUES('hello world');
-  INSERT INTO ft VALUES('one two three');
-}
-
-do_catchsql_test 8.1 {
-  INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, 'hello world');
-} {1 {'delete' may not be used with a contentless_delete=1 table}}
-
-do_execsql_test 8.2 {
-  BEGIN;
-    INSERT INTO ft(rowid, x) VALUES(3, 'four four four');
-    DELETE FROM ft WHERE rowid=3;
-  COMMIT;
-  SELECT rowid FROM ft('four');
-} {}
-
-finish_test

ext/fts5/test/fts5contentless2.test

@@ -1,207 +0,0 @@
-# 2023 July 19
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-#
-# This file contains tests for the content= and content_rowid= options.
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5contentless2
-
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-
-proc vocab {} {
-  list aaa bbb ccc ddd eee fff ggg hhh iii jjj kkk lll mmm nnn ooo ppp
-}
-
-proc document {nToken} {
-  set doc [list]
-  set vocab [vocab]
-  for {set ii 0} {$ii < $nToken} {incr ii} {
-    lappend doc [lindex $vocab [expr int(rand()*[llength $vocab])]]
-  }
-  set doc
-}
-db func document document
-
-proc contains {doc token} {
-  expr {[lsearch $doc $token]>=0}
-}
-db func contains contains
-
-proc do_compare_tables_test {tn} {
-  uplevel [list do_test $tn {
-    foreach v [vocab] {
-      set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $v) }]
-      set l2 [execsql { SELECT rowid FROM t2($v) }]
-      if {$l1!=$l2} { error "1: query mismatch ($l1) ($l2)" }
-
-      set w "[string range $v 0 1]*"
-      set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $w) }]
-      set l2 [execsql { SELECT rowid FROM t2($w) }]
-      if {$l1!=$l2} { error "2: query mismatch ($l1) ($l2)" }
-
-      set w "[string range $v 0 0]*"
-      set l1 [execsql { SELECT rowid FROM t1 WHERE contains(doc, $w) }]
-      set l2 [execsql { SELECT rowid FROM t2($w) }]
-      if {$l1!=$l2} { error "2: query mismatch ($l1) ($l2)" }
-
-      set l1 [execsql { 
-        SELECT rowid FROM t1 WHERE contains(doc, $v) ORDER BY rowid DESC 
-      }]
-      set l2 [execsql { SELECT rowid FROM t2($v) ORDER BY rowid DESC }]
-      if {$l1!=$l2} { error "1: query mismatch ($l1) ($l2)" }
-    }
-    set {} {}
-  } {}]
-}
-
-proc lshuffle {in} {
-  set L [list]
-  set ret [list]
-  foreach elem $in { lappend L [list [expr rand()] $elem] }
-  foreach pair [lsort -index 0 $L] { lappend ret [lindex $pair 1] }
-  set ret
-}
-
-expr srand(0)
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE t2 USING fts5(
-      doc, prefix=2, content=, contentless_delete=1
-  );
-
-  CREATE TABLE t1(doc);
-  CREATE TRIGGER tr1 AFTER DELETE ON t1 BEGIN
-    DELETE FROM t2 WHERE rowid = old.rowid;
-  END;
-}
-
-set SMALLEST64 -9223372036854775808
-set LARGEST64   9223372036854775807
-
-foreach {tn r1 r2} {
-  1   0               50
-  2   $SMALLEST64     $SMALLEST64+50
-  3   $LARGEST64-50   $LARGEST64
-  4   -50             -1
-} {
-  set r1 [expr $r1]
-  set r2 [expr $r2]
-
-  do_test 1.1.$tn {
-    execsql BEGIN
-    for {set ii $r1} {$ii <= $r2} {incr ii} {
-      execsql { INSERT INTO t1(rowid, doc) VALUES ($ii, document(8)); }
-    }
-    execsql COMMIT
-  } {}
-}
-do_test 1.2 {
-  db eval { SELECT rowid, doc FROM t1 } {
-    execsql { INSERT INTO t2(rowid, doc) VALUES($rowid, $doc) }
-  }
-} {}
-
-foreach {tn rowid} {
-  1  $SMALLEST64
-  2  0
-  3  -5
-  4  -30
-  5  $LARGEST64
-  6  $LARGEST64-1
-} {
-  set rowid [expr $rowid]
-  do_execsql_test 1.3.$tn.1 {
-    DELETE FROM t1 WHERE rowid=$rowid
-  }
-  do_compare_tables_test 1.3.$tn.2
-}
-
-set iTest 1
-foreach r [lshuffle [execsql {SELECT rowid FROM t1}]] {
-  if {($iTest % 50)==0} {
-    execsql { INSERT INTO t2(t2) VALUES('optimize') }
-  }
-  if {($iTest % 5)==0} {
-    execsql { INSERT INTO t2(t2, rank) VALUES('merge', 5) }
-  }
-  do_execsql_test 1.4.$iTest.1($r) {
-    DELETE FROM t1 WHERE rowid=$r
-  }
-  do_compare_tables_test 1.4.$iTest.2
-  incr iTest
-}
-
-do_execsql_test 1.5 {
-  SELECT * FROM t1
-} {}
-
-#-------------------------------------------------------------------------
-reset_db
-db func document document
-
-do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE t2 USING fts5(doc, content=, contentless_delete=1);
-  WITH s(i) AS (
-    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
-  )
-  INSERT INTO t2(rowid, doc) SELECT i, i || ' ' || i FROM s;
-}
-
-do_execsql_test 2.1 {
-  BEGIN;
-    DELETE FROM t2 WHERE rowid=32;
-    DELETE FROM t2 WHERE rowid=64;
-    DELETE FROM t2 WHERE rowid=96;
-    DELETE FROM t2 WHERE rowid=128;
-    DELETE FROM t2 WHERE rowid=160;
-    DELETE FROM t2 WHERE rowid=192;
-  COMMIT;
-}
-
-do_execsql_test 2.2 {
-  SELECT * FROM t2('128');
-} {}
-
-#-------------------------------------------------------------------------
-
-foreach {tn step} {
-  1     3 
-  2     7
-  3     15
-} {
-  set step [expr $step]
-
-  reset_db
-  db func document document
-  do_execsql_test 3.$tn.0 {
-    CREATE VIRTUAL TABLE t2 USING fts5(doc, content=, contentless_delete=1);
-    INSERT INTO t2(t2, rank) VALUES('pgsz', 100);
-    WITH s(i) AS (
-        SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
-    )
-    INSERT INTO t2(rowid, doc) SELECT i, i || ' ' || i FROM s;
-  }
-  do_execsql_test 3.$tn.1 {
-    DELETE FROM t2 WHERE (rowid % $step)==0
-  }
-  do_execsql_test 3.$tn.2 {
-    SELECT * FROM t2( $step * 5 )
-  } {}
-}
-
-
-
-finish_test

ext/fts5/test/fts5contentless3.test

@@ -1,195 +0,0 @@
-# 2023 July 21
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-#
-# This file contains tests for the content= and content_rowid= options.
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5contentless3
-
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1);
-  BEGIN;
-    INSERT INTO ft VALUES('one one one');
-    INSERT INTO ft VALUES('two two two');
-    INSERT INTO ft VALUES('three three three');
-    INSERT INTO ft VALUES('four four four');
-    INSERT INTO ft VALUES('five five five');
-    INSERT INTO ft VALUES('six six six');
-    INSERT INTO ft VALUES('seven seven seven');
-    INSERT INTO ft VALUES('eight eight eight');
-    INSERT INTO ft VALUES('nine nine nine');
-  COMMIT;
-
-  DELETE FROM ft WHERE rowid=3;
-}
-
-proc myhex {hex} { binary decode hex $hex }
-db func myhex myhex
-
-do_execsql_test 1.1 {
-  UPDATE ft_data SET block = 
-    myhex('04000000 00000001'                   ||
-          '01020304 01020304 01020304 01020304' ||
-          '01020304 01020304 01020304 01020304'
-    )
-  WHERE id = (SELECT max(id) FROM ft_data);
-}
-
-do_execsql_test 1.2 {
-  DELETE FROM ft WHERE rowid=1
-}
-
-do_execsql_test 1.3 {
-  SELECT rowid FROM ft('two');
-} {2}
-
-do_execsql_test 1.3 {
-  UPDATE ft_data SET block = 
-    myhex('08000000 00000001'                                               ||
-      '0000000001020304 0000000001020304 0000000001020304 0000000001020304' ||
-      '0000000001020304 0000000001020304 0000000001020304 0000000001020304'
-    )
-  WHERE id = (SELECT max(id) FROM ft_data);
-}
-
-do_execsql_test 1.4 {
-  SELECT rowid FROM ft('two');
-} {2}
-
-do_execsql_test 1.5 {
-  DELETE FROM ft WHERE rowid=4
-}
-
-do_execsql_test 1.6 {
-  UPDATE ft_data SET block = myhex('04000000 00000000')
-    WHERE id = (SELECT max(id) FROM ft_data);
-}
-do_execsql_test 1.7 {
-  SELECT rowid FROM ft('two');
-} {2}
-
-do_execsql_test 1.8 {
-  UPDATE ft_data SET block = myhex('04000000 00000000')
-    WHERE id = (SELECT max(id) FROM ft_data);
-}
-do_execsql_test 1.9 {
-  DELETE FROM ft WHERE rowid=8
-} {}
-
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1);
-  INSERT INTO ft VALUES('one one one');
-  INSERT INTO ft VALUES('two two two');
-  INSERT INTO ft VALUES('three three three');
-  INSERT INTO ft VALUES('four four four');
-  INSERT INTO ft VALUES('five five five');
-  INSERT INTO ft VALUES('six six six');
-  INSERT INTO ft VALUES('seven seven seven');
-  INSERT INTO ft VALUES('eight eight eight');
-  INSERT INTO ft VALUES('nine nine nine');
-}
-
-do_execsql_test 2.1 {
-  INSERT INTO ft(ft) VALUES('optimize');
-}
-do_execsql_test 2.2 {
-  SELECT count(*) FROM ft_data
-} {3}
-do_execsql_test 2.3 {
-  DELETE FROM ft WHERE rowid=5
-}
-do_execsql_test 2.4 {
-  SELECT count(*) FROM ft_data
-} {4}
-
-# Check that an 'optimize' works (rewrites the index) if there is a single
-# segment with one or more tombstone hash pages.
-do_execsql_test 2.5 {
-  INSERT INTO ft(ft) VALUES('optimize');
-}
-do_execsql_test 2.6 {
-  SELECT count(*) FROM ft_data
-} {3}
-
-# Check that an 'optimize' is a no-op if there is a single segment
-# and no tombstone hash pages.
-do_execsql_test 2.7 {
-  INSERT INTO ft(ft) VALUES('optimize');
-  SELECT rowid FROM ft_data;
-} [db eval {SELECT rowid FROM ft_data}]
-
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 3.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(x, content=, contentless_delete=1);
-  INSERT INTO ft(ft, rank) VALUES('pgsz', 64);
-  WITH s(i) AS (
-    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
-  )
-  INSERT INTO ft(rowid, x) SELECT i, i||' '||i||' '||i||' '||i FROM s;
-  INSERT INTO ft(ft) VALUES('optimize');
-}
-
-do_execsql_test 3.1 {
-  SELECT count(*) FROM ft_data
-} {200}
-
-do_execsql_test 3.2 {
-  DELETE FROM ft WHERE (rowid % 50)==0;
-  SELECT count(*) FROM ft_data;
-} {203}
-
-do_execsql_test 3.3 {
-  INSERT INTO ft(ft, rank) VALUES('merge', 500);
-  SELECT rowid FROM ft_data;
-} [db eval {SELECT rowid FROM ft_data}]
-
-do_execsql_test 3.4 {
-  INSERT INTO ft(ft, rank) VALUES('merge', -1000);
-  SELECT count(*) FROM ft_data;
-} {197}
-
-do_execsql_test 3.5 {
-  DELETE FROM ft WHERE (rowid % 50)==1;
-  SELECT count(*) FROM ft_data;
-} {200}
-
-do_execsql_test 3.6 {
-  SELECT level, segment, npgtombstone FROM fts5_structure(
-    (SELECT block FROM ft_data WHERE id=10)
-  )
-} {1 0 3}
-
-do_test 3.6 {
-  while 1 {
-    set nChange [db total_changes]
-    execsql { INSERT INTO ft(ft, rank) VALUES('merge', -5) }
-    if {([db total_changes] - $nChange)<2} break
-  }
-} {}
-
-do_execsql_test 3.7 {
-  SELECT level, segment, npgtombstone FROM fts5_structure(
-    (SELECT block FROM ft_data WHERE id=10)
-  )
-} {2 0 0}
-
-
-finish_test

ext/fts5/test/fts5contentless4.test

@@ -1,247 +0,0 @@
-# 2023 July 21
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-#
-# This file contains tests for the content= and content_rowid= options.
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5contentless4
-
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-
-proc document {n} {
-  set vocab [list A B C D E F G H I J K L M N O P Q R S T U V W X Y Z]
-  set ret [list]
-  for {set ii 0} {$ii < $n} {incr ii} {
-    lappend ret [lindex $vocab [expr int(rand()*[llength $vocab])]]
-  }
-  set ret
-}
-db func document document
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
-  INSERT INTO ft(ft, rank) VALUES('pgsz', 240);
-  WITH s(i) AS (
-    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
-  )
-  INSERT INTO ft SELECT document(12) FROM s;
-}
-
-do_execsql_test 1.1 {
-  INSERT INTO ft(ft) VALUES('optimize');
-}
-
-do_execsql_test 1.2 {
-  SELECT level, segment, nentry, nentrytombstone FROM fts5_structure((
-    SELECT block FROM ft_data WHERE id=10
-  ))
-} {0 0 1000 0}
-
-do_execsql_test 1.3 {
-  DELETE FROM ft WHERE rowid < 50
-}
-
-do_execsql_test 1.4 {
-  SELECT level, segment, nentry, nentrytombstone FROM fts5_structure((
-    SELECT block FROM ft_data WHERE id=10
-  ))
-} {0 0 1000 49}
-
-do_execsql_test 1.5 {
-  DELETE FROM ft WHERE rowid < 1000
-}
-
-do_execsql_test 1.6 {
-  SELECT level, segment, nentry, nentrytombstone FROM fts5_structure((
-    SELECT block FROM ft_data WHERE id=10
-  ))
-} {1 0 1 0}
-
-#--------------------------------------------------------------------------
-reset_db
-db func document document
-
-do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
-}
-
-do_test 2.1 {
-  for {set ii 0} {$ii < 5000} {incr ii} {
-    execsql { INSERT INTO ft VALUES( document(12) ) }
-  }
-} {}
-
-do_execsql_test 2.2 {
-  SELECT sum(nentry) - sum(nentrytombstone) FROM fts5_structure((
-    SELECT block FROM ft_data WHERE id=10
-  ))
-} {5000}
-
-for {set ii 5000} {$ii >= 0} {incr ii -100} {
-  do_execsql_test 2.3.$ii {
-    DELETE FROM ft WHERE rowid > $ii
-  }
-  do_execsql_test 2.3.$ii.2 {
-    SELECT 
-      CAST((total(nentry) - total(nentrytombstone)) AS integer) 
-    FROM 
-      fts5_structure( (SELECT block FROM ft_data WHERE id=10) )
-  } $ii
-}
-
-execsql_pp {
-  SELECT * FROM fts5_structure((
-    SELECT block FROM ft_data WHERE id=10
-  ))
-}
-
-do_test 2.4 {
-  for {set ii 0} {$ii < 5000} {incr ii} {
-    execsql { INSERT INTO ft VALUES( document(12) ) }
-  }
-} {}
-
-for {set ii 1} {$ii <= 5000} {incr ii 10} {
-  do_execsql_test 2.3.$ii {
-    DELETE FROM ft WHERE rowid = $ii;
-    INSERT INTO ft VALUES( document(12) );
-    INSERT INTO ft(ft, rank) VALUES('merge', -10);
-  }
-
-  do_execsql_test 2.3.$ii.2 {
-    SELECT 
-      CAST((total(nentry) - total(nentrytombstone)) AS integer) 
-    FROM 
-      fts5_structure( (SELECT block FROM ft_data WHERE id=10) )
-  } 5000
-}
-
-#-------------------------------------------------------------------------
-reset_db
-db func document document
-do_execsql_test 3.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=1);
-  WITH s(i) AS (
-    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
-  )
-  INSERT INTO ft SELECT document(12) FROM s;
-}
-
-do_catchsql_test 3.1 {
-  INSERT INTO ft(ft, rank) VALUES('deletemerge', 'text');
-} {1 {SQL logic error}}
-do_catchsql_test 3.2 {
-  INSERT INTO ft(ft, rank) VALUES('deletemerge', 50);
-} {0 {}}
-do_execsql_test 3.3 {
-  SELECT * FROM ft_config WHERE k='deletemerge'
-} {deletemerge 50}
-do_catchsql_test 3.4 {
-  INSERT INTO ft(ft, rank) VALUES('deletemerge', 101);
-} {0 {}}
-do_execsql_test 3.5 {
-  SELECT * FROM ft_config WHERE k='deletemerge'
-} {deletemerge 101}
-
-do_execsql_test 3.6 {
-  DELETE FROM ft WHERE rowid<95
-}
-
-do_execsql_test 3.7 {
-  SELECT nentrytombstone, nentry FROM fts5_structure((
-    SELECT block FROM ft_data WHERE id=10
-  ))
-} {94 100}
-
-do_execsql_test 3.8 {
-  DELETE FROM ft WHERE rowid=95
-}
-
-do_execsql_test 3.9 {
-  SELECT nentrytombstone, nentry FROM fts5_structure((
-    SELECT block FROM ft_data WHERE id=10
-  ))
-} {95 100}
-
-do_execsql_test 3.10 {
-  DELETE FROM ft;
-  WITH s(i) AS (
-    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100
-  )
-  INSERT INTO ft SELECT document(12) FROM s;
-  INSERT INTO ft(ft, rank) VALUES('deletemerge', 50);
-}
-
-do_execsql_test 3.11 {
-  DELETE FROM ft WHERE rowid<95
-}
-
-do_execsql_test 3.12 {
-  SELECT nentrytombstone, nentry FROM fts5_structure((
-    SELECT block FROM ft_data WHERE id=10
-  ))
-} {0 6}
-
-#-------------------------------------------------------------------------
-reset_db
-db func document document
-do_execsql_test 4.0 {
-  CREATE VIRTUAL TABLE x1 USING fts5(x, content='', contentless_delete=1);
-  INSERT INTO x1(x1, rank) VALUES('usermerge', 16);
-  INSERT INTO x1(x1, rank) VALUES('deletemerge', 40);
-  INSERT INTO x1 VALUES('one');
-  INSERT INTO x1 VALUES('two');
-  INSERT INTO x1 VALUES('three');
-  INSERT INTO x1 VALUES('four');
-  INSERT INTO x1 VALUES('five');
-  INSERT INTO x1 VALUES('six');
-  INSERT INTO x1 VALUES('seven');
-  INSERT INTO x1 VALUES('eight');
-  INSERT INTO x1 VALUES('nine');
-  INSERT INTO x1 VALUES('ten');
-}
-
-do_execsql_test 4.1 {
-  SELECT level, segment FROM fts5_structure((
-    SELECT block FROM x1_data WHERE id=10
-  ))
-} {
-  0 0 0 1 0 2 0 3 0 4 0 5 0 6 0 7 0 8 0 9
-}
-
-for {set ii 1} {$ii < 4} {incr ii} {
-  do_execsql_test 4.2.$ii {
-    DELETE FROM x1 WHERE rowid = $ii;
-    INSERT INTO x1(x1, rank) VALUES('merge', 5);
-    SELECT level, segment FROM fts5_structure((
-          SELECT block FROM x1_data WHERE id=10
-    ))
-  } {
-    0 0 0 1 0 2 0 3 0 4 0 5 0 6 0 7 0 8 0 9
-  }
-}
-
-do_execsql_test 4.3 {
-  DELETE FROM x1 WHERE rowid = $ii;
-  INSERT INTO x1(x1, rank) VALUES('merge', 5);
-  SELECT level, segment, nentry FROM fts5_structure((
-        SELECT block FROM x1_data WHERE id=10
-  ))
-} {
-  1 0   6
-}
-
-finish_test

ext/fts5/test/fts5contentless5.test

@@ -1,59 +0,0 @@
-# 2023 August 7
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-#
-# This file contains tests for the content= and content_rowid= options.
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5contentless5
-
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-unset -nocomplain res
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, content='', contentless_delete=1);
-  INSERT INTO t1 VALUES('A', 'B', 'C');
-  INSERT INTO t1 VALUES('D', 'E', 'F');
-  INSERT INTO t1 VALUES('G', 'H', 'I');
-}
-
-do_execsql_test 1.01 {
-  CREATE TABLE t2(x, y);
-  INSERT INTO t2 VALUES('x', 'y');
-}
-
-# explain_i "UPDATE t1 SET a='a' WHERE t1.rowid=1"    
-breakpoint
-explain_i "UPDATE t1 SET a='a' FROM t2 WHERE t1.rowid=1 AND b IS NULL"     
-
-#breakpoint
-#explain_i "UPDATE t1 SET a='a' WHERE b IS NULL AND rowid=?"
-
-foreach {tn up err} {
-  1   "UPDATE t1 SET a='a', b='b', c='c' WHERE rowid=1"                  0
-  2   "UPDATE t1 SET a='a', b='b'        WHERE rowid=1"                  1
-  3   "UPDATE t1 SET        b='b', c='c' WHERE rowid=1"                  1
-  4   "UPDATE t1 SET a='a',        c='c' WHERE rowid=1"                  1
-  5   "UPDATE t1 SET a='a',        c='c' WHERE t1.rowid=1 AND b IS NULL" 1
-  6   "UPDATE t1 SET a='a' FROM t2 WHERE t1.rowid=1"                     1
-  7   "UPDATE t1 SET a='a', b='b', c='c' FROM t2 WHERE t1.rowid=1"       0
-} {
-
-  set res(0) {0 {}}
-  set res(1) {1 {cannot UPDATE a subset of columns on fts5 contentless-delete table: t1}}
-  do_catchsql_test 1.$tn $up $res($err)
-}
-
-finish_test

ext/fts5/test/fts5corrupt.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -48,10 +48,6 @@ do_test 1.3 {
   }
   catchsql { INSERT INTO t1(t1) VALUES('integrity-check') }
 } {1 {database disk image is malformed}}
-do_execsql_test 1.3b {
-  PRAGMA integrity_check(t1);
-} {{malformed inverted index for FTS5 table main.t1}}
-
 
 do_test 1.4 {
   db_restore_and_reopen
@@ -99,6 +95,6 @@ sqlite3_db_config db DEFENSIVE 0
 do_catchsql_test 3.1 {
   DELETE FROM t3_content WHERE rowid = 3;
   SELECT * FROM t3 WHERE t3 MATCH 'o';
-} {1 {fts5: missing row 3 from content table 'main'.'t3_content'}}
+} {1 {database disk image is malformed}}
 
 finish_test

ext/fts5/test/fts5corrupt2.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt2
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -100,7 +100,6 @@ set lrowid [db one {SELECT max(rowid) FROM t1_data WHERE (rowid & $mask)=0}]
 set nbyte [db one {SELECT length(block) FROM t1_data WHERE rowid=$lrowid}]
 set all [db eval {SELECT rowid FROM t1}]
 sqlite3_db_config db DEFENSIVE 0
-unset -nocomplain res
 for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} {
   do_execsql_test 2.$i.1 {
     BEGIN;
@@ -153,7 +152,7 @@ foreach {tn hdr} {
       execsql BEGIN
 
       set fd [db incrblob main x3_data block $rowid]
-      fconfigure $fd -translation binary
+      fconfigure $fd -encoding binary -translation binary
       set existing [read $fd [string length $hdr]]
       seek $fd 0
       puts -nonewline $fd $hdr
@@ -168,9 +167,6 @@ foreach {tn hdr} {
       do_test 3.$tn.$tn2.2 {
         catchsql { INSERT INTO x3(x3) VALUES('integrity-check') }
       } {1 {database disk image is malformed}}
-      do_execsql_test 3.$tn.$tn2.3 {
-        PRAGMA integrity_check(x3);
-      } {{malformed inverted index for FTS5 table main.x3}}
     }
 
     execsql ROLLBACK
@@ -239,7 +235,7 @@ foreach {tn hdr} {
       execsql BEGIN
 
       set fd [db incrblob main x5_data block $rowid]
-      fconfigure $fd -translation binary
+      fconfigure $fd -encoding binary -translation binary
       puts -nonewline $fd $hdr
       close $fd
 

ext/fts5/test/fts5corrupt3.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt3
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -4267,7 +4267,7 @@ do_test 35.0 {
 
 do_catchsql_test 35.1 {
   SELECT * FROM t1 WHERE t1 MATCH 'e*';
-} {1 {fts5: missing row 14 from content table 'main'.'t1_content'}}
+} {1 {database disk image is malformed}}
 
 #-------------------------------------------------------------------------
 reset_db
@@ -8958,6 +8958,7 @@ do_catchsql_test 61.2 {
   SELECT * FROM t3 ORDER BY rowid;
 } {/*malformed database schema*/}
 
+breakpoint
 #-------------------------------------------------------------------------
 do_test 62.0 {
   sqlite3 db {}
@@ -10767,7 +10768,6 @@ do_catchsql_test 73.1 {
 reset_db
 do_test 74.0 {
   sqlite3 db {}
-  sqlite3_fts5_register_matchinfo db
   db deserialize [decode_hexdb {
 | size 106496 pagesize 4096 filename x.db
 | page 1 offset 0
@@ -14587,19 +14587,14 @@ do_test 74.0 {
 | end x.db
 }]} {}
 
-do_catchsql_test 74.0.5 {
-  SELECT matchinfo(1,2);
-} {1 {unable to use function matchinfo in the requested context}}
-
 do_catchsql_test 74.1 {
-  SELECT rowid, quote(matchinfo(t1,'pxyb<s')) FROM t1 WHERE t1 MATCH 'e*';
-} {1 {unrecognized matchinfo flag: <}}
+  SELECT rowid, quote(matchinfo(t1,'p�xyb<s')) FROM t1 WHERE t1 MATCH 'e*';
+} {1 {unable to use function matchinfo in the requested context}}
 
 #-------------------------------------------------------------------------
 reset_db
 do_test 75.0 {
   sqlite3 db {}
-  sqlite3_fts5_register_matchinfo db
   db deserialize [decode_hexdb {
 | size 32768 pagesize 4096 filename crash-033d665d5caa8d.db
 | page 1 offset 0
@@ -14796,7 +14791,7 @@ do_test 75.0 {
 
 do_catchsql_test 75.1 {
   SELECT rowid, quote(matchinfo(t1,'pcxybs')) FROM t1 WHERE t1 MATCH 'e*';
-} {1 {database disk image is malformed}}
+} {1 {unable to use function matchinfo in the requested context}}
 
 #-------------------------------------------------------------------------
 reset_db

ext/fts5/test/fts5corrupt4.test

@@ -14,7 +14,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt4
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5corrupt5.test

@@ -15,9 +15,9 @@
 #
 
 source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5corrupt5
+set testprefix fts5corrupt3
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -793,664 +793,6 @@ do_catchsql_test 4.5 {
   REPLACE INTO t1(rowid,a,b,rowid) VALUES(200,1,2,3);
 } {1 {database disk image is malformed}}
 
-#-------------------------------------------------------------------------
-reset_db
-do_test 5.0 {
-  sqlite3 db {}
-  db deserialize [decode_hexdb {
-.open --hexdb
-| size 28672 pagesize 4096 filename crash-0c6d3451d11597.db
-| page 1 offset 0
-|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
-|     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07   .....@  ........
-|     32: 00 00 00 00 00 00 00 00 00 00 00 07 00 00 00 04   ................
-|     96: 00 00 00 00 0d 00 00 00 07 0d d2 00 0f c4 0f 6d   ...............m
-|    112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00   .....N..........
-|   3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74   .........1tablet
-|   3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45   2t2.CREATE TABLE
-|   3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61    t2(x)V.......ta
-|   3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63   blet1_configt1_c
-|   3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42   onfig.CREATE TAB
-|   3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b   LE 't1_config'(k
-|   3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29    PRIMARY KEY, v)
-|   3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05    WITHOUT ROWID[.
-|   3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64   ..!!...tablet1_d
-|   3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65   ocsizet1_docsize
-|   3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74   .CREATE TABLE 't
-|   3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e   1_docsize'(id IN
-|   3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45   TEGER PRIMARY KE
-|   3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21   Y, sz BLOB)U...!
-|   3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65   !.wtablet1_conte
-|   3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45   ntt1_content.CRE
-|   3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f   ATE TABLE 't1_co
-|   3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45   ntent'(id INTEGE
-|   3824: 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 63   R PRIMARY KEY, c
-|   3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65   0)i.......-table
-|   3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45   t1_idxt1_idx.CRE
-|   3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64   ATE TABLE 't1_id
-|   3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20   x'(segid, term, 
-|   3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45   pgno, PRIMARY KE
-|   3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20   Y(segid, term)) 
-|   3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07   WITHOUT ROWIDU..
-|   3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61   ......tablet1_da
-|   3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45   tat1_data.CREATE
-|   3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27    TABLE 't1_data'
-|   4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d   (id INTEGER PRIM
-|   4016: 41 52 b9 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42   AR. KEY, block B
-|   4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c   LOB):......ctabl
-|   4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54   et1t1CREATE VIRT
-|   4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49   UAL TABLE t1 USI
-|   4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29   NG fts5(content)
-| page 2 offset 4096
-|      0: 0d 00 00 00 03 0f bd 00 0f e8 0f ef 0f bd 00 00   ................
-|     16: 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
-|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 24 84 80   .............$..
-|   4032: 80 80 80 01 03 00 4e 00 00 00 1e 06 30 61 62 61   ......N.....0aba
-|   4048: 63 6b 01 02 02 04 02 66 74 02 02 02 04 04 6e 64   ck.....ft.....nd
-|   4064: 6f 6e 03 02 02 04 0a 07 05 01 03 00 10 03 03 0f   on..............
-|   4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 11   ...$............
-| page 3 offset 8192
-|      0: 0a 00 00 00 01 0f 00 00 00 00 00 00 00 00 00 00   ................
-|   4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02   ................
-| page 4 offset 12288
-|      0: 0d 00 00 00 03 0f e0 00 0f f6 0f ec 0f e0 00 00   ................
-|   4064: 0a 03 03 00 1b 61 62 61 6e 64 6f 6e 08 02 03 00   .....abandon....
-|   4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b   .abaft.....aback
-| page 5 offset 16384
-|      0: 0d 00 00 00 03 0f ee 00 0f fa 0f f4 0f ee 00 00   ................
-|     16: 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
-|   4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 03   ................
-|   4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01   ................
-| page 6 offset 20480
-|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
-|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
-| page 7 offset 24576
-|      0: 0d 00 00 10 03 0f d6 00 0f f4 10 e1 0f d6 00 00   ................
-|     16: 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00   ................
-|   4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c   ..........rebuil
-|   4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63   d...+integrity-c
-|   4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 00 00 00   heck....optim...
-| end crash-0c6d3451d11597.db
-}]} {}
-
-do_execsql_test 5.1 {
-  INSERT INTO t1(t1,rank) VALUES('secure-delete',1);
-}
-do_catchsql_test 5.4 {
-  UPDATE t1 SET content=randomblob(500);
-} {1 {database disk image is malformed}}
-
-#-------------------------------------------------------------------------
-reset_db
-do_test 6.0 {
-  sqlite3 db {}
-  db deserialize [decode_hexdb {
-.open --hexdb
-| size 32768 pagesize 4096 filename crash-42fa37b694d45a.db
-| page 1 offset 0
-|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
-|     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 07   .....@  ........
-|     96: 00 00 00 00 0d 00 00 00 07 0d d2 00 0f c4 0f 6d   ...............m
-|    112: 0f 02 0e ab 0e 4e 0d f6 0d d2 00 00 00 00 00 00   .....N..........
-|   3536: 00 00 22 07 06 17 11 11 01 31 74 61 62 6c 65 74   .........1tablet
-|   3552: 32 74 32 07 43 52 45 41 54 45 20 54 41 42 4c 45   2t2.CREATE TABLE
-|   3568: 20 74 32 28 78 29 56 06 06 17 1f 1f 01 7d 74 61    t2(x)V.......ta
-|   3584: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63   blet1_configt1_c
-|   3600: 6f 6e 66 69 67 06 43 52 45 41 54 45 20 54 41 42   onfig.CREATE TAB
-|   3616: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b   LE 't1_config'(k
-|   3632: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29    PRIMARY KEY, v)
-|   3648: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 05    WITHOUT ROWID[.
-|   3664: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64   ..!!...tablet1_d
-|   3680: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65   ocsizet1_docsize
-|   3696: 05 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74   .CREATE TABLE 't
-|   3712: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e   1_docsize'(id IN
-|   3728: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45   TEGER PRIMARY KE
-|   3744: 59 2c 20 73 7a 20 42 4c 4f 42 29 55 04 06 17 21   Y, sz BLOB)U...!
-|   3760: 21 01 77 74 61 62 6c 65 74 31 5f 63 6f 6e 74 65   !.wtablet1_conte
-|   3776: 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 04 43 52 45   ntt1_content.CRE
-|   3792: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f   ATE TABLE 't1_co
-|   3808: 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47 45   ntent'(id INTEGE
-|   3824: 52 20 50 52 49 4d 41 52 49 20 4b 45 59 2c 20 63   R PRIMARI KEY, c
-|   3840: 30 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65   0)i.......-table
-|   3856: 74 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45   t1_idxt1_idx.CRE
-|   3872: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64   ATE TABLE 't1_id
-|   3888: 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20   x'(segid, term, 
-|   3904: 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45   pgno, PRIMARY KE
-|   3920: 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20   Y(segid, term)) 
-|   3936: 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07   WITHOUT ROWIDU..
-|   3952: 17 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61   ......tablet1_da
-|   3968: 74 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45   tat1_data.CREATE
-|   3984: 20 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27    TABLE 't1_data'
-|   4000: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d   (id INTEGER PRIM
-|   4016: 41 52 b9 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42   AR. KEY, block B
-|   4032: 4c 4f 42 29 3a 01 06 17 11 11 08 63 74 61 62 6c   LOB):......ctabl
-|   4048: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54   et1t1CREATE VIRT
-|   4064: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49   UAL TABLE t1 USI
-|   4080: 4e 47 20 66 74 73 35 28 63 6f 6e 74 65 6e 74 29   NG fts5(content)
-| page 2 offset 4096
-|      0: 0d 00 00 00 03 0f bd 00 0f e8 0f ef 0f bd f0 00   ................
-|     16: 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
-|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 24 84 80   .............$..
-|   4032: 80 80 80 01 03 00 4e 00 10 00 1e 06 30 61 62 61   ......N.....0aba
-|   4048: 63 6c 01 02 02 04 02 66 74 02 5f 02 04 04 6e 64   cl.....ft._...nd
-|   4064: 6f 6e 02 02 02 04 0a 07 05 01 03 00 10 03 03 0f   on..............
-|   4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 11   ...$............
-| page 3 offset 8192
-|      0: 0a 00 00 00 01 0f 00 01 00 00 00 00 00 00 00 00   ................
-|   4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02   ................
-| page 4 offset 12288
-|      0: 0d 00 00 00 03 0f e0 00 0f f6 0f ec 0f e0 00 00   ................
-|   4064: 0a 03 03 00 1b 61 62 61 6e 64 6f 6e 08 02 03 00   .....abandon....
-|   4080: 17 61 62 61 66 74 08 01 03 00 17 61 62 61 63 6b   .abaft.....aback
-| page 5 offset 16384
-|      0: 0d 00 00 00 03 0f ee 00 0f fa 0f 00 00 00 00 00   ................
-|   4064: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 04 03   ................
-|   4080: 03 00 0e 01 04 02 03 00 0e 01 04 01 03 00 0e 01   ................
-| page 6 offset 20480
-|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
-|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
-| page 7 offset 24576
-|      0: 0d 00 00 10 03 0f d6 00 0f 00 00 00 00 00 00 00   ................
-|   4048: 00 00 00 00 00 00 09 03 02 1b 72 65 62 75 69 6c   ..........rebuil
-|   4064: 64 11 02 02 2b 69 6e 74 65 67 72 69 74 79 2d 63   d...+integrity-c
-|   4080: 68 65 63 6b 0a 01 02 1d 6f 70 74 69 6d 00 00 00   heck....optim...
-| page 8 offset 28672
-|      0: 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
-| end crash-42fa37b694d45a.db
-}]} {}
-
-do_execsql_test 6.1 {
-  INSERT INTO t1(t1,rank) VALUES('secure-delete',1);
-}
-do_catchsql_test 6.2 {
-  UPDATE t1 SET content=randomblob(500) WHERE t1;
-} {1 {constraint failed}}
-
-#-------------------------------------------------------------------------
-reset_db
-do_test 7.0 {
-  sqlite3 db {}
-  db deserialize [decode_hexdb {
-.open --hexdb
-| size 40960 pagesize 4096 filename crash-d8b4a99207c10b.db
-| page 1 offset 0
-|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
-|     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a   .....@  ........
-|     32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04   ................
-|     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
-|     96: 00 00 00 00 0d 00 00 00 0d 0b 62 00 0f 97 0f 40   ..........b....@
-|    112: 0e d5 0e 75 0e 18 0d c0 0d 66 0d 0f 0c a4 0c 44   ...u.....f.....D
-|    128: 0b ec 0b a7 0b 62 00 00 00 00 00 00 00 00 00 00   .....b..........
-|   2912: 00 00 43 0d 06 17 11 11 08 75 74 61 62 6c 65 74   ..C......utablet
-|   2928: 34 74 34 43 52 45 41 54 45 20 56 49 52 54 55 41   4t4CREATE VIRTUA
-|   2944: 4c 20 54 41 42 4c 45 20 74 34 20 55 53 49 4e 47   L TABLE t4 USING
-|   2960: 20 66 74 73 35 76 6f 63 61 62 28 27 74 32 27 2c    fts5vocab('t2',
-|   2976: 20 27 72 6f 77 27 29 43 0c 06 17 11 11 08 75 74    'row')C......ut
-|   2992: 61 62 6c 65 74 33 74 33 43 52 45 41 54 45 20 56   ablet3t3CREATE V
-|   3008: 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 33 20   IRTUAL TABLE t3 
-|   3024: 55 53 49 4e 47 20 66 74 73 35 76 6f 63 61 62 28   USING fts5vocab(
-|   3040: 27 74 31 27 2c 20 27 72 6f 77 27 29 56 0b 06 17   't1', 'row')V...
-|   3056: 1f 1f 01 7d 74 61 62 6c 65 74 32 5f 63 6f 6e 66   ....tablet2_conf
-|   3072: 69 67 74 32 5f 63 6f 6e 66 69 67 0a 43 52 45 41   igt2_config.CREA
-|   3088: 54 45 20 54 41 42 4c 45 20 27 74 32 5f 63 6f 6e   TE TABLE 't2_con
-|   3104: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b   fig'(k PRIMARY K
-|   3120: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52   EY, v) WITHOUT R
-|   3136: 4f 57 49 44 5e 0a 07 17 21 21 01 81 07 74 61 62   OWID^...!!...tab
-|   3152: 6c 65 74 32 5f 63 6f 6e 74 65 6e 74 74 32 5f 63   let2_contentt2_c
-|   3168: 6f 6e 74 65 6e 74 09 43 52 45 41 54 45 20 54 41   ontent.CREATE TA
-|   3184: 42 4c 45 20 27 74 32 5f 63 6f 6e 74 65 6e 74 27   BLE 't2_content'
-|   3200: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d   (id INTEGER PRIM
-|   3216: 41 52 59 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c   ARY KEY, c0, c1,
-|   3232: 20 63 32 29 69 09 07 17 19 19 01 81 2d 74 61 62    c2)i.......-tab
-|   3248: 6c 65 74 32 5f 69 64 78 74 32 5f 69 64 78 08 43   let2_idxt2_idx.C
-|   3264: 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 32 5f   REATE TABLE 't2_
-|   3280: 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72 6d   idx'(segid, term
-|   3296: 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20   , pgno, PRIMARY 
-|   3312: 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d 29   KEY(segid, term)
-|   3328: 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 55   ) WITHOUT ROWIDU
-|   3344: 08 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 32 5f   ........tablet2_
-|   3360: 64 61 74 61 74 32 5f 64 61 74 61 07 43 52 45 41   datat2_data.CREA
-|   3376: 54 45 20 54 41 42 4c 45 20 27 74 32 5f 64 61 74   TE TABLE 't2_dat
-|   3392: 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 52   a'(id INTEGER PR
-|   3408: 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b   IMARY KEY, block
-|   3424: 20 42 4c 4f 42 29 58 07 07 17 11 11 08 81 1d 74    BLOB)X........t
-|   3440: 61 62 6c 65 74 32 74 32 43 52 45 41 54 45 20 56   ablet2t2CREATE V
-|   3456: 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 32 20   IRTUAL TABLE t2 
-|   3472: 55 53 49 4e 47 20 66 74 73 35 28 27 61 27 2c 5b   USING fts5('a',[
-|   3488: 62 5d 2c 22 63 22 2c 64 65 74 61 69 6c 3d 6e 6f   b],.c.,detail=no
-|   3504: 6e 65 2c 63 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29   ne,columnsize=0)
-|   3520: 56 06 06 17 1f 1f 01 7d 74 61 62 6c 65 74 31 5f   V.......tablet1_
-|   3536: 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 69 67 06   configt1_config.
-|   3552: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31   CREATE TABLE 't1
-|   3568: 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 49 4d 41   _config'(k PRIMA
-|   3584: 52 59 20 4b 45 59 2c 20 76 29 20 57 49 54 48 4f   RY KEY, v) WITHO
-|   3600: 55 54 20 52 4f 57 49 44 5b 05 07 17 21 21 01 81   UT ROWID[...!!..
-|   3616: 01 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65   .tablet1_docsize
-|   3632: 74 31 5f 64 6f 63 73 69 7a 65 05 43 52 45 41 54   t1_docsize.CREAT
-|   3648: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73   E TABLE 't1_docs
-|   3664: 69 7a 65 27 28 69 64 20 49 4e 54 45 47 45 52 20   ize'(id INTEGER 
-|   3680: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 73 7a 20   PRIMARY KEY, sz 
-|   3696: 42 4c 4f 42 29 5e 04 07 17 21 21 01 81 07 74 61   BLOB)^...!!...ta
-|   3712: 62 6c 65 74 31 5f 63 6f 6e 74 65 6e 74 74 31 5f   blet1_contentt1_
-|   3728: 63 6f 6e 74 65 6e 74 04 43 52 45 41 54 45 20 54   content.CREATE T
-|   3744: 41 42 4c 45 20 27 74 31 5f 63 6f 6e 74 65 6e 74   ABLE 't1_content
-|   3760: 27 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49   '(id INTEGER PRI
-|   3776: 4d 41 52 59 20 4b 45 59 2c 20 63 30 2c 20 63 31   MARY KEY, c0, c1
-|   3792: 2c 20 63 32 29 69 03 07 17 19 19 01 81 2d 74 61   , c2)i.......-ta
-|   3808: 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 64 78 03   blet1_idxt1_idx.
-|   3824: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31   CREATE TABLE 't1
-|   3840: 5f 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72   _idx'(segid, ter
-|   3856: 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59   m, pgno, PRIMARY
-|   3872: 20 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d    KEY(segid, term
-|   3888: 29 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44   )) WITHOUT ROWID
-|   3904: 55 02 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 31   U........tablet1
-|   3920: 5f 64 61 74 61 74 31 5f 64 61 74 61 02 43 52 45   _datat1_data.CRE
-|   3936: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 61   ATE TABLE 't1_da
-|   3952: 74 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50   ta'(id INTEGER P
-|   3968: 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63   RIMARY KEY, bloc
-|   3984: 6b 20 42 4c 4f 42 29 67 01 07 17 11 11 08 81 3b   k BLOB)g.......;
-|   4000: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20   tablet1t1CREATE 
-|   4016: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31   VIRTUAL TABLE t1
-|   4032: 20 55 53 49 4e 47 20 66 74 73 35 28 61 2c 62 20    USING fts5(a,b 
-|   4048: 75 6e 69 6e 64 65 78 65 64 2c 63 2c 74 6f 6b 65   unindexed,c,toke
-|   4064: 6e 69 7a 65 3d 22 70 6f 72 74 65 72 20 61 73 63   nize=.porter asc
-|   4080: 69 69 22 2c 74 6f 6b 65 6e 64 61 74 61 3d 31 29   ii.,tokendata=1)
-| page 2 offset 4096
-|      0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c   ..h............|
-|     16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00   .*..............
-|   3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00   .......0........
-|   3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80   ..........7.....
-|   3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01   ...t.....0a.....
-|   3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01   b.....c.....g...
-|   3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01   ....h.......i...
-|   3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00   ............*...
-|   3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80   ............%...
-|   3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02   .....P.....0g...
-|   3984: 01 02 02 01 01 68 02 08 03 01 02 03 01 01 69 02   .....h........i.
-|   4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 7f f1 03   ........7.......
-|   4016: 00 74 00 00 00 2e 02 30 61 01 02 02 01 01 62 01   .t.....0a.....b.
-|   4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02   ....c.....g.....
-|   4048: 01 01 68 01 06 01 02 03 01 01 69 01 06 01 02 04   ..h.......i.....
-|   4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00   ................
-|   4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01   ...$............
-| page 3 offset 8192
-|      0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00   ................
-|   4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c   ................
-|   4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02   ................
-| page 4 offset 12288
-|      0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00   ................
-|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03   ................
-|   4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67   .....a b cg h ig
-|   4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69    h i.......g h i
-|   4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17   a b cg h i......
-|   4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69   .a b cd e fg h i
-| page 5 offset 16384
-|      0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00   ................
-|   4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03   ................
-|   4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03   ................
-| page 6 offset 20480
-|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
-|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
-| page 7 offset 24576
-|      0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00   ................
-|   3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84   ..............A.
-|   4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61   ............4.0a
-|   4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01   .....b.....c....
-|   4032: 01 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01   .d...e...f...g..
-|   4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06   ...h.....i......
-|   4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f   ................
-|   4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01   ...$............
-| page 8 offset 28672
-|      0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00   ................
-|   4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02   ................
-| page 9 offset 32768
-|      0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00   ................
-|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03   ................
-|   4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67   .....a b cg h ig
-|   4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69    h i.......g h i
-|   4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17   a b cg h i......
-|   4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69   .a b cd e fg h i
-| page 10 offset 36864
-|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
-|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
-| end crash-d8b4a99207c10b.db
-}]} {}
-
-do_catchsql_test 7.1 {
-  SELECT snippet(t1, -1, '.', '..', '[', ']'), 
-         highlight(t1, 2, '[', ']') 
-           FROM t1('g + h') 
-           WHERE rank MATCH 'bm25(1.0, 1.0)' ORDER BY rank;
-} {1 {database disk image is malformed}}
-
-#-------------------------------------------------------------------------
-reset_db
-do_test 8.0 {
-  sqlite3 db {}
-  db deserialize [decode_hexdb {
-.open --hexdb
-| size 20480 pagesize 4096 filename crash-d57c01958e48ab.db
-| page 1 offset 0
-|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
-|     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 05   .....@  ........
-|     32: 00 00 00 00 00 00 00 00 00 00 00 05 00 00 00 04   ................
-|     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
-|     96: 00 00 00 00 0d 00 00 00 05 0e 10 00 0f 97 0f 40   ...............@
-|    112: 0e d5 0e 68 0e 10 01 00 00 00 00 00 00 00 00 00   ...h............
-|   3600: 56 05 06 17 1f 1f 01 7d 74 61 62 6c 65 74 31 5f   V.......tablet1_
-|   3616: 63 6f 6e 66 69 67 74 31 5f 63 6f 6e 66 69 67 05   configt1_config.
-|   3632: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31   CREATE TABLE 't1
-|   3648: 5f 63 6f 6e 66 69 67 27 28 6b 20 50 52 49 4d 41   _config'(k PRIMA
-|   3664: 52 59 20 4b 45 59 2c 20 76 29 20 57 49 54 48 4f   RY KEY, v) WITHO
-|   3680: 55 54 20 52 4f 57 49 44 6b 04 07 17 21 21 01 81   UT ROWIDk...!!..
-|   3696: 21 74 61 62 6c 65 74 31 5f 64 6f 63 73 69 7a 65   !tablet1_docsize
-|   3712: 74 31 5f 64 6f 63 73 69 7a 65 04 43 52 45 41 54   t1_docsize.CREAT
-|   3728: 45 20 54 41 42 4c 45 20 27 74 31 5f 64 6f 63 73   E TABLE 't1_docs
-|   3744: 69 7a 65 27 28 69 64 20 49 4e 54 45 47 45 52 20   ize'(id INTEGER 
-|   3760: 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 73 7a 20   PRIMARY KEY, sz 
-|   3776: 42 4c 4f 42 2c 20 6f 72 69 67 69 6e 20 49 4e 54   BLOB, origin INT
-|   3792: 45 47 45 52 29 69 03 07 17 19 19 01 81 2d 74 61   EGER)i.......-ta
-|   3808: 62 6c 65 74 31 5f 69 64 78 74 31 5f 69 64 78 03   blet1_idxt1_idx.
-|   3824: 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74 31   CREATE TABLE 't1
-|   3840: 5f 69 64 78 27 28 73 65 67 69 64 2c 20 74 65 72   _idx'(segid, ter
-|   3856: 6d 2c 20 70 67 6e 6f 2c 20 50 52 49 4d 41 52 59   m, pgno, PRIMARY
-|   3872: 20 4b 45 59 28 73 65 67 69 64 2c 20 74 65 72 6d    KEY(segid, term
-|   3888: 29 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44   )) WITHOUT ROWID
-|   3904: 55 02 07 17 1b 1b 01 81 01 74 61 62 6c 65 74 31   U........tablet1
-|   3920: 5f 64 61 74 61 74 31 5f 64 61 74 61 02 43 52 45   _datat1_data.CRE
-|   3936: 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 64 61   ATE TABLE 't1_da
-|   3952: 74 61 27 28 69 64 20 49 4e 54 45 47 45 52 20 50   ta'(id INTEGER P
-|   3968: 52 49 4d 41 52 59 20 4b 45 59 2c 20 62 6c 6f 63   RIMARY KEY, bloc
-|   3984: 6b 20 42 4c 4f 42 29 67 01 07 17 11 11 08 81 3b   k BLOB)g.......;
-|   4000: 74 61 62 6c 65 74 31 74 31 43 52 45 41 54 45 20   tablet1t1CREATE 
-|   4016: 56 49 52 54 55 41 4c 20 54 41 42 4c 45 20 74 31   VIRTUAL TABLE t1
-|   4032: 20 55 53 49 4e 47 20 66 74 73 35 28 61 2c 20 62    USING fts5(a, b
-|   4048: 2c 20 63 6f 6e 74 65 6e 74 3d 27 27 2c 20 63 6f   , content='', co
-|   4064: 6e 74 65 6e 74 6c 65 73 73 5f 64 65 6c 65 74 65   ntentless_delete
-|   4080: 3d 31 2c 20 74 6f 6b 65 6e 64 61 74 61 3d 31 29   =1, tokendata=1)
-| page 2 offset 4096
-|      0: 0d 0f eb 00 03 0e 17 00 0f e2 0e 17 0e 31 00 00   .............1..
-|     16: 01 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00   ................
-|   3600: 00 00 00 00 00 00 00 18 0a 03 00 36 00 00 00 00   ...........6....
-|   3616: ff 00 00 01 01 01 01 00 01 01 01 01 01 01 00 00   ................
-|   3632: 07 83 29 84 80 80 80 80 01 04 00 86 56 00 00 01   ..).........V...
-|   3648: 96 04 30 61 61 61 01 02 02 01 04 02 04 01 08 02   ..0aaa..........
-|   3664: 04 04 04 01 10 02 04 04 04 04 04 04 04 01 20 02   .............. .
-|   3680: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 01   ................
-|   3696: 40 02 04 04 04 04 04 04 04 04 04 04 04 04 04 04   @...............
-|   3712: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04   ................
-|   3728: 04 01 81 00 02 04 04 04 04 04 04 04 04 04 04 04   ................
-|   3744: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04   ................
-|   3760: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04   ................
-|   3776: 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04 04   ................
-|   3792: 04 04 04 04 02 02 62 63 01 06 01 01 02 01 03 62   ......bc.......b
-|   3808: 62 62 02 02 03 01 04 03 06 01 08 03 06 06 06 01   bb..............
-|   3824: 10 03 06 06 06 06 06 06 06 01 20 03 06 06 06 06   .......... .....
-|   3840: 06 06 06 06 06 06 06 06 06 06 06 01 40 03 06 06   ............@...
-|   3856: 06 06 06 06 06 06 06 06 06 06 06 06 06 06 06 06   ................
-|   3872: 06 06 06 06 06 06 06 06 06 06 16 06 06 02 02 63   ...............c
-|   3888: 64 02 06 01 01 02 01 03 63 63 63 03 02 05 01 04   d.......ccc.....
-|   3904: 05 0a 01 08 05 0a 0a 0a 01 10 05 0a 0a 0a 0a 0a   ................
-|   3920: 0a 0a 01 20 05 0a 0a 0a 0a 0a 0a 0a 0a 0a 0a 0a   ... ............
-|   3936: 0a 0a 0a 0a 02 02 64 65 03 06 01 01 02 01 03 64   ......de.......d
-|   3952: 64 64 04 02 09 01 04 09 12 01 08 09 12 12 12 01   dd..............
-|   3968: 10 09 12 12 12 12 12 12 12 02 02 65 66 04 06 01   ...........ef...
-|   3984: 01 02 01 03 65 65 65 05 02 11 01 04 11 22 01 08   ....eee.........
-|   4000: 11 22 22 22 02 02 66 67 05 06 01 01 02 01 03 66   ......fg.......f
-|   4016: 56 66 06 02 21 01 04 21 42 02 02 67 68 06 06 01   Vf..!..!B..gh...
-|   4032: 01 02 cb 03 67 67 67 07 02 41 02 02 68 69 07 06   ....ggg..A..hi..
-|   4048: 01 01 02 04 81 13 09 50 09 2e 09 1c 09 12 09 0c   .......P........
-|   4064: 09 08 07 01 03 00 14 07 81 77 07 00 00 00 15 22   .........w......
-|   4080: 00 00 00 00 ff 00 00 01 00 00 00 00 00 00 05 0c   ................
-| page 3 offset 8192
-|      0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00   ................
-|   4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02   ................
-| page 4 offset 12288
-|      0: 0d 00 00 00 07 0f c8 00 0f f8 0f f0 0f e8 0f e0   ................
-|     16: 0f d8 0f d0 0f c8 00 00 00 00 00 00 00 00 00 00   ................
-|   4032: 00 00 00 00 00 00 00 00 06 07 04 00 10 09 7f 01   ................
-|   4048: 06 06 04 00 10 09 3f 01 06 05 04 00 10 09 1f 01   ......?.........
-|   4064: 06 04 04 00 10 09 0f 01 06 03 04 00 10 09 07 01   ................
-|   4080: 06 02 04 00 10 09 03 01 06 01 04 00 10 09 01 01   ................
-| page 5 offset 16384
-|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
-|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
-| end crash-d57c01958e48ab.db
-}]} {}
-
-do_catchsql_test 8.1 {
-  SELECT rowid FROM t1('a* NOT ý‘') ;
-} {0 {1 2 3 4 5 6 7}}
-
-#-------------------------------------------------------------------------
-reset_db
-do_test 9.0 {
-  sqlite3 db {}
-  db deserialize [decode_hexdb {
-.open --hexdb
-| size 32768 pagesize 4096 filename crash-c76a16c24c8ba6.db
-| page 1 offset 0
-|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
-|     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 08   .....@  ........
-|     32: 00 00 00 02 00 00 00 01 00 00 00 09 00 00 00 04   ................
-|     96: 00 00 00 00 0d 0f c7 00 07 0d 92 00 0f 8d 0f 36   ...............6
-|    112: 0e cb 0e 6b 0e 0e 0d b6 0d 92 0d 92 00 00 00 00   ...k............
-|   3472: 00 00 22 08 06 17 11 11 01 31 74 61 62 6c 65 74   .........1tablet
-|   3488: 32 74 32 08 43 52 45 41 54 45 20 54 41 42 4c 45   2t2.CREATE TABLE
-|   3504: 20 74 32 28 78 29 56 07 06 17 1f 1f 01 7d 74 61    t2(x)V.......ta
-|   3520: 62 6c 65 74 31 5f 63 6f 6e 66 69 67 74 31 5f 63   blet1_configt1_c
-|   3536: 6f 6e 66 69 67 07 43 52 45 41 54 45 20 54 41 42   onfig.CREATE TAB
-|   3552: 4c 45 20 27 74 31 5f 63 6f 6e 66 69 67 27 28 6b   LE 't1_config'(k
-|   3568: 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20 76 29    PRIMARY KEY, v)
-|   3584: 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44 5b 06    WITHOUT ROWID[.
-|   3600: 07 17 21 21 01 81 01 74 61 62 6c 65 74 31 5f 64   ..!!...tablet1_d
-|   3616: 6f 63 73 69 7a 65 74 31 5f 64 6f 63 73 69 7a 65   ocsizet1_docsize
-|   3632: 06 43 52 45 41 54 45 20 54 41 42 4c 45 20 27 74   .CREATE TABLE 't
-|   3648: 31 5f 64 6f 63 73 69 7a 65 27 28 69 64 20 49 4e   1_docsize'(id IN
-|   3664: 54 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45   TEGER PRIMARY KE
-|   3680: 59 2c 20 73 7a 20 42 4c 4f 42 29 5e 05 07 17 21   Y, sz BLOB)^...!
-|   3696: 21 01 81 07 74 61 62 6c 65 74 31 5f 63 6f 6e 74   !...tablet1_cont
-|   3712: 65 6e 74 74 31 5f 63 6f 6e 74 65 6e 74 05 43 52   entt1_content.CR
-|   3728: 45 41 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63   EATE TABLE 't1_c
-|   3744: 6f 6e 74 65 6e 74 27 28 69 64 20 49 4e 54 45 47   ontent'(id INTEG
-|   3760: 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20   ER PRIMARY KEY, 
-|   3776: 63 30 2c 20 63 31 2c 20 63 32 29 69 04 07 17 19   c0, c1, c2)i....
-|   3792: 19 01 81 2d 74 61 62 6c 65 74 31 5f 69 64 78 74   ...-tablet1_idxt
-|   3808: 31 5f 69 64 78 04 43 52 45 41 54 45 20 54 41 42   1_idx.CREATE TAB
-|   3824: 4c 45 20 27 74 31 5f 69 64 78 27 28 73 65 67 69   LE 't1_idx'(segi
-|   3840: 64 2c 20 74 65 72 6d 2c 20 70 67 6e 6f 2c 20 50   d, term, pgno, P
-|   3856: 52 49 4d 41 52 59 20 4b 45 59 28 73 65 67 69 64   RIMARY KEY(segid
-|   3872: 2c 20 74 65 72 6d 29 29 20 57 49 54 48 4f 55 54   , term)) WITHOUT
-|   3888: 20 52 4f 57 49 44 55 03 07 17 1b 1b 01 81 01 74    ROWIDU........t
-|   3904: 61 62 6c 65 74 31 5f 64 61 74 61 74 31 5f 64 61   ablet1_datat1_da
-|   3920: 74 61 03 43 52 45 41 54 45 20 54 41 42 4c 45 20   ta.CREATE TABLE 
-|   3936: 27 74 31 5f 64 61 74 61 27 28 69 64 20 49 4e 54   't1_data'(id INT
-|   3952: 45 47 45 52 20 50 52 49 4d 41 52 59 20 4b 45 59   EGER PRIMARY KEY
-|   3968: 2c 20 62 6c 6f 63 6b 20 42 4c 4f 42 29 38 02 06   , block BLOB)8..
-|   3984: 17 11 11 08 5f 74 61 62 6c 65 74 31 74 31 43 52   ...._tablet1t1CR
-|   4000: 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41 42   EATE VIRTUAL TAB
-|   4016: 4c 45 20 74 31 20 55 53 49 4e 47 20 66 74 73 35   LE t1 USING fts5
-|   4032: 28 61 2c 62 2c 63 29 00 00 00 00 00 00 00 00 00   (a,b,c).........
-| page 3 offset 8192
-|      0: 0d 00 00 00 03 0c 94 00 0f e6 0f ef 0c 94 00 00   ................
-|   3216: 00 00 00 00 86 4a 84 80 80 80 80 01 04 00 8d 18   .....J..........
-|   3232: 00 00 03 2b 02 30 30 01 02 06 01 02 06 01 02 06   ...+.00.........
-|   3248: 1f 02 03 01 02 03 01 02 03 01 08 32 30 31 36 30   ...........20160
-|   3264: 36 30 39 01 02 07 01 02 07 01 02 07 01 01 34 01   609...........4.
-|   3280: 02 05 01 02 05 01 02 05 01 01 35 01 02 04 01 02   ..........5.....
-|   3296: 04 01 02 04 02 07 30 30 30 30 30 30 30 1c 02 04   ......0000000...
-|   3312: 01 02 04 01 02 04 01 06 62 69 6e 61 72 79 03 06   ........binary..
-|   3328: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01   ................
-|   3344: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02   ................
-|   3360: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02   ................
-|   3376: 03 06 01 02 02 03 06 01 02 02 01 08 63 6f 6d 70   ............comp
-|   3392: 69 6c 65 72 01 02 02 01 02 02 01 02 02 01 06 64   iler...........d
-|   3408: 62 73 74 61 74 07 02 03 01 02 03 01 02 03 02 04   bstat...........
-|   3424: 65 62 75 67 04 02 02 01 02 02 01 02 02 01 06 65   ebug...........e
-|   3440: 6e 61 62 6c 65 07 02 02 01 02 02 01 02 02 01 02   nable...........
-|   3456: 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02   ................
-|   3472: 01 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01   ................
-|   3488: 02 02 01 02 02 01 02 02 01 02 02 01 02 02 01 02   ................
-|   3504: 02 01 02 02 02 08 78 74 65 6e 73 69 6f 6e 1f 02   ......xtension..
-|   3520: 04 01 02 04 01 02 04 01 04 66 74 73 34 0a 02 03   .........fts4...
-|   3536: 01 02 03 01 02 03 04 01 35 0d 02 03 01 02 03 01   ........5.......
-|   3552: 02 03 01 03 67 63 63 01 02 03 01 02 03 01 02 03   ....gcc.........
-|   3568: 02 06 65 6f 70 6f 6c 79 10 02 03 01 02 03 01 02   ..eopoly........
-|   3584: 03 01 05 6a 73 6f 6e 31 13 02 03 01 02 03 01 02   ...json1........
-|   3600: 03 01 04 6c 6f 61 64 1f 02 03 01 02 03 01 02 03   ...load.........
-|   3616: 01 03 6d 61 78 1c 02 02 01 02 02 01 02 02 02 05   ..max...........
-|   3632: 65 6d 6f 72 79 1c 02 03 01 02 03 01 02 03 04 04   emory...........
-|   3648: 73 79 73 35 16 02 03 01 02 03 01 02 03 01 06 6e   sys5...........n
-|   3664: 6f 63 61 73 65 02 06 01 02 02 03 06 01 02 02 03   ocase...........
-|   3680: 06 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06   ................
-|   3696: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01   ................
-|   3712: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02   ................
-|   3728: 02 01 04 6f 6d 69 74 1f 02 02 01 02 02 01 02 02   ...omit.........
-|   3744: 01 05 72 74 72 65 65 19 02 03 01 02 03 01 02 03   ..rtree.........
-|   3760: 04 02 69 6d 01 06 01 02 02 03 06 01 02 02 03 06   ..im............
-|   3776: 01 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01   ................
-|   3792: 02 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02   ................
-|   3808: 02 03 06 01 02 02 03 06 01 02 02 03 06 01 02 02   ................
-|   3824: 01 0a 74 68 72 65 61 64 73 61 66 65 03 57 34 56   ..threadsafe.W4V
-|   3840: 94 64 91 46 85 84 04 76 74 61 62 07 02 04 01 02   .d.F...vtab.....
-|   3856: 04 01 02 04 01 01 78 01 06 01 01 02 01 06 01 01   ......x.........
-|   3872: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 10 02   ................
-|   3888: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01   ................
-|   3904: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06   ................
-|   3920: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01   ................
-|   3936: 01 02 01 06 01 01 10 01 06 01 01 02 01 06 01 01   ................
-|   3952: 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02   ................
-|   3968: 01 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01   ................
-|   3984: 06 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06   ................
-|   4000: 01 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01   ................
-|   4016: 01 02 01 06 01 01 02 01 06 01 01 02 01 06 01 01   ................
-|   4032: 02 01 06 01 01 02 01 06 01 01 02 04 15 13 0c 0c   ................
-|   4048: 12 44 13 11 0f 47 13 0f 0c 0e 11 10 0f 0e 10 0f   .D...G..........
-|   4064: 44 0f 10 40 15 0f 07 01 03 00 14 24 5a 24 24 0f   D..@.......$Z$$.
-|   4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01   ...$............
-| page 4 offset 12288
-|      0: 0a 00 00 00 01 0f fa 00 00 00 00 00 00 00 00 00   ................
-|   4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02   ................
-| page 5 offset 16384
-|      0: 0d 00 00 00 24 0c 0a 00 0f d8 0f af 0f 86 0f 74   ....$..........t
-|     16: 0f 61 0f 4e 0f 2f 0f 0f 0e ef 0e d7 0e be 0e a5   .a.N./..........
-|     32: 0e 8d 0e 74 0e 5b 0e 40 0e 24 0e 08 0d ef 0d d5   ...t.[.@.$......
-|     48: 0d bb 0d a0 0d 84 0d 68 0d 4f 0d 35 0d 1b 0c fb   .......h.O.5....
-|     64: 0c da 0c b9 0c 99 0c 78 0c 57 0c 3e 0c 24 0c 0a   .......x.W.>.$..
-|   3072: 00 00 00 00 00 00 00 00 00 00 18 24 05 00 25 0f   ...........$..%.
-|   3088: 19 54 48 52 45 41 44 53 41 46 45 3d 30 58 42 49   .THREADSAFE=0XBI
-|   3104: 4e 41 52 59 18 23 05 00 25 0f 19 54 48 52 45 41   NARY.#..%..THREA
-|   3120: 44 53 41 46 45 3d 30 58 4e 4f 43 41 53 45 17 22   DSAFE=0XNOCASE..
-|   3136: 05 00 25 0f 17 54 48 52 45 41 44 53 31 46 45 3d   ..%..THREADS1FE=
-|   3152: 30 58 52 64 52 49 4d 1f 21 05 00 33 0f 19 4f 4d   0XRdRIM.!..3..OM
-|   3168: 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49 4f   IT LOAD EXTENSIO
-|   3184: 4e 58 42 49 4e 41 52 59 1f 20 05 00 33 0f 19 4f   NXBINARY. ..3..O
-|   3200: 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53 49   MIT LOAD EXTENSI
-|   3216: 4f 4e 58 4e 4f 43 41 53 45 1e 1f 05 00 33 0f 17   ONXNOCASE....3..
-|   3232: 4f 4d 49 54 20 4c 4f 41 44 20 45 58 54 45 4e 53   OMIT LOAD EXTENS
-|   3248: 49 4f 4e 58 52 54 52 49 4d 1f 1e 05 00 33 0f 19   IONXRTRIM....3..
-|   3264: 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30 30   MAX MEMORY=50000
-|   3280: 30 30 30 58 42 49 4e 41 52 59 1f 1d 05 00 33 0f   000XBINARY....3.
-|   3296: 19 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30 30   .MAX MEMORY=5000
-|   3312: 30 30 30 30 58 4e 4f 43 41 53 45 1e 1c 05 00 33   0000XNOCASE....3
-|   3328: 0f 17 4d 41 58 20 4d 45 4d 4f 52 59 3d 35 30 30   ..MAX MEMORY=500
-|   3344: 30 30 30 30 30 58 52 54 52 49 4d 18 1b 05 00 25   00000XRTRIM....%
-|   3360: 0f 19 45 4e 41 42 4c 45 20 52 54 52 45 45 58 42   ..ENABLE RTREEXB
-|   3376: 49 4e 41 52 59 18 1a 05 00 25 0f 19 45 4e 41 42   INARY....%..ENAB
-|   3392: 4c 45 20 52 54 52 45 45 58 4e 4f 43 41 53 45 17   LE RTREEXNOCASE.
-|   3408: 19 05 00 25 0f 17 45 4e 41 42 4c 45 20 52 54 52   ...%..ENABLE RTR
-|   3424: 45 45 58 52 54 52 49 4d 1a 18 05 00 29 0f 19 45   EEXRTRIM....)..E
-|   3440: 4e 41 42 4b 45 20 4d 45 4d 53 59 53 35 58 42 49   NABKE MEMSYS5XBI
-|   3456: 4e 41 52 59 1a 17 05 00 29 0f 19 45 4e 41 42 4c   NARY....)..ENABL
-|   3472: 42 60 2d 45 4d 53 59 53 35 58 4e 4f 43 41 53 45   B`-EMSYS5XNOCASE
-|   3488: 19 16 05 00 29 0f 17 45 4e 41 42 4c 45 20 4d 45   ....)..ENABLE ME
-|   3504: 4d 53 59 53 35 58 52 54 52 49 4d 18 15 05 00 25   MSYS5XRTRIM....%
-|   3520: 0f 19 45 4e 41 42 4c 45 20 4a 53 4f 4e 31 58 42   ..ENABLE JSON1XB
-|   3536: 49 4e 41 52 59 18 14 05 00 25 0f 19 45 4e 41 42   INARY....%..ENAB
-|   3552: 4c 45 20 4a 53 4f 4e 31 58 4e 4f 43 41 53 45 17   LE JSON1XNOCASE.
-|   3568: 13 05 00 25 0f 17 45 4e 41 42 4c 45 20 4a 53 4f   ...%..ENABLE JSO
-|   3584: 4e 31 58 52 54 52 49 4d 1a 12 05 00 29 0f 19 45   N1XRTRIM....)..E
-|   3600: 4e 41 42 4c 45 20 47 45 4f 50 4f 4c 59 58 42 49   NABLE GEOPOLYXBI
-|   3616: 4e 41 52 59 1a 11 05 00 39 0f 19 45 4e 41 42 4c   NARY....9..ENABL
-|   3632: 45 20 47 45 4f 50 4f 4c 59 58 4e 4f 43 41 53 45   E GEOPOLYXNOCASE
-|   3648: 19 10 05 00 29 0f 17 45 4e 41 42 4c 45 20 47 45   ....)..ENABLE GE
-|   3664: 4f 50 4f 4c 59 58 52 54 52 49 4d 17 0f 05 00 23   OPOLYXRTRIM....#
-|   3680: 0f 19 45 4e 41 42 4c 45 20 46 54 53 35 58 42 49   ..ENABLE FTS5XBI
-|   3696: 4e 41 52 59 17 0e 05 00 23 0f 19 45 4e 41 42 4c   NARY....#..ENABL
-|   3712: 45 20 46 54 53 35 58 4e 4f 43 41 53 45 16 0d 05   E FTS5XNOCASE...
-|   3728: 00 23 0f 17 45 4e 41 42 4c 45 20 46 54 53 35 58   .#..ENABLE FTS5X
-|   3744: 52 54 52 49 4d 17 0c 05 00 23 0f 19 45 4e 41 42   RTRIM....#..ENAB
-|   3760: 4c 45 20 46 54 53 34 58 42 49 4e 41 52 59 17 0b   LE FTS4XBINARY..
-|   3776: 05 00 23 0f 19 45 4e 41 42 4c 45 20 46 54 53 34   ..#..ENABLE FTS4
-|   3792: 58 4e 4f 43 41 53 45 16 0a 05 00 23 0f 17 45 4e   XNOCASE....#..EN
-|   3808: 41 42 4c 45 20 46 54 53 34 58 52 54 52 49 4d 1e   ABLE FTS4XRTRIM.
-|   3824: 09 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53   ...1..ENABLE DBS
-|   3840: 54 41 54 20 56 54 41 42 58 42 49 4e 41 52 59 1e   TAT VTABXBINARY.
-|   3856: 08 05 00 31 0f 19 45 4e 41 42 4c 45 20 44 42 53   ...1..ENABLE DBS
-|   3872: 54 41 54 20 56 54 24 15 48 4e 4f 43 41 53 45 1d   TAT VT$.HNOCASE.
-|   3888: 07 05 00 31 0f 17 45 4e 41 42 4c 45 20 44 42 53   ...1..ENABLE DBS
-|   3904: 54 41 54 20 56 54 41 42 58 52 54 52 49 4d 11 06   TAT VTABXRTRIM..
-|   3920: 05 00 17 0f 19 44 45 42 55 47 58 42 49 4e 41 52   .....DEBUGXBINAR
-|   3936: 59 11 05 05 00 17 0f 19 44 45 42 55 47 58 4e 4f   Y.......DEBUGXNO
-|   3952: 43 41 53 45 10 04 05 00 17 0f 17 44 45 42 55 47   CASE.......DEBUG
-|   3968: 58 52 54 52 49 4d 27 03 05 00 43 0f 19 43 4f 4d   XRTRIM'...C..COM
-|   3984: 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e 30 20   PILER=gcc-5.4.0 
-|   4000: 32 30 31 36 30 36 30 39 58 42 49 4e 41 52 59 27   20160609XBINARY'
-|   4016: 02 05 00 43 0f 19 43 4f 4d 50 49 4c 45 52 3c 67   ...C..COMPILER<g
-|   4032: 63 63 2d 35 2e 34 2e 30 20 32 30 31 36 30 36 30   cc-5.4.0 2016060
-|   4048: 39 58 4e 4f 43 41 53 45 26 01 05 00 43 0f 17 43   9XNOCASE&...C..C
-|   4064: 4f 4d 50 49 4c 45 52 3d 67 63 63 2d 35 2e 34 2e   OMPILER=gcc-5.4.
-|   4080: 30 20 32 30 31 36 30 36 30 39 58 52 54 52 49 4d   0 20160609XRTRIM
-| page 6 offset 20480
-|      0: 0d 00 00 00 24 0e e0 00 0f f8 0f f0 0f e8 0f e0   ....$...........
-|     16: 0f d8 0f d0 0f c8 0f c0 0f b8 0f b0 0f a8 0f a0   ................
-|     32: 1f 98 0f 90 0f 88 0f 80 0f 78 0f 70 0f 68 0f 60   .........x.p.h.`
-|     48: 0f 58 0f 50 0f 48 0f 40 0f 38 0f 30 0f 28 0f 20   .X.P.H.@.8.0.(. 
-|     64: 0f 18 0f 10 0f 08 0f 00 0e f8 0e f0 0e e8 0e e0   ................
-|   3808: 06 24 03 00 12 02 01 01 06 23 03 00 12 02 01 01   .$.......#......
-|   3824: 06 22 03 00 12 02 01 01 06 21 03 00 12 03 01 01   .........!......
-|   3840: 06 20 03 00 12 03 01 01 06 1f 03 00 12 03 01 01   . ..............
-|   3856: 06 1e 03 00 12 03 01 01 06 1d 03 00 12 03 01 01   ................
-|   3872: 06 1c 03 00 12 03 01 01 06 1b 03 00 12 02 01 01   ................
-|   3888: 06 1a 03 00 12 02 01 01 06 19 03 00 12 02 01 01   ................
-|   3904: 06 18 03 00 12 02 01 01 06 17 03 00 12 02 01 01   ................
-|   3920: 06 15 f3 00 12 02 01 01 06 15 03 00 12 02 01 01   ................
-|   3936: 06 14 03 00 12 02 01 01 06 13 03 00 12 02 01 01   ................
-|   3952: 06 12 03 00 12 02 01 01 06 11 03 00 12 02 01 01   ................
-|   3968: 06 10 03 00 12 02 01 01 06 0f 03 00 12 02 01 01   ................
-|   3984: 06 0e 03 00 12 02 01 01 06 0d 03 00 12 02 01 01   ................
-|   4000: 06 0c 03 00 12 02 01 01 06 0b 03 00 12 02 01 01   ................
-|   4016: 06 0a 03 00 12 02 01 01 06 09 03 00 12 03 01 01   ................
-|   4032: 06 08 03 00 12 03 01 01 06 07 03 00 12 03 01 01   ................
-|   4048: 06 06 03 00 12 01 01 01 06 05 03 00 12 01 01 01   ................
-|   4064: 06 04 03 00 12 01 01 01 06 03 03 00 12 06 01 01   ................
-|   4080: 06 02 03 00 12 06 01 01 06 01 03 00 12 06 01 01   ................
-| page 7 offset 24576
-|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
-|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
-| page 8 offset 28672
-|      0: 0d 00 00 00 03 0f d6 00 0f f4 0f e9 0f d6 00 00   ................
-|   4048: 00 00 00 00 00 00 11 03 02 2b 69 6e 74 65 67 72   .........+integr
-|   4064: 69 74 79 2d 63 68 65 63 6b 09 02 02 1b 72 65 62   ity-check....reb
-|   4080: 75 69 6c 64 0a 01 02 1d 6f 70 74 69 00 00 00 00   uild....opti....
-| end crash-c76a16c24c8ba6.db
-}]} {}
-
-#.testctrl prng_seed 1 db
-#.testctrl internal_functions
-#.testctrl json_selfcheck on
-#
-
-do_execsql_test 9.1 { 
-  UPDATE t1 SET b=quote(zeroblob(current_date)) WHERE t1 MATCH 't*';
-  SAVEPOINT a;
-  UPDATE t1 SET b=quote(zeroblob(current_date)) WHERE t1 MATCH 't*';
-  INSERT INTO t1(t1,rank) VALUES('secure-delete',1);
-}
-do_catchsql_test 9.2 {
-  DELETE FROM t1;
-} {1 {database disk image is malformed}}
-
 sqlite3_fts5_may_be_corrupt 0
 finish_test
 

ext/fts5/test/fts5corrupt6.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt6
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5corrupt7.test

@@ -1,128 +0,0 @@
-# 2023 April 30
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5corrupt7
-
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-sqlite3_fts5_may_be_corrupt 1
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x);
-  INSERT INTO t1(t1, rank) VALUES('pgsz', 32);
-}
-
-set doc [string repeat "a b " 30]
-
-do_execsql_test 1.1 {
-  BEGIN;
-    INSERT INTO t1(rowid, x) VALUES(123, $doc);
-    INSERT INTO t1(rowid, x) VALUES(124, $doc);
-  COMMIT;
-}
-
-execsql_pp {
-  SELECT id, fts5_decode(id, block), quote(block) FROM t1_data
-}
-
-set rows [db eval { SELECT rowid FROM t1_data }]
-db_save_and_close
-
-foreach r $rows {
-  db_restore_and_reopen
-
-  proc edit_block {b} {
-    binary scan $b c* in
-    set out [lreplace $in 0 1 255 255]
-    binary format c* $out
-  }
-  db func edit_block edit_block
-
-  do_execsql_test 1.2.$r.1 {
-    UPDATE t1_data SET block = edit_block(block) WHERE rowid=$r;
-  }
-
-  do_execsql_test 1.2.$r.2 {
-    INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
-  }
-
-  do_test 1.2.$r.3 {
-    catchsql { DELETE FROM t1 WHERE rowid=123; }
-    catchsql { DELETE FROM t1 WHERE rowid=124; }
-    set {} {}
-  } {}
-
-  db close
-}
-
-foreach r $rows {
-set r 137438953475
-  db_restore_and_reopen
-
-  proc edit_block {b} {
-    binary scan $b c* in
-    set out [lreplace $in end end 127]
-    binary format c* $out
-  }
-  db func edit_block edit_block
-
-  do_execsql_test 1.2.$r.1 {
-    UPDATE t1_data SET block = edit_block(block) WHERE rowid=$r;
-  }
-
-  do_execsql_test 1.2.$r.2 {
-    INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
-  }
-
-  do_test 1.2.$r.3 {
-    catchsql { DELETE FROM t1 WHERE rowid=124; }
-    catchsql { DELETE FROM t1 WHERE rowid=123; }
-    set {} {}
-  } {}
-
-  db close
-}
-
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x);
-  BEGIN;
-    INSERT INTO t1 VALUES('abc');
-    INSERT INTO t1 VALUES('b d d d');
-  COMMIT;
-  INSERT INTO t1(t1, rank) VALUES('secure-delete', 1);
-}
-
-execsql_pp {
-  SELECT id, quote(block) FROM t1_data
-}
-
-do_execsql_test 2.1 {
-  SELECT quote(block) FROM t1_data WHERE id > 10;
-}        {X'0000001A04306162630102020101620202020101640206030303040806'}
-
-do_execsql_test 2.2 {
-  UPDATE t1_data SET 
-    block=X'0000001A04306162630102025501620202020101640206030303040806'
-  WHERE id>10
-}
-
-do_catchsql_test 2.3 {
-  DELETE FROM t1 WHERE rowid = 1
-} {1 {database disk image is malformed}}
-
-finish_test

ext/fts5/test/fts5dlidx.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5dlidx
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5doclist.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5doclist
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5ea.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ea
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5eb.test

@@ -13,7 +13,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5eb
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -86,18 +86,15 @@ do_execsql_test 3.0 {
   INSERT INTO e1 VALUES ('just a few words with a / inside');
 }
 do_execsql_test 3.1 {
-  SELECT rowid, format('%g',bm25(e1)) FROM e1 WHERE e1 MATCH '"just"' ORDER BY rank;
+  SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"just"' ORDER BY rank;
 } {1 -1e-06}
 do_execsql_test 3.2 {
   SELECT rowid FROM e1 WHERE e1 MATCH '"/" OR "just"'
 } 1
 do_execsql_test 3.3 {
-  SELECT rowid, format('%g',bm25(e1)) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank;
+  SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank;
 } {1 -1e-06}
 
-do_execsql_test 3.4 "
-  SELECT fts5_expr_tcl('e AND \" \"');
-" {{AND [nearset -- {e}] [{}]}}
 
 
 finish_test

ext/fts5/test/fts5expr.test

@@ -1,48 +0,0 @@
-# 2024 August 8
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The
-# focus of this script is testing the FTS5 module.
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5expr
-
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE x1 USING fts5(a);
-  INSERT INTO x1(rowid, a) VALUES (113, 'fts5 expr test');
-}
-
-do_execsql_test 1.1 {
-  SELECT rowid FROM x1('expr');
-} {113}
-
-for {set ii 0} {$ii < 300} {incr ii} {
-  set expr "expr "
-  append expr [string repeat "NOT abcd " $ii]
-
-  if {$ii<257} {
-    set res {0 113}
-  } else {
-    set res {1 {fts5 expression tree is too large (maximum depth 256)}}
-  }
-  do_catchsql_test 1.1.$ii {
-    SELECT rowid FROM x1($expr)
-  } $res
-}
-
-finish_test
-

ext/fts5/test/fts5fault6.test

@@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl]
 source $testdir/malloc_common.tcl
 set testprefix fts5fault6
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5fault8.test

@@ -57,6 +57,7 @@ foreach_detail_mode $testprefix {
 
 } ;# foreach_detail_mode...
 
+
 do_execsql_test 4.0 {
   CREATE VIRTUAL TABLE x2 USING fts5(a);
   INSERT INTO x2(x2, rank) VALUES('crisismerge', 2);
@@ -79,18 +80,5 @@ do_faultsim_test 4 -faults oom-* -prep {
   faultsim_test_result {0 {}} {1 SQLITE_NOMEM}
 }
 
-set TMPDBERROR {1 {unable to open a temporary database file for storing temporary tables}}
-
-do_faultsim_test 5 -faults oom-t* -prep {
-  faultsim_restore_and_reopen
-  execsql { PRAGMA temp_store = memory }
-} -body {
-  execsql { PRAGMA integrity_check }
-} -test {
-  if {[string match {*error code=7*} $testresult]==0} {
-    faultsim_test_result {0 ok} {1 SQLITE_NOMEM} $::TMPDBERROR
-  }
-}
-
 
 finish_test

ext/fts5/test/fts5faultF.test

@@ -1,111 +0,0 @@
-# 2023 July 20
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-#
-# This file is focused on OOM errors. Particularly those that may occur
-# when using contentless_delete=1 databases.
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-source $testdir/malloc_common.tcl
-set testprefix fts5faultF
-
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-
-faultsim_save_and_close
-do_faultsim_test 1 -prep {
-  faultsim_restore_and_reopen
-} -body {
-  execsql { 
-    CREATE VIRTUAL TABLE t1 USING fts5(x, y, content=, contentless_delete=1) 
-  }
-} -test {
-  faultsim_test_result {0 {}} {1 {vtable constructor failed: t1}} 
-}
-
-reset_db
-do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(doc, content=, contentless_delete=1);
-  BEGIN;
-    INSERT INTO t1(rowid, doc) VALUES(1, 'a b c d');
-    INSERT INTO t1(rowid, doc) VALUES(2, 'a b c d');
-    INSERT INTO t1(rowid, doc) VALUES(3, 'a b c d');
-    INSERT INTO t1(rowid, doc) VALUES(4, 'a b c d');
-  COMMIT;
-  DELETE FROM t1 WHERE rowid IN (2, 4);
-}
-
-do_faultsim_test 2 -prep {
-  sqlite3 db test.db
-  execsql { SELECT rowid FROM t1 }
-} -body {
-  execsql { 
-    SELECT rowid FROM t1('b c');
-  }
-} -test {
-  faultsim_test_result {0 {1 3}}
-}
-
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 3.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(doc, content=, contentless_delete=1);
-  BEGIN;
-    INSERT INTO t1(rowid, doc) VALUES(1, 'a b c d');
-    INSERT INTO t1(rowid, doc) VALUES(2, 'a b c d');
-    INSERT INTO t1(rowid, doc) VALUES(3, 'a b c d');
-    INSERT INTO t1(rowid, doc) VALUES(4, 'a b c d');
-  COMMIT;
-}
-
-faultsim_save_and_close
-do_faultsim_test 3 -prep {
-  faultsim_restore_and_reopen
-  execsql { SELECT rowid FROM t1 }
-} -body {
-  execsql { 
-    INSERT INTO t1(rowid, doc) VALUES(5, 'a b c d');
-  }
-} -test {
-  faultsim_test_result {0 {}}
-}
-
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 4.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(doc, content=, contentless_delete=1);
-  INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
-  WITH s(i) AS (
-    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
-  )
-  INSERT INTO t1(rowid, doc) SELECT i, 'a b c d' FROM s;
-}
-
-do_execsql_test 4.1 { DELETE FROM t1 WHERE rowid <= 25 }
-
-faultsim_save_and_close
-do_faultsim_test 4 -faults oom-t* -prep {
-  faultsim_restore_and_reopen
-  execsql { SELECT rowid FROM t1 }
-} -body {
-  execsql { 
-    DELETE FROM t1 WHERE rowid < 100
-  }
-} -test {
-  faultsim_test_result {0 {}}
-}
-
-
-finish_test
-

ext/fts5/test/fts5faultG.test

@@ -1,76 +0,0 @@
-# 2010 June 15
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-source $testdir/malloc_common.tcl
-set testprefix fts5faultG
-
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-
-set ::testprefix fts5faultG
-
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a);
-  INSERT INTO t1 VALUES('test renaming the table');
-  INSERT INTO t1 VALUES(' after it has been written');
-  INSERT INTO t1 VALUES(' actually other stuff instead');
-}
-faultsim_save_and_close
-do_faultsim_test 1 -faults oom* -prep { 
-  faultsim_restore_and_reopen
-  execsql {
-    BEGIN;
-      DELETE FROM t1 WHERE rowid=2;
-  }
-} -body {
-  execsql {
-    DELETE FROM t1;
-  }
-} -test {
-  catchsql { COMMIT }
-  faultsim_integrity_check
-  faultsim_test_result {0 {}}
-}
-
-reset_db
-do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, content=, contentless_delete=1);
-  BEGIN;
-    INSERT INTO t1 VALUES('here''s some text');
-    INSERT INTO t1 VALUES('useful stuff, text');
-    INSERT INTO t1 VALUES('what would we do without text!'); 
-  COMMIT;
-}
-faultsim_save_and_close
-do_faultsim_test 2 -faults oom* -prep { 
-  faultsim_restore_and_reopen
-  execsql {
-    BEGIN;
-      DELETE FROM t1 WHERE rowid=2;
-  }
-} -body {
-  execsql {
-    INSERT INTO t1(t1) VALUES('optimize');
-  }
-} -test {
-  faultsim_integrity_check
-  faultsim_test_result {0 {}}
-}
-
-
-
-finish_test

ext/fts5/test/fts5faultH.test

@@ -1,150 +0,0 @@
-# 2010 June 15
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-source $testdir/malloc_common.tcl
-set testprefix fts5faultG
-
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-
-set ::testprefix fts5faultH
-
-sqlite3_fts5_register_origintext db
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(
-      x, tokenize="origintext unicode61", tokendata=1
-  );
-
-  BEGIN;
-    INSERT INTO t1 VALUES('oNe tWo thRee');
-    INSERT INTO t1 VALUES('One Two Three');
-    INSERT INTO t1 VALUES('onE twO threE');
-  COMMIT;
-  BEGIN;
-    INSERT INTO t1 VALUES('one two three');
-    INSERT INTO t1 VALUES('one two three');
-    INSERT INTO t1 VALUES('one two three');
-  COMMIT;
-}
-
-do_faultsim_test 1 -faults oom* -prep { 
-} -body {
-  execsql {
-    SELECT rowid FROM t1('three');
-  }
-} -test {
-  faultsim_integrity_check
-  faultsim_test_result {0 {1 2 3 4 5 6}}
-}
-
-
-reset_db
-sqlite3_fts5_register_origintext db
-do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(
-      x, tokenize="origintext unicode61", tokendata=1
-  );
-  INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
-
-  BEGIN;
-    INSERT INTO t1(rowid, x) VALUES(10, 'aaa bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(12, 'bbb bbb bbb');
-    INSERT INTO t1(rowid, x) VALUES(13, 'bbb bbb bbb');
-    INSERT INTO t1(rowid, x) VALUES(14, 'bbb BBB bbb');
-    INSERT INTO t1(rowid, x) VALUES(15, 'bbb bbb bbb');
-    INSERT INTO t1(rowid, x) VALUES(16, 'bbb bbb bbb');
-    INSERT INTO t1(rowid, x) VALUES(17, 'bbb bbb bbb');
-    INSERT INTO t1(rowid, x) VALUES(18, 'bbb bbb bbb');
-    INSERT INTO t1(rowid, x) VALUES(19, 'bbb bbb bbb');
-    INSERT INTO t1(rowid, x) VALUES(20, 'bbb bbb bbb');
-    INSERT INTO t1(rowid, x) VALUES(21, 'bbb bbb bbb');
-    INSERT INTO t1(rowid, x) VALUES(22, 'bbb bbb bbb');
-    INSERT INTO t1(rowid, x) VALUES(23, 'bbb bbb bbb');
-    INSERT INTO t1(rowid, x) VALUES(24, 'aaa bbb BBB');
-  COMMIT;
-}
-
-do_faultsim_test 2 -faults oom* -prep { 
-} -body {
-  execsql {
-    SELECT rowid FROM t1('BBB AND AAA');
-  }
-} -test {
-  faultsim_integrity_check
-  faultsim_test_result {0 {10 24}}
-}
-
-reset_db
-sqlite3_fts5_register_origintext db
-do_execsql_test 3.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(
-      x, tokenize="origintext unicode61", tokendata=1
-  );
-  INSERT INTO t1(t1, rank) VALUES('pgsz', 64);
-
-  INSERT INTO t1(rowid, x) VALUES(9, 'bbb Bbb BBB');
-  BEGIN;
-    INSERT INTO t1(rowid, x) VALUES(10, 'aaa bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(11, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(12, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(13, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(14, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(15, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(16, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(17, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(18, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(19, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(20, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(21, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(22, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(23, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(24, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(25, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(26, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(27, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(28, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(29, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(30, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(31, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(32, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(33, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(34, 'bbb Bbb BBB');
-    INSERT INTO t1(rowid, x) VALUES(35, 'aaa bbb BBB');
-  COMMIT;
-}
-
-do_faultsim_test 3.1 -faults oom* -prep { 
-} -body {
-  execsql {
-    SELECT rowid FROM t1('BBB AND AAA');
-  }
-} -test {
-  faultsim_integrity_check
-  faultsim_test_result {0 {10 35}}
-}
-do_faultsim_test 3.2 -faults oom* -prep { 
-} -body {
-  execsql {
-    SELECT count(*) FROM t1('BBB');
-  }
-} -test {
-  faultsim_integrity_check
-  faultsim_test_result {0 27}
-}
-
-
-finish_test

ext/fts5/test/fts5first.test

@@ -22,7 +22,6 @@ do_execsql_test 1.0 {
   CREATE VIRTUAL TABLE x1 USING fts5(a, b);
 }
 
-unset -nocomplain res
 foreach {tn expr ok} {
   1 {^abc}           1
   2 {^abc + def}     1

ext/fts5/test/fts5full.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5full
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5hash.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5hash
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5integrity.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5integrity
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -77,9 +77,6 @@ do_catchsql_test 4.2 {
     UPDATE aa_docsize SET sz = X'44' WHERE rowid = 3;
     INSERT INTO aa(aa) VALUES('integrity-check'); 
 } {1 {database disk image is malformed}}
-do_execsql_test 4.2.1 {
-  PRAGMA integrity_check(aa);
-} {{malformed inverted index for FTS5 table main.aa}}
 
 do_catchsql_test 4.3 { 
   ROLLBACK;
@@ -153,7 +150,6 @@ do_execsql_test 5.3 {
   INSERT INTO gg(gg) VALUES('integrity-check');
 }
 
-unset -nocomplain res
 do_test 5.4.1 {
   set ok 0
   for {set i 0} {$i < 10000} {incr i} {
@@ -321,92 +317,4 @@ do_catchsql_test 10.5.3 {
   INSERT INTO vt0(vt0) VALUES('integrity-check');
 } {0 {}}
 
-reset_db
-proc slang {in} {return [string map {th d e eh} $in]}
-db function slang -deterministic -innocuous slang
-do_execsql_test 11.0 {
-  CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c TEXT AS (slang(b)));
-  INSERT INTO t1(b) VALUES('the quick fox jumps over the lazy brown dog');
-  SELECT c FROM t1;
-} {{deh quick fox jumps ovehr deh lazy brown dog}}
-
-do_execsql_test 11.1 {
-  CREATE VIRTUAL TABLE t2 USING fts5(content="t1", c);
-  INSERT INTO t2(t2) VALUES('rebuild');
-  SELECT rowid FROM t2 WHERE t2 MATCH 'deh';
-} {1}
-
-do_execsql_test 11.2 {
-  PRAGMA integrity_check(t2);
-} {ok}
-db close
-sqlite3 db test.db
-
-# FIX ME?
-#
-# FTS5 integrity-check does not care if the content table is unreadable or
-# does not exist.  It only looks for internal inconsistencies in the
-# inverted index.
-#
-do_execsql_test 11.3 {
-  PRAGMA integrity_check(t2);
-} {ok}
-do_execsql_test 11.4 {
-  DROP TABLE t1;
-  PRAGMA integrity_check(t2);
-} {ok}
-
-#-------------------------------------------------------------------
-reset_db
-
-do_execsql_test 12.1 {
-  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
-  INSERT INTO x1 VALUES('one', 'two');
-  INSERT INTO x1 VALUES('three', 'four');
-  INSERT INTO x1 VALUES('five', 'six');
-}
-
-do_execsql_test 12.2 {
-  PRAGMA integrity_check
-} {ok}
-
-db close
-sqlite3 db test.db -readonly 1
-
-explain_i {
-  PRAGMA integrity_check
-  }
-do_execsql_test 12.3 {
-  PRAGMA integrity_check
-} {ok}
-
-
-#-------------------------------------------------------------------
-reset_db
-do_execsql_test 13.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=ascii);
-  INSERT INTO t1 VALUES('a b c'), ('d e f');
-  PRAGMA integrity_check;
-} {ok}
-
-db close
-sqlite3 db test.db
-do_catchsql_test 13.2 {
-  PRAGMA integrity_check;
-} {0 ok}
-
-do_execsql_test 13.3 {
-  PRAGMA writable_schema = 1;
-  UPDATE sqlite_schema SET sql = 'CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=blah)'
-  WHERE name = 't1';
-}
-
-db close
-sqlite3 db test.db
-breakpoint
-do_catchsql_test 13.4 {
-  PRAGMA integrity_check;
-} {1 {SQL logic error}}
-
-
 finish_test

ext/fts5/test/fts5interrupt.test

@@ -33,7 +33,6 @@ proc progress_handler {args} {
   return 0
 }
 
-unset -nocomplain res
 foreach {tn sql} {
   1 { INSERT INTO t1(rowid, a) VALUES(0, 'z z z z') }
   2 { COMMIT }
@@ -65,3 +64,4 @@ foreach {tn sql} {
 }
 
 finish_test
+

ext/fts5/test/fts5lastrowid.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5lastrowid
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return