Merge the latest branch-3.46 changes into wal2 as a new sub-branch

called wal2-3.46.

FossilOrigin-Name: b917d07fc66c5af9e4839c4b9ed2e54bc18cca5209d54f32b34b65d31dfc3740

Makefile.in

@@ -121,6 +121,10 @@ TCLSH_CMD = @TCLSH_CMD@
 #
 TCLLIBDIR = @TCLLIBDIR@
 
+# The suffix used on shared libraries.  Ex:  ".dll", ".so", ".dylib"
+#
+SHLIB_SUFFIX = @TCL_SHLIB_SUFFIX@
+
 # If gcov support was enabled by the configure script, add the appropriate
 # flags here.  It's not always as easy as just having the user add the right
 # CFLAGS / LDFLAGS, because libtool wants to use CFLAGS when linking, which
@@ -424,6 +428,7 @@ TESTSRC += \
   $(TOP)/ext/expert/sqlite3expert.c \
   $(TOP)/ext/expert/test_expert.c \
   $(TOP)/ext/misc/amatch.c \
+  $(TOP)/ext/misc/bgckpt.c \
   $(TOP)/ext/misc/appendvfs.c \
   $(TOP)/ext/misc/basexx.c \
   $(TOP)/ext/misc/carray.c \
@@ -450,7 +455,6 @@ TESTSRC += \
   $(TOP)/ext/misc/remember.c \
   $(TOP)/ext/misc/series.c \
   $(TOP)/ext/misc/spellfix.c \
-  $(TOP)/ext/misc/stmtrand.c \
   $(TOP)/ext/misc/totype.c \
   $(TOP)/ext/misc/unionvtab.c \
   $(TOP)/ext/misc/wholenumber.c \
@@ -796,10 +800,6 @@ has_tclsh85:
 	sh $(TOP)/tool/cktclsh.sh 8.5 $(TCLSH_CMD)
 	touch has_tclsh85
 
-has_tclconfig:
-	@ if test x"$(HAVE_TCL)" != "x1"; then	echo 'ERROR: Requires access to "tclConfig.sh" which "configure" was not able to locate'; exit 1; fi
-	touch has_tclconfig
-
 
 # This target creates a directory named "tsrc" and fills it with
 # copies of all of the C source code and header files needed to
@@ -1122,7 +1122,7 @@ tclsqlite-shell.lo:	$(TOP)/src/tclsqlite.c $(HDR)
 tclsqlite-stubs.lo:	$(TOP)/src/tclsqlite.c $(HDR)
 	$(LTCOMPILE) -DUSE_TCL_STUBS=1 -o $@ -c $(TOP)/src/tclsqlite.c
 
-tclsqlite3$(TEXE):	has_tclconfig tclsqlite-shell.lo libsqlite3.la
+tclsqlite3$(TEXE):	tclsqlite-shell.lo libsqlite3.la
 	$(LTLINK) -o $@ tclsqlite-shell.lo \
 		 libsqlite3.la $(LIBTCL)
 
@@ -1309,7 +1309,7 @@ TESTFIXTURE_SRC1 = sqlite3.c
 TESTFIXTURE_SRC = $(TESTSRC) $(TOP)/src/tclsqlite.c
 TESTFIXTURE_SRC += $(TESTFIXTURE_SRC$(USE_AMALGAMATION))
 
-testfixture$(TEXE):	has_tclconfig has_tclsh85 $(TESTFIXTURE_SRC)
+testfixture$(TEXE):	has_tclsh85 $(TESTFIXTURE_SRC)
 	$(LTLINK) -DSQLITE_NO_SYNC=1 $(TEMP_STORE) $(TESTFIXTURE_FLAGS) \
 		-o $@ $(TESTFIXTURE_SRC) $(LIBTCL) $(TLIBS)
 
@@ -1408,13 +1408,13 @@ shelltest: $(TESTPROGS)
 sqlite3_analyzer.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/spaceanal.tcl $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in has_tclsh85
 	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqlite3_analyzer.c.in >sqlite3_analyzer.c
 
-sqlite3_analyzer$(TEXE): has_tclconfig sqlite3_analyzer.c
+sqlite3_analyzer$(TEXE): sqlite3_analyzer.c
 	$(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS)
 
 sqltclsh.c: sqlite3.c $(TOP)/src/tclsqlite.c $(TOP)/tool/sqltclsh.tcl $(TOP)/ext/misc/appendvfs.c $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in has_tclsh85
 	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/tool/sqltclsh.c.in >sqltclsh.c
 
-sqltclsh$(TEXE): has_tclconfig sqltclsh.c
+sqltclsh$(TEXE): sqltclsh.c
 	$(LTLINK) sqltclsh.c -o $@ $(LIBTCL) $(TLIBS)
 
 sqlite3_expert$(TEXE): $(TOP)/ext/expert/sqlite3expert.h $(TOP)/ext/expert/sqlite3expert.c $(TOP)/ext/expert/expert.c sqlite3.c
@@ -1433,7 +1433,7 @@ CHECKER_DEPS =\
 sqlite3_checker.c:	$(CHECKER_DEPS) has_tclsh85
 	$(TCLSH_CMD) $(TOP)/tool/mkccode.tcl $(TOP)/ext/repair/sqlite3_checker.c.in >$@
 
-sqlite3_checker$(TEXE):	has_tclconfig sqlite3_checker.c
+sqlite3_checker$(TEXE):	sqlite3_checker.c
 	$(LTLINK) sqlite3_checker.c -o $@ $(LIBTCL) $(TLIBS)
 
 dbdump$(TEXE): $(TOP)/ext/misc/dbdump.c sqlite3.lo
@@ -1558,37 +1558,12 @@ install:	sqlite3$(TEXE) lib_install sqlite3.h sqlite3.pc ${HAVE_TCL:1=tcl_instal
 
 pkgIndex.tcl:
 	echo 'package ifneeded sqlite3 $(RELEASE) [list load [file join $$dir libtclsqlite3[info sharedlibextension]] sqlite3]' > $@
-
 tcl_install:	lib_install libtclsqlite3.la pkgIndex.tcl
 	$(INSTALL) -d $(DESTDIR)$(TCLLIBDIR)
 	$(LTINSTALL) libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)
 	rm -f $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.la $(DESTDIR)$(TCLLIBDIR)/libtclsqlite3.a
 	$(INSTALL) -m 0644 pkgIndex.tcl $(DESTDIR)$(TCLLIBDIR)
 
-# Build the SQLite TCL extension in a way that make it compatible
-# with whatever version of TCL is running as $TCLSH_CMD, possibly defined
-# by --with-tclsh=
-#
-tclextension:	tclsqlite3.c
-	$(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --build-only --cc $(CC) $(CFLAGS) $(OPT_FEATURE_FLAGS) $(OPTS)
-
-# Install the SQLite TCL extension in a way that is appropriate for $TCLSH_CMD
-# to find it.
-#
-tclextension-install:	tclsqlite3.c
-	$(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --cc $(CC) $(CFLAGS) $(OPT_FEATURE_FLAGS) $(OPTS)
-
-# Install the SQLite TCL extension that is used by $TCLSH_CMD
-#
-tclextension-uninstall: 
-	$(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --uninstall
-
-# List all installed the SQLite TCL extension that is are accessible
-# by $TCLSH_CMD, included prior versions.
-#
-tclextension-list: 
-	$(TCLSH_CMD) $(TOP)/tool/buildtclext.tcl --info
-
 clean:
 	rm -f *.lo *.la *.o sqlite3$(TEXE) libsqlite3.la
 	rm -f sqlite3.h opcodes.*

Makefile.msc

@@ -1572,6 +1572,7 @@ TESTEXT = \
   $(TOP)\ext\misc\amatch.c \
   $(TOP)\ext\misc\appendvfs.c \
   $(TOP)\ext\misc\basexx.c \
+  $(TOP)\ext\misc\bgckpt.c \
   $(TOP)\ext\misc\carray.c \
   $(TOP)\ext\misc\cksumvfs.c \
   $(TOP)\ext\misc\closure.c \
@@ -1596,7 +1597,6 @@ TESTEXT = \
   $(TOP)\ext\misc\remember.c \
   $(TOP)\ext\misc\series.c \
   $(TOP)\ext\misc\spellfix.c \
-  $(TOP)\ext\misc\stmtrand.c \
   $(TOP)\ext\misc\totype.c \
   $(TOP)\ext\misc\unionvtab.c \
   $(TOP)\ext\misc\wholenumber.c \
@@ -1820,19 +1820,6 @@ pkgIndex.tcl:	$(TOP)\VERSION
 
 $(SQLITE3TCLDLL):	libtclsqlite3.lib $(LIBRESOBJS) tclsqlite3.def pkgIndex.tcl
 	$(LD) $(LDFLAGS) $(LTLINKOPTS) $(LTLIBPATHS) /DLL /DEF:tclsqlite3.def /OUT:$@ libtclsqlite3.lib $(LIBRESOBJS) $(LTLIBS) $(TLIBS)
-
-tclextension:	$(SQLITE3TCLDLL)
-
-tclextension-install:	$(SQLITE3TCLDLL)
-	$(TCLSH_CMD) $(TOP)\tool\buildtclext.tcl --install-only
-
-tclextension-uninstall:
-	$(TCLSH_CMD) $(TOP)\tool\buildtclext.tcl --uninstall
-
-tclextension-list:
-	$(TCLSH_CMD) $(TOP)\tool\buildtclext.tcl --info
-
-
 # <</mark>>
 
 $(SQLITE3DLL):	$(LIBOBJ) $(LIBRESOBJS) $(CORE_LINK_DEP)

README.md

@@ -1,19 +1,14 @@
 <h1 align="center">SQLite Source Repository</h1>
 
 This repository contains the complete source code for the
-[SQLite database engine](https://sqlite.org/), including
-many test scripts.  However, other test scripts
+[SQLite database engine](https://sqlite.org/).  Some test scripts
+are also included.  However, many other test scripts
 and most of the documentation are managed separately.
 
-See the [on-line documentation](https://sqlite.org/) for more information
-about what SQLite is and how it works from a user's perspective.  This
-README file is about the source code that goes into building SQLite,
-not about how SQLite is used.
-
 ## Version Control
 
 SQLite sources are managed using
-[Fossil](https://fossil-scm.org/), a distributed version control system
+[Fossil](https://www.fossil-scm.org/), a distributed version control system
 that was specifically designed and written to support SQLite development.
 The [Fossil repository](https://sqlite.org/src/timeline) contains the urtext.
 
@@ -73,11 +68,12 @@ archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows:
      then click on the "Tarball" or "ZIP Archive" links on the information
      page.
 
-To access sources directly using Fossil, first install Fossil version 2.0 or later.
-Source tarballs and precompiled binaries available
-[here](https://www.fossil-scm.org/home/uv/download.html).  Fossil is
+If you do want to use Fossil to check out the source tree,
+first install Fossil version 2.0 or later.
+(Source tarballs and precompiled binaries available
+[here](https://www.fossil-scm.org/fossil/uv/download.html).  Fossil is
 a stand-alone program.  To install, simply download or build the single
-executable file and put that file someplace on your $PATH.
+executable file and put that file someplace on your $PATH.)
 Then run commands like this:
 
         mkdir -p ~/sqlite ~/Fossils
@@ -85,8 +81,8 @@ Then run commands like this:
         fossil clone https://www.sqlite.org/src ~/Fossils/sqlite.fossil
         fossil open ~/Fossils/sqlite.fossil
 
-After setting up a repository using the steps above, you can do
-bandwidth-efficient updates to the latest version using:
+After setting up a repository using the steps above, you can always
+update to the latest version using:
 
         fossil update trunk   ;# latest trunk check-in
         fossil update release ;# latest official release
@@ -103,30 +99,15 @@ script found at the root of the source tree.  Then run "make".
 
 For example:
 
-        tar xzf sqlite.tar.gz      ;#  Unpack the source tree into "sqlite"
-        mkdir bld                  ;#  Build will occur in a sibling directory
-        cd bld                     ;#  Change to the build directory
-        ../sqlite/configure        ;#  Run the configure script
-        make sqlite3               ;#  Builds the "sqlite3" command-line tool
-        make sqlite3.c             ;#  Build the "amalgamation" source file
-        make mdevtest              ;#  Run development tests (requires tcl-dev)
-        make releasetest           ;#  Run full release tests (requires tcl-dev)
-        make sqldiff               ;#  Builds the "sqldiff" command-line tool
-        make sqlite3_analyzer      ;#  Builds the "sqlite3_analyzer" tool (requires tcl-dev)
-        make tclextension-install  ;#  Build and install the SQLite TCL extension
+        tar xzf sqlite.tar.gz    ;#  Unpack the source tree into "sqlite"
+        mkdir bld                ;#  Build will occur in a sibling directory
+        cd bld                   ;#  Change to the build directory
+        ../sqlite/configure      ;#  Run the configure script
+        make                     ;#  Builds the "sqlite3" command-line tool
+        make sqlite3.c           ;#  Build the "amalgamation" source file
+        make devtest             ;#  Run some tests (requires Tcl)
 
-See the makefile for additional targets.  For debugging builds, the
-core developers typically run "configure" with options like this:
-
-        ../sqlite/configure --enable-all --enable-debug CFLAGS='-O0 -g'
-
-For release builds, the core developers usually do:
-
-        ../sqlite/configure --enable-all
-
-Almost all makefile targets require a "tclsh" TCL interpreter
-version 8.6 or later.  The targets marked with "(requires tcl-dev)" also require
-the TCL development libraries.
+See the makefile for additional targets.
 
 The configure script uses autoconf 2.61 and libtool.  If the configure
 script does not work out for you, there is a generic makefile named
@@ -147,61 +128,48 @@ TCL library, using a command like this:
 
         set TCLDIR=c:\Tcl
 
-SQLite uses "tclsh.exe" as part of the build process, and so that
-program will need to be somewhere on your %PATH%.  SQLite itself
-does not contain any TCL code, but it does use TCL to help with the
-build process and to run tests.
+SQLite uses "tclsh.exe" as part of the build process, and so that utility
+program will need to be somewhere on your %PATH%.  The finished SQLite library
+does not contain any TCL code, but it does use TCL to help with the build process
+and to run tests.
 
 Build using Makefile.msc.  Example:
 
-        nmake /f Makefile.msc sqlite3.exe
+        nmake /f Makefile.msc
         nmake /f Makefile.msc sqlite3.c
-        nmake /f Makefile.msc mdevtest
+        nmake /f Makefile.msc devtest
         nmake /f Makefile.msc releasetest
-        nmake /f Makefile.msc tclextension-install
  
 There are many other makefile targets.  See comments in Makefile.msc for
 details.
 
-## Source Tree Map
+## Source Code Tour
 
-  *  **src/** - This directory contains the primary source code for the
-     SQLite core.  For historical reasons, C-code used for testing is
-     also found here.  Source files intended for testing begin with "`test`".
-     The `tclsqlite3.c` and `tclsqlite3.h` files are the TCL interface
-     for SQLite and are also not part of the core.
+Most of the core source files are in the **src/** subdirectory.  The
+**src/** folder also contains files used to build the "testfixture" test
+harness. The names of the source files used by "testfixture" all begin
+with "test".
+The **src/** also contains the "shell.c" file
+which is the main program for the "sqlite3.exe"
+[command-line shell](https://sqlite.org/cli.html) and
+the "tclsqlite.c" file which implements the
+[Tcl bindings](https://sqlite.org/tclsqlite.html) for SQLite.
+(Historical note:  SQLite began as a Tcl
+extension and only later escaped to the wild as an independent library.)
 
-  *  **test/** - This directory and its subdirectories contains code used
-     for testing.  Files that end in "`.test`" are TCL scripts that run
-     tests using an augmented TCL interpreter named "testfixture".  Use
-     a command like "`make testfixture`" (unix) or 
-     "`nmake /f Makefile.msc testfixture.exe`" (windows) to build that
-     augmented TCL interpreter, then run individual tests using commands like
-     "`testfixture test/main.test`".  This test/ subdirectory also contains
-     additional C code modules and scripts for other kinds of testing.
+Test scripts and programs are found in the **test/** subdirectory.
+Additional test code is found in other source repositories.
+See [How SQLite Is Tested](https://www.sqlite.org/testing.html) for
+additional information.
 
-  *  **tool/** - This directory contains programs and scripts used to
-     build some of the machine-generated code that goes into the SQLite
-     core, as well as to build and run tests and perform diagnostics.
-     The source code to [the Lemon parser generator](./doc/lemon.html) is
-     found here.  There are also TCL scripts used to build and/or transform
-     source code files.  For example, the tool/mksqlite3h.tcl script reads
-     the src/sqlite.h.in file and uses it as a template to construct
-     the deliverable "sqlite3.h" file that defines the SQLite interface.
+The **ext/** subdirectory contains code for extensions.  The
+Full-text search engine is in **ext/fts3**.  The R-Tree engine is in
+**ext/rtree**.  The **ext/misc** subdirectory contains a number of
+smaller, single-file extensions, such as a REGEXP operator.
 
-  *  **ext/** - Various extensions to SQLite are found under this
-     directory.  For example, the FTS5 subsystem is in "ext/fts5/".
-     Some of these extensions (ex: FTS3/4, FTS5, RTREE) might get built
-     into the SQLite amalgamation, but not all of them.  The
-     "ext/misc/" subdirectory contains an assortment of one-file extensions,
-     many of which are omitted from the SQLite core, but which are included
-     in the [SQLite CLI](https://sqlite.org/cli.html).
-     
-  *  **doc/** - Some documentation files about SQLite internals are found
-     here.  Note, however, that the primary documentation designed for
-     application developers and users of SQLite is in a completely separate
-     repository.  Note also that the primary API documentation is derived
-     from specially constructed comments in the src/sqlite.h.in file.
+The **tool/** subdirectory contains various scripts and programs used
+for building generated source code files or for testing or for generating
+accessory programs such as "sqlite3_analyzer(.exe)".
 
 ### Generated Source Code Files
 
@@ -284,37 +252,31 @@ individual source file exceeds 32K lines in length.
 SQLite is modular in design.
 See the [architectural description](https://www.sqlite.org/arch.html)
 for details. Other documents that are useful in
-helping to understand how SQLite works include the
+(helping to understand how SQLite works include the
 [file format](https://www.sqlite.org/fileformat2.html) description,
 the [virtual machine](https://www.sqlite.org/opcode.html) that runs
 prepared statements, the description of
 [how transactions work](https://www.sqlite.org/atomiccommit.html), and
 the [overview of the query planner](https://www.sqlite.org/optoverview.html).
 
-Decades of effort have gone into optimizing SQLite, both
+Years of effort have gone into optimizing SQLite, both
 for small size and high performance.  And optimizations tend to result in
 complex code.  So there is a lot of complexity in the current SQLite
 implementation.  It will not be the easiest library in the world to hack.
 
-### Key source code files
+Key files:
 
   *  **sqlite.h.in** - This file defines the public interface to the SQLite
      library.  Readers will need to be familiar with this interface before
-     trying to understand how the library works internally.  This file is
-     really a template that is transformed into the "sqlite3.h" deliverable
-     using a script invoked by the makefile.
+     trying to understand how the library works internally.
 
   *  **sqliteInt.h** - this header file defines many of the data objects
      used internally by SQLite.  In addition to "sqliteInt.h", some
-     subsystems inside of sQLite have their own header files.  These internal
-     interfaces are not for use by applications.  They can and do change
-     from one release of SQLite to the next.
+     subsystems have their own header files.
 
   *  **parse.y** - This file describes the LALR(1) grammar that SQLite uses
      to parse SQL statements, and the actions that are taken at each step
-     in the parsing process.  The file is processed by the
-     [Lemon Parser Generator](./doc/lemon.html) to produce the actual C code
-     used for parsing.
+     in the parsing process.
 
   *  **vdbe.c** - This file implements the virtual machine that runs
      prepared statements.  There are various helper files whose names
@@ -357,11 +319,6 @@ implementation.  It will not be the easiest library in the world to hack.
      (and some other test programs too) is built and run when you type
      "make test".
 
-  *  **VERSION**, **manifest**, and **manifest.uuid** - These files define
-     the current SQLite version number.  The "VERSION" file is human generated,
-     but the "manifest" and "manifest.uuid" files are automatically generated
-     by the [Fossil version control system](https://fossil-scm/).
-
 There are many other source files.  Each has a succinct header comment that
 describes its purpose and role within the larger system.
 
@@ -379,7 +336,7 @@ The `manifest.uuid` file should contain the SHA3-256 hash of the
 `manifest` file. If all of the above hash comparisons are correct, then
 you can be confident that your source tree is authentic and unadulterated.
 Details on the format for the `manifest` files are available
-[on the Fossil website](https://fossil-scm.org/home/doc/trunk/www/fileformat.wiki#manifest).
+[on the Fossil website](https://fossil-scm.org/fossil/doc/trunk/www/fileformat.wiki#manifest).
 
 The process of checking source code authenticity is automated by the 
 makefile:
@@ -400,7 +357,3 @@ The main SQLite website is [https://sqlite.org/](https://sqlite.org/)
 with geographically distributed backups at
 [https://www2.sqlite.org/](https://www2.sqlite.org) and
 [https://www3.sqlite.org/](https://www3.sqlite.org).
-
-Contact the SQLite developers through the
-[SQLite Forum](https://sqlite.org/forum/).  In an emergency, you
-can send private email to the lead developer at drh at sqlite dot org.

VERSION

@@ -1 +1 @@
-3.47.0
+3.46.1

autoconf/tea/configure.ac

@@ -19,7 +19,7 @@ dnl	to configure the system for the local environment.
 # so that we create the export library with the dll.
 #-----------------------------------------------------------------------
 
-AC_INIT([sqlite],[3.47.0])
+AC_INIT([sqlite],[3.46.1])
 
 #--------------------------------------------------------------------
 # Call TEA_INIT as the first TEA_ macro to set up initial vars.

autoconf/tea/tclconfig/tcl.m4

@@ -4064,4 +4064,4 @@ AC_DEFUN([TEA_ZIPFS_SUPPORT], [
 
 # Local Variables:
 # mode: autoconf
-# End:
+# End:

autoconf/tea/win/rules.vc

@@ -708,3 +708,4 @@ TK_INCLUDES     = -I"$(_TKDIR)\generic" -I"$(_TKDIR)\win" -I"$(_TKDIR)\xlib"
 !message *** Link options '$(LINKERFLAGS)'
 
 !endif
+

configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for sqlite 3.47.0.
+# Generated by GNU Autoconf 2.69 for sqlite 3.46.1.
 #
 #
 # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -726,8 +726,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='sqlite'
 PACKAGE_TARNAME='sqlite'
-PACKAGE_VERSION='3.47.0'
-PACKAGE_STRING='sqlite 3.47.0'
+PACKAGE_VERSION='3.46.1'
+PACKAGE_STRING='sqlite 3.46.1'
 PACKAGE_BUGREPORT=''
 PACKAGE_URL=''
 
@@ -781,6 +781,18 @@ TARGET_HAVE_EDITLINE
 TARGET_HAVE_READLINE
 TARGET_READLINE_INC
 TARGET_READLINE_LIBS
+HAVE_TCL
+TCL_SHLIB_SUFFIX
+TCL_STUB_LIB_SPEC
+TCL_STUB_LIB_FLAG
+TCL_STUB_LIB_FILE
+TCL_LIB_SPEC
+TCL_LIB_FLAG
+TCL_LIB_FILE
+TCL_INCLUDE_SPEC
+TCL_SRC_DIR
+TCL_BIN_DIR
+TCL_VERSION
 TARGET_EXEEXT
 SQLITE_OS_WIN
 SQLITE_OS_UNIX
@@ -794,10 +806,6 @@ RELEASE
 VERSION
 program_prefix
 TCLLIBDIR
-HAVE_TCL
-TCL_STUB_LIB_SPEC
-TCL_LIB_SPEC
-TCL_INCLUDE_SPEC
 TCLSH_CMD
 INSTALL_DATA
 INSTALL_SCRIPT
@@ -886,13 +894,12 @@ enable_fast_install
 with_gnu_ld
 enable_libtool_lock
 enable_largefile
-with_tclsh
-with_tcl
-enable_tcl
 with_wasi_sdk
 enable_threadsafe
 enable_releasemode
 enable_tempstore
+enable_tcl
+with_tcl
 enable_editline
 enable_readline
 with_readline_lib
@@ -1465,7 +1472,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures sqlite 3.47.0 to adapt to many kinds of systems.
+\`configure' configures sqlite 3.46.1 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1530,7 +1537,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of sqlite 3.47.0:";;
+     short | recursive ) echo "Configuration of sqlite 3.46.1:";;
    esac
   cat <<\_ACEOF
 
@@ -1544,12 +1551,11 @@ Optional Features:
                           optimize for fast installation [default=yes]
   --disable-libtool-lock  avoid locking (might break parallel builds)
   --disable-largefile     omit support for large files
-  --disable-tcl           omit building accessory programs that require
-                          TCL-dev
   --disable-threadsafe    Disable mutexing
   --enable-releasemode    Support libtool link to release mode
   --enable-tempstore      Use an in-ram database for temporary tables
                           (never,no,yes,always)
+  --disable-tcl           do not build TCL extension
   --enable-editline       enable BSD editline support
   --disable-readline      disable readline support
   --enable-debug          enable debugging & verbose explain
@@ -1577,10 +1583,10 @@ Optional Packages:
   --with-pic              try to use only PIC/non-PIC objects [default=use
                           both]
   --with-gnu-ld           assume the C compiler uses GNU ld [default=no]
-  --with-tclsh=PATHNAME   full pathname of a tclsh to use
-  --with-tcl=DIR          directory containing (tclConfig.sh)
   --with-wasi-sdk=DIR     directory containing the WASI SDK. Triggers
                           cross-compile to WASM.
+  --with-tcl=DIR          directory containing tcl configuration
+                          (tclConfig.sh)
   --with-readline-lib     specify readline library
   --with-readline-inc     specify readline include paths
   --with-linenoise=DIR    source directory for linenoise library
@@ -1662,7 +1668,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-sqlite configure 3.47.0
+sqlite configure 3.46.1
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2081,7 +2087,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by sqlite $as_me 3.47.0, which was
+It was created by sqlite $as_me 3.46.1, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -3939,13 +3945,13 @@ if ${lt_cv_nm_interface+:} false; then :
 else
   lt_cv_nm_interface="BSD nm"
   echo "int some_variable = 0;" > conftest.$ac_ext
-  (eval echo "\"\$as_me:3942: $ac_compile\"" >&5)
+  (eval echo "\"\$as_me:3948: $ac_compile\"" >&5)
   (eval "$ac_compile" 2>conftest.err)
   cat conftest.err >&5
-  (eval echo "\"\$as_me:3945: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
+  (eval echo "\"\$as_me:3951: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
   (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
   cat conftest.err >&5
-  (eval echo "\"\$as_me:3948: output\"" >&5)
+  (eval echo "\"\$as_me:3954: output\"" >&5)
   cat conftest.out >&5
   if $GREP 'External.*some_variable' conftest.out > /dev/null; then
     lt_cv_nm_interface="MS dumpbin"
@@ -5151,7 +5157,7 @@ ia64-*-hpux*)
   ;;
 *-*-irix6*)
   # Find out which ABI we are using.
-  echo '#line 5154 "configure"' > conftest.$ac_ext
+  echo '#line 5160 "configure"' > conftest.$ac_ext
   if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
   (eval $ac_compile) 2>&5
   ac_status=$?
@@ -6676,11 +6682,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:6679: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:6685: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>conftest.err)
    ac_status=$?
    cat conftest.err >&5
-   echo "$as_me:6683: \$? = $ac_status" >&5
+   echo "$as_me:6689: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s "$ac_outfile"; then
      # The compiler can only warn and ignore the option if not recognized
      # So say no if there are warnings other than the usual output.
@@ -7015,11 +7021,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7018: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7024: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>conftest.err)
    ac_status=$?
    cat conftest.err >&5
-   echo "$as_me:7022: \$? = $ac_status" >&5
+   echo "$as_me:7028: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s "$ac_outfile"; then
      # The compiler can only warn and ignore the option if not recognized
      # So say no if there are warnings other than the usual output.
@@ -7120,11 +7126,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7123: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7129: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>out/conftest.err)
    ac_status=$?
    cat out/conftest.err >&5
-   echo "$as_me:7127: \$? = $ac_status" >&5
+   echo "$as_me:7133: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s out/conftest2.$ac_objext
    then
      # The compiler can only warn and ignore the option if not recognized
@@ -7175,11 +7181,11 @@ else
    -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
    -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
    -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7178: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7184: $lt_compile\"" >&5)
    (eval "$lt_compile" 2>out/conftest.err)
    ac_status=$?
    cat out/conftest.err >&5
-   echo "$as_me:7182: \$? = $ac_status" >&5
+   echo "$as_me:7188: \$? = $ac_status" >&5
    if (exit $ac_status) && test -s out/conftest2.$ac_objext
    then
      # The compiler can only warn and ignore the option if not recognized
@@ -9555,7 +9561,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 9558 "configure"
+#line 9564 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -9651,7 +9657,7 @@ else
   lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
   lt_status=$lt_dlunknown
   cat > conftest.$ac_ext <<_LT_EOF
-#line 9654 "configure"
+#line 9660 "configure"
 #include "confdefs.h"
 
 #if HAVE_DLFCN_H
@@ -10308,31 +10314,11 @@ done
 USE_AMALGAMATION=1
 
 #########
-# Figure out all the name of a working tclsh and parameters needed to compile against Tcl.
-# The --with-tcl= and/or --with-tclsh= configuration arguments might be useful for this.
+# See whether we can run specific tclsh versions known to work well;
+# if not, then we fall back to plain tclsh.
+# TODO: try other versions before falling back?
 #
-
-# Check whether --with-tclsh was given.
-if test "${with_tclsh+set}" = set; then :
-  withval=$with_tclsh;
-fi
-
-
-# Check whether --with-tcl was given.
-if test "${with_tcl+set}" = set; then :
-  withval=$with_tcl;
-fi
-
-# Check whether --enable-tcl was given.
-if test "${enable_tcl+set}" = set; then :
-  enableval=$enable_tcl; use_tcl=$enableval
-else
-  use_tcl=yes
-fi
-
-original_use_tcl=${use_tcl}
-if test x"${with_tclsh}" == x -a x"${with_tcl}" == x; then
-  for ac_prog in tclsh8.6 tclsh tclsh9.0
+for ac_prog in tclsh8.7 tclsh8.6 tclsh8.5 tclsh
 do
   # Extract the first word of "$ac_prog", so it can be a program name with args.
 set dummy $ac_prog; ac_word=$2
@@ -10375,99 +10361,10 @@ fi
 done
 test -n "$TCLSH_CMD" || TCLSH_CMD="none"
 
-  with_tclsh=${TCLSH_CMD}
-fi
-if test x"${with_tclsh}" != x -a x"${with_tclsh}" != xnone; then
-  TCLSH_CMD=${with_tclsh}
-  { $as_echo "$as_me:${as_lineno-$LINENO}: result: using tclsh at \"$TCLSH_CMD\"" >&5
-$as_echo "using tclsh at \"$TCLSH_CMD\"" >&6; }
-  if test x"${use_tcl}" = "xyes"; then
-    with_tcl=`${with_tclsh} <${srcdir}/tool/find_tclconfig.tcl`
-    if test x"${with_tcl}" != x; then
-      { $as_echo "$as_me:${as_lineno-$LINENO}: result: $TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}" >&5
-$as_echo "$TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}" >&6; }
-    else
-      { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $TCLSH_CMD is unable to recommend a tclConfig.sh" >&5
-$as_echo "$as_me: WARNING: $TCLSH_CMD is unable to recommend a tclConfig.sh" >&2;}
-      use_tcl=no
-    fi
-  fi
-fi
-if test x"${use_tcl}" = "xyes"; then
-  if test x"${with_tcl}" != x; then
-    if test -r ${with_tcl}/tclConfig.sh; then
-      tclconfig="${with_tcl}/tclConfig.sh"
-    else
-      for i in tcl8.6 tcl9.0 lib; do
-        if test -r ${with_tcl}/$i/tclConfig.sh; then
-          tclconfig=${with_tcl}/$i/tclConfig.sh
-          break
-        fi
-      done
-    fi
-    if test ! -r "${tclconfig}"; then
-      as_fn_error $? "no tclConfig.sh file found under ${with_tcl}" "$LINENO" 5
-    fi
-  else
-    # If we have not yet found a tclConfig.sh file, look in $libdir whic is
-    # set automatically by autoconf or by the --prefix command-line option.
-    # See https://sqlite.org/forum/forumpost/e04e693439a22457
-    libdir=${prefix}/lib
-    if test -r ${libdir}/tclConfig.sh; then
-      tclconfig=${libdir}/tclConfig.sh
-    else
-      for i in tcl8.6 tcl9.0 lib; do
-        if test -r ${libdir}/$i/tclConfig.sh; then
-          tclconfig=${libdir}/$i/tclConfig.sh
-          break
-        fi
-      done
-    fi
-    if test ! -r "${tclconfig}"; then
-      as_fn_error $? "cannot find a usable tclConfig.sh file.
-        Use --with-tcl=DIR to specify a directory where tclConfig.sh can be found.
-        SQLite does not use TCL internally, but TCL is required to build SQLite
-        from canonical sources and TCL is required for testing." "$LINENO" 5
-    fi
-  fi
-  { $as_echo "$as_me:${as_lineno-$LINENO}: result: loading TCL configuration from ${tclconfig}" >&5
-$as_echo "loading TCL configuration from ${tclconfig}" >&6; }
-  . ${tclconfig}
-
-
-
-  # There are lots of other configuration variables that are provided by the
-  # tclConfig.sh file and that could be included here.  But as of right now,
-  # TCL_LIB_SPEC is the only what that the Makefile uses.
-  HAVE_TCL=1
-elif test x"${original_use_tcl}" = "xno"; then
-  { $as_echo "$as_me:${as_lineno-$LINENO}: result: unable to run tests because of --disable-tcl" >&5
-$as_echo "unable to run tests because of --disable-tcl" >&6; }
-  HAVE_TCL=0
-else
-  { $as_echo "$as_me:${as_lineno-$LINENO}: result: unable to run tests because no tclConfig.sh file could be located" >&5
-$as_echo "unable to run tests because no tclConfig.sh file could be located" >&6; }
-  HAVE_TCL=0
-fi
-
-if test x"$TCLSH_CMD" == x; then
-  TCLSH_CMD=${TCL_EXEC_PREFIX}/bin/tclsh${TCL_VERSION}
-  if test ! -x ${TCLSH_CMD}; then
-    TCLSH_CMD_2=${TCL_EXEC_PREFIX}/bin/tclsh
-    if test ! -x ${TCLSH_CMD_2}; then
-      { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: cannot find a usable tclsh at either ${TCLSH_CMD} or ${TCLSH_CMD_2}" >&5
-$as_echo "$as_me: WARNING: cannot find a usable tclsh at either ${TCLSH_CMD} or ${TCLSH_CMD_2}" >&2;}
-      TCLSH_CMD=none
-    else
-      TCLSH_CMD=${TCLSH_CMD_2}
-    fi
-  fi
-fi
 if test "$TCLSH_CMD" = "none"; then
   # If we can't find a local tclsh, then building the amalgamation will fail.
   # We act as though --disable-amalgamation has been used.
-  { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Warning: can't find tclsh - defaulting to non-amalgamation build." >&5
-$as_echo "$as_me: WARNING: Warning: can't find tclsh - defaulting to non-amalgamation build." >&2;}
+  echo "Warning: can't find tclsh - defaulting to non-amalgamation build."
   USE_AMALGAMATION=0
   TCLSH_CMD="tclsh"
 fi
@@ -10475,6 +10372,7 @@ fi
 
 
 if test "x${TCLLIBDIR+set}" != "xset" ; then
+  TCLLIBDIR='$(libdir)'
   for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` ; do
     if test -d $i ; then
       TCLLIBDIR=$i
@@ -10871,6 +10769,213 @@ fi
 
 
 
+##########
+# Figure out all the parameters needed to compile against Tcl.
+#
+# This code is derived from the SC_PATH_TCLCONFIG and SC_LOAD_TCLCONFIG
+# macros in the in the tcl.m4 file of the standard TCL distribution.
+# Those macros could not be used directly since we have to make some
+# minor changes to accomodate systems that do not have TCL installed.
+#
+# Check whether --enable-tcl was given.
+if test "${enable_tcl+set}" = set; then :
+  enableval=$enable_tcl; use_tcl=$enableval
+else
+  use_tcl=yes
+fi
+
+if test "${use_tcl}" = "yes" ; then
+
+# Check whether --with-tcl was given.
+if test "${with_tcl+set}" = set; then :
+  withval=$with_tcl; with_tclconfig=${withval}
+fi
+
+  { $as_echo "$as_me:${as_lineno-$LINENO}: checking for Tcl configuration" >&5
+$as_echo_n "checking for Tcl configuration... " >&6; }
+  if ${ac_cv_c_tclconfig+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+
+    # First check to see if --with-tcl was specified.
+    if test x"${with_tclconfig}" != x ; then
+      if test -f "${with_tclconfig}/tclConfig.sh" ; then
+        ac_cv_c_tclconfig=`(cd ${with_tclconfig}; pwd)`
+      else
+        as_fn_error $? "${with_tclconfig} directory doesn't contain tclConfig.sh" "$LINENO" 5
+      fi
+    fi
+
+    # Start autosearch by asking tclsh
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      if test x"$cross_compiling" = xno; then
+        for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}`
+        do
+          if test -f "$i/tclConfig.sh" ; then
+            ac_cv_c_tclconfig="$i"
+            break
+          fi
+        done
+      fi
+    fi
+
+    # On ubuntu 14.10, $auto_path on tclsh is not quite correct.
+    # So try again after applying corrections.
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      if test x"$cross_compiling" = xno; then
+        for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD} | sed 's,/tcltk/tcl,/tcl,g'`
+        do
+          if test -f "$i/tclConfig.sh" ; then
+            ac_cv_c_tclconfig="$i"
+            break
+          fi
+        done
+      fi
+    fi
+
+    # Recent versions of Xcode on Macs hid the tclConfig.sh file
+    # in a strange place.
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      if test x"$cross_compiling" = xno; then
+        for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib
+        do
+          if test -f "$i/tclConfig.sh" ; then
+            ac_cv_c_tclconfig="$i"
+            break
+          fi
+        done
+      fi
+    fi
+
+    # then check for a private Tcl installation
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      for i in \
+            ../tcl \
+            `ls -dr ../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \
+            `ls -dr ../tcl[8-9].[0-9] 2>/dev/null` \
+            `ls -dr ../tcl[8-9].[0-9]* 2>/dev/null` \
+            ../../tcl \
+            `ls -dr ../../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \
+            `ls -dr ../../tcl[8-9].[0-9] 2>/dev/null` \
+            `ls -dr ../../tcl[8-9].[0-9]* 2>/dev/null` \
+            ../../../tcl \
+            `ls -dr ../../../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \
+            `ls -dr ../../../tcl[8-9].[0-9] 2>/dev/null` \
+            `ls -dr ../../../tcl[8-9].[0-9]* 2>/dev/null`
+      do
+        if test -f "$i/unix/tclConfig.sh" ; then
+          ac_cv_c_tclconfig=`(cd $i/unix; pwd)`
+          break
+        fi
+      done
+    fi
+
+    # check in a few common install locations
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      for i in \
+            `ls -d ${libdir} 2>/dev/null` \
+            `ls -d /usr/local/lib 2>/dev/null` \
+            `ls -d /usr/contrib/lib 2>/dev/null` \
+            `ls -d /usr/lib 2>/dev/null`
+      do
+        if test -f "$i/tclConfig.sh" ; then
+           ac_cv_c_tclconfig=`(cd $i; pwd)`
+           break
+        fi
+      done
+    fi
+
+    # check in a few other private locations
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      for i in \
+         ${srcdir}/../tcl \
+         `ls -dr ${srcdir}/../tcl[8-9].[0-9].[0-9]* 2>/dev/null` \
+         `ls -dr ${srcdir}/../tcl[8-9].[0-9] 2>/dev/null` \
+         `ls -dr ${srcdir}/../tcl[8-9].[0-9]* 2>/dev/null`
+      do
+        if test -f "$i/unix/tclConfig.sh" ; then
+          ac_cv_c_tclconfig=`(cd $i/unix; pwd)`
+          break
+        fi
+      done
+    fi
+
+fi
+
+
+  if test x"${ac_cv_c_tclconfig}" = x ; then
+    use_tcl=no
+    { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: Can't find Tcl configuration definitions" >&5
+$as_echo "$as_me: WARNING: Can't find Tcl configuration definitions" >&2;}
+    { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: *** Without Tcl the regression tests cannot be executed ***" >&5
+$as_echo "$as_me: WARNING: *** Without Tcl the regression tests cannot be executed ***" >&2;}
+    { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: *** Consider using --with-tcl=... to define location of Tcl ***" >&5
+$as_echo "$as_me: WARNING: *** Consider using --with-tcl=... to define location of Tcl ***" >&2;}
+  else
+    TCL_BIN_DIR=${ac_cv_c_tclconfig}
+    { $as_echo "$as_me:${as_lineno-$LINENO}: result: found $TCL_BIN_DIR/tclConfig.sh" >&5
+$as_echo "found $TCL_BIN_DIR/tclConfig.sh" >&6; }
+
+    { $as_echo "$as_me:${as_lineno-$LINENO}: checking for existence of $TCL_BIN_DIR/tclConfig.sh" >&5
+$as_echo_n "checking for existence of $TCL_BIN_DIR/tclConfig.sh... " >&6; }
+    if test -f "$TCL_BIN_DIR/tclConfig.sh" ; then
+      { $as_echo "$as_me:${as_lineno-$LINENO}: result: loading" >&5
+$as_echo "loading" >&6; }
+      . $TCL_BIN_DIR/tclConfig.sh
+    else
+      { $as_echo "$as_me:${as_lineno-$LINENO}: result: file not found" >&5
+$as_echo "file not found" >&6; }
+    fi
+
+    #
+    # If the TCL_BIN_DIR is the build directory (not the install directory),
+    # then set the common variable name to the value of the build variables.
+    # For example, the variable TCL_LIB_SPEC will be set to the value
+    # of TCL_BUILD_LIB_SPEC. An extension should make use of TCL_LIB_SPEC
+    # instead of TCL_BUILD_LIB_SPEC since it will work with both an
+    # installed and uninstalled version of Tcl.
+    #
+
+    if test -f $TCL_BIN_DIR/Makefile ; then
+      TCL_LIB_SPEC=${TCL_BUILD_LIB_SPEC}
+      TCL_STUB_LIB_SPEC=${TCL_BUILD_STUB_LIB_SPEC}
+      TCL_STUB_LIB_PATH=${TCL_BUILD_STUB_LIB_PATH}
+    fi
+
+    #
+    # eval is required to do the TCL_DBGX substitution
+    #
+
+    eval "TCL_LIB_FILE=\"${TCL_LIB_FILE}\""
+    eval "TCL_LIB_FLAG=\"${TCL_LIB_FLAG}\""
+    eval "TCL_LIB_SPEC=\"${TCL_LIB_SPEC}\""
+
+    eval "TCL_STUB_LIB_FILE=\"${TCL_STUB_LIB_FILE}\""
+    eval "TCL_STUB_LIB_FLAG=\"${TCL_STUB_LIB_FLAG}\""
+    eval "TCL_STUB_LIB_SPEC=\"${TCL_STUB_LIB_SPEC}\""
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  fi
+fi
+if test "${use_tcl}" = "no" ; then
+  HAVE_TCL=""
+else
+  HAVE_TCL=1
+fi
+
+
 ##########
 # Figure out what C libraries are required to compile programs
 # that use "readline()" library.
@@ -12376,7 +12481,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by sqlite $as_me 3.47.0, which was
+This file was extended by sqlite $as_me 3.46.1, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -12442,7 +12547,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-sqlite config.status 3.47.0
+sqlite config.status 3.46.1
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

configure.ac

@@ -116,101 +116,15 @@ AC_CHECK_FUNCS([fdatasync gmtime_r isnan localtime_r localtime_s malloc_usable_s
 USE_AMALGAMATION=1
 
 #########
-# Figure out all the name of a working tclsh and parameters needed to compile against Tcl.
-# The --with-tcl= and/or --with-tclsh= configuration arguments might be useful for this.
-#
-AC_ARG_WITH(tclsh, AS_HELP_STRING([--with-tclsh=PATHNAME],[full pathname of a tclsh to use]))
-AC_ARG_WITH(tcl, AS_HELP_STRING([--with-tcl=DIR],[directory containing (tclConfig.sh)]))
-AC_ARG_ENABLE(tcl, AS_HELP_STRING([--disable-tcl],[omit building accessory programs that require TCL-dev]),
-      [use_tcl=$enableval],[use_tcl=yes])
-original_use_tcl=${use_tcl}
-if test x"${with_tclsh}" == x -a x"${with_tcl}" == x; then
-  AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.6 tclsh tclsh9.0],none)
-  with_tclsh=${TCLSH_CMD}
-fi
-if test x"${with_tclsh}" != x -a x"${with_tclsh}" != xnone; then
-  TCLSH_CMD=${with_tclsh}
-  AC_MSG_RESULT([using tclsh at "$TCLSH_CMD"])
-  if test x"${use_tcl}" = "xyes"; then
-    with_tcl=`${with_tclsh} <${srcdir}/tool/find_tclconfig.tcl`
-    if test x"${with_tcl}" != x; then
-      AC_MSG_RESULT([$TCLSH_CMD recommends the tclConfig.sh at ${with_tcl}])
-    else
-      AC_MSG_WARN([$TCLSH_CMD is unable to recommend a tclConfig.sh])
-      use_tcl=no
-    fi
-  fi
-fi
-if test x"${use_tcl}" = "xyes"; then
-  if test x"${with_tcl}" != x; then
-    if test -r ${with_tcl}/tclConfig.sh; then
-      tclconfig="${with_tcl}/tclConfig.sh"
-    else
-      for i in tcl8.6 tcl9.0 lib; do
-        if test -r ${with_tcl}/$i/tclConfig.sh; then
-          tclconfig=${with_tcl}/$i/tclConfig.sh
-          break
-        fi
-      done
-    fi
-    if test ! -r "${tclconfig}"; then
-      AC_MSG_ERROR([no tclConfig.sh file found under ${with_tcl}])
-    fi
-  else
-    # If we have not yet found a tclConfig.sh file, look in $libdir whic is
-    # set automatically by autoconf or by the --prefix command-line option.
-    # See https://sqlite.org/forum/forumpost/e04e693439a22457
-    libdir=${prefix}/lib
-    if test -r ${libdir}/tclConfig.sh; then
-      tclconfig=${libdir}/tclConfig.sh
-    else
-      for i in tcl8.6 tcl9.0 lib; do
-        if test -r ${libdir}/$i/tclConfig.sh; then
-          tclconfig=${libdir}/$i/tclConfig.sh
-          break
-        fi
-      done
-    fi
-    if test ! -r "${tclconfig}"; then
-      AC_MSG_ERROR([cannot find a usable tclConfig.sh file. 
-        Use --with-tcl=DIR to specify a directory where tclConfig.sh can be found.
-        SQLite does not use TCL internally, but TCL is required to build SQLite
-        from canonical sources and TCL is required for testing.])
-    fi
-  fi
-  AC_MSG_RESULT([loading TCL configuration from ${tclconfig}])
-  . ${tclconfig}
-  AC_SUBST(TCL_INCLUDE_SPEC)
-  AC_SUBST(TCL_LIB_SPEC)
-  AC_SUBST(TCL_STUB_LIB_SPEC)
-  # There are lots of other configuration variables that are provided by the
-  # tclConfig.sh file and that could be included here.  But as of right now,
-  # TCL_LIB_SPEC is the only what that the Makefile uses.
-  HAVE_TCL=1
-elif test x"${original_use_tcl}" = "xno"; then
-  AC_MSG_RESULT([unable to run tests because of --disable-tcl])
-  HAVE_TCL=0
-else
-  AC_MSG_RESULT([unable to run tests because no tclConfig.sh file could be located])
-  HAVE_TCL=0
-fi
-AC_SUBST(HAVE_TCL)
-if test x"$TCLSH_CMD" == x; then
-  TCLSH_CMD=${TCL_EXEC_PREFIX}/bin/tclsh${TCL_VERSION}
-  if test ! -x ${TCLSH_CMD}; then
-    TCLSH_CMD_2=${TCL_EXEC_PREFIX}/bin/tclsh
-    if test ! -x ${TCLSH_CMD_2}; then
-      AC_MSG_WARN([cannot find a usable tclsh at either ${TCLSH_CMD} or ${TCLSH_CMD_2}])
-      TCLSH_CMD=none
-    else
-      TCLSH_CMD=${TCLSH_CMD_2}
-    fi
-  fi
-fi
+# See whether we can run specific tclsh versions known to work well;
+# if not, then we fall back to plain tclsh.
+# TODO: try other versions before falling back?
+# 
+AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.7 tclsh8.6 tclsh8.5 tclsh], none)
 if test "$TCLSH_CMD" = "none"; then
   # If we can't find a local tclsh, then building the amalgamation will fail.
   # We act as though --disable-amalgamation has been used.
-  AC_MSG_WARN([Warning: can't find tclsh - defaulting to non-amalgamation build.])
+  echo "Warning: can't find tclsh - defaulting to non-amalgamation build."
   USE_AMALGAMATION=0
   TCLSH_CMD="tclsh"
 fi
@@ -218,6 +132,7 @@ AC_SUBST(TCLSH_CMD)
 
 AC_ARG_VAR([TCLLIBDIR], [Where to install tcl plugin])
 if test "x${TCLLIBDIR+set}" != "xset" ; then
+  TCLLIBDIR='$(libdir)'
   for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}` ; do
     if test -d $i ; then
       TCLLIBDIR=$i
@@ -424,6 +339,190 @@ AC_SUBST(SQLITE_OS_UNIX)
 AC_SUBST(SQLITE_OS_WIN)
 AC_SUBST(TARGET_EXEEXT)
 
+##########
+# Figure out all the parameters needed to compile against Tcl.
+#
+# This code is derived from the SC_PATH_TCLCONFIG and SC_LOAD_TCLCONFIG
+# macros in the in the tcl.m4 file of the standard TCL distribution.
+# Those macros could not be used directly since we have to make some
+# minor changes to accomodate systems that do not have TCL installed.
+#
+AC_ARG_ENABLE(tcl, AS_HELP_STRING([--disable-tcl],[do not build TCL extension]),
+      [use_tcl=$enableval],[use_tcl=yes])
+if test "${use_tcl}" = "yes" ; then
+  AC_ARG_WITH(tcl, AS_HELP_STRING([--with-tcl=DIR],[directory containing tcl configuration (tclConfig.sh)]), with_tclconfig=${withval})
+  AC_MSG_CHECKING([for Tcl configuration])
+  AC_CACHE_VAL(ac_cv_c_tclconfig,[
+    # First check to see if --with-tcl was specified.
+    if test x"${with_tclconfig}" != x ; then
+      if test -f "${with_tclconfig}/tclConfig.sh" ; then
+        ac_cv_c_tclconfig=`(cd ${with_tclconfig}; pwd)`
+      else
+        AC_MSG_ERROR([${with_tclconfig} directory doesn't contain tclConfig.sh])
+      fi
+    fi
+
+    # Start autosearch by asking tclsh
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      if test x"$cross_compiling" = xno; then
+        for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD}`
+        do
+          if test -f "$i/tclConfig.sh" ; then
+            ac_cv_c_tclconfig="$i"
+            break
+          fi
+        done
+      fi
+    fi
+
+    # On ubuntu 14.10, $auto_path on tclsh is not quite correct.
+    # So try again after applying corrections.
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      if test x"$cross_compiling" = xno; then
+        for i in `echo 'puts stdout $auto_path' | ${TCLSH_CMD} | sed 's,/tcltk/tcl,/tcl,g'`
+        do
+          if test -f "$i/tclConfig.sh" ; then
+            ac_cv_c_tclconfig="$i"
+            break
+          fi
+        done
+      fi
+    fi
+
+    # Recent versions of Xcode on Macs hid the tclConfig.sh file
+    # in a strange place.
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      if test x"$cross_compiling" = xno; then
+        for i in /Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX*.sdk/usr/lib
+        do
+          if test -f "$i/tclConfig.sh" ; then
+            ac_cv_c_tclconfig="$i"
+            break
+          fi
+        done
+      fi
+    fi
+
+    # then check for a private Tcl installation
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      for i in \
+            ../tcl \
+            `ls -dr ../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \
+            `ls -dr ../tcl[[8-9]].[[0-9]] 2>/dev/null` \
+            `ls -dr ../tcl[[8-9]].[[0-9]]* 2>/dev/null` \
+            ../../tcl \
+            `ls -dr ../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \
+            `ls -dr ../../tcl[[8-9]].[[0-9]] 2>/dev/null` \
+            `ls -dr ../../tcl[[8-9]].[[0-9]]* 2>/dev/null` \
+            ../../../tcl \
+            `ls -dr ../../../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \
+            `ls -dr ../../../tcl[[8-9]].[[0-9]] 2>/dev/null` \
+            `ls -dr ../../../tcl[[8-9]].[[0-9]]* 2>/dev/null`
+      do
+        if test -f "$i/unix/tclConfig.sh" ; then
+          ac_cv_c_tclconfig=`(cd $i/unix; pwd)`
+          break
+        fi
+      done
+    fi
+
+    # check in a few common install locations
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      for i in \
+            `ls -d ${libdir} 2>/dev/null` \
+            `ls -d /usr/local/lib 2>/dev/null` \
+            `ls -d /usr/contrib/lib 2>/dev/null` \
+            `ls -d /usr/lib 2>/dev/null`
+      do
+        if test -f "$i/tclConfig.sh" ; then
+           ac_cv_c_tclconfig=`(cd $i; pwd)`
+           break
+        fi
+      done
+    fi
+
+    # check in a few other private locations
+    if test x"${ac_cv_c_tclconfig}" = x ; then
+      for i in \
+         ${srcdir}/../tcl \
+         `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]].[[0-9]]* 2>/dev/null` \
+         `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]] 2>/dev/null` \
+         `ls -dr ${srcdir}/../tcl[[8-9]].[[0-9]]* 2>/dev/null`
+      do
+        if test -f "$i/unix/tclConfig.sh" ; then
+          ac_cv_c_tclconfig=`(cd $i/unix; pwd)`
+          break
+        fi
+      done
+    fi
+  ])
+
+  if test x"${ac_cv_c_tclconfig}" = x ; then
+    use_tcl=no
+    AC_MSG_WARN(Can't find Tcl configuration definitions)
+    AC_MSG_WARN(*** Without Tcl the regression tests cannot be executed ***)
+    AC_MSG_WARN(*** Consider using --with-tcl=... to define location of Tcl ***)
+  else
+    TCL_BIN_DIR=${ac_cv_c_tclconfig}
+    AC_MSG_RESULT(found $TCL_BIN_DIR/tclConfig.sh)
+
+    AC_MSG_CHECKING([for existence of $TCL_BIN_DIR/tclConfig.sh])
+    if test -f "$TCL_BIN_DIR/tclConfig.sh" ; then
+      AC_MSG_RESULT([loading])
+      . $TCL_BIN_DIR/tclConfig.sh
+    else
+      AC_MSG_RESULT([file not found])
+    fi
+    
+    #
+    # If the TCL_BIN_DIR is the build directory (not the install directory),
+    # then set the common variable name to the value of the build variables.
+    # For example, the variable TCL_LIB_SPEC will be set to the value
+    # of TCL_BUILD_LIB_SPEC. An extension should make use of TCL_LIB_SPEC
+    # instead of TCL_BUILD_LIB_SPEC since it will work with both an
+    # installed and uninstalled version of Tcl.
+    #
+    
+    if test -f $TCL_BIN_DIR/Makefile ; then
+      TCL_LIB_SPEC=${TCL_BUILD_LIB_SPEC}
+      TCL_STUB_LIB_SPEC=${TCL_BUILD_STUB_LIB_SPEC}
+      TCL_STUB_LIB_PATH=${TCL_BUILD_STUB_LIB_PATH}
+    fi
+    
+    #
+    # eval is required to do the TCL_DBGX substitution
+    #
+    
+    eval "TCL_LIB_FILE=\"${TCL_LIB_FILE}\""
+    eval "TCL_LIB_FLAG=\"${TCL_LIB_FLAG}\""
+    eval "TCL_LIB_SPEC=\"${TCL_LIB_SPEC}\""
+    
+    eval "TCL_STUB_LIB_FILE=\"${TCL_STUB_LIB_FILE}\""
+    eval "TCL_STUB_LIB_FLAG=\"${TCL_STUB_LIB_FLAG}\""
+    eval "TCL_STUB_LIB_SPEC=\"${TCL_STUB_LIB_SPEC}\""
+    
+    AC_SUBST(TCL_VERSION)
+    AC_SUBST(TCL_BIN_DIR)
+    AC_SUBST(TCL_SRC_DIR)
+    AC_SUBST(TCL_INCLUDE_SPEC)
+    
+    AC_SUBST(TCL_LIB_FILE)
+    AC_SUBST(TCL_LIB_FLAG)
+    AC_SUBST(TCL_LIB_SPEC)
+    
+    AC_SUBST(TCL_STUB_LIB_FILE)
+    AC_SUBST(TCL_STUB_LIB_FLAG)
+    AC_SUBST(TCL_STUB_LIB_SPEC)
+    AC_SUBST(TCL_SHLIB_SUFFIX)
+  fi
+fi
+if test "${use_tcl}" = "no" ; then
+  HAVE_TCL=""
+else
+  HAVE_TCL=1
+fi
+AC_SUBST(HAVE_TCL)
+
 ##########
 # Figure out what C libraries are required to compile programs
 # that use "readline()" library.

doc/compile-for-windows.md

@@ -59,16 +59,8 @@ canonical source on a new Windows 11 PC, as of 2023-11-01:
       <li>  `nmake /f makefile.msc sqlite3.c`
       <li>  `nmake /f makefile.msc devtest`
       <li>  `nmake /f makefile.msc releasetest`
-      <li>  `nmake /f makefile.msc sqlite3.exe`
       </ul>
 
-  7.  For a debugging build of the CLI, where the ".treetrace" and ".wheretrace"
-      commands work, add the DEBUG=3 argument to nmake.  Like this:
-      <ul>
-      <li> `nmake /f makefile.msc DEBUG=3 clean sqlite3.exe`
-      </ul>
-   
-
 ## 32-bit Builds
 
 Doing a 32-bit build is just like doing a 64-bit build with the

doc/wal2.md

@@ -0,0 +1,98 @@
+
+Wal2 Mode Notes
+===============
+
+## Activating/Deactivating Wal2 Mode
+
+"Wal2" mode is very similar to "wal" mode. To change a database to wal2 mode,
+use the command:
+
+>
+     PRAGMA journal_mode = wal2;
+
+It is not possible to change a database directly from "wal" mode to "wal2"
+mode. Instead, it must first be changed to rollback mode. So, to change a wal
+mode database to wal2 mode, the following two commands may be used:
+
+>
+     PRAGMA journal_mode = delete;
+     PRAGMA journal_mode = wal2;
+
+A database in wal2 mode may only be accessed by versions of SQLite compiled
+from this branch. Attempting to use any other version of SQLite results in an
+SQLITE_NOTADB error. A wal2 mode database may be changed back to rollback mode
+(making it accessible by all versions of SQLite) using:
+
+>
+     PRAGMA journal_mode = delete;
+
+## The Advantage of Wal2 Mode
+
+In legacy wal mode, when a writer writes data to the database, it doesn't
+modify the database file directly. Instead, it appends new data to the
+"<database>-wal" file. Readers read data from both the original database
+file and the "<database>-wal" file. At some point, data is copied from the
+"<database>-wal" file into the database file, after which the wal file can
+be deleted or overwritten. Copying data from the wal file into the database
+file is called a "checkpoint", and may be done explictly (either by "PRAGMA
+wal_checkpoint" or sqlite3_wal_checkpoint_v2()), or
+automatically (by configuring "PRAGMA wal_autocheckpoint" - this is the
+default).
+
+Checkpointers do not block writers, and writers do not block checkpointers.
+However, if a writer writes to the database while a checkpoint is ongoing,
+then the new data is appended to the end of the wal file. This means that,
+even following the checkpoint, the wal file cannot be overwritten or deleted,
+and so all subsequent transactions must also be appended to the wal file. The
+work of the checkpointer is not wasted - SQLite remembers which parts of the
+wal file have already been copied into the db file so that the next checkpoint
+does not have to do so again - but it does mean that the wal file may grow
+indefinitely if the checkpointer never gets a chance to finish without a
+writer appending to the wal file. There are also circumstances in which
+long-running readers may prevent a checkpointer from checkpointing the entire
+wal file - also causing the wal file to grow indefinitely in a busy system.
+
+Wal2 mode does not have this problem. In wal2 mode, wal files do not grow
+indefinitely even if the checkpointer never has a chance to finish
+uninterrupted.
+
+In wal2 mode, the system uses two wal files instead of one. The files are named
+"<database>-wal" and "<database>-wal2", where "<database>" is of
+course the name of the database file. When data is written to the database, the
+writer begins by appending the new data to the first wal file. Once the first
+wal file has grown large enough, writers switch to appending data to the second
+wal file. At this point the first wal file can be checkpointed (after which it
+can be overwritten). Then, once the second wal file has grown large enough and
+the first wal file has been checkpointed, writers switch back to the first wal
+file. And so on.
+
+## Application Programming
+
+From the point of view of the user, the main differences between wal and 
+wal2 mode are to do with checkpointing:
+
+  * In wal mode, a checkpoint may be attempted at any time. In wal2 
+    mode, the checkpointer has to wait until writers have switched 
+    to the "other" wal file before a checkpoint can take place.
+
+  * In wal mode, the wal-hook (callback registered using
+    sqlite3_wal_hook()) is invoked after a transaction is committed
+    with the total number of pages in the wal file as an argument. In wal2
+    mode, the argument is either the total number of uncheckpointed pages in
+    both wal files, or - if the "other" wal file is empty or already
+    checkpointed - 0.
+
+Clients are recommended to use the same strategies for checkpointing wal2 mode
+databases as for wal databases - by registering a wal-hook using
+sqlite3_wal_hook() and attempting a checkpoint when the parameter
+exceeds a certain threshold.
+
+However, it should be noted that although the wal-hook is invoked after each
+transaction is committed to disk and database locks released, it is still
+invoked from within the sqlite3_step() call used to execute the "COMMIT"
+command. In BEGIN CONCURRENT systems, where the "COMMIT" is often protected by
+an application mutex, this may reduce concurrency. In such systems, instead of
+executing a checkpoint from within the wal-hook, a thread might defer this
+action until after the application mutex has been released.
+
+

ext/expert/test_expert.c

@@ -16,7 +16,15 @@
 #include "sqlite3expert.h"
 #include <assert.h>
 #include <string.h>
-#include "tclsqlite.h"
+
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 

ext/fts3/fts3_snippet.c

@@ -398,7 +398,6 @@ static int fts3SnippetNextCandidate(SnippetIter *pIter){
       return 1;
     }
 
-    assert( pIter->nSnippet>=0 );
     pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1;
     for(i=0; i<pIter->nPhrase; i++){
       SnippetPhrase *pPhrase = &pIter->aPhrase[i];

ext/fts3/fts3_term.c

@@ -78,8 +78,6 @@ static int fts3termConnectMethod(
     iIndex = atoi(argv[4]);
     argc--;
   }
-  
-  *ppVtab = 0;
 
   /* The user should specify a single argument - the name of an fts3 table. */
   if( argc!=4 ){
@@ -97,17 +95,12 @@ static int fts3termConnectMethod(
   rc = sqlite3_declare_vtab(db, FTS3_TERMS_SCHEMA);
   if( rc!=SQLITE_OK ) return rc;
 
-  nByte = sizeof(Fts3termTable);
-  p = (Fts3termTable *)sqlite3Fts3MallocZero(nByte);
+  nByte = sizeof(Fts3termTable) + sizeof(Fts3Table) + nDb + nFts3 + 2;
+  p = (Fts3termTable *)sqlite3_malloc64(nByte);
   if( !p ) return SQLITE_NOMEM;
+  memset(p, 0, (size_t)nByte);
 
-  p->pFts3Tab = (Fts3Table*)sqlite3Fts3MallocZero(
-      sizeof(Fts3Table) + nDb + nFts3 + 2
-  );
-  if( p->pFts3Tab==0 ){
-    sqlite3_free(p);
-    return SQLITE_NOMEM;
-  }
+  p->pFts3Tab = (Fts3Table *)&p[1];
   p->pFts3Tab->zDb = (char *)&p->pFts3Tab[1];
   p->pFts3Tab->zName = &p->pFts3Tab->zDb[nDb+1];
   p->pFts3Tab->db = db;
@@ -137,7 +130,6 @@ static int fts3termDisconnectMethod(sqlite3_vtab *pVtab){
     sqlite3_finalize(pFts3->aStmt[i]);
   }
   sqlite3_free(pFts3->zSegmentsTbl);
-  sqlite3_free(pFts3);
   sqlite3_free(p);
   return SQLITE_OK;
 }

ext/fts3/fts3_test.c

@@ -18,7 +18,14 @@
 ** that the sqlite3_tokenizer_module.xLanguage() method is invoked correctly.
 */
 
-#include "tclsqlite.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 #include <string.h>
 #include <assert.h>
 
@@ -160,8 +167,7 @@ static int SQLITE_TCLAPI fts3_near_match_cmd(
   Tcl_Obj *pPhrasecount = 0;
   
   Tcl_Obj **apExprToken;
-  Tcl_Size nExprToken;
-  Tcl_Size nn;
+  int nExprToken;
 
   UNUSED_PARAMETER(clientData);
 
@@ -195,25 +201,23 @@ static int SQLITE_TCLAPI fts3_near_match_cmd(
     }
   }
 
-  rc = Tcl_ListObjGetElements(interp, objv[1], &nn, &apDocToken);
-  doc.nToken = (int)nn;
+  rc = Tcl_ListObjGetElements(interp, objv[1], &doc.nToken, &apDocToken);
   if( rc!=TCL_OK ) goto near_match_out;
   doc.aToken = (NearToken *)ckalloc(doc.nToken*sizeof(NearToken));
   for(ii=0; ii<doc.nToken; ii++){
-    doc.aToken[ii].z = Tcl_GetStringFromObj(apDocToken[ii], &nn);
-    doc.aToken[ii].n = (int)nn;
+    doc.aToken[ii].z = Tcl_GetStringFromObj(apDocToken[ii], &doc.aToken[ii].n);
   }
 
   rc = Tcl_ListObjGetElements(interp, objv[2], &nExprToken, &apExprToken);
   if( rc!=TCL_OK ) goto near_match_out;
 
-  nPhrase = (int)(nExprToken + 1) / 2;
+  nPhrase = (nExprToken + 1) / 2;
   aPhrase = (NearPhrase *)ckalloc(nPhrase * sizeof(NearPhrase));
   memset(aPhrase, 0, nPhrase * sizeof(NearPhrase));
   for(ii=0; ii<nPhrase; ii++){
     Tcl_Obj *pPhrase = apExprToken[ii*2];
     Tcl_Obj **apToken;
-    Tcl_Size nToken;
+    int nToken;
     int jj;
 
     rc = Tcl_ListObjGetElements(interp, pPhrase, &nToken, &apToken);
@@ -223,12 +227,11 @@ static int SQLITE_TCLAPI fts3_near_match_cmd(
       rc = TCL_ERROR;
       goto near_match_out;
     }
-    for(jj=0; jj<(int)nToken; jj++){
+    for(jj=0; jj<nToken; jj++){
       NearToken *pT = &aPhrase[ii].aToken[jj];
-      pT->z = Tcl_GetStringFromObj(apToken[jj], &nn);
-      pT->n = (int)nn;
+      pT->z = Tcl_GetStringFromObj(apToken[jj], &pT->n);
     }
-    aPhrase[ii].nToken = (int)nToken;
+    aPhrase[ii].nToken = nToken;
   }
   for(ii=1; ii<nPhrase; ii++){
     Tcl_Obj *pNear = apExprToken[2*ii-1];

ext/fts3/fts3_tokenizer.c

@@ -226,7 +226,11 @@ int sqlite3Fts3InitTokenizer(
 
 #ifdef SQLITE_TEST
 
-#include "tclsqlite.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#endif
 #include <string.h>
 
 /*

ext/fts3/unicode/mkunicode.tcl

@@ -628,9 +628,6 @@ proc print_categories {lMap} {
         $caseP
         $caseS
         $caseZ
-
-        default:
-          return 1;
       }
       return 0;
     }

ext/fts5/extract_api_docs.tcl

@@ -108,11 +108,8 @@ proc get_tokenizer_docs {data} {
       append res "<dt><b>$line</b></dt><dd><p style=margin-top:0>\n"
       continue
     }
-    if {[regexp {FTS5_TOKENIZER} $line]} {
-      set line </dl><p>
-    }
     if {[regexp {SYNONYM SUPPORT} $line]} {
-      set line "<h3>Synonym Support</h3>"
+      set line "</dl><h3>Synonym Support</h3>"
     }
     if {[string trim $line] == ""} {
       append res "<p>\n"

ext/fts5/fts5.h

@@ -238,10 +238,6 @@ struct Fts5PhraseIter {
 **   (i.e. if it is a contentless table), then this API always iterates
 **   through an empty set (all calls to xPhraseFirst() set iCol to -1).
 **
-**   In all cases, matches are visited in (column ASC, offset ASC) order.
-**   i.e. all those in column 0, sorted by offset, followed by those in 
-**   column 1, etc.
-**
 ** xPhraseNext()
 **   See xPhraseFirst above.
 **
@@ -308,32 +304,9 @@ struct Fts5PhraseIter {
 **
 **   This API can be quite slow if used with an FTS5 table created with the
 **   "detail=none" or "detail=column" option.
-**
-** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale)
-**   If parameter iCol is less than zero, or greater than or equal to the
-**   number of columns in the table, SQLITE_RANGE is returned.
-**
-**   Otherwise, this function attempts to retrieve the locale associated
-**   with column iCol of the current row. Usually, there is no associated
-**   locale, and output parameters (*pzLocale) and (*pnLocale) are set
-**   to NULL and 0, respectively. However, if the fts5_locale() function
-**   was used to associate a locale with the value when it was inserted
-**   into the fts5 table, then (*pzLocale) is set to point to a nul-terminated
-**   buffer containing the name of the locale in utf-8 encoding. (*pnLocale) 
-**   is set to the size in bytes of the buffer, not including the 
-**   nul-terminator.
-**
-**   If successful, SQLITE_OK is returned. Or, if an error occurs, an
-**   SQLite error code is returned. The final value of the output parameters
-**   is undefined in this case.
-**
-** xTokenize_v2:
-**   Tokenize text using the tokenizer belonging to the FTS5 table. This
-**   API is the same as the xTokenize() API, except that it allows a tokenizer
-**   locale to be specified.
 */
 struct Fts5ExtensionApi {
-  int iVersion;                   /* Currently always set to 4 */
+  int iVersion;                   /* Currently always set to 3 */
 
   void *(*xUserData)(Fts5Context*);
 
@@ -375,15 +348,6 @@ struct Fts5ExtensionApi {
       const char **ppToken, int *pnToken
   );
   int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*);
-
-  /* Below this point are iVersion>=4 only */
-  int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn);
-  int (*xTokenize_v2)(Fts5Context*,
-    const char *pText, int nText, /* Text to tokenize */
-    const char *pLoc, int nLoc,   /* Locale to pass to tokenizer */
-    void *pCtx,                   /* Context passed to xToken() */
-    int (*xToken)(void*, int, const char*, int, int, int)       /* Callback */
-  );
 };
 
 /* 
@@ -396,7 +360,6 @@ struct Fts5ExtensionApi {
 ** Applications may also register custom tokenizer types. A tokenizer 
 ** is registered by providing fts5 with a populated instance of the 
 ** following structure. All structure methods must be defined, setting
-**
 ** any member of the fts5_tokenizer struct to NULL leads to undefined
 ** behaviour. The structure methods are expected to function as follows:
 **
@@ -405,7 +368,7 @@ struct Fts5ExtensionApi {
 **   A tokenizer instance is required to actually tokenize text.
 **
 **   The first argument passed to this function is a copy of the (void*)
-**   pointer provided by the application when the fts5_tokenizer_v2 object
+**   pointer provided by the application when the fts5_tokenizer object
 **   was registered with FTS5 (the third argument to xCreateTokenizer()). 
 **   The second and third arguments are an array of nul-terminated strings
 **   containing the tokenizer arguments, if any, specified following the
@@ -429,7 +392,7 @@ struct Fts5ExtensionApi {
 **   argument passed to this function is a pointer to an Fts5Tokenizer object
 **   returned by an earlier call to xCreate().
 **
-**   The third argument indicates the reason that FTS5 is requesting
+**   The second argument indicates the reason that FTS5 is requesting
 **   tokenization of the supplied text. This is always one of the following
 **   four values:
 **
@@ -453,13 +416,6 @@ struct Fts5ExtensionApi {
 **            on a columnsize=0 database.  
 **   </ul>
 **
-**   The sixth and seventh arguments passed to xTokenize() - pLocale and
-**   nLocale - are a pointer to a buffer containing the locale to use for
-**   tokenization (e.g. "en_US") and its size in bytes, respectively. The
-**   pLocale buffer is not nul-terminated. pLocale may be passed NULL (in
-**   which case nLocale is always 0) to indicate that the tokenizer should
-**   use its default locale.
-**
 **   For each token in the input string, the supplied callback xToken() must
 **   be invoked. The first argument to it should be a copy of the pointer
 **   passed as the second argument to xTokenize(). The third and fourth
@@ -483,29 +439,6 @@ struct Fts5ExtensionApi {
 **   may abandon the tokenization and return any error code other than
 **   SQLITE_OK or SQLITE_DONE.
 **
-**   If the tokenizer is registered using an fts5_tokenizer_v2 object,
-**   then the xTokenize() method has two additional arguments - pLocale
-**   and nLocale. These specify the locale that the tokenizer should use
-**   for the current request. If pLocale and nLocale are both 0, then the
-**   tokenizer should use its default locale. Otherwise, pLocale points to
-**   an nLocale byte buffer containing the name of the locale to use as utf-8 
-**   text. pLocale is not nul-terminated.
-**
-** FTS5_TOKENIZER
-**
-** There is also an fts5_tokenizer object. This is an older version of
-** fts5_tokenizer_v2. It is similar except that:
-**
-**  <ul>
-**    <li> There is no "iVersion" field, and
-**    <li> The xTokenize() method does not take a locale argument.
-**  </ul>
-**
-** fts5_tokenizer tokenizers should be registered with the xCreateTokenizer()
-** function, instead of xCreateTokenizer_v2(). Tokenizers implementations 
-** registered using either API may be retrieved using both xFindTokenizer()
-** and xFindTokenizer_v2().
-**
 ** SYNONYM SUPPORT
 **
 **   Custom tokenizers may also support synonyms. Consider a case in which a
@@ -614,33 +547,6 @@ struct Fts5ExtensionApi {
 **   inefficient.
 */
 typedef struct Fts5Tokenizer Fts5Tokenizer;
-typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2;
-struct fts5_tokenizer_v2 {
-  int iVersion;             /* Currently always 2 */
-
-  int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
-  void (*xDelete)(Fts5Tokenizer*);
-  int (*xTokenize)(Fts5Tokenizer*, 
-      void *pCtx,
-      int flags,            /* Mask of FTS5_TOKENIZE_* flags */
-      const char *pText, int nText, 
-      const char *pLocale, int nLocale,
-      int (*xToken)(
-        void *pCtx,         /* Copy of 2nd argument to xTokenize() */
-        int tflags,         /* Mask of FTS5_TOKEN_* flags */
-        const char *pToken, /* Pointer to buffer containing token */
-        int nToken,         /* Size of token in bytes */
-        int iStart,         /* Byte offset of token within input text */
-        int iEnd            /* Byte offset of end of token within input text */
-      )
-  );
-};
-
-/*
-** New code should use the fts5_tokenizer_v2 type to define tokenizer
-** implementations. The following type is included for legacy applications
-** that still use it.
-*/
 typedef struct fts5_tokenizer fts5_tokenizer;
 struct fts5_tokenizer {
   int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
@@ -660,7 +566,6 @@ struct fts5_tokenizer {
   );
 };
 
-
 /* Flags that may be passed as the third argument to xTokenize() */
 #define FTS5_TOKENIZE_QUERY     0x0001
 #define FTS5_TOKENIZE_PREFIX    0x0002
@@ -680,7 +585,7 @@ struct fts5_tokenizer {
 */
 typedef struct fts5_api fts5_api;
 struct fts5_api {
-  int iVersion;                   /* Currently always set to 3 */
+  int iVersion;                   /* Currently always set to 2 */
 
   /* Create a new tokenizer */
   int (*xCreateTokenizer)(
@@ -707,25 +612,6 @@ struct fts5_api {
     fts5_extension_function xFunction,
     void (*xDestroy)(void*)
   );
-
-  /* APIs below this point are only available if iVersion>=3 */
-
-  /* Create a new tokenizer */
-  int (*xCreateTokenizer_v2)(
-    fts5_api *pApi,
-    const char *zName,
-    void *pUserData,
-    fts5_tokenizer_v2 *pTokenizer,
-    void (*xDestroy)(void*)
-  );
-
-  /* Find an existing tokenizer */
-  int (*xFindTokenizer_v2)(
-    fts5_api *pApi,
-    const char *zName,
-    void **ppUserData,
-    fts5_tokenizer_v2 **ppTokenizer
-  );
 };
 
 /*

ext/fts5/fts5Int.h

@@ -59,22 +59,6 @@ typedef sqlite3_uint64 u64;
 # define LARGEST_INT64  (0xffffffff|(((i64)0x7fffffff)<<32))
 # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64)
 
-/* The uptr type is an unsigned integer large enough to hold a pointer
-*/
-#if defined(HAVE_STDINT_H)
-  typedef uintptr_t uptr;
-#elif SQLITE_PTRSIZE==4
-  typedef u32 uptr;
-#else
-  typedef u64 uptr;
-#endif
-
-#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC
-# define EIGHT_BYTE_ALIGNMENT(X)   ((((uptr)(X) - (uptr)0)&3)==0)
-#else
-# define EIGHT_BYTE_ALIGNMENT(X)   ((((uptr)(X) - (uptr)0)&7)==0)
-#endif
-
 #endif
 
 /* Truncate very long tokens to this many bytes. Hard limit is 
@@ -158,18 +142,6 @@ struct Fts5Colset {
 */
 
 typedef struct Fts5Config Fts5Config;
-typedef struct Fts5TokenizerConfig Fts5TokenizerConfig;
-
-struct Fts5TokenizerConfig {
-  Fts5Tokenizer *pTok;
-  fts5_tokenizer_v2 *pApi2;
-  fts5_tokenizer *pApi1;
-  const char **azArg;
-  int nArg;
-  int ePattern;                   /* FTS_PATTERN_XXX constant */
-  const char *pLocale;            /* Current locale to use */
-  int nLocale;                    /* Size of pLocale in bytes */
-};
 
 /*
 ** An instance of the following structure encodes all information that can
@@ -209,12 +181,9 @@ struct Fts5TokenizerConfig {
 **
 **       INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex);
 **
-** bLocale:
-**   Set to true if locale=1 was specified when the table was created.
 */
 struct Fts5Config {
   sqlite3 *db;                    /* Database handle */
-  Fts5Global *pGlobal;            /* Global fts5 object for handle db */
   char *zDb;                      /* Database holding FTS index (e.g. "main") */
   char *zName;                    /* Name of FTS index */
   int nCol;                       /* Number of columns */
@@ -228,12 +197,12 @@ struct Fts5Config {
   char *zContentRowid;            /* "content_rowid=" option value */ 
   int bColumnsize;                /* "columnsize=" option value (dflt==1) */
   int bTokendata;                 /* "tokendata=" option value (dflt==0) */
-  int bLocale;                    /* "locale=" option value (dflt==0) */
   int eDetail;                    /* FTS5_DETAIL_XXX value */
   char *zContentExprlist;
-  Fts5TokenizerConfig t;
+  Fts5Tokenizer *pTok;
+  fts5_tokenizer *pTokApi;
   int bLock;                      /* True when table is preparing statement */
-  
+  int ePattern;                   /* FTS_PATTERN_XXX constant */
 
   /* Values loaded from the %_config table */
   int iVersion;                   /* fts5 file format 'version' */
@@ -299,8 +268,6 @@ int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);
 
 int sqlite3Fts5ConfigParseRank(const char*, char**, char**);
 
-void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...);
-
 /*
 ** End of interface to code in fts5_config.c.
 **************************************************************************/
@@ -345,7 +312,7 @@ char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
 void sqlite3Fts5Put32(u8*, int);
 int sqlite3Fts5Get32(const u8*);
 
-#define FTS5_POS2COLUMN(iPos) (int)((iPos >> 32) & 0x7FFFFFFF)
+#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
 #define FTS5_POS2OFFSET(iPos) (int)(iPos & 0x7FFFFFFF)
 
 typedef struct Fts5PoslistReader Fts5PoslistReader;
@@ -630,23 +597,18 @@ struct Fts5Table {
   Fts5Index *pIndex;              /* Full-text index */
 };
 
-int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig);
+int sqlite3Fts5GetTokenizer(
+  Fts5Global*, 
+  const char **azArg,
+  int nArg,
+  Fts5Config*,
+  char **pzErr
+);
 
 Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64);
 
 int sqlite3Fts5FlushToDisk(Fts5Table*);
 
-int sqlite3Fts5ExtractText(
-  Fts5Config *pConfig,
-  sqlite3_value *pVal,            /* Value to extract text from */
-  int bContent,                   /* Loaded from content table */
-  int *pbResetTokenizer,          /* OUT: True if ClearLocale() required */
-  const char **ppText,            /* OUT: Pointer to text buffer */
-  int *pnText                     /* OUT: Size of (*ppText) in bytes */
-);
-
-void sqlite3Fts5ClearLocale(Fts5Config *pConfig);
-
 /*
 ** End of interface to code in fts5.c.
 **************************************************************************/
@@ -726,7 +688,7 @@ int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
 int sqlite3Fts5DropAll(Fts5Config*);
 int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);
 
-int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**, int);
+int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**);
 int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
 int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);
 
@@ -752,9 +714,6 @@ int sqlite3Fts5StorageOptimize(Fts5Storage *p);
 int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
 int sqlite3Fts5StorageReset(Fts5Storage *p);
 
-void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage*);
-int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel);
-
 /*
 ** End of interface to code in fts5_storage.c.
 **************************************************************************/
@@ -907,7 +866,6 @@ int sqlite3Fts5TokenizerPattern(
     int (*xCreate)(void*, const char**, int, Fts5Tokenizer**),
     Fts5Tokenizer *pTok
 );
-int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig*);
 /*
 ** End of interface to code in fts5_tokenizer.c.
 **************************************************************************/

ext/fts5/fts5_aux.c

@@ -226,7 +226,6 @@ static int fts5HighlightCb(
   return rc;
 }
 
-
 /*
 ** Implementation of highlight() function.
 */
@@ -257,19 +256,12 @@ static void fts5HighlightFunction(
     sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC);
     rc = SQLITE_OK;
   }else if( ctx.zIn ){
-    const char *pLoc = 0;         /* Locale of column iCol */
-    int nLoc = 0;                 /* Size of pLoc in bytes */
     if( rc==SQLITE_OK ){
       rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter);
     }
 
     if( rc==SQLITE_OK ){
-      rc = pApi->xColumnLocale(pFts, iCol, &pLoc, &nLoc);
-    }
-    if( rc==SQLITE_OK ){
-      rc = pApi->xTokenize_v2(
-          pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx, fts5HighlightCb
-      );
+      rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
     }
     if( ctx.bOpen ){
       fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1);
@@ -466,8 +458,6 @@ static void fts5SnippetFunction(
   memset(&sFinder, 0, sizeof(Fts5SFinder));
   for(i=0; i<nCol; i++){
     if( iCol<0 || iCol==i ){
-      const char *pLoc = 0;       /* Locale of column iCol */
-      int nLoc = 0;               /* Size of pLoc in bytes */
       int nDoc;
       int nDocsize;
       int ii;
@@ -475,10 +465,8 @@ static void fts5SnippetFunction(
       sFinder.nFirst = 0;
       rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc);
       if( rc!=SQLITE_OK ) break;
-      rc = pApi->xColumnLocale(pFts, i, &pLoc, &nLoc);
-      if( rc!=SQLITE_OK ) break;
-      rc = pApi->xTokenize_v2(pFts, 
-          sFinder.zDoc, nDoc, pLoc, nLoc, (void*)&sFinder, fts5SentenceFinderCb
+      rc = pApi->xTokenize(pFts, 
+          sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb
       );
       if( rc!=SQLITE_OK ) break;
       rc = pApi->xColumnSize(pFts, i, &nDocsize);
@@ -536,9 +524,6 @@ static void fts5SnippetFunction(
     rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
   }
   if( ctx.zIn ){
-    const char *pLoc = 0;         /* Locale of column iBestCol */
-    int nLoc = 0;                 /* Bytes in pLoc */
-
     if( rc==SQLITE_OK ){
       rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
     }
@@ -557,12 +542,7 @@ static void fts5SnippetFunction(
     }
 
     if( rc==SQLITE_OK ){
-      rc = pApi->xColumnLocale(pFts, iBestCol, &pLoc, &nLoc);
-    }
-    if( rc==SQLITE_OK ){
-      rc = pApi->xTokenize_v2(
-          pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx,fts5HighlightCb
-      );
+      rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
     }
     if( ctx.bOpen ){
       fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1);

ext/fts5/fts5_config.c

@@ -234,6 +234,7 @@ static int fts5ConfigSetEnum(
 ** eventually free any such error message using sqlite3_free().
 */
 static int fts5ConfigParseSpecial(
+  Fts5Global *pGlobal,
   Fts5Config *pConfig,            /* Configuration object to update */
   const char *zCmd,               /* Special command to parse */
   const char *zArg,               /* Argument to parse */
@@ -297,11 +298,12 @@ static int fts5ConfigParseSpecial(
   if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){
     const char *p = (const char*)zArg;
     sqlite3_int64 nArg = strlen(zArg) + 1;
-    char **azArg = sqlite3Fts5MallocZero(&rc, (sizeof(char*) + 2) * nArg);
+    char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg);
+    char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2);
+    char *pSpace = pDel;
 
-    if( azArg ){
-      char *pSpace = (char*)&azArg[nArg];
-      if( pConfig->t.azArg ){
+    if( azArg && pSpace ){
+      if( pConfig->pTok ){
         *pzErr = sqlite3_mprintf("multiple tokenize=... directives");
         rc = SQLITE_ERROR;
       }else{
@@ -324,14 +326,16 @@ static int fts5ConfigParseSpecial(
           *pzErr = sqlite3_mprintf("parse error in tokenize directive");
           rc = SQLITE_ERROR;
         }else{
-          pConfig->t.azArg = (const char**)azArg;
-          pConfig->t.nArg = nArg;
-          azArg = 0;
+          rc = sqlite3Fts5GetTokenizer(pGlobal, 
+              (const char**)azArg, (int)nArg, pConfig,
+              pzErr
+          );
         }
       }
     }
-    sqlite3_free(azArg);
 
+    sqlite3_free(azArg);
+    sqlite3_free(pDel);
     return rc;
   }
 
@@ -380,16 +384,6 @@ static int fts5ConfigParseSpecial(
     return rc;
   }
 
-  if( sqlite3_strnicmp("locale", zCmd, nCmd)==0 ){
-    if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
-      *pzErr = sqlite3_mprintf("malformed locale=... directive");
-      rc = SQLITE_ERROR;
-    }else{
-      pConfig->bLocale = (zArg[0]=='1');
-    }
-    return rc;
-  }
-
   if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){
     const Fts5Enum aDetail[] = {
       { "none", FTS5_DETAIL_NONE },
@@ -418,6 +412,16 @@ static int fts5ConfigParseSpecial(
   return SQLITE_ERROR;
 }
 
+/*
+** Allocate an instance of the default tokenizer ("simple") at 
+** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error
+** code if an error occurs.
+*/
+static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){
+  assert( pConfig->pTok==0 && pConfig->pTokApi==0 );
+  return sqlite3Fts5GetTokenizer(pGlobal, 0, 0, pConfig, 0);
+}
+
 /*
 ** Gobble up the first bareword or quoted word from the input buffer zIn.
 ** Return a pointer to the character immediately following the last in
@@ -550,7 +554,6 @@ int sqlite3Fts5ConfigParse(
   *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config));
   if( pRet==0 ) return SQLITE_NOMEM;
   memset(pRet, 0, sizeof(Fts5Config));
-  pRet->pGlobal = pGlobal;
   pRet->db = db;
   pRet->iCookie = -1;
 
@@ -599,7 +602,7 @@ int sqlite3Fts5ConfigParse(
         rc = SQLITE_ERROR;
       }else{
         if( bOption ){
-          rc = fts5ConfigParseSpecial(pRet, 
+          rc = fts5ConfigParseSpecial(pGlobal, pRet, 
             ALWAYS(zOne)?zOne:"",
             zTwo?zTwo:"",
             pzErr
@@ -637,6 +640,13 @@ int sqlite3Fts5ConfigParse(
     rc = SQLITE_ERROR;
   }
 
+  /* If a tokenizer= option was successfully parsed, the tokenizer has
+  ** already been allocated. Otherwise, allocate an instance of the default
+  ** tokenizer (unicode61) now.  */
+  if( rc==SQLITE_OK && pRet->pTok==0 ){
+    rc = fts5ConfigDefaultTokenizer(pGlobal, pRet);
+  }
+
   /* If no zContent option was specified, fill in the default values. */
   if( rc==SQLITE_OK && pRet->zContent==0 ){
     const char *zTail = 0;
@@ -678,14 +688,9 @@ int sqlite3Fts5ConfigParse(
 void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
   if( pConfig ){
     int i;
-    if( pConfig->t.pTok ){
-      if( pConfig->t.pApi1 ){
-        pConfig->t.pApi1->xDelete(pConfig->t.pTok);
-      }else{
-        pConfig->t.pApi2->xDelete(pConfig->t.pTok);
-      }
+    if( pConfig->pTok ){
+      pConfig->pTokApi->xDelete(pConfig->pTok);
     }
-    sqlite3_free((char*)pConfig->t.azArg);
     sqlite3_free(pConfig->zDb);
     sqlite3_free(pConfig->zName);
     for(i=0; i<pConfig->nCol; i++){
@@ -760,24 +765,10 @@ int sqlite3Fts5Tokenize(
   void *pCtx,                     /* Context passed to xToken() */
   int (*xToken)(void*, int, const char*, int, int, int)    /* Callback */
 ){
-  int rc = SQLITE_OK;
-  if( pText ){
-    if( pConfig->t.pTok==0 ){
-      rc = sqlite3Fts5LoadTokenizer(pConfig);
-    }
-    if( rc==SQLITE_OK ){
-      if( pConfig->t.pApi1 ){
-        rc = pConfig->t.pApi1->xTokenize(
-            pConfig->t.pTok, pCtx, flags, pText, nText, xToken
-        );
-      }else{
-        rc = pConfig->t.pApi2->xTokenize(pConfig->t.pTok, pCtx, flags, 
-            pText, nText, pConfig->t.pLocale, pConfig->t.nLocale, xToken
-        );
-      }
-    }
-  }
-  return rc;
+  if( pText==0 ) return SQLITE_OK;
+  return pConfig->pTokApi->xTokenize(
+      pConfig->pTok, pCtx, flags, pText, nText, xToken
+  );
 }
 
 /*
@@ -1031,10 +1022,13 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
    && iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE
   ){
     rc = SQLITE_ERROR;
-    sqlite3Fts5ConfigErrmsg(pConfig, "invalid fts5 file format "
-        "(found %d, expected %d or %d) - run 'rebuild'",
-        iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE
-    );
+    if( pConfig->pzErrmsg ){
+      assert( 0==*pConfig->pzErrmsg );
+      *pConfig->pzErrmsg = sqlite3_mprintf("invalid fts5 file format "
+          "(found %d, expected %d or %d) - run 'rebuild'",
+          iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE
+      );
+    }
   }else{
     pConfig->iVersion = iVersion;
   }
@@ -1044,26 +1038,3 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
   }
   return rc;
 }
-
-/*
-** Set (*pConfig->pzErrmsg) to point to an sqlite3_malloc()ed buffer 
-** containing the error message created using printf() style formatting
-** string zFmt and its trailing arguments.
-*/
-void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){
-  va_list ap;                     /* ... printf arguments */
-  char *zMsg = 0;
-
-  va_start(ap, zFmt);
-  zMsg = sqlite3_vmprintf(zFmt, ap);
-  if( pConfig->pzErrmsg ){
-    assert( *pConfig->pzErrmsg==0 );
-    *pConfig->pzErrmsg = zMsg;
-  }else{
-    sqlite3_free(zMsg);
-  }
-
-  va_end(ap);
-}
-
-

ext/fts5/fts5_expr.c

@@ -54,7 +54,7 @@ struct Fts5Expr {
 
 /*
 ** eType:
-**   Expression node type. Usually one of:
+**   Expression node type. Always one of:
 **
 **       FTS5_AND                 (nChild, apChild valid)
 **       FTS5_OR                  (nChild, apChild valid)
@@ -62,10 +62,6 @@ struct Fts5Expr {
 **       FTS5_STRING              (pNear valid)
 **       FTS5_TERM                (pNear valid)
 **
-**   An expression node with eType==0 may also exist. It always matches zero
-**   rows. This is created when a phrase containing no tokens is parsed.
-**   e.g. "".
-**
 ** iHeight:
 **   Distance from this node to furthest leaf. This is always 0 for nodes
 **   of type FTS5_STRING and FTS5_TERM. For all other nodes it is one 
@@ -286,12 +282,11 @@ int sqlite3Fts5ExprNew(
   }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF );
   sqlite3Fts5ParserFree(pEngine, fts5ParseFree);
 
-  assert( sParse.pExpr || sParse.rc!=SQLITE_OK );
   assert_expr_depth_ok(sParse.rc, sParse.pExpr);
 
   /* If the LHS of the MATCH expression was a user column, apply the
   ** implicit column-filter.  */
-  if( sParse.rc==SQLITE_OK && iCol<pConfig->nCol ){
+  if( iCol<pConfig->nCol && sParse.pExpr && sParse.rc==SQLITE_OK ){
     int n = sizeof(Fts5Colset);
     Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n);
     if( pColset ){
@@ -308,7 +303,15 @@ int sqlite3Fts5ExprNew(
       sParse.rc = SQLITE_NOMEM;
       sqlite3Fts5ParseNodeFree(sParse.pExpr);
     }else{
-      pNew->pRoot = sParse.pExpr;
+      if( !sParse.pExpr ){
+        const int nByte = sizeof(Fts5ExprNode);
+        pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&sParse.rc, nByte);
+        if( pNew->pRoot ){
+          pNew->pRoot->bEof = 1;
+        }
+      }else{
+        pNew->pRoot = sParse.pExpr;
+      }
       pNew->pIndex = 0;
       pNew->pConfig = pConfig;
       pNew->apExprPhrase = sParse.apPhrase;
@@ -1126,7 +1129,7 @@ static int fts5ExprNodeTest_STRING(
           }
         }else{
           Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
-          if( pIter->iRowid==iLast ) continue;
+          if( pIter->iRowid==iLast || pIter->bEof ) continue;
           bMatch = 0;
           if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
             return rc;
@@ -1648,6 +1651,9 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset(
   Fts5ExprNearset *pRet = 0;
 
   if( pParse->rc==SQLITE_OK ){
+    if( pPhrase==0 ){
+      return pNear;
+    }
     if( pNear==0 ){
       sqlite3_int64 nByte;
       nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*);
@@ -1869,7 +1875,6 @@ Fts5ExprPhrase *sqlite3Fts5ParseTerm(
     }else if( sCtx.pPhrase->nTerm ){
       sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix;
     }
-    assert( pParse->apPhrase!=0 );
     pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase;
   }
 
@@ -1889,7 +1894,7 @@ int sqlite3Fts5ExprClonePhrase(
   Fts5ExprPhrase *pOrig = 0;      /* The phrase extracted from pExpr */
   Fts5Expr *pNew = 0;             /* Expression to return via *ppNew */
   TokenCtx sCtx = {0,0,0};        /* Context object for fts5ParseTokenize */
-  if( !pExpr || iPhrase<0 || iPhrase>=pExpr->nPhrase ){
+  if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){
     rc = SQLITE_RANGE;
   }else{
     pOrig = pExpr->apExprPhrase[iPhrase];
@@ -2257,9 +2262,6 @@ static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
   }
 }
 
-/*
-** Add pSub as a child of p.
-*/
 static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){
   int ii = p->nChild;
   if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){
@@ -2404,23 +2406,19 @@ Fts5ExprNode *sqlite3Fts5ParseNode(
                   "fts5: %s queries are not supported (detail!=full)", 
                   pNear->nPhrase==1 ? "phrase": "NEAR"
               );
-              sqlite3Fts5ParseNodeFree(pRet);
+              sqlite3_free(pRet);
               pRet = 0;
-              pNear = 0;
-              assert( pLeft==0 && pRight==0 );
             }
           }
         }else{
-          assert( pNear==0 );
           fts5ExprAddChildren(pRet, pLeft);
           fts5ExprAddChildren(pRet, pRight);
-          pLeft = pRight = 0;
           if( pRet->iHeight>SQLITE_FTS5_MAX_EXPR_DEPTH ){
             sqlite3Fts5ParseError(pParse, 
                 "fts5 expression tree is too large (maximum depth %d)", 
                 SQLITE_FTS5_MAX_EXPR_DEPTH
             );
-            sqlite3Fts5ParseNodeFree(pRet);
+            sqlite3_free(pRet);
             pRet = 0;
           }
         }
@@ -2472,8 +2470,6 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
         );
 
     if( pRight->eType==FTS5_EOF ){
-      assert( pParse->apPhrase!=0 );
-      assert( pParse->nPhrase>0 );
       assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] );
       sqlite3Fts5ParseNodeFree(pRight);
       pRet = pLeft;
@@ -3106,7 +3102,6 @@ static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){
   pNode->iRowid = iRowid;
   pNode->bEof = 0;
   switch( pNode->eType ){
-    case 0:
     case FTS5_TERM:
     case FTS5_STRING:
       return (pNode->pNear->apPhrase[0]->poslist.n>0);

ext/fts5/fts5_index.c

@@ -831,12 +831,11 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){
     if( rc==SQLITE_OK ){
       u8 *aOut = 0;               /* Read blob data into this buffer */
       int nByte = sqlite3_blob_bytes(p->pReader);
-      int szData = (sizeof(Fts5Data) + 7) & ~7;
-      sqlite3_int64 nAlloc = szData + nByte + FTS5_DATA_PADDING;
+      sqlite3_int64 nAlloc = sizeof(Fts5Data) + nByte + FTS5_DATA_PADDING;
       pRet = (Fts5Data*)sqlite3_malloc64(nAlloc);
       if( pRet ){
         pRet->nn = nByte;
-        aOut = pRet->p = (u8*)pRet + szData;
+        aOut = pRet->p = (u8*)&pRet[1];
       }else{
         rc = SQLITE_NOMEM;
       }
@@ -859,7 +858,6 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){
   }
 
   assert( (pRet==0)==(p->rc!=SQLITE_OK) );
-  assert( pRet==0 || EIGHT_BYTE_ALIGNMENT( pRet->p ) );
   return pRet;
 }
 

ext/fts5/fts5_storage.c

@@ -16,40 +16,13 @@
 
 #include "fts5Int.h"
 
-/*
-** pSavedRow:
-**   SQL statement FTS5_STMT_LOOKUP2 is a copy of FTS5_STMT_LOOKUP, it 
-**   does a by-rowid lookup to retrieve a single row from the %_content 
-**   table or equivalent external-content table/view.
-**
-**   However, FTS5_STMT_LOOKUP2 is only used when retrieving the original
-**   values for a row being UPDATEd. In that case, the SQL statement is
-**   not reset and pSavedRow is set to point at it. This is so that the
-**   insert operation that follows the delete may access the original 
-**   row values for any new values for which sqlite3_value_nochange() returns
-**   true. i.e. if the user executes:
-**
-**        CREATE VIRTUAL TABLE ft USING fts5(a, b, c, locale=1);
-**        ...
-**        UPDATE fts SET a=?, b=? WHERE rowid=?;
-**
-**   then the value passed to the xUpdate() method of this table as the 
-**   new.c value is an sqlite3_value_nochange() value. So in this case it
-**   must be read from the saved row stored in Fts5Storage.pSavedRow.
-**
-**   This is necessary - using sqlite3_value_nochange() instead of just having
-**   SQLite pass the original value back via xUpdate() - so as not to discard
-**   any locale information associated with such values.
-**
-*/
 struct Fts5Storage {
   Fts5Config *pConfig;
   Fts5Index *pIndex;
   int bTotalsValid;               /* True if nTotalRow/aTotalSize[] are valid */
   i64 nTotalRow;                  /* Total number of rows in FTS table */
   i64 *aTotalSize;                /* Total sizes of each column */ 
-  sqlite3_stmt *pSavedRow;
-  sqlite3_stmt *aStmt[12];
+  sqlite3_stmt *aStmt[11];
 };
 
 
@@ -63,15 +36,14 @@ struct Fts5Storage {
 # error "FTS5_STMT_LOOKUP mismatch" 
 #endif
 
-#define FTS5_STMT_LOOKUP2         3
-#define FTS5_STMT_INSERT_CONTENT  4
-#define FTS5_STMT_REPLACE_CONTENT 5
-#define FTS5_STMT_DELETE_CONTENT  6
-#define FTS5_STMT_REPLACE_DOCSIZE 7
-#define FTS5_STMT_DELETE_DOCSIZE  8
-#define FTS5_STMT_LOOKUP_DOCSIZE  9
-#define FTS5_STMT_REPLACE_CONFIG 10
-#define FTS5_STMT_SCAN           11
+#define FTS5_STMT_INSERT_CONTENT  3
+#define FTS5_STMT_REPLACE_CONTENT 4
+#define FTS5_STMT_DELETE_CONTENT  5
+#define FTS5_STMT_REPLACE_DOCSIZE  6
+#define FTS5_STMT_DELETE_DOCSIZE  7
+#define FTS5_STMT_LOOKUP_DOCSIZE  8
+#define FTS5_STMT_REPLACE_CONFIG 9
+#define FTS5_STMT_SCAN 10
 
 /*
 ** Prepare the two insert statements - Fts5Storage.pInsertContent and
@@ -101,7 +73,6 @@ static int fts5StorageGetStmt(
       "SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC",
       "SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC",
       "SELECT %s FROM %s T WHERE T.%Q=?",               /* LOOKUP  */
-      "SELECT %s FROM %s T WHERE T.%Q=?",               /* LOOKUP2  */
 
       "INSERT INTO %Q.'%q_content' VALUES(%s)",         /* INSERT_CONTENT  */
       "REPLACE INTO %Q.'%q_content' VALUES(%s)",        /* REPLACE_CONTENT */
@@ -117,8 +88,6 @@ static int fts5StorageGetStmt(
     Fts5Config *pC = p->pConfig;
     char *zSql = 0;
 
-    assert( ArraySize(azStmt)==ArraySize(p->aStmt) );
-
     switch( eStmt ){
       case FTS5_STMT_SCAN:
         zSql = sqlite3_mprintf(azStmt[eStmt], 
@@ -135,7 +104,6 @@ static int fts5StorageGetStmt(
         break;
 
       case FTS5_STMT_LOOKUP:
-      case FTS5_STMT_LOOKUP2:
         zSql = sqlite3_mprintf(azStmt[eStmt], 
             pC->zContentExprlist, pC->zContent, pC->zContentRowid
         );
@@ -182,7 +150,7 @@ static int fts5StorageGetStmt(
       rc = SQLITE_NOMEM;
     }else{
       int f = SQLITE_PREPARE_PERSISTENT;
-      if( eStmt>FTS5_STMT_LOOKUP2 ) f |= SQLITE_PREPARE_NO_VTAB;
+      if( eStmt>FTS5_STMT_LOOKUP ) f |= SQLITE_PREPARE_NO_VTAB;
       p->pConfig->bLock++;
       rc = sqlite3_prepare_v3(pC->db, zSql, -1, f, &p->aStmt[eStmt], 0);
       p->pConfig->bLock--;
@@ -431,49 +399,15 @@ static int fts5StorageInsertCallback(
   return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken);
 }
 
-/*
-** This function is used as part of an UPDATE statement that modifies the
-** rowid of a row. In that case, this function is called first to set
-** Fts5Storage.pSavedRow to point to a statement that may be used to 
-** access the original values of the row being deleted - iDel.
-**
-** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
-** It is not considered an error if row iDel does not exist. In this case
-** pSavedRow is not set and SQLITE_OK returned.
-*/
-int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel){
-  int rc = SQLITE_OK;
-  sqlite3_stmt *pSeek = 0;
-
-  assert( p->pSavedRow==0 );
-  rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+1, &pSeek, 0);
-  if( rc==SQLITE_OK ){
-    sqlite3_bind_int64(pSeek, 1, iDel);
-    if( sqlite3_step(pSeek)!=SQLITE_ROW ){
-      rc = sqlite3_reset(pSeek);
-    }else{
-      p->pSavedRow = pSeek;
-    }
-  }
-
-  return rc;
-}
-
 /*
 ** If a row with rowid iDel is present in the %_content table, add the
 ** delete-markers to the FTS index necessary to delete it. Do not actually
 ** remove the %_content row at this time though.
-**
-** If parameter bSaveRow is true, then Fts5Storage.pSavedRow is left
-** pointing to a statement (FTS5_STMT_LOOKUP2) that may be used to access
-** the original values of the row being deleted. This is used by UPDATE
-** statements.
 */
 static int fts5StorageDeleteFromIndex(
   Fts5Storage *p, 
   i64 iDel, 
-  sqlite3_value **apVal,
-  int bSaveRow                    /* True to set pSavedRow */
+  sqlite3_value **apVal
 ){
   Fts5Config *pConfig = p->pConfig;
   sqlite3_stmt *pSeek = 0;        /* SELECT to read row iDel from %_data */
@@ -482,21 +416,12 @@ static int fts5StorageDeleteFromIndex(
   int iCol;
   Fts5InsertCtx ctx;
 
-  assert( bSaveRow==0 || apVal==0 );
-  assert( bSaveRow==0 || bSaveRow==1 );
-  assert( FTS5_STMT_LOOKUP2==FTS5_STMT_LOOKUP+1 );
-
   if( apVal==0 ){
-    if( p->pSavedRow && bSaveRow ){
-      pSeek = p->pSavedRow;
-      p->pSavedRow = 0;
-    }else{
-      rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+bSaveRow, &pSeek, 0);
-      if( rc!=SQLITE_OK ) return rc;
-      sqlite3_bind_int64(pSeek, 1, iDel);
-      if( sqlite3_step(pSeek)!=SQLITE_ROW ){
-        return sqlite3_reset(pSeek);
-      }
+    rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0);
+    if( rc!=SQLITE_OK ) return rc;
+    sqlite3_bind_int64(pSeek, 1, iDel);
+    if( sqlite3_step(pSeek)!=SQLITE_ROW ){
+      return sqlite3_reset(pSeek);
     }
   }
 
@@ -504,32 +429,26 @@ static int fts5StorageDeleteFromIndex(
   ctx.iCol = -1;
   for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
     if( pConfig->abUnindexed[iCol-1]==0 ){
-      sqlite3_value *pVal = 0;
-      const char *pText = 0;
-      int nText = 0;
-      int bReset = 0;
-
+      const char *zText;
+      int nText;
       assert( pSeek==0 || apVal==0 );
       assert( pSeek!=0 || apVal!=0 );
       if( pSeek ){
-        pVal = sqlite3_column_value(pSeek, iCol);
+        zText = (const char*)sqlite3_column_text(pSeek, iCol);
+        nText = sqlite3_column_bytes(pSeek, iCol);
+      }else if( ALWAYS(apVal) ){
+        zText = (const char*)sqlite3_value_text(apVal[iCol-1]);
+        nText = sqlite3_value_bytes(apVal[iCol-1]);
       }else{
-        pVal = apVal[iCol-1];
+        continue;
       }
-
-      rc = sqlite3Fts5ExtractText(
-          pConfig, pVal, pSeek!=0, &bReset, &pText, &nText
+      ctx.szCol = 0;
+      rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, 
+          zText, nText, (void*)&ctx, fts5StorageInsertCallback
       );
-      if( rc==SQLITE_OK ){
-        ctx.szCol = 0;
-        rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, 
-            pText, nText, (void*)&ctx, fts5StorageInsertCallback
-        );
-        p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
-        if( rc==SQLITE_OK && p->aTotalSize[iCol-1]<0 ){
-          rc = FTS5_CORRUPT;
-        }
-        if( bReset ) sqlite3Fts5ClearLocale(pConfig);
+      p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
+      if( p->aTotalSize[iCol-1]<0 ){
+        rc = FTS5_CORRUPT;
       }
     }
   }
@@ -539,29 +458,11 @@ static int fts5StorageDeleteFromIndex(
     p->nTotalRow--;
   }
 
-  if( rc==SQLITE_OK && bSaveRow ){
-    assert( p->pSavedRow==0 );
-    p->pSavedRow = pSeek;
-  }else{
-    rc2 = sqlite3_reset(pSeek);
-    if( rc==SQLITE_OK ) rc = rc2;
-  }
+  rc2 = sqlite3_reset(pSeek);
+  if( rc==SQLITE_OK ) rc = rc2;
   return rc;
 }
 
-/*
-** Reset any saved statement pSavedRow. Zero pSavedRow as well. This
-** should be called by the xUpdate() method of the fts5 table before 
-** returning from any operation that may have set Fts5Storage.pSavedRow.
-*/
-void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage *pStorage){
-  assert( pStorage->pSavedRow==0 
-       || pStorage->pSavedRow==pStorage->aStmt[FTS5_STMT_LOOKUP2] 
-  );
-  sqlite3_reset(pStorage->pSavedRow);
-  pStorage->pSavedRow = 0;
-}
-
 /*
 ** This function is called to process a DELETE on a contentless_delete=1
 ** table. It adds the tombstone required to delete the entry with rowid 
@@ -618,12 +519,12 @@ static int fts5StorageInsertDocsize(
         rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin);
         sqlite3_bind_int64(pReplace, 3, iOrigin);
       }
-    }
-    if( rc==SQLITE_OK ){
-      sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
-      sqlite3_step(pReplace);
-      rc = sqlite3_reset(pReplace);
-      sqlite3_bind_null(pReplace, 2);
+      if( rc==SQLITE_OK ){
+        sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
+        sqlite3_step(pReplace);
+        rc = sqlite3_reset(pReplace);
+        sqlite3_bind_null(pReplace, 2);
+      }
     }
   }
   return rc;
@@ -677,12 +578,7 @@ static int fts5StorageSaveTotals(Fts5Storage *p){
 /*
 ** Remove a row from the FTS table.
 */
-int sqlite3Fts5StorageDelete(
-  Fts5Storage *p,                 /* Storage object */
-  i64 iDel,                       /* Rowid to delete from table */
-  sqlite3_value **apVal,          /* Optional - values to remove from index */
-  int bSaveRow                    /* If true, set pSavedRow for deleted row */
-){
+int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
   Fts5Config *pConfig = p->pConfig;
   int rc;
   sqlite3_stmt *pDel = 0;
@@ -699,7 +595,7 @@ int sqlite3Fts5StorageDelete(
     if( p->pConfig->bContentlessDelete ){
       rc = fts5StorageContentlessDelete(p, iDel);
     }else{
-      rc = fts5StorageDeleteFromIndex(p, iDel, apVal, bSaveRow);
+      rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
     }
   }
 
@@ -788,21 +684,14 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){
     for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
       ctx.szCol = 0;
       if( pConfig->abUnindexed[ctx.iCol]==0 ){
-        int bReset = 0;           /* True if tokenizer locale must be reset */
-        int nText = 0;            /* Size of pText in bytes */
-        const char *pText = 0;    /* Pointer to buffer containing text value */
-        sqlite3_value *pVal = sqlite3_column_value(pScan, ctx.iCol+1);
-
-        rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &pText, &nText);
-        if( rc==SQLITE_OK ){
-          rc = sqlite3Fts5Tokenize(pConfig, 
-              FTS5_TOKENIZE_DOCUMENT,
-              pText, nText,
-              (void*)&ctx,
-              fts5StorageInsertCallback
-          );
-          if( bReset ) sqlite3Fts5ClearLocale(pConfig);
-        }
+        const char *zText = (const char*)sqlite3_column_text(pScan, ctx.iCol+1);
+        int nText = sqlite3_column_bytes(pScan, ctx.iCol+1);
+        rc = sqlite3Fts5Tokenize(pConfig, 
+            FTS5_TOKENIZE_DOCUMENT,
+            zText, nText,
+            (void*)&ctx,
+            fts5StorageInsertCallback
+        );
       }
       sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
       p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
@@ -886,31 +775,7 @@ int sqlite3Fts5StorageContentInsert(
     int i;                        /* Counter variable */
     rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0);
     for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
-      sqlite3_value *pVal = apVal[i];
-      if( sqlite3_value_nochange(pVal) && p->pSavedRow ){
-        /* This is an UPDATE statement, and column (i-2) was not modified.
-        ** Retrieve the value from Fts5Storage.pSavedRow instead. */
-        pVal = sqlite3_column_value(p->pSavedRow, i-1);
-      }else if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE ){
-        assert( pConfig->bLocale );
-        assert( i>1 );
-        if( pConfig->abUnindexed[i-2] ){
-          /* At attempt to insert an fts5_locale() value into an UNINDEXED
-          ** column. Strip the locale away and just bind the text.  */
-          const char *pText = 0;
-          int nText = 0;
-          rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, 0, &pText, &nText);
-          sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT);
-        }else{
-          const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal);
-          int nBlob = sqlite3_value_bytes(pVal);
-          assert( nBlob>4 );
-          sqlite3_bind_blob(pInsert, i, pBlob+4, nBlob-4, SQLITE_TRANSIENT);
-        }
-        continue;
-      }
-
-      rc = sqlite3_bind_value(pInsert, i, pVal);
+      rc = sqlite3_bind_value(pInsert, i, apVal[i]);
     }
     if( rc==SQLITE_OK ){
       sqlite3_step(pInsert);
@@ -945,24 +810,14 @@ int sqlite3Fts5StorageIndexInsert(
   for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
     ctx.szCol = 0;
     if( pConfig->abUnindexed[ctx.iCol]==0 ){
-      int bReset = 0;             /* True if tokenizer locale must be reset */
-      int nText = 0;              /* Size of pText in bytes */
-      const char *pText = 0;      /* Pointer to buffer containing text value */
-      sqlite3_value *pVal = apVal[ctx.iCol+2];
-      int bDisk = 0;
-      if( p->pSavedRow && sqlite3_value_nochange(pVal) ){
-        pVal = sqlite3_column_value(p->pSavedRow, ctx.iCol+1);
-        bDisk = 1;
-      }
-      rc = sqlite3Fts5ExtractText(pConfig, pVal, bDisk, &bReset, &pText,&nText);
-      if( rc==SQLITE_OK ){
-        assert( bReset==0 || pConfig->bLocale );
-        rc = sqlite3Fts5Tokenize(pConfig, 
-            FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx,
-            fts5StorageInsertCallback
-        );
-        if( bReset ) sqlite3Fts5ClearLocale(pConfig);
-      }
+      const char *zText = (const char*)sqlite3_value_text(apVal[ctx.iCol+2]);
+      int nText = sqlite3_value_bytes(apVal[ctx.iCol+2]);
+      rc = sqlite3Fts5Tokenize(pConfig, 
+          FTS5_TOKENIZE_DOCUMENT,
+          zText, nText,
+          (void*)&ctx,
+          fts5StorageInsertCallback
+      );
     }
     sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
     p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
@@ -1133,22 +988,14 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){
             rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
           }
           if( rc==SQLITE_OK ){
-            int bReset = 0;        /* True if tokenizer locale must be reset */
-            int nText = 0;         /* Size of pText in bytes */
-            const char *pText = 0; /* Pointer to buffer containing text value */
-
-            rc = sqlite3Fts5ExtractText(pConfig, 
-                sqlite3_column_value(pScan, i+1), 1, &bReset, &pText, &nText
+            const char *zText = (const char*)sqlite3_column_text(pScan, i+1);
+            int nText = sqlite3_column_bytes(pScan, i+1);
+            rc = sqlite3Fts5Tokenize(pConfig, 
+                FTS5_TOKENIZE_DOCUMENT,
+                zText, nText,
+                (void*)&ctx,
+                fts5StorageIntegrityCallback
             );
-            if( rc==SQLITE_OK ){
-              rc = sqlite3Fts5Tokenize(pConfig, 
-                  FTS5_TOKENIZE_DOCUMENT,
-                  pText, nText,
-                  (void*)&ctx,
-                  fts5StorageIntegrityCallback
-              );
-              if( bReset ) sqlite3Fts5ClearLocale(pConfig);
-            }
           }
           if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
             rc = FTS5_CORRUPT;

ext/fts5/fts5_tcl.c

@@ -14,7 +14,14 @@
 
 
 #ifdef SQLITE_TEST
-#include "tclsqlite.h"
+#if defined(INCLUDE_SQLITE_TCL_H)
+#  include "sqlite_tcl.h"
+#else
+#  include "tcl.h"
+#  ifndef SQLITE_TCLAPI
+#    define SQLITE_TCLAPI
+#  endif
+#endif
 
 #ifdef SQLITE_ENABLE_FTS5
 
@@ -240,7 +247,6 @@ static int SQLITE_TCLAPI xF5tApi(
 
     { "xQueryToken",       2, "IPHRASE ITERM" },      /* 18 */
     { "xInstToken",        2, "IDX ITERM" },          /* 19 */
-    { "xColumnLocale",     1, "COL" },                /* 20 */
     { 0, 0, 0}
   };
 
@@ -291,12 +297,12 @@ static int SQLITE_TCLAPI xF5tApi(
       break;
     }
     CASE(3, "xTokenize") {
-      Tcl_Size nText;
+      int nText;
       char *zText = Tcl_GetStringFromObj(objv[2], &nText);
       F5tFunction ctx;
       ctx.interp = interp;
       ctx.pScript = objv[3];
-      rc = p->pApi->xTokenize(p->pFts, zText, (int)nText, &ctx, xTokenizeCb);
+      rc = p->pApi->xTokenize(p->pFts, zText, nText, &ctx, xTokenizeCb);
       if( rc==SQLITE_OK ){
         Tcl_ResetResult(interp);
       }
@@ -529,20 +535,6 @@ static int SQLITE_TCLAPI xF5tApi(
       break;
     }
 
-    CASE(20, "xColumnLocale") {
-      const char *z = 0;
-      int n = 0;
-      int iCol;
-      if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){
-        return TCL_ERROR;
-      }
-      rc = p->pApi->xColumnLocale(p->pFts, iCol, &z, &n);
-      if( rc==SQLITE_OK && z ){
-        Tcl_SetObjResult(interp, Tcl_NewStringObj(z, n));
-      }
-      break;
-    }
-
     default: 
       assert( 0 );
       break;
@@ -613,16 +605,15 @@ static void xF5tFunction(
     sqlite3_result_error(pCtx, Tcl_GetStringResult(p->interp), -1);
   }else{
     Tcl_Obj *pVar = Tcl_GetObjResult(p->interp);
+    int n;
     const char *zType = (pVar->typePtr ? pVar->typePtr->name : "");
     char c = zType[0];
     if( c=='b' && strcmp(zType,"bytearray")==0 && pVar->bytes==0 ){
       /* Only return a BLOB type if the Tcl variable is a bytearray and
       ** has no string representation. */
-      Tcl_Size nn;
-      unsigned char *data = Tcl_GetByteArrayFromObj(pVar, &nn);
-      sqlite3_result_blob(pCtx, data, (int)nn, SQLITE_TRANSIENT);
+      unsigned char *data = Tcl_GetByteArrayFromObj(pVar, &n);
+      sqlite3_result_blob(pCtx, data, n, SQLITE_TRANSIENT);
     }else if( c=='b' && strcmp(zType,"boolean")==0 ){
-      int n;
       Tcl_GetIntFromObj(0, pVar, &n);
       sqlite3_result_int(pCtx, n);
     }else if( c=='d' && strcmp(zType,"double")==0 ){
@@ -635,9 +626,8 @@ static void xF5tFunction(
       Tcl_GetWideIntFromObj(0, pVar, &v);
       sqlite3_result_int64(pCtx, v);
     }else{
-      Tcl_Size nn;
-      unsigned char *data = (unsigned char *)Tcl_GetStringFromObj(pVar, &nn);
-      sqlite3_result_text(pCtx, (char*)data, (int)nn, SQLITE_TRANSIENT);
+      unsigned char *data = (unsigned char *)Tcl_GetStringFromObj(pVar, &n);
+      sqlite3_result_text(pCtx, (char *)data, n, SQLITE_TRANSIENT);
     }
   }
 }
@@ -730,7 +720,7 @@ static int SQLITE_TCLAPI f5tTokenize(
   Tcl_Obj *CONST objv[]
 ){
   char *zText;
-  Tcl_Size nText;
+  int nText;
   sqlite3 *db = 0;
   fts5_api *pApi = 0;
   Fts5Tokenizer *pTok = 0;
@@ -739,7 +729,7 @@ static int SQLITE_TCLAPI f5tTokenize(
   void *pUserdata;
   int rc;
 
-  Tcl_Size nArg;
+  int nArg;
   const char **azArg;
   F5tTokenizeCtx ctx;
 
@@ -771,7 +761,7 @@ static int SQLITE_TCLAPI f5tTokenize(
     return TCL_ERROR;
   }
 
-  rc = tokenizer.xCreate(pUserdata, &azArg[1], (int)(nArg-1), &pTok);
+  rc = tokenizer.xCreate(pUserdata, &azArg[1], nArg-1, &pTok);
   if( rc!=SQLITE_OK ){
     Tcl_AppendResult(interp, "error in tokenizer.xCreate()", 0);
     return TCL_ERROR;
@@ -783,7 +773,7 @@ static int SQLITE_TCLAPI f5tTokenize(
   ctx.pRet = pRet;
   ctx.zInput = zText;
   rc = tokenizer.xTokenize(
-      pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, zText,(int)nText, xTokenizeCb2
+      pTok, (void*)&ctx, FTS5_TOKENIZE_DOCUMENT, zText, nText, xTokenizeCb2
   );
   tokenizer.xDelete(pTok);
   if( rc!=SQLITE_OK ){
@@ -811,32 +801,18 @@ typedef struct F5tTokenizerInstance F5tTokenizerInstance;
 struct F5tTokenizerContext {
   void *pCtx;
   int (*xToken)(void*, int, const char*, int, int, int);
-  F5tTokenizerInstance *pInst;
 };
 
 struct F5tTokenizerModule {
   Tcl_Interp *interp;
   Tcl_Obj *pScript;
-  void *pParentCtx;
-  fts5_tokenizer_v2 parent_v2;
-  fts5_tokenizer parent;
   F5tTokenizerContext *pContext;
 };
 
-/*
-** zLocale:
-**   Within a call to xTokenize_v2(), pLocale/nLocale store the locale
-**   passed to the call by fts5. This can be retrieved by a Tcl tokenize 
-**   script using [sqlite3_fts5_locale].
-*/
 struct F5tTokenizerInstance {
   Tcl_Interp *interp;
   Tcl_Obj *pScript;
-  F5tTokenizerModule *pModule;
-  Fts5Tokenizer *pParent;
   F5tTokenizerContext *pContext;
-  const char *pLocale;
-  int nLocale;
 };
 
 static int f5tTokenizerCreate(
@@ -845,20 +821,11 @@ static int f5tTokenizerCreate(
   int nArg, 
   Fts5Tokenizer **ppOut
 ){
-  Fts5Tokenizer *pParent = 0;
   F5tTokenizerModule *pMod = (F5tTokenizerModule*)pCtx;
   Tcl_Obj *pEval;
   int rc = TCL_OK;
   int i;
 
-  assert( pMod->parent_v2.xCreate==0 || pMod->parent.xCreate==0 );
-  if( pMod->parent_v2.xCreate ){
-    rc = pMod->parent_v2.xCreate(pMod->pParentCtx, 0, 0, &pParent);
-  }
-  if( pMod->parent.xCreate ){
-    rc = pMod->parent.xCreate(pMod->pParentCtx, 0, 0, &pParent);
-  }
-
   pEval = Tcl_DuplicateObj(pMod->pScript);
   Tcl_IncrRefCount(pEval);
   for(i=0; rc==TCL_OK && i<nArg; i++){
@@ -878,8 +845,6 @@ static int f5tTokenizerCreate(
     pInst->interp = pMod->interp;
     pInst->pScript = Tcl_GetObjResult(pMod->interp);
     pInst->pContext = pMod->pContext;
-    pInst->pParent = pParent;
-    pInst->pModule = pMod;
     Tcl_IncrRefCount(pInst->pScript);
     *ppOut = (Fts5Tokenizer*)pInst;
   }
@@ -890,21 +855,11 @@ static int f5tTokenizerCreate(
 
 static void f5tTokenizerDelete(Fts5Tokenizer *p){
   F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
-  if( pInst ){
-    if( pInst->pParent ){
-      if( pInst->pModule->parent_v2.xDelete ){
-        pInst->pModule->parent_v2.xDelete(pInst->pParent);
-      }else{
-        pInst->pModule->parent.xDelete(pInst->pParent);
-      }
-    }
-    Tcl_DecrRefCount(pInst->pScript);
-    ckfree((char *)pInst);
-  }
+  Tcl_DecrRefCount(pInst->pScript);
+  ckfree((char *)pInst);
 }
 
-
-static int f5tTokenizerReallyTokenize(
+static int f5tTokenizerTokenize(
   Fts5Tokenizer *p, 
   void *pCtx,
   int flags,
@@ -912,7 +867,6 @@ static int f5tTokenizerReallyTokenize(
   int (*xToken)(void*, int, const char*, int, int, int)
 ){
   F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
-  F5tTokenizerInstance *pOldInst = 0;
   void *pOldCtx;
   int (*xOldToken)(void*, int, const char*, int, int, int);
   Tcl_Obj *pEval;
@@ -921,11 +875,9 @@ static int f5tTokenizerReallyTokenize(
 
   pOldCtx = pInst->pContext->pCtx;
   xOldToken = pInst->pContext->xToken;
-  pOldInst = pInst->pContext->pInst;
 
   pInst->pContext->pCtx = pCtx;
   pInst->pContext->xToken = xToken;
-  pInst->pContext->pInst = pInst;
 
   assert( 
       flags==FTS5_TOKENIZE_DOCUMENT
@@ -961,105 +913,9 @@ static int f5tTokenizerReallyTokenize(
 
   pInst->pContext->pCtx = pOldCtx;
   pInst->pContext->xToken = xOldToken;
-  pInst->pContext->pInst = pOldInst;
   return rc;
 }
 
-typedef struct CallbackCtx CallbackCtx;
-struct CallbackCtx {
-  Fts5Tokenizer *p;
-  void *pCtx;
-  int flags;
-  int (*xToken)(void*, int, const char*, int, int, int);
-};
-
-static int f5tTokenizeCallback(
-  void *pCtx, 
-  int tflags, 
-  const char *z, int n, 
-  int iStart, int iEnd
-){
-  CallbackCtx *p = (CallbackCtx*)pCtx;
-  return f5tTokenizerReallyTokenize(p->p, p->pCtx, p->flags, z, n, p->xToken);
-}
-
-static int f5tTokenizerTokenize_v2(
-  Fts5Tokenizer *p, 
-  void *pCtx,
-  int flags,
-  const char *pText, int nText, 
-  const char *pLoc, int nLoc, 
-  int (*xToken)(void*, int, const char*, int, int, int)
-){
-  int rc = SQLITE_OK;
-  F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
-
-  pInst->pLocale = pLoc;
-  pInst->nLocale = nLoc;
-
-  if( pInst->pParent ){
-    CallbackCtx ctx;
-    ctx.p = p;
-    ctx.pCtx = pCtx;
-    ctx.flags = flags;
-    ctx.xToken = xToken;
-    if( pInst->pModule->parent_v2.xTokenize ){
-      rc = pInst->pModule->parent_v2.xTokenize(
-          pInst->pParent, (void*)&ctx, flags, pText, nText, 
-          pLoc, nLoc, f5tTokenizeCallback
-      );
-    }else{
-      rc = pInst->pModule->parent.xTokenize(
-          pInst->pParent, (void*)&ctx, flags, pText, nText, f5tTokenizeCallback
-      );
-    }
-  }else{
-    rc = f5tTokenizerReallyTokenize(p, pCtx, flags, pText, nText, xToken);
-  }
-
-  pInst->pLocale = 0;
-  pInst->nLocale = 0;
-  return rc;
-}
-static int f5tTokenizerTokenize(
-  Fts5Tokenizer *p, 
-  void *pCtx,
-  int flags,
-  const char *pText, int nText, 
-  int (*xToken)(void*, int, const char*, int, int, int)
-){
-  return f5tTokenizerTokenize_v2(p, pCtx, flags, pText, nText, 0, 0, xToken);
-}
-
-/*
-** sqlite3_fts5_locale 
-*/
-static int SQLITE_TCLAPI f5tTokenizerLocale(
-  void * clientData,
-  Tcl_Interp *interp,
-  int objc,
-  Tcl_Obj *CONST objv[]
-){
-  F5tTokenizerContext *p = (F5tTokenizerContext*)clientData;
-
-  if( objc!=1 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "");
-    return TCL_ERROR;
-  }
-
-  if( p->xToken==0 ){
-    Tcl_AppendResult(interp, 
-        "sqlite3_fts5_locale may only be used by tokenizer callback", 0
-    );
-    return TCL_ERROR;
-  }
-
-  Tcl_SetObjResult(interp, 
-      Tcl_NewStringObj(p->pInst->pLocale, p->pInst->nLocale)
-  );
-  return TCL_OK;
-}
-
 /*
 ** sqlite3_fts5_token ?-colocated? TEXT START END
 */
@@ -1072,13 +928,13 @@ static int SQLITE_TCLAPI f5tTokenizerReturn(
   F5tTokenizerContext *p = (F5tTokenizerContext*)clientData;
   int iStart;
   int iEnd;
-  Tcl_Size nToken;
+  int nToken;
   int tflags = 0;
   char *zToken;
   int rc;
 
   if( objc==5 ){
-    Tcl_Size nArg;
+    int nArg;
     char *zArg = Tcl_GetStringFromObj(objv[1], &nArg);
     if( nArg<=10 && nArg>=2 && memcmp("-colocated", zArg, nArg)==0 ){
       tflags |= FTS5_TOKEN_COLOCATED;
@@ -1103,7 +959,7 @@ static int SQLITE_TCLAPI f5tTokenizerReturn(
     return TCL_ERROR;
   }
 
-  rc = p->xToken(p->pCtx, tflags, zToken, (int)nToken, iStart, iEnd);
+  rc = p->xToken(p->pCtx, tflags, zToken, nToken, iStart, iEnd);
   Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE);
   return rc==SQLITE_OK ? TCL_OK : TCL_ERROR;
 
@@ -1145,112 +1001,32 @@ static int SQLITE_TCLAPI f5tCreateTokenizer(
   fts5_api *pApi;
   char *zName;
   Tcl_Obj *pScript;
+  fts5_tokenizer t;
   F5tTokenizerModule *pMod;
-  int rc = SQLITE_OK;
-  int bV2 = 0;                    /* True to use _v2 API */
-  int iVersion = 2;               /* Value for _v2.iVersion */
-  const char *zParent = 0;        /* Name of parent tokenizer, if any */
-  int ii = 0;
+  int rc;
 
-  if( objc<4 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "?OPTIONS? DB NAME SCRIPT");
+  if( objc!=4 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "DB NAME SCRIPT");
     return TCL_ERROR;
   }
-
-  /* Parse any options. Set stack variables bV2 and zParent. */
-  for(ii=1; ii<objc-3; ii++){
-    int iOpt = 0;
-    const char *azOpt[] = { "-v2", "-parent", "-version", 0 };
-    if( Tcl_GetIndexFromObj(interp, objv[ii], azOpt, "OPTION", 0, &iOpt) ){
-      return TCL_ERROR;
-    }
-    switch( iOpt ){
-      case 0: /* -v2 */ {
-        bV2 = 1;
-        break;
-      }
-      case 1: /* -parent */ {
-        ii++;
-        if( ii==objc-3 ){
-          Tcl_AppendResult(
-              interp, "option requires an argument: -parent", (char*)0
-          );
-          return TCL_ERROR;
-        }
-        zParent = Tcl_GetString(objv[ii]);
-        break;
-      }
-      case 2: /* -version */ {
-        ii++;
-        if( ii==objc-3 ){
-          Tcl_AppendResult(
-              interp, "option requires an argument: -version", (char*)0
-          );
-          return TCL_ERROR;
-        }
-        if( Tcl_GetIntFromObj(interp, objv[ii], &iVersion) ){
-          return TCL_ERROR;
-        }
-        break;
-      }
-      default:
-        assert( 0 );
-        break;
-    }
-  }
-
-  if( f5tDbAndApi(interp, objv[objc-3], &db, &pApi) ){
+  if( f5tDbAndApi(interp, objv[1], &db, &pApi) ){
     return TCL_ERROR;
   }
-  zName = Tcl_GetString(objv[objc-2]);
-  pScript = objv[objc-1];
+  zName = Tcl_GetString(objv[2]);
+  pScript = objv[3];
+
+  t.xCreate = f5tTokenizerCreate;
+  t.xTokenize = f5tTokenizerTokenize;
+  t.xDelete = f5tTokenizerDelete;
 
   pMod = (F5tTokenizerModule*)ckalloc(sizeof(F5tTokenizerModule));
-  memset(pMod, 0, sizeof(F5tTokenizerModule));
   pMod->interp = interp;
   pMod->pScript = pScript;
-  Tcl_IncrRefCount(pScript);
   pMod->pContext = pContext;
-  if( zParent ){
-    if( bV2 ){
-      fts5_tokenizer_v2 *pParent = 0;
-      rc = pApi->xFindTokenizer_v2(pApi, zParent, &pMod->pParentCtx, &pParent);
-      if( rc==SQLITE_OK ){
-        memcpy(&pMod->parent_v2, pParent, sizeof(fts5_tokenizer_v2));
-        pMod->parent_v2.xDelete(0);
-      }
-    }else{
-      rc = pApi->xFindTokenizer(pApi, zParent, &pMod->pParentCtx,&pMod->parent);
-      if( rc==SQLITE_OK ){
-        pMod->parent.xDelete(0);
-      }
-    }
-  }
-
-  if( rc==SQLITE_OK ){
-    void *pModCtx = (void*)pMod;
-    if( bV2==0 ){
-      fts5_tokenizer t;
-      t.xCreate = f5tTokenizerCreate;
-      t.xTokenize = f5tTokenizerTokenize;
-      t.xDelete = f5tTokenizerDelete;
-      rc = pApi->xCreateTokenizer(pApi, zName, pModCtx, &t, f5tDelTokenizer);
-    }else{
-      fts5_tokenizer_v2 t2;
-      memset(&t2, 0, sizeof(t2));
-      t2.iVersion = iVersion;
-      t2.xCreate = f5tTokenizerCreate;
-      t2.xTokenize = f5tTokenizerTokenize_v2;
-      t2.xDelete = f5tTokenizerDelete;
-      rc = pApi->xCreateTokenizer_v2(pApi, zName, pModCtx, &t2,f5tDelTokenizer);
-    }
-  }
-
+  Tcl_IncrRefCount(pScript);
+  rc = pApi->xCreateTokenizer(pApi, zName, (void*)pMod, &t, f5tDelTokenizer);
   if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, (
-          bV2 ? "error in fts5_api.xCreateTokenizer_v2()"
-          : "error in fts5_api.xCreateTokenizer()"
-    ), 0);
+    Tcl_AppendResult(interp, "error in fts5_api.xCreateTokenizer()", 0);
     return TCL_ERROR;
   }
 
@@ -1307,7 +1083,7 @@ static int SQLITE_TCLAPI f5tTokenHash(
   Tcl_Obj *CONST objv[]
 ){
   char *z;
-  Tcl_Size n;
+  int n;
   unsigned int iVal;
   int nSlot;
 
@@ -1320,7 +1096,7 @@ static int SQLITE_TCLAPI f5tTokenHash(
   }
   z = Tcl_GetStringFromObj(objv[2], &n);
 
-  iVal = f5t_fts5HashKey(nSlot, z, (int)n);
+  iVal = f5t_fts5HashKey(nSlot, z, n);
   Tcl_SetObjResult(interp, Tcl_NewIntObj(iVal));
   return TCL_OK;
 }
@@ -1557,7 +1333,6 @@ int Fts5tcl_Init(Tcl_Interp *interp){
   } aCmd[] = {
     { "sqlite3_fts5_create_tokenizer",   f5tCreateTokenizer, 1 },
     { "sqlite3_fts5_token",              f5tTokenizerReturn, 1 },
-    { "sqlite3_fts5_locale",             f5tTokenizerLocale, 1 },
     { "sqlite3_fts5_tokenize",           f5tTokenize, 0 },
     { "sqlite3_fts5_create_function",    f5tCreateFunction, 0 },
     { "sqlite3_fts5_may_be_corrupt",     f5tMayBeCorrupt, 0 },

ext/fts5/fts5_tokenize.c

@@ -79,7 +79,7 @@ static int fts5AsciiCreate(
       int i;
       memset(p, 0, sizeof(AsciiTokenizer));
       memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar));
-      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
+      for(i=0; rc==SQLITE_OK && i<nArg-1; i+=2){
         const char *zArg = azArg[i+1];
         if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
           fts5AsciiAddExceptions(p, zArg, 1);
@@ -90,6 +90,7 @@ static int fts5AsciiCreate(
           rc = SQLITE_ERROR;
         }
       }
+      if( rc==SQLITE_OK && i<nArg ) rc = SQLITE_ERROR;
       if( rc!=SQLITE_OK ){
         fts5AsciiDelete((Fts5Tokenizer*)p);
         p = 0;
@@ -381,7 +382,7 @@ static int fts5UnicodeCreate(
       }
 
       /* Search for a "categories" argument */
-      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
+      for(i=0; rc==SQLITE_OK && i<nArg-1; i+=2){
         if( 0==sqlite3_stricmp(azArg[i], "categories") ){
           zCat = azArg[i+1];
         }
@@ -390,7 +391,7 @@ static int fts5UnicodeCreate(
         rc = unicodeSetCategories(p, zCat);
       }
 
-      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
+      for(i=0; rc==SQLITE_OK && i<nArg-1; i+=2){
         const char *zArg = azArg[i+1];
         if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
           if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
@@ -415,6 +416,8 @@ static int fts5UnicodeCreate(
           rc = SQLITE_ERROR;
         }
       }
+      if( i<nArg && rc==SQLITE_OK ) rc = SQLITE_ERROR;
+
     }else{
       rc = SQLITE_NOMEM;
     }
@@ -553,7 +556,7 @@ static int fts5UnicodeTokenize(
 
 typedef struct PorterTokenizer PorterTokenizer;
 struct PorterTokenizer {
-  fts5_tokenizer_v2 tokenizer_v2; /* Parent tokenizer module */
+  fts5_tokenizer tokenizer;       /* Parent tokenizer module */
   Fts5Tokenizer *pTokenizer;      /* Parent tokenizer instance */
   char aBuf[FTS5_PORTER_MAX_TOKEN + 64];
 };
@@ -565,7 +568,7 @@ static void fts5PorterDelete(Fts5Tokenizer *pTok){
   if( pTok ){
     PorterTokenizer *p = (PorterTokenizer*)pTok;
     if( p->pTokenizer ){
-      p->tokenizer_v2.xDelete(p->pTokenizer);
+      p->tokenizer.xDelete(p->pTokenizer);
     }
     sqlite3_free(p);
   }
@@ -584,7 +587,6 @@ static int fts5PorterCreate(
   PorterTokenizer *pRet;
   void *pUserdata = 0;
   const char *zBase = "unicode61";
-  fts5_tokenizer_v2 *pV2 = 0;
 
   if( nArg>0 ){
     zBase = azArg[0];
@@ -593,15 +595,14 @@ static int fts5PorterCreate(
   pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer));
   if( pRet ){
     memset(pRet, 0, sizeof(PorterTokenizer));
-    rc = pApi->xFindTokenizer_v2(pApi, zBase, &pUserdata, &pV2);
+    rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer);
   }else{
     rc = SQLITE_NOMEM;
   }
   if( rc==SQLITE_OK ){
     int nArg2 = (nArg>0 ? nArg-1 : 0);
-    const char **az2 = (nArg2 ? &azArg[1] : 0);
-    memcpy(&pRet->tokenizer_v2, pV2, sizeof(fts5_tokenizer_v2));
-    rc = pRet->tokenizer_v2.xCreate(pUserdata, az2, nArg2, &pRet->pTokenizer);
+    const char **azArg2 = (nArg2 ? &azArg[1] : 0);
+    rc = pRet->tokenizer.xCreate(pUserdata, azArg2, nArg2, &pRet->pTokenizer);
   }
 
   if( rc!=SQLITE_OK ){
@@ -1252,7 +1253,6 @@ static int fts5PorterTokenize(
   void *pCtx,
   int flags,
   const char *pText, int nText,
-  const char *pLoc, int nLoc,
   int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
 ){
   PorterTokenizer *p = (PorterTokenizer*)pTokenizer;
@@ -1260,8 +1260,8 @@ static int fts5PorterTokenize(
   sCtx.xToken = xToken;
   sCtx.pCtx = pCtx;
   sCtx.aBuf = p->aBuf;
-  return p->tokenizer_v2.xTokenize(
-      p->pTokenizer, (void*)&sCtx, flags, pText, nText, pLoc, nLoc, fts5PorterCb
+  return p->tokenizer.xTokenize(
+      p->pTokenizer, (void*)&sCtx, flags, pText, nText, fts5PorterCb
   );
 }
 
@@ -1291,46 +1291,41 @@ static int fts5TriCreate(
   Fts5Tokenizer **ppOut
 ){
   int rc = SQLITE_OK;
-  TrigramTokenizer *pNew = 0;
+  TrigramTokenizer *pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
   UNUSED_PARAM(pUnused);
-  if( nArg%2 ){
-    rc = SQLITE_ERROR;
+  if( pNew==0 ){
+    rc = SQLITE_NOMEM;
   }else{
     int i;
-    pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
-    if( pNew==0 ){
-      rc = SQLITE_NOMEM;
-    }else{
-      pNew->bFold = 1;
-      pNew->iFoldParam = 0;
-  
-      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
-        const char *zArg = azArg[i+1];
-        if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
-          if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
-            rc = SQLITE_ERROR;
-          }else{
-            pNew->bFold = (zArg[0]=='0');
-          }
-        }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
-          if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
-            rc = SQLITE_ERROR;
-          }else{
-            pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
-          }
-        }else{
+    pNew->bFold = 1;
+    pNew->iFoldParam = 0;
+    for(i=0; rc==SQLITE_OK && i<nArg-1; i+=2){
+      const char *zArg = azArg[i+1];
+      if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
+        if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
           rc = SQLITE_ERROR;
+        }else{
+          pNew->bFold = (zArg[0]=='0');
         }
-      }
-  
-      if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
+      }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
+        if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
+          rc = SQLITE_ERROR;
+        }else{
+          pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
+        }
+      }else{
         rc = SQLITE_ERROR;
       }
-  
-      if( rc!=SQLITE_OK ){
-        fts5TriDelete((Fts5Tokenizer*)pNew);
-        pNew = 0;
-      }
+    }
+    if( i<nArg && rc==SQLITE_OK ) rc = SQLITE_ERROR;
+
+    if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
+      rc = SQLITE_ERROR;
+    }
+
+    if( rc!=SQLITE_OK ){
+      fts5TriDelete((Fts5Tokenizer*)pNew);
+      pNew = 0;
     }
   }
   *ppOut = (Fts5Tokenizer*)pNew;
@@ -1435,16 +1430,6 @@ int sqlite3Fts5TokenizerPattern(
   return FTS5_PATTERN_NONE;
 }
 
-/*
-** Return true if the tokenizer described by p->azArg[] is the trigram
-** tokenizer. This tokenizer needs to be loaded before xBestIndex is
-** called for the first time in order to correctly handle LIKE/GLOB.
-*/
-int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig *p){
-  return (p->nArg>=1 && 0==sqlite3_stricmp(p->azArg[0], "trigram"));
-}
-
-
 /*
 ** Register all built-in tokenizers with FTS5.
 */
@@ -1455,6 +1440,7 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){
   } aBuiltin[] = {
     { "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}},
     { "ascii",     {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }},
+    { "porter",    {fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize }},
     { "trigram",   {fts5TriCreate, fts5TriDelete, fts5TriTokenize}},
   };
   
@@ -1469,19 +1455,6 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){
         0
     );
   }
-  if( rc==SQLITE_OK ){
-    fts5_tokenizer_v2 sPorter = {
-      2,
-      fts5PorterCreate,
-      fts5PorterDelete,
-      fts5PorterTokenize
-    };
-    rc = pApi->xCreateTokenizer_v2(pApi,
-        "porter",
-        (void*)pApi,
-        &sPorter,
-        0
-    );
-  }
+
   return rc;
 }

ext/fts5/fts5_unicode2.c

@@ -364,9 +364,6 @@ int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){
             default: return 1;          }
           break;
 
-
-    default:
-      return 1;
   }
   return 0;
 }

ext/fts5/fts5_vocab.c

@@ -64,7 +64,6 @@ struct Fts5VocabCursor {
 
   int nLeTerm;                    /* Size of zLeTerm in bytes */
   char *zLeTerm;                  /* (term <= $zLeTerm) paramater, or NULL */
-  int colUsed;                    /* Copy of sqlite3_index_info.colUsed */
 
   /* These are used by 'col' tables only */
   int iCol;
@@ -91,11 +90,9 @@ struct Fts5VocabCursor {
 /*
 ** Bits for the mask used as the idxNum value by xBestIndex/xFilter.
 */
-#define FTS5_VOCAB_TERM_EQ 0x0100
-#define FTS5_VOCAB_TERM_GE 0x0200
-#define FTS5_VOCAB_TERM_LE 0x0400
-
-#define FTS5_VOCAB_COLUSED_MASK 0xFF
+#define FTS5_VOCAB_TERM_EQ 0x01
+#define FTS5_VOCAB_TERM_GE 0x02
+#define FTS5_VOCAB_TERM_LE 0x04
 
 
 /*
@@ -272,13 +269,11 @@ static int fts5VocabBestIndexMethod(
   int iTermEq = -1;
   int iTermGe = -1;
   int iTermLe = -1;
-  int idxNum = (int)pInfo->colUsed;
+  int idxNum = 0;
   int nArg = 0;
 
   UNUSED_PARAM(pUnused);
 
-  assert( (pInfo->colUsed & FTS5_VOCAB_COLUSED_MASK)==pInfo->colUsed );
-
   for(i=0; i<pInfo->nConstraint; i++){
     struct sqlite3_index_constraint *p = &pInfo->aConstraint[i];
     if( p->usable==0 ) continue;
@@ -370,7 +365,7 @@ static int fts5VocabOpenMethod(
       if( rc==SQLITE_OK ){
         pVTab->zErrMsg = sqlite3_mprintf(
             "no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl
-        );
+            );
         rc = SQLITE_ERROR;
       }
     }else{
@@ -530,19 +525,9 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){
 
         switch( pTab->eType ){
           case FTS5_VOCAB_ROW:
-            /* Do not bother counting the number of instances if the "cnt"
-            ** column is not being read (according to colUsed).  */
-            if( eDetail==FTS5_DETAIL_FULL && (pCsr->colUsed & 0x04) ){
-              while( iPos<nPos ){
-                u32 ii;
-                fts5FastGetVarint32(pPos, iPos, ii);
-                if( ii==1 ){ 
-                  /* New column in the position list */
-                  fts5FastGetVarint32(pPos, iPos, ii);
-                }else{
-                  /* An instance - increment pCsr->aCnt[] */
-                  pCsr->aCnt[0]++;
-                }
+            if( eDetail==FTS5_DETAIL_FULL ){
+              while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){
+                pCsr->aCnt[0]++;
               }
             }
             pCsr->aDoc[0]++;
@@ -640,7 +625,6 @@ static int fts5VocabFilterMethod(
   if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++];
   if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++];
   if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++];
-  pCsr->colUsed = (idxNum & FTS5_VOCAB_COLUSED_MASK);
 
   if( pEq ){
     zTerm = (const char *)sqlite3_value_text(pEq);

ext/fts5/test/fts5_common.tcl

@@ -51,10 +51,6 @@ proc fts5_test_poslist2 {cmd} {
   sort_poslist $res
 }
 
-proc fts5_test_insttoken {cmd iInst iToken} {
-  $cmd xInstToken $iInst $iToken
-}
-
 proc fts5_test_collist {cmd} {
   set res [list]
 
@@ -82,9 +78,6 @@ proc fts5_test_columnsize {cmd} {
 proc fts5_columntext {cmd iCol} {
   $cmd xColumnText $iCol
 }
-proc fts5_columnlocale {cmd iCol} {
-  $cmd xColumnLocale $iCol
-}
 
 proc fts5_test_columntext {cmd} {
   set res [list]
@@ -94,14 +87,6 @@ proc fts5_test_columntext {cmd} {
   set res
 }
 
-proc fts5_test_columnlocale {cmd} {
-  set res [list]
-  for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
-    lappend res [$cmd xColumnLocale $i]
-  }
-  set res
-}
-
 proc fts5_test_columntotalsize {cmd} {
   set res [list]
   for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
@@ -129,10 +114,6 @@ proc fts5_test_rowcount {cmd} {
   $cmd xRowCount
 }
 
-proc fts5_test_rowid {cmd} {
-  $cmd xRowid
-}
-
 proc test_queryphrase_cb {cnt cmd} {
   upvar $cnt L 
   for {set i 0} {$i < [$cmd xInstCount]} {incr i} {
@@ -180,21 +161,17 @@ proc fts5_aux_test_functions {db} {
   foreach f {
     fts5_test_columnsize
     fts5_test_columntext
-    fts5_test_columnlocale
     fts5_test_columntotalsize
     fts5_test_poslist
     fts5_test_poslist2
     fts5_test_collist
-    fts5_test_insttoken
     fts5_test_tokenize
     fts5_test_rowcount
-    fts5_test_rowid
     fts5_test_all
 
     fts5_test_queryphrase
     fts5_test_phrasecount
     fts5_columntext
-    fts5_columnlocale
     fts5_queryphrase
     fts5_collist
   } {

ext/fts5/test/fts5aa.test

@@ -440,7 +440,7 @@ do_execsql_test 16.1 {
 proc funk {} {
   db eval { UPDATE n1_config SET v=50 WHERE k='version' }
   set fd [db incrblob main n1_data block 10]
-  fconfigure $fd -translation binary
+  fconfigure $fd -encoding binary -translation binary
 #  puts -nonewline $fd "\x44\x45"
   close $fd
 }
@@ -453,7 +453,7 @@ db func funk funk
 # statement no longer fails.
 #
 do_catchsql_test 16.2 {
-  SELECT funk(), format('%g',bm25(n1)), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
+  SELECT funk(), bm25(n1), funk() FROM n1 WHERE n1 MATCH 'a+b+c+d'
 } {0 {{} -1e-06 {}}}
 # {1 {SQL logic error}}
 

ext/fts5/test/fts5ab.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ab
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5ac.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ac
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5ad.test

@@ -18,7 +18,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ad
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5ae.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ae
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5af.test

@@ -18,7 +18,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5af
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5ag.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ag
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5ah.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ah
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -163,17 +163,6 @@ do_execsql_test 1.8.2 {
   SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid < 'text';
 } {10000}
 
-do_execsql_test 1.8.3 {
-  SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid<5000 AND rowid < 'text';
-} {4999}
-do_execsql_test 1.8.4 {
-  SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid>5000 AND rowid > 'text';
-} {0}
-
-do_catchsql_test 1.9 {
-  SELECT * FROM t1('*xy');
-} {1 {unknown special query: xy}}
-
 } ;# foreach_detail_mode
 
 #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}

ext/fts5/test/fts5ai.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ai
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5aj.test

@@ -19,7 +19,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5aj
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5ak.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ak
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5al.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5al
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -293,16 +293,6 @@ do_catchsql_test 4.4.4 {
   SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL
 } {1 {parse error in rank function: }}
 
-# Check that the second and subsequent rank= constraints are ignored.
-#
-do_catchsql_test 4.3.3 {
-  SELECT *, rank FROM t3
-  WHERE t3 MATCH 'a' AND 
-  rank MATCH 'nosuch()' AND
-  rank MATCH 'rowidmod(3)'  
-  ORDER BY rank ASC
-} {1 {unable to use function MATCH in the requested context}}
-
 } ;# foreach_detail_mode
 
 

ext/fts5/test/fts5alter.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5alter
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5auto.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5auto
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5aux.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5aux
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -399,6 +399,6 @@ db close
 sqlite3 db test.db
 do_catchsql_test 13.4 {
   SELECT highlight(t1, 0, '[', ']') FROM t1
-} {1 {SQL logic error}}
+} {1 {no such tokenizer: blah}}
 
 finish_test

ext/fts5/test/fts5aux2.test

@@ -1,71 +0,0 @@
-# 2024 June 24
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-#
-# Tests focusing on the auxiliary function APIs.
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5aux
-
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
-  INSERT INTO x1 VALUES('a b', 'c d');
-  INSERT INTO x1 VALUES('d e', 'a b');
-  INSERT INTO x1 VALUES('a b', 'e f');
-  INSERT INTO x1 VALUES('d e', 'c d');
-}
-
-fts5_aux_test_functions db
-do_execsql_test 1.1 {
-  SELECT fts5_test_all(x1) FROM x1 WHERE rowid=2
-} [list [list {*}{
-  columnsize {2 2}
-  columntext {{d e} {a b}}
-  columntotalsize {8 8}
-  poslist {}
-  tokenize {{d e} {a b}}
-  rowcount 4
-}]]
-
-do_execsql_test 1.2 {
-  SELECT fts5_test_columntext(x1) FROM x1
-} {
-  {{a b} {c d}}
-  {{d e} {a b}}
-  {{a b} {e f}}
-  {{d e} {c d}}
-}
-
-do_execsql_test 1.3 {
-  SELECT fts5_test_rowid(x1) FROM x1
-} {
-  1 2 3 4
-}
-do_execsql_test 1.4 {
-  SELECT fts5_test_phrasecount(x1) FROM x1
-} {
-  0 0 0 0
-}
-do_catchsql_test 1.5 {
-  SELECT fts5_queryphrase(x1, 0) FROM x1
-} {1 SQLITE_RANGE} 
-do_execsql_test 1.6 {
-  SELECT fts5_test_rowcount(x1) FROM x1
-} {4 4 4 4}
-
-
-finish_test

ext/fts5/test/fts5auxdata.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5auxdata
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5bigpl.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5bigpl
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5blob.test

@@ -1,166 +0,0 @@
-# 2024 July 30
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-#
-# This file verifies that:
-#
-#   * blob values may be written to locale=0 tables.
-#
-#   * blob values - other than fts5_locale() values - may not be written 
-#     to locale=0 tables. This is an SQLITE_MISMATCH error
-#
-#   * blob values may be returned by queries on the external-content table
-#     of a locale=0 table.
-#
-#   * blob values not may be returned by queries on the external-content 
-#     table of a locale=1 table, apart from fts5_locale() blobs. This is an
-#     SQLITE_MISMATCH error.
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5blob
-
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-
-# Test that blobs may be stored in normal locale=0 tables.
-#
-foreach {tn enc} {
-  1 utf8  
-  2 utf16 
-} {
-  reset_db
-  fts5_aux_test_functions db
-
-  execsql "PRAGMA encoding = $enc"
-
-  execsql "
-    CREATE VIRTUAL TABLE t1 USING fts5(x, y);
-  "
-  do_execsql_test 1.$tn.0 {
-    CREATE VIRTUAL TABLE tt USING fts5vocab('t1', 'instance');
-    INSERT INTO t1(rowid, x, y) VALUES(1, 555, X'0000000041424320444546');
-    INSERT INTO t1(rowid, x, y) VALUES(2, 666, X'41424300444546');
-    INSERT INTO t1(rowid, x, y) VALUES(3, 777, 'xyz');
-  }
-
-  do_execsql_test 1.$tn.1 {
-    SELECT rowid, quote(x), quote(y) FROM t1
-  } {
-    1 555 X'0000000041424320444546'
-    2 666 X'41424300444546'
-    3 777 'xyz'
-  }
-
-  do_execsql_test 1.$tn.2 {
-    DELETE FROM t1 WHERE rowid=2;
-    DELETE FROM t1 WHERE rowid=1;
-  }
-
-  do_execsql_test 1.$tn.3 {
-    PRAGMA integrity_check;
-  } {ok}
-}
-
-#--------------------------------------------------------------------------
-# Test that a blob may be stored and retrieved in an unindexed column of
-# a regular table with locale=1.
-#
-reset_db
-do_execsql_test 2.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(x, y UNINDEXED, locale=1);
-  INSERT INTO t1(rowid, x, y) VALUES(12, 'twelve', X'0000000041424320444546');
-}
-
-do_execsql_test 2.1 {
-  select rowid, x, quote(y) FROM t1
-} {
-  12 twelve X'0000000041424320444546'
-}
-
-#--------------------------------------------------------------------------
-# Test that blobs may not be written to any type of table with locale=1
-# set. Except, they may be written to UNINDEXED columns.
-#
-reset_db
-do_execsql_test 3.0 {
-  CREATE TABLE t1(a, b);
-
-  CREATE VIRTUAL TABLE x1 USING fts5(a, b, locale=1);
-  CREATE VIRTUAL TABLE x2 USING fts5(a, b, locale=1, content=t2);
-  CREATE VIRTUAL TABLE x3 USING fts5(a, b, locale=1, content=);
-}
-
-do_catchsql_test 3.1 {
-  INSERT INTO x1(rowid, a, b) VALUES(113, 'hello world', X'123456');
-} {1 {datatype mismatch}}
-do_catchsql_test 3.2 {
-  INSERT INTO x2(rowid, a, b) VALUES(113, 'hello world', X'123456');
-} {1 {datatype mismatch}}
-do_catchsql_test 3.3 {
-  INSERT INTO x3(rowid, a, b) VALUES(113, 'hello world', X'123456');
-} {1 {datatype mismatch}}
-
-
-#--------------------------------------------------------------------------
-# Test that fts5_locale() values may not be written to any type of table
-# without locale=1 set. Even to an UNINDEXED column.
-#
-reset_db
-do_execsql_test 3.0 {
-  CREATE TABLE t1(a, b);
-
-  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
-  CREATE VIRTUAL TABLE x2 USING fts5(a, b, content=t2);
-  CREATE VIRTUAL TABLE x3 USING fts5(a, b, content=);
-
-  CREATE VIRTUAL TABLE x4 USING fts5(a, b, c UNINDEXED);
-}
-
-do_catchsql_test 3.1 {
-  INSERT INTO x1(rowid, a, b) 
-    VALUES(113, 'hello world', fts5_locale('en_AU', 'abc'));
-} {1 {fts5_locale() requires locale=1}}
-do_catchsql_test 3.2 {
-  INSERT INTO x2(rowid, a, b) 
-    VALUES(113, 'hello world', fts5_locale('en_AU', 'abc'));
-} {1 {fts5_locale() requires locale=1}}
-do_catchsql_test 3.3 {
-  INSERT INTO x3(rowid, a, b) 
-    VALUES(113, 'hello world', fts5_locale('en_AU', 'abc'));
-} {1 {fts5_locale() requires locale=1}}
-do_catchsql_test 3.4 {
-  INSERT INTO x4(rowid, a, b, c) 
-    VALUES(113, 'hello world', 'yesno', fts5_locale('en_AU', 'abc'));
-} {1 {fts5_locale() requires locale=1}}
-
-
-#-------------------------------------------------------------------------
-#
-reset_db
-do_execsql_test 4.0 {
-  CREATE VIRTUAL TABLE x1 USING fts5(x);
-}
-
-foreach {tn sql} {
-  1 { INSERT INTO x1(rowid, x) VALUES(4.5, 'abcd') }
-  2 { INSERT INTO x1(rowid, x) VALUES('xyz', 'abcd') }
-  3 { INSERT INTO x1(rowid, x) VALUES(X'001122', 'abcd') }
-} {
-  do_catchsql_test 4.1.$tn $sql {1 {datatype mismatch}}
-}
-
-
-finish_test
-
-

ext/fts5/test/fts5cat.test

@@ -55,22 +55,5 @@ do_execsql_test 1.5 {
   SELECT * FROM t4t
 } {สนามกีฬา 1 1}
 
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 2.0 "
-  CREATE VIRTUAL TABLE x1 USING fts5(c, 
-      tokenize=\"unicode61 categories ' \t'\");
-"
-
-do_catchsql_test 2.1 "
-  CREATE VIRTUAL TABLE x2 USING fts5(c, 
-      tokenize=\"unicode61 categories 'N*\t\tMYZ'\");
-" {1 {error in tokenizer constructor}}
-
-do_catchsql_test 2.2 "
-  CREATE VIRTUAL TABLE x2 USING fts5(c, 
-      tokenize=\"unicode61 categories 'N*\t\tXYZ'\");
-" {1 {error in tokenizer constructor}}
-
 
 finish_test

ext/fts5/test/fts5colset.test

@@ -79,7 +79,7 @@ foreach_detail_mode $::testprefix {
 
   do_catchsql_test 4.1 {
     SELECT * FROM t1 WHERE rowid MATCH 'a'
-  } {1 {no query solution}}
+  } {1 {unable to use function MATCH in the requested context}}
 }
 
 #-------------------------------------------------------------------------

ext/fts5/test/fts5columnsize.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5columnsize
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5config.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5config
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5content.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5content
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -328,41 +328,4 @@ do_execsql_test 8.3.2 { INSERT INTO t1(t1) VALUES('rebuild') }
 do_execsql_test 8.3.3 { SELECT * FROM t1 WHERE rowid=1 } {one}
 do_execsql_test 8.3.4 { SELECT rowid FROM t1('two') } {2}
 
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 9.1 {
-  CREATE TABLE t1(a INTEGER PRIMARY KEY, b);
-  INSERT INTO t1 VALUES(1, 'one two three');
-  INSERT INTO t1 VALUES(2, 'one two three');
-
-  CREATE VIRTUAL TABLE ft USING fts5(b, content=t1, content_rowid=a);
-  INSERT INTO ft(ft) VALUES('rebuild');
-}
-
-do_execsql_test 9.2 {
-  SELECT rowid, b FROM ft('two');
-} {
-  1 {one two three}
-  2 {one two three}
-}
-
-do_execsql_test 9.3 {
-  DELETE FROM t1 WHERE a=2;
-} 
-
-do_catchsql_test 9.4 {
-  SELECT rowid FROM ft('two');
-} {0 {1 2}}
-
-do_catchsql_test 9.5 {
-  SELECT * FROM ft('two');
-} {1 {fts5: missing row 2 from content table 'main'.'t1'}}
-
-fts5_aux_test_functions db
-
-do_catchsql_test 9.6 {
-  SELECT rowid, fts5_columntext(ft, 0) FROM ft('two');
-} {1 SQLITE_CORRUPT_VTAB}
-
 finish_test
-

ext/fts5/test/fts5contentless.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5contentless
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -267,24 +267,4 @@ do_execsql_test 8.2 {
   SELECT rowid FROM ft('four');
 } {}
 
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 9.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=0);
-  INSERT INTO ft VALUES('hello world');
-  INSERT INTO ft VALUES('one two three');
-}
-
-do_catchsql_test 9.1 {
-  INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, 'hello world');
-} {0 {}}
-
-do_catchsql_test 9.2 {
-  CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', contentless_delete=2);
-} {1 {malformed contentless_delete=... directive}}
-
-do_catchsql_test 9.3 {
-  CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', contentless_delete=11);
-} {1 {malformed contentless_delete=... directive}}
-
 finish_test

ext/fts5/test/fts5contentless2.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5contentless2
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5contentless3.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5contentless3
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5contentless4.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5contentless4
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5contentless5.test

@@ -15,12 +15,11 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5contentless5
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
 }
-unset -nocomplain res
 
 do_execsql_test 1.0 {
   CREATE VIRTUAL TABLE t1 USING fts5(a, b, c, content='', contentless_delete=1);

ext/fts5/test/fts5corrupt.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -99,27 +99,6 @@ sqlite3_db_config db DEFENSIVE 0
 do_catchsql_test 3.1 {
   DELETE FROM t3_content WHERE rowid = 3;
   SELECT * FROM t3 WHERE t3 MATCH 'o';
-} {1 {fts5: missing row 3 from content table 'main'.'t3_content'}}
-
-#--------------------------------------------------------------------
-#
-reset_db
-do_execsql_test 4.0 {
-  CREATE VIRTUAL TABLE t2 USING fts5(x);
-  INSERT INTO t2 VALUES('one two three');
-  INSERT INTO t2 VALUES('four five six');
-  INSERT INTO t2 VALUES('seven eight nine');
-  INSERT INTO t2 VALUES('ten eleven twelve');
-}
-do_execsql_test 4.1 {
-  SELECT hex(block) FROM t2_data WHERE id=1;
-} {040C}
-do_execsql_test 4.2 {
-  UPDATE t2_data SET block = X'0402' WHERE id=1
-}
-breakpoint
-do_catchsql_test 4.3 {
-  DELETE FROM t2 WHERE rowid=3
 } {1 {database disk image is malformed}}
 
 finish_test

ext/fts5/test/fts5corrupt2.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt2
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -100,7 +100,6 @@ set lrowid [db one {SELECT max(rowid) FROM t1_data WHERE (rowid & $mask)=0}]
 set nbyte [db one {SELECT length(block) FROM t1_data WHERE rowid=$lrowid}]
 set all [db eval {SELECT rowid FROM t1}]
 sqlite3_db_config db DEFENSIVE 0
-unset -nocomplain res
 for {set i [expr $nbyte-2]} {$i>=0} {incr i -1} {
   do_execsql_test 2.$i.1 {
     BEGIN;
@@ -153,7 +152,7 @@ foreach {tn hdr} {
       execsql BEGIN
 
       set fd [db incrblob main x3_data block $rowid]
-      fconfigure $fd -translation binary
+      fconfigure $fd -encoding binary -translation binary
       set existing [read $fd [string length $hdr]]
       seek $fd 0
       puts -nonewline $fd $hdr
@@ -239,7 +238,7 @@ foreach {tn hdr} {
       execsql BEGIN
 
       set fd [db incrblob main x5_data block $rowid]
-      fconfigure $fd -translation binary
+      fconfigure $fd -encoding binary -translation binary
       puts -nonewline $fd $hdr
       close $fd
 

ext/fts5/test/fts5corrupt3.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt3
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -4267,7 +4267,7 @@ do_test 35.0 {
 
 do_catchsql_test 35.1 {
   SELECT * FROM t1 WHERE t1 MATCH 'e*';
-} {1 {fts5: missing row 14 from content table 'main'.'t1_content'}}
+} {1 {database disk image is malformed}}
 
 #-------------------------------------------------------------------------
 reset_db
@@ -8958,6 +8958,7 @@ do_catchsql_test 61.2 {
   SELECT * FROM t3 ORDER BY rowid;
 } {/*malformed database schema*/}
 
+breakpoint
 #-------------------------------------------------------------------------
 do_test 62.0 {
   sqlite3 db {}
@@ -10767,7 +10768,6 @@ do_catchsql_test 73.1 {
 reset_db
 do_test 74.0 {
   sqlite3 db {}
-  sqlite3_fts5_register_matchinfo db
   db deserialize [decode_hexdb {
 | size 106496 pagesize 4096 filename x.db
 | page 1 offset 0
@@ -14587,19 +14587,14 @@ do_test 74.0 {
 | end x.db
 }]} {}
 
-do_catchsql_test 74.0.5 {
-  SELECT matchinfo(1,2);
-} {1 {unable to use function matchinfo in the requested context}}
-
 do_catchsql_test 74.1 {
-  SELECT rowid, quote(matchinfo(t1,'pxyb<s')) FROM t1 WHERE t1 MATCH 'e*';
-} {1 {unrecognized matchinfo flag: <}}
+  SELECT rowid, quote(matchinfo(t1,'p�xyb<s')) FROM t1 WHERE t1 MATCH 'e*';
+} {1 {unable to use function matchinfo in the requested context}}
 
 #-------------------------------------------------------------------------
 reset_db
 do_test 75.0 {
   sqlite3 db {}
-  sqlite3_fts5_register_matchinfo db
   db deserialize [decode_hexdb {
 | size 32768 pagesize 4096 filename crash-033d665d5caa8d.db
 | page 1 offset 0
@@ -14796,7 +14791,7 @@ do_test 75.0 {
 
 do_catchsql_test 75.1 {
   SELECT rowid, quote(matchinfo(t1,'pcxybs')) FROM t1 WHERE t1 MATCH 'e*';
-} {1 {database disk image is malformed}}
+} {1 {unable to use function matchinfo in the requested context}}
 
 #-------------------------------------------------------------------------
 reset_db
@@ -15524,160 +15519,6 @@ do_catchsql_test 80.1 {
 SELECT snippet(rowid, -1, '.', '..', '[', '(]'),snippet(rowid, -1, '.', '.', '', '(]'), highlight(t1, 29, 1 , '') FROM t1('g+ h') WHERE rank MATCH 'bm25(1.0, 10)' ORDER BY NOT (SELECT 1 FROM t1('g+ æ') WHERE rank MATCH 'bm25(1.0, 10)' ORDER BY rank);
 } {1 {database disk image is malformed}}
 
-#-------------------------------------------------------------------------
-reset_db
-do_test 81.0 {
-  sqlite3 db {}
-  db deserialize [decode_hexdb {
-.open --hexdb
-| size 40960 pagesize 4096 filename crash-44e8035a976422.db
-| page 1 offset 0
-|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
-|     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a   .....@  ........
-|     32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04   ................
-|     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
-|     96: 00 00 00 00 0d 00 00 00 0d 0b 6e 00 0f a3 0f 4c   ..........n....L
-|    112: 0e e1 0e 81 0e 24 0d cc 0d 72 0d 1b 0c b0 0c 50   .....$...r.....P
-|    128: 0b f8 0b b3 0b 6e 00 00 00 00 00 00 00 00 00 00   .....n..........
-|   2912: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 43 0d   ..............C.
-|   2928: 06 17 11 11 08 75 74 61 62 6c 65 74 34 74 34 43   .....utablet4t4C
-|   2944: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41   REATE VIRTUAL TA
-|   2960: 42 4c 45 20 74 34 20 55 53 49 4e 47 20 66 74 73   BLE t4 USING fts
-|   2976: 35 76 6f 63 61 62 28 27 74 32 27 2c 20 27 72 6f   5vocab('t2', 'ro
-|   2992: 77 27 29 43 0c 06 17 11 11 08 75 74 61 62 6c 65   w')C......utable
-|   3008: 74 33 74 33 43 52 45 41 54 45 20 56 49 52 54 55   t3t3CREATE VIRTU
-|   3024: 41 4c 20 54 41 42 4c 45 20 74 33 20 55 53 49 4e   AL TABLE t3 USIN
-|   3040: 47 20 66 74 73 35 76 6f 63 61 62 28 27 74 31 27   G fts5vocab('t1'
-|   3056: 2c 20 27 72 6f 77 27 29 56 0b 06 17 1f 1f 01 7d   , 'row')V.......
-|   3072: 74 61 62 6c 65 74 32 5f 63 6f 6e 66 69 67 74 32   tablet2_configt2
-|   3088: 5f 63 6f 6e 66 69 67 0a 43 52 45 41 54 45 20 54   _config.CREATE T
-|   3104: 41 42 4c 45 20 27 74 32 5f 63 6f 6e 66 69 67 27   ABLE 't2_config'
-|   3120: 28 6b 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20   (k PRIMARY KEY, 
-|   3136: 76 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44   v) WITHOUT ROWID
-|   3152: 5e 0a 07 17 21 21 01 81 07 74 61 62 6c 65 74 32   ^...!!...tablet2
-|   3168: 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 6f 6e 74 65   _contentt2_conte
-|   3184: 6e 74 09 43 52 45 41 54 45 20 54 41 42 4c 45 20   nt.CREATE TABLE 
-|   3200: 27 74 32 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20   't2_content'(id 
-|   3216: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20   INTEGER PRIMARY 
-|   3232: 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 29   KEY, c0, c1, c2)
-|   3248: 69 09 07 17 19 19 01 81 2d 74 61 62 6c 65 74 32   i.......-tablet2
-|   3264: 5f 69 64 78 74 32 5f 69 64 78 08 43 52 45 41 54   _idxt2_idx.CREAT
-|   3280: 45 20 54 41 42 4c 45 20 27 74 32 5f 69 64 78 27   E TABLE 't2_idx'
-|   3296: 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67   (segid, term, pg
-|   3312: 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28   no, PRIMARY KEY(
-|   3328: 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49   segid, term)) WI
-|   3344: 54 48 4f 55 54 20 52 4f 57 49 44 55 08 07 17 1b   THOUT ROWIDU....
-|   3360: 1b 01 81 01 74 61 62 6c 65 74 32 5f 64 61 74 61   ....tablet2_data
-|   3376: 74 32 5f 64 61 74 61 07 43 52 45 41 54 45 20 54   t2_data.CREATE T
-|   3392: 41 42 4c 45 20 27 74 32 5f 64 61 74 61 27 28 69   ABLE 't2_data'(i
-|   3408: 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52   d INTEGER PRIMAR
-|   3424: 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c 4f   Y KEY, block BLO
-|   3440: 42 29 58 07 07 17 11 11 08 81 1d 74 61 62 6c 65   B)X........table
-|   3456: 74 32 74 32 43 52 45 41 54 45 20 56 49 52 54 55   t2t2CREATE VIRTU
-|   3472: 41 4c 20 54 41 42 4c 45 20 74 32 20 55 53 49 4e   AL TABLE t2 USIN
-|   3488: 47 20 66 74 73 35 28 27 61 27 2c 5b 62 5d 2c 22   G fts5('a',[b],.
-|   3504: 63 22 2c 64 65 74 61 69 6c 3d 6e 6f 6e 65 2c 63   c.,detail=none,c
-|   3520: 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 56 06 06 17   olumnsize=0)V...
-|   3536: 1f 1f 01 7d 74 61 62 6c 65 74 31 5f 63 6f 6e 66   ....tablet1_conf
-|   3552: 69 67 74 31 5f 63 6f 6e 66 69 67 06 43 52 45 41   igt1_config.CREA
-|   3568: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f 6e   TE TABLE 't1_con
-|   3584: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b   fig'(k PRIMARY K
-|   3600: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52   EY, v) WITHOUT R
-|   3616: 4f 57 49 44 5b 05 07 17 21 21 01 81 01 74 61 62   OWID[...!!...tab
-|   3632: 6c 65 74 31 5f 64 6f 63 73 69 7a 65 74 31 5f 64   let1_docsizet1_d
-|   3648: 6f 63 73 69 7a 65 05 43 52 45 41 54 45 20 54 41   ocsize.CREATE TA
-|   3664: 42 4c 45 20 27 74 31 5f 64 6f 63 73 69 7a 65 27   BLE 't1_docsize'
-|   3680: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d   (id INTEGER PRIM
-|   3696: 41 52 59 20 4b 45 59 2c 20 73 7a 20 42 4c 4f 42   ARY KEY, sz BLOB
-|   3712: 29 5e 04 07 17 21 21 01 81 07 74 61 62 6c 65 74   )^...!!...tablet
-|   3728: 31 5f 63 6f 6e 74 65 6e 74 74 31 5f 63 6f 6e 74   1_contentt1_cont
-|   3744: 65 6e 74 04 43 52 45 41 54 45 20 54 41 42 4c 45   ent.CREATE TABLE
-|   3760: 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 69 64    't1_content'(id
-|   3776: 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59    INTEGER PRIMARY
-|   3792: 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32    KEY, c0, c1, c2
-|   3808: 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 74   )i.......-tablet
-|   3824: 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 41   1_idxt1_idx.CREA
-|   3840: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 78   TE TABLE 't1_idx
-|   3856: 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70   '(segid, term, p
-|   3872: 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59   gno, PRIMARY KEY
-|   3888: 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57   (segid, term)) W
-|   3904: 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 17   ITHOUT ROWIDU...
-|   3920: 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 74   .....tablet1_dat
-|   3936: 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 20   at1_data.CREATE 
-|   3952: 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 28   TABLE 't1_data'(
-|   3968: 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41   id INTEGER PRIMA
-|   3984: 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c   RY KEY, block BL
-|   4000: 4f 42 29 5b 01 07 17 11 11 08 81 23 74 61 62 6c   OB)[.......#tabl
-|   4016: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54   et1t1CREATE VIRT
-|   4032: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49   UAL TABLE t1 USI
-|   4048: 4e 47 20 66 74 73 35 28 61 2c 62 20 75 6e 69 6e   NG fts5(a,b unin
-|   4064: 64 65 78 65 64 2c 63 2c 74 6f 6b 65 6e 69 7a 65   dexed,c,tokenize
-|   4080: 3d 22 70 6f 72 74 65 72 20 61 73 63 69 69 22 29   =.porter ascii.)
-| page 2 offset 4096
-|      0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c   ..h............|
-|     16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00   .*..............
-|   3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00   .......0........
-|   3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80   ..........7.....
-|   3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01   ...t.....0a.....
-|   3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01   b.....c.....g...
-|   3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01   ....h.......i...
-|   3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00   ............*...
-|   3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80   ............%...
-|   3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02   .....P.....0g...
-|   3984: 01 02 02 01 01 68 02 08 03 8d 02 03 01 01 6a 42   .....h........jB
-|   4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 80 01 03   ........7.......
-|   4016: 00 74 00 00 00 2e 02 30 61 01 12 02 01 01 62 01   .t.....0a.....b.
-|   4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02   ....c.....g.....
-|   4048: 01 01 68 01 05 01 02 03 01 01 69 01 06 01 02 04   ..h.......i.....
-|   4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00   ................
-|   4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01   ...$............
-| page 3 offset 8192
-|      0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00   ................
-|   4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c   ................
-|   4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02   ................
-| page 4 offset 12288
-|      0: 0d 00 00 00 03 0f be 00 0f ea 00 00 00 00 00 00   ................
-|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03   ................
-|   4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67   .....a b cg h ig
-|   4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69    h i.......g h i
-|   4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17   a b cg h i......
-|   4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69   .a b cd e fg h i
-| page 5 offset 16384
-|      0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00   ................
-|   4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03   ................
-|   4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03   ................
-| page 6 offset 20480
-|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
-|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
-| page 7 offset 24576
-|      0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00   ................
-|   3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84   ..............A.
-|   4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61   ............4.0a
-|   4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01   .....b.....c....
-|   4032: e6 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01   .d...e...f...g..
-|   4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06   ...h.....i......
-|   4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f   ................
-|   4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01   ...$............
-| page 8 offset 28672
-|      0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00   ................
-|   4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02   ................
-| page 9 offset 32768
-|      0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00   ................
-|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03   ................
-|   4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67   .....a b cg h ig
-|   4048: 20 68 21 69 14 02 05 00 17 17 17 67 20 68 20 69    h!i.......g h i
-|   4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17   a b cg h i......
-|   4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69   .a b cd e fg h i
-| page 10 offset 36864
-|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
-|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
-| end crash-44e8035a976422.db
-}]} {}
-
-do_catchsql_test 81.2 {
-  UPDATE t1 SET b=zeroblob(299);
-} {1 {database disk image is malformed}}
-
 
 sqlite3_fts5_may_be_corrupt 0
 finish_test

ext/fts5/test/fts5corrupt4.test

@@ -14,7 +14,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt4
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5corrupt5.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt5
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5corrupt6.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt6
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5corrupt7.test

@@ -13,7 +13,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5corrupt7
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5dlidx.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5dlidx
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5doclist.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5doclist
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5ea.test

@@ -16,7 +16,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5ea
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5eb.test

@@ -13,7 +13,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5eb
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -86,13 +86,13 @@ do_execsql_test 3.0 {
   INSERT INTO e1 VALUES ('just a few words with a / inside');
 }
 do_execsql_test 3.1 {
-  SELECT rowid, format('%g',bm25(e1)) FROM e1 WHERE e1 MATCH '"just"' ORDER BY rank;
+  SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"just"' ORDER BY rank;
 } {1 -1e-06}
 do_execsql_test 3.2 {
   SELECT rowid FROM e1 WHERE e1 MATCH '"/" OR "just"'
 } 1
 do_execsql_test 3.3 {
-  SELECT rowid, format('%g',bm25(e1)) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank;
+  SELECT rowid, bm25(e1) FROM e1 WHERE e1 MATCH '"/" OR "just"' ORDER BY rank;
 } {1 -1e-06}
 
 do_execsql_test 3.4 "

ext/fts5/test/fts5expr.test

@@ -1,52 +0,0 @@
-# 2024 August 8
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#*************************************************************************
-# This file implements regression tests for SQLite library.  The
-# focus of this script is testing the FTS5 module.
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5expr
-
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE x1 USING fts5(a);
-  INSERT INTO x1(rowid, a) VALUES (113, 'fts5 expr test');
-}
-
-do_execsql_test 1.1 {
-  SELECT rowid FROM x1('expr');
-} {113}
-
-for {set ii 0} {$ii < 300} {incr ii} {
-  set expr "expr "
-  append expr [string repeat "NOT abcd " $ii]
-
-  if {$ii<257} {
-    set res {0 113}
-  } else {
-    set res {1 {fts5 expression tree is too large (maximum depth 256)}}
-  }
-  do_catchsql_test 1.1.$ii {
-    SELECT rowid FROM x1($expr)
-  } $res
-}
-
-do_execsql_test 1.2 {
-  SELECT rowid FROM x1 WHERE a MATCH '"..."'
-} {}
-
-finish_test
-

ext/fts5/test/fts5fault6.test

@@ -16,7 +16,7 @@ source [file join [file dirname [info script]] fts5_common.tcl]
 source $testdir/malloc_common.tcl
 set testprefix fts5fault6
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5faultG.test

@@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl]
 source $testdir/malloc_common.tcl
 set testprefix fts5faultG
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5faultH.test

@@ -14,7 +14,7 @@ source [file join [file dirname [info script]] fts5_common.tcl]
 source $testdir/malloc_common.tcl
 set testprefix fts5faultG
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5faultI.test

@@ -1,237 +0,0 @@
-# 2010 June 15
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-source $testdir/malloc_common.tcl
-set testprefix fts5faultI
-
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-
-set ::testprefix fts5faultI
-
-do_execsql_test 1.0 {
-  PRAGMA encoding = utf16;
-  CREATE VIRTUAL TABLE t1 USING fts5(x, locale=1);
-  INSERT INTO t1 VALUES('origintext unicode61 ascii porter trigram');
-}
-
-faultsim_save_and_close
-faultsim_restore_and_reopen
-
-do_faultsim_test 1 -faults oom* -prep { 
-} -body {
-  execsql {
-    SELECT rowid FROM t1(fts5_locale('en_US', 'origintext'));
-  }
-} -test {
-  faultsim_test_result {0 1}
-}
-
-do_faultsim_test 2 -faults oom* -prep { 
-  faultsim_restore_and_reopen
-  execsql {
-    SELECT * FROM t1('ascii');
-  }
-} -body {
-  execsql {
-    UPDATE t1 SET rowid=rowid+1;
-  }
-} -test {
-  faultsim_test_result {0 {}}
-}
-
-fts5_aux_test_functions db
-do_faultsim_test 3 -faults oom* -prep { 
-} -body {
-  execsql {
-    SELECT fts5_columnlocale(t1, 0) FROM t1('unicode*');
-  }
-} -test {
-  faultsim_test_result {0 {{}}} {1 SQLITE_NOMEM}
-}
-
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 4.0 {
-  CREATE VIRTUAL TABLE w1 USING fts5(a);
-}
-faultsim_save_and_close
-
-do_faultsim_test 4 -faults oom* -prep { 
-  faultsim_restore_and_reopen
-  execsql {
-    BEGIN;
-      INSERT INTO w1 VALUES('token token token');
-  }
-} -body {
-  execsql {
-    INSERT INTO w1(w1, rank) VALUES('rank', 'bm25()');
-  }
-} -test {
-  faultsim_test_result {0 {}}
-}
-
-do_faultsim_test 5 -faults oom* -prep { 
-  faultsim_restore_and_reopen
-  execsql { 
-    BEGIN;
-      INSERT INTO w1 VALUES('one');
-      SAVEPOINT one;
-        INSERT INTO w1 VALUES('two');
-      ROLLBACK TO one;
-  }
-
-} -body {
-  execsql {
-    INSERT INTO w1 VALUES('string');
-  }
-} -test {
-  faultsim_test_result {0 {}}
-}
-
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 5.0 {
-  CREATE VIRTUAL TABLE w1 USING fts5(a);
-  INSERT INTO w1 VALUES('one two three');
-}
-fts5_aux_test_functions db
-
-do_faultsim_test 5 -faults oom* -prep {
-} -body {
-  execsql {
-    SELECT fts5_test_insttoken(w1, 0, 0) FROM w1('two');
-  }
-} -test {
-  faultsim_test_result {0 two} {1 SQLITE_NOMEM}
-}
-
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 6.0 {
-  CREATE VIRTUAL TABLE w1 USING fts5(a);
-  INSERT INTO w1 VALUES('one two three');
-}
-fts5_aux_test_functions db
-faultsim_save_and_close
-
-do_faultsim_test 6 -faults oom* -prep {
-  faultsim_restore_and_reopen
-  db eval {
-    BEGIN;
-      INSERT INTO w1 VALUES('four five six');
-      SAVEPOINT abc;
-        INSERT INTO w1 VALUES('seven eight nine');
-        SAVEPOINT def;
-          INSERT INTO w1 VALUES('ten eleven twelve');
-  }
-} -body {
-  execsql {
-    RELEASE abc;
-  }
-} -test {
-  faultsim_test_result {0 {}}
-}
-
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 7.0 {
-  CREATE VIRTUAL TABLE w1 USING fts5(a);
-  INSERT INTO w1 VALUES('one two three');
-  INSERT INTO w1 VALUES('three two one');
-  DELETE FROM w1_content WHERE rowid=1;
-}
-
-faultsim_save_and_close
-
-do_faultsim_test 7 -faults oom* -prep {
-  faultsim_restore_and_reopen
-  db eval { SELECT * FROM w1 }
-} -body {
-  execsql {
-    PRAGMA integrity_check;
-  }
-} -test {
-}
-
-#-------------------------------------------------------------------------
-reset_db
-fts5_tclnum_register db
-fts5_aux_test_functions db
-
-do_execsql_test 8.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(
-      x, tokenize = "tclnum query", detail=columns
-  );
-  INSERT INTO ft VALUES('one two three i ii iii');
-  INSERT INTO ft VALUES('four five six iv v vi');
-  INSERT INTO ft VALUES('eight nine ten viii ix x');
-} {}
-
-do_faultsim_test 8.1 -faults oom* -prep {
-} -body {
-  execsql {
-    SELECT fts5_test_collist (ft) FROM ft('one two');
-  }
-} -test {
-  faultsim_test_result {0 {{0.0 1.0}}} {1 {SQL logic error}} {1 SQLITE_NOMEM}
-}
-
-do_faultsim_test 8.2 -faults oom* -prep {
-} -body {
-  execsql {
-    SELECT rowid FROM ft('one two') ORDER BY rank;
-  }
-} -test {
-  faultsim_test_result {0 1} {1 {SQL logic error}} {1 SQLITE_NOMEM}
-}
-
-#-------------------------------------------------------------------------
-reset_db
-
-do_execsql_test 9.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(x);
-  INSERT INTO ft VALUES('one two three i ii iii');
-  INSERT INTO ft VALUES('four five six iv v vi');
-  INSERT INTO ft VALUES('eight nine ten viii ix x');
-} {}
-
-faultsim_save_and_close
-
-do_faultsim_test 9.1 -faults oom* -prep {
-  faultsim_restore_and_reopen
-} -body {
-  execsql {
-    UPDATE ft SET rowid=4 WHERE rowid=1
-  }
-} -test {
-  faultsim_test_result {0 {}}
-}
-
-do_faultsim_test 9.2 -faults oom* -prep {
-  faultsim_restore_and_reopen
-} -body {
-  execsql {
-    SELECT rowid FROM ft WHERE x MATCH 'one AND two AND three'
-  }
-} -test {
-  faultsim_test_result {0 1}
-}
-
-
-
-finish_test
-

ext/fts5/test/fts5first.test

@@ -22,7 +22,6 @@ do_execsql_test 1.0 {
   CREATE VIRTUAL TABLE x1 USING fts5(a, b);
 }
 
-unset -nocomplain res
 foreach {tn expr ok} {
   1 {^abc}           1
   2 {^abc + def}     1

ext/fts5/test/fts5full.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5full
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5hash.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5hash
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5integrity.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5integrity
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -153,7 +153,6 @@ do_execsql_test 5.3 {
   INSERT INTO gg(gg) VALUES('integrity-check');
 }
 
-unset -nocomplain res
 do_test 5.4.1 {
   set ok 0
   for {set i 0} {$i < 10000} {incr i} {
@@ -406,7 +405,7 @@ sqlite3 db test.db
 breakpoint
 do_catchsql_test 13.4 {
   PRAGMA integrity_check;
-} {1 {SQL logic error}}
+} {1 {no such tokenizer: blah}}
 
 
 finish_test

ext/fts5/test/fts5interrupt.test

@@ -33,7 +33,6 @@ proc progress_handler {args} {
   return 0
 }
 
-unset -nocomplain res
 foreach {tn sql} {
   1 { INSERT INTO t1(rowid, a) VALUES(0, 'z z z z') }
   2 { COMMIT }
@@ -65,3 +64,4 @@ foreach {tn sql} {
 }
 
 finish_test
+

ext/fts5/test/fts5lastrowid.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5lastrowid
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5locale.test

@@ -1,576 +0,0 @@
-# 2014 Dec 20
-#
-# The author disclaims copyright to this source code.  In place of
-# a legal notice, here is a blessing:
-#
-#    May you do good and not evil.
-#    May you find forgiveness for yourself and forgive others.
-#    May you share freely, never taking more than you give.
-#
-#***********************************************************************
-#
-# Tests focusing on the built-in fts5 tokenizers. 
-#
-
-source [file join [file dirname [info script]] fts5_common.tcl]
-set testprefix fts5locale
-
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
-ifcapable !fts5 {
-  finish_test
-  return
-}
-
-proc transform_token {locale token} {
-  switch -- $locale {
-    reverse {
-      set ret ""
-      foreach c [split $token ""] {
-        set ret "$c$ret"
-      }
-      set token $ret
-    }
-
-    default {
-      # no-op
-    }
-  }
-
-  set token
-}
-
-proc tcl_create {args} { return "tcl_tokenize" }
-proc tcl_tokenize {tflags text} {
-  set iToken 1
-  set bSkip 0
-  if {[sqlite3_fts5_locale]=="second"} { set bSkip 1 }
-  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
-    incr iToken
-    if {(($iToken) % ($bSkip + 1))} continue
-
-    set w [transform_token [sqlite3_fts5_locale] $w]
-    sqlite3_fts5_token $w $iStart $iEnd
-  }
-}
-
-#-------------------------------------------------------------------------
-# Check that queries can have a locale attached to them.
-#
-reset_db
-sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
-
-do_execsql_test 1.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl);
-  INSERT INTO t1 VALUES('abc');
-  INSERT INTO t1 VALUES('cba');
-} {}
-
-do_execsql_test 1.1 {
-  SELECT rowid, a FROM t1( fts5_locale('en_US', 'abc') );
-} {1 abc}
-
-do_execsql_test 1.2 {
-  SELECT rowid, a FROM t1( fts5_locale('reverse', 'abc') );
-} {2 cba}
-
-#-------------------------------------------------------------------------
-# Test that the locale= option exists and seems to accept values. And
-# that fts5_locale() values may only be inserted into an internal-content
-# table if the locale=1 option was specified.
-#
-reset_db
-sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
-
-do_execsql_test 2.1 {
-  CREATE VIRTUAL TABLE b1 USING fts5(x, y, locale=1, tokenize=tcl);
-  CREATE VIRTUAL TABLE b2 USING fts5(x, y, locale=0, tokenize=tcl);
-
-  CREATE VIRTUAL TABLE ttt USING fts5vocab('b1', instance);
-}
-
-do_catchsql_test 2.2.1 {
-  CREATE VIRTUAL TABLE b3 USING fts5(x, y, locale=2);
-} {1 {malformed locale=... directive}}
-do_catchsql_test 2.2.2 {
-  CREATE VIRTUAL TABLE b3 USING fts5(x, y, locale=111);
-} {1 {malformed locale=... directive}}
-
-do_catchsql_test 2.3 {
-  INSERT INTO b1(b1, rank) VALUES('locale', 0);
-} {1 {SQL logic error}}
-
-do_execsql_test 2.4 {
-  INSERT INTO b1 VALUES('abc', 'one two three');
-  INSERT INTO b1 VALUES('def', fts5_locale('reverse', 'four five six'));
-}
-
-do_execsql_test 2.5 {
-  INSERT INTO b2 VALUES('abc', 'one two three');
-}
-
-do_catchsql_test 2.6 {
-  INSERT INTO b2 VALUES('def', fts5_locale('reverse', 'four five six'));
-} {1 {fts5_locale() requires locale=1}}
-
-do_execsql_test 2.7  { SELECT rowid FROM b1('one')  } {1}
-do_execsql_test 2.8  { SELECT rowid FROM b1('four') } {}
-do_execsql_test 2.9  { SELECT rowid FROM b1('ruof') } 2
-do_execsql_test 2.10 { SELECT rowid FROM b1(fts5_locale('reverse', 'five'))} 2
-
-do_execsql_test 2.11 {
-  SELECT x, quote(y) FROM b1
-} {
-  abc {'one two three'}
-  def {'four five six'}
-}
-
-do_execsql_test 2.12  { SELECT quote(y) FROM b1('ruof') } {
-  {'four five six'}
-}
-
-do_execsql_test 2.13 {
-  INSERT INTO b1(b1) VALUES('integrity-check');
-}
-do_execsql_test 2.14 {
-  INSERT INTO b1(b1) VALUES('rebuild');
-}
-do_execsql_test 2.15 {
-  INSERT INTO b1(b1) VALUES('integrity-check');
-}
-
-do_execsql_test 2.16 {
-  DELETE FROM b1 WHERE rowid=2
-}
-do_execsql_test 2.17 {
-  INSERT INTO b1(b1) VALUES('integrity-check');
-}
-
-do_execsql_test 2.18 {
-  INSERT INTO b1(rowid, x, y) VALUES(
-      test_setsubtype(45, 76), 'abc def', 'def abc'
-  );
-  INSERT INTO b1(b1) VALUES('integrity-check');
-}
-
-#-------------------------------------------------------------------------
-# Test the 'delete' command with contentless tables.
-#
-reset_db
-sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
-
-do_execsql_test 3.1 {
-  CREATE VIRTUAL TABLE c1 USING fts5(x, content=, tokenize=tcl, locale=1);
-  CREATE VIRTUAL TABLE c2 USING fts5vocab('c1', instance);
-
-  INSERT INTO c1 VALUES('hello world');
-  INSERT INTO c1 VALUES( fts5_locale('reverse', 'one two three') );
-}
-
-do_execsql_test 3.2 {
-  SELECT DISTINCT term FROM c2 ORDER BY 1
-} {
-  eerht eno hello owt world
-}
-
-do_execsql_test 3.3 {
-  INSERT INTO c1(c1, rowid, x) 
-    VALUES('delete', 2, fts5_locale('reverse', 'one two three') );
-}
-
-do_execsql_test 3.4 {
-  SELECT DISTINCT term FROM c2 ORDER BY 1
-} {
-  hello world
-}
-
-#-------------------------------------------------------------------------
-# Test that an UPDATE that updates a subset of the columns does not
-# magically discard the locale from those columns not updated.
-#
-reset_db
-sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
-
-do_execsql_test 4.1 {
-  CREATE VIRTUAL TABLE d1 USING fts5(x, y, locale=1, tokenize=tcl);
-  CREATE VIRTUAL TABLE d2 USING fts5vocab('d1', instance);
-
-  INSERT INTO d1(rowid, x, y) VALUES(1, 'abc', 'def');
-  INSERT INTO d1(rowid, x, y) VALUES(2, 'ghi', fts5_locale('reverse', 'hello'));
-}
-
-do_execsql_test 4.2 {
-  SELECT DISTINCT term FROM d2 ORDER BY 1
-} {
-  abc def ghi olleh
-}
-
-do_execsql_test 4.3 {
-  UPDATE d1 SET x='jkl' WHERE rowid=2;
-}
-
-do_execsql_test 4.4 {
-  SELECT DISTINCT term FROM d2 ORDER BY 1
-} {
-  abc def jkl olleh
-}
-
-do_execsql_test 4.5 {
-  SELECT rowid, * FROM d1
-} {
-  1 abc def
-  2 jkl hello
-}
-
-do_execsql_test 4.6 {
-  UPDATE d1 SET rowid=4 WHERE rowid=2
-}
-
-do_execsql_test 4.7 {
-  SELECT rowid, * FROM d1
-} {
-  1 abc def
-  4 jkl hello
-}
-
-fts5_aux_test_functions db
-
-do_execsql_test 4.8.1 {
-  SELECT fts5_test_columntext(d1) FROM d1('jkl')
-} {{jkl hello}}
-do_execsql_test 4.8.2 {
-  SELECT fts5_test_columntext(d1) FROM d1(fts5_locale('reverse', 'hello'))
-} {{jkl hello}}
-
-do_execsql_test 4.9 {
-  SELECT fts5_test_columnlocale(d1) FROM d1(fts5_locale('reverse', 'hello'))
-} {{{} reverse}}
-
-do_execsql_test 4.10 {
-  SELECT fts5_test_columnlocale(d1) FROM d1
-} {
-  {{} {}}
-  {{} reverse}
-}
-
-#-------------------------------------------------------------------------
-# Test that if an fts5_locale() value is written to an UNINDEXED 
-# column it is stored as text. This is so that blobs and other values
-# can also be stored as is.
-#
-reset_db
-sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
-
-do_execsql_test 5.1 {
-  CREATE VIRTUAL TABLE t1 USING fts5(
-      x, y UNINDEXED, locale=1, tokenize=tcl
-  );
-
-  INSERT INTO t1(rowid, x, y) VALUES(111,
-      fts5_locale('reverse', 'one two three'),
-      fts5_locale('reverse', 'four five six')
-  );
-}
-
-do_execsql_test 5.2 {
-  SELECT rowid, x, y FROM t1
-} {
-  111 {one two three} {four five six}
-}
-
-do_execsql_test 5.3 {
-  SELECT typeof(c0), typeof(c1) FROM t1_content
-} {
-  blob text
-}
-
-#-------------------------------------------------------------------------
-
-foreach {tn opt} {
-  1 {}
-  2 {, columnsize=0}
-} {
-  reset_db
-  sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
-  do_execsql_test 6.$tn.1 "
-    CREATE VIRTUAL TABLE y1 USING fts5(t, locale=1, tokenize=tcl $opt);
-  "
-
-  do_execsql_test 6.$tn.2 {
-    INSERT INTO y1(rowid, t) VALUES
-      (1, fts5_locale('second', 'the city of London')),
-      (2, fts5_locale('second', 'shall have all the old')),
-      (3, fts5_locale('second', 'Liberties and Customs')),
-      (4, fts5_locale('second', 'which it hath been used to have'));
-  }
-
-  fts5_aux_test_functions db
-
-  do_execsql_test 5.$tn.3 {
-    SELECT fts5_test_columnsize(y1) FROM y1
-  } {
-    2 3 2 4
-  }
-
-  do_execsql_test 5.$tn.4 {
-    SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall');
-  } {
-    2 3
-  }
-
-  do_execsql_test 5.$tn.5 {
-    SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall');
-  } {
-    2 3
-  }
-
-  do_execsql_test 5.$tn.6 {
-    SELECT rowid, fts5_test_columnsize(y1) FROM y1('have');
-  } {
-    4 4
-  }
-
-  do_execsql_test 5.$tn.7 {
-    SELECT rowid, highlight(y1, 0, '[', ']') FROM y1('have');
-  } {
-    4 {which it hath been used to [have]}
-  }
-
-  do_execsql_test 5.$tn.8 {
-    SELECT rowid, 
-           highlight(y1, 0, '[', ']'),
-           snippet(y1, 0, '[', ']', '...', 10)
-    FROM y1('Liberties + Customs');
-  } {
-    3 {[Liberties and Customs]}
-      {[Liberties and Customs]}
-  }
-}
-
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 6.0 {
-  CREATE VIRTUAL TABLE x1 USING fts5(x);
-}
-do_catchsql_test 6.1 {
-  INSERT INTO x1(rowid, x) VALUES(123, fts5_locale('en_AU', 'hello world'));
-} {1 {fts5_locale() requires locale=1}}
-
-do_execsql_test 6.2 {
-  SELECT typeof( fts5_locale(NULL, 'xyz') ), typeof( fts5_locale('', 'abc') );
-} {text text}
-
-#--------------------------------------------------------------------------
-# Test that fts5_locale() works with external-content tables.
-#
-reset_db
-sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
-
-do_execsql_test 7.1 {
-  CREATE TABLE t1(ii INTEGER PRIMARY KEY, bb BLOB, tt TEXT, locale TEXT);
-  CREATE VIEW v1 AS 
-      SELECT ii AS rowid, bb, fts5_locale(locale, tt) AS tt FROM t1;
-
-  CREATE VIRTUAL TABLE ft USING fts5(
-      bb, tt, locale=1, tokenize=tcl, content=v1
-  );
-
-  INSERT INTO t1 VALUES(1, NULL, 'one two three', NULL);
-  INSERT INTO t1 VALUES(2, '7800616263', 'four five six', 'reverse');
-  INSERT INTO t1 VALUES(3, '000000007800616263', 'seven eight nine', 'second');
-}
-
-do_execsql_test 7.2 {
-  INSERT INTO ft(ft) VALUES('rebuild');
-  INSERT INTO ft(ft) VALUES('integrity-check');
-}
-
-do_execsql_test 7.3 {
-  SELECT rowid, quote(bb), quote(tt) FROM ft
-} {
-  1 NULL {'one two three'}
-  2 '7800616263' {'four five six'}
-  3 '000000007800616263' {'seven eight nine'}
-}
-
-do_execsql_test 7.4 { SELECT rowid FROM ft('six'); } 
-do_execsql_test 7.5 { SELECT rowid FROM ft(fts5_locale('reverse','six')); } 2
-
-fts5_aux_test_functions db
-
-do_execsql_test 7.6 {
-  SELECT fts5_test_columnlocale(ft) FROM ft;
-} {
-  {{} {}} {{} reverse} {{} second}
-}
-
-#-------------------------------------------------------------------------
-# Test that the porter tokenizer works with locales.
-#
-reset_db
-sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
-
-do_execsql_test 8.1 {
-  CREATE VIRTUAL TABLE ft USING fts5(tt, locale=1, tokenize="porter tcl");
-  CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', instance);
-
-  INSERT INTO ft(rowid, tt) VALUES
-    (111, fts5_locale('second', 'the porter tokenizer is a wrapper tokenizer')),
-    (222, fts5_locale('reverse', 'This value may also be set'));
-}
-
-do_execsql_test 8.1 {
-  SELECT DISTINCT term FROM vocab ORDER BY 1
-} {
-  a eb eulav osla sihT te the token yam
-}
-
-#-------------------------------------------------------------------------
-# Test that position-lists (used by xInst, xPhraseFirst etc.) work with
-# locales and modes other than detail=full.
-#
-foreach {tn detail} {
-  1 detail=full
-  2 detail=none
-  3 detail=column
-} {
-  reset_db
-  sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
-  do_execsql_test 9.$tn.0 "
-    CREATE VIRTUAL TABLE ft USING fts5(tt, locale=1, tokenize=tcl, $detail);
-  "
-  do_execsql_test 9.$tn.1 {
-    CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', instance);
-    INSERT INTO ft(rowid, tt) VALUES
-      (-1, fts5_locale('second', 'it is an ancient mariner'));
-  }
-
-  do_execsql_test 9.$tn.2 {
-    SELECT DISTINCT term FROM vocab 
-  } {an it mariner}
-
-  do_execsql_test 9.$tn.3 {
-    SELECT highlight(ft, 0, '[', ']') FROM ft('mariner')
-  } {{it is an ancient [mariner]}}
-}
-
-#-------------------------------------------------------------------------
-# Check some corrupt fts5_locale() blob formats are detected.
-#
-foreach_detail_mode $::testprefix {
-
-  reset_db
-  sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
-  fts5_aux_test_functions db
-  do_execsql_test 10.1 {
-    CREATE TABLE x1(ii INTEGER PRIMARY KEY, x);
-    CREATE VIRTUAL TABLE ft USING fts5(x, 
-        content=x1, content_rowid=ii, locale=1, detail=%DETAIL%, columnsize=0
-    );
-
-    CREATE VIRTUAL TABLE ft2 USING fts5(
-        x, locale=1, detail=%DETAIL%, columnsize=0
-    );
-  }
-  
-  foreach {tn v} {
-    1   X'001122'
-    2   X'0011223344'
-    3   X'00E0B2EB68656c6c6f'
-    4   X'00E0B2EB0068656c6c6f'
-  } {
-    do_execsql_test 10.2.$tn.0 { INSERT INTO ft(ft) VALUES('delete-all') }
-    do_execsql_test 10.2.$tn.1 { DELETE FROM x1; }
-    do_execsql_test 10.2.$tn.2 " INSERT INTO x1 VALUES(NULL, $v) "
-  
-    do_catchsql_test 10.2.$tn.3 {
-      INSERT INTO ft(ft) VALUES('rebuild');
-    } {1 {SQL logic error}}
-  
-    do_catchsql_test 10.2.$tn.4 "
-      SELECT * FROM ft( test_setsubtype($v, 76) );
-    " {1 {SQL logic error}}
-  
-    do_execsql_test 10.2.$tn.5 {
-      INSERT INTO ft(rowid, x) VALUES(1, 'hello world');
-    }
-  
-    if {"%DETAIL%"!="full"} {
-      do_catchsql_test 10.2.$tn.6 {
-        SELECT fts5_test_poslist(ft) FROM ft('world');
-      } {1 SQLITE_ERROR}
-  
-      do_catchsql_test 10.2.$tn.7 {
-        SELECT fts5_test_columnsize(ft) FROM ft('world');
-      } {1 SQLITE_ERROR}
-
-      do_catchsql_test 10.2.$tn.7 {
-        SELECT fts5_test_columnlocale(ft) FROM ft('world');
-      } {1 SQLITE_ERROR}
-    }
-
-    do_catchsql_test 10.2.$tn.8 {
-      SELECT * FROM ft('hello')
-    } {1 {SQL logic error}}
-
-    do_catchsql_test 10.2.$tn.9 {
-      PRAGMA integrity_check;
-    } {0 ok}
-
-    do_execsql_test 10.2.$tn.10 {
-      DELETE FROM x1;
-      INSERT INTO x1(ii, x) VALUES(1, 'hello world');
-    }
-
-    do_catchsql_test 10.2.$tn.11 "
-      INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, test_setsubtype($v,76) )
-    " {1 {SQL logic error}}
-
-    do_catchsql_test 10.2.$tn.12 "
-      INSERT INTO ft(rowid, x) VALUES(2, test_setsubtype($v,76) )
-    " {1 {SQL logic error}}
-
-    do_execsql_test 10.2.$tn.13 {
-      INSERT INTO ft2(rowid, x) VALUES(1, 'hello world');
-    }
-    do_execsql_test 10.2.$tn.14 "UPDATE ft2_content SET c0=$v"
-
-    do_catchsql_test 10.2.$tn.15 {
-      PRAGMA integrity_check;
-    } {1 {SQL logic error}}
-
-    do_execsql_test 10.2.$tn.16 {
-      DELETE FROM ft2_content;
-      INSERT INTO ft2(ft2) VALUES('rebuild');
-    }
-  }
-
-}
-
-#-------------------------------------------------------------------------
-#
-reset_db
-sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
-fts5_aux_test_functions db
-do_execsql_test 11.0 {
-  CREATE VIRTUAL TABLE x1 USING fts5(abc, locale=1);
-  INSERT INTO x1(rowid, abc) VALUES(123, fts5_locale('en_US', 'one two three'));
-}
-
-do_catchsql_test 11.1 {
-  SELECT fts5_columnlocale(x1, -1) FROM x1('two');
-} {1 SQLITE_RANGE}
-do_catchsql_test 11.2 {
-  SELECT fts5_columnlocale(x1, 1) FROM x1('two');
-} {1 SQLITE_RANGE}
-
-#-------------------------------------------------------------------------
-#
-reset_db
-do_test 12.0 {
-  list [catch {
-    sqlite3_fts5_create_tokenizer -v2 -version 3 db tcl tcl_create
-  } msg] $msg
-} {1 {error in fts5_api.xCreateTokenizer_v2()}}
-
-finish_test
-

ext/fts5/test/fts5matchinfo.test

@@ -517,7 +517,7 @@ fts5_aux_test_functions db
 do_execsql_test 15.3 {
   SELECT fts5_test_all(t1) FROM t1 LIMIT 1;
 } {
-  {columnsize {1 1} columntext {c d} columntotalsize {2 2} poslist {} tokenize {c d} rowcount 2}
+  {columnsize {0 0} columntext {c d} columntotalsize {2 2} poslist {} tokenize {c d} rowcount 2}
 }
 
 finish_test

ext/fts5/test/fts5merge.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5merge
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5misc.test

@@ -35,21 +35,21 @@ do_catchsql_test 1.1.2 {
 
 do_catchsql_test 1.2.1 { 
   SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*id'); 
-} {1 {no such cursor: 4}}
+} {0 {{}}}
 
 do_catchsql_test 1.2.2 {
   SELECT a FROM t1
     WHERE rank = (SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*id'));
-} {1 {no such cursor: 6}}
+} {0 {}}
 
 do_catchsql_test 1.3.1 { 
   SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads'); 
-} {1 {no such cursor: 1}}
+} {1 {no such cursor: 2}}
 
 do_catchsql_test 1.3.2 {
   SELECT a FROM t1
     WHERE rank = (SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads'));
-} {1 {no such cursor: 1}}
+} {1 {no such cursor: 2}}
 
 db close
 sqlite3 db test.db
@@ -59,11 +59,6 @@ do_catchsql_test 1.3.3 {
     WHERE rank = (SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads'));
 } {1 {no such cursor: 1}}
 
-fts5_aux_test_functions db
-do_catchsql_test 1.3.4 {
-  SELECT fts5_columntext(t1) FROM t1('*reads');
-} {1 {no such cursor: 1}}
-
 #-------------------------------------------------------------------------
 reset_db
 do_execsql_test 2.0 {
@@ -540,130 +535,5 @@ do_execsql_test 19.0 {
   COMMIT;
 }
 
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 20.0 {
-  CREATE VIRTUAL TABLE x1 USING fts5(a);
-  INSERT INTO x1(rowid, a) VALUES
-      (1, 'a b c d'),
-      (2, 'x b c d'),
-      (3, 'x y z d'),
-      (4, 'a y c x');
-}
-
-do_execsql_test 20.1 {
-  SELECT rowid FROM x1 WHERE x1 MATCH 'a' AND x1 MATCH 'b';
-} {1}
-
-do_execsql_test 20.2 {
-  SELECT rowid FROM x1 WHERE x1 MATCH 'a' AND x1 MATCH 'y';
-} {4}
-
-do_execsql_test 20.3 {
-  SELECT rowid FROM x1 WHERE x1 MATCH 'a' OR x1 MATCH 'y';
-} {1 4 3}
-
-do_execsql_test 20.4 {
-  SELECT rowid FROM x1 WHERE x1 MATCH 'a' OR (x1 MATCH 'y' AND x1 MATCH 'd');
-} {1 4 3}
-
-do_execsql_test 20.5 {
-  SELECT rowid FROM x1 WHERE x1 MATCH 'z' OR (x1 MATCH 'a' AND x1 MATCH 'd');
-} {3 1}
-
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 21.0 {
-  CREATE TABLE t1(ii INTEGER, x TEXT, y TEXT);
-  CREATE VIRTUAL TABLE xyz USING fts5(content_rowid=ii, content=t1, x, y);
-  INSERT INTO t1 VALUES(1, 'one', 'i');
-  INSERT INTO t1 VALUES(2, 'two', 'ii');
-  INSERT INTO t1 VALUES(3, 'tree', 'iii');
-  INSERT INTO xyz(xyz) VALUES('rebuild');
-}
-
-do_execsql_test 21.1 {
-  UPDATE xyz SET y='TWO' WHERE rowid=2;
-  UPDATE t1 SET y='TWO' WHERE ii=2;
-}
-
-do_execsql_test 21.2 {
-  PRAGMA integrity_check
-} {ok}
-
-breakpoint
-sqlite3_db_config db DEFENSIVE 1
-do_execsql_test 21.3 {
-  CREATE TABLE xyz_notashadow(x, y);
-  DROP TABLE xyz_notashadow;
-}
-sqlite3_db_config db DEFENSIVE 0
-
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 22.0 {
-  SELECT fts5(NULL);
-} {{}}
-do_execsql_test 22.1 {
-  SELECT count(*) FROM (
-    SELECT fts5_source_id()
-  )
-} {1}
-execsql_pp {
-  SELECT fts5_source_id()
-}
-
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 23.0 {
-  CREATE VIRTUAL TABLE x1 USING fts5(x);
-  INSERT INTO x1 VALUES('one + two + three');
-  INSERT INTO x1 VALUES('one + xyz + three');
-  INSERT INTO x1 VALUES('xyz + two + xyz');
-}
-do_execsql_test 23.1 {
-  SELECT rowid FROM x1('one + two + three');
-} {1}
-
-do_execsql_test 23.2 {
-  SELECT rowid FROM x1('^".." AND one');
-} {}
-
-do_execsql_test 23.3 {
-  SELECT rowid FROM x1('abc NEAR ".." NEAR def');
-} {}
-
-#-------------------------------------------------------------------------
-reset_db
-do_execsql_test 24.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, detail='none');
-  INSERT INTO t1(a) VALUES('a');
-}
-
-do_execsql_test 24.2 {
-  SELECT rank FROM ( SELECT rank FROM t1('a NOT "" NOT def') ) ORDER BY 1;
-} {-1e-06}
-
-do_execsql_test 24.3 {
-  SELECT rank FROM ( SELECT rank FROM t1('a NOT � NOT def') ) ORDER BY 1;
-} {-1e-06}
-
-do_execsql_test 24.4 {
-  SELECT rank FROM ( SELECT rank FROM t1('a NOT "" NOT def') );
-} {-1e-06}
-
-#-------------------------------------------------------------------------
-reset_db
-fts5_aux_test_functions db
-
-do_execsql_test 25.0 {
-  CREATE VIRTUAL TABLE t1 USING fts5(a, detail='none', content='');
-  INSERT INTO t1(a) VALUES('a b c');
-}
-
-do_execsql_test 25.0 {
-  SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank;
-} {{}}
-
 finish_test
 

ext/fts5/test/fts5near.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5near
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5optimize.test

@@ -14,7 +14,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5optimize
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5optimize2.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5optimize2
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5optimize3.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5optimize2
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5origintext.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5origintext
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return
@@ -261,8 +261,8 @@ do_execsql_test 5.3 {
 reset_db
 sqlite3_fts5_register_origintext db
 do_execsql_test 6.0 {
-  CREATE VIRTUAL TABLE ft USING fts5( 
-      x, y, tokenize='origintext unicode61', detail=%DETAIL%, tokendata=0
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, y, tokenize='origintext unicode61', detail=%DETAIL%
   );
 
   INSERT INTO ft VALUES('One Two', 'Three two');
@@ -291,22 +291,6 @@ do_execsql_test 6.3 {
   SELECT rowid, tokens(ft) FROM ft('Three*');
 } {1 {{}} 2 {{}}}
 
-fts5_aux_test_functions db
-do_catchsql_test 6.4 {
-  SELECT fts5_test_insttoken(ft, -1, 0) FROM ft('one');
-} {1 SQLITE_RANGE}
-
-do_catchsql_test 6.5 {
-  SELECT fts5_test_insttoken(ft, 1, 0) FROM ft('one');
-} {1 SQLITE_RANGE}
-
-do_catchsql_test 6.6 {
-  CREATE VIRTUAL TABLE ft2 USING fts5(x, tokendata=2);
-} {1 {malformed tokendata=... directive}}
-do_catchsql_test 6.7 {
-  CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', tokendata=11);
-} {1 {malformed tokendata=... directive}}
-
 }
 
 finish_test

ext/fts5/test/fts5origintext2.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5origintext2
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5origintext3.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5origintext3
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5origintext4.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5origintext4
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5origintext5.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5origintext
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5phrase.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5phrase
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5plan.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5plan
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5porter.test

@@ -17,7 +17,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5porter
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5porter2.test

@@ -18,7 +18,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5porter2
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return

ext/fts5/test/fts5prefix.test

@@ -15,7 +15,7 @@
 source [file join [file dirname [info script]] fts5_common.tcl]
 set testprefix fts5prefix
 
-# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+# If SQLITE_ENABLE_FTS5 is defined, omit this file.
 ifcapable !fts5 {
   finish_test
   return