Remove redundant #ifdef accidentally inserted into mkkeywordhash.c

FossilOrigin-Name: b08f3b74495e125f92c8968472fc8028c73d8df5400195df3081e3d8df8bd985
Add the AGGREGATE clause to the pipeline.
2024-08-26 14:20:20 +00:00 · 2024-08-26 14:06:08 +00:00 · 2024-08-26 13:20:38 +00:00 · 2024-08-26 12:26:26 +00:00 · 2024-08-26 02:53:31 +00:00 · 2024-08-26 00:15:35 +00:00
59 changed files with 4422 additions and 612 deletions
@@ -117,6 +117,11 @@ HAVE_TCL = @HAVE_TCL@
 #
 TCLSH_CMD = @TCLSH_CMD@

+# Additional options when running tests using testrunner.tcl
+# This is usually either blank, or else --status
+#
+TSTRNNR_OPTS = @TSTRNNR_OPTS@
+
 # Where do we want to install the tcl plugin
 #
 TCLLIBDIR = @TCLLIBDIR@
@@ -1363,10 +1368,10 @@ testrunner:	testfixture$(TEXE)
 devtest:	srctree-check testfixture$(TEXE) fuzztest testrunner

 mdevtest: srctree-check has_tclsh85
-	$(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest
+	$(TCLSH_CMD) $(TOP)/test/testrunner.tcl mdevtest $(TSTRNNR_OPTS)

 sdevtest: has_tclsh85
-	$(TCLSH_CMD) $(TOP)/test/testrunner.tcl sdevtest
+	$(TCLSH_CMD) $(TOP)/test/testrunner.tcl sdevtest $(TSTRNNR_OPTS)

 # Validate that various generated files in the source tree
 # are up-to-date.
@@ -1377,7 +1382,7 @@ srctree-check:	$(TOP)/tool/srctree-check.tcl
 # Testing for a release
 #
 releasetest: srctree-check testfixture$(TEXE)
-	./testfixture$(TEXE) $(TOP)/test/testrunner.tcl release
+	./testfixture$(TEXE) $(TOP)/test/testrunner.tcl release $(TSTRNNR_OPTS)

 # Minimal testing that runs in less than 3 minutes
 #
@@ -1,14 +1,19 @@
 <h1 align="center">SQLite Source Repository</h1>

 This repository contains the complete source code for the
-[SQLite database engine](https://sqlite.org/).  Some test scripts
-are also included.  However, many other test scripts
+[SQLite database engine](https://sqlite.org/), including
+many test scripts.  However, other test scripts
 and most of the documentation are managed separately.

+See the [on-line documentation](https://sqlite.org/) for more information
+about what SQLite is and how it works from a user's perspective.  This
+README file is about the source code that goes into building SQLite,
+not about how SQLite is used.
+
 ## Version Control

 SQLite sources are managed using
-[Fossil](https://www.fossil-scm.org/), a distributed version control system
+[Fossil](https://fossil-scm.org/), a distributed version control system
 that was specifically designed and written to support SQLite development.
 The [Fossil repository](https://sqlite.org/src/timeline) contains the urtext.

@@ -68,12 +73,11 @@ archives or [SQLite archives](https://sqlite.org/cli.html#sqlar) as follows:
     then click on the "Tarball" or "ZIP Archive" links on the information
     page.

-If you do want to use Fossil to check out the source tree,
-first install Fossil version 2.0 or later.
-(Source tarballs and precompiled binaries available
-[here](https://www.fossil-scm.org/fossil/uv/download.html).  Fossil is
+To access sources directly using Fossil, first install Fossil version 2.0 or later.
+Source tarballs and precompiled binaries available
+[here](https://www.fossil-scm.org/home/uv/download.html).  Fossil is
 a stand-alone program.  To install, simply download or build the single
-executable file and put that file someplace on your $PATH.)
+executable file and put that file someplace on your $PATH.
 Then run commands like this:

        mkdir -p ~/sqlite ~/Fossils
@@ -81,8 +85,8 @@ Then run commands like this:
        fossil clone https://www.sqlite.org/src ~/Fossils/sqlite.fossil
        fossil open ~/Fossils/sqlite.fossil

-After setting up a repository using the steps above, you can always
-update to the latest version using:
+After setting up a repository using the steps above, you can do
+bandwidth-efficient updates to the latest version using:

        fossil update trunk   ;# latest trunk check-in
        fossil update release ;# latest official release
@@ -99,15 +103,30 @@ script found at the root of the source tree.  Then run "make".

 For example:

-        tar xzf sqlite.tar.gz    ;#  Unpack the source tree into "sqlite"
-        mkdir bld                ;#  Build will occur in a sibling directory
-        cd bld                   ;#  Change to the build directory
-        ../sqlite/configure      ;#  Run the configure script
-        make                     ;#  Builds the "sqlite3" command-line tool
-        make sqlite3.c           ;#  Build the "amalgamation" source file
-        make devtest             ;#  Run some tests (requires Tcl)
+        tar xzf sqlite.tar.gz      ;#  Unpack the source tree into "sqlite"
+        mkdir bld                  ;#  Build will occur in a sibling directory
+        cd bld                     ;#  Change to the build directory
+        ../sqlite/configure        ;#  Run the configure script
+        make sqlite3               ;#  Builds the "sqlite3" command-line tool
+        make sqlite3.c             ;#  Build the "amalgamation" source file
+        make mdevtest              ;#  Run development tests (requires tcl-dev)
+        make releasetest           ;#  Run full release tests (requires tcl-dev)
+        make sqldiff               ;#  Builds the "sqldiff" command-line tool
+        make sqlite3_analyzer      ;#  Builds the "sqlite3_analyzer" tool (requires tcl-dev)
+        make tclextension-install  ;#  Build and install the SQLite TCL extension

-See the makefile for additional targets.
+See the makefile for additional targets.  For debugging builds, the
+core developers typically run "configure" with options like this:
+
+        ../sqlite/configure --enable-all --enable-debug CFLAGS='-O0 -g'
+
+For release builds, the core developers usually do:
+
+        ../sqlite/configure --enable-all
+
+Almost all makefile targets require a "tclsh" TCL interpreter
+version 8.6 or later.  The targets marked with "(requires tcl-dev)" also require
+the TCL development libraries.

 The configure script uses autoconf 2.61 and libtool.  If the configure
 script does not work out for you, there is a generic makefile named
@@ -128,48 +147,61 @@ TCL library, using a command like this:

        set TCLDIR=c:\Tcl

-SQLite uses "tclsh.exe" as part of the build process, and so that utility
-program will need to be somewhere on your %PATH%.  The finished SQLite library
-does not contain any TCL code, but it does use TCL to help with the build process
-and to run tests.
+SQLite uses "tclsh.exe" as part of the build process, and so that
+program will need to be somewhere on your %PATH%.  SQLite itself
+does not contain any TCL code, but it does use TCL to help with the
+build process and to run tests.

 Build using Makefile.msc.  Example:

-        nmake /f Makefile.msc
+        nmake /f Makefile.msc sqlite3.exe
        nmake /f Makefile.msc sqlite3.c
-        nmake /f Makefile.msc devtest
+        nmake /f Makefile.msc mdevtest
        nmake /f Makefile.msc releasetest
+        nmake /f Makefile.msc tclextension-install
 
 There are many other makefile targets.  See comments in Makefile.msc for
 details.

-## Source Code Tour
+## Source Tree Map

-Most of the core source files are in the **src/** subdirectory.  The
-**src/** folder also contains files used to build the "testfixture" test
-harness. The names of the source files used by "testfixture" all begin
-with "test".
-The **src/** also contains the "shell.c" file
-which is the main program for the "sqlite3.exe"
-[command-line shell](https://sqlite.org/cli.html) and
-the "tclsqlite.c" file which implements the
-[Tcl bindings](https://sqlite.org/tclsqlite.html) for SQLite.
-(Historical note:  SQLite began as a Tcl
-extension and only later escaped to the wild as an independent library.)
+  *  **src/** - This directory contains the primary source code for the
+     SQLite core.  For historical reasons, C-code used for testing is
+     also found here.  Source files intended for testing begin with "`test`".
+     The `tclsqlite3.c` and `tclsqlite3.h` files are the TCL interface
+     for SQLite and are also not part of the core.

-Test scripts and programs are found in the **test/** subdirectory.
-Additional test code is found in other source repositories.
-See [How SQLite Is Tested](https://www.sqlite.org/testing.html) for
-additional information.
+  *  **test/** - This directory and its subdirectories contains code used
+     for testing.  Files that end in "`.test`" are TCL scripts that run
+     tests using an augmented TCL interpreter named "testfixture".  Use
+     a command like "`make testfixture`" (unix) or 
+     "`nmake /f Makefile.msc testfixture.exe`" (windows) to build that
+     augmented TCL interpreter, then run individual tests using commands like
+     "`testfixture test/main.test`".  This test/ subdirectory also contains
+     additional C code modules and scripts for other kinds of testing.

-The **ext/** subdirectory contains code for extensions.  The
-Full-text search engine is in **ext/fts3**.  The R-Tree engine is in
-**ext/rtree**.  The **ext/misc** subdirectory contains a number of
-smaller, single-file extensions, such as a REGEXP operator.
+  *  **tool/** - This directory contains programs and scripts used to
+     build some of the machine-generated code that goes into the SQLite
+     core, as well as to build and run tests and perform diagnostics.
+     The source code to [the Lemon parser generator](./doc/lemon.html) is
+     found here.  There are also TCL scripts used to build and/or transform
+     source code files.  For example, the tool/mksqlite3h.tcl script reads
+     the src/sqlite.h.in file and uses it as a template to construct
+     the deliverable "sqlite3.h" file that defines the SQLite interface.

-The **tool/** subdirectory contains various scripts and programs used
-for building generated source code files or for testing or for generating
-accessory programs such as "sqlite3_analyzer(.exe)".
+  *  **ext/** - Various extensions to SQLite are found under this
+     directory.  For example, the FTS5 subsystem is in "ext/fts5/".
+     Some of these extensions (ex: FTS3/4, FTS5, RTREE) might get built
+     into the SQLite amalgamation, but not all of them.  The
+     "ext/misc/" subdirectory contains an assortment of one-file extensions,
+     many of which are omitted from the SQLite core, but which are included
+     in the [SQLite CLI](https://sqlite.org/cli.html).
+     
+  *  **doc/** - Some documentation files about SQLite internals are found
+     here.  Note, however, that the primary documentation designed for
+     application developers and users of SQLite is in a completely separate
+     repository.  Note also that the primary API documentation is derived
+     from specially constructed comments in the src/sqlite.h.in file.

 ### Generated Source Code Files

@@ -252,31 +284,37 @@ individual source file exceeds 32K lines in length.
 SQLite is modular in design.
 See the [architectural description](https://www.sqlite.org/arch.html)
 for details. Other documents that are useful in
-(helping to understand how SQLite works include the
+helping to understand how SQLite works include the
 [file format](https://www.sqlite.org/fileformat2.html) description,
 the [virtual machine](https://www.sqlite.org/opcode.html) that runs
 prepared statements, the description of
 [how transactions work](https://www.sqlite.org/atomiccommit.html), and
 the [overview of the query planner](https://www.sqlite.org/optoverview.html).

-Years of effort have gone into optimizing SQLite, both
+Decades of effort have gone into optimizing SQLite, both
 for small size and high performance.  And optimizations tend to result in
 complex code.  So there is a lot of complexity in the current SQLite
 implementation.  It will not be the easiest library in the world to hack.

-Key files:
+### Key source code files

  *  **sqlite.h.in** - This file defines the public interface to the SQLite
     library.  Readers will need to be familiar with this interface before
-     trying to understand how the library works internally.
+     trying to understand how the library works internally.  This file is
+     really a template that is transformed into the "sqlite3.h" deliverable
+     using a script invoked by the makefile.

  *  **sqliteInt.h** - this header file defines many of the data objects
     used internally by SQLite.  In addition to "sqliteInt.h", some
-     subsystems have their own header files.
+     subsystems inside of sQLite have their own header files.  These internal
+     interfaces are not for use by applications.  They can and do change
+     from one release of SQLite to the next.

  *  **parse.y** - This file describes the LALR(1) grammar that SQLite uses
     to parse SQL statements, and the actions that are taken at each step
-     in the parsing process.
+     in the parsing process.  The file is processed by the
+     [Lemon Parser Generator](./doc/lemon.html) to produce the actual C code
+     used for parsing.

  *  **vdbe.c** - This file implements the virtual machine that runs
     prepared statements.  There are various helper files whose names
@@ -319,6 +357,11 @@ Key files:
     (and some other test programs too) is built and run when you type
     "make test".

+  *  **VERSION**, **manifest**, and **manifest.uuid** - These files define
+     the current SQLite version number.  The "VERSION" file is human generated,
+     but the "manifest" and "manifest.uuid" files are automatically generated
+     by the [Fossil version control system](https://fossil-scm/).
+
 There are many other source files.  Each has a succinct header comment that
 describes its purpose and role within the larger system.

@@ -336,7 +379,7 @@ The `manifest.uuid` file should contain the SHA3-256 hash of the
 `manifest` file. If all of the above hash comparisons are correct, then
 you can be confident that your source tree is authentic and unadulterated.
 Details on the format for the `manifest` files are available
-[on the Fossil website](https://fossil-scm.org/fossil/doc/trunk/www/fileformat.wiki#manifest).
+[on the Fossil website](https://fossil-scm.org/home/doc/trunk/www/fileformat.wiki#manifest).

 The process of checking source code authenticity is automated by the 
 makefile:
@@ -357,3 +400,7 @@ The main SQLite website is [https://sqlite.org/](https://sqlite.org/)
 with geographically distributed backups at
 [https://www2.sqlite.org/](https://www2.sqlite.org) and
 [https://www3.sqlite.org/](https://www3.sqlite.org).
+
+Contact the SQLite developers through the
+[SQLite Forum](https://sqlite.org/forum/).  In an emergency, you
+can send private email to the lead developer at drh at sqlite dot org.
@@ -793,6 +793,7 @@ HAVE_WASI_SDK
 RELEASE
 VERSION
 program_prefix
+TSTRNNR_OPTS
 TCLLIBDIR
 HAVE_TCL
 TCL_STUB_LIB_SPEC
@@ -889,6 +890,7 @@ enable_largefile
 with_tclsh
 with_tcl
 enable_tcl
+enable_test_status
 with_wasi_sdk
 enable_threadsafe
 enable_releasemode
@@ -1546,6 +1548,7 @@ Optional Features:
  --disable-largefile     omit support for large files
  --disable-tcl           omit building accessory programs that require
                          TCL-dev
+  --enable-test-status    Full-screen status of tests
  --disable-threadsafe    Disable mutexing
  --enable-releasemode    Support libtool link to release mode
  --enable-tempstore      Use an in-ram database for temporary tables
@@ -3939,13 +3942,13 @@ if ${lt_cv_nm_interface+:} false; then :
 else
  lt_cv_nm_interface="BSD nm"
  echo "int some_variable = 0;" > conftest.$ac_ext
-  (eval echo "\"\$as_me:3942: $ac_compile\"" >&5)
+  (eval echo "\"\$as_me:3945: $ac_compile\"" >&5)
  (eval "$ac_compile" 2>conftest.err)
  cat conftest.err >&5
-  (eval echo "\"\$as_me:3945: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
+  (eval echo "\"\$as_me:3948: $NM \\\"conftest.$ac_objext\\\"\"" >&5)
  (eval "$NM \"conftest.$ac_objext\"" 2>conftest.err > conftest.out)
  cat conftest.err >&5
-  (eval echo "\"\$as_me:3948: output\"" >&5)
+  (eval echo "\"\$as_me:3951: output\"" >&5)
  cat conftest.out >&5
  if $GREP 'External.*some_variable' conftest.out > /dev/null; then
    lt_cv_nm_interface="MS dumpbin"
@@ -5151,7 +5154,7 @@ ia64-*-hpux*)
  ;;
 *-*-irix6*)
  # Find out which ABI we are using.
-  echo '#line 5154 "configure"' > conftest.$ac_ext
+  echo '#line 5157 "configure"' > conftest.$ac_ext
  if { { eval echo "\"\$as_me\":${as_lineno-$LINENO}: \"$ac_compile\""; } >&5
  (eval $ac_compile) 2>&5
  ac_status=$?
@@ -6676,11 +6679,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:6679: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:6682: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>conftest.err)
   ac_status=$?
   cat conftest.err >&5
-   echo "$as_me:6683: \$? = $ac_status" >&5
+   echo "$as_me:6686: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s "$ac_outfile"; then
     # The compiler can only warn and ignore the option if not recognized
     # So say no if there are warnings other than the usual output.
@@ -7015,11 +7018,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7018: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7021: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>conftest.err)
   ac_status=$?
   cat conftest.err >&5
-   echo "$as_me:7022: \$? = $ac_status" >&5
+   echo "$as_me:7025: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s "$ac_outfile"; then
     # The compiler can only warn and ignore the option if not recognized
     # So say no if there are warnings other than the usual output.
@@ -7120,11 +7123,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7123: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7126: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>out/conftest.err)
   ac_status=$?
   cat out/conftest.err >&5
-   echo "$as_me:7127: \$? = $ac_status" >&5
+   echo "$as_me:7130: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s out/conftest2.$ac_objext
   then
     # The compiler can only warn and ignore the option if not recognized
@@ -7175,11 +7178,11 @@ else
   -e 's:.*FLAGS}\{0,1\} :&$lt_compiler_flag :; t' \
   -e 's: [^ ]*conftest\.: $lt_compiler_flag&:; t' \
   -e 's:$: $lt_compiler_flag:'`
-   (eval echo "\"\$as_me:7178: $lt_compile\"" >&5)
+   (eval echo "\"\$as_me:7181: $lt_compile\"" >&5)
   (eval "$lt_compile" 2>out/conftest.err)
   ac_status=$?
   cat out/conftest.err >&5
-   echo "$as_me:7182: \$? = $ac_status" >&5
+   echo "$as_me:7185: \$? = $ac_status" >&5
   if (exit $ac_status) && test -s out/conftest2.$ac_objext
   then
     # The compiler can only warn and ignore the option if not recognized
@@ -9555,7 +9558,7 @@ else
  lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  lt_status=$lt_dlunknown
  cat > conftest.$ac_ext <<_LT_EOF
-#line 9558 "configure"
+#line 9561 "configure"
 #include "confdefs.h"

 #if HAVE_DLFCN_H
@@ -9651,7 +9654,7 @@ else
  lt_dlunknown=0; lt_dlno_uscore=1; lt_dlneed_uscore=2
  lt_status=$lt_dlunknown
  cat > conftest.$ac_ext <<_LT_EOF
-#line 9654 "configure"
+#line 9657 "configure"
 #include "confdefs.h"

 #if HAVE_DLFCN_H
@@ -10484,6 +10487,23 @@ if test "x${TCLLIBDIR+set}" != "xset" ; then
  TCLLIBDIR="${TCLLIBDIR}/sqlite3"
 fi

+#########
+# Set up options for running tests.
+#
+# Check whether --enable-test-status was given.
+if test "${enable_test_status+set}" = set; then :
+  enableval=$enable_test_status; use_vt100=$enableval
+else
+  use_vt100=no
+fi
+
+if test $use_vt100 != no; then
+  TSTRNNR_OPTS=--status
+else
+  TSTRNNR_OPTS=
+fi
+
+

 #########
 # Set up an appropriate program prefix
@@ -227,6 +227,18 @@ if test "x${TCLLIBDIR+set}" != "xset" ; then
  TCLLIBDIR="${TCLLIBDIR}/sqlite3"
 fi

+#########
+# Set up options for running tests.
+#
+AC_ARG_ENABLE(test-status, AS_HELP_STRING([--enable-test-status],[Full-screen status of tests]),
+      [use_vt100=$enableval],[use_vt100=no])
+if test $use_vt100 != no; then
+  TSTRNNR_OPTS=--status
+else
+  TSTRNNR_OPTS=
+fi
+AC_SUBST(TSTRNNR_OPTS)
+

 #########
 # Set up an appropriate program prefix
@@ -628,6 +628,9 @@ proc print_categories {lMap} {
        $caseP
        $caseS
        $caseZ
+
+        default:
+          return 1;
      }
      return 0;
    }
@@ -82,7 +82,7 @@ proc get_struct_docs {data names} {
        set current_doc ""
      }
      set subject n/a
-      regexp {^ *([[:alpha:]]*)} $line -> subject
+      regexp {^ *([[:alnum:]_]*)} $line -> subject
      if {[lsearch $names $subject]>=0} {
        set current_header $subject
      } else {
@@ -108,8 +108,11 @@ proc get_tokenizer_docs {data} {
      append res "<dt><b>$line</b></dt><dd><p style=margin-top:0>\n"
      continue
    }
+    if {[regexp {FTS5_TOKENIZER} $line]} {
+      set line </dl><p>
+    }
    if {[regexp {SYNONYM SUPPORT} $line]} {
-      set line "</dl><h3>Synonym Support</h3>"
+      set line "<h3>Synonym Support</h3>"
    }
    if {[string trim $line] == ""} {
      append res "<p>\n"
@@ -308,9 +308,32 @@ struct Fts5PhraseIter {
 **
 **   This API can be quite slow if used with an FTS5 table created with the
 **   "detail=none" or "detail=column" option.
+**
+** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale)
+**   If parameter iCol is less than zero, or greater than or equal to the
+**   number of columns in the table, SQLITE_RANGE is returned.
+**
+**   Otherwise, this function attempts to retrieve the locale associated
+**   with column iCol of the current row. Usually, there is no associated
+**   locale, and output parameters (*pzLocale) and (*pnLocale) are set
+**   to NULL and 0, respectively. However, if the fts5_locale() function
+**   was used to associate a locale with the value when it was inserted
+**   into the fts5 table, then (*pzLocale) is set to point to a nul-terminated
+**   buffer containing the name of the locale in utf-8 encoding. (*pnLocale) 
+**   is set to the size in bytes of the buffer, not including the 
+**   nul-terminator.
+**
+**   If successful, SQLITE_OK is returned. Or, if an error occurs, an
+**   SQLite error code is returned. The final value of the output parameters
+**   is undefined in this case.
+**
+** xTokenize_v2:
+**   Tokenize text using the tokenizer belonging to the FTS5 table. This
+**   API is the same as the xTokenize() API, except that it allows a tokenizer
+**   locale to be specified.
 */
 struct Fts5ExtensionApi {
-  int iVersion;                   /* Currently always set to 3 */
+  int iVersion;                   /* Currently always set to 4 */

  void *(*xUserData)(Fts5Context*);

@@ -352,6 +375,15 @@ struct Fts5ExtensionApi {
      const char **ppToken, int *pnToken
  );
  int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*);
+
+  /* Below this point are iVersion>=4 only */
+  int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn);
+  int (*xTokenize_v2)(Fts5Context*,
+    const char *pText, int nText,      /* Text to tokenize */
+    const char *pLocale, int nLocale,  /* Locale to pass to tokenizer */
+    void *pCtx,                        /* Context passed to xToken() */
+    int (*xToken)(void*, int, const char*, int, int, int)       /* Callback */
+  );
 };

 /* 
@@ -364,6 +396,7 @@ struct Fts5ExtensionApi {
 ** Applications may also register custom tokenizer types. A tokenizer 
 ** is registered by providing fts5 with a populated instance of the 
 ** following structure. All structure methods must be defined, setting
+**
 ** any member of the fts5_tokenizer struct to NULL leads to undefined
 ** behaviour. The structure methods are expected to function as follows:
 **
@@ -372,7 +405,7 @@ struct Fts5ExtensionApi {
 **   A tokenizer instance is required to actually tokenize text.
 **
 **   The first argument passed to this function is a copy of the (void*)
-**   pointer provided by the application when the fts5_tokenizer object
+**   pointer provided by the application when the fts5_tokenizer_v2 object
 **   was registered with FTS5 (the third argument to xCreateTokenizer()). 
 **   The second and third arguments are an array of nul-terminated strings
 **   containing the tokenizer arguments, if any, specified following the
@@ -396,7 +429,7 @@ struct Fts5ExtensionApi {
 **   argument passed to this function is a pointer to an Fts5Tokenizer object
 **   returned by an earlier call to xCreate().
 **
-**   The second argument indicates the reason that FTS5 is requesting
+**   The third argument indicates the reason that FTS5 is requesting
 **   tokenization of the supplied text. This is always one of the following
 **   four values:
 **
@@ -420,6 +453,13 @@ struct Fts5ExtensionApi {
 **            on a columnsize=0 database.  
 **   </ul>
 **
+**   The sixth and seventh arguments passed to xTokenize() - pLocale and
+**   nLocale - are a pointer to a buffer containing the locale to use for
+**   tokenization (e.g. "en_US") and its size in bytes, respectively. The
+**   pLocale buffer is not nul-terminated. pLocale may be passed NULL (in
+**   which case nLocale is always 0) to indicate that the tokenizer should
+**   use its default locale.
+**
 **   For each token in the input string, the supplied callback xToken() must
 **   be invoked. The first argument to it should be a copy of the pointer
 **   passed as the second argument to xTokenize(). The third and fourth
@@ -443,6 +483,29 @@ struct Fts5ExtensionApi {
 **   may abandon the tokenization and return any error code other than
 **   SQLITE_OK or SQLITE_DONE.
 **
+**   If the tokenizer is registered using an fts5_tokenizer_v2 object,
+**   then the xTokenize() method has two additional arguments - pLocale
+**   and nLocale. These specify the locale that the tokenizer should use
+**   for the current request. If pLocale and nLocale are both 0, then the
+**   tokenizer should use its default locale. Otherwise, pLocale points to
+**   an nLocale byte buffer containing the name of the locale to use as utf-8 
+**   text. pLocale is not nul-terminated.
+**
+** FTS5_TOKENIZER
+**
+** There is also an fts5_tokenizer object. This is an older version of
+** fts5_tokenizer_v2. It is similar except that:
+**
+**  <ul>
+**    <li> There is no "iVersion" field, and
+**    <li> The xTokenize() method does not take a locale argument.
+**  </ul>
+**
+** fts5_tokenizer tokenizers should be registered with the xCreateTokenizer()
+** function, instead of xCreateTokenizer_v2(). Tokenizers implementations 
+** registered using either API may be retrieved using both xFindTokenizer()
+** and xFindTokenizer_v2().
+**
 ** SYNONYM SUPPORT
 **
 **   Custom tokenizers may also support synonyms. Consider a case in which a
@@ -551,6 +614,33 @@ struct Fts5ExtensionApi {
 **   inefficient.
 */
 typedef struct Fts5Tokenizer Fts5Tokenizer;
+typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2;
+struct fts5_tokenizer_v2 {
+  int iVersion;             /* Currently always 2 */
+
+  int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
+  void (*xDelete)(Fts5Tokenizer*);
+  int (*xTokenize)(Fts5Tokenizer*, 
+      void *pCtx,
+      int flags,            /* Mask of FTS5_TOKENIZE_* flags */
+      const char *pText, int nText, 
+      const char *pLocale, int nLocale,
+      int (*xToken)(
+        void *pCtx,         /* Copy of 2nd argument to xTokenize() */
+        int tflags,         /* Mask of FTS5_TOKEN_* flags */
+        const char *pToken, /* Pointer to buffer containing token */
+        int nToken,         /* Size of token in bytes */
+        int iStart,         /* Byte offset of token within input text */
+        int iEnd            /* Byte offset of end of token within input text */
+      )
+  );
+};
+
+/*
+** New code should use the fts5_tokenizer_v2 type to define tokenizer
+** implementations. The following type is included for legacy applications
+** that still use it.
+*/
 typedef struct fts5_tokenizer fts5_tokenizer;
 struct fts5_tokenizer {
  int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut);
@@ -570,6 +660,7 @@ struct fts5_tokenizer {
  );
 };

+
 /* Flags that may be passed as the third argument to xTokenize() */
 #define FTS5_TOKENIZE_QUERY     0x0001
 #define FTS5_TOKENIZE_PREFIX    0x0002
@@ -589,7 +680,7 @@ struct fts5_tokenizer {
 */
 typedef struct fts5_api fts5_api;
 struct fts5_api {
-  int iVersion;                   /* Currently always set to 2 */
+  int iVersion;                   /* Currently always set to 3 */

  /* Create a new tokenizer */
  int (*xCreateTokenizer)(
@@ -616,6 +707,25 @@ struct fts5_api {
    fts5_extension_function xFunction,
    void (*xDestroy)(void*)
  );
+
+  /* APIs below this point are only available if iVersion>=3 */
+
+  /* Create a new tokenizer */
+  int (*xCreateTokenizer_v2)(
+    fts5_api *pApi,
+    const char *zName,
+    void *pUserData,
+    fts5_tokenizer_v2 *pTokenizer,
+    void (*xDestroy)(void*)
+  );
+
+  /* Find an existing tokenizer */
+  int (*xFindTokenizer_v2)(
+    fts5_api *pApi,
+    const char *zName,
+    void **ppUserData,
+    fts5_tokenizer_v2 **ppTokenizer
+  );
 };

 /*
@@ -162,10 +162,13 @@ typedef struct Fts5TokenizerConfig Fts5TokenizerConfig;

 struct Fts5TokenizerConfig {
  Fts5Tokenizer *pTok;
-  fts5_tokenizer *pTokApi;
+  fts5_tokenizer_v2 *pApi2;
+  fts5_tokenizer *pApi1;
  const char **azArg;
  int nArg;
  int ePattern;                   /* FTS_PATTERN_XXX constant */
+  const char *pLocale;            /* Current locale to use */
+  int nLocale;                    /* Size of pLocale in bytes */
 };

 /*
@@ -206,6 +209,8 @@ struct Fts5TokenizerConfig {
 **
 **       INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex);
 **
+** bLocale:
+**   Set to true if locale=1 was specified when the table was created.
 */
 struct Fts5Config {
  sqlite3 *db;                    /* Database handle */
@@ -223,10 +228,12 @@ struct Fts5Config {
  char *zContentRowid;            /* "content_rowid=" option value */ 
  int bColumnsize;                /* "columnsize=" option value (dflt==1) */
  int bTokendata;                 /* "tokendata=" option value (dflt==0) */
+  int bLocale;                    /* "locale=" option value (dflt==0) */
  int eDetail;                    /* FTS5_DETAIL_XXX value */
  char *zContentExprlist;
  Fts5TokenizerConfig t;
  int bLock;                      /* True when table is preparing statement */
+  

  /* Values loaded from the %_config table */
  int iVersion;                   /* fts5 file format 'version' */
@@ -292,6 +299,8 @@ int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, int*);

 int sqlite3Fts5ConfigParseRank(const char*, char**, char**);

+void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...);
+
 /*
 ** End of interface to code in fts5_config.c.
 **************************************************************************/
@@ -336,7 +345,7 @@ char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...);
 void sqlite3Fts5Put32(u8*, int);
 int sqlite3Fts5Get32(const u8*);

-#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32)
+#define FTS5_POS2COLUMN(iPos) (int)((iPos >> 32) & 0x7FFFFFFF)
 #define FTS5_POS2OFFSET(iPos) (int)(iPos & 0x7FFFFFFF)

 typedef struct Fts5PoslistReader Fts5PoslistReader;
@@ -627,6 +636,17 @@ Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64);

 int sqlite3Fts5FlushToDisk(Fts5Table*);

+int sqlite3Fts5ExtractText(
+  Fts5Config *pConfig,
+  sqlite3_value *pVal,            /* Value to extract text from */
+  int bContent,                   /* Loaded from content table */
+  int *pbResetTokenizer,          /* OUT: True if ClearLocale() required */
+  const char **ppText,            /* OUT: Pointer to text buffer */
+  int *pnText                     /* OUT: Size of (*ppText) in bytes */
+);
+
+void sqlite3Fts5ClearLocale(Fts5Config *pConfig);
+
 /*
 ** End of interface to code in fts5.c.
 **************************************************************************/
@@ -706,7 +726,7 @@ int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName);
 int sqlite3Fts5DropAll(Fts5Config*);
 int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **);

-int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**);
+int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**, int);
 int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*);
 int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64);

@@ -732,6 +752,9 @@ int sqlite3Fts5StorageOptimize(Fts5Storage *p);
 int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge);
 int sqlite3Fts5StorageReset(Fts5Storage *p);

+void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage*);
+int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel);
+
 /*
 ** End of interface to code in fts5_storage.c.
 **************************************************************************/
@@ -226,6 +226,7 @@ static int fts5HighlightCb(
  return rc;
 }

+
 /*
 ** Implementation of highlight() function.
 */
@@ -256,12 +257,19 @@ static void fts5HighlightFunction(
    sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC);
    rc = SQLITE_OK;
  }else if( ctx.zIn ){
+    const char *pLoc = 0;         /* Locale of column iCol */
+    int nLoc = 0;                 /* Size of pLoc in bytes */
    if( rc==SQLITE_OK ){
      rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter);
    }

    if( rc==SQLITE_OK ){
-      rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
+      rc = pApi->xColumnLocale(pFts, iCol, &pLoc, &nLoc);
+    }
+    if( rc==SQLITE_OK ){
+      rc = pApi->xTokenize_v2(
+          pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx, fts5HighlightCb
+      );
    }
    if( ctx.bOpen ){
      fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1);
@@ -458,6 +466,8 @@ static void fts5SnippetFunction(
  memset(&sFinder, 0, sizeof(Fts5SFinder));
  for(i=0; i<nCol; i++){
    if( iCol<0 || iCol==i ){
+      const char *pLoc = 0;       /* Locale of column iCol */
+      int nLoc = 0;               /* Size of pLoc in bytes */
      int nDoc;
      int nDocsize;
      int ii;
@@ -465,8 +475,10 @@ static void fts5SnippetFunction(
      sFinder.nFirst = 0;
      rc = pApi->xColumnText(pFts, i, &sFinder.zDoc, &nDoc);
      if( rc!=SQLITE_OK ) break;
-      rc = pApi->xTokenize(pFts, 
-          sFinder.zDoc, nDoc, (void*)&sFinder,fts5SentenceFinderCb
+      rc = pApi->xColumnLocale(pFts, i, &pLoc, &nLoc);
+      if( rc!=SQLITE_OK ) break;
+      rc = pApi->xTokenize_v2(pFts, 
+          sFinder.zDoc, nDoc, pLoc, nLoc, (void*)&sFinder, fts5SentenceFinderCb
      );
      if( rc!=SQLITE_OK ) break;
      rc = pApi->xColumnSize(pFts, i, &nDocsize);
@@ -524,6 +536,9 @@ static void fts5SnippetFunction(
    rc = pApi->xColumnSize(pFts, iBestCol, &nColSize);
  }
  if( ctx.zIn ){
+    const char *pLoc = 0;         /* Locale of column iBestCol */
+    int nLoc = 0;                 /* Bytes in pLoc */
+
    if( rc==SQLITE_OK ){
      rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter);
    }
@@ -542,7 +557,12 @@ static void fts5SnippetFunction(
    }

    if( rc==SQLITE_OK ){
-      rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb);
+      rc = pApi->xColumnLocale(pFts, iBestCol, &pLoc, &nLoc);
+    }
+    if( rc==SQLITE_OK ){
+      rc = pApi->xTokenize_v2(
+          pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx,fts5HighlightCb
+      );
    }
    if( ctx.bOpen ){
      fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1);
@@ -726,6 +746,53 @@ static void fts5Bm25Function(
  }
 }

+/*
+** Implementation of fts5_get_locale() function.
+*/
+static void fts5GetLocaleFunction(
+  const Fts5ExtensionApi *pApi,   /* API offered by current FTS version */
+  Fts5Context *pFts,              /* First arg to pass to pApi functions */
+  sqlite3_context *pCtx,          /* Context for returning result/error */
+  int nVal,                       /* Number of values in apVal[] array */
+  sqlite3_value **apVal           /* Array of trailing arguments */
+){
+  int iCol = 0;
+  int eType = 0;
+  int rc = SQLITE_OK;
+  const char *zLocale = 0;
+  int nLocale = 0;
+
+  /* xColumnLocale() must be available */
+  assert( pApi->iVersion>=4 );
+
+  if( nVal!=1 ){
+    const char *z = "wrong number of arguments to function fts5_get_locale()";
+    sqlite3_result_error(pCtx, z, -1);
+    return;
+  }
+
+  eType = sqlite3_value_numeric_type(apVal[0]);
+  if( eType!=SQLITE_INTEGER ){
+    const char *z = "non-integer argument passed to function fts5_get_locale()";
+    sqlite3_result_error(pCtx, z, -1);
+    return;
+  }
+
+  iCol = sqlite3_value_int(apVal[0]);
+  if( iCol<0 || iCol>=pApi->xColumnCount(pFts) ){
+    sqlite3_result_error_code(pCtx, SQLITE_RANGE);
+    return;
+  }
+
+  rc = pApi->xColumnLocale(pFts, iCol, &zLocale, &nLocale);
+  if( rc!=SQLITE_OK ){
+    sqlite3_result_error_code(pCtx, rc);
+    return;
+  }
+
+  sqlite3_result_text(pCtx, zLocale, nLocale, SQLITE_TRANSIENT);
+}
+
 int sqlite3Fts5AuxInit(fts5_api *pApi){
  struct Builtin {
    const char *zFunc;            /* Function name (nul-terminated) */
@@ -733,9 +800,10 @@ int sqlite3Fts5AuxInit(fts5_api *pApi){
    fts5_extension_function xFunc;/* Callback function */
    void (*xDestroy)(void*);      /* Destructor function */
  } aBuiltin [] = {
-    { "snippet",   0, fts5SnippetFunction, 0 },
-    { "highlight", 0, fts5HighlightFunction, 0 },
-    { "bm25",      0, fts5Bm25Function,    0 },
+    { "snippet",         0, fts5SnippetFunction,   0 },
+    { "highlight",       0, fts5HighlightFunction, 0 },
+    { "bm25",            0, fts5Bm25Function,      0 },
+    { "fts5_get_locale", 0, fts5GetLocaleFunction, 0 },
  };
  int rc = SQLITE_OK;             /* Return code */
  int i;                          /* To iterate through builtin functions */
@@ -380,6 +380,16 @@ static int fts5ConfigParseSpecial(
    return rc;
  }

+  if( sqlite3_strnicmp("locale", zCmd, nCmd)==0 ){
+    if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){
+      *pzErr = sqlite3_mprintf("malformed locale=... directive");
+      rc = SQLITE_ERROR;
+    }else{
+      pConfig->bLocale = (zArg[0]=='1');
+    }
+    return rc;
+  }
+
  if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){
    const Fts5Enum aDetail[] = {
      { "none", FTS5_DETAIL_NONE },
@@ -669,7 +679,11 @@ void sqlite3Fts5ConfigFree(Fts5Config *pConfig){
  if( pConfig ){
    int i;
    if( pConfig->t.pTok ){
-      pConfig->t.pTokApi->xDelete(pConfig->t.pTok);
+      if( pConfig->t.pApi1 ){
+        pConfig->t.pApi1->xDelete(pConfig->t.pTok);
+      }else{
+        pConfig->t.pApi2->xDelete(pConfig->t.pTok);
+      }
    }
    sqlite3_free((char*)pConfig->t.azArg);
    sqlite3_free(pConfig->zDb);
@@ -752,9 +766,15 @@ int sqlite3Fts5Tokenize(
      rc = sqlite3Fts5LoadTokenizer(pConfig);
    }
    if( rc==SQLITE_OK ){
-      rc = pConfig->t.pTokApi->xTokenize(
-          pConfig->t.pTok, pCtx, flags, pText, nText, xToken
-      );
+      if( pConfig->t.pApi1 ){
+        rc = pConfig->t.pApi1->xTokenize(
+            pConfig->t.pTok, pCtx, flags, pText, nText, xToken
+        );
+      }else{
+        rc = pConfig->t.pApi2->xTokenize(pConfig->t.pTok, pCtx, flags, 
+            pText, nText, pConfig->t.pLocale, pConfig->t.nLocale, xToken
+        );
+      }
    }
  }
  return rc;
@@ -1011,13 +1031,10 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
   && iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE
  ){
    rc = SQLITE_ERROR;
-    if( pConfig->pzErrmsg ){
-      assert( 0==*pConfig->pzErrmsg );
-      *pConfig->pzErrmsg = sqlite3_mprintf("invalid fts5 file format "
-          "(found %d, expected %d or %d) - run 'rebuild'",
-          iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE
-      );
-    }
+    sqlite3Fts5ConfigErrmsg(pConfig, "invalid fts5 file format "
+        "(found %d, expected %d or %d) - run 'rebuild'",
+        iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE
+    );
  }else{
    pConfig->iVersion = iVersion;
  }
@@ -1027,3 +1044,26 @@ int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
  }
  return rc;
 }
+
+/*
+** Set (*pConfig->pzErrmsg) to point to an sqlite3_malloc()ed buffer 
+** containing the error message created using printf() style formatting
+** string zFmt and its trailing arguments.
+*/
+void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){
+  va_list ap;                     /* ... printf arguments */
+  char *zMsg = 0;
+
+  va_start(ap, zFmt);
+  zMsg = sqlite3_vmprintf(zFmt, ap);
+  if( pConfig->pzErrmsg ){
+    assert( *pConfig->pzErrmsg==0 );
+    *pConfig->pzErrmsg = zMsg;
+  }else{
+    sqlite3_free(zMsg);
+  }
+
+  va_end(ap);
+}
+
+
@@ -286,11 +286,12 @@ int sqlite3Fts5ExprNew(
  }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF );
  sqlite3Fts5ParserFree(pEngine, fts5ParseFree);

+  assert( sParse.pExpr || sParse.rc!=SQLITE_OK );
  assert_expr_depth_ok(sParse.rc, sParse.pExpr);

  /* If the LHS of the MATCH expression was a user column, apply the
  ** implicit column-filter.  */
-  if( iCol<pConfig->nCol && sParse.pExpr && sParse.rc==SQLITE_OK ){
+  if( sParse.rc==SQLITE_OK && iCol<pConfig->nCol ){
    int n = sizeof(Fts5Colset);
    Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n);
    if( pColset ){
@@ -307,15 +308,7 @@ int sqlite3Fts5ExprNew(
      sParse.rc = SQLITE_NOMEM;
      sqlite3Fts5ParseNodeFree(sParse.pExpr);
    }else{
-      if( !sParse.pExpr ){
-        const int nByte = sizeof(Fts5ExprNode);
-        pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&sParse.rc, nByte);
-        if( pNew->pRoot ){
-          pNew->pRoot->bEof = 1;
-        }
-      }else{
-        pNew->pRoot = sParse.pExpr;
-      }
+      pNew->pRoot = sParse.pExpr;
      pNew->pIndex = 0;
      pNew->pConfig = pConfig;
      pNew->apExprPhrase = sParse.apPhrase;
@@ -1133,7 +1126,7 @@ static int fts5ExprNodeTest_STRING(
          }
        }else{
          Fts5IndexIter *pIter = pPhrase->aTerm[j].pIter;
-          if( pIter->iRowid==iLast || pIter->bEof ) continue;
+          if( pIter->iRowid==iLast ) continue;
          bMatch = 0;
          if( fts5ExprAdvanceto(pIter, bDesc, &iLast, &rc, &pNode->bEof) ){
            return rc;
@@ -1655,9 +1648,6 @@ Fts5ExprNearset *sqlite3Fts5ParseNearset(
  Fts5ExprNearset *pRet = 0;

  if( pParse->rc==SQLITE_OK ){
-    if( pPhrase==0 ){
-      return pNear;
-    }
    if( pNear==0 ){
      sqlite3_int64 nByte;
      nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*);
@@ -16,13 +16,40 @@

 #include "fts5Int.h"

+/*
+** pSavedRow:
+**   SQL statement FTS5_STMT_LOOKUP2 is a copy of FTS5_STMT_LOOKUP, it 
+**   does a by-rowid lookup to retrieve a single row from the %_content 
+**   table or equivalent external-content table/view.
+**
+**   However, FTS5_STMT_LOOKUP2 is only used when retrieving the original
+**   values for a row being UPDATEd. In that case, the SQL statement is
+**   not reset and pSavedRow is set to point at it. This is so that the
+**   insert operation that follows the delete may access the original 
+**   row values for any new values for which sqlite3_value_nochange() returns
+**   true. i.e. if the user executes:
+**
+**        CREATE VIRTUAL TABLE ft USING fts5(a, b, c, locale=1);
+**        ...
+**        UPDATE fts SET a=?, b=? WHERE rowid=?;
+**
+**   then the value passed to the xUpdate() method of this table as the 
+**   new.c value is an sqlite3_value_nochange() value. So in this case it
+**   must be read from the saved row stored in Fts5Storage.pSavedRow.
+**
+**   This is necessary - using sqlite3_value_nochange() instead of just having
+**   SQLite pass the original value back via xUpdate() - so as not to discard
+**   any locale information associated with such values.
+**
+*/
 struct Fts5Storage {
  Fts5Config *pConfig;
  Fts5Index *pIndex;
  int bTotalsValid;               /* True if nTotalRow/aTotalSize[] are valid */
  i64 nTotalRow;                  /* Total number of rows in FTS table */
  i64 *aTotalSize;                /* Total sizes of each column */ 
-  sqlite3_stmt *aStmt[11];
+  sqlite3_stmt *pSavedRow;
+  sqlite3_stmt *aStmt[12];
 };


@@ -36,14 +63,15 @@ struct Fts5Storage {
 # error "FTS5_STMT_LOOKUP mismatch" 
 #endif

-#define FTS5_STMT_INSERT_CONTENT  3
-#define FTS5_STMT_REPLACE_CONTENT 4
-#define FTS5_STMT_DELETE_CONTENT  5
-#define FTS5_STMT_REPLACE_DOCSIZE  6
-#define FTS5_STMT_DELETE_DOCSIZE  7
-#define FTS5_STMT_LOOKUP_DOCSIZE  8
-#define FTS5_STMT_REPLACE_CONFIG 9
-#define FTS5_STMT_SCAN 10
+#define FTS5_STMT_LOOKUP2         3
+#define FTS5_STMT_INSERT_CONTENT  4
+#define FTS5_STMT_REPLACE_CONTENT 5
+#define FTS5_STMT_DELETE_CONTENT  6
+#define FTS5_STMT_REPLACE_DOCSIZE 7
+#define FTS5_STMT_DELETE_DOCSIZE  8
+#define FTS5_STMT_LOOKUP_DOCSIZE  9
+#define FTS5_STMT_REPLACE_CONFIG 10
+#define FTS5_STMT_SCAN           11

 /*
 ** Prepare the two insert statements - Fts5Storage.pInsertContent and
@@ -73,6 +101,7 @@ static int fts5StorageGetStmt(
      "SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC",
      "SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC",
      "SELECT %s FROM %s T WHERE T.%Q=?",               /* LOOKUP  */
+      "SELECT %s FROM %s T WHERE T.%Q=?",               /* LOOKUP2  */

      "INSERT INTO %Q.'%q_content' VALUES(%s)",         /* INSERT_CONTENT  */
      "REPLACE INTO %Q.'%q_content' VALUES(%s)",        /* REPLACE_CONTENT */
@@ -88,6 +117,8 @@ static int fts5StorageGetStmt(
    Fts5Config *pC = p->pConfig;
    char *zSql = 0;

+    assert( ArraySize(azStmt)==ArraySize(p->aStmt) );
+
    switch( eStmt ){
      case FTS5_STMT_SCAN:
        zSql = sqlite3_mprintf(azStmt[eStmt], 
@@ -104,6 +135,7 @@ static int fts5StorageGetStmt(
        break;

      case FTS5_STMT_LOOKUP:
+      case FTS5_STMT_LOOKUP2:
        zSql = sqlite3_mprintf(azStmt[eStmt], 
            pC->zContentExprlist, pC->zContent, pC->zContentRowid
        );
@@ -150,7 +182,7 @@ static int fts5StorageGetStmt(
      rc = SQLITE_NOMEM;
    }else{
      int f = SQLITE_PREPARE_PERSISTENT;
-      if( eStmt>FTS5_STMT_LOOKUP ) f |= SQLITE_PREPARE_NO_VTAB;
+      if( eStmt>FTS5_STMT_LOOKUP2 ) f |= SQLITE_PREPARE_NO_VTAB;
      p->pConfig->bLock++;
      rc = sqlite3_prepare_v3(pC->db, zSql, -1, f, &p->aStmt[eStmt], 0);
      p->pConfig->bLock--;
@@ -399,15 +431,49 @@ static int fts5StorageInsertCallback(
  return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken);
 }

+/*
+** This function is used as part of an UPDATE statement that modifies the
+** rowid of a row. In that case, this function is called first to set
+** Fts5Storage.pSavedRow to point to a statement that may be used to 
+** access the original values of the row being deleted - iDel.
+**
+** SQLITE_OK is returned if successful, or an SQLite error code otherwise.
+** It is not considered an error if row iDel does not exist. In this case
+** pSavedRow is not set and SQLITE_OK returned.
+*/
+int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel){
+  int rc = SQLITE_OK;
+  sqlite3_stmt *pSeek = 0;
+
+  assert( p->pSavedRow==0 );
+  rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+1, &pSeek, 0);
+  if( rc==SQLITE_OK ){
+    sqlite3_bind_int64(pSeek, 1, iDel);
+    if( sqlite3_step(pSeek)!=SQLITE_ROW ){
+      rc = sqlite3_reset(pSeek);
+    }else{
+      p->pSavedRow = pSeek;
+    }
+  }
+
+  return rc;
+}
+
 /*
 ** If a row with rowid iDel is present in the %_content table, add the
 ** delete-markers to the FTS index necessary to delete it. Do not actually
 ** remove the %_content row at this time though.
+**
+** If parameter bSaveRow is true, then Fts5Storage.pSavedRow is left
+** pointing to a statement (FTS5_STMT_LOOKUP2) that may be used to access
+** the original values of the row being deleted. This is used by UPDATE
+** statements.
 */
 static int fts5StorageDeleteFromIndex(
  Fts5Storage *p, 
  i64 iDel, 
-  sqlite3_value **apVal
+  sqlite3_value **apVal,
+  int bSaveRow                    /* True to set pSavedRow */
 ){
  Fts5Config *pConfig = p->pConfig;
  sqlite3_stmt *pSeek = 0;        /* SELECT to read row iDel from %_data */
@@ -416,12 +482,21 @@ static int fts5StorageDeleteFromIndex(
  int iCol;
  Fts5InsertCtx ctx;

+  assert( bSaveRow==0 || apVal==0 );
+  assert( bSaveRow==0 || bSaveRow==1 );
+  assert( FTS5_STMT_LOOKUP2==FTS5_STMT_LOOKUP+1 );
+
  if( apVal==0 ){
-    rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0);
-    if( rc!=SQLITE_OK ) return rc;
-    sqlite3_bind_int64(pSeek, 1, iDel);
-    if( sqlite3_step(pSeek)!=SQLITE_ROW ){
-      return sqlite3_reset(pSeek);
+    if( p->pSavedRow && bSaveRow ){
+      pSeek = p->pSavedRow;
+      p->pSavedRow = 0;
+    }else{
+      rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+bSaveRow, &pSeek, 0);
+      if( rc!=SQLITE_OK ) return rc;
+      sqlite3_bind_int64(pSeek, 1, iDel);
+      if( sqlite3_step(pSeek)!=SQLITE_ROW ){
+        return sqlite3_reset(pSeek);
+      }
    }
  }

@@ -429,26 +504,32 @@ static int fts5StorageDeleteFromIndex(
  ctx.iCol = -1;
  for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){
    if( pConfig->abUnindexed[iCol-1]==0 ){
-      const char *zText;
-      int nText;
+      sqlite3_value *pVal = 0;
+      const char *pText = 0;
+      int nText = 0;
+      int bReset = 0;
+
      assert( pSeek==0 || apVal==0 );
      assert( pSeek!=0 || apVal!=0 );
      if( pSeek ){
-        zText = (const char*)sqlite3_column_text(pSeek, iCol);
-        nText = sqlite3_column_bytes(pSeek, iCol);
-      }else if( ALWAYS(apVal) ){
-        zText = (const char*)sqlite3_value_text(apVal[iCol-1]);
-        nText = sqlite3_value_bytes(apVal[iCol-1]);
+        pVal = sqlite3_column_value(pSeek, iCol);
      }else{
-        continue;
+        pVal = apVal[iCol-1];
      }
-      ctx.szCol = 0;
-      rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, 
-          zText, nText, (void*)&ctx, fts5StorageInsertCallback
+
+      rc = sqlite3Fts5ExtractText(
+          pConfig, pVal, pSeek!=0, &bReset, &pText, &nText
      );
-      p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
-      if( p->aTotalSize[iCol-1]<0 && rc==SQLITE_OK ){
-        rc = FTS5_CORRUPT;
+      if( rc==SQLITE_OK ){
+        ctx.szCol = 0;
+        rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, 
+            pText, nText, (void*)&ctx, fts5StorageInsertCallback
+        );
+        p->aTotalSize[iCol-1] -= (i64)ctx.szCol;
+        if( rc==SQLITE_OK && p->aTotalSize[iCol-1]<0 ){
+          rc = FTS5_CORRUPT;
+        }
+        if( bReset ) sqlite3Fts5ClearLocale(pConfig);
      }
    }
  }
@@ -458,11 +539,29 @@ static int fts5StorageDeleteFromIndex(
    p->nTotalRow--;
  }

-  rc2 = sqlite3_reset(pSeek);
-  if( rc==SQLITE_OK ) rc = rc2;
+  if( rc==SQLITE_OK && bSaveRow ){
+    assert( p->pSavedRow==0 );
+    p->pSavedRow = pSeek;
+  }else{
+    rc2 = sqlite3_reset(pSeek);
+    if( rc==SQLITE_OK ) rc = rc2;
+  }
  return rc;
 }

+/*
+** Reset any saved statement pSavedRow. Zero pSavedRow as well. This
+** should be called by the xUpdate() method of the fts5 table before 
+** returning from any operation that may have set Fts5Storage.pSavedRow.
+*/
+void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage *pStorage){
+  assert( pStorage->pSavedRow==0 
+       || pStorage->pSavedRow==pStorage->aStmt[FTS5_STMT_LOOKUP2] 
+  );
+  sqlite3_reset(pStorage->pSavedRow);
+  pStorage->pSavedRow = 0;
+}
+
 /*
 ** This function is called to process a DELETE on a contentless_delete=1
 ** table. It adds the tombstone required to delete the entry with rowid 
@@ -519,12 +618,12 @@ static int fts5StorageInsertDocsize(
        rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin);
        sqlite3_bind_int64(pReplace, 3, iOrigin);
      }
-      if( rc==SQLITE_OK ){
-        sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
-        sqlite3_step(pReplace);
-        rc = sqlite3_reset(pReplace);
-        sqlite3_bind_null(pReplace, 2);
-      }
+    }
+    if( rc==SQLITE_OK ){
+      sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC);
+      sqlite3_step(pReplace);
+      rc = sqlite3_reset(pReplace);
+      sqlite3_bind_null(pReplace, 2);
    }
  }
  return rc;
@@ -578,7 +677,12 @@ static int fts5StorageSaveTotals(Fts5Storage *p){
 /*
 ** Remove a row from the FTS table.
 */
-int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
+int sqlite3Fts5StorageDelete(
+  Fts5Storage *p,                 /* Storage object */
+  i64 iDel,                       /* Rowid to delete from table */
+  sqlite3_value **apVal,          /* Optional - values to remove from index */
+  int bSaveRow                    /* If true, set pSavedRow for deleted row */
+){
  Fts5Config *pConfig = p->pConfig;
  int rc;
  sqlite3_stmt *pDel = 0;
@@ -595,7 +699,7 @@ int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){
    if( p->pConfig->bContentlessDelete ){
      rc = fts5StorageContentlessDelete(p, iDel);
    }else{
-      rc = fts5StorageDeleteFromIndex(p, iDel, apVal);
+      rc = fts5StorageDeleteFromIndex(p, iDel, apVal, bSaveRow);
    }
  }

@@ -684,14 +788,21 @@ int sqlite3Fts5StorageRebuild(Fts5Storage *p){
    for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
      ctx.szCol = 0;
      if( pConfig->abUnindexed[ctx.iCol]==0 ){
-        const char *zText = (const char*)sqlite3_column_text(pScan, ctx.iCol+1);
-        int nText = sqlite3_column_bytes(pScan, ctx.iCol+1);
-        rc = sqlite3Fts5Tokenize(pConfig, 
-            FTS5_TOKENIZE_DOCUMENT,
-            zText, nText,
-            (void*)&ctx,
-            fts5StorageInsertCallback
-        );
+        int bReset = 0;           /* True if tokenizer locale must be reset */
+        int nText = 0;            /* Size of pText in bytes */
+        const char *pText = 0;    /* Pointer to buffer containing text value */
+        sqlite3_value *pVal = sqlite3_column_value(pScan, ctx.iCol+1);
+
+        rc = sqlite3Fts5ExtractText(pConfig, pVal, 1, &bReset, &pText, &nText);
+        if( rc==SQLITE_OK ){
+          rc = sqlite3Fts5Tokenize(pConfig, 
+              FTS5_TOKENIZE_DOCUMENT,
+              pText, nText,
+              (void*)&ctx,
+              fts5StorageInsertCallback
+          );
+          if( bReset ) sqlite3Fts5ClearLocale(pConfig);
+        }
      }
      sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
      p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
@@ -775,7 +886,31 @@ int sqlite3Fts5StorageContentInsert(
    int i;                        /* Counter variable */
    rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0);
    for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){
-      rc = sqlite3_bind_value(pInsert, i, apVal[i]);
+      sqlite3_value *pVal = apVal[i];
+      if( sqlite3_value_nochange(pVal) && p->pSavedRow ){
+        /* This is an UPDATE statement, and column (i-2) was not modified.
+        ** Retrieve the value from Fts5Storage.pSavedRow instead. */
+        pVal = sqlite3_column_value(p->pSavedRow, i-1);
+      }else if( sqlite3_value_subtype(pVal)==FTS5_LOCALE_SUBTYPE ){
+        assert( pConfig->bLocale );
+        assert( i>1 );
+        if( pConfig->abUnindexed[i-2] ){
+          /* At attempt to insert an fts5_locale() value into an UNINDEXED
+          ** column. Strip the locale away and just bind the text.  */
+          const char *pText = 0;
+          int nText = 0;
+          rc = sqlite3Fts5ExtractText(pConfig, pVal, 0, 0, &pText, &nText);
+          sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT);
+        }else{
+          const u8 *pBlob = (const u8*)sqlite3_value_blob(pVal);
+          int nBlob = sqlite3_value_bytes(pVal);
+          assert( nBlob>4 );
+          sqlite3_bind_blob(pInsert, i, pBlob+4, nBlob-4, SQLITE_TRANSIENT);
+        }
+        continue;
+      }
+
+      rc = sqlite3_bind_value(pInsert, i, pVal);
    }
    if( rc==SQLITE_OK ){
      sqlite3_step(pInsert);
@@ -810,14 +945,24 @@ int sqlite3Fts5StorageIndexInsert(
  for(ctx.iCol=0; rc==SQLITE_OK && ctx.iCol<pConfig->nCol; ctx.iCol++){
    ctx.szCol = 0;
    if( pConfig->abUnindexed[ctx.iCol]==0 ){
-      const char *zText = (const char*)sqlite3_value_text(apVal[ctx.iCol+2]);
-      int nText = sqlite3_value_bytes(apVal[ctx.iCol+2]);
-      rc = sqlite3Fts5Tokenize(pConfig, 
-          FTS5_TOKENIZE_DOCUMENT,
-          zText, nText,
-          (void*)&ctx,
-          fts5StorageInsertCallback
-      );
+      int bReset = 0;             /* True if tokenizer locale must be reset */
+      int nText = 0;              /* Size of pText in bytes */
+      const char *pText = 0;      /* Pointer to buffer containing text value */
+      sqlite3_value *pVal = apVal[ctx.iCol+2];
+      int bDisk = 0;
+      if( p->pSavedRow && sqlite3_value_nochange(pVal) ){
+        pVal = sqlite3_column_value(p->pSavedRow, ctx.iCol+1);
+        bDisk = 1;
+      }
+      rc = sqlite3Fts5ExtractText(pConfig, pVal, bDisk, &bReset, &pText,&nText);
+      if( rc==SQLITE_OK ){
+        assert( bReset==0 || pConfig->bLocale );
+        rc = sqlite3Fts5Tokenize(pConfig, 
+            FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx,
+            fts5StorageInsertCallback
+        );
+        if( bReset ) sqlite3Fts5ClearLocale(pConfig);
+      }
    }
    sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol);
    p->aTotalSize[ctx.iCol] += (i64)ctx.szCol;
@@ -988,14 +1133,22 @@ int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){
            rc = sqlite3Fts5TermsetNew(&ctx.pTermset);
          }
          if( rc==SQLITE_OK ){
-            const char *zText = (const char*)sqlite3_column_text(pScan, i+1);
-            int nText = sqlite3_column_bytes(pScan, i+1);
-            rc = sqlite3Fts5Tokenize(pConfig, 
-                FTS5_TOKENIZE_DOCUMENT,
-                zText, nText,
-                (void*)&ctx,
-                fts5StorageIntegrityCallback
+            int bReset = 0;        /* True if tokenizer locale must be reset */
+            int nText = 0;         /* Size of pText in bytes */
+            const char *pText = 0; /* Pointer to buffer containing text value */
+
+            rc = sqlite3Fts5ExtractText(pConfig, 
+                sqlite3_column_value(pScan, i+1), 1, &bReset, &pText, &nText
            );
+            if( rc==SQLITE_OK ){
+              rc = sqlite3Fts5Tokenize(pConfig, 
+                  FTS5_TOKENIZE_DOCUMENT,
+                  pText, nText,
+                  (void*)&ctx,
+                  fts5StorageIntegrityCallback
+              );
+              if( bReset ) sqlite3Fts5ClearLocale(pConfig);
+            }
          }
          if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){
            rc = FTS5_CORRUPT;
@@ -240,6 +240,7 @@ static int SQLITE_TCLAPI xF5tApi(

    { "xQueryToken",       2, "IPHRASE ITERM" },      /* 18 */
    { "xInstToken",        2, "IDX ITERM" },          /* 19 */
+    { "xColumnLocale",     1, "COL" },                /* 20 */
    { 0, 0, 0}
  };

@@ -528,6 +529,20 @@ static int SQLITE_TCLAPI xF5tApi(
      break;
    }

+    CASE(20, "xColumnLocale") {
+      const char *z = 0;
+      int n = 0;
+      int iCol;
+      if( Tcl_GetIntFromObj(interp, objv[2], &iCol) ){
+        return TCL_ERROR;
+      }
+      rc = p->pApi->xColumnLocale(p->pFts, iCol, &z, &n);
+      if( rc==SQLITE_OK && z ){
+        Tcl_SetObjResult(interp, Tcl_NewStringObj(z, n));
+      }
+      break;
+    }
+
    default: 
      assert( 0 );
      break;
@@ -796,18 +811,32 @@ typedef struct F5tTokenizerInstance F5tTokenizerInstance;
 struct F5tTokenizerContext {
  void *pCtx;
  int (*xToken)(void*, int, const char*, int, int, int);
+  F5tTokenizerInstance *pInst;
 };

 struct F5tTokenizerModule {
  Tcl_Interp *interp;
  Tcl_Obj *pScript;
+  void *pParentCtx;
+  fts5_tokenizer_v2 parent_v2;
+  fts5_tokenizer parent;
  F5tTokenizerContext *pContext;
 };

+/*
+** zLocale:
+**   Within a call to xTokenize_v2(), pLocale/nLocale store the locale
+**   passed to the call by fts5. This can be retrieved by a Tcl tokenize 
+**   script using [sqlite3_fts5_locale].
+*/
 struct F5tTokenizerInstance {
  Tcl_Interp *interp;
  Tcl_Obj *pScript;
+  F5tTokenizerModule *pModule;
+  Fts5Tokenizer *pParent;
  F5tTokenizerContext *pContext;
+  const char *pLocale;
+  int nLocale;
 };

 static int f5tTokenizerCreate(
@@ -816,11 +845,20 @@ static int f5tTokenizerCreate(
  int nArg, 
  Fts5Tokenizer **ppOut
 ){
+  Fts5Tokenizer *pParent = 0;
  F5tTokenizerModule *pMod = (F5tTokenizerModule*)pCtx;
  Tcl_Obj *pEval;
  int rc = TCL_OK;
  int i;

+  assert( pMod->parent_v2.xCreate==0 || pMod->parent.xCreate==0 );
+  if( pMod->parent_v2.xCreate ){
+    rc = pMod->parent_v2.xCreate(pMod->pParentCtx, 0, 0, &pParent);
+  }
+  if( pMod->parent.xCreate ){
+    rc = pMod->parent.xCreate(pMod->pParentCtx, 0, 0, &pParent);
+  }
+
  pEval = Tcl_DuplicateObj(pMod->pScript);
  Tcl_IncrRefCount(pEval);
  for(i=0; rc==TCL_OK && i<nArg; i++){
@@ -840,6 +878,8 @@ static int f5tTokenizerCreate(
    pInst->interp = pMod->interp;
    pInst->pScript = Tcl_GetObjResult(pMod->interp);
    pInst->pContext = pMod->pContext;
+    pInst->pParent = pParent;
+    pInst->pModule = pMod;
    Tcl_IncrRefCount(pInst->pScript);
    *ppOut = (Fts5Tokenizer*)pInst;
  }
@@ -850,11 +890,21 @@ static int f5tTokenizerCreate(

 static void f5tTokenizerDelete(Fts5Tokenizer *p){
  F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
-  Tcl_DecrRefCount(pInst->pScript);
-  ckfree((char *)pInst);
+  if( pInst ){
+    if( pInst->pParent ){
+      if( pInst->pModule->parent_v2.xDelete ){
+        pInst->pModule->parent_v2.xDelete(pInst->pParent);
+      }else{
+        pInst->pModule->parent.xDelete(pInst->pParent);
+      }
+    }
+    Tcl_DecrRefCount(pInst->pScript);
+    ckfree((char *)pInst);
+  }
 }

-static int f5tTokenizerTokenize(
+
+static int f5tTokenizerReallyTokenize(
  Fts5Tokenizer *p, 
  void *pCtx,
  int flags,
@@ -862,6 +912,7 @@ static int f5tTokenizerTokenize(
  int (*xToken)(void*, int, const char*, int, int, int)
 ){
  F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
+  F5tTokenizerInstance *pOldInst = 0;
  void *pOldCtx;
  int (*xOldToken)(void*, int, const char*, int, int, int);
  Tcl_Obj *pEval;
@@ -870,9 +921,11 @@ static int f5tTokenizerTokenize(

  pOldCtx = pInst->pContext->pCtx;
  xOldToken = pInst->pContext->xToken;
+  pOldInst = pInst->pContext->pInst;

  pInst->pContext->pCtx = pCtx;
  pInst->pContext->xToken = xToken;
+  pInst->pContext->pInst = pInst;

  assert( 
      flags==FTS5_TOKENIZE_DOCUMENT
@@ -908,9 +961,105 @@ static int f5tTokenizerTokenize(

  pInst->pContext->pCtx = pOldCtx;
  pInst->pContext->xToken = xOldToken;
+  pInst->pContext->pInst = pOldInst;
  return rc;
 }

+typedef struct CallbackCtx CallbackCtx;
+struct CallbackCtx {
+  Fts5Tokenizer *p;
+  void *pCtx;
+  int flags;
+  int (*xToken)(void*, int, const char*, int, int, int);
+};
+
+static int f5tTokenizeCallback(
+  void *pCtx, 
+  int tflags, 
+  const char *z, int n, 
+  int iStart, int iEnd
+){
+  CallbackCtx *p = (CallbackCtx*)pCtx;
+  return f5tTokenizerReallyTokenize(p->p, p->pCtx, p->flags, z, n, p->xToken);
+}
+
+static int f5tTokenizerTokenize_v2(
+  Fts5Tokenizer *p, 
+  void *pCtx,
+  int flags,
+  const char *pText, int nText, 
+  const char *pLoc, int nLoc, 
+  int (*xToken)(void*, int, const char*, int, int, int)
+){
+  int rc = SQLITE_OK;
+  F5tTokenizerInstance *pInst = (F5tTokenizerInstance*)p;
+
+  pInst->pLocale = pLoc;
+  pInst->nLocale = nLoc;
+
+  if( pInst->pParent ){
+    CallbackCtx ctx;
+    ctx.p = p;
+    ctx.pCtx = pCtx;
+    ctx.flags = flags;
+    ctx.xToken = xToken;
+    if( pInst->pModule->parent_v2.xTokenize ){
+      rc = pInst->pModule->parent_v2.xTokenize(
+          pInst->pParent, (void*)&ctx, flags, pText, nText, 
+          pLoc, nLoc, f5tTokenizeCallback
+      );
+    }else{
+      rc = pInst->pModule->parent.xTokenize(
+          pInst->pParent, (void*)&ctx, flags, pText, nText, f5tTokenizeCallback
+      );
+    }
+  }else{
+    rc = f5tTokenizerReallyTokenize(p, pCtx, flags, pText, nText, xToken);
+  }
+
+  pInst->pLocale = 0;
+  pInst->nLocale = 0;
+  return rc;
+}
+static int f5tTokenizerTokenize(
+  Fts5Tokenizer *p, 
+  void *pCtx,
+  int flags,
+  const char *pText, int nText, 
+  int (*xToken)(void*, int, const char*, int, int, int)
+){
+  return f5tTokenizerTokenize_v2(p, pCtx, flags, pText, nText, 0, 0, xToken);
+}
+
+/*
+** sqlite3_fts5_locale 
+*/
+static int SQLITE_TCLAPI f5tTokenizerLocale(
+  void * clientData,
+  Tcl_Interp *interp,
+  int objc,
+  Tcl_Obj *CONST objv[]
+){
+  F5tTokenizerContext *p = (F5tTokenizerContext*)clientData;
+
+  if( objc!=1 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "");
+    return TCL_ERROR;
+  }
+
+  if( p->xToken==0 ){
+    Tcl_AppendResult(interp, 
+        "sqlite3_fts5_locale may only be used by tokenizer callback", 0
+    );
+    return TCL_ERROR;
+  }
+
+  Tcl_SetObjResult(interp, 
+      Tcl_NewStringObj(p->pInst->pLocale, p->pInst->nLocale)
+  );
+  return TCL_OK;
+}
+
 /*
 ** sqlite3_fts5_token ?-colocated? TEXT START END
 */
@@ -996,32 +1145,112 @@ static int SQLITE_TCLAPI f5tCreateTokenizer(
  fts5_api *pApi;
  char *zName;
  Tcl_Obj *pScript;
-  fts5_tokenizer t;
  F5tTokenizerModule *pMod;
-  int rc;
+  int rc = SQLITE_OK;
+  int bV2 = 0;                    /* True to use _v2 API */
+  int iVersion = 2;               /* Value for _v2.iVersion */
+  const char *zParent = 0;        /* Name of parent tokenizer, if any */
+  int ii = 0;

-  if( objc!=4 ){
-    Tcl_WrongNumArgs(interp, 1, objv, "DB NAME SCRIPT");
+  if( objc<4 ){
+    Tcl_WrongNumArgs(interp, 1, objv, "?OPTIONS? DB NAME SCRIPT");
    return TCL_ERROR;
  }
-  if( f5tDbAndApi(interp, objv[1], &db, &pApi) ){
+
+  /* Parse any options. Set stack variables bV2 and zParent. */
+  for(ii=1; ii<objc-3; ii++){
+    int iOpt = 0;
+    const char *azOpt[] = { "-v2", "-parent", "-version", 0 };
+    if( Tcl_GetIndexFromObj(interp, objv[ii], azOpt, "OPTION", 0, &iOpt) ){
+      return TCL_ERROR;
+    }
+    switch( iOpt ){
+      case 0: /* -v2 */ {
+        bV2 = 1;
+        break;
+      }
+      case 1: /* -parent */ {
+        ii++;
+        if( ii==objc-3 ){
+          Tcl_AppendResult(
+              interp, "option requires an argument: -parent", (char*)0
+          );
+          return TCL_ERROR;
+        }
+        zParent = Tcl_GetString(objv[ii]);
+        break;
+      }
+      case 2: /* -version */ {
+        ii++;
+        if( ii==objc-3 ){
+          Tcl_AppendResult(
+              interp, "option requires an argument: -version", (char*)0
+          );
+          return TCL_ERROR;
+        }
+        if( Tcl_GetIntFromObj(interp, objv[ii], &iVersion) ){
+          return TCL_ERROR;
+        }
+        break;
+      }
+      default:
+        assert( 0 );
+        break;
+    }
+  }
+
+  if( f5tDbAndApi(interp, objv[objc-3], &db, &pApi) ){
    return TCL_ERROR;
  }
-  zName = Tcl_GetString(objv[2]);
-  pScript = objv[3];
-
-  t.xCreate = f5tTokenizerCreate;
-  t.xTokenize = f5tTokenizerTokenize;
-  t.xDelete = f5tTokenizerDelete;
+  zName = Tcl_GetString(objv[objc-2]);
+  pScript = objv[objc-1];

  pMod = (F5tTokenizerModule*)ckalloc(sizeof(F5tTokenizerModule));
+  memset(pMod, 0, sizeof(F5tTokenizerModule));
  pMod->interp = interp;
  pMod->pScript = pScript;
-  pMod->pContext = pContext;
  Tcl_IncrRefCount(pScript);
-  rc = pApi->xCreateTokenizer(pApi, zName, (void*)pMod, &t, f5tDelTokenizer);
+  pMod->pContext = pContext;
+  if( zParent ){
+    if( bV2 ){
+      fts5_tokenizer_v2 *pParent = 0;
+      rc = pApi->xFindTokenizer_v2(pApi, zParent, &pMod->pParentCtx, &pParent);
+      if( rc==SQLITE_OK ){
+        memcpy(&pMod->parent_v2, pParent, sizeof(fts5_tokenizer_v2));
+        pMod->parent_v2.xDelete(0);
+      }
+    }else{
+      rc = pApi->xFindTokenizer(pApi, zParent, &pMod->pParentCtx,&pMod->parent);
+      if( rc==SQLITE_OK ){
+        pMod->parent.xDelete(0);
+      }
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    void *pModCtx = (void*)pMod;
+    if( bV2==0 ){
+      fts5_tokenizer t;
+      t.xCreate = f5tTokenizerCreate;
+      t.xTokenize = f5tTokenizerTokenize;
+      t.xDelete = f5tTokenizerDelete;
+      rc = pApi->xCreateTokenizer(pApi, zName, pModCtx, &t, f5tDelTokenizer);
+    }else{
+      fts5_tokenizer_v2 t2;
+      memset(&t2, 0, sizeof(t2));
+      t2.iVersion = iVersion;
+      t2.xCreate = f5tTokenizerCreate;
+      t2.xTokenize = f5tTokenizerTokenize_v2;
+      t2.xDelete = f5tTokenizerDelete;
+      rc = pApi->xCreateTokenizer_v2(pApi, zName, pModCtx, &t2,f5tDelTokenizer);
+    }
+  }
+
  if( rc!=SQLITE_OK ){
-    Tcl_AppendResult(interp, "error in fts5_api.xCreateTokenizer()", 0);
+    Tcl_AppendResult(interp, (
+          bV2 ? "error in fts5_api.xCreateTokenizer_v2()"
+          : "error in fts5_api.xCreateTokenizer()"
+    ), 0);
    return TCL_ERROR;
  }

@@ -1328,6 +1557,7 @@ int Fts5tcl_Init(Tcl_Interp *interp){
  } aCmd[] = {
    { "sqlite3_fts5_create_tokenizer",   f5tCreateTokenizer, 1 },
    { "sqlite3_fts5_token",              f5tTokenizerReturn, 1 },
+    { "sqlite3_fts5_locale",             f5tTokenizerLocale, 1 },
    { "sqlite3_fts5_tokenize",           f5tTokenize, 0 },
    { "sqlite3_fts5_create_function",    f5tCreateFunction, 0 },
    { "sqlite3_fts5_may_be_corrupt",     f5tMayBeCorrupt, 0 },
@@ -79,7 +79,7 @@ static int fts5AsciiCreate(
      int i;
      memset(p, 0, sizeof(AsciiTokenizer));
      memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar));
-      for(i=0; rc==SQLITE_OK && i<nArg-1; i+=2){
+      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
        const char *zArg = azArg[i+1];
        if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
          fts5AsciiAddExceptions(p, zArg, 1);
@@ -90,7 +90,6 @@ static int fts5AsciiCreate(
          rc = SQLITE_ERROR;
        }
      }
-      if( rc==SQLITE_OK && i<nArg ) rc = SQLITE_ERROR;
      if( rc!=SQLITE_OK ){
        fts5AsciiDelete((Fts5Tokenizer*)p);
        p = 0;
@@ -382,7 +381,7 @@ static int fts5UnicodeCreate(
      }

      /* Search for a "categories" argument */
-      for(i=0; rc==SQLITE_OK && i<nArg-1; i+=2){
+      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
        if( 0==sqlite3_stricmp(azArg[i], "categories") ){
          zCat = azArg[i+1];
        }
@@ -391,7 +390,7 @@ static int fts5UnicodeCreate(
        rc = unicodeSetCategories(p, zCat);
      }

-      for(i=0; rc==SQLITE_OK && i<nArg-1; i+=2){
+      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
        const char *zArg = azArg[i+1];
        if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
          if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
@@ -416,8 +415,6 @@ static int fts5UnicodeCreate(
          rc = SQLITE_ERROR;
        }
      }
-      if( i<nArg && rc==SQLITE_OK ) rc = SQLITE_ERROR;
-
    }else{
      rc = SQLITE_NOMEM;
    }
@@ -556,7 +553,7 @@ static int fts5UnicodeTokenize(

 typedef struct PorterTokenizer PorterTokenizer;
 struct PorterTokenizer {
-  fts5_tokenizer tokenizer;       /* Parent tokenizer module */
+  fts5_tokenizer_v2 tokenizer_v2; /* Parent tokenizer module */
  Fts5Tokenizer *pTokenizer;      /* Parent tokenizer instance */
  char aBuf[FTS5_PORTER_MAX_TOKEN + 64];
 };
@@ -568,7 +565,7 @@ static void fts5PorterDelete(Fts5Tokenizer *pTok){
  if( pTok ){
    PorterTokenizer *p = (PorterTokenizer*)pTok;
    if( p->pTokenizer ){
-      p->tokenizer.xDelete(p->pTokenizer);
+      p->tokenizer_v2.xDelete(p->pTokenizer);
    }
    sqlite3_free(p);
  }
@@ -587,6 +584,7 @@ static int fts5PorterCreate(
  PorterTokenizer *pRet;
  void *pUserdata = 0;
  const char *zBase = "unicode61";
+  fts5_tokenizer_v2 *pV2 = 0;

  if( nArg>0 ){
    zBase = azArg[0];
@@ -595,14 +593,15 @@ static int fts5PorterCreate(
  pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer));
  if( pRet ){
    memset(pRet, 0, sizeof(PorterTokenizer));
-    rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer);
+    rc = pApi->xFindTokenizer_v2(pApi, zBase, &pUserdata, &pV2);
  }else{
    rc = SQLITE_NOMEM;
  }
  if( rc==SQLITE_OK ){
    int nArg2 = (nArg>0 ? nArg-1 : 0);
-    const char **azArg2 = (nArg2 ? &azArg[1] : 0);
-    rc = pRet->tokenizer.xCreate(pUserdata, azArg2, nArg2, &pRet->pTokenizer);
+    const char **az2 = (nArg2 ? &azArg[1] : 0);
+    memcpy(&pRet->tokenizer_v2, pV2, sizeof(fts5_tokenizer_v2));
+    rc = pRet->tokenizer_v2.xCreate(pUserdata, az2, nArg2, &pRet->pTokenizer);
  }

  if( rc!=SQLITE_OK ){
@@ -1253,6 +1252,7 @@ static int fts5PorterTokenize(
  void *pCtx,
  int flags,
  const char *pText, int nText,
+  const char *pLoc, int nLoc,
  int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd)
 ){
  PorterTokenizer *p = (PorterTokenizer*)pTokenizer;
@@ -1260,8 +1260,8 @@ static int fts5PorterTokenize(
  sCtx.xToken = xToken;
  sCtx.pCtx = pCtx;
  sCtx.aBuf = p->aBuf;
-  return p->tokenizer.xTokenize(
-      p->pTokenizer, (void*)&sCtx, flags, pText, nText, fts5PorterCb
+  return p->tokenizer_v2.xTokenize(
+      p->pTokenizer, (void*)&sCtx, flags, pText, nText, pLoc, nLoc, fts5PorterCb
  );
 }

@@ -1291,41 +1291,46 @@ static int fts5TriCreate(
  Fts5Tokenizer **ppOut
 ){
  int rc = SQLITE_OK;
-  TrigramTokenizer *pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
+  TrigramTokenizer *pNew = 0;
  UNUSED_PARAM(pUnused);
-  if( pNew==0 ){
-    rc = SQLITE_NOMEM;
+  if( nArg%2 ){
+    rc = SQLITE_ERROR;
  }else{
    int i;
-    pNew->bFold = 1;
-    pNew->iFoldParam = 0;
-    for(i=0; rc==SQLITE_OK && i<nArg-1; i+=2){
-      const char *zArg = azArg[i+1];
-      if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
-        if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
-          rc = SQLITE_ERROR;
+    pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew));
+    if( pNew==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      pNew->bFold = 1;
+      pNew->iFoldParam = 0;
+  
+      for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
+        const char *zArg = azArg[i+1];
+        if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
+          if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
+            rc = SQLITE_ERROR;
+          }else{
+            pNew->bFold = (zArg[0]=='0');
+          }
+        }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
+          if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
+            rc = SQLITE_ERROR;
+          }else{
+            pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
+          }
        }else{
-          pNew->bFold = (zArg[0]=='0');
-        }
-      }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
-        if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
          rc = SQLITE_ERROR;
-        }else{
-          pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0;
        }
-      }else{
+      }
+  
+      if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
        rc = SQLITE_ERROR;
      }
-    }
-    if( i<nArg && rc==SQLITE_OK ) rc = SQLITE_ERROR;
-
-    if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
-      rc = SQLITE_ERROR;
-    }
-
-    if( rc!=SQLITE_OK ){
-      fts5TriDelete((Fts5Tokenizer*)pNew);
-      pNew = 0;
+  
+      if( rc!=SQLITE_OK ){
+        fts5TriDelete((Fts5Tokenizer*)pNew);
+        pNew = 0;
+      }
    }
  }
  *ppOut = (Fts5Tokenizer*)pNew;
@@ -1450,7 +1455,6 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){
  } aBuiltin[] = {
    { "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}},
    { "ascii",     {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }},
-    { "porter",    {fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize }},
    { "trigram",   {fts5TriCreate, fts5TriDelete, fts5TriTokenize}},
  };
  
@@ -1465,6 +1469,19 @@ int sqlite3Fts5TokenizerInit(fts5_api *pApi){
        0
    );
  }
-
+  if( rc==SQLITE_OK ){
+    fts5_tokenizer_v2 sPorter = {
+      2,
+      fts5PorterCreate,
+      fts5PorterDelete,
+      fts5PorterTokenize
+    };
+    rc = pApi->xCreateTokenizer_v2(pApi,
+        "porter",
+        (void*)pApi,
+        &sPorter,
+        0
+    );
+  }
  return rc;
 }
@@ -364,6 +364,9 @@ int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){
            default: return 1;          }
          break;

+
+    default:
+      return 1;
  }
  return 0;
 }
@@ -51,6 +51,10 @@ proc fts5_test_poslist2 {cmd} {
  sort_poslist $res
 }

+proc fts5_test_insttoken {cmd iInst iToken} {
+  $cmd xInstToken $iInst $iToken
+}
+
 proc fts5_test_collist {cmd} {
  set res [list]

@@ -78,6 +82,9 @@ proc fts5_test_columnsize {cmd} {
 proc fts5_columntext {cmd iCol} {
  $cmd xColumnText $iCol
 }
+proc fts5_columnlocale {cmd iCol} {
+  $cmd xColumnLocale $iCol
+}

 proc fts5_test_columntext {cmd} {
  set res [list]
@@ -87,6 +94,14 @@ proc fts5_test_columntext {cmd} {
  set res
 }

+proc fts5_test_columnlocale {cmd} {
+  set res [list]
+  for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
+    lappend res [$cmd xColumnLocale $i]
+  }
+  set res
+}
+
 proc fts5_test_columntotalsize {cmd} {
  set res [list]
  for {set i 0} {$i < [$cmd xColumnCount]} {incr i} {
@@ -165,10 +180,12 @@ proc fts5_aux_test_functions {db} {
  foreach f {
    fts5_test_columnsize
    fts5_test_columntext
+    fts5_test_columnlocale
    fts5_test_columntotalsize
    fts5_test_poslist
    fts5_test_poslist2
    fts5_test_collist
+    fts5_test_insttoken
    fts5_test_tokenize
    fts5_test_rowcount
    fts5_test_rowid
@@ -177,6 +194,7 @@ proc fts5_aux_test_functions {db} {
    fts5_test_queryphrase
    fts5_test_phrasecount
    fts5_columntext
+    fts5_columnlocale
    fts5_queryphrase
    fts5_collist
  } {
@@ -163,6 +163,17 @@ do_execsql_test 1.8.2 {
  SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid < 'text';
 } {10000}

+do_execsql_test 1.8.3 {
+  SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid<5000 AND rowid < 'text';
+} {4999}
+do_execsql_test 1.8.4 {
+  SELECT count(*) FROM t1 WHERE t1 MATCH 'x' AND rowid>5000 AND rowid > 'text';
+} {0}
+
+do_catchsql_test 1.9 {
+  SELECT * FROM t1('*xy');
+} {1 {unknown special query: xy}}
+
 } ;# foreach_detail_mode

 #db eval {SELECT rowid, fts5_decode(rowid, block) aS r FROM t1_data} {puts $r}
@@ -293,6 +293,16 @@ do_catchsql_test 4.4.4 {
  SELECT *, rank FROM t3 WHERE t3 MATCH 'a' AND rank MATCH NULL
 } {1 {parse error in rank function: }}

+# Check that the second and subsequent rank= constraints are ignored.
+#
+do_catchsql_test 4.3.3 {
+  SELECT *, rank FROM t3
+  WHERE t3 MATCH 'a' AND 
+  rank MATCH 'nosuch()' AND
+  rank MATCH 'rowidmod(3)'  
+  ORDER BY rank ASC
+} {1 {unable to use function MATCH in the requested context}}
+
 } ;# foreach_detail_mode


@@ -0,0 +1,166 @@
+# 2024 July 30
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file verifies that:
+#
+#   * blob values may be written to locale=0 tables.
+#
+#   * blob values - other than fts5_locale() values - may not be written 
+#     to locale=0 tables. This is an SQLITE_MISMATCH error
+#
+#   * blob values may be returned by queries on the external-content table
+#     of a locale=0 table.
+#
+#   * blob values not may be returned by queries on the external-content 
+#     table of a locale=1 table, apart from fts5_locale() blobs. This is an
+#     SQLITE_MISMATCH error.
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5blob
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+# Test that blobs may be stored in normal locale=0 tables.
+#
+foreach {tn enc} {
+  1 utf8  
+  2 utf16 
+} {
+  reset_db
+  fts5_aux_test_functions db
+
+  execsql "PRAGMA encoding = $enc"
+
+  execsql "
+    CREATE VIRTUAL TABLE t1 USING fts5(x, y);
+  "
+  do_execsql_test 1.$tn.0 {
+    CREATE VIRTUAL TABLE tt USING fts5vocab('t1', 'instance');
+    INSERT INTO t1(rowid, x, y) VALUES(1, 555, X'0000000041424320444546');
+    INSERT INTO t1(rowid, x, y) VALUES(2, 666, X'41424300444546');
+    INSERT INTO t1(rowid, x, y) VALUES(3, 777, 'xyz');
+  }
+
+  do_execsql_test 1.$tn.1 {
+    SELECT rowid, quote(x), quote(y) FROM t1
+  } {
+    1 555 X'0000000041424320444546'
+    2 666 X'41424300444546'
+    3 777 'xyz'
+  }
+
+  do_execsql_test 1.$tn.2 {
+    DELETE FROM t1 WHERE rowid=2;
+    DELETE FROM t1 WHERE rowid=1;
+  }
+
+  do_execsql_test 1.$tn.3 {
+    PRAGMA integrity_check;
+  } {ok}
+}
+
+#--------------------------------------------------------------------------
+# Test that a blob may be stored and retrieved in an unindexed column of
+# a regular table with locale=1.
+#
+reset_db
+do_execsql_test 2.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(x, y UNINDEXED, locale=1);
+  INSERT INTO t1(rowid, x, y) VALUES(12, 'twelve', X'0000000041424320444546');
+}
+
+do_execsql_test 2.1 {
+  select rowid, x, quote(y) FROM t1
+} {
+  12 twelve X'0000000041424320444546'
+}
+
+#--------------------------------------------------------------------------
+# Test that blobs may not be written to any type of table with locale=1
+# set. Except, they may be written to UNINDEXED columns.
+#
+reset_db
+do_execsql_test 3.0 {
+  CREATE TABLE t1(a, b);
+
+  CREATE VIRTUAL TABLE x1 USING fts5(a, b, locale=1);
+  CREATE VIRTUAL TABLE x2 USING fts5(a, b, locale=1, content=t2);
+  CREATE VIRTUAL TABLE x3 USING fts5(a, b, locale=1, content=);
+}
+
+do_catchsql_test 3.1 {
+  INSERT INTO x1(rowid, a, b) VALUES(113, 'hello world', X'123456');
+} {1 {datatype mismatch}}
+do_catchsql_test 3.2 {
+  INSERT INTO x2(rowid, a, b) VALUES(113, 'hello world', X'123456');
+} {1 {datatype mismatch}}
+do_catchsql_test 3.3 {
+  INSERT INTO x3(rowid, a, b) VALUES(113, 'hello world', X'123456');
+} {1 {datatype mismatch}}
+
+
+#--------------------------------------------------------------------------
+# Test that fts5_locale() values may not be written to any type of table
+# without locale=1 set. Even to an UNINDEXED column.
+#
+reset_db
+do_execsql_test 3.0 {
+  CREATE TABLE t1(a, b);
+
+  CREATE VIRTUAL TABLE x1 USING fts5(a, b);
+  CREATE VIRTUAL TABLE x2 USING fts5(a, b, content=t2);
+  CREATE VIRTUAL TABLE x3 USING fts5(a, b, content=);
+
+  CREATE VIRTUAL TABLE x4 USING fts5(a, b, c UNINDEXED);
+}
+
+do_catchsql_test 3.1 {
+  INSERT INTO x1(rowid, a, b) 
+    VALUES(113, 'hello world', fts5_locale('en_AU', 'abc'));
+} {1 {fts5_locale() requires locale=1}}
+do_catchsql_test 3.2 {
+  INSERT INTO x2(rowid, a, b) 
+    VALUES(113, 'hello world', fts5_locale('en_AU', 'abc'));
+} {1 {fts5_locale() requires locale=1}}
+do_catchsql_test 3.3 {
+  INSERT INTO x3(rowid, a, b) 
+    VALUES(113, 'hello world', fts5_locale('en_AU', 'abc'));
+} {1 {fts5_locale() requires locale=1}}
+do_catchsql_test 3.4 {
+  INSERT INTO x4(rowid, a, b, c) 
+    VALUES(113, 'hello world', 'yesno', fts5_locale('en_AU', 'abc'));
+} {1 {fts5_locale() requires locale=1}}
+
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(x);
+}
+
+foreach {tn sql} {
+  1 { INSERT INTO x1(rowid, x) VALUES(4.5, 'abcd') }
+  2 { INSERT INTO x1(rowid, x) VALUES('xyz', 'abcd') }
+  3 { INSERT INTO x1(rowid, x) VALUES(X'001122', 'abcd') }
+} {
+  do_catchsql_test 4.1.$tn $sql {1 {datatype mismatch}}
+}
+
+
+finish_test
+
+
@@ -55,5 +55,22 @@ do_execsql_test 1.5 {
  SELECT * FROM t4t
 } {สนามกีฬา 1 1}

+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0 "
+  CREATE VIRTUAL TABLE x1 USING fts5(c, 
+      tokenize=\"unicode61 categories ' \t'\");
+"
+
+do_catchsql_test 2.1 "
+  CREATE VIRTUAL TABLE x2 USING fts5(c, 
+      tokenize=\"unicode61 categories 'N*\t\tMYZ'\");
+" {1 {error in tokenizer constructor}}
+
+do_catchsql_test 2.2 "
+  CREATE VIRTUAL TABLE x2 USING fts5(c, 
+      tokenize=\"unicode61 categories 'N*\t\tXYZ'\");
+" {1 {error in tokenizer constructor}}
+

 finish_test
@@ -267,4 +267,24 @@ do_execsql_test 8.2 {
  SELECT rowid FROM ft('four');
 } {}

+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 9.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, content='', contentless_delete=0);
+  INSERT INTO ft VALUES('hello world');
+  INSERT INTO ft VALUES('one two three');
+}
+
+do_catchsql_test 9.1 {
+  INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, 'hello world');
+} {0 {}}
+
+do_catchsql_test 9.2 {
+  CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', contentless_delete=2);
+} {1 {malformed contentless_delete=... directive}}
+
+do_catchsql_test 9.3 {
+  CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', contentless_delete=11);
+} {1 {malformed contentless_delete=... directive}}
+
 finish_test
@@ -101,4 +101,25 @@ do_catchsql_test 3.1 {
  SELECT * FROM t3 WHERE t3 MATCH 'o';
 } {1 {fts5: missing row 3 from content table 'main'.'t3_content'}}

+#--------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE t2 USING fts5(x);
+  INSERT INTO t2 VALUES('one two three');
+  INSERT INTO t2 VALUES('four five six');
+  INSERT INTO t2 VALUES('seven eight nine');
+  INSERT INTO t2 VALUES('ten eleven twelve');
+}
+do_execsql_test 4.1 {
+  SELECT hex(block) FROM t2_data WHERE id=1;
+} {040C}
+do_execsql_test 4.2 {
+  UPDATE t2_data SET block = X'0402' WHERE id=1
+}
+breakpoint
+do_catchsql_test 4.3 {
+  DELETE FROM t2 WHERE rowid=3
+} {1 {database disk image is malformed}}
+
 finish_test
@@ -15524,6 +15524,160 @@ do_catchsql_test 80.1 {
 SELECT snippet(rowid, -1, '.', '..', '[', '(]'),snippet(rowid, -1, '.', '.', '', '(]'), highlight(t1, 29, 1 , '') FROM t1('g+ h') WHERE rank MATCH 'bm25(1.0, 10)' ORDER BY NOT (SELECT 1 FROM t1('g+ æ') WHERE rank MATCH 'bm25(1.0, 10)' ORDER BY rank);
 } {1 {database disk image is malformed}}

+#-------------------------------------------------------------------------
+reset_db
+do_test 81.0 {
+  sqlite3 db {}
+  db deserialize [decode_hexdb {
+.open --hexdb
+| size 40960 pagesize 4096 filename crash-44e8035a976422.db
+| page 1 offset 0
+|      0: 53 51 4c 69 74 65 20 66 6f 72 6d 61 74 20 33 00   SQLite format 3.
+|     16: 10 00 01 01 00 40 20 20 00 00 00 00 00 00 00 0a   .....@  ........
+|     32: 00 00 00 00 00 00 00 00 00 00 00 0d 00 00 00 04   ................
+|     48: 00 00 00 00 00 00 00 00 00 00 00 01 00 00 00 00   ................
+|     96: 00 00 00 00 0d 00 00 00 0d 0b 6e 00 0f a3 0f 4c   ..........n....L
+|    112: 0e e1 0e 81 0e 24 0d cc 0d 72 0d 1b 0c b0 0c 50   .....$...r.....P
+|    128: 0b f8 0b b3 0b 6e 00 00 00 00 00 00 00 00 00 00   .....n..........
+|   2912: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 43 0d   ..............C.
+|   2928: 06 17 11 11 08 75 74 61 62 6c 65 74 34 74 34 43   .....utablet4t4C
+|   2944: 52 45 41 54 45 20 56 49 52 54 55 41 4c 20 54 41   REATE VIRTUAL TA
+|   2960: 42 4c 45 20 74 34 20 55 53 49 4e 47 20 66 74 73   BLE t4 USING fts
+|   2976: 35 76 6f 63 61 62 28 27 74 32 27 2c 20 27 72 6f   5vocab('t2', 'ro
+|   2992: 77 27 29 43 0c 06 17 11 11 08 75 74 61 62 6c 65   w')C......utable
+|   3008: 74 33 74 33 43 52 45 41 54 45 20 56 49 52 54 55   t3t3CREATE VIRTU
+|   3024: 41 4c 20 54 41 42 4c 45 20 74 33 20 55 53 49 4e   AL TABLE t3 USIN
+|   3040: 47 20 66 74 73 35 76 6f 63 61 62 28 27 74 31 27   G fts5vocab('t1'
+|   3056: 2c 20 27 72 6f 77 27 29 56 0b 06 17 1f 1f 01 7d   , 'row')V.......
+|   3072: 74 61 62 6c 65 74 32 5f 63 6f 6e 66 69 67 74 32   tablet2_configt2
+|   3088: 5f 63 6f 6e 66 69 67 0a 43 52 45 41 54 45 20 54   _config.CREATE T
+|   3104: 41 42 4c 45 20 27 74 32 5f 63 6f 6e 66 69 67 27   ABLE 't2_config'
+|   3120: 28 6b 20 50 52 49 4d 41 52 59 20 4b 45 59 2c 20   (k PRIMARY KEY, 
+|   3136: 76 29 20 57 49 54 48 4f 55 54 20 52 4f 57 49 44   v) WITHOUT ROWID
+|   3152: 5e 0a 07 17 21 21 01 81 07 74 61 62 6c 65 74 32   ^...!!...tablet2
+|   3168: 5f 63 6f 6e 74 65 6e 74 74 32 5f 63 6f 6e 74 65   _contentt2_conte
+|   3184: 6e 74 09 43 52 45 41 54 45 20 54 41 42 4c 45 20   nt.CREATE TABLE 
+|   3200: 27 74 32 5f 63 6f 6e 74 65 6e 74 27 28 69 64 20   't2_content'(id 
+|   3216: 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59 20   INTEGER PRIMARY 
+|   3232: 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32 29   KEY, c0, c1, c2)
+|   3248: 69 09 07 17 19 19 01 81 2d 74 61 62 6c 65 74 32   i.......-tablet2
+|   3264: 5f 69 64 78 74 32 5f 69 64 78 08 43 52 45 41 54   _idxt2_idx.CREAT
+|   3280: 45 20 54 41 42 4c 45 20 27 74 32 5f 69 64 78 27   E TABLE 't2_idx'
+|   3296: 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70 67   (segid, term, pg
+|   3312: 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59 28   no, PRIMARY KEY(
+|   3328: 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57 49   segid, term)) WI
+|   3344: 54 48 4f 55 54 20 52 4f 57 49 44 55 08 07 17 1b   THOUT ROWIDU....
+|   3360: 1b 01 81 01 74 61 62 6c 65 74 32 5f 64 61 74 61   ....tablet2_data
+|   3376: 74 32 5f 64 61 74 61 07 43 52 45 41 54 45 20 54   t2_data.CREATE T
+|   3392: 41 42 4c 45 20 27 74 32 5f 64 61 74 61 27 28 69   ABLE 't2_data'(i
+|   3408: 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52   d INTEGER PRIMAR
+|   3424: 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c 4f   Y KEY, block BLO
+|   3440: 42 29 58 07 07 17 11 11 08 81 1d 74 61 62 6c 65   B)X........table
+|   3456: 74 32 74 32 43 52 45 41 54 45 20 56 49 52 54 55   t2t2CREATE VIRTU
+|   3472: 41 4c 20 54 41 42 4c 45 20 74 32 20 55 53 49 4e   AL TABLE t2 USIN
+|   3488: 47 20 66 74 73 35 28 27 61 27 2c 5b 62 5d 2c 22   G fts5('a',[b],.
+|   3504: 63 22 2c 64 65 74 61 69 6c 3d 6e 6f 6e 65 2c 63   c.,detail=none,c
+|   3520: 6f 6c 75 6d 6e 73 69 7a 65 3d 30 29 56 06 06 17   olumnsize=0)V...
+|   3536: 1f 1f 01 7d 74 61 62 6c 65 74 31 5f 63 6f 6e 66   ....tablet1_conf
+|   3552: 69 67 74 31 5f 63 6f 6e 66 69 67 06 43 52 45 41   igt1_config.CREA
+|   3568: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 63 6f 6e   TE TABLE 't1_con
+|   3584: 66 69 67 27 28 6b 20 50 52 49 4d 41 52 59 20 4b   fig'(k PRIMARY K
+|   3600: 45 59 2c 20 76 29 20 57 49 54 48 4f 55 54 20 52   EY, v) WITHOUT R
+|   3616: 4f 57 49 44 5b 05 07 17 21 21 01 81 01 74 61 62   OWID[...!!...tab
+|   3632: 6c 65 74 31 5f 64 6f 63 73 69 7a 65 74 31 5f 64   let1_docsizet1_d
+|   3648: 6f 63 73 69 7a 65 05 43 52 45 41 54 45 20 54 41   ocsize.CREATE TA
+|   3664: 42 4c 45 20 27 74 31 5f 64 6f 63 73 69 7a 65 27   BLE 't1_docsize'
+|   3680: 28 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d   (id INTEGER PRIM
+|   3696: 41 52 59 20 4b 45 59 2c 20 73 7a 20 42 4c 4f 42   ARY KEY, sz BLOB
+|   3712: 29 5e 04 07 17 21 21 01 81 07 74 61 62 6c 65 74   )^...!!...tablet
+|   3728: 31 5f 63 6f 6e 74 65 6e 74 74 31 5f 63 6f 6e 74   1_contentt1_cont
+|   3744: 65 6e 74 04 43 52 45 41 54 45 20 54 41 42 4c 45   ent.CREATE TABLE
+|   3760: 20 27 74 31 5f 63 6f 6e 74 65 6e 74 27 28 69 64    't1_content'(id
+|   3776: 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41 52 59    INTEGER PRIMARY
+|   3792: 20 4b 45 59 2c 20 63 30 2c 20 63 31 2c 20 63 32    KEY, c0, c1, c2
+|   3808: 29 69 03 07 17 19 19 01 81 2d 74 61 62 6c 65 74   )i.......-tablet
+|   3824: 31 5f 69 64 78 74 31 5f 69 64 78 03 43 52 45 41   1_idxt1_idx.CREA
+|   3840: 54 45 20 54 41 42 4c 45 20 27 74 31 5f 69 64 78   TE TABLE 't1_idx
+|   3856: 27 28 73 65 67 69 64 2c 20 74 65 72 6d 2c 20 70   '(segid, term, p
+|   3872: 67 6e 6f 2c 20 50 52 49 4d 41 52 59 20 4b 45 59   gno, PRIMARY KEY
+|   3888: 28 73 65 67 69 64 2c 20 74 65 72 6d 29 29 20 57   (segid, term)) W
+|   3904: 49 54 48 4f 55 54 20 52 4f 57 49 44 55 02 07 17   ITHOUT ROWIDU...
+|   3920: 1b 1b 01 81 01 74 61 62 6c 65 74 31 5f 64 61 74   .....tablet1_dat
+|   3936: 61 74 31 5f 64 61 74 61 02 43 52 45 41 54 45 20   at1_data.CREATE 
+|   3952: 54 41 42 4c 45 20 27 74 31 5f 64 61 74 61 27 28   TABLE 't1_data'(
+|   3968: 69 64 20 49 4e 54 45 47 45 52 20 50 52 49 4d 41   id INTEGER PRIMA
+|   3984: 52 59 20 4b 45 59 2c 20 62 6c 6f 63 6b 20 42 4c   RY KEY, block BL
+|   4000: 4f 42 29 5b 01 07 17 11 11 08 81 23 74 61 62 6c   OB)[.......#tabl
+|   4016: 65 74 31 74 31 43 52 45 41 54 45 20 56 49 52 54   et1t1CREATE VIRT
+|   4032: 55 41 4c 20 54 41 42 4c 45 20 74 31 20 55 53 49   UAL TABLE t1 USI
+|   4048: 4e 47 20 66 74 73 35 28 61 2c 62 20 75 6e 69 6e   NG fts5(a,b unin
+|   4064: 64 65 78 65 64 2c 63 2c 74 6f 6b 65 6e 69 7a 65   dexed,c,tokenize
+|   4080: 3d 22 70 6f 72 74 65 72 20 61 73 63 69 69 22 29   =.porter ascii.)
+| page 2 offset 4096
+|      0: 0d 0f 68 00 05 0f 13 00 0f e6 0f 13 0f a8 0f 7c   ..h............|
+|     16: 0f 2a 00 00 00 00 00 00 00 00 00 00 00 00 00 00   .*..............
+|   3856: 00 00 00 15 0a 03 00 30 00 00 00 00 01 03 03 00   .......0........
+|   3872: 03 01 01 01 02 01 01 03 01 01 37 8c 80 80 80 80   ..........7.....
+|   3888: 01 03 00 74 00 00 00 2e 02 30 61 03 02 02 01 01   ...t.....0a.....
+|   3904: 62 03 02 03 01 01 63 03 02 04 01 01 67 03 06 01   b.....c.....g...
+|   3920: 02 02 01 01 68 03 06 01 02 03 01 01 69 03 06 01   ....h.......i...
+|   3936: 02 04 04 06 06 06 08 08 0f ef 00 14 2a 00 00 00   ............*...
+|   3952: 00 01 02 02 00 02 01 01 01 02 01 01 25 88 80 80   ............%...
+|   3968: 80 80 01 03 00 50 00 00 00 1f 02 30 67 02 08 02   .....P.....0g...
+|   3984: 01 02 02 01 01 68 02 08 03 8d 02 03 01 01 6a 42   .....h........jB
+|   4000: 08 04 01 02 04 04 09 09 37 84 80 80 80 80 01 03   ........7.......
+|   4016: 00 74 00 00 00 2e 02 30 61 01 12 02 01 01 62 01   .t.....0a.....b.
+|   4032: 02 03 01 01 63 01 02 04 01 01 67 01 06 01 02 02   ....c.....g.....
+|   4048: 01 01 68 01 05 01 02 03 01 01 69 01 06 01 02 04   ..h.......i.....
+|   4064: 04 06 06 06 08 08 07 01 03 00 14 03 09 00 09 00   ................
+|   4080: 00 00 11 24 00 00 00 00 01 01 01 00 01 01 01 01   ...$............
+| page 3 offset 8192
+|      0: 0a 00 00 00 03 0f ec 00 0f fa 0f f3 0f ec 00 00   ................
+|   4064: 00 00 00 00 00 00 00 00 00 00 00 00 06 04 01 0c   ................
+|   4080: 01 03 02 06 04 01 0c 01 02 02 05 04 09 0c 01 02   ................
+| page 4 offset 12288
+|      0: 0d 00 00 00 03 0f be 00 0f ea 00 00 00 00 00 00   ................
+|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03   ................
+|   4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67   .....a b cg h ig
+|   4048: 20 68 20 69 14 02 05 00 17 17 17 67 20 68 20 69    h i.......g h i
+|   4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17   a b cg h i......
+|   4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69   .a b cd e fg h i
+| page 5 offset 16384
+|      0: 0d 00 00 00 03 0f e8 00 0f f8 0f f0 0f e8 00 00   ................
+|   4064: 00 00 00 00 00 00 00 00 06 03 03 00 12 03 00 03   ................
+|   4080: 06 02 03 00 12 03 00 03 06 01 03 00 12 03 00 03   ................
+| page 6 offset 20480
+|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
+| page 7 offset 24576
+|      0: 0d 00 00 00 03 0f 9e 00 0f e6 0f ef 0f 9e 00 00   ................
+|   3984: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 41 84   ..............A.
+|   4000: 80 80 80 80 01 04 00 81 06 00 00 00 34 02 30 61   ............4.0a
+|   4016: 01 01 01 01 01 62 01 01 01 01 01 63 01 01 01 01   .....b.....c....
+|   4032: e6 64 01 01 01 65 01 01 01 66 01 01 01 67 01 01   .d...e...f...g..
+|   4048: 01 01 01 68 01 01 01 01 01 69 01 01 01 04 06 06   ...h.....i......
+|   4064: 06 04 04 04 06 06 07 01 03 00 14 03 09 09 09 0f   ................
+|   4080: 0a 03 00 24 00 00 00 00 01 01 01 00 01 01 01 01   ...$............
+| page 8 offset 28672
+|      0: 0a 00 00 00 01 0f fa 00 0f fa 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 00 00 00 00 00 00 05 04 09 0c 01 02   ................
+| page 9 offset 32768
+|      0: 0d 00 00 00 03 0f be 00 0f ea 0f d4 0f be 00 00   ................
+|   4016: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 14 03   ................
+|   4032: 05 00 17 17 17 61 20 62 20 63 67 20 68 20 69 67   .....a b cg h ig
+|   4048: 20 68 21 69 14 02 05 00 17 17 17 67 20 68 20 69    h!i.......g h i
+|   4064: 61 20 62 20 63 67 20 68 20 69 14 01 05 00 17 17   a b cg h i......
+|   4080: 17 61 20 62 20 63 64 20 65 20 66 67 20 68 20 69   .a b cd e fg h i
+| page 10 offset 36864
+|      0: 0a 00 00 00 01 0f f4 00 0f f4 00 00 00 00 00 00   ................
+|   4080: 00 00 00 00 0b 03 1b 01 76 65 72 73 69 6f 6e 04   ........version.
+| end crash-44e8035a976422.db
+}]} {}
+
+do_catchsql_test 81.2 {
+  UPDATE t1 SET b=zeroblob(299);
+} {1 {database disk image is malformed}}
+

 sqlite3_fts5_may_be_corrupt 0
 finish_test
@@ -44,5 +44,9 @@ for {set ii 0} {$ii < 300} {incr ii} {
  } $res
 }

+do_execsql_test 1.2 {
+  SELECT rowid FROM x1 WHERE a MATCH '"..."'
+} {}
+
 finish_test

@@ -0,0 +1,261 @@
+# 2010 June 15
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+source $testdir/malloc_common.tcl
+set testprefix fts5faultI
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+set ::testprefix fts5faultI
+
+do_execsql_test 1.0 {
+  PRAGMA encoding = utf16;
+  CREATE VIRTUAL TABLE t1 USING fts5(x, locale=1);
+  INSERT INTO t1 VALUES('origintext unicode61 ascii porter trigram');
+}
+
+faultsim_save_and_close
+faultsim_restore_and_reopen
+
+do_faultsim_test 1 -faults oom* -prep { 
+} -body {
+  execsql {
+    SELECT rowid FROM t1(fts5_locale('en_US', 'origintext'));
+  }
+} -test {
+  faultsim_test_result {0 1}
+}
+
+do_faultsim_test 2 -faults oom* -prep { 
+  faultsim_restore_and_reopen
+  execsql {
+    SELECT * FROM t1('ascii');
+  }
+} -body {
+  execsql {
+    UPDATE t1 SET rowid=rowid+1;
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+fts5_aux_test_functions db
+do_faultsim_test 3 -faults oom* -prep { 
+} -body {
+  execsql {
+    SELECT fts5_columnlocale(t1, 0) FROM t1('unicode*');
+  }
+} -test {
+  faultsim_test_result {0 {{}}} {1 SQLITE_NOMEM}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 4.0 {
+  CREATE VIRTUAL TABLE w1 USING fts5(a);
+}
+faultsim_save_and_close
+
+do_faultsim_test 4 -faults oom* -prep { 
+  faultsim_restore_and_reopen
+  execsql {
+    BEGIN;
+      INSERT INTO w1 VALUES('token token token');
+  }
+} -body {
+  execsql {
+    INSERT INTO w1(w1, rank) VALUES('rank', 'bm25()');
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+do_faultsim_test 5 -faults oom* -prep { 
+  faultsim_restore_and_reopen
+  execsql { 
+    BEGIN;
+      INSERT INTO w1 VALUES('one');
+      SAVEPOINT one;
+        INSERT INTO w1 VALUES('two');
+      ROLLBACK TO one;
+  }
+
+} -body {
+  execsql {
+    INSERT INTO w1 VALUES('string');
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 5.0 {
+  CREATE VIRTUAL TABLE w1 USING fts5(a);
+  INSERT INTO w1 VALUES('one two three');
+}
+fts5_aux_test_functions db
+
+do_faultsim_test 5 -faults oom* -prep {
+} -body {
+  execsql {
+    SELECT fts5_test_insttoken(w1, 0, 0) FROM w1('two');
+  }
+} -test {
+  faultsim_test_result {0 two} {1 SQLITE_NOMEM}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE w1 USING fts5(a);
+  INSERT INTO w1 VALUES('one two three');
+}
+fts5_aux_test_functions db
+faultsim_save_and_close
+
+do_faultsim_test 6 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  db eval {
+    BEGIN;
+      INSERT INTO w1 VALUES('four five six');
+      SAVEPOINT abc;
+        INSERT INTO w1 VALUES('seven eight nine');
+        SAVEPOINT def;
+          INSERT INTO w1 VALUES('ten eleven twelve');
+  }
+} -body {
+  execsql {
+    RELEASE abc;
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 7.0 {
+  CREATE VIRTUAL TABLE w1 USING fts5(a);
+  INSERT INTO w1 VALUES('one two three');
+  INSERT INTO w1 VALUES('three two one');
+  DELETE FROM w1_content WHERE rowid=1;
+}
+
+faultsim_save_and_close
+
+do_faultsim_test 7 -faults oom* -prep {
+  faultsim_restore_and_reopen
+  db eval { SELECT * FROM w1 }
+} -body {
+  execsql {
+    PRAGMA integrity_check;
+  }
+} -test {
+}
+
+#-------------------------------------------------------------------------
+reset_db
+fts5_tclnum_register db
+fts5_aux_test_functions db
+
+do_execsql_test 8.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(
+      x, tokenize = "tclnum query", detail=columns
+  );
+  INSERT INTO ft VALUES('one two three i ii iii');
+  INSERT INTO ft VALUES('four five six iv v vi');
+  INSERT INTO ft VALUES('eight nine ten viii ix x');
+} {}
+
+do_faultsim_test 8.1 -faults oom* -prep {
+} -body {
+  execsql {
+    SELECT fts5_test_collist (ft) FROM ft('one two');
+  }
+} -test {
+  faultsim_test_result {0 {{0.0 1.0}}} {1 {SQL logic error}} {1 SQLITE_NOMEM}
+}
+
+do_faultsim_test 8.2 -faults oom* -prep {
+} -body {
+  execsql {
+    SELECT rowid FROM ft('one two') ORDER BY rank;
+  }
+} -test {
+  faultsim_test_result {0 1} {1 {SQL logic error}} {1 SQLITE_NOMEM}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 9.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x);
+  INSERT INTO ft VALUES('one two three i ii iii');
+  INSERT INTO ft VALUES('four five six iv v vi');
+  INSERT INTO ft VALUES('eight nine ten viii ix x');
+} {}
+
+faultsim_save_and_close
+
+do_faultsim_test 9.1 -faults oom* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql {
+    UPDATE ft SET rowid=4 WHERE rowid=1
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+do_faultsim_test 9.2 -faults oom* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql {
+    SELECT rowid FROM ft WHERE x MATCH 'one AND two AND three'
+  }
+} -test {
+  faultsim_test_result {0 1}
+}
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 10.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(x, locale=1);
+  INSERT INTO ft VALUES(fts5_locale('hello', 'one two three i ii iii'));
+  INSERT INTO ft VALUES('four five six iv v vi');
+  INSERT INTO ft VALUES('eight nine ten viii ix x');
+} {}
+
+do_execsql_test 10.1 {
+  SELECT fts5_get_locale(ft, 0) FROM ft WHERE x MATCH 'one AND two AND three'
+} {hello}
+
+faultsim_save_and_close
+do_faultsim_test 10 -faults oom* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql {
+    SELECT fts5_get_locale(ft, 0) FROM ft WHERE x MATCH 'one AND two AND three'
+  }
+} -test {
+  faultsim_test_result {0 hello}
+}
+
+
+finish_test
+
@@ -0,0 +1,667 @@
+# 2014 Dec 20
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focusing on the built-in fts5 tokenizers. 
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5locale
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+proc transform_token {locale token} {
+  switch -- $locale {
+    reverse {
+      set ret ""
+      foreach c [split $token ""] {
+        set ret "$c$ret"
+      }
+      set token $ret
+    }
+
+    default {
+      # no-op
+    }
+  }
+
+  set token
+}
+
+proc tcl_create {args} { return "tcl_tokenize" }
+proc tcl_tokenize {tflags text} {
+  set iToken 1
+  set bSkip 0
+  if {[sqlite3_fts5_locale]=="second"} { set bSkip 1 }
+  foreach {w iStart iEnd} [fts5_tokenize_split $text] {
+    incr iToken
+    if {(($iToken) % ($bSkip + 1))} continue
+
+    set w [transform_token [sqlite3_fts5_locale] $w]
+    sqlite3_fts5_token $w $iStart $iEnd
+  }
+}
+
+#-------------------------------------------------------------------------
+# Check that queries can have a locale attached to them.
+#
+reset_db
+sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
+
+do_execsql_test 1.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=tcl);
+  INSERT INTO t1 VALUES('abc');
+  INSERT INTO t1 VALUES('cba');
+} {}
+
+do_execsql_test 1.1 {
+  SELECT rowid, a FROM t1( fts5_locale('en_US', 'abc') );
+} {1 abc}
+
+do_execsql_test 1.2 {
+  SELECT rowid, a FROM t1( fts5_locale('reverse', 'abc') );
+} {2 cba}
+
+#-------------------------------------------------------------------------
+# Test that the locale= option exists and seems to accept values. And
+# that fts5_locale() values may only be inserted into an internal-content
+# table if the locale=1 option was specified.
+#
+reset_db
+sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
+
+do_execsql_test 2.1 {
+  CREATE VIRTUAL TABLE b1 USING fts5(x, y, locale=1, tokenize=tcl);
+  CREATE VIRTUAL TABLE b2 USING fts5(x, y, locale=0, tokenize=tcl);
+
+  CREATE VIRTUAL TABLE ttt USING fts5vocab('b1', instance);
+}
+
+do_catchsql_test 2.2.1 {
+  CREATE VIRTUAL TABLE b3 USING fts5(x, y, locale=2);
+} {1 {malformed locale=... directive}}
+do_catchsql_test 2.2.2 {
+  CREATE VIRTUAL TABLE b3 USING fts5(x, y, locale=111);
+} {1 {malformed locale=... directive}}
+
+do_catchsql_test 2.3 {
+  INSERT INTO b1(b1, rank) VALUES('locale', 0);
+} {1 {SQL logic error}}
+
+do_execsql_test 2.4 {
+  INSERT INTO b1 VALUES('abc', 'one two three');
+  INSERT INTO b1 VALUES('def', fts5_locale('reverse', 'four five six'));
+}
+
+do_execsql_test 2.5 {
+  INSERT INTO b2 VALUES('abc', 'one two three');
+}
+
+do_catchsql_test 2.6 {
+  INSERT INTO b2 VALUES('def', fts5_locale('reverse', 'four five six'));
+} {1 {fts5_locale() requires locale=1}}
+
+do_execsql_test 2.7  { SELECT rowid FROM b1('one')  } {1}
+do_execsql_test 2.8  { SELECT rowid FROM b1('four') } {}
+do_execsql_test 2.9  { SELECT rowid FROM b1('ruof') } 2
+do_execsql_test 2.10 { SELECT rowid FROM b1(fts5_locale('reverse', 'five'))} 2
+
+do_execsql_test 2.11 {
+  SELECT x, quote(y) FROM b1
+} {
+  abc {'one two three'}
+  def {'four five six'}
+}
+
+do_execsql_test 2.12  { SELECT quote(y) FROM b1('ruof') } {
+  {'four five six'}
+}
+
+do_execsql_test 2.13 {
+  INSERT INTO b1(b1) VALUES('integrity-check');
+}
+do_execsql_test 2.14 {
+  INSERT INTO b1(b1) VALUES('rebuild');
+}
+do_execsql_test 2.15 {
+  INSERT INTO b1(b1) VALUES('integrity-check');
+}
+
+do_execsql_test 2.16 {
+  DELETE FROM b1 WHERE rowid=2
+}
+do_execsql_test 2.17 {
+  INSERT INTO b1(b1) VALUES('integrity-check');
+}
+
+do_execsql_test 2.18 {
+  INSERT INTO b1(rowid, x, y) VALUES(
+      test_setsubtype(45, 76), 'abc def', 'def abc'
+  );
+  INSERT INTO b1(b1) VALUES('integrity-check');
+}
+
+#-------------------------------------------------------------------------
+# Test the 'delete' command with contentless tables.
+#
+reset_db
+sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
+
+do_execsql_test 3.1 {
+  CREATE VIRTUAL TABLE c1 USING fts5(x, content=, tokenize=tcl, locale=1);
+  CREATE VIRTUAL TABLE c2 USING fts5vocab('c1', instance);
+
+  INSERT INTO c1 VALUES('hello world');
+  INSERT INTO c1 VALUES( fts5_locale('reverse', 'one two three') );
+}
+
+do_execsql_test 3.2 {
+  SELECT DISTINCT term FROM c2 ORDER BY 1
+} {
+  eerht eno hello owt world
+}
+
+do_execsql_test 3.3 {
+  INSERT INTO c1(c1, rowid, x) 
+    VALUES('delete', 2, fts5_locale('reverse', 'one two three') );
+}
+
+do_execsql_test 3.4 {
+  SELECT DISTINCT term FROM c2 ORDER BY 1
+} {
+  hello world
+}
+
+#-------------------------------------------------------------------------
+# Test that an UPDATE that updates a subset of the columns does not
+# magically discard the locale from those columns not updated.
+#
+reset_db
+sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
+
+do_execsql_test 4.1 {
+  CREATE VIRTUAL TABLE d1 USING fts5(x, y, locale=1, tokenize=tcl);
+  CREATE VIRTUAL TABLE d2 USING fts5vocab('d1', instance);
+
+  INSERT INTO d1(rowid, x, y) VALUES(1, 'abc', 'def');
+  INSERT INTO d1(rowid, x, y) VALUES(2, 'ghi', fts5_locale('reverse', 'hello'));
+}
+
+do_execsql_test 4.2 {
+  SELECT DISTINCT term FROM d2 ORDER BY 1
+} {
+  abc def ghi olleh
+}
+
+do_execsql_test 4.3 {
+  UPDATE d1 SET x='jkl' WHERE rowid=2;
+}
+
+do_execsql_test 4.4 {
+  SELECT DISTINCT term FROM d2 ORDER BY 1
+} {
+  abc def jkl olleh
+}
+
+do_execsql_test 4.5 {
+  SELECT rowid, * FROM d1
+} {
+  1 abc def
+  2 jkl hello
+}
+
+do_execsql_test 4.6 {
+  UPDATE d1 SET rowid=4 WHERE rowid=2
+}
+
+do_execsql_test 4.7 {
+  SELECT rowid, * FROM d1
+} {
+  1 abc def
+  4 jkl hello
+}
+
+fts5_aux_test_functions db
+
+do_execsql_test 4.8.1 {
+  SELECT fts5_test_columntext(d1) FROM d1('jkl')
+} {{jkl hello}}
+do_execsql_test 4.8.2 {
+  SELECT fts5_test_columntext(d1) FROM d1(fts5_locale('reverse', 'hello'))
+} {{jkl hello}}
+
+do_execsql_test 4.9 {
+  SELECT fts5_test_columnlocale(d1) FROM d1(fts5_locale('reverse', 'hello'))
+} {{{} reverse}}
+
+do_execsql_test 4.10 {
+  SELECT fts5_test_columnlocale(d1) FROM d1
+} {
+  {{} {}}
+  {{} reverse}
+}
+
+#-------------------------------------------------------------------------
+# Test that if an fts5_locale() value is written to an UNINDEXED 
+# column it is stored as text. This is so that blobs and other values
+# can also be stored as is.
+#
+reset_db
+sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
+
+do_execsql_test 5.1 {
+  CREATE VIRTUAL TABLE t1 USING fts5(
+      x, y UNINDEXED, locale=1, tokenize=tcl
+  );
+
+  INSERT INTO t1(rowid, x, y) VALUES(111,
+      fts5_locale('reverse', 'one two three'),
+      fts5_locale('reverse', 'four five six')
+  );
+}
+
+do_execsql_test 5.2 {
+  SELECT rowid, x, y FROM t1
+} {
+  111 {one two three} {four five six}
+}
+
+do_execsql_test 5.3 {
+  SELECT typeof(c0), typeof(c1) FROM t1_content
+} {
+  blob text
+}
+
+#-------------------------------------------------------------------------
+
+foreach {tn opt} {
+  1 {}
+  2 {, columnsize=0}
+} {
+  reset_db
+  sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
+  do_execsql_test 6.$tn.1 "
+    CREATE VIRTUAL TABLE y1 USING fts5(t, locale=1, tokenize=tcl $opt);
+  "
+
+  do_execsql_test 6.$tn.2 {
+    INSERT INTO y1(rowid, t) VALUES
+      (1, fts5_locale('second', 'the city of London')),
+      (2, fts5_locale('second', 'shall have all the old')),
+      (3, fts5_locale('second', 'Liberties and Customs')),
+      (4, fts5_locale('second', 'which it hath been used to have'));
+  }
+
+  fts5_aux_test_functions db
+
+  do_execsql_test 5.$tn.3 {
+    SELECT fts5_test_columnsize(y1) FROM y1
+  } {
+    2 3 2 4
+  }
+
+  do_execsql_test 5.$tn.4 {
+    SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall');
+  } {
+    2 3
+  }
+
+  do_execsql_test 5.$tn.5 {
+    SELECT rowid, fts5_test_columnsize(y1) FROM y1('shall');
+  } {
+    2 3
+  }
+
+  do_execsql_test 5.$tn.6 {
+    SELECT rowid, fts5_test_columnsize(y1) FROM y1('have');
+  } {
+    4 4
+  }
+
+  do_execsql_test 5.$tn.7 {
+    SELECT rowid, highlight(y1, 0, '[', ']') FROM y1('have');
+  } {
+    4 {which it hath been used to [have]}
+  }
+
+  do_execsql_test 5.$tn.8 {
+    SELECT rowid, 
+           highlight(y1, 0, '[', ']'),
+           snippet(y1, 0, '[', ']', '...', 10)
+    FROM y1('Liberties + Customs');
+  } {
+    3 {[Liberties and Customs]}
+      {[Liberties and Customs]}
+  }
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 6.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(x);
+}
+do_catchsql_test 6.1 {
+  INSERT INTO x1(rowid, x) VALUES(123, fts5_locale('en_AU', 'hello world'));
+} {1 {fts5_locale() requires locale=1}}
+
+do_execsql_test 6.2 {
+  SELECT typeof( fts5_locale(NULL, 'xyz') ), typeof( fts5_locale('', 'abc') );
+} {text text}
+
+#--------------------------------------------------------------------------
+# Test that fts5_locale() works with external-content tables.
+#
+reset_db
+sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
+
+do_execsql_test 7.1 {
+  CREATE TABLE t1(ii INTEGER PRIMARY KEY, bb BLOB, tt TEXT, locale TEXT);
+  CREATE VIEW v1 AS 
+      SELECT ii AS rowid, bb, fts5_locale(locale, tt) AS tt FROM t1;
+
+  CREATE VIRTUAL TABLE ft USING fts5(
+      bb, tt, locale=1, tokenize=tcl, content=v1
+  );
+
+  INSERT INTO t1 VALUES(1, NULL, 'one two three', NULL);
+  INSERT INTO t1 VALUES(2, '7800616263', 'four five six', 'reverse');
+  INSERT INTO t1 VALUES(3, '000000007800616263', 'seven eight nine', 'second');
+}
+
+do_execsql_test 7.2 {
+  INSERT INTO ft(ft) VALUES('rebuild');
+  INSERT INTO ft(ft) VALUES('integrity-check');
+}
+
+do_execsql_test 7.3 {
+  SELECT rowid, quote(bb), quote(tt) FROM ft
+} {
+  1 NULL {'one two three'}
+  2 '7800616263' {'four five six'}
+  3 '000000007800616263' {'seven eight nine'}
+}
+
+do_execsql_test 7.4 { SELECT rowid FROM ft('six'); } 
+do_execsql_test 7.5 { SELECT rowid FROM ft(fts5_locale('reverse','six')); } 2
+
+fts5_aux_test_functions db
+
+do_execsql_test 7.6 {
+  SELECT fts5_test_columnlocale(ft) FROM ft;
+} {
+  {{} {}} {{} reverse} {{} second}
+}
+
+#-------------------------------------------------------------------------
+# Test that the porter tokenizer works with locales.
+#
+reset_db
+sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
+
+do_execsql_test 8.1 {
+  CREATE VIRTUAL TABLE ft USING fts5(tt, locale=1, tokenize="porter tcl");
+  CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', instance);
+
+  INSERT INTO ft(rowid, tt) VALUES
+    (111, fts5_locale('second', 'the porter tokenizer is a wrapper tokenizer')),
+    (222, fts5_locale('reverse', 'This value may also be set'));
+}
+
+do_execsql_test 8.1 {
+  SELECT DISTINCT term FROM vocab ORDER BY 1
+} {
+  a eb eulav osla sihT te the token yam
+}
+
+#-------------------------------------------------------------------------
+# Test that position-lists (used by xInst, xPhraseFirst etc.) work with
+# locales and modes other than detail=full.
+#
+foreach {tn detail} {
+  1 detail=full
+  2 detail=none
+  3 detail=column
+} {
+  reset_db
+  sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
+  do_execsql_test 9.$tn.0 "
+    CREATE VIRTUAL TABLE ft USING fts5(tt, locale=1, tokenize=tcl, $detail);
+  "
+  do_execsql_test 9.$tn.1 {
+    CREATE VIRTUAL TABLE vocab USING fts5vocab('ft', instance);
+    INSERT INTO ft(rowid, tt) VALUES
+      (-1, fts5_locale('second', 'it is an ancient mariner'));
+  }
+
+  do_execsql_test 9.$tn.2 {
+    SELECT DISTINCT term FROM vocab 
+  } {an it mariner}
+
+  do_execsql_test 9.$tn.3 {
+    SELECT highlight(ft, 0, '[', ']') FROM ft('mariner')
+  } {{it is an ancient [mariner]}}
+}
+
+#-------------------------------------------------------------------------
+# Check some corrupt fts5_locale() blob formats are detected.
+#
+foreach_detail_mode $::testprefix {
+
+  reset_db
+  sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
+  fts5_aux_test_functions db
+  do_execsql_test 10.1 {
+    CREATE TABLE x1(ii INTEGER PRIMARY KEY, x);
+    CREATE VIRTUAL TABLE ft USING fts5(x, 
+        content=x1, content_rowid=ii, locale=1, detail=%DETAIL%, columnsize=0
+    );
+
+    CREATE VIRTUAL TABLE ft2 USING fts5(
+        x, locale=1, detail=%DETAIL%, columnsize=0
+    );
+  }
+  
+  foreach {tn v} {
+    1   X'001122'
+    2   X'0011223344'
+    3   X'00E0B2EB68656c6c6f'
+    4   X'00E0B2EB0068656c6c6f'
+  } {
+    do_execsql_test 10.2.$tn.0 { INSERT INTO ft(ft) VALUES('delete-all') }
+    do_execsql_test 10.2.$tn.1 { DELETE FROM x1; }
+    do_execsql_test 10.2.$tn.2 " INSERT INTO x1 VALUES(NULL, $v) "
+  
+    do_catchsql_test 10.2.$tn.3 {
+      INSERT INTO ft(ft) VALUES('rebuild');
+    } {1 {SQL logic error}}
+  
+    do_catchsql_test 10.2.$tn.4 "
+      SELECT * FROM ft( test_setsubtype($v, 76) );
+    " {1 {SQL logic error}}
+  
+    do_execsql_test 10.2.$tn.5 {
+      INSERT INTO ft(rowid, x) VALUES(1, 'hello world');
+    }
+  
+    if {"%DETAIL%"!="full"} {
+      do_catchsql_test 10.2.$tn.6 {
+        SELECT fts5_test_poslist(ft) FROM ft('world');
+      } {1 SQLITE_ERROR}
+  
+      do_catchsql_test 10.2.$tn.7 {
+        SELECT fts5_test_columnsize(ft) FROM ft('world');
+      } {1 SQLITE_ERROR}
+
+      do_catchsql_test 10.2.$tn.7 {
+        SELECT fts5_test_columnlocale(ft) FROM ft('world');
+      } {1 SQLITE_ERROR}
+    }
+
+    do_catchsql_test 10.2.$tn.8 {
+      SELECT * FROM ft('hello')
+    } {1 {SQL logic error}}
+
+    do_catchsql_test 10.2.$tn.9 {
+      PRAGMA integrity_check;
+    } {0 ok}
+
+    do_execsql_test 10.2.$tn.10 {
+      DELETE FROM x1;
+      INSERT INTO x1(ii, x) VALUES(1, 'hello world');
+    }
+
+    do_catchsql_test 10.2.$tn.11 "
+      INSERT INTO ft(ft, rowid, x) VALUES('delete', 1, test_setsubtype($v,76) )
+    " {1 {SQL logic error}}
+
+    do_catchsql_test 10.2.$tn.12 "
+      INSERT INTO ft(rowid, x) VALUES(2, test_setsubtype($v,76) )
+    " {1 {SQL logic error}}
+
+    do_execsql_test 10.2.$tn.13 {
+      INSERT INTO ft2(rowid, x) VALUES(1, 'hello world');
+    }
+    do_execsql_test 10.2.$tn.14 "UPDATE ft2_content SET c0=$v"
+
+    do_catchsql_test 10.2.$tn.15 {
+      PRAGMA integrity_check;
+    } {1 {SQL logic error}}
+
+    do_execsql_test 10.2.$tn.16 {
+      DELETE FROM ft2_content;
+      INSERT INTO ft2(ft2) VALUES('rebuild');
+    }
+  }
+
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+sqlite3_fts5_create_tokenizer -v2 db tcl tcl_create
+fts5_aux_test_functions db
+do_execsql_test 11.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(abc, locale=1);
+  INSERT INTO x1(rowid, abc) VALUES(123, fts5_locale('en_US', 'one two three'));
+}
+
+do_catchsql_test 11.1 {
+  SELECT fts5_columnlocale(x1, -1) FROM x1('two');
+} {1 SQLITE_RANGE}
+do_catchsql_test 11.2 {
+  SELECT fts5_columnlocale(x1, 1) FROM x1('two');
+} {1 SQLITE_RANGE}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_test 12.0 {
+  list [catch {
+    sqlite3_fts5_create_tokenizer -v2 -version 3 db tcl tcl_create
+  } msg] $msg
+} {1 {error in fts5_api.xCreateTokenizer_v2()}}
+
+#-------------------------------------------------------------------------
+# Tests for auxiliary function fts5_get_locale().
+#
+reset_db
+
+# Check that if the table does not support locale=1, fts5_get_locale()
+# always returns NULL.
+do_execsql_test 13.1.0 {
+  CREATE VIRTUAL TABLE nolocale USING fts5(a, b);
+  INSERT INTO nolocale VALUES('one two three', 'four five six');
+  INSERT INTO nolocale VALUES('three two one', 'seven eight nine');
+}
+do_execsql_test 13.1.1 {
+  SELECT fts5_get_locale(nolocale, 0) IS NULL FROM nolocale;
+} {1 1}
+do_execsql_test 13.1.2 {
+  SELECT fts5_get_locale(nolocale, 1) IS NULL FROM nolocale('one + two');
+} {1}
+do_execsql_test 13.1.3 {
+  SELECT fts5_get_locale(nolocale, 0) IS NULL FROM nolocale('one AND two');
+} {1 1}
+do_execsql_test 13.1.4 {
+  SELECT 
+      fts5_get_locale(nolocale, 1) IS NULL 
+  FROM nolocale('three AND two') ORDER BY rank
+} {1 1}
+do_catchsql_test 13.1.5 {
+  SELECT fts5_get_locale(nolocale, 2) IS NULL FROM nolocale('three AND two');
+} {1 {column index out of range}}
+do_catchsql_test 13.1.6 {
+  SELECT fts5_get_locale(nolocale, -1) IS NULL FROM nolocale('three AND two');
+} {1 {column index out of range}}
+do_catchsql_test 13.1.7 {
+  SELECT fts5_get_locale(nolocale) IS NULL FROM nolocale('three AND two');
+} {1 {wrong number of arguments to function fts5_get_locale()}}
+do_catchsql_test 13.1.8 {
+  SELECT fts5_get_locale(nolocale, 0, 0) IS NULL FROM nolocale('three AND two');
+} {1 {wrong number of arguments to function fts5_get_locale()}}
+do_catchsql_test 13.1.9 {
+  SELECT fts5_get_locale(nolocale, 'text') FROM nolocale('three AND two');
+} {1 {non-integer argument passed to function fts5_get_locale()}}
+
+
+# Check that if the table does support locale=1, fts5_get_locale()
+# returns the locale of the identified row/column.
+do_execsql_test 13.2.0 {
+  CREATE VIRTUAL TABLE ft USING fts5(a, b, locale=1);
+  INSERT INTO ft VALUES(
+      fts5_locale('th_TH', 'one two three'), 'four five six seven'
+  );
+  INSERT INTO ft VALUES(
+      'three two one', fts5_locale('en_AU', 'seven eight nine')
+  );
+}
+
+do_execsql_test 13.2.1 {
+  SELECT quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) FROM ft
+} { 'th_TH' NULL NULL 'en_AU' }
+do_execsql_test 13.2.2 {
+  SELECT 
+    quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) 
+  FROM ft('one AND three')
+} { 'th_TH' NULL NULL 'en_AU' }
+do_execsql_test 13.2.3 {
+  SELECT 
+    quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) 
+  FROM ft('one AND three') ORDER BY rank
+} { NULL 'en_AU' 'th_TH' NULL }
+do_execsql_test 13.2.4 {
+  SELECT 
+    quote(fts5_get_locale(ft, 0)), quote(fts5_get_locale(ft, 1)) 
+  FROM ft('one AND three') ORDER BY rowid
+} { 'th_TH' NULL NULL 'en_AU' }
+
+do_execsql_test 13.2.5 {
+  SELECT 
+    quote(fts5_get_locale(ft, '0')), quote(fts5_get_locale(ft, 1)) 
+  FROM ft('one AND three') ORDER BY rowid
+} { 'th_TH' NULL NULL 'en_AU' }
+
+do_catchsql_test 13.2.6 {
+  SELECT 
+    quote(fts5_get_locale(ft, '0.0')), quote(fts5_get_locale(ft, 1)) 
+  FROM ft('one AND three') ORDER BY rowid
+} {1 {non-integer argument passed to function fts5_get_locale()}}
+do_catchsql_test 13.2.7 {
+  SELECT 
+    quote(fts5_get_locale(ft, 0.0)), quote(fts5_get_locale(ft, 1)) 
+  FROM ft('one AND three') ORDER BY rowid
+} {1 {non-integer argument passed to function fts5_get_locale()}}
+
+finish_test
+
@@ -21,8 +21,6 @@ ifcapable !fts5 {
  return
 }

-if 0 {
-
 do_execsql_test 1.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a);
 }
@@ -37,21 +35,21 @@ do_catchsql_test 1.1.2 {

 do_catchsql_test 1.2.1 { 
  SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*id'); 
-} {0 {{}}}
+} {1 {no such cursor: 4}}

 do_catchsql_test 1.2.2 {
  SELECT a FROM t1
    WHERE rank = (SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*id'));
-} {0 {}}
+} {1 {no such cursor: 6}}

 do_catchsql_test 1.3.1 { 
  SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads'); 
-} {1 {no such cursor: 2}}
+} {1 {no such cursor: 1}}

 do_catchsql_test 1.3.2 {
  SELECT a FROM t1
    WHERE rank = (SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads'));
-} {1 {no such cursor: 2}}
+} {1 {no such cursor: 1}}

 db close
 sqlite3 db test.db
@@ -61,6 +59,11 @@ do_catchsql_test 1.3.3 {
    WHERE rank = (SELECT highlight(t1, 4, '<b>', '</b>') FROM t1('*reads'));
 } {1 {no such cursor: 1}}

+fts5_aux_test_functions db
+do_catchsql_test 1.3.4 {
+  SELECT fts5_columntext(t1) FROM t1('*reads');
+} {1 {no such cursor: 1}}
+
 #-------------------------------------------------------------------------
 reset_db
 do_execsql_test 2.0 {
@@ -569,24 +572,98 @@ do_execsql_test 20.5 {
 } {3 1}

 #-------------------------------------------------------------------------
-}
 reset_db
 do_execsql_test 21.0 {
+  CREATE TABLE t1(ii INTEGER, x TEXT, y TEXT);
+  CREATE VIRTUAL TABLE xyz USING fts5(content_rowid=ii, content=t1, x, y);
+  INSERT INTO t1 VALUES(1, 'one', 'i');
+  INSERT INTO t1 VALUES(2, 'two', 'ii');
+  INSERT INTO t1 VALUES(3, 'tree', 'iii');
+  INSERT INTO xyz(xyz) VALUES('rebuild');
+}
+
+do_execsql_test 21.1 {
+  UPDATE xyz SET y='TWO' WHERE rowid=2;
+  UPDATE t1 SET y='TWO' WHERE ii=2;
+}
+
+do_execsql_test 21.2 {
+  PRAGMA integrity_check
+} {ok}
+
+breakpoint
+sqlite3_db_config db DEFENSIVE 1
+do_execsql_test 21.3 {
+  CREATE TABLE xyz_notashadow(x, y);
+  DROP TABLE xyz_notashadow;
+}
+sqlite3_db_config db DEFENSIVE 0
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 22.0 {
+  SELECT fts5(NULL);
+} {{}}
+do_execsql_test 22.1 {
+  SELECT count(*) FROM (
+    SELECT fts5_source_id()
+  )
+} {1}
+execsql_pp {
+  SELECT fts5_source_id()
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 23.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(x);
+  INSERT INTO x1 VALUES('one + two + three');
+  INSERT INTO x1 VALUES('one + xyz + three');
+  INSERT INTO x1 VALUES('xyz + two + xyz');
+}
+do_execsql_test 23.1 {
+  SELECT rowid FROM x1('one + two + three');
+} {1}
+
+do_execsql_test 23.2 {
+  SELECT rowid FROM x1('^".." AND one');
+} {}
+
+do_execsql_test 23.3 {
+  SELECT rowid FROM x1('abc NEAR ".." NEAR def');
+} {}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 24.0 {
  CREATE VIRTUAL TABLE t1 USING fts5(a, detail='none');
  INSERT INTO t1(a) VALUES('a');
 }

-do_execsql_test 21.2 {
+do_execsql_test 24.2 {
  SELECT rank FROM ( SELECT rank FROM t1('a NOT "" NOT def') ) ORDER BY 1;
 } {-1e-06}

-do_execsql_test 21.3 {
+do_execsql_test 24.3 {
  SELECT rank FROM ( SELECT rank FROM t1('a NOT � NOT def') ) ORDER BY 1;
 } {-1e-06}

-do_execsql_test 21.4 {
+do_execsql_test 24.4 {
  SELECT rank FROM ( SELECT rank FROM t1('a NOT "" NOT def') );
 } {-1e-06}

+#-------------------------------------------------------------------------
+reset_db
+fts5_aux_test_functions db
+
+do_execsql_test 25.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, detail='none', content='');
+  INSERT INTO t1(a) VALUES('a b c');
+}
+
+do_execsql_test 25.0 {
+  SELECT fts5_test_poslist(t1) FROM t1('b') ORDER BY rank;
+} {{}}
+
 finish_test

@@ -261,8 +261,8 @@ do_execsql_test 5.3 {
 reset_db
 sqlite3_fts5_register_origintext db
 do_execsql_test 6.0 {
-  CREATE VIRTUAL TABLE ft USING fts5(
-      x, y, tokenize='origintext unicode61', detail=%DETAIL%
+  CREATE VIRTUAL TABLE ft USING fts5( 
+      x, y, tokenize='origintext unicode61', detail=%DETAIL%, tokendata=0
  );

  INSERT INTO ft VALUES('One Two', 'Three two');
@@ -291,6 +291,22 @@ do_execsql_test 6.3 {
  SELECT rowid, tokens(ft) FROM ft('Three*');
 } {1 {{}} 2 {{}}}

+fts5_aux_test_functions db
+do_catchsql_test 6.4 {
+  SELECT fts5_test_insttoken(ft, -1, 0) FROM ft('one');
+} {1 SQLITE_RANGE}
+
+do_catchsql_test 6.5 {
+  SELECT fts5_test_insttoken(ft, 1, 0) FROM ft('one');
+} {1 SQLITE_RANGE}
+
+do_catchsql_test 6.6 {
+  CREATE VIRTUAL TABLE ft2 USING fts5(x, tokendata=2);
+} {1 {malformed tokendata=... directive}}
+do_catchsql_test 6.7 {
+  CREATE VIRTUAL TABLE ft2 USING fts5(x, content='', tokendata=11);
+} {1 {malformed tokendata=... directive}}
+
 }

 finish_test
@@ -58,6 +58,10 @@ do_execsql_test 2.1 {
  pragma quick_check;
 } {ok}

+do_catchsql_test 2.2 {
+  INSERT INTO xyz(xyz, rank) VALUES('secure-delete', 'hello world');
+} {1 {SQL logic error}}
+



@@ -480,4 +480,33 @@ do_execsql_test 22.0 {
 do_catchsql_test 22.1 {SELECT * FROM x1('')}   {1 {fts5: syntax error near ""}}
 do_catchsql_test 22.2 {SELECT * FROM x1(NULL)} {1 {fts5: syntax error near ""}}

+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 23.0 {
+  CREATE VIRTUAL TABLE x1 USING fts5(x);
+  SELECT count(*) FROM x1_data;
+} {2}
+
+do_execsql_test 23.1 {
+  BEGIN;
+    INSERT INTO x1 VALUES('a b c d');
+    INSERT INTO x1 VALUES('a b c d');
+    INSERT INTO x1 VALUES('a b c d');
+}
+
+do_execsql_test 23.2 {
+  SELECT count(*) FROM x1_data;
+} {2}
+
+do_execsql_test 23.3 {
+  INSERT INTO x1(x1) VALUES('flush');
+  SELECT count(*) FROM x1_data;
+} {3}
+
+do_execsql_test 23.4 {
+  ROLLBACK;
+  SELECT count(*) FROM x1_data;
+} {2}
+
+
 finish_test
@@ -0,0 +1,77 @@
+# 2024 Aug 10
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Tests focusing on the built-in fts5 tokenizers. 
+#
+
+source [file join [file dirname [info script]] fts5_common.tcl]
+set testprefix fts5tokenizer3
+
+# If SQLITE_ENABLE_FTS5 is not defined, omit this file.
+ifcapable !fts5 {
+  finish_test
+  return
+}
+
+
+proc get_sod {args} { return "split_on_dot" }
+proc get_lowercase    {args} { return "lowercase" }
+
+proc lowercase {flags txt} {
+  set n [string length $txt]
+  sqlite3_fts5_token [string tolower $txt] 0 $n
+  return 0
+}
+
+proc split_on_dot {flags txt} {
+  set iOff 0
+  foreach t [split $txt "."] {
+    set n [string length $txt]
+    sqlite3_fts5_token $t $iOff [expr $iOff+$n]
+    incr iOff [expr {$n+1}]
+  }
+  return ""
+}
+
+foreach {tn script} {
+  1 {
+    sqlite3_fts5_create_tokenizer db lowercase get_lowercase
+    sqlite3_fts5_create_tokenizer -parent lowercase db split_on_dot get_sod
+  }
+  2 {
+    sqlite3_fts5_create_tokenizer -v2 db lowercase get_lowercase
+    sqlite3_fts5_create_tokenizer -parent lowercase db split_on_dot get_sod
+  }
+  3 {
+    sqlite3_fts5_create_tokenizer db lowercase get_lowercase
+    sqlite3_fts5_create_tokenizer -v2 -parent lowercase db split_on_dot get_sod
+  }
+  4 {
+    sqlite3_fts5_create_tokenizer -v2 db lowercase get_lowercase
+    sqlite3_fts5_create_tokenizer -v2 -parent lowercase db split_on_dot get_sod
+  }
+} {
+  reset_db
+  eval $script
+
+  do_execsql_test 1.$tn.0 {
+    CREATE VIRTUAL TABLE t1 USING fts5(x, tokenize=split_on_dot);
+    CREATE VIRTUAL TABLE t1vocab USING fts5vocab(t1, instance);
+    INSERT INTO t1 VALUES('ABC.Def.ghi');
+  }
+
+  do_execsql_test 1.$tn.1 {
+    SELECT term FROM t1vocab ORDER BY 1
+  } {abc def ghi}
+}
+
+
+finish_test
@@ -56,6 +56,7 @@ foreach {tn like res} {
  7 {ABCDEFG%} 1
  8 {%รุงเ%}    2
  9 {%งเ%}     2
+  10 {%"งเ"%}  {}
 } {
  do_execsql_test 1.3.$tn {
    SELECT rowid FROM t1 WHERE y LIKE $like
@@ -200,6 +201,12 @@ do_eqp_test 6.3 {
 do_eqp_test 6.4 {
  SELECT * FROM ci1 WHERE x GLOB ?
 } {VIRTUAL TABLE INDEX 0:G0}
+do_eqp_test 6.5 {
+  SELECT * FROM ci1 WHERE x < ?
+} {{SCAN ci1 VIRTUAL TABLE INDEX 0:}}
+do_eqp_test 6.6 {
+  SELECT * FROM ci0 WHERE x < ?
+} {{SCAN ci0 VIRTUAL TABLE INDEX 0:}}

 reset_db
 do_execsql_test 7.0 {
@@ -256,4 +263,85 @@ do_execsql_test 8.3 {
  {[abcde]}
 }

+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 9.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(
+    a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, 
+    tokenize=trigram
+  );
+
+  INSERT INTO t1(rowid, a12) VALUES(111, 'thats a tricky case though');
+  INSERT INTO t1(rowid, a12) VALUES(222, 'the query planner cannot do');
+}
+
+do_execsql_test 9.1 {
+  SELECT rowid FROM t1 WHERE a12 LIKE '%tricky%'
+} {111}
+
+do_execsql_test 9.2 {
+  SELECT rowid FROM t1 WHERE a12 LIKE '%tricky%' AND a12 LIKE '%case%'
+} {111}
+
+do_execsql_test 9.3 {
+  SELECT rowid FROM t1 WHERE a12 LIKE NULL
+} {}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 10.0 {
+  CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=trigram);
+}
+
+do_test 10.1 {
+  foreach {val} {
+    "abc \UFFjkl\UFF"
+    "abc \UFFFjkl\UFFF"
+    "abc \UFFFFjkl\UFFFF"
+    "abc \UFFFFFjkl\UFFFFF"
+    "\UFFjkl\UFF abc"
+    "\UFFFjkl\UFFF abc"
+    "\UFFFFjkl\UFFFF abc"
+    "\UFFFFFjkl\UFFFFF abc"
+    "\U10001jkl\U10001 abc"
+  } {
+    execsql { INSERT INTO t1 VALUES( $val ) }
+  }
+} {}
+
+do_test 10.2 {
+  foreach {val} {
+    X'E18000626320646566'
+    X'61EDA0806320646566'
+    X'61EDA0806320646566'
+    X'61EFBFBE6320646566'
+    X'76686920E18000626320646566'
+    X'7668692061EDA0806320646566'
+    X'7668692061EDA0806320646566'
+    X'7668692061EFBFBE6320646566'
+  } {
+    execsql " INSERT INTO t1 VALUES( $val ) "
+  }
+} {}
+
+do_test 10.3 {
+  set a [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0x62}]
+  set b [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0x62}]
+  set c [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}]
+  set d [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}]
+  execsql {
+    INSERT INTO t1 VALUES($a);
+    INSERT INTO t1 VALUES($b);
+    INSERT INTO t1 VALUES($c);
+    INSERT INTO t1 VALUES($d);
+
+    INSERT INTO t1 VALUES('abcd' || $a);
+    INSERT INTO t1 VALUES('abcd' || $b);
+    INSERT INTO t1 VALUES('abcd' || $c);
+    INSERT INTO t1 VALUES('abcd' || $d);
+  }
+} {}
+
+
+
 finish_test
@@ -470,4 +470,24 @@ do_execsql_test 8.2.3 {
  SELECT rowid FROM t4 WHERE t4 MATCH 'a' ORDER BY rowid ASC;
 } {2 4}

+#-------------------------------------------------------------------------
+
+foreach {tn val bErr} {
+  1 0 0
+  2 1 0
+  3 2 0
+  4 3  1
+  5 11 1
+} {
+  reset_db
+  set aRes(0) {0 {}}
+  set aRes(1) {1 {error in tokenizer constructor}}
+  set res $aRes($bErr)
+  do_catchsql_test 9.1.$tn "
+    CREATE VIRTUAL TABLE bl USING fts5(
+        s, tokenize='trigram remove_diacritics $val'
+    );
+  " $res
+}
+
 finish_test
@@ -90,6 +90,26 @@
 ** and a very large cost if either start or stop are unavailable.  This
 ** encourages the query planner to order joins such that the bounds of the
 ** series are well-defined.
+**
+** Update on 2024-08-22:
+** xBestIndex now also looks for equality and inequality constraints against
+** the value column and uses those constraints as additional bounds against
+** the sequence range.  Thus, a query like this:
+**
+**     SELECT value FROM generate_series($SA,$EA)
+**      WHERE value BETWEEN $SB AND $EB;
+**
+** Is logically the same as:
+**
+**     SELECT value FROM generate_series(max($SA,$SB),min($EA,$EB));
+**
+** Constraints on the value column can server as substitutes for constraints
+** on the hidden start and stop columns.  So, the following two queries
+** are equivalent:
+**
+**     SELECT value FROM generate_series($S,$E);
+**     SELECT value FROM generate_series WHERE value BETWEEN $S and $E;
+**
 */
 #include "sqlite3ext.h"
 SQLITE_EXTENSION_INIT1
@@ -131,8 +151,10 @@ static sqlite3_int64 genSeqMember(
 typedef unsigned char u8;

 typedef struct SequenceSpec {
-  sqlite3_int64 iBase;         /* Starting value ("start") */
-  sqlite3_int64 iTerm;         /* Given terminal value ("stop") */
+  sqlite3_int64 iOBase;        /* Original starting value ("start") */
+  sqlite3_int64 iOTerm;        /* Original terminal value ("stop") */
+  sqlite3_int64 iBase;         /* Starting value to actually use */
+  sqlite3_int64 iTerm;         /* Terminal value to actually use */
  sqlite3_int64 iStep;         /* Increment ("step") */
  sqlite3_uint64 uSeqIndexMax; /* maximum sequence index (aka "n") */
  sqlite3_uint64 uSeqIndexNow; /* Current index during generation */
@@ -325,9 +347,9 @@ static int seriesColumn(
  series_cursor *pCur = (series_cursor*)cur;
  sqlite3_int64 x = 0;
  switch( i ){
-    case SERIES_COLUMN_START:  x = pCur->ss.iBase; break;
-    case SERIES_COLUMN_STOP:   x = pCur->ss.iTerm; break;
-    case SERIES_COLUMN_STEP:   x = pCur->ss.iStep;   break;
+    case SERIES_COLUMN_START:  x = pCur->ss.iOBase;     break;
+    case SERIES_COLUMN_STOP:   x = pCur->ss.iOTerm;     break;
+    case SERIES_COLUMN_STEP:   x = pCur->ss.iStep;      break;
    default:                   x = pCur->ss.iValueNow;  break;
  }
  sqlite3_result_int64(ctx, x);
@@ -335,7 +357,9 @@ static int seriesColumn(
 }

 #ifndef LARGEST_UINT64
+#define LARGEST_INT64  (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32))
 #define LARGEST_UINT64 (0xffffffff|(((sqlite3_uint64)0xffffffff)<<32))
+#define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64)
 #endif

 /*
@@ -376,13 +400,18 @@ static int seriesEof(sqlite3_vtab_cursor *cur){
 ** parameter.  (idxStr is not used in this implementation.)  idxNum
 ** is a bitmask showing which constraints are available:
 **
-**   0x01:    start=VALUE
-**   0x02:    stop=VALUE
-**   0x04:    step=VALUE
-**   0x08:    descending order
-**   0x10:    ascending order
-**   0x20:    LIMIT  VALUE
-**   0x40:    OFFSET  VALUE
+**   0x0001:    start=VALUE
+**   0x0002:    stop=VALUE
+**   0x0004:    step=VALUE
+**   0x0008:    descending order
+**   0x0010:    ascending order
+**   0x0020:    LIMIT  VALUE
+**   0x0040:    OFFSET  VALUE
+**   0x0080:    value=VALUE
+**   0x0100:    value>=VALUE
+**   0x0200:    value>VALUE
+**   0x1000:    value<=VALUE
+**   0x2000:    value<VALUE
 **
 ** This routine should initialize the cursor and position it so that it
 ** is pointing at the first row, or pointing off the end of the table
@@ -395,6 +424,12 @@ static int seriesFilter(
 ){
  series_cursor *pCur = (series_cursor *)pVtabCursor;
  int i = 0;
+  int returnNoRows = 0;
+  sqlite3_int64 iMin = SMALLEST_INT64;
+  sqlite3_int64 iMax = LARGEST_INT64;
+  sqlite3_int64 iLimit = 0;
+  sqlite3_int64 iOffset = 0;
+
  (void)idxStrUnused;
  if( idxNum & 0x01 ){
    pCur->ss.iBase = sqlite3_value_int64(argv[i++]);
@@ -416,16 +451,98 @@ static int seriesFilter(
  }else{
    pCur->ss.iStep = 1;
  }
+
+  /* If there are constraints on the value column but there are
+  ** no constraints on  the start, stop, and step columns, then
+  ** initialize the default range to be the entire range of 64-bit signed
+  ** integers.  This range will contracted by the value column constraints
+  ** further below.
+  */
+  if( (idxNum & 0x05)==0 && (idxNum & 0x0380)!=0 ){
+    pCur->ss.iBase = SMALLEST_INT64;
+  }
+  if( (idxNum & 0x06)==0 && (idxNum & 0x3080)!=0 ){
+    pCur->ss.iTerm = LARGEST_INT64;
+  }
+  pCur->ss.iOBase = pCur->ss.iBase;
+  pCur->ss.iOTerm = pCur->ss.iTerm;
+
+  /* Extract the LIMIT and OFFSET values, but do not apply them yet.
+  ** The range must first be constrained by the limits on value.
+  */
  if( idxNum & 0x20 ){
-    sqlite3_int64 iLimit = sqlite3_value_int64(argv[i++]);
-    sqlite3_int64 iTerm;
+    iLimit = sqlite3_value_int64(argv[i++]);
    if( idxNum & 0x40 ){
-      sqlite3_int64 iOffset = sqlite3_value_int64(argv[i++]);
-      if( iOffset>0 ){
-        pCur->ss.iBase += pCur->ss.iStep*iOffset;
+      iOffset = sqlite3_value_int64(argv[i++]);
+    }
+  }
+
+  if( idxNum & 0x3380 ){
+    /* Extract the maximum range of output values determined by
+    ** constraints on the "value" column.
+    */
+    if( idxNum & 0x0080 ){
+      iMin = iMax = sqlite3_value_int64(argv[i++]);
+    }else{
+      if( idxNum & 0x0300 ){
+        iMin = sqlite3_value_int64(argv[i++]);
+        if( idxNum & 0x0200 ){
+          if( iMin==LARGEST_INT64 ){
+            returnNoRows = 1;
+          }else{
+            iMin++;
+          }
+        }
+      }
+      if( idxNum & 0x3000 ){
+        iMax = sqlite3_value_int64(argv[i++]);
+        if( idxNum & 0x2000 ){
+          if( iMax==SMALLEST_INT64 ){
+            returnNoRows = 1;
+          }else{
+            iMax--;
+          }
+        }
+      }
+      if( iMin>iMax ){
+        returnNoRows = 1;
      }
    }
+
+    /* Try to reduce the range of values to be generated based on
+    ** constraints on the "value" column.
+    */
+    if( pCur->ss.iStep>0 ){
+      sqlite3_int64 szStep = pCur->ss.iStep;
+      if( pCur->ss.iBase<iMin ){
+        sqlite3_uint64 d = iMin - pCur->ss.iBase;
+        pCur->ss.iBase += ((d+szStep-1)/szStep)*szStep;
+      }
+      if( pCur->ss.iTerm>iMax ){
+        sqlite3_uint64 d = pCur->ss.iTerm - iMax;
+        pCur->ss.iTerm -= ((d+szStep-1)/szStep)*szStep;
+      }
+    }else{
+      sqlite3_int64 szStep = -pCur->ss.iStep;
+      assert( szStep>0 );
+      if( pCur->ss.iBase>iMax ){
+        sqlite3_uint64 d = pCur->ss.iBase - iMax;
+        pCur->ss.iBase -= ((d+szStep-1)/szStep)*szStep;
+      }
+      if( pCur->ss.iTerm<iMin ){
+        sqlite3_uint64 d = iMin - pCur->ss.iTerm;
+        pCur->ss.iTerm += ((d+szStep-1)/szStep)*szStep;
+      }
+    }
+  }
+
+  /* Apply LIMIT and OFFSET constraints, if any */
+  if( idxNum & 0x20 ){
+    if( iOffset>0 ){
+      pCur->ss.iBase += pCur->ss.iStep*iOffset;
+    }
    if( iLimit>=0 ){
+      sqlite3_int64 iTerm;
      iTerm = pCur->ss.iBase + (iLimit - 1)*pCur->ss.iStep;
      if( pCur->ss.iStep<0 ){
        if( iTerm>pCur->ss.iTerm ) pCur->ss.iTerm = iTerm;
@@ -434,16 +551,21 @@ static int seriesFilter(
      }
    }
  }
+
+
  for(i=0; i<argc; i++){
    if( sqlite3_value_type(argv[i])==SQLITE_NULL ){
      /* If any of the constraints have a NULL value, then return no rows.
      ** See ticket https://www.sqlite.org/src/info/fac496b61722daf2 */
-      pCur->ss.iBase = 1;
-      pCur->ss.iTerm = 0;
-      pCur->ss.iStep = 1;
+      returnNoRows = 1;
      break;
    }
  }
+  if( returnNoRows ){
+    pCur->ss.iBase = 1;
+    pCur->ss.iTerm = 0;
+    pCur->ss.iStep = 1;
+  }
  if( idxNum & 0x08 ){
    pCur->ss.isReversing = pCur->ss.iStep > 0;
  }else{
@@ -464,13 +586,35 @@ static int seriesFilter(
 **
 ** The query plan is represented by bits in idxNum:
 **
-**   0x01  start = $value  -- constraint exists
-**   0x02  stop = $value   -- constraint exists
-**   0x04  step = $value   -- constraint exists
-**   0x08  output is in descending order
-**   0x10  output is in ascending order
-**   0x20  LIMIT $value    -- constraint exists
-**   0x40  OFFSET $value   -- constraint exists
+**   0x0001  start = $num
+**   0x0002  stop = $num
+**   0x0004  step = $num
+**   0x0008  output is in descending order
+**   0x0010  output is in ascending order
+**   0x0020  LIMIT $num
+**   0x0040  OFFSET $num
+**   0x0080  value = $num
+**   0x0100  value >= $num
+**   0x0200  value > $num
+**   0x1000  value <= $num
+**   0x2000  value < $num
+**
+** Only one of 0x0100 or 0x0200 will be returned.  Similarly, only
+** one of 0x1000 or 0x2000 will be returned.  If the 0x0080 is set, then
+** none of the 0xff00 bits will be set.
+**
+** The order of parameters passed to xFilter is as follows:
+**
+**    * The argument to start= if bit 0x0001 is in the idxNum mask
+**    * The argument to stop= if bit 0x0002 is in the idxNum mask
+**    * The argument to step= if bit 0x0004 is in the idxNum mask
+**    * The argument to LIMIT if bit 0x0020 is in the idxNum mask
+**    * The argument to OFFSET if bit 0x0040 is in the idxNum mask
+**    * The argument to value=, or value>= or value> if any of
+**      bits 0x0380 are in the idxNum mask
+**    * The argument to value<= or value< if either of bits 0x3000
+**      are in the mask
+**
 */
 static int seriesBestIndex(
  sqlite3_vtab *pVTab,
@@ -483,7 +627,9 @@ static int seriesBestIndex(
 #endif
  int unusableMask = 0;  /* Mask of unusable constraints */
  int nArg = 0;          /* Number of arguments that seriesFilter() expects */
-  int aIdx[5];           /* Constraints on start, stop, step, LIMIT, OFFSET */
+  int aIdx[7];           /* Constraints on start, stop, step, LIMIT, OFFSET,
+                         ** and value.  aIdx[5] covers value=, value>=, and
+                         ** value>,  aIdx[6] covers value<= and value< */
  const struct sqlite3_index_constraint *pConstraint;

  /* This implementation assumes that the start, stop, and step columns
@@ -491,7 +637,7 @@ static int seriesBestIndex(
  assert( SERIES_COLUMN_STOP == SERIES_COLUMN_START+1 );
  assert( SERIES_COLUMN_STEP == SERIES_COLUMN_START+2 );

-  aIdx[0] = aIdx[1] = aIdx[2] = aIdx[3] = aIdx[4] = -1;
+  aIdx[0] = aIdx[1] = aIdx[2] = aIdx[3] = aIdx[4] = aIdx[5] = aIdx[6] = -1;
  pConstraint = pIdxInfo->aConstraint;
  for(i=0; i<pIdxInfo->nConstraint; i++, pConstraint++){
    int iCol;    /* 0 for start, 1 for stop, 2 for step */
@@ -512,7 +658,50 @@ static int seriesBestIndex(
      }
      continue;
    }
-    if( pConstraint->iColumn<SERIES_COLUMN_START ) continue;
+    if( pConstraint->iColumn==SERIES_COLUMN_VALUE ){
+      switch( op ){
+        case SQLITE_INDEX_CONSTRAINT_EQ:
+        case SQLITE_INDEX_CONSTRAINT_IS: {
+          idxNum |=  0x0080;
+          idxNum &= ~0x3300;
+          aIdx[5] = i;
+          aIdx[6] = -1;
+          bStartSeen = 1;
+          break;
+        }
+        case SQLITE_INDEX_CONSTRAINT_GE: {
+          if( idxNum & 0x0080 ) break;
+          idxNum |=  0x0100;
+          idxNum &= ~0x0200;
+          aIdx[5] = i;
+          bStartSeen = 1;
+          break;
+        }
+        case SQLITE_INDEX_CONSTRAINT_GT: {
+          if( idxNum & 0x0080 ) break;
+          idxNum |=  0x0200;
+          idxNum &= ~0x0100;
+          aIdx[5] = i;
+          bStartSeen = 1;
+          break;
+        }
+        case SQLITE_INDEX_CONSTRAINT_LE: {
+          if( idxNum & 0x0080 ) break;
+          idxNum |=  0x1000;
+          idxNum &= ~0x2000;
+          aIdx[6] = i;
+          break;
+        }
+        case SQLITE_INDEX_CONSTRAINT_LT: {
+          if( idxNum & 0x0080 ) break;
+          idxNum |=  0x2000;
+          idxNum &= ~0x1000;
+          aIdx[6] = i;
+          break;
+        }
+      }
+      continue;
+    }
    iCol = pConstraint->iColumn - SERIES_COLUMN_START;
    assert( iCol>=0 && iCol<=2 );
    iMask = 1 << iCol;
@@ -534,7 +723,7 @@ static int seriesBestIndex(
    idxNum &= ~0x60;
    aIdx[4] = 0;
  }
-  for(i=0; i<5; i++){
+  for(i=0; i<7; i++){
    if( (j = aIdx[i])>=0 ){
      pIdxInfo->aConstraintUsage[j].argvIndex = ++nArg;
      pIdxInfo->aConstraintUsage[j].omit =
@@ -582,6 +771,9 @@ static int seriesBestIndex(
    pIdxInfo->estimatedRows = 2147483647;
  }
  pIdxInfo->idxNum = idxNum;
+#ifdef SQLITE_INDEX_SCAN_HEX
+  pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_HEX;
+#endif
  return SQLITE_OK;
 }

@@ -1,12 +1,12 @@
-C Fix\sa\sname\sresolution\sissue\swith\sCTEs.
-D 2024-08-20T22:44:40.653
+C Remove\sredundant\s#ifdef\saccidentally\sinserted\sinto\smkkeywordhash.c
+D 2024-08-26T14:20:20.876
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
-F Makefile.in 209e9c64edf499b0ea7a15448699101c01d15828af35a3439d32fc25fdf1b787
+F Makefile.in 101cb90f75c16ca3f8e7088364322441f9b01d6fdcef4a9c7f5fdcf78a3601a4
 F Makefile.linux-gcc f3842a0b1efbfbb74ac0ef60e56b301836d05b4d867d014f714fa750048f1ab6
 F Makefile.msc 6c3fe8b6ce60e73f34a148c957d78b4648745c8d30e792423aa1a8d8bf12d065
-F README.md 6358805260a03ebead84e168bbf3740ddf3f683b477e478567186aa7afb490d3
+F README.md 3d47ef5758e2de29d6f4e1aca714d51424baba86c8b561fef330f6ef2dc9282c
 F VERSION 0db40f92c04378404eb45bff93e9e42c148c7e54fd3da99469ed21e22411f5a6
 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50
 F art/icon-243x273.gif 9750b734f82fdb3dc43127753d5e6fbf3b62c9f4e136c2fbf573b2f57ea87af5
@@ -35,8 +35,8 @@ F autoconf/tea/win/nmakehlp.c b01f822eabbe1ed2b64e70882d97d48402b42d2689a1ea0034
 F autoconf/tea/win/rules.vc 7b3bb2ef32ade0f3f14d951231811678722725e3bca240dd9727ae0dfe10f6a5
 F config.guess 883205ddf25b46f10c181818bf42c09da9888884af96f79e1719264345053bd6
 F config.sub c2d0260f17f3e4bc0b6808fccf1b291cb5e9126c14fc5890efc77b9fd0175559
-F configure 956e132a3e2481e5016e9ecaa0a2bdd9df95c1d923fa4fb2aa2cf9f0b3a54ad9 x
-F configure.ac fd9ff064b142682964971fec3d867235e51e1f6de30a031e33f683b7699338ff
+F configure 49523f0a070b583cea040d26eff53a65fb0893eca4663b1343a4d5a9a964da53 x
+F configure.ac a100ebf7a07f5dedd319ef547dd467d1676ed059b85a7877aa9c44ac309f7000
 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad
 F doc/F2FS.txt c1d4a0ae9711cfe0e1d8b019d154f1c29e0d3abfe820787ba1e9ed7691160fcd
 F doc/compile-for-windows.md e8635eea9153dcd6a51fd2740666ebc4492b3813cb1ac31cd8e99150df91762d
@@ -89,29 +89,29 @@ F ext/fts3/tool/fts3cov.sh c331d006359456cf6f8f953e37f2b9c7d568f3863f00bb5f7eb87
 F ext/fts3/tool/fts3view.c 413c346399159df81f86c4928b7c4a455caab73bfbc8cd68f950f632e5751674
 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c
 F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7
-F ext/fts3/unicode/mkunicode.tcl d5aebf022fa4577ee8cdf27468f0d847879993959101f6dbd6348ef0cfc324a7
+F ext/fts3/unicode/mkunicode.tcl 63db9624ccf70d4887836c320eda93ab552f21008f3be7ede551eac3ead62baa
 F ext/fts3/unicode/parseunicode.tcl a981bd6466d12dd17967515801c3ff23f74a281be1a03cf1e6f52a6959fc77eb
-F ext/fts5/extract_api_docs.tcl bc3a0ca78be7d3df08e7602c00ca48021ebae40682d75eb001bfdf6e54ffb44e
-F ext/fts5/fts5.h 6b49ce6eb2e395e7fd84557b21d32f5de8041f2fada4c617e481e99427e24b6e
-F ext/fts5/fts5Int.h 41fb3a2dd40e818cc96c6f4176dbdf2aaa8f57043cfc9a8f2676e7e6a72ad764
-F ext/fts5/fts5_aux.c 4584e88878e54828bf7d4d0d83deedd232ec60628b7731be02bad6adb62304b1
+F ext/fts5/extract_api_docs.tcl 009cf59c77afa86d137b0cca3e3b1a5efbe2264faa2df233f9a7aa8563926d15
+F ext/fts5/fts5.h c72759fd15179934f10b71ae51ee999e8558f696335e4ec2d049243b764354ee
+F ext/fts5/fts5Int.h 26a71a09cefa4ef6b4516b204ed48da3e1380970a19b3482eea7c5d805655360
+F ext/fts5/fts5_aux.c 65a0468dd177d6093aa9ae1622e6d86b0136b8d267c62c0ad6493ad1e9a3d759
 F ext/fts5/fts5_buffer.c 0eec58bff585f1a44ea9147eae5da2447292080ea435957f7488c70673cb6f09
-F ext/fts5/fts5_config.c 68cb87a49215f8e7028000b681df4057c430a4a6afbd676463886da94c9e1c37
-F ext/fts5/fts5_expr.c 4b7734db98393d6f7fbc5c9c71ebcabe70110f7df08f6b136d096a1eaee0f56a
+F ext/fts5/fts5_config.c 353d2a0d12678cae6ab5b9ce54aed8dac0825667b69248b5a4ed81cbefc109ea
+F ext/fts5/fts5_expr.c 9a56f53700d1860f0ee2f373c2b9074eaf2a7aa0637d0e27a6476de26a3fee33
 F ext/fts5/fts5_hash.c adda4272be401566a6e0ba1acbe70ee5cb97fce944bc2e04dc707152a0ec91b1
 F ext/fts5/fts5_index.c eb9a0dda3bc6ef969a6be8d2746af56856e67251810ddba08622b45be8477abe
-F ext/fts5/fts5_main.c 6ec7a7d005c632d86e510ddfaca56b197a5b20b61848415764b91bd27d1e4f84
-F ext/fts5/fts5_storage.c 1d7e08d4331da2f3f7e78e70eef2ed6a013d91ba16175c651adbc5ad672235aa
-F ext/fts5/fts5_tcl.c 5ca3e3e35010d326f5b821a563e4fcde3913e052935f5c2c72c264122a26b48f
+F ext/fts5/fts5_main.c 3d8b778f65fe5be218f6a8e4019048f092c0c09621b035ce3e8804093036b578
+F ext/fts5/fts5_storage.c 9a9b880be12901f1962ae2a5a7e1b74348b3099a1e728764e419f75d98e3e612
+F ext/fts5/fts5_tcl.c 1dcf08028141c40a32634bdcf2d5601622ce4edc48f82ac4ce0cbe0a92a6961d
 F ext/fts5/fts5_test_mi.c 08c11ec968148d4cb4119d96d819f8c1f329812c568bac3684f5464be177d3ee
 F ext/fts5/fts5_test_tok.c 3cb0a9b508b30d17ef025ccddd26ae3dc8ddffbe76c057616e59a9aa85d36f3b
-F ext/fts5/fts5_tokenize.c 63ebe9057ed3f4dfc49944bc4aee3d3b745cc2faff73bc152ed3554ed3bf9cf4
-F ext/fts5/fts5_unicode2.c eca63dbc797f8ff0572e97caf4631389c0ab900d6364861b915bdd4735973f00
+F ext/fts5/fts5_tokenize.c ae9c4fa93174ef06ffc138bd4280a1c37f7e13624d3d2706aad4b80573f23c41
+F ext/fts5/fts5_unicode2.c 6f9b0fb79a8facaed76628ffd4eb9c16d7f2b84b52872784f617cf3422a9b043
 F ext/fts5/fts5_varint.c e64d2113f6e1bfee0032972cffc1207b77af63319746951bf1d09885d1dadf80
 F ext/fts5/fts5_vocab.c e4830b00809e5da53bc10f93adc59e321407b0f801c7f4167c0e47f5552267e0
 F ext/fts5/fts5parse.y eb526940f892ade5693f22ffd6c4f2702543a9059942772526eac1fde256bb05
 F ext/fts5/mkportersteps.tcl 5acf962d2e0074f701620bb5308155fa1e4a63ba
-F ext/fts5/test/fts5_common.tcl bc33c6cc65e5d390f28a68aeeb3a526dadd2c3a947d2466ee1986c1a4002df56
+F ext/fts5/test/fts5_common.tcl c5aa7cf7148b6dcffb5b61520ae18212baf169936af734ab265143f59db328fe
 F ext/fts5/test/fts5aa.test 015c81b84d53bfcedd77d624202c8b02e9f0cbbb4b51688e3a9c9f90bccbb4ac
 F ext/fts5/test/fts5ab.test 4bdb619fee409e11417e8827e320b857e42e926a01a0408fc9f143ec870a6ced
 F ext/fts5/test/fts5ac.test 4a73626de86f3d17c95738034880c4f0de8d54741fb943d819b528373657e59b
@@ -119,11 +119,11 @@ F ext/fts5/test/fts5ad.test 058e616612964e61d19f70295f0e6eaedceb4b29b1fbf4f85961
 F ext/fts5/test/fts5ae.test 3d49edbd50bb0684199a2e7568aeb30d1d29718f5c0f61751983740fa836d15f
 F ext/fts5/test/fts5af.test ae81f08b8da4c5f9b3ec1ef538a4ab6b7c278e92fa9058d6dc5d842c5d9771b9
 F ext/fts5/test/fts5ag.test 6667807b5d3fbf460892e756763fbe3d87a2fffe345a06514ba010ca6f6641f7
-F ext/fts5/test/fts5ah.test ac327281c8910cf9b85738a2655003fd0c6a8a76189ef34f3d086b8f9e54263b
+F ext/fts5/test/fts5ah.test e1f01314b35745a30e1b494b46045b82005d71cae74f1ebd9f1338566b77f9fc
 F ext/fts5/test/fts5ai.test cbe26d78030998f535bc103f37915350b137a822c71a9db439a077d7666a3539
 F ext/fts5/test/fts5aj.test 53c8508dab4acca3e691a4c51eca4b3b018319ab8635e540103d5bbdc91543c9
 F ext/fts5/test/fts5ak.test 25e2f8afdcff30d98ca9dee8c5cacca2f26db17501c9401f16d99ee036f70e8d
-F ext/fts5/test/fts5al.test 842c50fd9b287e3fa988dfcab436b27c432866f7406a56aaf3c67f316952cc02
+F ext/fts5/test/fts5al.test f0e655606771b2b5dbaf70e7f0044d560257cf3531d5eea40df58d0d7add8c39
 F ext/fts5/test/fts5alter.test ebbee06419c2d3cee5ef7ebb5ba6a9996f1aa374035361c0acd37368cc5f64f3
 F ext/fts5/test/fts5auto.test 2278de323172ced485d2844cb1357d00036ac1665f27e70fa1a48ce57bf31c2c
 F ext/fts5/test/fts5aux.test 27210687338133b1e9bc0dd669322fca59fd432439f40b126895e2d7c2f899d6
@@ -132,7 +132,8 @@ F ext/fts5/test/fts5auxdata.test 372549088ff792655f73e62b9dfaf4863ce74f5e604c06c
 F ext/fts5/test/fts5bigid.test 2860854c2561a57594192b00c33a29f91cb85e25f3d6c03b5c2b8f62708f39dd
 F ext/fts5/test/fts5bigpl.test 8f09858aab866c33593560e6480b2b6975ae7ff29ca32ad7b77e2da61402f8ef
 F ext/fts5/test/fts5bigtok.test 541119e616c637caea925a8c028c37c2c29e94383e00aa2f9198d530724b6e36
-F ext/fts5/test/fts5cat.test daba0b80659460b0cb60bd1f40b402478a761fe7ea414c3c94c2be25568cc33a
+F ext/fts5/test/fts5blob.test caa33369e93e99ff494cd1103506ae34c5afbc0bcc369ed5e58e135144e33689
+F ext/fts5/test/fts5cat.test bf67dd335f964482ee658287521b81e2b88697b45eb7f73933e15f198ed447cb
 F ext/fts5/test/fts5circref.test f880dfd0d99f6fb73b88ccacb0927d18e833672fd906cc47d6b4e529419eaa62
 F ext/fts5/test/fts5colset.test 544f4998cdbfe06a3123887fc0221612e8aa8192cdaff152872f1aadb10e6897
 F ext/fts5/test/fts5columnsize.test 0af91d63985afdf663455d4b572b935238380140d74079eac362760866d3297b
@@ -140,14 +141,14 @@ F ext/fts5/test/fts5config.test 017daf10d2642496e97402baa0134de8b5b46b9c37e53c22
 F ext/fts5/test/fts5conflict.test bf6030a77dbb1bedfcc42e589ed7980846c995765d77460551e448b56d741244
 F ext/fts5/test/fts5connect.test 08030168fc96fc278fa81f28654fb7e90566f33aff269c073e19b3ae9126b2f4
 F ext/fts5/test/fts5content.test d5c0c2142e64cb305f0968de70c01f8e59dbc3ecc56520c22e739e5dd99ea3bb
-F ext/fts5/test/fts5contentless.test b107465f8cd27dde6313b9c60b61d7158a7753b9c663c5c553695f826bb3c0a5
+F ext/fts5/test/fts5contentless.test 606f063b29ba0f46d4b79aa36cdd1ef4dab5de53eae8c881d731af75a4894aca
 F ext/fts5/test/fts5contentless2.test 70ffe6c611d8f278240da56734df8a77948f04e2739b358439e9bdcf56ced35f
 F ext/fts5/test/fts5contentless3.test 75eaae5ad6b284ee447788943974d323228f27cc35a1681da997135cff95bc6a
 F ext/fts5/test/fts5contentless4.test ec34dc69ef474ca9997dae6d91e072906e0e9a5a4b05ea89964c863833b6eff8
 F ext/fts5/test/fts5contentless5.test 40cdcb4fe751672450829c5a96bd32c25fc2f6076279dd2ce5c58ac9a390132a
-F ext/fts5/test/fts5corrupt.test a9bda1ded5112ebf1ee85c5381bd1fe8974952e2523cede4d5072804d2011503
+F ext/fts5/test/fts5corrupt.test 6485f721b88ba355ca5d701e7ee87a4efa3ea578d8e6adb26f51ef956c8328bd
 F ext/fts5/test/fts5corrupt2.test 335911e3f68b9625d850325f9e29a128db3f4276a8c9d4e32134580da8f924c4
-F ext/fts5/test/fts5corrupt3.test b5f35d72af85b1d5a092b3d5e437f7944d142dd0b0c87b928fd0436a0aec6987
+F ext/fts5/test/fts5corrupt3.test 621e9bca3e7299f487e1b29ff4179d9fc9560f5847dfc5b50a16010c9d2a0e5f
 F ext/fts5/test/fts5corrupt4.test dc08d19f5b8943e95a7778a7d8da592042504faf18dd93f68f7d7a0d7d7dd733
 F ext/fts5/test/fts5corrupt5.test 11b47126f5772cc37b67e3e8b2ed05895c4d07c05338bc07e4eea225bfe32c76
 F ext/fts5/test/fts5corrupt6.test 2d72db743db7b5d9c9a6d0cfef24d799ed1aa5e8192b66c40e871a37ed9eed06
@@ -159,7 +160,7 @@ F ext/fts5/test/fts5dlidx.test a7c42b0a74dc7c8aa1a46d586e0aadda4b6cc42c24450f8d3
 F ext/fts5/test/fts5doclist.test b7cb84758504519746957802db9cd31187bb4e0028b89d9087ba06e26cc4155f
 F ext/fts5/test/fts5ea.test cefdf66024550fa7920c03395c71ce5046235ed1a1a7a469d79b19e7aad5afb5
 F ext/fts5/test/fts5eb.test 401f756fdb77083aeba8b696c1e0ad4d834c39dbd6f17e492bb55a2ad64b4296
-F ext/fts5/test/fts5expr.test 7e1b2d075b63b727a624a378c2c09f94296a93dc4ae968aad67f8d9f3810c266
+F ext/fts5/test/fts5expr.test c7e208813df7a90badc856fde3796da79569b39382e0fdb43042127f3b8e06a7
 F ext/fts5/test/fts5fault1.test d28a65caee75db6897c3cf1358c5230d3bb2a3bf7fb31062c19c7e5382b3d2bd
 F ext/fts5/test/fts5fault2.test 69c8fdbef830cd0d450908d4504d5bb86609e255af99c421c20a0756251fe344
 F ext/fts5/test/fts5fault3.test da2f9e3e56ff5740d68ebdd6877c97089e7ed28ddff28a0da87a6afea27e5522
@@ -176,6 +177,7 @@ F ext/fts5/test/fts5faultE.test 844586ce71dab4be85bb86880e87b624d089f851654cd22e
 F ext/fts5/test/fts5faultF.test 4abef99f86e99d9f0c6460dd68c586a766b6b9f1f660ada55bf2e8266bd1bbc1
 F ext/fts5/test/fts5faultG.test 0544411ffcb3e19b42866f757a8a5e0fb8fef3a62c06f61d14deebc571bb7ea9
 F ext/fts5/test/fts5faultH.test 2b2b5b8cb1b3fd7679f488c06e22af44107fbc6137eaf45b3e771dc7b149312d
+F ext/fts5/test/fts5faultI.test e1e8a927122a8c3be8614a59717e838cad0505ba0a6f404983da8430e7aeb14d
 F ext/fts5/test/fts5first.test bfd685b96905bf541d99d8644e0a7219d1d833455a08ab64e344071a613b6ba9
 F ext/fts5/test/fts5full.test 97d263c1072f4a560929cca31e70f65d2ae232610e17e6affcf7e979df59547b
 F ext/fts5/test/fts5fuzz1.test 238d8c45f3b81342aa384de3e581ff2fa330bf922a7b69e484bbc06051a1080e
@@ -185,10 +187,11 @@ F ext/fts5/test/fts5interrupt.test 20d04204d3e341b104c0c24a41596b6393a3a81eba104
 F ext/fts5/test/fts5lastrowid.test f36298a1fb9f988bde060a274a7ce638faa9c38a31400f8d2d27ea9373e0c4a1
 F ext/fts5/test/fts5leftjoin.test c0b4cafb9661379e576dc4405c0891d8fcc2782680740513c4d1fc114b43d4ad
 F ext/fts5/test/fts5limits.test 8ab67cf5d311c124b6ceb0062d0297767176df4572d955fce79fa43004dff01c
+F ext/fts5/test/fts5locale.test 797cf6f5e017462ab11313ce884b9f1df8ff063811e74ef42190cd19ed6b600b
 F ext/fts5/test/fts5matchinfo.test 877520582feb86bbfd95ab780099bcba4526f18ac75ee34979144cf86ba3a5a3
 F ext/fts5/test/fts5merge.test 2654df0bcdb2d117c2d38b6aeb0168061be01c643f9e9194b36c43a2970e8082
 F ext/fts5/test/fts5merge2.test 3ebad1a59d6ad3fb66eff6523a09e95dc6367cbefb3cd73196801dea0425c8e2
-F ext/fts5/test/fts5misc.test b1682a40061bc58dcc62bbad48938fc5214d2ac6a868a8899c0c3d1930f1115d
+F ext/fts5/test/fts5misc.test 60bb2be4a2d83d7a45047c1812781e2e337a27efa539d86356ef7f4acaf08eab
 F ext/fts5/test/fts5multi.test a15bc91cdb717492e6e1b66fec1c356cb57386b980c7ba5af1915f97fe878581
 F ext/fts5/test/fts5multiclient.test 5ff811c028d6108045ffef737f1e9f05028af2458e456c0937c1d1b8dea56d45
 F ext/fts5/test/fts5near.test 33d60867581066e5db7016deb5d651628125d7ff4e0233a88175aa5b65874c74
@@ -196,7 +199,7 @@ F ext/fts5/test/fts5onepass.test f9b7d9b2c334900c6542a869760290e2ab5382af8fbd618
 F ext/fts5/test/fts5optimize.test 264b9101721c17d06d1d174feb743fda3ddc89fad41dee980fef821428258e47
 F ext/fts5/test/fts5optimize2.test 795d4ae5f66a7239cf8d5aef4c2ea96aeb8bcd907bd9be0cfe22064fc71a44ed
 F ext/fts5/test/fts5optimize3.test 1653029284e10e0715246819893ba30565c4ead0d0fc470adae92c353ea857d3
-F ext/fts5/test/fts5origintext.test 87c34c78f201b1e22ac93ac6bbe6196dde59f0a7266b1aeb938604a0eb9d5552
+F ext/fts5/test/fts5origintext.test 2015f69bc8abd111152a8e66211fd2d45026378001e07c054159aa4f84e6691d
 F ext/fts5/test/fts5origintext2.test f4505ff79bf7369f2b8b10b9cef7476049d844e20b37f29cad3a8b8d5ac6f9ba
 F ext/fts5/test/fts5origintext3.test 45c33cf0c91a9ca0e36d298462db3edc7c8fe45fd185649a9dbfd66bb670058b
 F ext/fts5/test/fts5origintext4.test 0d3ef0a8038f471dbc83001c34fe5f7ae39b571bfc209670771eb28bc0fc50e8
@@ -220,9 +223,9 @@ F ext/fts5/test/fts5secure4.test 0d10a80590c07891478700af7793b232962042677432b98
 F ext/fts5/test/fts5secure5.test c07a68ced5951567ac116c22f2d2aafae497e47fe9fcb6a335c22f9c7a4f2c3a
 F ext/fts5/test/fts5secure6.test 74bf04733cc523bccca519bb03d3b4e2ed6f6e3db7c59bf6be82c88a0ac857fd
 F ext/fts5/test/fts5secure7.test fd03d0868d64340a1db8615b02e5508fea409de13910114e4f19eaefc120777a
-F ext/fts5/test/fts5secure8.test e68c0ac4447f415ff3e4e82531e99548289286f9f3a29c8cd53036113fe28602
+F ext/fts5/test/fts5secure8.test 808ade9d172ed07b24b85c57dd53b6d2b1aba018b4e634d267ce572221de80e0
 F ext/fts5/test/fts5securefault.test c34a28c7cd2f31a8b8907563889e1329a97da975c08df2d951422bcef8e2ebc5
-F ext/fts5/test/fts5simple.test 847fb828262328744733847dc76d6b5d4a6bd4c5d9b282cb819f6504340e061a
+F ext/fts5/test/fts5simple.test ed7c3815c9fa1c16166258cb98edb2e014c63c7589958d76c5487df0df913d61
 F ext/fts5/test/fts5simple2.test d10d963a357b8ec77b99032e4c816459b4dbdb1f6eee25eada7ef3ed245cb2dc
 F ext/fts5/test/fts5simple3.test 146ec3dc8f5763d6212641c9f0a2f1cba41679353d2add7b963beceb115dc7f4
 F ext/fts5/test/fts5synonym.test becc8cea6cfc958a50b30c572c68cbfdf7455971d0fe988202ce67638d2c6cf6
@@ -231,12 +234,13 @@ F ext/fts5/test/fts5tok1.test 1f7817499f5971450d8c4a652114b3d833393c8134e32422d0
 F ext/fts5/test/fts5tok2.test dcacb32d4a2a3f0dd3215d4a3987f78ae4be21a2
 F ext/fts5/test/fts5tokenizer.test 7937cec672b148223fff8746d21d3e7ed0965fd7caf35ccdc888a005bb452f98
 F ext/fts5/test/fts5tokenizer2.test ddb8b10fbe4b84b2a75812671f127774c1d2e3e2bf82d2e0e4f0bb1cd8a2b2d6
-F ext/fts5/test/fts5trigram.test be914555deb8504dde682bd5aa343d00c4da37dfad20709a5bac30d5f97f2ef5
+F ext/fts5/test/fts5tokenizer3.test eea778f7bb7024c3e904e28915f9d53286141671b138722148be22a9c758bdc3
+F ext/fts5/test/fts5trigram.test fb9ee982edd76280ce979905a2251081cd04ae4c470248bd5d391b2d096430ab
 F ext/fts5/test/fts5trigram2.test 6fde9de7f63a6b4aa18dc731be56dbd6be4e755c9b13dcd55479e200d1df0e61
 F ext/fts5/test/fts5ubsan.test 9a2dcf399dc8d0e0de661f0d93884d1d27e5b7f0693cfceb97dd24d818df5dd2
 F ext/fts5/test/fts5umlaut.test a42fe2fe6387c40c49ab27ccbd070e1ae38e07f38d05926482cc0bccac9ad602
 F ext/fts5/test/fts5unicode.test 41898f7e476e6515cd4b737c02a442cda5a580a74509788aa9072a2074948e0e
-F ext/fts5/test/fts5unicode2.test a5c38179b311a188b24376772309389b073c996f52b79bb9ca760a19e62043ea
+F ext/fts5/test/fts5unicode2.test 3bbd30152f9f760bf13886e5b1e5ec23ff62f56758ddda5d9c775a6082fb4c7c
 F ext/fts5/test/fts5unicode3.test f4891a3dac3b49c3d7c0fdb29566e9eb0ecff35263370c89f9661b1952b20818
 F ext/fts5/test/fts5unicode4.test 728c8f0caafb05567f524ad313d9f8b780fa45987b8a8df04eff87923c74b4d0
 F ext/fts5/test/fts5unindexed.test 168838d2c385e131120bbf5b516d2432a5fabc4caa2259c932e1d49ae209a4ae
@@ -413,7 +417,7 @@ F ext/misc/regexp.c 4bdd0045912f81c84908bd535ec5ad3b1c8540b4287c70ab840709636240
 F ext/misc/remember.c add730f0f7e7436cd15ea3fd6a90fd83c3f706ab44169f7f048438b7d6baa69c
 F ext/misc/rot13.c 51ac5f51e9d5fd811db58a9c23c628ad5f333c173f1fc53c8491a3603d38556c
 F ext/misc/scrub.c 2a44b0d44c69584c0580ad2553f6290a307a49df4668941d2812135bfb96a946
-F ext/misc/series.c d96e5aac21658c6b5d54f918ac140460ec7197734c1a4fba806950831a7b1e7a
+F ext/misc/series.c a6089b5e8e3002bd1e5d9877cee6aead0b9a6426e406c09a399817db9e9ae823
 F ext/misc/sha1.c 4011aef176616872b2a0d5bccf0ecfb1f7ce3fe5c3d107f3a8e949d8e1e3f08d
 F ext/misc/shathree.c 1821d90a0040c9accdbe3e3527d378d30569475d758aa70f6848924c0b430e8c
 F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52
@@ -702,13 +706,13 @@ F src/btmutex.c 79a43670447eacc651519a429f6ece9fd638563cf95b469d6891185ddae2b522
 F src/btree.c 8b42fc7d9efdb2df05c30e8f91ff6cfbd979724ae24bf90269028468b7a13333
 F src/btree.h 55066f513eb095db935169dab1dc2f7c7a747ef223c533f5d4ad4dfed346cbd0
 F src/btreeInt.h 98aadb6dcb77b012cab2574d6a728fad56b337fc946839b9898c4b4c969e30b6
-F src/build.c c5522b5faf8128227678e194275cefaeb4d063f55dbe70bcff745f1b63a187cf
+F src/build.c 3a1840d9d171ce2d24f4c1f7acda7266ab796c664290c1acba65ff98ce2bd01e
 F src/callback.c db3a45e376deff6a16c0058163fe0ae2b73a2945f3f408ca32cf74960b28d490
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c 64e4b1227b4ed123146f0aa2989131d1fbd9b927b11e80c9d58c6a68f9cd5ce3
-F src/date.c 13dd752847afb32ed70510ad7345a5b9c841f51ad904dba5d010f1fa3a6a324e
+F src/date.c 89ce1ff20512a7fa5070ba6e7dd5c171148ca7d580955795bf97c79c2456144a
 F src/dbpage.c 80e46e1df623ec40486da7a5086cb723b0275a6e2a7b01d9f9b5da0f04ba2782
-F src/dbstat.c 3b677254d512fcafd4d0b341bf267b38b235ccfddbef24f9154e19360fa22e43
+F src/dbstat.c 73362c0df0f40ad5523a6f5501224959d0976757b511299bf892313e79d14f5c
 F src/delete.c 444c4d1eaac40103461e3b6f0881846dd3aafc1cec1dd169d3482fa331667da7
 F src/expr.c 6d5f2c38fe3ec06a7eac599dac822788b36064124e20112a844e9cd5156cb239
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
@@ -723,7 +727,7 @@ F src/insert.c f8d1a0f8ee258411009c6b7f2d93170e351bd19f5ad89d57e1180644297cbe70
 F src/json.c 5b6a1d6015997b9ee848a32948720bdb26a0ef2de5a2127ebf7355ce66dbdc0d
 F src/legacy.c d7874bc885906868cd51e6c2156698f2754f02d9eee1bae2d687323c3ca8e5aa
 F src/loadext.c 7432c944ff197046d67a1207790a1b13eec4548c85a9457eb0896bb3641dfb36
-F src/main.c 8a59d297ec77e6b78550433bfccb95a1b26f2fb69aaaf233206e21579a1cfcc1
+F src/main.c a520c325c7400b249242945f2c602acb19662c40f5db1c04d88664cfaa8b85ac
 F src/malloc.c 410e570b30c26cc36e3372577df50f7a96ee3eed5b2b161c6b6b48773c650c5e
 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
 F src/mem1.c 3bb59158c38e05f6270e761a9f435bf19827a264c13d1631c58b84bdc96d73b2
@@ -744,12 +748,12 @@ F src/os.h 1ff5ae51d339d0e30d8a9d814f4b8f8e448169304d83a7ed9db66a65732f3e63
 F src/os_common.h 6c0eb8dd40ef3e12fe585a13e709710267a258e2c8dd1c40b1948a1d14582e06
 F src/os_kv.c 4d39e1f1c180b11162c6dc4aa8ad34053873a639bac6baae23272fc03349986a
 F src/os_setup.h 6011ad7af5db4e05155f385eb3a9b4470688de6f65d6166b8956e58a3d872107
-F src/os_unix.c 2ea8d3ed496b8d1f9332a9505653424e5464fd797ea9d91f8e2e62f9dd0298d0
+F src/os_unix.c 6e3e4fc75904ff85184091dbab996e6e35c1799e771788961cc3b4fcbe8f852c
 F src/os_win.c 6ff43bac175bd9ed79e7c0f96840b139f2f51d01689a638fd05128becf94908a
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c b08600ebf0db90b6d1e9b8b6577c6fa3877cbe1a100bd0b2899e4c6e9adad4b3
 F src/pager.h 4b1140d691860de0be1347474c51fee07d5420bd7f802d38cbab8ea4ab9f538a
-F src/parse.y 5972b7d00af4c8d96fdad781af1ea1d5d51fc3b907ad61bda60e49503274e5ed
+F src/parse.y f4c8ded81b05059c284bc9512aaa07f6ecd29add585e35d53d72a67894b6e4d7
 F src/pcache.c 588cc3c5ccaaadde689ed35ce5c5c891a1f7b1f4d1f56f6cf0143b74d8ee6484
 F src/pcache.h 1497ce1b823cf00094bb0cf3bac37b345937e6f910890c626b16512316d3abf5
 F src/pcache1.c 49516ad7718a3626f28f710fa7448ef1fce3c07fd169acbb4817341950264319
@@ -758,11 +762,11 @@ F src/pragma.h e690a356c18e98414d2e870ea791c1be1545a714ba623719deb63f7f226d8bb7
 F src/prepare.c d99931f45416652895e502328ca49fe782cfc4e1ebdcda13b3736d991ebf42ce
 F src/printf.c 6a87534ebfb9e5346011191b1f3a7ebc457f5938c7e4feeea478ecf53f6a41b2
 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
-F src/resolve.c 9afed5fd7b9111633bdb74a73cdc47324e28e4dc6c27113e3e9aee38fb9422ab
+F src/resolve.c 2c127880c0634962837f16f2f48a295e514357af959330cc038de73015d5b5e8
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
-F src/select.c c1c28650d3ea5dc0670dd658600b963c29a5c31b685941a5df583b02631d04ff
-F src/shell.c.in 94571558b0fb28c37a5cf6dbd6ea27285341023a28a8cb5795cd2768fab67704
-F src/sqlite.h.in 1ad9110150773c38ebababbad11b5cb361bcd3997676dec1c91ac5e0416a7b86
+F src/select.c 4b14337a2742f0c0beeba490e9a05507e9b4b12184b9cd12773501d08d48e3fe
+F src/shell.c.in 5a05e1a1d187ab916afe9a628d7e9b20b5997273b595bd748db324814e0ccca9
+F src/sqlite.h.in f07bff4225a1244efd604a0ffef81ed69f29d3dbaed7e22f906f26229ba3ca9e
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 3f046c04ea3595d6bfda99b781926b17e672fd6d27da2ba6d8d8fc39981dcb54
 F src/sqliteInt.h 28c878bdf528879afefe1994ac007d094f8061f2fdacdc55d6055d7e9341151e
@@ -823,8 +827,8 @@ F src/test_windirent.h da2e5b73c32d09905fbdd00f27cd802212a32a58ead882736fe4f5eb7
 F src/test_window.c 6d80e11fba89a1796525e6f0048ff0c7789aa2c6b0b11c80827dc1437bd8ea72
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
-F src/tokenize.c 3f703cacdab728d7741e5a6ac242006d74fe1c2754d4f03ed889d7253259bd68
-F src/treeview.c e98dbc8068d27f3d0bd2a4cebfce6a7c21c776edc23d80e12c67a70cd2ba3fe5
+F src/tokenize.c 6fac7f14dd021fecfdd8be93c27937804c4b665d2da9e667a10e4b5f04c6e085
+F src/treeview.c 88aa39b754f5ef7214385c1bbbdd2f3dc20efafeed0cf590e8d1199b9c6e44aa
 F src/trigger.c 0bb986a5b96047fd597c6aac28588853df56064e576e6b81ba777ef2ccaac461
 F src/update.c 0e01aa6a3edf9ec112b33eb714b9016a81241497b1fb7c3e74332f4f71756508
 F src/upsert.c 215328c3f91623c520ec8672c44323553f12caeb4f01b1090ebdca99fdf7b4f1
@@ -846,10 +850,10 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 887fc4ca3f020ebb2e376f222069570834ac63bf50111ef0cbf3ae417048ed89
 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452
 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014
-F src/where.c b6f79b189fcba459bb80420d3b4102f42a399be36ff29a3deff4ae4888fee46d
-F src/whereInt.h 6444b888ce395cb80511284b8a73b63472d34247fcb1b125ee06a54fa6ae878e
-F src/wherecode.c 137797b0de9ddf1ff43e5b0edffcc76fb05184ed651fc4f5a0a01a45c0b89d04
-F src/whereexpr.c 6a72cf607548765a262f216e87373fd675a4646f9cc4278fc519b66cf03dbc13
+F src/where.c c046dd58c3410f7b7528e1e6317cb876398557bad346d568ed8562321a7d002d
+F src/whereInt.h a5d079c346a658b7a6e9e47bb943d021e02fa1e6aed3b964ca112112a4892192
+F src/wherecode.c 5172d647798134e7c92536ddffe7e530c393d79b5dedd648b88faf2646c65baf
+F src/whereexpr.c 44f41ae554c7572e1de1485b3169b233ee04d464b2ee5881687ede3bf07cacfa
 F src/window.c 499d48f315a09242dc68f2fac635ed27dcf6bbb0d9ab9084857898c64489e975
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test ce1aafc86e110685b324e9a763eab4f2a73f737842ec3b687bd965867de90627
@@ -1060,6 +1064,7 @@ F test/date.test c8ff835023f2107b57ce7a45c92265d51c98a23fc93231e998f12d850831aad
 F test/date2.test 7e12ec14aaf4d5e6294b4ba140445b0eca06ea50062a9c3a69c4ee13d0b6f8b1
 F test/date3.test a1b77abf05c6772fe5ca2337cac1398892f2a41e62bce7e6be0f4a08a0e64ae5
 F test/date4.test 75dc8401e8c0639a228cd26a6eaa4ff5ea8ccda912b9853d1c9462c476670e17
+F test/date5.test 14ba189bc4d03efc371dd5302e035764f6633355a3e13acb4a45e7b33530231e
 F test/dbdata.test 042f49acff3438f940eeba5868d3af080ae64ddf26ae78f80c92bec3ca7d8603
 F test/dbfuzz.c 73047c920d6210e5912c87cdffd9a1c281d4252e
 F test/dbfuzz001.test 6c9a4622029d69dc38926f115864b055cb2f39badd25ec22cbfb130c8ba8e9c3
@@ -1436,7 +1441,7 @@ F test/memjournal2.test dbc2c5cb5f7b38950f4f6dc3e73fcecf0fcbed3fc32c7ce913bba164
 F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2
 F test/memsubsys1.test 86b8158752af9188ed5b32a30674a1ef71183e6bc4e6808e815cd658ca9058a6
 F test/memsubsys2.test 774b93cb09ca50d1b759bb7c645baa2a9ce172edc3a3da67d5150a26a9fc2a08
-F test/merge1.test 2de6d6ef8d25402764b1aab49d8f9d7f89208c89a6674e437f76de4c812157b8
+F test/merge1.test 7dd9dc6838bcd0623a069485fe3a8dd498a051c16e1877cf84f506c0d6a29b43
 F test/minmax.test fe638b55d77d2375531a8f549b338eafcd9adfbd2f72df37ed77d9b26ca0a71a
 F test/minmax2.test cf9311babb6f0518d04e42fd6a42c619531c4309a9dd790a2c4e9b3bc595e0de
 F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354
@@ -1695,7 +1700,7 @@ F test/sync.test 89539f4973c010eda5638407e71ca7fddbcd8e0594f4c9980229f804d433309
 F test/sync2.test 8f9f7d4f6d5be8ca8941a8dadcc4299e558cb6a1ff653a9469146c7a76ef2039
 F test/syscall.test a067468b43b8cb2305e9f9fe414e5f40c875bb5d2cba5f00b8154396e95fcf37
 F test/sysfault.test c9f2b0d8d677558f74de750c75e12a5454719d04
-F test/tabfunc01.test f150d206294471d20f50029e6b46b76b87a7a010b16dc57eb44245c76dd02802
+F test/tabfunc01.test 6002a5f37b76355f173c75c2b3b03173b19d6a8b078c5baaa4c78bbcd0fa6323
 F test/table.test 7862a00b58b5541511a26757ea9c5c7c3f8298766e98aa099deec703d9c0a8e0
 F test/tableapi.test ecbcc29c4ab62c1912c3717c48ea5c5e59f7d64e4a91034e6148bd2b82f177f4
 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930
@@ -1707,7 +1712,7 @@ F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30
 F test/temptable2.test 76821347810ecc88203e6ef0dd6897b6036ac788e9dd3e6b04fd4d1631311a16
 F test/temptable3.test d11a0974e52b347e45ee54ef1923c91ed91e4637
 F test/temptrigger.test 38f0ca479b1822d3117069e014daabcaacefffcc
-F test/tester.tcl e88c498c369cff6bf0898db6d04088685066730be51821ef775ef13fd2b1d077
+F test/tester.tcl 2c203a2dd664298f239f0ec3ce22fbc65b5f021c1e09edbae8452af8a694e052
 F test/testrunner.tcl 5d02deeba7a53baeadae6aa7641d90aac58fdfa3a7bcac85cfcfd752b1aab87c
 F test/testrunner_data.tcl c5ae2b1f9a99210b0600d002fb3af1fee350997cee9416551e83b93501360ebf
 F test/thread001.test a0985c117eab62c0c65526e9fa5d1360dd1cac5b03bde223902763274ce21899
@@ -2064,7 +2069,7 @@ F test/windowA.test 6d63dc1260daa17141a55007600581778523a8b420629f1282d2acfc36af
 F test/windowB.test aad7c31739999f68a98a813cfd78390918fc70f56d2d925317a1523cab548ecf
 F test/windowC.test 6fd75f5bb2f1343d34e470e36e68f0ff638d8a42f6aa7d99471261b31a0d42f2
 F test/windowD.test 65cf5a765fb8072450e8a0de2979ce7f09a38d87724fe1280c6444073e3da49b
-F test/windowE.test 6ba0c8048e4cc02b942e56640f8fcd50fd7ca72c876656c40f6baf42e316684c
+F test/windowE.test c98507e0b0d95980ad25845db758557757be2d7054198ccf522c7d277057a3df
 F test/windowerr.tcl f5acd6fbc210d7b5546c0e879d157888455cd4a17a1d3f28f07c1c8a387019e0
 F test/windowerr.test a8b752402109c15aa1c5efe1b93ccb0ce1ef84fa964ae1cd6684dd0b3cc1819b
 F test/windowfault.test 15094c1529424e62f798bc679e3fe9dfab6e8ba2f7dfe8c923b6248c31660a7c
@@ -2126,7 +2131,7 @@ F tool/merge-test.tcl de76b62f2de2a92d4c1ca4f976bce0aea6899e0229e250479b229b2a19
 F tool/mkautoconfamal.sh cbdcf993fa83dccbef7fb77b39cdeb31ef9f77d9d88c9e343b58d35ca3898a6a
 F tool/mkccode.tcl 86463e68ce9c15d3041610fedd285ce32a5cf7a58fc88b3202b8b76837650dbe x
 F tool/mkctimec.tcl 060e9785e9503bf51f8b1b11b542bdeef90fd0ceb0738154f6762acec0c61e5f x
-F tool/mkkeywordhash.c b9faa0ae7e14e4dbbcd951cddd786bf46b8a65bb07b129ba8c0cfade723aaffd
+F tool/mkkeywordhash.c 8a0a0401a6e0f01ca6e02f58b294a813d067554474875f4fe0222c2b8b0d6ed3
 F tool/mkmsvcmin.tcl d76c45efda1cce2d4005bcea7b8a22bb752e3256009f331120fb4fecb14ebb7a
 F tool/mkopcodec.tcl 33d20791e191df43209b77d37f0ff0904620b28465cca6990cf8d60da61a07ef
 F tool/mkopcodeh.tcl 2b4e6967a670ef21bf53a164964c35c6163277d002a4c6f56fa231d68c88d023
@@ -2205,8 +2210,8 @@ F vsixtest/vsixtest.tcl 6195aba1f12a5e10efc2b8c0009532167be5e301abe5b31385638080
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c1bbed19f4348ec31c3cb4723eaa8f1554bb58ef4de7feb3a0f39612785f13d5
-R 1c80ace1a0877505f5ca5ebb5b98295b
+P 16d32676e2e98a7710787de16850c328e7519e893f81ed56f6155a8fa9cbc3a4
+R a100ff1c9bda3fc624920cd66e34ae7c
 U drh
-Z ef82496d97c0311a270f63ff1146b092
+Z 58c98d8befdb125e16f0435abf5b1a98
 # Remove this line to create a well-formed Fossil manifest.
@@ -1 +1 @@
-4fa8235dd59cd683d6c6c97bfe181a9637be7c054d435323c903b9dbd74aff02
+b08f3b74495e125f92c8968472fc8028c73d8df5400195df3081e3d8df8bd985
@@ -4965,8 +4965,6 @@ void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){
    /* Check invariants on SrcItem */
    assert( !pItem->fg.isIndexedBy || !pItem->fg.isTabFunc );
    assert( !pItem->fg.isCte || !pItem->fg.isIndexedBy );
-    assert( !pItem->fg.hadSchema || !pItem->fg.isSubquery );
-    assert( !pItem->fg.hadSchema || pItem->fg.fixedSchema );
    assert( !pItem->fg.fixedSchema || !pItem->fg.isSubquery );
    assert( !pItem->fg.isSubquery || (pItem->u4.pSubq!=0 && 
                                      pItem->u4.pSubq->pSelect!=0) );
@@ -271,8 +271,8 @@ static void computeJD(DateTime *p){
    Y--;
    M += 12;
  }
-  A = Y/100;
-  B = 2 - A + (A/4);
+  A = (Y+4800)/100;
+  B = 38 - A + (A/4);
  X1 = 36525*(Y+4716)/100;
  X2 = 306001*(M+1)/10000;
  p->iJD = (sqlite3_int64)((X1 + X2 + D + B - 1524.5 ) * 86400000);
@@ -456,7 +456,7 @@ static int validJulianDay(sqlite3_int64 iJD){
 ** Compute the Year, Month, and Day from the julian day number.
 */
 static void computeYMD(DateTime *p){
-  int Z, A, B, C, D, E, X1;
+  int Z, alpha, A, B, C, D, E, X1;
  if( p->validYMD ) return;
  if( !p->validJD ){
    p->Y = 2000;
@@ -467,8 +467,8 @@ static void computeYMD(DateTime *p){
    return;
  }else{
    Z = (int)((p->iJD + 43200000)/86400000);
-    A = (int)((Z - 1867216.25)/36524.25);
-    A = Z + 1 + A - (A/4);
+    alpha = (int)((Z + 32044.75)/36524.25) - 52;
+    A = Z + 1 + alpha - ((alpha+100)/4) + 25;
    B = A + 1524;
    C = (int)((B - 122.1)/365.25);
    D = (36525*(C&32767))/100;
@@ -279,6 +279,7 @@ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
    pIdxInfo->orderByConsumed = 1;
    pIdxInfo->idxNum |= 0x08;
  }
+  pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_HEX;

  return SQLITE_OK;
 }
@@ -4393,6 +4393,18 @@ int sqlite3_test_control(int op, ...){
      break;
    }

+    /*  sqlite3_test_control(SQLITE_TESTCTRL_GETOPT, sqlite3 *db, int *N)
+    **
+    ** Write the current optimization settings into *N.  A zero bit means that
+    ** the optimization is on, and a 1 bit means that the optimization is off.
+    */
+    case SQLITE_TESTCTRL_GETOPT: {
+      sqlite3 *db = va_arg(ap, sqlite3*);
+      int *pN = va_arg(ap, int*);
+      *pN = db->dbOptFlags;
+      break;
+    }
+
    /*   sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, onoff, xAlt);
    **
    ** If parameter onoff is 1, subsequent calls to localtime() fail.
@@ -322,7 +322,7 @@ static pid_t randomnessPid = 0;
 #define UNIXFILE_EXCL        0x01     /* Connections from one process only */
 #define UNIXFILE_RDONLY      0x02     /* Connection is read only */
 #define UNIXFILE_PERSIST_WAL 0x04     /* Persistent WAL mode */
-#ifndef SQLITE_DISABLE_DIRSYNC
+#if !defined(SQLITE_DISABLE_DIRSYNC) && !defined(_AIX)
 # define UNIXFILE_DIRSYNC    0x08     /* Directory sync needed */
 #else
 # define UNIXFILE_DIRSYNC    0x00
@@ -268,7 +268,7 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,A,Y);}
  GENERATED ALWAYS
 %endif
  MATERIALIZED
-  REINDEX RENAME CTIME_KW IF
+  REINDEX RENAME CTIME_KW IF AGGREGATE
  .
 %wildcard ANY.

@@ -295,7 +295,8 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,A,Y);}
 %left CONCAT PTR.
 %left COLLATE.
 %right BITNOT.
-%nonassoc ON.
+%nonassoc ON FROM.
+%nonassoc JOIN_KW JOIN.

 // An IDENTIFIER can be a generic identifier, or one of several
 // keywords.  Any non-standard keyword can also be an identifier.
@@ -624,6 +625,233 @@ oneselect(A) ::= SELECT distinct(D) selcollist(W) from(X) where_opt(Y)
 }
 %endif

+%ifndef SQLITE_OMIT_SUBQUERY
+%include {
+  SQLITE_NOINLINE SrcList *sqlite3SrcListAppendSubFrom(
+    Parse *pParse,           /* Parsing context */
+    SrcList *pFrom,          /* LHS of the FROM clause */
+    SrcList *pSubFrom,       /* New parenthesized sub-FROM to append */
+    Token *pAs,              /* AS-Name of the sub-FROM */
+    OnOrUsing *pOnUsing      /* ON or USING clause associated with the join */
+  ){
+    if( pFrom==0 && pAs->n==0 && pOnUsing->pOn==0 && pOnUsing->pUsing==0 ){
+      pFrom = pSubFrom;
+    }else if( ALWAYS(pSubFrom!=0) && pSubFrom->nSrc==1 ){
+      pFrom = sqlite3SrcListAppendFromTerm(pParse,pFrom,0,0,pAs,0,pOnUsing);
+      if( pFrom ){
+        SrcItem *pNew = &pFrom->a[pFrom->nSrc-1];
+        SrcItem *pOld = pSubFrom->a;
+        assert( pOld->fg.fixedSchema==0 );
+        pNew->zName = pOld->zName;
+        assert( pOld->fg.fixedSchema==0 );
+        if( pOld->fg.isSubquery ){
+          pNew->fg.isSubquery = 1;
+          pNew->u4.pSubq = pOld->u4.pSubq;
+          pOld->u4.pSubq = 0;
+          pOld->fg.isSubquery = 0;
+          assert( pNew->u4.pSubq!=0 && pNew->u4.pSubq->pSelect!=0 );
+          if( (pNew->u4.pSubq->pSelect->selFlags & SF_NestedFrom)!=0 ){
+            pNew->fg.isNestedFrom = 1;
+          }
+        }else{
+          pNew->u4.zDatabase = pOld->u4.zDatabase;
+          pOld->u4.zDatabase = 0;
+        }
+        if( pOld->fg.isTabFunc ){
+          pNew->u1.pFuncArg = pOld->u1.pFuncArg;
+          pOld->u1.pFuncArg = 0;
+          pOld->fg.isTabFunc = 0;
+          pNew->fg.isTabFunc = 1;
+        }
+        pOld->zName = 0;
+      }
+      sqlite3SrcListDelete(pParse->db, pSubFrom);
+    }else{
+      Select *pSubquery;
+      sqlite3SrcListShiftJoinType(pParse,pSubFrom);
+      pSubquery = sqlite3SelectNew(pParse,0,pSubFrom,0,0,0,0,SF_NestedFrom,0);
+      pFrom = sqlite3SrcListAppendFromTerm(pParse,pFrom,0,0,pAs,pSubquery,pOnUsing);
+    }
+    return pFrom;
+  }
+}
+%endif SQLITE_OMIT_SUBQUERY
+
+%ifndef SQLITE_OMIT_FROM_FIRST
+%include {
+  /* The Pipe object represents a pipe under construction.  It is a transient
+  ** object found only on the LALR(1) parser stack. */
+  typedef struct Pipe Pipe;
+  struct Pipe {
+    SrcList *pFrom;          /* The FROM clause */
+    ExprList *pProj;         /* The projection - list of columns to return */
+    ExprList *pOrderBy;      /* The ORDER BY clause */
+    ExprList *pGroupBy;      /* The GROUP BY clause */
+    Expr *pLimit;            /* The LIMIT or LIMIT/OFFSET clause */
+    Expr *pWhere;            /* The WHERE clause */
+    Expr *pHaving;           /* The HAVING clause */
+    int selFlags;            /* SF_DISTINCT or SF_ALL or 0 */
+    u8 bSeenAgg;             /* True if AGGREGATE has been seen */
+    u8 bNeedPush;            /* PipePush needed before adding terms to pProj */
+    Token sAs;               /* Value of a pending AS clause */
+  };
+
+  /* Delete a pipe object */
+  void sqlite3PipeDelete(sqlite3 *db, Pipe *p){
+    sqlite3SrcListDelete(db, p->pFrom);
+    sqlite3ExprListDelete(db, p->pProj);
+    sqlite3ExprListDelete(db, p->pOrderBy);
+    sqlite3ExprListDelete(db, p->pGroupBy);
+    sqlite3ExprDelete(db, p->pLimit);
+    sqlite3ExprDelete(db, p->pWhere);
+    sqlite3ExprDelete(db, p->pHaving);
+    sqlite3DbFree(db, p);
+  }
+
+  /* Generate a Select from the current content of a Pipe.  Reset the Pipe
+  ** to be empty, except do not remove the AS token if there is one and do
+  ** not delete the Pipe.
+  */
+  Select *sqlite3SelectFromPipe(Parse *pParse, Pipe *p){
+    Token sAs = p->sAs;
+    Select *pSel = sqlite3SelectNew(pParse, p->pProj, p->pFrom, p->pWhere,
+                                    p->pGroupBy, p->pHaving, p->pOrderBy,
+                                    p->selFlags, p->pLimit);
+    memset(p, 0, sizeof(*p));
+    p->sAs = sAs;
+    return pSel;
+  }
+
+  /* Take the current content of the pipe and make it "FROM <subquery>" pipe.
+  */
+  void sqlite3PipePush(Parse *pParse, Pipe *pPipe, int jointype){
+    Select *pSel = sqlite3SelectFromPipe(pParse, pPipe);
+    SrcList *pFrom = pPipe->pFrom;
+    if( pFrom ) pFrom->a[pFrom->nSrc-1].fg.jointype = jointype;
+    pPipe->pFrom = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&pPipe->sAs,pSel,0);
+    pPipe->sAs.z = 0;
+  }
+  
+}
+%type pipeline {Pipe*}
+%destructor pipeline {sqlite3PipeDelete(pParse->db,$$);}
+
+oneselect(A) ::= pipeline(X).  {
+  Pipe *pPipe = X;
+  Select *pSel = sqlite3SelectFromPipe(pParse,pPipe);
+  sqlite3DbFree(pParse->db, pPipe);
+  A = pSel; /*A-overwrites-X */
+}
+pipeline(A) ::= FROM seltablist(X). {
+  Pipe *pPipe = sqlite3DbMallocZero(pParse->db, sizeof(*pPipe));
+  A = pPipe;
+  if( pPipe ){
+    pPipe->pFrom = X;
+  }else{
+    sqlite3SrcListDelete(pParse->db, X);
+  }
+}
+pipeline(A) ::= pipeline(A) pipe pipejoinop(J) nm(Y) dbnm(D) as(Z) on_using(N). {
+  Pipe *p = A;
+  if( p->bNeedPush ) sqlite3PipePush(pParse, p, J);
+  p->pFrom = sqlite3SrcListAppendFromTerm(pParse, p->pFrom, &Y, &D, &Z, 0, &N);
+}
+pipeline(A) ::= pipeline(A) pipe 
+                pipejoinop(J) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) on_using(N). {
+  Pipe *p = A;
+  if( p->bNeedPush ) sqlite3PipePush(pParse, p, J);
+  p->pFrom = sqlite3SrcListAppendFromTerm(pParse, p->pFrom, &Y, &D, &Z, 0, &N);
+  sqlite3SrcListFuncArgs(pParse, p->pFrom, E);
+}
+pipeline(A) ::= pipeline(A) pipe pipejoinop(J) LP select(S) RP as(Z) on_using(N). {
+  Pipe *p = A;
+  if( p->bNeedPush ) sqlite3PipePush(pParse, p, J);
+  p->pFrom = sqlite3SrcListAppendFromTerm(pParse,p->pFrom,0,0,&Z,S,&N);
+}  
+pipeline(A) ::= pipeline(A) pipe pipejoinop(J) LP seltablist(F) RP as(Z) on_using(N).{
+  Pipe *p = A;
+  if( p->bNeedPush ) sqlite3PipePush(pParse, p, J);
+  p->pFrom = sqlite3SrcListAppendSubFrom(pParse,p->pFrom,F,&Z,&N);
+}
+pipeline(A) ::= pipeline(A) pipe WHERE expr(W). {
+  Pipe *p = A;
+  if( p->bSeenAgg ){
+    p->pHaving = sqlite3ExprAnd(pParse, A->pHaving, W);
+  }else{
+    p->pWhere = sqlite3ExprAnd(pParse, A->pWhere, W);
+  }
+  p->bNeedPush = 1;
+}
+pipeline(A) ::= pipeline(A) pipe HAVING expr(W). {
+  Pipe *p = A;
+  p->pHaving = sqlite3ExprAnd(pParse, A->pHaving, W);
+  p->bNeedPush = 1;
+}
+pipeline(A) ::= pipeline(A) pipe AGGREGATE selcollist(W) groupby_opt(G). {
+  Pipe *p = A;
+  ExprList *pProj = W;
+  ExprList *pGroupby = G;
+  if( p->pProj ) sqlite3PipePush(pParse, p, 0);
+  if( pGroupby ){
+    int i;
+    for(i=0; i<pGroupby->nExpr; i++){
+      pProj = sqlite3ExprListAppend(pParse, pProj,
+                            sqlite3ExprDup(pParse->db, pGroupby->a[i].pExpr, 0));
+    }
+    p->pGroupBy = pGroupby;
+  }
+  p->pProj = pProj;
+  p->bSeenAgg = 1;
+  p->bNeedPush = 1;
+}
+pipeline(A) ::= pipeline(A) pipe SELECT distinct(D) selcollist(W). {
+  Pipe *p = A;
+  if( p->pProj ) sqlite3PipePush(pParse, p, 0);
+  p->pProj = W;
+  p->selFlags = D;
+  p->bNeedPush = 1;
+}
+pipeline(A) ::= pipeline(A) pipe ORDER BY sortlist(X). {
+  Pipe *p = A;
+  if( p->pOrderBy || p->pLimit ) sqlite3PipePush(pParse, p, 0);
+  p->pOrderBy = X;
+  p->bNeedPush = 1;
+}
+pipeline(A) ::= pipeline(A) pipe LIMIT expr(X). {
+  Pipe *p = A;
+  Expr *pLimit = sqlite3PExpr(pParse,TK_LIMIT,X,0);
+  if( pLimit ){
+    sqlite3ExprDelete(pParse->db, p->pLimit);
+    p->pLimit = pLimit;
+  }
+  p->bNeedPush = 1;
+}
+pipeline(A) ::= pipeline(A) pipe LIMIT expr(X) OFFSET expr(Y). {
+  Pipe *p = A;
+  Expr *pLimit = sqlite3PExpr(pParse,TK_LIMIT,X,Y);
+  if( pLimit ){
+    sqlite3ExprDelete(pParse->db, p->pLimit);
+    p->pLimit = pLimit;
+  }
+  p->bNeedPush = 1;
+}
+pipeline(A) ::= pipeline(A) PIPE AS nm(X).     {A->sAs = X; A->bNeedPush = 1;}
+pipeline(A) ::= pipeline(A) pipe VIEW nm(X).   {A->sAs = X; A->bNeedPush = 1;}
+//pipeline ::= pipeline pipe DISTINCT ON nexprlist.
+
+pipe ::= .
+pipe ::= PIPE.
+
+%type pipejoinop {int}
+pipejoinop(X) ::= JOIN.              { X = JT_INNER; }
+pipejoinop(X) ::= JOIN_KW(A) JOIN.
+                  {X = sqlite3JoinType(pParse,&A,0,0);  /*X-overwrites-A*/}
+pipejoinop(X) ::= JOIN_KW(A) nm(B) JOIN.
+                  {X = sqlite3JoinType(pParse,&A,&B,0); /*X-overwrites-A*/}
+pipejoinop(X) ::= JOIN_KW(A) nm(B) nm(C) JOIN.
+                  {X = sqlite3JoinType(pParse,&A,&B,&C);/*X-overwrites-A*/}
+
+%endif FROM_first

 // Single row VALUES clause.
 //
@@ -733,44 +961,7 @@ seltablist(A) ::= stl_prefix(A) nm(Y) dbnm(D) LP exprlist(E) RP as(Z) on_using(N
    A = sqlite3SrcListAppendFromTerm(pParse,A,0,0,&Z,S,&N);
  }
  seltablist(A) ::= stl_prefix(A) LP seltablist(F) RP as(Z) on_using(N). {
-    if( A==0 && Z.n==0 && N.pOn==0 && N.pUsing==0 ){
-      A = F;
-    }else if( ALWAYS(F!=0) && F->nSrc==1 ){
-      A = sqlite3SrcListAppendFromTerm(pParse,A,0,0,&Z,0,&N);
-      if( A ){
-        SrcItem *pNew = &A->a[A->nSrc-1];
-        SrcItem *pOld = F->a;
-        assert( pOld->fg.fixedSchema==0 );
-        pNew->zName = pOld->zName;
-        assert( pOld->fg.fixedSchema==0 );
-        if( pOld->fg.isSubquery ){
-          pNew->fg.isSubquery = 1;
-          pNew->u4.pSubq = pOld->u4.pSubq;
-          pOld->u4.pSubq = 0;
-          pOld->fg.isSubquery = 0;
-          assert( pNew->u4.pSubq!=0 && pNew->u4.pSubq->pSelect!=0 );
-          if( (pNew->u4.pSubq->pSelect->selFlags & SF_NestedFrom)!=0 ){
-            pNew->fg.isNestedFrom = 1;
-          }
-        }else{
-          pNew->u4.zDatabase = pOld->u4.zDatabase;
-          pOld->u4.zDatabase = 0;
-        }
-        if( pOld->fg.isTabFunc ){
-          pNew->u1.pFuncArg = pOld->u1.pFuncArg;
-          pOld->u1.pFuncArg = 0;
-          pOld->fg.isTabFunc = 0;
-          pNew->fg.isTabFunc = 1;
-        }
-        pOld->zName = 0;
-      }
-      sqlite3SrcListDelete(pParse->db, F);
-    }else{
-      Select *pSubquery;
-      sqlite3SrcListShiftJoinType(pParse,F);
-      pSubquery = sqlite3SelectNew(pParse,0,F,0,0,0,0,SF_NestedFrom,0);
-      A = sqlite3SrcListAppendFromTerm(pParse,A,0,0,&Z,pSubquery,&N);
-    }
+    A = sqlite3SrcListAppendSubFrom(pParse,A,F,&Z,&N);
  }
 %endif  SQLITE_OMIT_SUBQUERY

@@ -1300,9 +1300,9 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){
          sqlite3WalkExprList(pWalker, pExpr->pLeft->x.pList);
        }
 #ifndef SQLITE_OMIT_WINDOWFUNC
-        if( pWin ){
+        if( pWin && pParse->nErr==0 ){
          Select *pSel = pNC->pWinSelect;
-          assert( pWin==0 || (ExprUseYWin(pExpr) && pWin==pExpr->y.pWin) );
+          assert( ExprUseYWin(pExpr) && pWin==pExpr->y.pWin );
          if( IN_RENAME_OBJECT==0 ){
            sqlite3WindowUpdate(pParse, pSel ? pSel->pWinDefn : 0, pWin, pDef);
            if( pParse->db->mallocFailed ) break;
@@ -6784,6 +6784,7 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){
  for(i=0, pF=pAggInfo->aFunc; i<pAggInfo->nFunc; i++, pF++){
    ExprList *pList;
    assert( ExprUseXList(pF->pFExpr) );
+    if( pParse->nErr ) return;
    pList = pF->pFExpr->x.pList;
    if( pF->iOBTab>=0 ){
      /* For an ORDER BY aggregate, calls to OP_AggStep were deferred.  Inputs
@@ -6993,6 +6994,7 @@ static void updateAccumulator(
    if( addrNext ){
      sqlite3VdbeResolveLabel(v, addrNext);
    }
+    if( pParse->nErr ) return;
  }
  if( regHit==0 && pAggInfo->nAccumulator ){
    regHit = regAcc;
@@ -7002,6 +7004,7 @@ static void updateAccumulator(
  }
  for(i=0, pC=pAggInfo->aCol; i<pAggInfo->nAccumulator; i++, pC++){
    sqlite3ExprCode(pParse, pC->pCExpr, AggInfoColumnReg(pAggInfo,i));
+    if( pParse->nErr ) return;
  }

  pAggInfo->directMode = 0;
@@ -7337,7 +7340,7 @@ static int fromClauseTermCanBeCoroutine(
 }

 /*
-** Generate code for the SELECT statement given in the p argument. 
+** Generate byte-code for the SELECT statement given in the p argument. 
 **
 ** The results are returned according to the SelectDest structure.
 ** See comments in sqliteInt.h for further information.
@@ -7348,6 +7351,40 @@ static int fromClauseTermCanBeCoroutine(
 **
 ** This routine does NOT free the Select structure passed in.  The
 ** calling function needs to do that.
+**
+** This is a long function.  The following is an outline of the processing
+** steps, with tags referencing various milestones:
+**
+**  *  Resolve names and similar preparation                tag-select-0100
+**  *  Scan of the FROM clause                              tag-select-0200
+**      +  OUTER JOIN strength reduction                      tag-select-0220
+**      +  Sub-query ORDER BY removal                         tag-select-0230
+**      +  Query flattening                                   tag-select-0240
+**  *  Separate subroutine for compound-SELECT              tag-select-0300
+**  *  WHERE-clause constant propagation                    tag-select-0330
+**  *  Count()-of-VIEW optimization                         tag-select-0350
+**  *  Scan of the FROM clause again                        tag-select-0400
+**      +  Authorize unreferenced tables                      tag-select-0410
+**      +  Predicate push-down optimization                   tag-select-0420
+**      +  Omit unused subquery columns optimization          tag-select-0440
+**      +  Generate code to implement subqueries              tag-select-0480
+**         -  Co-routines                                       tag-select-0482
+**         -  Reuse previously computed CTE                     tag-select-0484
+**         -  REuse previously computed VIEW                    tag-select-0486
+**         -  Materialize a VIEW or CTE                         tag-select-0488
+**  *  DISTINCT ORDER BY -> GROUP BY optimization           tag-select-0500
+**  *  Set up for ORDER BY                                  tag-select-0600
+**  *  Create output table                                  tag-select-0630
+**  *  Prepare registers for LIMIT                          tag-select-0650
+**  *  Setup for DISTINCT                                   tag-select-0680
+**  *  Generate code for non-aggregate and non-GROUP BY     tag-select-0700
+**  *  Generate code for aggregate and/or GROUP BY          tag-select-0800
+**      +  GROUP BY queries                                   tag-select-0810
+**      +  non-GROUP BY queries                               tag-select-0820
+**         -  Special case of count() w/o GROUP BY              tag-select-0821
+**         -  General case of non-GROUP BY aggregates           tag-select-0822
+**  *  Sort results, as needed                              tag-select-0900
+**  *  Internal self-checks                                 tag-select-1000
 */
 int sqlite3Select(
  Parse *pParse,         /* The parser context */
@@ -7391,6 +7428,7 @@ int sqlite3Select(
  }
 #endif

+  /* tag-select-0100 */
  assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo );
  assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo );
  assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue );
@@ -7477,6 +7515,7 @@ int sqlite3Select(

  /* Try to do various optimizations (flattening subqueries, and strength
  ** reduction of join operators) in the FROM clause up into the main query
+  ** tag-select-0200
  */
 #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW)
  for(i=0; !p->pPrior && i<pTabList->nSrc; i++){
@@ -7499,6 +7538,7 @@ int sqlite3Select(
    ** way that the i-th table cannot be the NULL row of a join, then
    ** perform the appropriate simplification. This is called
    ** "OUTER JOIN strength reduction" in the SQLite documentation.
+    ** tag-select-0220
    */
    if( (pItem->fg.jointype & (JT_LEFT|JT_LTORJ))!=0
     && sqlite3ExprImpliesNonNullRow(p->pWhere, pItem->iCursor,
@@ -7569,7 +7609,8 @@ int sqlite3Select(
    if( (pSub->selFlags & SF_Aggregate)!=0 ) continue;
    assert( pSub->pGroupBy==0 );

-    /* If a FROM-clause subquery has an ORDER BY clause that is not
+    /* tag-select-0230:
+    ** If a FROM-clause subquery has an ORDER BY clause that is not
    ** really doing anything, then delete it now so that it does not
    ** interfere with query flattening.  See the discussion at
    ** https://sqlite.org/forum/forumpost/2d76f2bcf65d256a
@@ -7635,6 +7676,7 @@ int sqlite3Select(
      continue;
    }

+    /* tag-select-0240 */
    if( flattenSubquery(pParse, p, i, isAgg) ){
      if( pParse->nErr ) goto select_end;
      /* This subquery can be absorbed into its parent. */
@@ -7650,7 +7692,7 @@ int sqlite3Select(

 #ifndef SQLITE_OMIT_COMPOUND_SELECT
  /* Handle compound SELECT statements using the separate multiSelect()
-  ** procedure.
+  ** procedure.  tag-select-0300
  */
  if( p->pPrior ){
    rc = multiSelect(pParse, p, pDest);
@@ -7666,9 +7708,9 @@ int sqlite3Select(
 #endif

  /* Do the WHERE-clause constant propagation optimization if this is
-  ** a join.  No need to speed time on this operation for non-join queries
+  ** a join.  No need to spend time on this operation for non-join queries
  ** as the equivalent optimization will be handled by query planner in
-  ** sqlite3WhereBegin().
+  ** sqlite3WhereBegin().  tag-select-0330
  */
  if( p->pWhere!=0
   && p->pWhere->op==TK_AND
@@ -7685,6 +7727,7 @@ int sqlite3Select(
    TREETRACE(0x2000,pParse,p,("Constant propagation not helpful\n"));
  }

+  /* tag-select-0350 */
  if( OptimizationEnabled(db, SQLITE_QueryFlattener|SQLITE_CountOfView)
   && countOfViewOptimization(pParse, p)
  ){
@@ -7692,9 +7735,12 @@ int sqlite3Select(
    pTabList = p->pSrc;
  }

-  /* For each term in the FROM clause, do two things:
-  ** (1) Authorized unreferenced tables
-  ** (2) Generate code for all sub-queries
+  /* Loop over all terms in the FROM clause and do two things for each term:
+  **
+  **   (1) Authorize unreferenced tables
+  **   (2) Generate code for all sub-queries
+  **
+  ** tag-select-0400
  */
  for(i=0; i<pTabList->nSrc; i++){
    SrcItem *pItem = &pTabList->a[i];
@@ -7706,7 +7752,9 @@ int sqlite3Select(
    const char *zSavedAuthContext;
 #endif

-    /* Issue SQLITE_READ authorizations with a fake column name for any
+    /* Authorized unreferenced tables.  tag-select-0410
+    **
+    ** Issue SQLITE_READ authorizations with a fake column name for any
    ** tables that are referenced but from which no values are extracted.
    ** Examples of where these kinds of null SQLITE_READ authorizations
    ** would occur:
@@ -7742,10 +7790,9 @@ int sqlite3Select(
    pSubq = pItem->u4.pSubq;
    assert( pSubq!=0 );
    pSub = pSubq->pSelect;
-    if( pSubq->addrFillSub!=0 ) continue;

    /* The code for a subquery should only be generated once. */
-    assert( pSubq->addrFillSub==0 );
+    if( pSubq->addrFillSub!=0 ) continue;

    /* Increment Parse.nHeight by the height of the largest expression
    ** tree referred to by this, the parent select. The child select
@@ -7758,6 +7805,7 @@ int sqlite3Select(

    /* Make copies of constant WHERE-clause terms in the outer query down
    ** inside the subquery.  This can help the subquery to run more efficiently.
+    ** This is the "predicate push-down optimization".  tag-select-0420
    */
    if( OptimizationEnabled(db, SQLITE_PushDown)
     && (pItem->fg.isCte==0
@@ -7778,6 +7826,7 @@ int sqlite3Select(

    /* Convert unused result columns of the subquery into simple NULL
    ** expressions, to avoid unneeded searching and computation.
+    ** tag-select-0440
    */
    if( OptimizationEnabled(db, SQLITE_NullUnusedCols)
     && disableUnusedSubqueryResultColumns(pItem)
@@ -7795,11 +7844,11 @@ int sqlite3Select(
    zSavedAuthContext = pParse->zAuthContext;
    pParse->zAuthContext = pItem->zName;

-    /* Generate code to implement the subquery
+    /* Generate byte-code to implement the subquery  tag-select-0480
    */
    if( fromClauseTermCanBeCoroutine(pParse, pTabList, i, p->selFlags) ){
      /* Implement a co-routine that will return a single row of the result
-      ** set on each invocation.
+      ** set on each invocation.  tag-select-0482
      */
      int addrTop = sqlite3VdbeCurrentAddr(v)+1;
    
@@ -7814,13 +7863,14 @@ int sqlite3Select(
      pItem->fg.viaCoroutine = 1;
      pSubq->regResult = dest.iSdst;
      sqlite3VdbeEndCoroutine(v, pSubq->regReturn);
+      VdbeComment((v, "end %!S", pItem));
      sqlite3VdbeJumpHere(v, addrTop-1);
      sqlite3ClearTempRegCache(pParse);
    }else if( pItem->fg.isCte && pItem->u2.pCteUse->addrM9e>0 ){
      /* This is a CTE for which materialization code has already been
      ** generated.  Invoke the subroutine to compute the materialization,
-      ** the make the pItem->iCursor be a copy of the ephemeral table that
-      ** holds the result of the materialization. */
+      ** then make the pItem->iCursor be a copy of the ephemeral table that
+      ** holds the result of the materialization. tag-select-0484 */
      CteUse *pCteUse = pItem->u2.pCteUse;
      sqlite3VdbeAddOp2(v, OP_Gosub, pCteUse->regRtn, pCteUse->addrM9e);
      if( pItem->iCursor!=pCteUse->iCur ){
@@ -7830,7 +7880,7 @@ int sqlite3Select(
      pSub->nSelectRow = pCteUse->nRowEst;
    }else if( (pPrior = isSelfJoinView(pTabList, pItem, 0, i))!=0 ){
      /* This view has already been materialized by a prior entry in
-      ** this same FROM clause.  Reuse it. */
+      ** this same FROM clause.  Reuse it.  tag-select-0486 */
      Subquery *pPriorSubq;
      assert( pPrior->fg.isSubquery );
      pPriorSubq = pPrior->u4.pSubq;
@@ -7844,7 +7894,7 @@ int sqlite3Select(
    }else{
      /* Materialize the view.  If the view is not correlated, generate a
      ** subroutine to do the materialization so that subsequent uses of
-      ** the same view can reuse the materialization. */
+      ** the same view can reuse the materialization.  tag-select-0488 */
      int topAddr;
      int onceAddr = 0;
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
@@ -7904,7 +7954,9 @@ int sqlite3Select(
  }
 #endif

-  /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and
+  /* tag-select-0500
+  **
+  ** If the query is DISTINCT with an ORDER BY but is not an aggregate, and
  ** if the select-list is the same as the ORDER BY list, then this query
  ** can be rewritten as a GROUP BY. In other words, this:
  **
@@ -7954,7 +8006,7 @@ int sqlite3Select(
  ** If that is the case, then the OP_OpenEphemeral instruction will be
  ** changed to an OP_Noop once we figure out that the sorting index is
  ** not needed.  The sSort.addrSortIndex variable is used to facilitate
-  ** that change.
+  ** that change.  tag-select-0600
  */
  if( sSort.pOrderBy ){
    KeyInfo *pKeyInfo;
@@ -7971,6 +8023,7 @@ int sqlite3Select(
  }

  /* If the output is destined for a temporary table, open that table.
+  ** tag-select-0630
  */
  if( pDest->eDest==SRT_EphemTab ){
    sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pDest->iSDParm, pEList->nExpr);
@@ -7988,7 +8041,7 @@ int sqlite3Select(
    }
  }

-  /* Set the limiter.
+  /* Set the limiter.  tag-select-0650
  */
  iEnd = sqlite3VdbeMakeLabel(pParse);
  if( (p->selFlags & SF_FixedLimit)==0 ){
@@ -8000,7 +8053,7 @@ int sqlite3Select(
    sSort.sortFlags |= SORTFLAG_UseSorter;
  }

-  /* Open an ephemeral index to use for the distinct set.
+  /* Open an ephemeral index to use for the distinct set. tag-select-0680
  */
  if( p->selFlags & SF_Distinct ){
    sDistinct.tabTnct = pParse->nTab++;
@@ -8015,7 +8068,7 @@ int sqlite3Select(
  }

  if( !isAgg && pGroupBy==0 ){
-    /* No aggregate functions and no GROUP BY clause */
+    /* No aggregate functions and no GROUP BY clause.  tag-select-0700 */
    u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0)
                   | (p->selFlags & SF_FixedLimit);
 #ifndef SQLITE_OMIT_WINDOWFUNC
@@ -8088,8 +8141,8 @@ int sqlite3Select(
      sqlite3WhereEnd(pWInfo);
    }
  }else{
-    /* This case when there exist aggregate functions or a GROUP BY clause
-    ** or both */
+    /* This case is for when there exist aggregate functions or a GROUP BY
+    ** clause or both.  tag-select-0800 */
    NameContext sNC;    /* Name context for processing aggregate information */
    int iAMem;          /* First Mem address for storing current GROUP BY */
    int iBMem;          /* First Mem address for previous GROUP BY */
@@ -8208,7 +8261,7 @@ int sqlite3Select(


    /* Processing for aggregates with GROUP BY is very different and
-    ** much more complex than aggregates without a GROUP BY.
+    ** much more complex than aggregates without a GROUP BY.  tag-select-0810
    */
    if( pGroupBy ){
      KeyInfo *pKeyInfo;  /* Keying information for the group by clause */
@@ -8505,9 +8558,12 @@ int sqlite3Select(
      }
    } /* endif pGroupBy.  Begin aggregate queries without GROUP BY: */
    else {
+      /* Aggregate functions without GROUP BY. tag-select-0820 */
      Table *pTab;
      if( (pTab = isSimpleCount(p, pAggInfo))!=0 ){
-        /* If isSimpleCount() returns a pointer to a Table structure, then
+        /* tag-select-0821
+        **
+        ** If isSimpleCount() returns a pointer to a Table structure, then
        ** the SQL statement is of the form:
        **
        **   SELECT count(*) FROM <tbl>
@@ -8566,6 +8622,8 @@ int sqlite3Select(
        sqlite3VdbeAddOp1(v, OP_Close, iCsr);
        explainSimpleCount(pParse, pTab, pBest);
      }else{
+        /* The general case of an aggregate query without GROUP BY
+        ** tag-select-0822 */
        int regAcc = 0;           /* "populate accumulators" flag */
        ExprList *pDistinct = 0;
        u16 distFlag = 0;
@@ -8654,7 +8712,7 @@ int sqlite3Select(
  }

  /* If there is an ORDER BY clause, then we need to sort the results
-  ** and send them to the callback one by one.
+  ** and send them to the callback one by one.  tag-select-0900
  */
  if( sSort.pOrderBy ){
    assert( p->pEList==pEList );
@@ -8677,6 +8735,7 @@ select_end:
  assert( db->mallocFailed==0 || pParse->nErr!=0 );
  sqlite3ExprListDelete(db, pMinMaxOrderBy);
 #ifdef SQLITE_DEBUG
+  /* Internal self-checks.  tag-select-1000 */
  if( pAggInfo && !db->mallocFailed ){
 #if TREETRACE_ENABLED
    if( sqlite3TreeTrace & 0x20 ){
@@ -283,7 +283,7 @@ INCLUDE ../ext/consio/console_io.c
 #else
 /* For Fiddle, all console handling and emit redirection is omitted. */
 /* These next 3 macros are for emitting formatted output. When complaints
- * from the WASM build are issued for non-formatted output, (when a mere
+ * from the WASM build are issued for non-formatted output, when a mere
 * string literal is to be emitted, the ?putz(z) forms should be used.
 * (This permits compile-time checking of format string / argument mismatch.)
 */
@@ -534,6 +534,14 @@ static char *shell_strncpy(char *dest, const char *src, size_t n){
  return dest;
 }

+/*
+** strcpy() workalike to squelch an unwarranted link-time warning
+** from OpenBSD.
+*/
+static void shell_strcpy(char *dest, const char *src){
+  while( (*(dest++) = *(src++))!=0 ){}
+}
+
 /*
 ** Optionally disable dynamic continuation prompt.
 ** Unless disabled, the continuation prompt shows open SQL lexemes if any,
@@ -599,7 +607,7 @@ static char *dynamicContinuePrompt(void){
      size_t ncp = strlen(continuePrompt);
      size_t ndp = strlen(dynPrompt.zScannerAwaits);
      if( ndp > ncp-3 ) return continuePrompt;
-      strcpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits);
+      shell_strcpy(dynPrompt.dynamicPrompt, dynPrompt.zScannerAwaits);
      while( ndp<3 ) dynPrompt.dynamicPrompt[ndp++] = ' ';
      shell_strncpy(dynPrompt.dynamicPrompt+3, continuePrompt+3,
              PROMPT_LEN_MAX-4);
@@ -3548,10 +3556,11 @@ static void bind_prepared_stmt(ShellState *pArg, sqlite3_stmt *pStmt){
    }else if( strncmp(zVar, "$int_", 5)==0 ){
      sqlite3_bind_int(pStmt, i, atoi(&zVar[5]));
    }else if( strncmp(zVar, "$text_", 6)==0 ){
-      char *zBuf = sqlite3_malloc64( strlen(zVar)-5 );
+      size_t szVar = strlen(zVar);
+      char *zBuf = sqlite3_malloc64( szVar-5 );
      if( zBuf ){
-        memcpy(zBuf, &zVar[6], strlen(zVar)-5);
-        sqlite3_bind_text64(pStmt, i, zBuf, -1, sqlite3_free, SQLITE_UTF8);
+        memcpy(zBuf, &zVar[6], szVar-5);
+        sqlite3_bind_text64(pStmt, i, zBuf, szVar-6, sqlite3_free, SQLITE_UTF8);
      }
    }else{
      sqlite3_bind_null(pStmt, i);
@@ -6343,12 +6352,17 @@ static int shell_dbinfo_command(ShellState *p, int nArg, char **azArg){
 }
 #endif /* SQLITE_SHELL_HAVE_RECOVER */

+/*
+** Print the given string as an error message.
+*/
+static void shellEmitError(const char *zErr){
+  eputf("Error: %s\n", zErr);
+}
 /*
 ** Print the current sqlite3_errmsg() value to stderr and return 1.
 */
 static int shellDatabaseError(sqlite3 *db){
-  const char *zErr = sqlite3_errmsg(db);
-  eputf("Error: %s\n", zErr);
+  shellEmitError(sqlite3_errmsg(db));
  return 1;
 }

@@ -6893,7 +6907,7 @@ static int arErrorMsg(ArCommand *pAr, const char *zFmt, ...){
  va_start(ap, zFmt);
  z = sqlite3_vmprintf(zFmt, ap);
  va_end(ap);
-  eputf("Error: %s\n", z);
+  shellEmitError(z);
  if( pAr->fromCmdLine ){
    eputz("Use \"-A\" for more help\n");
  }else{
@@ -8124,7 +8138,7 @@ static int do_meta_command(char *zLine, ShellState *p){
    open_db(p, 0);
    pBackup = sqlite3_backup_init(pDest, "main", p->db, zDb);
    if( pBackup==0 ){
-      eputf("Error: %s\n", sqlite3_errmsg(pDest));
+      shellDatabaseError(pDest);
      close_db(pDest);
      return 1;
    }
@@ -8133,7 +8147,7 @@ static int do_meta_command(char *zLine, ShellState *p){
    if( rc==SQLITE_DONE ){
      rc = 0;
    }else{
-      eputf("Error: %s\n", sqlite3_errmsg(pDest));
+      shellDatabaseError(pDest);
      rc = 1;
    }
    close_db(pDest);
@@ -8309,7 +8323,7 @@ static int do_meta_command(char *zLine, ShellState *p){
    open_db(p, 0);
    rc = sqlite3_prepare_v2(p->db, "PRAGMA database_list", -1, &pStmt, 0);
    if( rc ){
-      eputf("Error: %s\n", sqlite3_errmsg(p->db));
+      shellDatabaseError(p->db);
      rc = 1;
    }else{
      while( sqlite3_step(pStmt)==SQLITE_ROW ){
@@ -9005,7 +9019,7 @@ static int do_meta_command(char *zLine, ShellState *p){
    zSql = 0;
    if( rc ){
      if (pStmt) sqlite3_finalize(pStmt);
-      eputf("Error: %s\n", sqlite3_errmsg(p->db));
+      shellDatabaseError(p->db);
      import_cleanup(&sCtx);
      rc = 1;
      goto meta_command_exit;
@@ -9049,7 +9063,7 @@ static int do_meta_command(char *zLine, ShellState *p){
    sqlite3_free(zSql);
    zSql = 0;
    if( rc ){
-      eputf("Error: %s\n", sqlite3_errmsg(p->db));
+      shellDatabaseError(p->db);
      if (pStmt) sqlite3_finalize(pStmt);
      import_cleanup(&sCtx);
      rc = 1;
@@ -9343,7 +9357,7 @@ static int do_meta_command(char *zLine, ShellState *p){
    open_db(p, 0);
    rc = sqlite3_load_extension(p->db, zFile, zProc, &zErrMsg);
    if( rc!=SQLITE_OK ){
-      eputf("Error: %s\n", zErrMsg);
+      shellEmitError(zErrMsg);
      sqlite3_free(zErrMsg);
      rc = 1;
    }
@@ -9965,7 +9979,7 @@ static int do_meta_command(char *zLine, ShellState *p){
    open_db(p, 0);
    pBackup = sqlite3_backup_init(p->db, zDb, pSrc, "main");
    if( pBackup==0 ){
-      eputf("Error: %s\n", sqlite3_errmsg(p->db));
+      shellDatabaseError(p->db);
      close_db(pSrc);
      return 1;
    }
@@ -9983,7 +9997,7 @@ static int do_meta_command(char *zLine, ShellState *p){
      eputz("Error: source database is busy\n");
      rc = 1;
    }else{
-      eputf("Error: %s\n", sqlite3_errmsg(p->db));
+      shellDatabaseError(p->db);
      rc = 1;
    }
    close_db(pSrc);
@@ -10080,7 +10094,7 @@ static int do_meta_command(char *zLine, ShellState *p){
      rc = sqlite3_prepare_v2(p->db, "SELECT name FROM pragma_database_list",
                              -1, &pStmt, 0);
      if( rc ){
-        eputf("Error: %s\n", sqlite3_errmsg(p->db));
+        shellDatabaseError(p->db);
        sqlite3_finalize(pStmt);
        rc = 1;
        goto meta_command_exit;
@@ -10149,7 +10163,7 @@ static int do_meta_command(char *zLine, ShellState *p){
      freeText(&sSelect);
    }
    if( zErrMsg ){
-      eputf("Error: %s\n", zErrMsg);
+      shellEmitError(zErrMsg);
      sqlite3_free(zErrMsg);
      rc = 1;
    }else if( rc != SQLITE_OK ){
@@ -10920,7 +10934,7 @@ static int do_meta_command(char *zLine, ShellState *p){
    {"json_selfcheck",     SQLITE_TESTCTRL_JSON_SELFCHECK ,0,"BOOLEAN"      },
    {"localtime_fault",    SQLITE_TESTCTRL_LOCALTIME_FAULT,0,"BOOLEAN"      },
    {"never_corrupt",      SQLITE_TESTCTRL_NEVER_CORRUPT,1, "BOOLEAN"       },
-    {"optimizations",      SQLITE_TESTCTRL_OPTIMIZATIONS,0,"DISABLE-MASK"   },
+    {"optimizations",      SQLITE_TESTCTRL_OPTIMIZATIONS,0,"DISABLE-MASK ..."},
 #ifdef YYCOVERAGE
    {"parser_coverage",    SQLITE_TESTCTRL_PARSER_COVERAGE,0,""             },
 #endif
@@ -10984,8 +10998,118 @@ static int do_meta_command(char *zLine, ShellState *p){
    }else{
      switch(testctrl){

+        /* Special processing for .testctrl opt MASK ...
+        ** Each MASK argument can be one of:
+        **
+        **      +LABEL       Enable the named optimization 
+        **
+        **      -LABEL       Disable the named optimization
+        **
+        **      INTEGER      Mask of optimizations to disable
+        */
+        case SQLITE_TESTCTRL_OPTIMIZATIONS: {
+          static const struct {
+             unsigned int mask;    /* Mask for this optimization */
+             unsigned int bDsply;  /* Display this on output */
+             const char *zLabel;   /* Name of optimization */
+          } aLabel[] = {
+            { 0x00000001, 1, "QueryFlattener" },
+            { 0x00000001, 0, "Flatten" },
+            { 0x00000002, 1, "WindowFunc" },
+            { 0x00000004, 1, "GroupByOrder" },
+            { 0x00000008, 1, "FactorOutConst" },
+            { 0x00000010, 1, "DistinctOpt" },
+            { 0x00000020, 1, "CoverIdxScan" },
+            { 0x00000040, 1, "OrderByIdxJoin" },
+            { 0x00000080, 1, "Transitive" },
+            { 0x00000100, 1, "OmitNoopJoin" },
+            { 0x00000200, 1, "CountOfView" },
+            { 0x00000400, 1, "CurosrHints" },
+            { 0x00000800, 1, "Stat4" },
+            { 0x00001000, 1, "PushDown" },
+            { 0x00002000, 1, "SimplifyJoin" },
+            { 0x00004000, 1, "SkipScan" },
+            { 0x00008000, 1, "PropagateConst" },
+            { 0x00010000, 1, "MinMaxOpt" },
+            { 0x00020000, 1, "SeekScan" },
+            { 0x00040000, 1, "OmitOrderBy" },
+            { 0x00080000, 1, "BloomFilter" },
+            { 0x00100000, 1, "BloomPulldown" },
+            { 0x00200000, 1, "BalancedMerge" },
+            { 0x00400000, 1, "ReleaseReg" },
+            { 0x00800000, 1, "FlttnUnionAll" },
+            { 0x01000000, 1, "IndexedEXpr" },
+            { 0x02000000, 1, "Coroutines" },
+            { 0x04000000, 1, "NullUnusedCols" },
+            { 0x08000000, 1, "OnePass" },
+            { 0x10000000, 1, "OrderBySubq" },
+            { 0xffffffff, 0, "All" },
+          };
+          unsigned int curOpt;
+          unsigned int newOpt;
+          int ii;
+          sqlite3_test_control(SQLITE_TESTCTRL_GETOPT, p->db, &curOpt);
+          newOpt = curOpt;
+          for(ii=2; ii<nArg; ii++){
+            const char *z = azArg[ii];
+            int useLabel = 0;
+            const char *zLabel = 0;
+            if( (z[0]=='+'|| z[0]=='-') && !IsDigit(z[1]) ){
+              useLabel = z[0];
+              zLabel = &z[1];
+            }else if( !IsDigit(z[0]) && z[0]!=0 && !IsDigit(z[1]) ){
+              useLabel = '+';
+              zLabel = z;
+            }else{
+              newOpt = (unsigned int)strtol(z,0,0);
+            }
+            if( useLabel ){
+              int jj;
+              for(jj=0; jj<ArraySize(aLabel); jj++){
+                if( sqlite3_stricmp(zLabel, aLabel[jj].zLabel)==0 ) break;
+              }
+              if( jj>=ArraySize(aLabel) ){
+                eputf("Error: no such optimization: \"%s\"\n", zLabel);
+                eputz("Should be one of:");
+                for(jj=0; jj<ArraySize(aLabel); jj++){
+                  eputf(" %s", aLabel[jj].zLabel);
+                }
+                eputz("\n");
+                rc = 1;
+                goto meta_command_exit;
+              }
+              if( useLabel=='+' ){
+                newOpt &= ~aLabel[jj].mask;
+              }else{
+                newOpt |= aLabel[jj].mask;
+              }
+            }
+          }
+          if( curOpt!=newOpt ){
+            sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,p->db,newOpt);
+          }else if( nArg<3 ){
+            curOpt = ~newOpt;
+          }
+          if( newOpt==0 ){
+            oputz("+All\n");
+          }else if( newOpt==0xffffffff ){
+            oputz("-All\n");
+          }else{
+            int jj;
+            for(jj=0; jj<ArraySize(aLabel); jj++){
+              unsigned int m = aLabel[jj].mask;
+              if( !aLabel[jj].bDsply  ) continue;
+              if( (curOpt&m)!=(newOpt&m) ){
+                oputf("%c%s\n", (newOpt & m)==0 ? '+' : '-',
+                      aLabel[jj].zLabel);
+              }
+            }
+          }
+          rc2 = isOk = 3;
+          break;
+        }
+
        /* sqlite3_test_control(int, db, int) */
-        case SQLITE_TESTCTRL_OPTIMIZATIONS:
        case SQLITE_TESTCTRL_FK_NO_ACTION:
          if( nArg==3 ){
            unsigned int opt = (unsigned int)strtol(azArg[2], 0, 0);
@@ -12728,7 +12852,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
        open_db(&data, 0);
        rc = shell_exec(&data, z, &zErrMsg);
        if( zErrMsg!=0 ){
-          eputf("Error: %s\n", zErrMsg);
+          shellEmitError(zErrMsg);
          if( bail_on_error ) return rc!=0 ? rc : 1;
        }else if( rc!=0 ){
          eputf("Error: unable to process SQL \"%s\"\n", z);
@@ -12782,7 +12906,7 @@ int SQLITE_CDECL wmain(int argc, wchar_t **wargv){
        rc = shell_exec(&data, azCmd[i], &zErrMsg);
        if( zErrMsg || rc ){
          if( zErrMsg!=0 ){
-            eputf("Error: %s\n", zErrMsg);
+            shellEmitError(zErrMsg);
          }else{
            eputf("Error: unable to process SQL: %s\n", azCmd[i]);
          }
@@ -7427,9 +7427,11 @@ struct sqlite3_module {
 ** will be returned by the strategy.
 **
 ** The xBestIndex method may optionally populate the idxFlags field with a
-** mask of SQLITE_INDEX_SCAN_* flags. Currently there is only one such flag -
-** SQLITE_INDEX_SCAN_UNIQUE. If the xBestIndex method sets this flag, SQLite
-** assumes that the strategy may visit at most one row.
+** mask of SQLITE_INDEX_SCAN_* flags. One such flag is
+** [SQLITE_INDEX_SCAN_HEX], which if set causes the [EXPLAIN QUERY PLAN]
+** output to show the idxNum has hex instead of as decimal.  Another flag is
+** SQLITE_INDEX_SCAN_UNIQUE, which if set indicates that the query plan will
+** return at most one row.
 **
 ** Additionally, if xBestIndex sets the SQLITE_INDEX_SCAN_UNIQUE flag, then
 ** SQLite also assumes that if a call to the xUpdate() method is made as
@@ -7493,7 +7495,9 @@ struct sqlite3_index_info {
 ** [sqlite3_index_info].idxFlags field to some combination of
 ** these bits.
 */
-#define SQLITE_INDEX_SCAN_UNIQUE      1     /* Scan visits at most 1 row */
+#define SQLITE_INDEX_SCAN_UNIQUE 0x00000001 /* Scan visits at most 1 row */
+#define SQLITE_INDEX_SCAN_HEX    0x00000002 /* Display idxNum as hex */
+                                            /* in EXPLAIN QUERY PLAN */

 /*
 ** CAPI3REF: Virtual Table Constraint Operator Codes
@@ -8330,6 +8334,7 @@ int sqlite3_test_control(int op, ...);
 #define SQLITE_TESTCTRL_JSON_SELFCHECK          14
 #define SQLITE_TESTCTRL_OPTIMIZATIONS           15
 #define SQLITE_TESTCTRL_ISKEYWORD               16  /* NOT USED */
+#define SQLITE_TESTCTRL_GETOPT                  16
 #define SQLITE_TESTCTRL_SCRATCHMALLOC           17  /* NOT USED */
 #define SQLITE_TESTCTRL_INTERNAL_FUNCTIONS      17
 #define SQLITE_TESTCTRL_LOCALTIME_FAULT         18
@@ -372,12 +372,17 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
      }
    }
    case CC_PIPE: {
-      if( z[1]!='|' ){
-        *tokenType = TK_BITOR;
-        return 1;
-      }else{
+      if( z[1]=='|' ){
        *tokenType = TK_CONCAT;
        return 2;
+#ifdef TK_PIPE
+      }else if( z[1]=='>' ){
+        *tokenType = TK_PIPE;
+        return 2;
+#endif
+      }else{
+        *tokenType = TK_BITOR;
+        return 1;
      }
    }
    case CC_COMMA: {
@@ -226,6 +226,9 @@ void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc){
    if( pItem->fg.viaCoroutine )   sqlite3_str_appendf(&x, " viaCoroutine");
    if( pItem->fg.notCte )         sqlite3_str_appendf(&x, " notCte");
    if( pItem->fg.isNestedFrom )   sqlite3_str_appendf(&x, " isNestedFrom");
+    if( pItem->fg.fixedSchema )    sqlite3_str_appendf(&x, " fixedSchema");
+    if( pItem->fg.hadSchema )      sqlite3_str_appendf(&x, " hadSchema");
+    if( pItem->fg.isSubquery )     sqlite3_str_appendf(&x, " isSubquery");

    sqlite3StrAccumFinish(&x);
    sqlite3TreeViewItem(pView, zLine, i<pSrc->nSrc-1);
@@ -237,12 +240,16 @@ void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc){
      sqlite3TreeViewIdList(pView, pItem->u3.pUsing, (--n)>0, "USING");
    }
    if( pItem->fg.isSubquery ){
+      assert( n==1 );
      if( pItem->pSTab ){
        Table *pTab = pItem->pSTab;
        sqlite3TreeViewColumnList(pView, pTab->aCol, pTab->nCol, 1);
      }
      assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem) );
-      sqlite3TreeViewSelect(pView, pItem->u4.pSubq->pSelect, (--n)>0);
+      sqlite3TreeViewPush(&pView, 0);
+      sqlite3TreeViewLine(pView, "SUBQUERY");
+      sqlite3TreeViewPop(&pView);
+      sqlite3TreeViewSelect(pView, pItem->u4.pSubq->pSelect, 0);
    }
    if( pItem->fg.isTabFunc ){
      sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:");
@@ -3987,6 +3987,7 @@ static int whereLoopAddBtree(
    pNew->prereq = mPrereq;
    pNew->nOut = rSize;
    pNew->u.btree.pIndex = pProbe;
+    pNew->u.btree.pOrderBy = 0;
    b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor);

    /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */
@@ -4349,6 +4350,7 @@ static int whereLoopAddVirtualOne(
  pNew->u.vtab.idxStr = pIdxInfo->idxStr;
  pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ?
      pIdxInfo->nOrderBy : 0);
+  pNew->u.vtab.bIdxNumHex = (pIdxInfo->idxFlags&SQLITE_INDEX_SCAN_HEX)!=0;
  pNew->rSetup = 0;
  pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost);
  pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows);
@@ -7122,15 +7124,17 @@ WhereInfo *sqlite3WhereBegin(
    pSrc = &pTabList->a[pLevel->iFrom];
    if( pSrc->fg.isMaterialized ){
      Subquery *pSubq;
+      int iOnce = 0;
      assert( pSrc->fg.isSubquery );
      pSubq = pSrc->u4.pSubq;
-      if( pSrc->fg.isCorrelated ){
-        sqlite3VdbeAddOp2(v, OP_Gosub, pSubq->regReturn, pSubq->addrFillSub);
+      if( pSrc->fg.isCorrelated==0 ){
+        iOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
      }else{
-        int iOnce = sqlite3VdbeAddOp0(v, OP_Once);  VdbeCoverage(v);
-        sqlite3VdbeAddOp2(v, OP_Gosub, pSubq->regReturn, pSubq->addrFillSub);
-        sqlite3VdbeJumpHere(v, iOnce);
+        iOnce = 0;
      }
+      sqlite3VdbeAddOp2(v, OP_Gosub, pSubq->regReturn, pSubq->addrFillSub);
+      VdbeComment((v, "materialize %!S", pSrc));
+      if( iOnce )  sqlite3VdbeJumpHere(v, iOnce);
    }
    assert( pTabList == pWInfo->pTabList );
    if( (wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))!=0 ){
@@ -149,6 +149,7 @@ struct WhereLoop {
      int idxNum;            /* Index number */
      u32 needFree : 1;      /* True if sqlite3_free(idxStr) is needed */
      u32 bOmitOffset : 1;   /* True to let virtual table handle offset */
+      u32 bIdxNumHex : 1;    /* Show idxNum as hex in EXPLAIN QUERY PLAN */
      i8 isOrdered;          /* True if satisfies ORDER BY */
      u16 omitMask;          /* Terms that may be omitted */
      char *idxStr;          /* Index identifier string */
@@ -200,7 +200,9 @@ int sqlite3WhereExplainOneScan(
    }
 #ifndef SQLITE_OMIT_VIRTUALTABLE
    else if( (flags & WHERE_VIRTUALTABLE)!=0 ){
-      sqlite3_str_appendf(&str, " VIRTUAL TABLE INDEX %d:%s",
+      sqlite3_str_appendall(&str, " VIRTUAL TABLE INDEX ");
+      sqlite3_str_appendf(&str,
+                  pLoop->u.vtab.bIdxNumHex ? "0x%x:%s" : "%d:%s",
                  pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr);
    }
 #endif
@@ -220,11 +220,20 @@ static int isLikeOrGlob(
  }
  if( z ){

-    /* Count the number of prefix characters prior to the first wildcard */
+    /* Count the number of prefix characters prior to the first wildcard.
+    ** If the underlying database has a UTF16LE encoding, then only consider
+    ** ASCII characters.  Note that the encoding of z[] is UTF8 - we are
+    ** dealing with only UTF8 here in this code, but the database engine
+    ** itself might be processing content using a different encoding. */
    cnt = 0;
    while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){
      cnt++;
-      if( c==wc[3] && z[cnt]!=0 ) cnt++;
+      if( c==wc[3] && z[cnt]!=0 ){
+        cnt++;
+      }else if( c>=0x80 && ENC(db)==SQLITE_UTF16LE ){
+         cnt--;
+         break;
+      }
    }

    /* The optimization is possible only if (1) the pattern does not begin
@@ -239,7 +248,7 @@ static int isLikeOrGlob(
      Expr *pPrefix;

      /* A "complete" match if the pattern ends with "*" or "%" */
-      *pisComplete = c==wc[0] && z[cnt+1]==0;
+      *pisComplete = c==wc[0] && z[cnt+1]==0 && ENC(db)!=SQLITE_UTF16LE;

      /* Get the pattern prefix.  Remove all escapes from the prefix. */
      pPrefix = sqlite3Expr(db, TK_STRING, (char*)z);
@@ -0,0 +1,86 @@
+# 2024-08-19
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# https://sqlite.org/forum/forumpost/eaa0a09786c6368b
+#
+# Apparently SQLite has been miscomputing leap-year dates before
+# the year 0400.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+# Skip this whole file if date and time functions are omitted
+# at compile-time
+#
+ifcapable {!datetime} {
+  finish_test
+  return
+}
+
+# Data sources:
+#   1-10  https://ssd.jpl.nasa.gov/tools/jdc/#/cd
+#   11    Jean Meeus, Astronomical Algorithms, ISBN 0-943396-61-1, p.59
+#   12    https://en.wikipedia.org/wiki/Julian_day
+#   
+# ID YEAR MONTH DAY JD
+set date5data {
+   1 2024     2  29 2460369.5
+   2 2024     3   1 2460370.5
+   3 2023     2  28 2460003.5
+   4 2023     3   1 2460004.5
+   5 2000     2  29 2451603.5
+   6 2000     3   1 2451604.5
+   7 1900     2  28 2415078.5
+   8 1900     3   1 2415079.5
+   9 1712     2  29 2346413.5
+  10 1712     3   1 2346414.5
+  11 1977     4  26 2443259.5
+  12 2013     1   1 2456293.5
+}
+
+foreach {id y m d jd} $date5data {
+  set date [format %04d-%02d-%02d $y $m $d]
+  do_execsql_test date5-jd$jd {
+    SELECT date($::jd);
+  } $date
+  do_execsql_test date5-cal/$date {
+    SELECT julianday($::date);
+  } $jd
+  for {set i 1} {$y+400*$i<=9999} {incr i} {
+    set y2 [expr {$y+400*$i}]
+    set date2 [format %04d-%02d-%02d $y2 $m $d]
+    set jd2 [expr {$jd+146097*$i}]
+    do_execsql_test date5-jd$jd2 {
+      SELECT date($::jd2);
+    } $date2
+    do_execsql_test date5-cal/$date2 {
+      SELECT julianday($::date2);
+    } $jd2
+  }
+  for {set i 1} {$y-400*$i>=-4712} {incr i} {
+    set y2 [expr {$y-400*$i}]
+    if {$y2<0} {
+      set date2 [format -%04d-%02d-%02d [expr {-$y2}] $m $d]
+    } else {
+      set date2 [format %04d-%02d-%02d $y2 $m $d]
+    }
+    set jd2 [expr {$jd-146097*$i}]
+    do_execsql_test date5-jd$jd2 {
+      SELECT date($::jd2);
+    } $date2
+    do_execsql_test date5-cal/$date2 {
+      SELECT julianday($::date2);
+    } $jd2
+  }
+}
+
+finish_test
@@ -64,21 +64,21 @@ do_eqp_test 101 {
     |     |--LEFT
     |     |  `--MERGE (UNION ALL)
     |     |     |--LEFT
-     |     |     |  `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+     |     |     |  `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
     |     |     `--RIGHT
-     |     |        `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+     |     |        `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
     |     `--RIGHT
-     |        `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+     |        `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
     `--RIGHT
        `--MERGE (UNION ALL)
           |--LEFT
           |  `--MERGE (UNION ALL)
           |     |--LEFT
-           |     |  `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+           |     |  `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
           |     `--RIGHT
-           |        `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+           |        `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
           `--RIGHT
-              `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+              `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
 }

 # Same test with the blanced-merge optimization
@@ -129,17 +129,17 @@ do_eqp_test 111 {
     |     |     |     |--LEFT
     |     |     |     |  `--MERGE (UNION ALL)
     |     |     |     |     |--LEFT
-     |     |     |     |     |  `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+     |     |     |     |     |  `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
     |     |     |     |     `--RIGHT
-     |     |     |     |        `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+     |     |     |     |        `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
     |     |     |     `--RIGHT
-     |     |     |        `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+     |     |     |        `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
     |     |     `--RIGHT
-     |     |        `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+     |     |        `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
     |     `--RIGHT
-     |        `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+     |        `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
     `--RIGHT
-        `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+        `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
 }

 finish_test
@@ -121,26 +121,26 @@ do_eqp_test tabfunc01-3.10 {
  SELECT value FROM generate_series(1,10) ORDER BY value;
 } {
  QUERY PLAN
-  `--SCAN generate_series VIRTUAL TABLE INDEX 19:
+  `--SCAN generate_series VIRTUAL TABLE INDEX 0x13:
 }
 do_eqp_test tabfunc01-3.11 {
  SELECT value FROM generate_series(1,10) ORDER BY +value;
 } {
  QUERY PLAN
-  |--SCAN generate_series VIRTUAL TABLE INDEX 3:
+  |--SCAN generate_series VIRTUAL TABLE INDEX 0x3:
  `--USE TEMP B-TREE FOR ORDER BY
 }
 do_eqp_test tabfunc01-3.12 {
  SELECT value FROM generate_series(1,10) ORDER BY value, stop;
 } {
  QUERY PLAN
-  `--SCAN generate_series VIRTUAL TABLE INDEX 19:
+  `--SCAN generate_series VIRTUAL TABLE INDEX 0x13:
 }
 do_eqp_test tabfunc01-3.13 {
  SELECT value FROM generate_series(1,10) ORDER BY stop, value;
 } {
  QUERY PLAN
-  |--SCAN generate_series VIRTUAL TABLE INDEX 3:
+  |--SCAN generate_series VIRTUAL TABLE INDEX 0x3:
  `--USE TEMP B-TREE FOR ORDER BY
 }

@@ -156,9 +156,9 @@ do_eqp_test tabfunc01-3.20 {
  QUERY PLAN
  `--MERGE (UNION ALL)
     |--LEFT
-     |  `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+     |  `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
     `--RIGHT
-        `--SCAN generate_series VIRTUAL TABLE INDEX 23:
+        `--SCAN generate_series VIRTUAL TABLE INDEX 0x17:
 }
  

@@ -985,7 +985,7 @@ proc query_plan_graph {sql} {
  }
  set a "\n  QUERY PLAN\n"
  append a [append_graph "  " dx cx 0]
-  regsub -all { 0x[A-F0-9]+\y} $a { xxxxxx} a
+  regsub -all {SUBQUERY 0x[A-F0-9]+\y} $a {SUBQUERY xxxxxx} a
  regsub -all {(MATERIALIZE|CO-ROUTINE|SUBQUERY) \d+\y} $a {\1 xxxxxx} a
  regsub -all {\((join|subquery)-\d+\)} $a {(\1-xxxxxx)} a
  return $a
@@ -54,5 +54,25 @@ do_execsql_test 1.3 {
  5 5,4 5,4,1 5,4,1,6 5,4,1,6,3 5,4,1,6,3,2
 }

+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 2.0 {
+  CREATE TABLE t1(x);
+}
+
+sqlite3_create_aggregate db
+
+breakpoint
+do_catchsql_test 2.1 {
+  SELECT min(x) OVER w1 FROM t1
+    WINDOW w1 AS (PARTITION BY x_count(x) OVER w1);
+} {1 {x_count() may not be used as a window function}}
+
+do_catchsql_test 2.2 {
+  SELECT min(x) FILTER (WHERE x_count(x) OVER w1) OVER w1 FROM t1
+    WINDOW w1 AS (PARTITION BY x OVER w1);
+} {1 {near "OVER": syntax error}}
+
+
 finish_test

@@ -164,6 +164,11 @@ struct Keyword {
 #else
 #  define RETURNING  0x00400000
 #endif
+#ifdef SQLITE_OMIT_FROM_FIRST
+#  define PIPE       0
+#else
+#  define PIPE       0x00800000
+#endif


 /*
@@ -174,6 +179,7 @@ static Keyword aKeywordTable[] = {
  { "ACTION",           "TK_ACTION",       FKEY,             0      },
  { "ADD",              "TK_ADD",          ALTER,            1      },
  { "AFTER",            "TK_AFTER",        TRIGGER,          0      },
+  { "AGGREGATE",        "TK_AGGREGATE",    PIPE,             10     },
  { "ALL",              "TK_ALL",          ALWAYS,           0      },
  { "ALTER",            "TK_ALTER",        ALTER,            0      },
  { "ALWAYS",           "TK_ALWAYS",       GENCOL,           0      },
Author	SHA1	Message	Date
drh	818d5bcf28	Remove redundant #ifdef accidentally inserted into mkkeywordhash.c FossilOrigin-Name: b08f3b74495e125f92c8968472fc8028c73d8df5400195df3081e3d8df8bd985	2024-08-26 14:20:20 +00:00
drh	8941cb4817	Add the AGGREGATE clause to the pipeline. FossilOrigin-Name: 16d32676e2e98a7710787de16850c328e7519e893f81ed56f6155a8fa9cbc3a4	2024-08-26 14:06:08 +00:00
drh	1082e86417	Working better now. FossilOrigin-Name: 4bdcc18d2cf4d11a7ca81117f6f8932fc8d280780d2c1e612189b2f5359cc778	2024-08-26 13:20:38 +00:00
drh	7267c510ac	Reduce rules added to the grammar. Kinda works, but there are still bugs. FossilOrigin-Name: 3c7a5cc6d0d186e68a5885299f7474ad2e31b839af404a649dbc8f5b8f1d0a62	2024-08-26 12:26:26 +00:00
drh	f53ed4e68e	Another version of the parser with a slightly richer syntax. The ghastly pipe operator is now optional, but is never required. Pipelines cannot be initiated from an arbitrary SELECT unless the arbitrary SELECT is a subquery on the initial FROM. FossilOrigin-Name: 8781d7352b926129027f73d1fe21df78fcb6d21ab3caeefe0f829437c7c1250b	2024-08-26 02:53:31 +00:00
drh	7210e4c5b4	Proof-of-concept grammar rules to parse Google-style "pipe" syntax for SQL, without the ghastly "\|>" operator. The grammar rules are not connected to working code. They just parse the syntax. Pipelines that begin with an ordinary SELECT statement must have the keyword "INTO" (rather than the "\|>" operator) separating the SELECT from the start of the pipeline, to avoid syntactic and semantic ambiguity. FossilOrigin-Name: 2c4bae3e68eea14a5edbc753a7a2d764924c95060aff95fa1d2141c11fa77fed	2024-08-26 00:15:35 +00:00
stephan	87ebadbf4e	shell.c.in: use eputz/oputz() instead of eputf/oputf() where appropriate to avoid compilation errors in -std=c99 mode (namely wasm builds). FossilOrigin-Name: 9ef8317faebc29d016bdf2e8c678fb21ca2cfa95272f1f18b461fcaf0e220ac5	2024-08-25 11:59:29 +00:00
drh	cc902fd25e	Fix harmless compiler warnings in fts5V2toV1Tokenize(). FossilOrigin-Name: df65d00f104f31741056686f2ba41ecb192c552012bffb17c2a8b5d4db058328	2024-08-24 20:05:59 +00:00
drh	306a8dcf3e	Fix compiler warnings in the CLI detected by MSVC. FossilOrigin-Name: 23ae505cbfde6dfd1dbb2216cf76c1e316b49f2ca84491a1aae28ad33f7777a9	2024-08-24 20:01:05 +00:00
drh	838d2d3078	Improved help message for the --enable-test-status configuration option. Fix the build for when that option is omitted. FossilOrigin-Name: b404a5fe3f0532f6d56b5e65c44a59379c17d175c21ac69a1f2ba8aadf453313	2024-08-24 19:06:52 +00:00
drh	1d5a12e6a5	New ./configure option --enable-test-status causes testrunner.tcl to be run with the --status option for targets like mdevtest and releasetest. FossilOrigin-Name: 9ca8513c26e54c97f2d3c6105c42144100044727f6fb4c28048f7cd7270dd35e	2024-08-24 19:02:33 +00:00
drh	f9eff9052d	Remove a term from an assert() that is no longer relevant. FossilOrigin-Name: 8b4766252b80075354a6b1406c5430fe999a6490aa3e079893fcc682d88734f7	2024-08-24 18:42:39 +00:00
dan	58b08d7702	Avoid a stack overflow that could be caused by a recursively defined WINDOW() with a strategically embedded error. FossilOrigin-Name: bada54bd6bf54190e40aa721b77081015957d204c7b6a9fdbe8c67bcf20798f8	2024-08-24 15:54:15 +00:00
drh	27095fc4a5	Ensure that the WhereLoop.u.btree.pOrderBy field added by the order-by-subquery optimization [7a0cdc7edb704a88] is correctly initialized. dbsqlfuzz 437be171a98687082a31ea6efc2fd5c5d64481c8. FossilOrigin-Name: b75f8b2252aefe7bb96fa237205a1a9dc88c46d9ec09bca7407e6c636b203e7e	2024-08-24 11:48:19 +00:00
stephan	7766c9943e	Eliminate use of strcpy() in shell.c.in to squelch an unwarranted link-time warning on OpenBSD. FossilOrigin-Name: 32e9bf7f972144eacbb2580151220bc4fc1cbcc8bcf586ffd8e10762abf75c7e	2024-08-23 21:20:50 +00:00
dan	789da35848	Fix a problem in the ext/fts5/extract_api_docs.tcl script. FossilOrigin-Name: 9a9d0f6301faefe324261f03543023ffb6a90823349c6946abb0df2f69b31f96	2024-08-23 17:40:29 +00:00
dan	8d4edd0d67	Add fts5 auxiliary function fts5_get_locale(). For querying the locale of a stored value. FossilOrigin-Name: 396f720f36a937145ac727c2750acfd26e4eda350b46d648d82a7e9985545a9f	2024-08-23 15:18:02 +00:00
drh	5804da0218	Enhance the generate_series() table-valued function such that it is able to recognize equality and inequality constraints on the "value" column and optimize its operating accordingly. FossilOrigin-Name: d50b784807333c5461a2d027778c746c799285b95bb1952f142b317ea2846460	2024-08-22 18:12:10 +00:00
drh	d55ab84451	Add the SQLITE_INDEX_SCAN_HEX bit to the sqlite3_index_info.idxFlags bitmask. When set, this bit causes the EXPLAIN QUERY PLAN output to show the idxNum value in hex rather than in decimal. This is purely a debugging aid. FossilOrigin-Name: 6c00e88ebdb41d6317bb8758825521614dedc2e6e6289ff415c5f0406eed815b	2024-08-22 16:22:08 +00:00
drh	889ce8ee90	Add SQLITE_TESTCTRL_GETOPT. Use it to implement improvements to the ".testctrl opt" dot-command in the CLI, and similar enhancements to TH3. FossilOrigin-Name: 6500baa9914b08ce19c361a69f19a80e5d800fcb82579d1c1e03601c7a664338	2024-08-21 20:25:31 +00:00
drh	be1fdbd272	Add the new SQLITE_TESTCTRL_OPTGET that retrieves the current optimization setting. Use this in the CLI to provide symbolic fine-grain control over optimization settings using the ".testctrl opt" dot-command. FossilOrigin-Name: 0ea4e1e8fc689cb1af3a8d21dd6af9d483115412b414e85e8a42480f50d65af2	2024-08-21 18:57:55 +00:00
drh	9904d824b8	Improved comments on byte-code subroutine calls to materialize subqueries. Should not affect production builds. FossilOrigin-Name: 6b024c85ef856e26cddac8ad862e16909b5149c3e3efb4e73ba54c97291f1cc4	2024-08-21 17:38:50 +00:00
stephan	5b5625f679	README.md: a markup link fix and use the canonical URL for Fossil. FossilOrigin-Name: c49bfee6b878bd96adfc852fdc76e98bf81e2fb82be8325672fbc7af04e7dffa	2024-08-21 13:57:30 +00:00
drh	59b171172f	Updates and enhancements to the README.md file. No code changes. FossilOrigin-Name: 3161b8028916bff9a13a04bfb1996bce7e1274a3c403fc58f4d798afd30c528f	2024-08-21 13:44:40 +00:00
drh	4cc86b6e92	Add code comments for a "table-of-contents" and various milestone marks in the 1300+ line long sqlite3Select() function, to help improve readiability and maintainability. Comment changes only - no functional changes. FossilOrigin-Name: 00cfbb9fa5136e6a7847da4e4ab30d320ca79c22acda9db2030e558d59b8c744	2024-08-21 12:01:46 +00:00
drh	91937a6961	Fix a minor typo in a comment. FossilOrigin-Name: 92c80af1129051c9eded8df170730ad6366b4f7715dead34c4364c8149d0dce9	2024-08-21 10:37:59 +00:00
drh	01972f55fe	Improvements to treetrace output for subqueries. Debug code only. FossilOrigin-Name: 0a835d18d9063b17a1d506fd989ebcf7f0d76593978a3bd5981be59618335f21	2024-08-21 10:32:00 +00:00
stephan	fb7d0c3c01	De-duplicate some error-output code in shell.c.in as part of tracking down the cause of [forum:5647ca2af1\|forum post 5647ca2af1]. FossilOrigin-Name: fd5904495e0c775174f9380bc6877fb9def3d30dec1c9315979ad4503a18d70b	2024-08-21 09:56:45 +00:00
drh	8c58fd78c3	Remove incorrect assert() statements in the SrcItem invariant validator. dbsqlfuzz 7f908865436ce531b0ace55c97a1c71d05c4e76c. FossilOrigin-Name: 81a1ede38823b8d50e221ecb6ce52a05d82a856099002a71d9f1ac561a587f05	2024-08-21 07:38:14 +00:00
drh	5648d717b1	Refactor the SrcItem object to move fields associated with subqueries out into a separate object named Subquery. This reduces the size of the SrcItem object by about 1/3rd and provides improved performance. FossilOrigin-Name: 484bcd75bc95491d8540c791c1c4d40d996cb465839564662e14f98739699bf1	2024-08-20 23:11:28 +00:00
drh	781163aa9d	When the database encoding is UTF-16LE and the GLOB optimization is used, it is ok to use the range search over an index, but it is not ok to disable the actual GLOB function call. FossilOrigin-Name: 3399698376761ab8c422f8ea02bfa2759afb606f08bedbd1cf7eee834229a9aa	2024-08-20 14:12:16 +00:00
drh	6ad4e9fd2a	Fix for the UTF-16LE problem reporte by [forum:/forumpost/dc7e1b7527e84343\|forum post dc7e1b7527e84343]. Because of the unexpected ordering of characters using the default collation (memcmp() order) in UTF-16LE, the LIKE/GLOB optimization restricts its attention to the pattern prefix that is all ASCII, which is the common case. FossilOrigin-Name: a5797ebdea423afc3d2d3bd8adaf1575d33a01f594c0c315afcb1499f1718e9b	2024-08-20 12:09:55 +00:00
drh	1c8ed5fa43	Earlier error detection while processing complex aggregate queries. dbsqlfuzz 5242c2f07f4aa031aa3c80461f18e9b7619ede9b FossilOrigin-Name: 70f4973078ffc72f4ff247234e6f8e695b40803c3e7c9ed12050d97195728352	2024-08-19 23:43:08 +00:00
dan	127812e645	Update some comments that are processed into the fts5.html webpage. FossilOrigin-Name: 3e06ab218bbd1ed75a24afb44e8df9ce84e9fc24701428cb8b3459760f44006d	2024-08-19 20:35:30 +00:00
dan	18557fdeb5	Add the fts5_locale() function and related functionality to fts5. FossilOrigin-Name: b9632900100bdbc913f83bfb03b32585cf07a192b9a7f26b9bebc7d91e63a610	2024-08-19 14:33:39 +00:00
drh	922cf3e2e4	Fix leap-year handling for dates prior to 0400-03-01. [forum:/forumpost/eaa0a09786c6368b\|Forum post eaa0a09786c6368b]. FossilOrigin-Name: 474b7e34b83bc5e85854bc3b386f31ff39b390549d89b94921f33bbc5b658d1d	2024-08-19 14:20:18 +00:00
drh	5a18c1f9cd	New date/time test cases to validate ancient dates. FossilOrigin-Name: 82719074f090d9d0a0b9baea0dee7b1dd40a272e3f3e45043d4a640c07989d5d	2024-08-19 13:53:49 +00:00
drh	aa2258e35b	Attempt to fix date/time calculations for days earlier than 0400-03-01. See [forum:/forumpost/eaa0a09786c6368b\|forum thread eaa0a09786c6368b]. FossilOrigin-Name: 00cae11fffaf50e2133915e851d41098d00b6b1458e1438bcb6664e1411d8498	2024-08-19 12:54:12 +00:00
dan	f5b1c3da36	Merge trunk changes into this branch. FossilOrigin-Name: 6787603bd0d8117e41e6b572c0d7cada0f6d7266a4ff2ad79f9471b9beb28a9e	2024-08-19 10:40:08 +00:00
dan	dd9a7e4540	Fix a problem with detail=none, content='' tables. FossilOrigin-Name: 4b8070816ada3f269d53c9cee20669f59e19a23c31c8509fc397c21dd097ec9f	2024-08-18 14:46:13 +00:00
drh	901b671455	Automatically disable directory fsync when compiling for AIX. FossilOrigin-Name: 8d170e07e20c21a5bb97fdc8c2d01d92178f5fe740e60d203e4621747cfce33e	2024-08-18 09:53:37 +00:00
dan	6c796631fc	Fix a broken assert() in fts5_storage.c. FossilOrigin-Name: 01116e32797e1b127ba79059d890fbe5075eee2b6e3ce4468e82862b0fc3aa46	2024-08-17 19:40:15 +00:00
dan	408a34925d	Merge trunk changes into this branch. FossilOrigin-Name: 6a8ff9ba5e71d817489093d8dff0a8d77365b4222773b941accbd58558d24379	2024-08-17 19:11:09 +00:00
dan	ef2401f669	Tests to improve coverage of fts5_expr.c. FossilOrigin-Name: f4b839e5265700b1a89066d1b6e0d0d010852a69c5da3d75d2c41624dbf3c0af	2024-08-17 19:07:13 +00:00
dan	b651084713	Add tests to restore coverage of fts5_tokenizer.c. FossilOrigin-Name: 8f9257361b05e368bf433e56d0698923b0f97d12e7c0ad7760aaab6746c0e467	2024-08-17 17:22:49 +00:00
dan	342984075b	Add tests to restore coverage of fts5_config.c. FossilOrigin-Name: 9d971b31df7ad4d68eb348f95d8f996071cf87d41c47033bde3fcc4dba732e06	2024-08-16 19:05:47 +00:00
dan	f0f240c0d8	Add tests to restore coverage of fts5_storage.c. FossilOrigin-Name: 38841881a08f4d75c2dfcbc1b2b45e311d7dda1287780f8ba1c8388f7adc1629	2024-08-16 17:54:00 +00:00
dan	db79dd1f88	Add tests to restore coverage of fts5_main.c. FossilOrigin-Name: 681edd08ab7024ddcd6447a64d8aad581b47b039481b697d91d26e6c6660f373	2024-08-16 16:20:43 +00:00
dan	41ebf965cd	Extra tests. FossilOrigin-Name: fb257540d32ced4a5fe0759d3ad9c1b36fd2b7d64d93cb0a0ecdb48355a68b7a	2024-08-15 20:33:05 +00:00
dan	189c41221d	Further tests and fixes for this branch. FossilOrigin-Name: d27985245a0e8c0d6b04323c98b26b6a8fb4e489fa8f5f3234252c7c198f23c8	2024-08-15 18:50:13 +00:00
dan	a7384e0de2	More tests for the new code on this branch. FossilOrigin-Name: 00792e807f1dde750e6ac9f9b0095cf60b9fa6ff7cf4b14440600a21de2ee61a	2024-08-13 21:15:43 +00:00
dan	7dad9b4297	Add tests and fix minor issues with the code on this branch. FossilOrigin-Name: def5243a7b4f1d50427ba607d31332db2d6ac34b181b0cbd2ed4911064f5637a	2024-08-13 15:04:00 +00:00
dan	3e38aa0821	Add extra documentation comments to fts5_main.c. FossilOrigin-Name: 1a7a9bd3f6a2b9347720f7f0386399e7f14b428500acf07d7c55b9dde1ec233a	2024-08-12 18:39:13 +00:00
dan	a4a8e91022	Fix problem with some fts5 aux function APIs with (locale=, detail=none/column) tables. FossilOrigin-Name: bf116dec6fb20cce69099cb8c031b0de511195482919a99ccfb3c498e9bf8125	2024-08-12 17:28:13 +00:00
dan	10e54e365a	Fix further issues to do with fts5 locale support. FossilOrigin-Name: e626123580065986f7df50b6140f00048944becce179b9391fbf09f97ac55485	2024-08-12 17:03:37 +00:00
dan	29d5e43ad4	Update the porter tokenizer to use locales. FossilOrigin-Name: 3291ce3a3359a80e51e4546a3d7a187cbe4c7530fca6632f0bb2728525efe212	2024-08-12 11:46:09 +00:00
dan	1286677773	Update the auxiliary function API to include xTokenize_x2() instead of xSetLocale(). FossilOrigin-Name: f7d56a1f2149f0da117167db62e2c28ec337e8da3403873b64cdfc6a951e2e8e	2024-08-12 11:13:56 +00:00
dan	9b118de9d5	Merge trunk changes into this branch. FossilOrigin-Name: b243007525a825b3daf8aa9bb2d3088efb853bd8b7c9ea3c0924fde193eb5c44	2024-08-10 20:03:01 +00:00
dan	7978910f85	Fix an uninitialized variable in fts5_tcl.c. FossilOrigin-Name: 924d3467cebd71ac8dfef8225965d71eda82ef1f0b43508af5b7ea78e2099cf3	2024-08-10 19:18:19 +00:00
dan	ff6ab9dd2b	Update the fts5_tokenizer_v2 API so that the locale is passed as parameter to xTokenize(), instead of via a separate call to xSetLocale(). FossilOrigin-Name: 03e63ed24e7a08817341e59b075ae2d4e3f7a5e5b37e0e6138359d5fd29a5e9e	2024-08-10 18:59:36 +00:00
dan	5eba4281b0	Ensure tokenizers registered with xCreateTokenizer_v2() can be accessed using xFindTokenizer(), and that those registered with xCreateTokenizer() work with xFindTokenizer_v2(). FossilOrigin-Name: a3ef7d47b582a66a85ebe802ab40114f77289d6b44a4b375ea7858c60f59b27f	2024-08-09 20:59:50 +00:00
dan	e317e7f4b4	Change things so that locale=1 is required to write fts5_locale() values to an fts5 table, and so that blobs may not be stored in indexed (i.e. not UNINDEXED) columns of these tables. FossilOrigin-Name: c98ccc12169419b8b27ead89ef0665de40320277c5daa748b80869337419e43e	2024-08-02 21:06:13 +00:00
dan	f971c15b1f	Clarify the role of Fts5Storage.pSavedRow in the new feature on this branch. FossilOrigin-Name: e8a61d5c48073fdd4d99d0b6fc70469b37af009f281336a44e3789e7eeed820d	2024-08-01 17:15:17 +00:00
dan	5bd8cc7dd5	Fix various problems with the code on this branch. FossilOrigin-Name: 8bd4ae7e95c7b6ce34db5ea705dc136e742a22f333d0e7370b485ebd736b5ec2	2024-07-31 20:49:00 +00:00
dan	8e3042b565	Add missing documentation for new features to fts5.h. FossilOrigin-Name: fa0da3b28e411affd45a918d6b7faba49f744ca8b4adf3cd5ce6609bb630499e	2024-07-31 15:46:41 +00:00
dan	c97cdbe523	Merge latest changes from trunk into this branch. FossilOrigin-Name: ff64939ec8399949393f1029fa6d514892fbf2bf1498545300cc5e120b9622a5	2024-07-31 10:52:38 +00:00
dan	9015b8822d	Fix cases surrounding blob values being stored in locale=1 fts5 tables. FossilOrigin-Name: ae435aff1785d5832821c19dc88ccf6c496a7ff55be80276b31e5c9abee723db	2024-07-30 20:39:58 +00:00
dan	8174305e17	Update the highlight() and snippet() functions to use locales when available. FossilOrigin-Name: 569ae56a673a2e101fac7003a1cb41c7a02a515adf55bd1e1f2a03c19eb2b085	2024-07-30 15:55:51 +00:00
dan	8f14c45197	Fix xColumnText() and xColumnSize() APIs. Add xColumnLocale() and xTokenizeSetLocale(). FossilOrigin-Name: 14006711d83d098e665c540b978b0e29aa8f458da1c2c8e9c2baf2ad5ebd502c	2024-07-29 20:31:17 +00:00
dan	b4afecc626	Fix a couple of asan problems on this branch. FossilOrigin-Name: 4e3e529313c86b42d5f1ba113d0714d1545bc6641e09e4537d622057f0352989	2024-07-29 15:00:11 +00:00
dan	53bef48fbe	Ensure that an fts5 UPDATE does not strip the locale from columns that it does not write to. FossilOrigin-Name: 69205264debd829573b1c777a5a493cfeb6083c4cdec106b1f819989f859ac75	2024-07-27 19:46:10 +00:00
dan	2ec78c0e4b	Add the fts5_locale() function, and begin adding the related functionality to fts5. FossilOrigin-Name: 8839ef7cfb49239e7f1c4812a53a93a672827c88d6921408b1d5062b352c87cc	2024-07-26 20:50:33 +00:00